From e5524392a9c99208448fe863952586e545776927 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 15:14:12 +0100 Subject: [PATCH 01/97] Add first draft of SCIP persistent solving --- pyomo/solvers/plugins/solvers/__init__.py | 2 + pyomo/solvers/plugins/solvers/scip_direct.py | 838 ++++++++++++++++++ .../plugins/solvers/scip_persistent.py | 185 ++++ pyomo/solvers/tests/checks/test_SCIPDirect.py | 335 +++++++ .../tests/checks/test_SCIPPersistent.py | 318 +++++++ pyomo/solvers/tests/solvers.py | 21 + 6 files changed, 1699 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/scip_direct.py create mode 100644 pyomo/solvers/plugins/solvers/scip_persistent.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 9b2507d876c..e8f4e00e31a 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,5 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.scip_direct +import pyomo.solvers.plugins.solvers.scip_persistent diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py new file mode 100644 index 00000000000..0aafb596007 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -0,0 +1,838 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +import re +import sys + +from pyomo.common.collections import ComponentSet, ComponentMap, Bunch +from pyomo.common.tempfiles import TempfileManager +from pyomo.core import Var +from pyomo.core.expr.numeric_expr import ( + SumExpression, + ProductExpression, + UnaryFunctionExpression, + PowExpression, + DivisionExpression, +) +from pyomo.core.expr.numvalue import is_fixed +from pyomo.core.expr.numvalue import value +from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn import generate_standard_repn +from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver +from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( + DirectOrPersistentSolver, +) +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.opt.results.results_ import SolverResults +from pyomo.opt.results.solution import Solution, SolutionStatus +from pyomo.opt.results.solver import TerminationCondition, SolverStatus +from pyomo.opt.base import SolverFactory +from pyomo.core.base.suffix import Suffix + + +logger = logging.getLogger("pyomo.solvers") + + +class DegreeError(ValueError): + pass + + +def _is_numeric(x): + try: + float(x) + except ValueError: + return False + return True + + +@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") +class SCIPDirect(DirectSolver): + + def __init__(self, **kwds): + kwds["type"] = "scipdirect" + DirectSolver.__init__(self, **kwds) + self._init() + self._solver_model = None + + def _init(self): + try: + import pyscipopt + + self._scip = pyscipopt + self._python_api_exists = True + self._version = str(self._scip.Model().version()) + self._version_major = self._version.split(".")[0] + except ImportError: + self._python_api_exists = False + except Exception as e: + print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + self._python_api_exists = False + + # Note: Undefined capabilities default to None + self._max_constraint_degree = None + self._max_obj_degree = 1 + self._capabilities.linear = True + self._capabilities.quadratic_objective = False + self._capabilities.quadratic_constraint = True + self._capabilities.integer = True + self._capabilities.sos1 = True + self._capabilities.sos2 = True + + # Dictionary used exclusively for SCIP, as we want the constraint expressions + self._pyomo_var_to_solver_var_expr_map = ComponentMap() + self._pyomo_con_to_solver_con_expr_map = dict() + + def _apply_solver(self): + StaleFlagManager.mark_all_as_stale() + + # Supress solver output if requested + if self._tee: + self._solver_model.hideOutput(quiet=False) + else: + self._solver_model.hideOutput(quiet=True) + + # Redirect solver output to a logfile if requested + if self._keepfiles: + # Only save log file when the user wants to keep it. + self._solver_model.setLogfile(self._log_file) + print("Solver log file: " + self._log_file) + + # Set user specified parameters + for key, option in self.options.items(): + try: + key_type = type(self._solver_model.getParam(key)) + except KeyError: + raise ValueError(f"Key {key} is an invalid parameter for SCIP") + + if key_type == str: + self._solver_model.setParam(key, option) + else: + if not _is_numeric(option): + raise ValueError( + f"Value {option} for parameter {key} is not a string and can't be converted to float" + ) + self._solver_model.setParam(key, float(option)) + + self._solver_model.optimize() + + # TODO: Check if this is even needed, or if it is sufficient to close the open file + # if self._keepfiles: + # self._solver_model.setLogfile(None) + + # FIXME: can we get a return code indicating if SCIP had a significant failure? + return Bunch(rc=None, log=None) + + def _get_expr_from_pyomo_repn(self, repn, max_degree=None): + referenced_vars = ComponentSet() + + new_expr = repn.constant + + if len(repn.linear_vars) > 0: + referenced_vars.update(repn.linear_vars) + new_expr += sum( + repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] + for i, var in enumerate(repn.linear_vars) + ) + + for i, v in enumerate(repn.quadratic_vars): + x, y = v + new_expr += ( + repn.quadratic_coefs[i] + * self._pyomo_var_to_solver_var_expr_map[x] + * self._pyomo_var_to_solver_var_expr_map[y] + ) + referenced_vars.add(x) + referenced_vars.add(y) + + # TODO: Introduce handling on non-linear expressions + if repn.nonlinear_expr is not None: + + def get_nl_expr_recursively(pyomo_expr): + if not hasattr(pyomo_expr, "args"): + if not isinstance(pyomo_expr, Var): + return float(pyomo_expr) + else: + referenced_vars.add(pyomo_expr) + return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] + scip_expr_list = [0 for i in range(pyomo_expr.nargs())] + for i in range(pyomo_expr.nargs()): + scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) + if isinstance(pyomo_expr, PowExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"PowExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] ** (scip_expr_list[1]) + elif isinstance(pyomo_expr, ProductExpression): + return self._scip.quickprod(scip_expr_list) + elif isinstance(pyomo_expr, SumExpression): + return self._scip.quicksum(scip_expr_list) + elif isinstance(pyomo_expr, DivisionExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] / scip_expr_list[1] + elif isinstance(pyomo_expr, UnaryFunctionExpression): + if len(scip_expr_list) != 1: + raise ValueError( + f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" + ) + if pyomo_expr.name == "sin": + return self._scip.sin(scip_expr_list[0]) + elif pyomo_expr.name == "cos": + return self._scip.cos(scip_expr_list[0]) + elif pyomo_expr.name == "exp": + return self._scip.exp(scip_expr_list[0]) + elif pyomo_expr.name == "log": + return self._scip.log(scip_expr_list[0]) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" + ) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" + ) + + new_expr += get_nl_expr_recursively(repn.nonlinear_expr) + + return new_expr, referenced_vars + + def _get_expr_from_pyomo_expr(self, expr, max_degree=None): + if max_degree is None or max_degree >= 2: + repn = generate_standard_repn(expr, quadratic=True) + else: + repn = generate_standard_repn(expr, quadratic=False) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + + return scip_expr, referenced_vars + + def _scip_lb_ub_from_var(self, var): + if var.is_fixed(): + val = var.value + return val, val + if var.has_lb(): + lb = value(var.lb) + else: + lb = -self._solver_model.infinity() + if var.has_ub(): + ub = value(var.ub) + else: + ub = self._solver_model.infinity() + return lb, ub + + def _add_var(self, var): + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._pyomo_var_to_solver_var_map[var] = scip_var.name + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = 0 + + def close(self): + """Frees SCIP resources used by this solver instance.""" + + if self._solver_model is not None: + self._solver_model.freeProb() + self._solver_model = None + + def __exit__(self, t, v, traceback): + super().__exit__(t, v, traceback) + self.close() + + def _set_instance(self, model, kwds={}): + DirectOrPersistentSolver._set_instance(self, model, kwds) + try: + self._solver_model = self._scip.Model() + except Exception: + e = sys.exc_info()[1] + msg = ( + "Unable to create SCIP model. " + "Have you installed PySCIPOpt correctly?\n\n\t" + + "Error message: {0}".format(e) + ) + raise Exception(msg) + + self._add_block(model) + + for var, n_ref in self._referenced_variables.items(): + if n_ref != 0: + if var.fixed: + if not self._output_fixed_variable_bounds: + raise ValueError( + "Encountered a fixed variable (%s) inside " + "an active objective or constraint " + "expression on model %s, which is usually " + "indicative of a preprocessing error. Use " + "the IO-option 'output_fixed_variable_bounds=True' " + "to suppress this error and fix the variable " + "by overwriting its bounds in the SCIP instance." + % (var.name, self._pyomo_model.name) + ) + + def _add_block(self, block): + DirectOrPersistentSolver._add_block(self, block) + + def _add_constraint(self, con): + if not con.active: + return None + + if is_fixed(con.body) and self._skip_trivial_constraints: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + + if con._linear_canonical_form: + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( + con.canonical_form(), self._max_constraint_degree + ) + else: + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + con.body, self._max_constraint_degree + ) + + if con.has_lb(): + if not is_fixed(con.lower): + raise ValueError( + "Lower bound of constraint {0} is not constant.".format(con) + ) + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError( + "Upper bound of constraint {0} is not constant.".format(con) + ) + + if con.equality: + scip_cons = self._solver_model.addCons( + scip_expr == value(con.lower), name=conname + ) + elif con.has_lb() and con.has_ub(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + ) + elif con.has_lb(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= scip_expr, name=conname + ) + elif con.has_ub(): + scip_cons = self._solver_model.addCons( + scip_expr <= value(con.upper), name=conname + ) + else: + raise ValueError( + "Constraint does not have a lower " + "or an upper bound: {0} \n".format(con) + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + self._vars_referenced_by_con[con] = referenced_vars + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _add_sos_constraint(self, con): + if not con.active: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + level = con.level + if level not in [1, 2]: + raise ValueError(f"Solver does not support SOS level {level} constraints") + + scip_vars = [] + weights = [] + + self._vars_referenced_by_con[con] = ComponentSet() + + if hasattr(con, "get_items"): + # aml sos constraint + sos_items = list(con.get_items()) + else: + # kernel sos constraint + sos_items = list(con.items()) + + for v, w in sos_items: + self._vars_referenced_by_con[con].add(v) + scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) + self._referenced_variables[v] += 1 + weights.append(w) + + if level == 1: + scip_cons = self._solver_model.addConsSOS1( + scip_vars, weights=weights, name=conname + ) + else: + scip_cons = self._solver_model.addConsSOS2( + scip_vars, weights=weights, name=conname + ) + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _scip_vtype_from_var(self, var): + """ + This function takes a pyomo variable and returns the appropriate SCIP variable type + :param var: pyomo.core.base.var.Var + :return: B, I, or C + """ + if var.is_binary(): + vtype = "B" + elif var.is_integer(): + vtype = "I" + elif var.is_continuous(): + vtype = "C" + else: + raise ValueError( + "Variable domain type is not recognized for {0}".format(var.domain) + ) + return vtype + + def _set_objective(self, obj): + if self._objective is not None: + for var in self._vars_referenced_by_obj: + self._referenced_variables[var] -= 1 + self._vars_referenced_by_obj = ComponentSet() + self._objective = None + + if obj.active is False: + raise ValueError("Cannot add inactive objective to solver.") + + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + obj.expr, self._max_obj_degree + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + + self._solver_model.setObjective(scip_expr, sense=sense) + self._objective = obj + self._vars_referenced_by_obj = referenced_vars + + self._needs_updated = True + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + "***The scip_direct solver plugin cannot extract solution suffix=" + + suffix + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + if scip.getStage() == 1: # SCIP Model is created but not yet optimized + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Model is loaded, but no solution information is available." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.unknown + elif status == "optimal": # optimal + self.results.solver.status = SolverStatus.ok + self.results.solver.termination_message = ( + "Model was solved to optimality (subject to tolerances), " + "and an optimal solution is available." + ) + self.results.solver.termination_condition = TerminationCondition.optimal + soln.status = SolutionStatus.optimal + elif status == "infeasible": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be infeasible" + ) + self.results.solver.termination_condition = TerminationCondition.infeasible + soln.status = SolutionStatus.infeasible + elif status == "inforunbd": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Problem proven to be infeasible or unbounded." + ) + self.results.solver.termination_condition = ( + TerminationCondition.infeasibleOrUnbounded + ) + soln.status = SolutionStatus.unsure + elif status == "unbounded": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be unbounded." + ) + self.results.solver.termination_condition = TerminationCondition.unbounded + soln.status = SolutionStatus.unbounded + elif status == "gaplimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the gap dropped below " + "the value specified in the " + "limits/gap parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "stallnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the stalling node limit " + "exceeded the value specified in the " + "limits/stallnodes parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "restartlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the total number of restarts " + "exceeded the value specified in the " + "limits/restarts parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "nodelimit" or status == "totalnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of " + "branch-and-cut nodes explored exceeded the limits specified " + "in the limits/nodes or limits/totalnodes parameter" + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxEvaluations + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "timelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the time expended exceeded " + "the value specified in the limits/time parameter." + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxTimeLimit + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "sollimit" or status == "bestsollimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of solutions found " + "reached the value specified in the limits/solutions or" + "limits/bestsol parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "memlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the memory used exceeded " + "the value specified in the limits/memory parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "userinterrupt": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization was terminated by the user." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + else: + self.results.solver.status = SolverStatus.error + self.results.solver.termination_message = ( + "Unhandled SCIP status (" + str(status) + ")" + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + + self.results.problem.name = scip.getProbName() + + if scip.getObjectiveSense() == "minimize": + self.results.problem.sense = minimize + elif scip.getObjectiveSense() == "maximize": + self.results.problem.sense = maximize + else: + raise RuntimeError( + f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" + ) + + self.results.problem.upper_bound = None + self.results.problem.lower_bound = None + if scip.getNSols() > 0: + scip_has_sol = True + else: + scip_has_sol = False + if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): + pass + else: + if n_bin_vars + n_int_vars == 0: + self.results.problem.upper_bound = scip.getObjVal() + self.results.problem.lower_bound = scip.getObjVal() + elif scip.getObjectiveSense() == "minimize": # minimizing + if scip_has_sol: + self.results.problem.upper_bound = scip.getObjVal() + else: + self.results.problem.upper_bound = scip.infinity() + self.results.problem.lower_bound = scip.getDualbound() + else: # maximizing + self.results.problem.upper_bound = scip.getDualbound() + if scip_has_sol: + self.results.problem.lower_bound = scip.getObjVal() + else: + self.results.problem.lower_bound = -scip.infinity() + + try: + soln.gap = ( + self.results.problem.upper_bound - self.results.problem.lower_bound + ) + except TypeError: + soln.gap = None + + # TODO: Should these values be of the transformed or the original problem? + self.results.problem.number_of_constraints = scip.getNConss() + # self.results.problem.number_of_nonzeros = None + self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_binary_variables = n_bin_vars + self.results.problem.number_of_integer_variables = n_int_vars + self.results.problem.number_of_continuous_variables = n_con_vars + self.results.problem.number_of_objectives = 1 + self.results.problem.number_of_solutions = scip.getNSols() + + # if a solve was stopped by a limit, we still need to check to + # see if there is a solution available - this may not always + # be the case, both in LP and MIP contexts. + if self._save_results: + """ + This code in this if statement is only needed for backwards compatibility. It is more efficient to set + _save_results to False and use load_vars, load_duals, etc. + """ + if scip.getNSols() > 0: + soln_variables = soln.variable + soln_constraints = soln.constraint + scip_sol = scip.getBestSol() + + scip_vars = scip.getVars() + scip_var_names = [scip_var.name for scip_var in scip_vars] + var_names = set(self._solver_var_to_pyomo_var_map.keys()) + assert set(scip_var_names) == var_names + var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] + + for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name] = {"Value": val} + + if extract_reduced_costs: + vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] + for scip_var, val, name in zip(scip_vars, vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name]["Rc"] = val + + if extract_duals or extract_slacks: + scip_cons = scip.getConss() + con_names = [cons.name for cons in scip_cons] + assert set(self._solver_con_to_pyomo_con_map.keys()) == set( + con_names + ) + for name in con_names: + soln_constraints[name] = {} + + if extract_duals: + vals = [scip.getDualSolVal(con) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Dual"] = val + + if extract_slacks: + vals = [scip.getSlack(con, scip_sol) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Slack"] = val + + elif self._load_solutions: + if scip.getNSols() > 0: + self.load_vars() + + if extract_reduced_costs: + self._load_rc() + + if extract_duals: + self._load_duals() + + if extract_slacks: + self._load_slacks() + + self.results.solution.insert(soln) + + # finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. + TempfileManager.pop(remove=not self._keepfiles) + + return DirectOrPersistentSolver._postsolve(self) + + def warm_start_capable(self): + return True + + def _warm_start(self): + scip_sol = self._solver_model.createSol() + for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): + if pyomo_var.value is not None: + scip_sol[scip_var] = value(pyomo_var) + self._solver_model.trySol(scip_sol, free=True) + + def _load_vars(self, vars_to_load=None): + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + var.set_value(val, skip_validation=True) + + def _load_rc(self, vars_to_load=None): + if not hasattr(self._pyomo_model, "rc"): + self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + rc = self._pyomo_model.rc + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [ + self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load + ] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + rc[var] = val + + def _load_duals(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "dual"): + self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + dual = self._pyomo_model.dual + scip_cons = self._solver_model.getConss() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + dual[pyomo_con] = vals[i] + + def _load_slacks(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "slack"): + self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + slack = self._pyomo_model.slack + scip_cons = self._solver_model.getConss() + scip_sol = self._solver_model.getBestSol() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [ + self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + ] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + slack[pyomo_con] = vals[i] + + def load_duals(self, cons_to_load=None): + """ + Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_duals(cons_to_load) + + def load_rc(self, vars_to_load): + """ + Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. + + Parameters + ---------- + vars_to_load: list of Var + """ + self._load_rc(vars_to_load) + + def load_slacks(self, cons_to_load=None): + """ + Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent + model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_slacks(cons_to_load) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py new file mode 100644 index 00000000000..408aa84633f --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -0,0 +1,185 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect +from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver +from pyomo.opt.base import SolverFactory + + +@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") +class SCIPPersistent(PersistentSolver, SCIPDirect): + """ + A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. + Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces + are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes + to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible + for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. + + Keyword Arguments + ----------------- + model: ConcreteModel + Passing a model to the constructor is equivalent to calling the set_instance method. + type: str + String indicating the class type of the solver instance. + name: str + String representing either the class type of the solver instance or an assigned name. + doc: str + Documentation for the solver + options: dict + Dictionary of solver options + """ + + def __init__(self, **kwds): + kwds["type"] = "scip_persistent" + PersistentSolver.__init__(self, **kwds) + SCIPDirect._init(self) + + self._pyomo_model = kwds.pop("model", None) + if self._pyomo_model is not None: + self.set_instance(self._pyomo_model, **kwds) + + def _remove_constraint(self, solver_conname): + con = self._solver_con_to_pyomo_con_map[solver_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_sos_constraint(self, solver_sos_conname): + con = self._solver_con_to_pyomo_con_map[solver_sos_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_var(self, solver_varname): + var = self._solver_var_to_pyomo_var_map[solver_varname] + scip_var = self._pyomo_var_to_solver_var_expr_map[var] + self._solver_model.delVar(scip_var) + + def _warm_start(self): + SCIPDirect._warm_start(self) + + def update_var(self, var): + """Update a single variable in the solver's model. + + This will update bounds, fix/unfix the variable as needed, and + update the variable type. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + + """ + # see PR #366 for discussion about handling indexed + # objects and keeping compatibility with the + # pyomo.kernel objects + # if var.is_indexed(): + # for child_var in var.values(): + # self.compile_var(child_var) + # return + if var not in self._pyomo_var_to_solver_var_map: + raise ValueError( + "The Var provided to compile_var needs to be added first: {0}".format( + var + ) + ) + scip_var = self._pyomo_var_to_solver_var_map[var] + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def write(self, filename, filetype=""): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + filetype: str + The file type (e.g., lp). + """ + self._solver_model.writeProblem(filename + filetype) + + def set_scip_param(self, param, val): + """ + Set a SCIP parameter. + + Parameters + ---------- + param: str + The SCIP parameter to set. Options include any SCIP parameter. + Please see the SCIP documentation for options. + val: any + The value to set the parameter to. See SCIP documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_scip_param(self, param): + """ + Get the value of the SCIP parameter. + + Parameters + ---------- + param: str or int or float + The SCIP parameter to get the value of. See SCIP documentation for possible options. + """ + return self._solver_model.getParam(param) + + def _add_column(self, var, obj_coef, constraints, coefficients): + """Add a column to the solver's model + + This will add the Pyomo variable var to the solver's + model, and put the coefficients on the associated + constraints in the solver model. If the obj_coef is + not zero, it will add obj_coef*var to the objective + of the solver's model. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + obj_coef: float + constraints: list of solver constraints + coefficients: list of coefficients to put on var in the associated constraint + """ + + # Set-up add var + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + # Add the variable to the model and then to all the constraints + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = len(coefficients) + + # Get the SCIP cons by passing through two dictionaries + pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_con] + for pyomo_con in pyomo_cons + ] + + for i, scip_con in enumerate(scip_cons): + if not scip_con.isLinear(): + raise ValueError( + "_add_column functionality not supported for non-linear constraints" + ) + self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) + con = self._solver_con_to_pyomo_con_map[scip_con.name] + self._vars_referenced_by_con[con].add(var) + + sense = self._solver_model.getObjectiveSense() + self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) + + def reset(self): + self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py new file mode 100644 index 00000000000..ee37f5ddcc8 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import sys + +import pyomo.common.unittest as unittest + +from pyomo.environ import ( + ConcreteModel, + AbstractModel, + Var, + Objective, + Block, + Constraint, + Suffix, + NonNegativeIntegers, + NonNegativeReals, + Integers, + Binary, + value, +) +from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +class SCIPDirectTests(unittest.TestCase): + def setUp(self): + self.stderr = sys.stderr + sys.stderr = None + + def tearDown(self): + sys.stderr = self.stderr + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var() + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_get_duals_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.Y = Var(within=NonNegativeReals) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + results = opt.solve(model, suffixes=["dual"], load_solutions=False) + + model.dual = Suffix(direction=Suffix.IMPORT) + model.solutions.load_from(results) + + self.assertAlmostEqual(model.dual[model.C1], 0.4) + self.assertAlmostEqual(model.dual[model.C2], 0.2) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = AbstractModel() + model.X = Var(within=Integers) + model.O = Objective(expr=model.X) + + instance = model.create_instance() + results = opt.solve(instance) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddVar(unittest.TestCase): + def test_add_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_var(model.X) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_multiple_variables(self): + """Test that: + - The variable is added correctly to `solver_model` + - Fixed variable bounds are set correctly + """ + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X1 = Var(within=Binary) + model.X2 = Var(within=NonNegativeReals) + model.X3 = Var(within=NonNegativeIntegers) + + model.X3.fix(5) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 3) + scip_vars = opt._solver_model.getVars() + vtypes = [scip_var.vtype() for scip_var in scip_vars] + assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes + lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] + ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] + assert 0 in lbs and 5 in lbs + assert ( + 1 in ubs + and 5 in ubs + and any([opt._solver_model.isInfinity(ub) for ub in ubs]) + ) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddCon(unittest.TestCase): + def test_add_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.C = Constraint(expr=model.X == 1) + + opt._add_constraint(model.C) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C = Constraint(expr=model.X == 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_multiple_constraints(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C1 = Constraint(expr=model.X == 1) + model.B.C2 = Constraint(expr=model.X <= 1) + model.B.C3 = Constraint(expr=model.X >= 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 3) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestLoadVars(unittest.TestCase): + def setUp(self): + opt = SolverFactory("scip_direct", solver_io="python") + model = ConcreteModel() + model.X = Var(within=NonNegativeReals, initialize=0) + model.Y = Var(within=NonNegativeReals, initialize=0) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + opt.solve(model, load_solutions=False, save_results=False) + + self._model = model + self._opt = opt + + def test_all_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars() + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + def test_only_specified_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.X]) + + self.assertFalse(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.Y]) + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py new file mode 100644 index 00000000000..0cf1aab65f6 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -0,0 +1,318 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.environ +import pyomo.common.unittest as unittest + +from pyomo.core import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeReals, + NonNegativeIntegers, + Reals, + Binary, + SOSConstraint, + Set, + sin, + cos, + exp, + log, +) +from pyomo.opt import SolverFactory + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestQuadraticObjective(unittest.TestCase): + def test_quadratic_objective_linear_surrogate_is_set(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.Y = Var(bounds=(-2, 2)) + m.Z = Var(within=Reals) + m.O = Objective(expr=m.Z) + m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) + m.C2 = Constraint(expr=m.Y >= -m.X + 2) + m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 1, places=3) + + opt.reset() + + opt.remove_constraint(m.C3) + del m.C3 + m.C3 = Constraint(expr=m.Z >= m.X**2) + opt.add_constraint(m.C3) + opt.solve() + self.assertAlmostEqual(m.X.value, 0, places=3) + self.assertAlmostEqual(m.Y.value, 2, places=3) + + def test_add_and_remove_sos(self): + m = ConcreteModel() + m.I = Set(initialize=[1, 2, 3]) + m.X = Var(m.I, bounds=(-2, 2)) + + m.C = SOSConstraint(var=m.X, sos=1) + + m.O = Objective(expr=m.X[1] + m.X[2]) + + opt = SolverFactory("scip_persistent") + + opt.set_instance(m) + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 2 + + opt.reset() + + opt.remove_sos_constraint(m.C) + del m.C + + m.C = SOSConstraint(var=m.X, sos=2) + opt.add_sos_constraint(m.C) + + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 1 + + def test_get_and_set_param(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.O = Objective(expr=m.X) + m.C3 = Constraint(expr=m.X <= 2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.set_scip_param("limits/time", 60) + + assert opt.get_scip_param("limits/time") == 60 + + def test_non_linear(self): + + PI = 3.141592653589793238462643 + NWIRES = 11 + DIAMETERS = [ + 0.207, + 0.225, + 0.244, + 0.263, + 0.283, + 0.307, + 0.331, + 0.362, + 0.394, + 0.4375, + 0.500, + ] + PRELOAD = 300.0 + MAXWORKLOAD = 1000.0 + MAXDEFLECT = 6.0 + DEFLECTPRELOAD = 1.25 + MAXFREELEN = 14.0 + MAXCOILDIAM = 3.0 + MAXSHEARSTRESS = 189000.0 + SHEARMOD = 11500000.0 + + m = ConcreteModel() + m.coil = Var(within=NonNegativeReals) + m.wire = Var(within=NonNegativeReals) + m.defl = Var( + bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) + ) + m.ncoils = Var(within=NonNegativeIntegers) + m.const1 = Var(within=NonNegativeReals) + m.const2 = Var(within=NonNegativeReals) + m.volume = Var(within=NonNegativeReals) + m.I = Set(initialize=[i for i in range(NWIRES)]) + m.y = Var(m.I, within=Binary) + + m.O = Objective(expr=m.volume) + + m.c1 = Constraint( + expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 + ) + + m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) + + m.c3 = Constraint( + expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 + == 0 + ) + + m.c4 = Constraint( + expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 + - MAXSHEARSTRESS * m.wire**2 + <= 0 + ) + + m.c5 = Constraint( + expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 + ) + + m.c6 = Constraint( + expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire + <= MAXFREELEN + ) + + m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) + + m.c8 = Constraint( + expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 + ) + + m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + + self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) + + def test_non_linear_unary_expressions(self): + + m = ConcreteModel() + m.X = Var(bounds=(1, 2)) + m.Y = Var(within=Reals) + + m.O = Objective(expr=m.Y) + + m.C = Constraint(expr=exp(m.X) == m.Y) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, exp(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=log(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 0, places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=sin(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, sin(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=cos(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 2, places=3) + self.assertAlmostEqual(m.Y.value, cos(2), places=3) + + def test_add_column(self): + m = ConcreteModel() + m.x = Var(within=NonNegativeReals) + m.c = Constraint(expr=(0, m.x, 1)) + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + self.assertAlmostEqual(m.x.value, 1) + + m.y = Var(within=NonNegativeReals) + + opt.reset() + + opt.add_column(m, m.y, -3, [m.c], [2]) + opt.solve() + + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0.5) + + def test_add_column_exceptions(self): + m = ConcreteModel() + m.x = Var() + m.c = Constraint(expr=(0, m.x, 1)) + m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) + m.cd = Constraint(expr=(0, -m.x, 1)) + m.cd.deactivate() + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + + # set_instance not called + self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) + + opt.set_instance(m) + + m2 = ConcreteModel() + m2.y = Var() + m2.c = Constraint(expr=(0, m.x, 1)) + + # different model than attached to opt + self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) + # pyomo var attached to different model + self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) + + z = Var() + # pyomo var floating + self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) + + m.y = Var() + # len(coefficients) == len(constraints) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) + + # add indexed constraint + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) + # add something not a _ConstraintData + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) + + # constraint not on solver model + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) + + # inactive constraint + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) + + opt.add_var(m.y) + # var already in solver model + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 918a801ae37..3ad944de8d1 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -376,6 +376,27 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) + # + # SCIP PERSISTENT + # + + _scip_persistent_capabilities = set( + [ + "linear", + "integer", + "quadratic_constraint", + "sos1", + "sos2", + ] + ) + + _test_solver_cases["scip_persistent", "python"] = initialize( + name="scip_persistent", + io="python", + capabilities=_scip_persistent_capabilities, + import_suffixes=["slack", "dual", "rc"], + ) + # # CONOPT # From 6a14f108636dc9afb4e854b2fb27512aeb719ad0 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 17:20:01 +0100 Subject: [PATCH 02/97] Add SCIPPersistent to docs --- doc/OnlineDocs/library_reference/solvers/index.rst | 1 + .../library_reference/solvers/scip_persistent.rst | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 doc/OnlineDocs/library_reference/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/index.rst b/doc/OnlineDocs/library_reference/solvers/index.rst index 400032df076..628f9cfdab0 100644 --- a/doc/OnlineDocs/library_reference/solvers/index.rst +++ b/doc/OnlineDocs/library_reference/solvers/index.rst @@ -9,3 +9,4 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst + scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst new file mode 100644 index 00000000000..63ed55b74e3 --- /dev/null +++ b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst @@ -0,0 +1,7 @@ +SCIPPersistent +================ + +.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent + :members: + :inherited-members: + :show-inheritance: \ No newline at end of file From c1079090567bbe95b290402a3918c936a0ded576 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 11:48:19 +0100 Subject: [PATCH 03/97] Add SCIp to Github action scripts --- .github/workflows/test_branches.yml | 6 ++++++ .github/workflows/test_pr_and_main.yml | 6 ++++++ pyomo/solvers/plugins/solvers/scip_persistent.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 55f903a37f9..89e789db5ba 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -268,6 +268,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 76ec6de951a..a6cf6ef7eec 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -298,6 +298,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 408aa84633f..e28c91073ab 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -182,4 +182,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): + """ This function is necessary to call before making any changes to the + SCIP model after optimizing. It frees solution run specific information + that is not automatically done when changes to an already solved model + are made. Making changes to an already optimized model, e.g. adding additional + constraints will raise an error unless this function is called. """ self._solver_model.freeTransform() From e00ded8e33e823b8a9146facff85166291e17d71 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 14:55:52 +0100 Subject: [PATCH 04/97] Remove 5.0.0 specific version. Add conda to workflow --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 89e789db5ba..1d61aaf2d77 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -271,7 +271,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -347,7 +347,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index a6cf6ef7eec..89fd90c41d0 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -301,7 +301,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -376,7 +376,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a From a0b625060217e04aa126dc9f2cb9a410e0968078 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:14:54 +0100 Subject: [PATCH 05/97] Standardise string formatting to fstring --- pyomo/solvers/plugins/solvers/scip_direct.py | 35 +++++++------------ .../plugins/solvers/scip_persistent.py | 8 ++--- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0aafb596007..e93e5579f26 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -74,7 +74,7 @@ def _init(self): except ImportError: self._python_api_exists = False except Exception as e: - print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") self._python_api_exists = False # Note: Undefined capabilities default to None @@ -104,7 +104,7 @@ def _apply_solver(self): if self._keepfiles: # Only save log file when the user wants to keep it. self._solver_model.setLogfile(self._log_file) - print("Solver log file: " + self._log_file) + print(f"Solver log file: {self._log_file}") # Set user specified parameters for key, option in self.options.items(): @@ -257,14 +257,14 @@ def __exit__(self, t, v, traceback): def _set_instance(self, model, kwds={}): DirectOrPersistentSolver._set_instance(self, model, kwds) + self.available() try: self._solver_model = self._scip.Model() except Exception: e = sys.exc_info()[1] msg = ( "Unable to create SCIP model. " - "Have you installed PySCIPOpt correctly?\n\n\t" - + "Error message: {0}".format(e) + f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" ) raise Exception(msg) @@ -275,14 +275,13 @@ def _set_instance(self, model, kwds={}): if var.fixed: if not self._output_fixed_variable_bounds: raise ValueError( - "Encountered a fixed variable (%s) inside " + f"Encountered a fixed variable {var.name} inside " "an active objective or constraint " - "expression on model %s, which is usually " + f"expression on model {self._pyomo_model.name}, which is usually " "indicative of a preprocessing error. Use " "the IO-option 'output_fixed_variable_bounds=True' " "to suppress this error and fix the variable " "by overwriting its bounds in the SCIP instance." - % (var.name, self._pyomo_model.name) ) def _add_block(self, block): @@ -308,14 +307,10 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): - raise ValueError( - "Lower bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Lower bound of constraint {con} is not constant.") if con.has_ub(): if not is_fixed(con.upper): - raise ValueError( - "Upper bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Upper bound of constraint {con} is not constant.") if con.equality: scip_cons = self._solver_model.addCons( @@ -335,8 +330,7 @@ def _add_constraint(self, con): ) else: raise ValueError( - "Constraint does not have a lower " - "or an upper bound: {0} \n".format(con) + f"Constraint does not have a lower or an upper bound: {con} \n" ) for var in referenced_vars: @@ -398,9 +392,7 @@ def _scip_vtype_from_var(self, var): elif var.is_continuous(): vtype = "C" else: - raise ValueError( - "Variable domain type is not recognized for {0}".format(var.domain) - ) + raise ValueError(f"Variable domain type is not recognized for {var.domain}") return vtype def _set_objective(self, obj): @@ -418,7 +410,7 @@ def _set_objective(self, obj): elif obj.sense == maximize: sense = "maximize" else: - raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + raise ValueError(f"Objective sense is not recognized: {obj.sense}") scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( obj.expr, self._max_obj_degree @@ -455,8 +447,7 @@ def _postsolve(self): flag = True if not flag: raise RuntimeError( - "***The scip_direct solver plugin cannot extract solution suffix=" - + suffix + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" ) scip = self._solver_model @@ -593,7 +584,7 @@ def _postsolve(self): else: self.results.solver.status = SolverStatus.error self.results.solver.termination_message = ( - "Unhandled SCIP status (" + str(status) + ")" + f"Unhandled SCIP status ({str(status)})" ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e28c91073ab..abb85b8dbca 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -84,9 +84,7 @@ def update_var(self, var): # return if var not in self._pyomo_var_to_solver_var_map: raise ValueError( - "The Var provided to compile_var needs to be added first: {0}".format( - var - ) + f"The Var provided to compile_var needs to be added first: {var}" ) scip_var = self._pyomo_var_to_solver_var_map[var] vtype = self._scip_vtype_from_var(var) @@ -182,9 +180,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): - """ This function is necessary to call before making any changes to the + """This function is necessary to call before making any changes to the SCIP model after optimizing. It frees solution run specific information that is not automatically done when changes to an already solved model are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called. """ + constraints will raise an error unless this function is called.""" self._solver_model.freeTransform() From d0816eb008bae43eff5bff54390873c98e5b7a1b Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:17:29 +0100 Subject: [PATCH 06/97] Add parameter link to docstring --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index abb85b8dbca..49fe224e72a 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -116,6 +116,7 @@ def set_scip_param(self, param, val): param: str The SCIP parameter to set. Options include any SCIP parameter. Please see the SCIP documentation for options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php val: any The value to set the parameter to. See SCIP documentation for possible values. """ @@ -129,6 +130,7 @@ def get_scip_param(self, param): ---------- param: str or int or float The SCIP parameter to get the value of. See SCIP documentation for possible options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php """ return self._solver_model.getParam(param) From 0e11f112161b6947ddf30d6530d6167ef174cf44 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:20:07 +0100 Subject: [PATCH 07/97] Remove redundant second objective sense check --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index e93e5579f26..c6285ff53cb 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -591,15 +591,6 @@ def _postsolve(self): self.results.problem.name = scip.getProbName() - if scip.getObjectiveSense() == "minimize": - self.results.problem.sense = minimize - elif scip.getObjectiveSense() == "maximize": - self.results.problem.sense = maximize - else: - raise RuntimeError( - f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" - ) - self.results.problem.upper_bound = None self.results.problem.lower_bound = None if scip.getNSols() > 0: From 068ec99277321743b611c5323be3858a5b505ce8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:35:54 +0100 Subject: [PATCH 08/97] Clean up _post_solve with a helper function for status handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 109 +++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index c6285ff53cb..9074d40870f 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -382,8 +382,16 @@ def _add_sos_constraint(self, con): def _scip_vtype_from_var(self, var): """ This function takes a pyomo variable and returns the appropriate SCIP variable type - :param var: pyomo.core.base.var.Var - :return: B, I, or C + + Parameters + ---------- + var: pyomo.core.base.var.Var + The pyomo variable that we want to retrieve the SCIP vtype of + + Returns + ------- + vtype: str + B for Binary, I for Integer, or C for Continuous """ if var.is_binary(): vtype = "B" @@ -425,52 +433,12 @@ def _set_objective(self, obj): self._needs_updated = True - def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - extract_duals = False - extract_slacks = False - extract_reduced_costs = False - for suffix in self._suffixes: - flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model + def _get_solver_solution_status(self, scip, soln): + """ """ + # Get the status of the SCIP Model currently status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() + # Go through each potential case and update appropriately if scip.getStage() == 1: # SCIP Model is created but not yet optimized self.results.solver.status = SolverStatus.aborted self.results.solver.termination_message = ( @@ -588,6 +556,55 @@ def _postsolve(self): ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error + return soln + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + soln = self._get_solver_solution_status(scip, soln) self.results.problem.name = scip.getProbName() From 63af6d8ce13a28b50a59c3d9a027d57db64d8ca6 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:38:45 +0100 Subject: [PATCH 09/97] Remove individual skip_test option --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index ee37f5ddcc8..cc9e114fed1 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -37,6 +37,7 @@ scip_available = False +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") class SCIPDirectTests(unittest.TestCase): def setUp(self): self.stderr = sys.stderr @@ -45,7 +46,6 @@ def setUp(self): def tearDown(self): sys.stderr = self.stderr - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -60,7 +60,6 @@ def test_infeasible_lp(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -77,7 +76,6 @@ def test_unbounded_lp(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -88,7 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_get_duals_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -108,7 +105,6 @@ def test_get_duals_lp(self): self.assertAlmostEqual(model.dual[model.C1], 0.4) self.assertAlmostEqual(model.dual[model.C2], 0.2) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -123,7 +119,6 @@ def test_infeasible_mip(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = AbstractModel() @@ -141,7 +136,6 @@ def test_unbounded_mip(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 4075d9ad8ed1200c3d06e74ca5d3a678dd4e7239 Mon Sep 17 00:00:00 2001 From: Mark Turner <64978342+Opt-Mucca@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:48:02 +0100 Subject: [PATCH 10/97] Update pyomo/solvers/plugins/solvers/scip_persistent.py Co-authored-by: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 49fe224e72a..572a1b638e0 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -96,7 +96,7 @@ def update_var(self, var): def write(self, filename, filetype=""): """ - Write the model to a file (e.g., and lp file). + Write the model to a file (e.g., an lp file). Parameters ---------- From f55fcc5aaa329049a6ac521738537fb8498f1a36 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:52:10 +0100 Subject: [PATCH 11/97] Update from the black command --- pyomo/solvers/tests/solvers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 3ad944de8d1..1a5c1671f19 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -381,13 +381,7 @@ def test_solver_cases(*args): # _scip_persistent_capabilities = set( - [ - "linear", - "integer", - "quadratic_constraint", - "sos1", - "sos2", - ] + ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] ) _test_solver_cases["scip_persistent", "python"] = initialize( From 91eae7b573624fe35e5fb579debde19efbed740a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 16:37:13 +0100 Subject: [PATCH 12/97] Fix typos --- pyomo/solvers/plugins/solvers/scip_direct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9074d40870f..5ba3395d1d2 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -94,7 +94,7 @@ def _init(self): def _apply_solver(self): StaleFlagManager.mark_all_as_stale() - # Supress solver output if requested + # Suppress solver output if requested if self._tee: self._solver_model.hideOutput(quiet=False) else: @@ -179,7 +179,7 @@ def get_nl_expr_recursively(pyomo_expr): elif isinstance(pyomo_expr, DivisionExpression): if len(scip_expr_list) != 2: raise ValueError( - f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" ) return scip_expr_list[0] / scip_expr_list[1] elif isinstance(pyomo_expr, UnaryFunctionExpression): From 5ee2007a05921ef90a5ef9b31e76a877c9899007 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 17:28:07 +0100 Subject: [PATCH 13/97] Replace trySol via more safe checkSol --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 5ba3395d1d2..25c668a0a06 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -729,7 +729,14 @@ def _warm_start(self): for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - self._solver_model.trySol(scip_sol, free=True) + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From f6ff0923ba9dbbc2bfd04990846c1446ca9e9ed8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Mar 2024 10:01:01 +0100 Subject: [PATCH 14/97] Adds support for partial solution loading --- pyomo/solvers/plugins/solvers/scip_direct.py | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 25c668a0a06..0d4ad722459 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -725,18 +725,30 @@ def warm_start_capable(self): return True def _warm_start(self): - scip_sol = self._solver_model.createSol() + partial_sol = False + for pyomo_var in self._pyomo_var_to_solver_var_expr_map: + if pyomo_var.value is None: + partial_sol = True + break + if partial_sol: + scip_sol = self._solver_model.createPartialSol() + else: + scip_sol = self._solver_model.createSol() for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - feasible = self._solver_model.checkSol(scip_sol) - if feasible: + if partial_sol: self._solver_model.addSol(scip_sol) del scip_sol else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - del scip_sol + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From e7ac980a6e58a9b1035bafc38cf3ee55420900b8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 21 Mar 2024 09:58:24 +0100 Subject: [PATCH 15/97] Add error handling for setting non-linear objective --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0d4ad722459..456b370eff1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -134,6 +134,15 @@ def _apply_solver(self): def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars = ComponentSet() + degree = repn.polynomial_degree() + if (max_degree is not None) and (degree > max_degree): + raise DegreeError( + "While SCIP supports general non-linear constraints, the objective must be linear. " + "Please reformulate the objective by introducing a new variable. " + "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " + "f(x) is the original non-linear objective." + ) + new_expr = repn.constant if len(repn.linear_vars) > 0: From 2540f650df319bcca59cf26d0bc524a7fab7de8c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:44:29 +0200 Subject: [PATCH 16/97] Remove dual and rc loading for SCIP. Fix bug of ranged rows --- pyomo/solvers/plugins/solvers/scip_direct.py | 151 ++++++------------ .../plugins/solvers/scip_persistent.py | 21 ++- pyomo/solvers/tests/solvers.py | 2 +- 3 files changed, 73 insertions(+), 101 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 456b370eff1..04440b59f9b 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -86,10 +86,12 @@ def _init(self): self._capabilities.integer = True self._capabilities.sos1 = True self._capabilities.sos2 = True + self._skip_trivial_constraints = True # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() + self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -239,6 +241,7 @@ def _scip_lb_ub_from_var(self, var): ub = value(var.ub) else: ub = self._solver_model.infinity() + return lb, ub def _add_var(self, var): @@ -327,7 +330,10 @@ def _add_constraint(self, con): ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + value(con.lower) <= scip_expr, name=conname + ) + self._solver_model.chgRhs( + scip_cons, value(con.upper) - value(con.body.constant) ) elif con.has_lb(): scip_cons = self._solver_model.addCons( @@ -346,6 +352,7 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -440,8 +447,6 @@ def _set_objective(self, obj): self._objective = obj self._vars_referenced_by_obj = referenced_vars - self._needs_updated = True - def _get_solver_solution_status(self, scip, soln): """ """ # Get the status of the SCIP Model currently @@ -569,24 +574,17 @@ def _get_solver_solution_status(self, scip, soln): def _postsolve(self): # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list + # constraint slacks. constraint duals and variable + # reduced-costs were removed as in SCIP they contain + # too many caveats. scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_duals = False extract_slacks = False - extract_reduced_costs = False for suffix in self._suffixes: flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True if re.match(suffix, "slack"): extract_slacks = True flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True if not flag: raise RuntimeError( f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" @@ -599,14 +597,6 @@ def _postsolve(self): n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - self.results = SolverResults() soln = Solution() @@ -667,6 +657,7 @@ def _postsolve(self): This code in this if statement is only needed for backwards compatibility. It is more efficient to set _save_results to False and use load_vars, load_duals, etc. """ + if scip.getNSols() > 0: soln_variables = soln.variable soln_constraints = soln.constraint @@ -683,42 +674,35 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_reduced_costs: - vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] - for scip_var, val, name in zip(scip_vars, vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name]["Rc"] = val - - if extract_duals or extract_slacks: - scip_cons = scip.getConss() - con_names = [cons.name for cons in scip_cons] - assert set(self._solver_con_to_pyomo_con_map.keys()) == set( - con_names - ) - for name in con_names: - soln_constraints[name] = {} - - if extract_duals: - vals = [scip.getDualSolVal(con) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Dual"] = val - if extract_slacks: - vals = [scip.getSlack(con, scip_sol) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Slack"] = val + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) + con_names = [cons.name for cons in scip_cons] + if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): + raise AssertionError( + f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" + ) + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + soln_constraints[cons.name] = {} + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + if pyomo_con.has_lb(): + lhs = value(pyomo_con.lower) + else: + lhs = -1e20 + if pyomo_con.has_ub(): + rhs = value(pyomo_con.upper) + else: + rhs = 1e20 + soln_constraints[cons.name]["Slack"] = min( + activity - lhs, rhs - activity + ) elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_reduced_costs: - self._load_rc() - - if extract_duals: - self._load_duals() - if extract_slacks: self._load_slacks() @@ -773,65 +757,36 @@ def _load_vars(self, vars_to_load=None): var.set_value(val, skip_validation=True) def _load_rc(self, vars_to_load=None): - if not hasattr(self._pyomo_model, "rc"): - self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - rc = self._pyomo_model.rc - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [ - self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load - ] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - rc[var] = val + raise NotImplementedError( + "SCIP via Pyomo does not support reduced cost loading." + ) def _load_duals(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "dual"): - self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = self._pyomo_model.dual - scip_cons = self._solver_model.getConss() - - if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] - else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - dual[pyomo_con] = vals[i] + raise NotImplementedError( + "SCIP via Pyomo does not support dual solution loading" + ) def _load_slacks(self, cons_to_load=None): if not hasattr(self._pyomo_model, "slack"): self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map slack = self._pyomo_model.slack - scip_cons = self._solver_model.getConss() scip_sol = self._solver_model.getBestSol() if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [ - self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_cons] + for pyomo_cons in cons_to_load ] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - slack[pyomo_con] = vals[i] + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + rhs = self._solver_model.getRhs(cons) + lhs = self._solver_model.getLhs(cons) + slack[pyomo_con] = min(activity - lhs, rhs - activity) def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 572a1b638e0..880380ced1f 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -8,7 +8,6 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ - from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver from pyomo.opt.base import SolverFactory @@ -50,16 +49,34 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._pyomo_con_to_solver_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] + + def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) + del self._pyomo_var_to_solver_var_expr_map[var] + del self._pyomo_var_to_solver_var_map[var] + del self._solver_var_to_pyomo_var_map[scip_var.name] + del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 1a5c1671f19..b66c1ca5af5 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack", "dual", "rc"], + import_suffixes=["slack"], ) # From 9e5d9442ea0900d36629a3f0677eb6c6ce8d7f19 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:48:34 +0200 Subject: [PATCH 17/97] Add safe con.body.constant check --- pyomo/solvers/plugins/solvers/scip_direct.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 04440b59f9b..1b5e81db302 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -332,9 +332,10 @@ def _add_constraint(self, con): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname ) - self._solver_model.chgRhs( - scip_cons, value(con.upper) - value(con.body.constant) - ) + rhs = value(con.upper) + if hasattr(con.body, "constant"): + rhs -= value(con.body.constant) + self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname From f90dfade88dafd2d150409efd0a216c34578c89d Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:33:26 +0200 Subject: [PATCH 18/97] Remove slack loading for SCIP --- pyomo/solvers/plugins/solvers/scip_direct.py | 72 +++---------------- .../plugins/solvers/scip_persistent.py | 15 ---- 2 files changed, 8 insertions(+), 79 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 1b5e81db302..57cfc213f3d 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -91,7 +91,6 @@ def _init(self): # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() - self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -353,7 +352,6 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -574,22 +572,17 @@ def _get_solver_solution_status(self, scip, soln): return soln def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint slacks. constraint duals and variable + # Constraint duals and variable # reduced-costs were removed as in SCIP they contain - # too many caveats. scan through the solver suffix list + # too many caveats. Slacks were removed as later + # planned interfaces do not intend to support. + # Scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_slacks = False for suffix in self._suffixes: - flag = False - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) scip = self._solver_model status = scip.getStatus() @@ -661,8 +654,6 @@ def _postsolve(self): if scip.getNSols() > 0: soln_variables = soln.variable - soln_constraints = soln.constraint - scip_sol = scip.getBestSol() scip_vars = scip.getVars() scip_var_names = [scip_var.name for scip_var in scip_vars] @@ -675,38 +666,10 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_slacks: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - con_names = [cons.name for cons in scip_cons] - if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): - raise AssertionError( - f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" - ) - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - soln_constraints[cons.name] = {} - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - if pyomo_con.has_lb(): - lhs = value(pyomo_con.lower) - else: - lhs = -1e20 - if pyomo_con.has_ub(): - rhs = value(pyomo_con.upper) - else: - rhs = 1e20 - soln_constraints[cons.name]["Slack"] = min( - activity - lhs, rhs - activity - ) - elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_slacks: - self._load_slacks() - self.results.solution.insert(soln) # finally, clean any temporary files registered with the temp file @@ -768,26 +731,7 @@ def _load_duals(self, cons_to_load=None): ) def _load_slacks(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "slack"): - self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - slack = self._pyomo_model.slack - scip_sol = self._solver_model.getBestSol() - - if cons_to_load is None: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - else: - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_cons] - for pyomo_cons in cons_to_load - ] - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - rhs = self._solver_model.getRhs(cons) - lhs = self._solver_model.getLhs(cons) - slack[pyomo_con] = min(activity - lhs, rhs - activity) + raise NotImplementedError("SCIP via Pyomo does not support slack loading") def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 880380ced1f..e3fe9e37b5d 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -49,34 +49,19 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._pyomo_con_to_solver_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] - def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) del self._pyomo_var_to_solver_var_expr_map[var] - del self._pyomo_var_to_solver_var_map[var] - del self._solver_var_to_pyomo_var_map[scip_var.name] - del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) From f703d1f71128a95d509aa9ea0b08d12de2dcb41a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:38:10 +0200 Subject: [PATCH 19/97] Remove dual loading test for SCIP --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index cc9e114fed1..5863a54bdcb 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -86,25 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - def test_get_duals_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.Y = Var(within=NonNegativeReals) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - results = opt.solve(model, suffixes=["dual"], load_solutions=False) - - model.dual = Suffix(direction=Suffix.IMPORT) - model.solutions.load_from(results) - - self.assertAlmostEqual(model.dual[model.C1], 0.4) - self.assertAlmostEqual(model.dual[model.C2], 0.2) - def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 5c02d32009990b8054440f0a6049bdf934247a79 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:42:00 +0200 Subject: [PATCH 20/97] Remove slack for suffix in tests --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 -- pyomo/solvers/tests/solvers.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 57cfc213f3d..a965e66362e 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -10,7 +10,6 @@ # ___________________________________________________________________________ import logging -import re import sys from pyomo.common.collections import ComponentSet, ComponentMap, Bunch @@ -36,7 +35,6 @@ from pyomo.opt.results.solution import Solution, SolutionStatus from pyomo.opt.results.solver import TerminationCondition, SolverStatus from pyomo.opt.base import SolverFactory -from pyomo.core.base.suffix import Suffix logger = logging.getLogger("pyomo.solvers") diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index b66c1ca5af5..ba1530c67cc 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack"], + import_suffixes=[], ) # From 8ebcf88365267e28a5b820eedce12a0d1bf5473c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 11:13:28 +0200 Subject: [PATCH 21/97] Remove TODO for nonlinear handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index a965e66362e..9061deac6ad 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -161,7 +161,6 @@ def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars.add(x) referenced_vars.add(y) - # TODO: Introduce handling on non-linear expressions if repn.nonlinear_expr is not None: def get_nl_expr_recursively(pyomo_expr): From 30d8cc62d5903e9e8dce4f0cabe79f22a5aba495 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 14:48:43 +0200 Subject: [PATCH 22/97] Skip LP_trivial_constraints for SCIP persistent --- pyomo/solvers/plugins/solvers/scip_direct.py | 6 ++++-- pyomo/solvers/tests/testcases.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9061deac6ad..39c3a4fd996 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -67,8 +67,10 @@ def _init(self): self._scip = pyscipopt self._python_api_exists = True - self._version = str(self._scip.Model().version()) - self._version_major = self._version.split(".")[0] + self._version = tuple( + int(k) for k in str(self._scip.Model().version()).split(".") + ) + self._version_major = self._version[0] except ImportError: self._python_api_exists = False except Exception as e: diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index 6bef40818d9..f586e22b1e1 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,6 +248,15 @@ "inside NL files. A ticket has been filed.", ) +# +# SCIP Persistent +# + +ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( + lambda v: v <= _trunk_version, + "SCIP does not allow empty constraints with no variables to be added to the Model.", +) + # # BARON # From 30e5e65bfd063b049f950b9f74c0d238187077ca Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:12:24 +0200 Subject: [PATCH 23/97] Add transformation for add_cons with non float/int rhs e.g. np.int --- pyomo/solvers/plugins/solvers/scip_direct.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 39c3a4fd996..7c26670c2b4 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -318,29 +318,38 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") + con_lower = value(con.lower) + if not isinstance(con_lower, (float, int)): + con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") + con_upper = value(con.upper) + if not isinstance(con_upper, (float, int)): + con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons( - scip_expr == value(con.lower), name=conname + scip_expr == con_lower, name=conname ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) - rhs = value(con.upper) + rhs = con_upper if hasattr(con.body, "constant"): - rhs -= value(con.body.constant) + con_constant = value(con.body.constant) + if not isinstance(con_constant, (float, int)): + con_body = float(con_constant) + rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) elif con.has_ub(): scip_cons = self._solver_model.addCons( - scip_expr <= value(con.upper), name=conname + scip_expr <= con_upper, name=conname ) else: raise ValueError( From 9104a921c55f9cd170bd1a7e93e1628869de2360 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:35:41 +0200 Subject: [PATCH 24/97] Add warning if type is converted. Tidy up logic --- pyomo/solvers/plugins/solvers/scip_direct.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 7c26670c2b4..6ce98d80e27 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -319,38 +319,38 @@ def _add_constraint(self, con): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") con_lower = value(con.lower) - if not isinstance(con_lower, (float, int)): + if type(con_lower) != float and type(con_lower) != int: + logger.warning( + f"Constraint {conname} has LHS type {type(value(con.lower))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if not isinstance(con_upper, (float, int)): + if type(con_upper) != float and type(con_upper) != int: + logger.warning( + f"Constraint {conname} has RHS type {type(value(con.upper))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_upper = float(con_upper) if con.equality: - scip_cons = self._solver_model.addCons( - scip_expr == con_lower, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) rhs = con_upper if hasattr(con.body, "constant"): con_constant = value(con.body.constant) if not isinstance(con_constant, (float, int)): - con_body = float(con_constant) + con_constant = float(con_constant) rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) elif con.has_ub(): - scip_cons = self._solver_model.addCons( - scip_expr <= con_upper, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) else: raise ValueError( f"Constraint does not have a lower or an upper bound: {con} \n" From f3f2d7c0334afbea37e9d8d24d60227f333ee4c1 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Nov 2024 17:13:03 +0100 Subject: [PATCH 25/97] Fix num. vars and cons from transformed. Silent warm start fail --- pyomo/solvers/plugins/solvers/scip_direct.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 6ce98d80e27..89dd25b86ee 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -641,10 +641,9 @@ def _postsolve(self): except TypeError: soln.gap = None - # TODO: Should these values be of the transformed or the original problem? - self.results.problem.number_of_constraints = scip.getNConss() + self.results.problem.number_of_constraints = scip.getNConss(transformed=False) # self.results.problem.number_of_nonzeros = None - self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars self.results.problem.number_of_continuous_variables = n_con_vars @@ -704,16 +703,13 @@ def _warm_start(self): scip_sol[scip_var] = value(pyomo_var) if partial_sol: self._solver_model.addSol(scip_sol) - del scip_sol else: - feasible = self._solver_model.checkSol(scip_sol) + feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) if feasible: self._solver_model.addSol(scip_sol) - del scip_sol else: logger.warning("Warm start solution was not accepted by SCIP") self._solver_model.freeSol(scip_sol) - del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From 7b18354386df3954cf9fc59e5f0ec9b587e316ed Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 19 Feb 2025 16:38:06 +0100 Subject: [PATCH 26/97] Add minor changes --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 89dd25b86ee..314fce40da5 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -322,19 +322,13 @@ def _add_constraint(self, con): if type(con_lower) != float and type(con_lower) != int: logger.warning( f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as type is not allowed for SCIP." + f"Converting to float as SCIP fails otherwise." ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if type(con_upper) != float and type(con_upper) != int: - logger.warning( - f"Constraint {conname} has RHS type {type(value(con.upper))}. " - f"Converting to float as type is not allowed for SCIP." - ) - con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) @@ -642,7 +636,6 @@ def _postsolve(self): soln.gap = None self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - # self.results.problem.number_of_nonzeros = None self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars From 27e3d108662f554966819c4ea5db52df1ca38ffc Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 21 Mar 2025 11:26:23 +0100 Subject: [PATCH 27/97] Change copyright 2024 to 2025 --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 +- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- pyomo/solvers/tests/checks/test_SCIPDirect.py | 2 +- pyomo/solvers/tests/checks/test_SCIPPersistent.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 314fce40da5..c862d9047c1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e3fe9e37b5d..bc64edc28a8 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index 5863a54bdcb..186de0eaf58 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py index 0cf1aab65f6..61cf7385352 100644 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain From b57ab07cde98cc387f8ee0c4f1724f209b05084a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:06:26 -0600 Subject: [PATCH 28/97] updating solution loader --- .../contrib/solver/common/solution_loader.py | 106 +++++++++++++++--- pyomo/contrib/solver/solvers/highs.py | 4 +- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 911d8bee50d..065c00185f6 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,7 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, NoReturn +from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -23,24 +23,75 @@ class SolutionLoaderBase: Intent of this class and its children is to load the solution back into the model. """ - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_solution_ids(self) -> List[Any]: """ - Load the solution of the primal variables into the value attribute of the variables. + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions + are available, this will return None + + Returns + ------- + solutions_ids: List[Any] + The identifiers for multiple solutions + """ + return NotImplemented + + def get_number_of_solutions(self) -> int: + """ + Returns + ------- + num_solutions: int + Indicates the number of solutions found + """ + return NotImplemented + + def load_solution(self, solution_id=None): + """ + Load the solution (everything that can be) back into the model + + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + # this should load everything it can + self.load_vars(solution_id=solution_id) + self.load_import_suffixes(solution_id=solution_id) + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, + ) -> None: + """ + Load the solution of the primal variables into the value attribute + of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If vars_to_load - is None, then the solution to all primal variables will be loaded. Even if - vars_to_load is specified, the values of other variables may also be - loaded depending on the interface. + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of + other variables may also be loaded depending on the interface. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. """ - for var, val in self.get_primals(vars_to_load=vars_to_load).items(): + for var, val in self.get_vars( + vars_to_load=vars_to_load, + solution_id=solution_id + ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -50,6 +101,9 @@ def get_primals( vars_to_load: list A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- @@ -57,11 +111,13 @@ def get_primals( Maps variables to solution values """ raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'get_primals'." + f"Derived class {self.__class__.__name__} failed to implement required method 'get_vars'." ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -71,16 +127,21 @@ def get_duals( cons_to_load: list A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- duals: dict Maps constraints to dual values """ - raise NotImplementedError(f'{type(self)} does not support the get_duals method') + return NotImplemented def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -90,15 +151,26 @@ def get_reduced_costs( vars_to_load: list A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- reduced_costs: ComponentMap Maps variables to reduced costs """ - raise NotImplementedError( - f'{type(self)} does not support the get_reduced_costs method' - ) + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + """ + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + return NotImplemented class PersistentSolutionLoader(SolutionLoaderBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 2fdac4942c8..6eb4afa828a 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -306,7 +306,9 @@ def _solve(self): self._solver_model.run() timer.stop('optimize') - return self._postsolve() + res = self._postsolve() + res.solver_log = ostreams[0].getvalue() + return res def _process_domain_and_bounds(self, var_id): _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[var_id] From 070811d91c992691f16bbf7b97d9de2310fc7398 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:20:11 -0600 Subject: [PATCH 29/97] refactoring gurobi interfaces --- .../solver/solvers/gurobi_persistent.py | 156 ++++++++++++------ 1 file changed, 108 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index ea3693c1c70..899b7915e80 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,44 +233,15 @@ def __init__(self): self.var2 = None -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - +class GurobiBase(SolverBase): CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) + super().__init__(**kwds) self._register_env_client() self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -280,12 +251,10 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - @property - def symbol_map(self): - return self._symbol_map + def _mipstart(self): + raise NotImplementedError('should be implemented by derived classes') - def _solve(self): - config = self._active_config + def _solve(self, config): timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -304,13 +273,7 @@ def _solve(self): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) + self._mipstart() for key, option in options.items(): self._solver_model.setParam(key, option) @@ -319,18 +282,99 @@ def _solve(self): self._solver_model.optimize(self._callback) timer.stop('optimize') - self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res + + +class GurobiDirect(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiQuadraticBase(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._vars = {} # from id(v) to v + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + @property + def symbol_map(self): + return self._symbol_map + + def _mipstart(self): + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id] + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def _proces_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain step: {step} (should be either 0 or 1)' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + +class GurobiDirectQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiPersistentQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_con_to_pyomo_con_map = {} + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._callback_func = None + self._last_results_object: Optional[Results] = None + + def _solve(self, config): + super()._solve(config) + self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] + _lb = var._lb + _ub = var._ub + _fixed = var.fixed + _domain_interval = var.domain.get_interval() + _value = var.value lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -372,6 +416,24 @@ def _process_domain_and_bounds( return lb, ub, vtype + +class _GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + + + def __init__(self, **kwds): + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) + def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -522,7 +584,6 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) - self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -654,7 +715,6 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From d70dbb52a28aeb3500bdd430919f43f94c7a862c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 15:36:39 -0600 Subject: [PATCH 30/97] revert_gurobi_persistent --- .../solver/solvers/gurobi_persistent.py | 156 ++++++------------ 1 file changed, 48 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index 899b7915e80..ea3693c1c70 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,15 +233,44 @@ def __init__(self): self.var2 = None -class GurobiBase(SolverBase): +class GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - super().__init__(**kwds) + treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) self._register_env_client() self._solver_model = None + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._solver_con_to_pyomo_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._range_constraints = OrderedSet() + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True self._callback = None + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -251,10 +280,12 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - def _mipstart(self): - raise NotImplementedError('should be implemented by derived classes') + @property + def symbol_map(self): + return self._symbol_map - def _solve(self, config): + def _solve(self): + config = self._active_config timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -273,7 +304,13 @@ def _solve(self, config): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - self._mipstart() + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id][0] + if pyomo_var.is_integer() and pyomo_var.value is not None: + self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) for key, option in options.items(): self._solver_model.setParam(key, option) @@ -282,99 +319,18 @@ def _solve(self, config): self._solver_model.optimize(self._callback) timer.stop('optimize') + self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res - - -class GurobiDirect(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiQuadraticBase(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._vars = {} # from id(v) to v - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - - @property - def symbol_map(self): - return self._symbol_map - - def _mipstart(self): - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id] - if pyomo_var.is_integer() and pyomo_var.value is not None: - gurobi_var.setAttr('Start', pyomo_var.value) - - def _proces_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if var.fixed: - lb = var.value - ub = lb - else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - -class GurobiDirectQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiPersistentQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_con_to_pyomo_con_map = {} - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._callback_func = None - self._last_results_object: Optional[Results] = None - - def _solve(self, config): - super()._solve(config) - self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _lb = var._lb - _ub = var._ub - _fixed = var.fixed - _domain_interval = var.domain.get_interval() - _value = var.value + _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -416,24 +372,6 @@ def _process_domain_and_bounds( return lb, ub, vtype - -class _GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - - def __init__(self, **kwds): - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -584,6 +522,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) + self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -715,6 +654,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] + self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From 5b1d3f9cfb551598599a8f2ecf99313517bf06f7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 20:43:45 -0600 Subject: [PATCH 31/97] refactoring gurobi interfaces --- .../contrib/solver/solvers/gurobi/__init__.py | 0 .../solver/solvers/gurobi/gurobi_direct.py | 201 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 328 ++++++++++++++++++ 3 files changed, 529 insertions(+) create mode 100644 pyomo/contrib/solver/solvers/gurobi/__init__.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py new file mode 100644 index 00000000000..5c36372ef72 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -0,0 +1,201 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging +from .gurobi_direct_base import GurobiDirectBase, gurobipy + + +class GurobiDirectSolutionLoader(SolutionLoaderBase): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + self._grb_model = grb_model + self._grb_cons = grb_cons + self._grb_vars = grb_vars + self._pyo_cons = pyo_cons + self._pyo_vars = pyo_vars + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._grb_model is not None: + self._grb_cons = None + self._grb_vars = None + self._pyo_cons = None + self._pyo_vars = None + # explicitly release the model + self._grb_model.dispose() + self._grb_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + for p_var, g_var in iterator: + p_var.set_value(g_var, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + def get_primals(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + def get_duals(self, cons_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + def dedup(_iter): + last = None + for con_info_dual in _iter: + if not con_info_dual[1] and con_info_dual[0][0] is last: + continue + last = con_info_dual[0][0] + yield con_info_dual + + iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) + if cons_to_load: + cons_to_load = set(cons_to_load) + iterator = filter( + lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator + ) + return {con_info[0]: dual for con_info, dual in iterator} + + def get_reduced_costs(self, vars_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + +class GurobiDirect(GurobiDirectBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._gurobi_vars = None + self._pyomo_vars = None + + def _pyomo_gurobi_var_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + + def _create_solver_model(self, pyomo_model, config): + timer = config.timer + + timer.start('compile_model') + repn = LinearStandardFormCompiler().write( + pyomo_model, mixed_form=True, set_sense=None + ) + timer.stop('compile_model') + + if len(repn.objectives) > 1: + raise IncompatibleModelError( + f"The {self.__class__.__name__} solver only supports models " + f"with zero or one objectives (received {len(repn.objectives)})." + ) + + timer.start('prepare_matrices') + inf = float('inf') + ninf = -inf + bounds = list(map(operator.attrgetter('bounds'), repn.columns)) + lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] + ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] + CON = gurobipy.GRB.CONTINUOUS + BIN = gurobipy.GRB.BINARY + INT = gurobipy.GRB.INTEGER + vtype = [ + ( + CON + if v.is_continuous() + else BIN if v.is_binary() else INT if v.is_integer() else '?' + ) + for v in repn.columns + ] + sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense = [sense_type[r[1]] for r in repn.rows] + timer.stop('prepare_matrices') + + gurobi_model = gurobipy.Model(env=self.env()) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) + # Note: calling gurobi_model.update() here is not + # necessary (it will happen as part of optimize()): + # gurobi_model.update() + timer.stop('transfer_model') + + self._pyomo_vars = repn.columns + self._gurobi_vars = x + + solution_loader = GurobiDirectSolutionLoader( + gurobi_model, A, x, repn.rows, repn.columns + ) + has_obj = len(repn.objectives) > 0 + + return gurobi_model, solution_loader, has_obj diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py new file mode 100644 index 00000000000..01c91b8b2ed --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -0,0 +1,328 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging + + +logger = logging.getLogger(__name__) + + +gurobipy, gurobipy_available = attempt_import('gurobipy') + + +class GurobiConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Gurobi.", + ), + ) + + +class GurobiDirectBase(SolverBase): + + _num_gurobipy_env_clients = 0 + _gurobipy_env = None + _available = None + _gurobipy_available = gurobipy_available + _tc_map = None + + CONFIG = GurobiConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._register_env_client() + + def __del__(self): + if not python_is_shutting_down(): + self._release_env_client() + + def available(self): + if self._available is None: + # this triggers the deferred import, and for the persistent + # interface, may update the _available flag + # + # Note that we set the _available flag on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't - is that true?) + if not self._gurobipy_available: + if self._available is None: + self.__class__._available = Availability.NotFound + else: + self.__class__._available = self._check_license() + return self._available + + @staticmethod + def release_license(): + if GurobiDirectBase._gurobipy_env is None: + return + if GurobiDirectBase._num_gurobipy_env_clients: + logger.warning( + "Call to GurobiDirectBase.release_license() with %s remaining " + "environment clients." % (GurobiDirectBase._num_gurobipy_env_clients,) + ) + GurobiDirectBase._gurobipy_env.close() + GurobiDirectBase._gurobipy_env = None + + @staticmethod + def env(): + if GurobiDirectBase._gurobipy_env is None: + with capture_output(capture_fd=True): + GurobiDirectBase._gurobipy_env = gurobipy.Env() + return GurobiDirectBase._gurobipy_env + + @staticmethod + def _register_env_client(): + GurobiDirectBase._num_gurobipy_env_clients += 1 + + @staticmethod + def _release_env_client(): + GurobiDirectBase._num_gurobipy_env_clients -= 1 + if GurobiDirectBase._num_gurobipy_env_clients <= 0: + # Note that _num_gurobipy_env_clients should never be <0, + # but if it is, release_license will issue a warning (that + # we want to know about) + GurobiDirectBase.release_license() + + def _check_license(self): + try: + model = gurobipy.Model(env=self.env()) + except gurobipy.GurobiError: + return Availability.BadLicense + + model.setParam('OutputFlag', 0) + try: + model.addVars(range(2001)) + model.optimize() + return Availability.FullLicense + except gurobipy.GurobiError: + return Availability.LimitedLicense + finally: + model.dispose() + + def version(self): + version = ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + return version + + def _create_solver_model(self, pyomo_model, config): + # should return gurobi_model, solution_loader, has_objective + raise NotImplementedError('should be implemented by derived classes') + + def _pyomo_gurobi_var_iter(self): + # generator of tuples (pyomo_var, gurobi_var) + raise NotImplementedError('should be implemented by derived classes') + + def _mipstart(self): + for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + config: GurobiConfig = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + + orig_cwd = os.getcwd() + try: + if config.working_dir: + os.chdir(config.working_dir) + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + options = config.solver_options + + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize() + timer.stop('optimize') + finally: + os.chdir(orig_cwd) + + res = self._postsolve( + grb_model=gurobi_model, + config=config, + ) + + res.solution_loader = solution_loader + res.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + return res + + def _get_tc_map(self): + if GurobiDirectBase._tc_map is None: + grb = gurobipy.GRB + tc = TerminationCondition + GurobiDirectBase._tc_map = { + grb.LOADED: tc.unknown, # problem is loaded, but no solution + grb.OPTIMAL: tc.convergenceCriteriaSatisfied, + grb.INFEASIBLE: tc.provenInfeasible, + grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, + grb.UNBOUNDED: tc.unbounded, + grb.CUTOFF: tc.objectiveLimit, + grb.ITERATION_LIMIT: tc.iterationLimit, + grb.NODE_LIMIT: tc.iterationLimit, + grb.TIME_LIMIT: tc.maxTimeLimit, + grb.SOLUTION_LIMIT: tc.unknown, + grb.INTERRUPTED: tc.interrupted, + grb.NUMERIC: tc.unknown, + grb.SUBOPTIMAL: tc.unknown, + grb.USER_OBJ_LIMIT: tc.objectiveLimit, + } + return GurobiDirectBase._tc_map + + def _postsolve(self, grb_model, config, has_obj): + status = grb_model.Status + + results = Results() + results.timing_info.gurobi_time = grb_model.Runtime + + if grb_model.SolCount > 0: + if status == gurobipy.GRB.OPTIMAL: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible + else: + results.solution_status = SolutionStatus.noSolution + + results.termination_condition = self._get_tc_map().get( + status, TerminationCondition.unknown + ) + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if math.isfinite(grb_model.ObjVal): + results.incumbent_objective = grb_model.ObjVal + else: + results.incumbent_objective = None + except (gurobipy.GurobiError, AttributeError): + results.incumbent_objective = None + try: + results.objective_bound = grb_model.ObjBound + except (gurobipy.GurobiError, AttributeError): + if grb_model.ModelSense == ObjectiveSense.minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf + else: + results.incumbent_objective = None + results.objective_bound = None + + results.iteration_count = grb_model.getAttr('IterCount') + + config.timer.start('load solution') + if config.load_solutions: + if grb_model.SolCount > 0: + results.solution_loader.load_vars() + else: + raise NoFeasibleSolutionError() + config.timer.stop('load solution') + + results.solver_config = config + results.solver_name = self.name + results.solver_version = self.version() + + return results From 4818130badff45f8873326b32710bfba2e87e7fd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 11 Aug 2025 06:23:20 -0600 Subject: [PATCH 32/97] refactoring gurobi interfaces --- .../solver/solvers/gurobi/gurobi_direct.py | 7 +- .../solvers/gurobi/gurobi_direct_base.py | 72 ++-- .../solvers/gurobi/gurobi_persistent.py | 335 ++++++++++++++++++ 3 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5c36372ef72..f4a33e2cc54 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -24,6 +24,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -127,6 +128,8 @@ def get_reduced_costs(self, vars_to_load=None): class GurobiDirect(GurobiDirectBase): + _minimum_version = (9, 0, 0) + def __init__(self, **kwds): super().__init__(**kwds) self._gurobi_vars = None @@ -135,8 +138,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars.tolist()) - def _create_solver_model(self, pyomo_model, config): - timer = config.timer + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 01c91b8b2ed..b314a39b49a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -85,12 +85,14 @@ class GurobiDirectBase(SolverBase): _available = None _gurobipy_available = gurobipy_available _tc_map = None + _minimum_version = (0, 0, 0) CONFIG = GurobiConfig() def __init__(self, **kwds): super().__init__(**kwds) self._register_env_client() + self._callback = None def __del__(self): if not python_is_shutting_down(): @@ -111,6 +113,8 @@ def available(self): self.__class__._available = Availability.NotFound else: self.__class__._available = self._check_license() + if self.version() < self._minimum_version: + self.__class__._available = Availability.BadVersion return self._available @staticmethod @@ -169,7 +173,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model, config): + def _create_solver_model(self, pyomo_model): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -184,29 +188,30 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config: GurobiConfig = self.config( - value=kwds, - preserve_implicit=True, - ) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - ostreams = [io.StringIO()] + config.tee - + orig_config = self.config orig_cwd = os.getcwd() try: + self.config = config = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + gurobi_model, solution_loader, has_obj = self._create_solver_model(model) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -227,15 +232,16 @@ def solve(self, model, **kwds) -> Results: gurobi_model.setParam(key, option) timer.start('optimize') - gurobi_model.optimize() + gurobi_model.optimize(self._callback) timer.stop('optimize') + + res = self._postsolve( + grb_model=gurobi_model, + has_obj=has_obj, + ) finally: os.chdir(orig_cwd) - - res = self._postsolve( - grb_model=gurobi_model, - config=config, - ) + self.config = orig_config res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() @@ -267,7 +273,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, config, has_obj): + def _postsolve(self, grb_model, has_obj): status = grb_model.Status results = Results() @@ -288,7 +294,7 @@ def _postsolve(self, grb_model, config, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result + and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -313,15 +319,19 @@ def _postsolve(self, grb_model, config, has_obj): results.iteration_count = grb_model.getAttr('IterCount') - config.timer.start('load solution') - if config.load_solutions: + self.config.timer.start('load solution') + if self.config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - config.timer.stop('load solution') + self.config.timer.stop('load solution') - results.solver_config = config + # self.config gets copied a the beginning of + # solve and restored at the end, so modifying + # results.solver_config will not actually + # modify self.config + results.solver_config = self.config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py new file mode 100644 index 00000000000..fa269c1d3c5 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import io +import logging +import math +from typing import List, Optional +from collections.abc import Iterable + +from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet +from pyomo.common.dependencies import attempt_import +from pyomo.common.errors import ApplicationError +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler +from pyomo.core.base.var import VarData +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.param import ParamData +from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types +from pyomo.repn import generate_standard_repn +from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.results import ( + Results, + TerminationCondition, + SolutionStatus, +) +from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig +from pyomo.contrib.solver.solvers.gurobi_direct import ( + GurobiConfigMixin, + GurobiSolverMixin, +) +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.persistent import ( + PersistentSolverUtils, + PersistentSolverMixin, +) +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.core.staleflag import StaleFlagManager +from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from pyomo.contrib.solver.common.util import get_objective +from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor + + +logger = logging.getLogger(__name__) + + +class GurobiSolutionLoader(PersistentSolutionLoader): + def load_vars(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + self._solver._load_vars( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + def get_primals(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + +class _MutableLowerBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('lb', value(self.expr)) + + +class _MutableUpperBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('ub', value(self.expr)) + + +class _MutableLinearCoefficient: + def __init__(self): + self.expr = None + self.var = None + self.con = None + self.gurobi_model = None + + def update(self): + self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + + +class _MutableRangeConstant: + def __init__(self): + self.lhs_expr = None + self.rhs_expr = None + self.con = None + self.slack_name = None + self.gurobi_model = None + + def update(self): + rhs_val = value(self.rhs_expr) + lhs_val = value(self.lhs_expr) + self.con.rhs = rhs_val + slack = self.gurobi_model.getVarByName(self.slack_name) + slack.ub = rhs_val - lhs_val + + +class _MutableConstant: + def __init__(self): + self.expr = None + self.con = None + + def update(self): + self.con.rhs = value(self.expr) + + +class _MutableQuadraticConstraint: + def __init__( + self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + ): + self.con = gurobi_con + self.gurobi_model = gurobi_model + self.constant = constant + self.last_constant_value = value(self.constant.expr) + self.linear_coefs = linear_coefs + self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + gurobi_expr = self.gurobi_model.getQCRow(self.con) + for ndx, coef in enumerate(self.linear_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_linear_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var + self.last_linear_coef_values[ndx] = current_coef_value + for ndx, coef in enumerate(self.quadratic_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + def get_updated_rhs(self): + return value(self.constant.expr) + + +class _MutableObjective: + def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): + self.gurobi_model = gurobi_model + self.constant = constant + self.linear_coefs = linear_coefs + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + for ndx, coef in enumerate(self.linear_coefs): + coef.var.obj = value(coef.expr) + self.gurobi_model.ObjCon = value(self.constant.expr) + + gurobi_expr = None + for ndx, coef in enumerate(self.quadratic_coefs): + if value(coef.expr) != self.last_quadratic_coef_values[ndx]: + if gurobi_expr is None: + self.gurobi_model.update() + gurobi_expr = self.gurobi_model.getObjective() + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + +class _MutableQuadraticCoefficient: + def __init__(self): + self.expr = None + self.var1 = None + self.var2 = None + + +class GurobiDirectQuadratic(GurobiDirectBase): + _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # from id(v) to v + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _create_solver_model(self, pyomo_model): + self._clear() + self._solver_model = gurobipy.Model(env=self.env()) + cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + self._add_constraints(cons) + sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + self._add_sos_constraints(sos) + obj = get_objective(pyomo_model) + self._set_objective(obj) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _pyomo_gurobi_var_iter(self): + for vid, v in self._vars.items(): + yield v, self._pyomo_var_to_solver_var_map[vid] + + def _process_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain: {var.domain}' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + def _add_variables(self, variables: List[VarData]): + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ) + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + + def _get_expr_from_pyomo_expr(self, expr): + repn = generate_standard_repn(expr, quadratic=True, compute_values=True) + + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + new_expr = gurobipy.LinExpr( + repn.linear_coefs, + [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + ) + else: + new_expr = 0.0 + + for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): + x, y = v + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y + + return new_expr, repn.constant + + def _add_constraints(self, cons: List[ConstraintData]): + gurobi_expr_list = [] + for con in cons: + lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + elif ub is None: + gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + elif lb == ub: + gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + else: + gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + + gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._pyomo_con_to_solver_con_map[con] = gurobipy_con + self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + +class GurobiPersistentQuadratic(GurobiDirectQuadratic): + _minimum_version = (7, 0, 0) From 7998fda55861cde77931d14d58b27444d9f3a501 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 10:04:55 -0600 Subject: [PATCH 33/97] refactoring gurobi interfaces --- pyomo/contrib/solver/plugins.py | 8 +- .../solvers/gurobi/gurobi_direct_base.py | 132 ++++++++++- .../solvers/gurobi/gurobi_persistent.py | 220 ++++++++++++++++-- .../solver/tests/solvers/test_solvers.py | 26 ++- 4 files changed, 350 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 86c05f2bd70..7630c614aa2 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt from .solvers.gurobi_persistent import GurobiPersistent -from .solvers.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from .solvers.highs import Highs @@ -31,6 +32,11 @@ def load(): legacy_name='gurobi_direct_v2', doc='Direct (scipy-based) interface to Gurobi', )(GurobiDirect) + SolverFactory.register( + name='gurobi_direct_quadratic', + legacy_name='gurobi_direct_quadratic_v2', + doc='Direct interface to Gurobi', + )(GurobiDirect) SolverFactory.register( name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index b314a39b49a..d26dbf54c83 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -78,6 +78,122 @@ def __init__( ) +def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if ( + solver_model.getAttr('NumIntVars') == 0 + and solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) + original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] + solver_model.setParam('SolutionNumber', solution_number) + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Xn", gurobi_vars_to_load) + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + solver_model.setParam('SolutionNumber', original_solution_number) + return res + + +def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + for v, val in _get_primals( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ).items(): + v.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + +def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if solver_model.SolCount == 0: + raise NoSolutionError() + + if solution_number != 0: + return _load_suboptimal_mip_solution( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ) + + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("X", gurobi_vars_to_load) + + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + return res + + +def _get_reduced_costs(solver_model, var_map, vars_to_load): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + res = ComponentMap() + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Rc", gurobi_vars_to_load) + + for var, val in zip(vars_to_load, vals): + res[var] = val + + return res + + +def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): + """ + solver_model: gurobipy.Model + con_map: Dict[ConstraintData, gurobipy.Constr] + Maps the pyomo constraint to the gurobipy constraint + linear_cons_to_load: List[ConstraintData] + quadratic_cons_to_load: List[ConstraintData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] + quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] + linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) + quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) + + duals = {} + for c, val in zip(linear_cons_to_load, linear_vals): + duals[c] = val + for c, val in zip(quadratic_cons_to_load, quadratic_vals): + duals[c] = val + return duals + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 @@ -191,10 +307,15 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - self.config = config = self.config( + config = self.config( value=kwds, preserve_implicit=True, ) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + if not self.available(): c = self.__class__ raise ApplicationError( @@ -237,13 +358,17 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, + solution_loader=solution_loader, has_obj=has_obj, ) finally: os.chdir(orig_cwd) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) self.config = orig_config - res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -273,10 +398,11 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, has_obj): + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status results = Results() + results.solution_loader = solution_loader results.timing_info.gurobi_time = grb_model.Runtime if grb_model.SolCount > 0: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index fa269c1d3c5..3ae6e86526c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -12,7 +12,7 @@ import io import logging import math -from typing import List, Optional +from typing import Dict, List, NoReturn, Optional, Sequence, Mapping from collections.abc import Iterable from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet @@ -53,9 +53,18 @@ PersistentSolverUtils, PersistentSolverMixin, ) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager -from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from .gurobi_direct_base import ( + GurobiConfig, + GurobiDirectBase, + gurobipy, + _load_suboptimal_mip_solution, + _load_vars, + _get_primals, + _get_duals, + _get_reduced_costs, +) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor @@ -63,7 +72,87 @@ logger = logging.getLogger(__name__) -class GurobiSolutionLoader(PersistentSolutionLoader): +class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._linear_cons = linear_cons + self._quadratic_cons = quadratic_cons + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> None: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = list(self._con_map.keys()) + linear_cons_to_load = [] + quadratic_cons_to_load = [] + for c in cons_to_load: + if c in self._linear_cons: + linear_cons_to_load.append(c) + else: + assert c in self._quadratic_cons + quadratic_cons_to_load.append(c) + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + linear_cons_to_load=linear_cons_to_load, + quadratic_cons_to_load=quadratic_cons_to_load, + ) + + +class GurobiPersistentSolutionLoader(PersistentSolutionLoader): def load_vars(self, vars_to_load=None, solution_number=0): self._assert_solution_still_valid() self._solver._load_vars( @@ -212,23 +301,50 @@ def __init__(self, **kwds): self._vars = {} # from id(v) to v self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _create_solver_model(self, pyomo_model): + timer = self.config.timer + timer.start('create gurobipy model') self._clear() self._solver_model = gurobipy.Model(env=self.env()) + timer.start('collect constraints') cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + timer.stop('collect constraints') + timer.start('translate constraints') self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') obj = get_objective(pyomo_model) + timer.stop('get objective') + timer.start('translate objective') self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = GurobiDirectQuadraticSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + timer.stop('create gurobipy model') + return self._solver_model, solution_loader, has_obj def _clear(self): self._solver_model = None self._vars = {} self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _pyomo_gurobi_var_iter(self): @@ -256,8 +372,10 @@ def _process_domain_and_bounds(self, var): lb = var.value ub = lb else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) return lb, ub, vtype def _add_variables(self, variables: List[VarData]): @@ -273,14 +391,12 @@ def _add_variables(self, variables: List[VarData]): gurobi_vars = self._solver_model.addVars( len(variables), lb=lbs, ub=ubs, vtype=vtypes - ) + ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - def _get_expr_from_pyomo_expr(self, expr): - repn = generate_standard_repn(expr, quadratic=True, compute_values=True) - + def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' @@ -289,18 +405,26 @@ def _get_expr_from_pyomo_expr(self, expr): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( repn.linear_coefs, - [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + vlist, ) else: new_expr = 0.0 - for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y return new_expr, repn.constant @@ -308,26 +432,72 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] for con in cons: lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + repn = generate_standard_repn(body, quadratic=True, compute_values=True) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) elif ub is None: - gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) else: - gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) - gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con + + def _set_objective(self, obj): + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5ab36554061..748e0127151 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,8 +30,9 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent -from pyomo.contrib.solver.solvers.gurobi_direct import GurobiDirect +# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -47,20 +48,31 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [('ipopt', Ipopt)] -qcp_solvers = [('gurobi_persistent', GurobiPersistent), ('ipopt', Ipopt)] +nlp_solvers = [ + ('ipopt', Ipopt), +] +qcp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), +] qp_solvers = qcp_solvers + [("highs", Highs)] -miqcqp_solvers = [('gurobi_persistent', GurobiPersistent)] +miqcqp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), +] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 909be8815c6c585a5c2c700b9444599868c26598 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:02:49 -0600 Subject: [PATCH 34/97] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 69 +- pyomo/contrib/solver/plugins.py | 3 +- .../solvers/gurobi/gurobi_persistent.py | 934 +++++++++++++++++- .../solver/tests/solvers/test_solvers.py | 11 +- 4 files changed, 925 insertions(+), 92 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..8f7238c2ee9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -162,9 +162,6 @@ def __init__( class Observer(abc.ABC): - def __init__(self): - pass - @abc.abstractmethod def add_variables(self, variables: List[VarData]): pass @@ -255,7 +252,7 @@ def set_instance(self, model): self._model = model self._add_block(model) - def _add_variables(self, variables: List[VarData]): + def add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') @@ -271,7 +268,7 @@ def _add_variables(self, variables: List[VarData]): for obs in self._observers: obs.add_variables(variables) - def _add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: List[ParamData]): for p in params: pid = id(p) if pid in self._referenced_params: @@ -287,7 +284,7 @@ def _check_for_new_vars(self, variables: List[VarData]): v_id = id(v) if v_id not in self._referenced_variables: new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): vars_to_remove = {} @@ -296,7 +293,7 @@ def _check_to_remove_vars(self, variables: List[VarData]): ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): new_params = {} @@ -304,7 +301,7 @@ def _check_for_new_params(self, params: List[ParamData]): pid = id(p) if pid not in self._referenced_params: new_params[pid] = p - self._add_parameters(list(new_params.values())) + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): params_to_remove = {} @@ -313,9 +310,9 @@ def _check_to_remove_params(self, params: List[ParamData]): ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + self.remove_parameters(list(params_to_remove.values())) - def _add_constraints(self, cons: List[ConstraintData]): + def add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -343,7 +340,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.add_constraints(cons) - def _add_sos_constraints(self, cons: List[SOSConstraintData]): + def add_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -373,7 +370,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): + def set_objective(self, obj: Optional[ObjectiveData]): vars_to_remove_check = [] params_to_remove_check = [] if self._objective is not None: @@ -414,12 +411,12 @@ def _set_objective(self, obj: Optional[ObjectiveData]): self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): - self._add_constraints( + self.add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) ) ) - self._add_sos_constraints( + self.add_sos_constraints( list( block.component_data_objects( SOSConstraint, descend_into=True, active=True @@ -427,9 +424,9 @@ def _add_block(self, block): ) ) obj = get_objective(block) - self._set_objective(obj) + self.set_objective(obj) - def _remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) vars_to_check = [] @@ -453,7 +450,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) vars_to_check = [] @@ -476,7 +473,7 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_variables(self, variables: List[VarData]): + def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) for v in variables: @@ -493,7 +490,7 @@ def _remove_variables(self, variables: List[VarData]): del self._referenced_variables[v_id] del self._vars[v_id] - def _remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: @@ -510,7 +507,7 @@ def _remove_parameters(self, params: List[ParamData]): del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -523,7 +520,7 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters(self, params): + def update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) for obs in self._observers: @@ -668,28 +665,28 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + self.add_sos_constraints(new_sos) + self.remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + self.add_constraints(new_cons) + self.remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -698,10 +695,10 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() - self._update_variables(vars_to_update) + self.update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -711,8 +708,8 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer.start('named expressions') cons_to_update, update_obj = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -730,11 +727,11 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = True if need_to_set_objective: - self._set_objective(new_obj) + self.set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + self.update_parameters(params_to_update) timer.stop('params') diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 7630c614aa2..f29c4f61c4e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -12,9 +12,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt -from .solvers.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3ae6e86526c..844502ca476 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import io import logging import math @@ -21,6 +22,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData @@ -67,6 +69,7 @@ ) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -152,18 +155,33 @@ def get_duals( ) -class GurobiPersistentSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): +class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): + def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: + super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + self._valid = True + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().load_vars(vars_to_load, solution_id) + + def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_primals(vars_to_load, solution_id) - def get_primals(self, vars_to_load=None, solution_number=0): + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().get_duals(cons_to_load) + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) class _MutableLowerBound: @@ -187,46 +205,61 @@ def update(self): class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None + def __init__(self, expr, pyomo_con, con_map, pyomo_var_id, var_map, gurobi_model): + self.expr = expr + self.pyomo_con = pyomo_con + self.pyomo_var_id = pyomo_var_id + self.con_map = con_map + self.var_map = var_map + self.gurobi_model = gurobi_model + + @property + def gurobi_var(self): + return self.var_map[self.pyomo_var_id] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + self.gurobi_model.chgCoeff(self.gurobi_con, self.gurobi_var, value(self.expr)) class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None + def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + self.lhs_expr = lhs_expr + self.rhs_expr = rhs_expr + self.pyomo_con = pyomo_con + self.con_map = con_map + self.slack_name = slack_name + self.gurobi_model = gurobi_model def update(self): rhs_val = value(self.rhs_expr) lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val + con = self.con_map[self.pyomo_con] + con.rhs = rhs_val slack = self.gurobi_model.getVarByName(self.slack_name) slack.ub = rhs_val - lhs_val class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None + def __init__(self, expr, pyomo_con, con_map): + self.expr = expr + self.pyomo_con = pyomo_con + self.con_map = con_map def update(self): - self.con.rhs = value(self.expr) + con = self.con_map[self.pyomo_con] + con.rhs = value(self.expr) class _MutableQuadraticConstraint: def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + self, gurobi_model, pyomo_con, con_map, constant, linear_coefs, quadratic_coefs ): - self.con = gurobi_con + self.pyomo_con = pyomo_con + self.con_map = con_map self.gurobi_model = gurobi_model self.constant = constant self.last_constant_value = value(self.constant.expr) @@ -235,8 +268,12 @@ def __init__( self.quadratic_coefs = quadratic_coefs self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) + gurobi_expr = self.gurobi_model.getQCRow(self.gurobi_con) for ndx, coef in enumerate(self.linear_coefs): current_coef_value = value(coef.expr) incremental_coef_value = ( @@ -260,14 +297,14 @@ def get_updated_rhs(self): class _MutableObjective: def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + self.constant: _MutableConstant = constant + self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs + self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs + self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) + coef.gurobi_var.obj = value(coef.expr) self.gurobi_model.ObjCon = value(self.constant.expr) gurobi_expr = None @@ -286,10 +323,19 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: - def __init__(self): + def __init__(self, expr, v1id, v2id, var_map): self.expr = None - self.var1 = None - self.var2 = None + self.var_map = var_map + self.v1id = v1id + self.v2id = v2id + + @property + def var1(self): + return self.var_map[self.v1id] + + @property + def var2(self): + return self.var_map[self.v2id] class GurobiDirectQuadratic(GurobiDirectBase): @@ -405,9 +451,10 @@ def _get_expr_from_pyomo_repn(self, repn): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( - repn.linear_coefs, + coef_list, vlist, ) else: @@ -424,9 +471,9 @@ def _get_expr_from_pyomo_repn(self, repn): for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + new_expr += value(coef) * gurobi_x * gurobi_y - return new_expr, repn.constant + return new_expr def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] @@ -437,20 +484,20 @@ def _add_constraints(self, cons: List[ConstraintData]): self._quadratic_cons.add(con) else: self._linear_cons.add(con) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) elif ub is None: - gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) + gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) @@ -495,11 +542,802 @@ def _set_objective(self, obj): raise ValueError(f'Objective sense is not recognized: {obj.sense}') repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + repn_constant = repn.constant self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - self._needs_updated = True -class GurobiPersistentQuadratic(GurobiDirectQuadratic): +class _GurobiObserver(Observer): + def __init__(self, opt: GurobiPersistentQuadratic) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class GurobiPersistent(GurobiDirectQuadratic): _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None + self._observer = _GurobiObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._constraint_ndx = 0 + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object = None + self._constraint_ndx = 0 + + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: + self.update() + else: + self.set_instance(pyomo_model) + + solution_loader = GurobiPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + has_obj = self._objective is not None + return self._solver_model, solution_loader, has_obj + + def release_license(self): + self._clear() + self.__class__.release_license() + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + self._needs_updated = False + return res + + def _process_domain_and_bounds(self, var): + res = super()._process_domain_and_bounds(var) + if not is_constant(var._lb): + mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) + if not is_constant(var._ub): + mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) + return res + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + super()._add_variables(variables) + self._vars_added_since_update.update(variables) + self._needs_updated = True + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = gurobipy.Model(env=self.env()) + timer.start('set_instance') + self._change_detector.set_instance(pyomo_model) + timer.stop('set_instance') + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + if self._needs_updated: + self._update_gurobi_model() + self._change_detector.update(timer=timer) + timer.stop('update') + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + gurobi_expr_list = [] + for ndx, con in enumerate(cons): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) + repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + mutable_constant = None + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif ub is None: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif con.equality: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + else: + assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + lhs_expr = lb - repn.constant + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + if not is_constant(lhs_expr) or not is_constant(rhs_expr): + conname = f'c{self._constraint_ndx}[{ndx}]' + mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + if len(repn.quadratic_vars) == 0: + if len(mlc_list) > 0: + self._mutable_helpers[con] = mlc_list + if mutable_constant is not None: + if con not in self._mutable_helpers: + self._mutable_helpers[con] = [] + self._mutable_helpers[con].append(mutable_constant) + else: + if mutable_constant is None: + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + mqc = _MutableQuadraticConstraint( + self._solver_model, + con, + self._pyomo_con_to_solver_con_map, + mutable_constant, + mlc_list, + mqc_list, + ) + self._mutable_quadratic_helpers[con] = mqc + + gurobi_cons = list(self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}' + ).values()) + self._constraint_ndx += 1 + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + super()._add_sos_constraints(cons) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _set_objective(self, obj): + self._invalidate_last_results() + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + self._mutable_objective = None + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + + # hack + # see PR #2454 + if self._objective is not None: + self._solver_model.setObjective(0) + self._solver_model.update() + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) + self._objective = obj + self._needs_updated = True + + def _update_gurobi_model(self): + self._solver_model.update() + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def _remove_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_con = self._pyomo_con_to_solver_con_map[con] + self._solver_model.remove(solver_con) + del self._pyomo_con_to_solver_con_map[con] + self._mutable_helpers.pop(con, None) + self._mutable_quadratic_helpers.pop(con, None) + self._needs_updated = True + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] + self._solver_model.remove(solver_sos_con) + del self._pyomo_sos_to_solver_sos_map[con] + self._needs_updated = True + + def _remove_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + v_id = id(var) + if var in self._vars_added_since_update: + self._update_gurobi_model() + solver_var = self._pyomo_var_to_solver_var_map[v_id] + self._solver_model.remove(solver_var) + del self._pyomo_var_to_solver_var_map[v_id] + self._mutable_bounds.pop(v_id, None) + self._needs_updated = True + + def _update_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + var_id = id(var) + if var_id not in self._pyomo_var_to_solver_var_map: + raise ValueError( + f'The Var provided to update_var needs to be added first: {var}' + ) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] + lb, ub, vtype = self._process_domain_and_bounds(var) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + self._needs_updated = True + + def _update_parameters(self, params: List[ParamData]): + self._invalidate_last_results() + for con, helpers in self._mutable_helpers.items(): + for helper in helpers: + helper.update() + for k, (v, helper) in self._mutable_bounds.items(): + helper.update() + + for con, helper in self._mutable_quadratic_helpers.items(): + if con in self._constraints_added_since_update: + self._update_gurobi_model() + gurobi_con = helper.gurobi_con + new_gurobi_expr = helper.get_updated_expression() + new_rhs = helper.get_updated_rhs() + new_sense = gurobi_con.qcsense + self._solver_model.remove(gurobi_con) + new_con = self._solver_model.addQConstr( + new_gurobi_expr, new_sense, new_rhs, + ) + self._pyomo_con_to_solver_con_map[con] = new_con + helper.pyomo_con = con + self._constraints_added_since_update.add(con) + + if self._mutable_objective is not None: + new_gurobi_expr = self._mutable_objective.get_updated_expression() + if new_gurobi_expr is not None: + if self._objective.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + else: + sense = gurobipy.GRB.MAXIMIZE + # TODO: need a test for when part of the object is linear + # and part of the objective is quadratic, but both + # parts have mutable coefficients + self._solver_model.setObjective(new_gurobi_expr, sense=sense) + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def get_model_attr(self, attr): + """ + Get the value of an attribute on the Gurobi model. + + Parameters + ---------- + attr: str + The attribute to get. See Gurobi documentation for descriptions of the attributes. + """ + if self._needs_updated: + self._update_gurobi_model() + return self._solver_model.getAttr(attr) + + def write(self, filename): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + """ + self._solver_model.write(filename) + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def set_linear_constraint_attr(self, con, attr, val): + """ + Set the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be modified. + attr: str + The attribute to be modified. Options are: + CBasis + DStart + Lazy + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'Sense', 'RHS', 'ConstrName'}: + raise ValueError( + f'Linear constraint attr {attr} cannot be set with' + ' the set_linear_constraint_attr method. Please use' + ' the remove_constraint and add_constraint methods.' + ) + self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) + self._needs_updated = True + + def set_var_attr(self, var, attr, val): + """ + Set the value of an attribute on a gurobi variable. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be modified. + attr: str + The attribute to be modified. Options are: + Start + VarHintVal + VarHintPri + BranchPriority + VBasis + PStart + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'LB', 'UB', 'VType', 'VarName'}: + raise ValueError( + f'Var attr {attr} cannot be set with' + ' the set_var_attr method. Please use' + ' the update_var method.' + ) + if attr == 'Obj': + raise ValueError( + 'Var attr Obj cannot be set with' + ' the set_var_attr method. Please use' + ' the set_objective method.' + ) + self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) + self._needs_updated = True + + def get_var_attr(self, var, attr): + """ + Get the value of an attribute on a gurobi var. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be retrieved. + attr: str + The attribute to get. See gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) + + def get_linear_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def get_sos_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi sos constraint. + + Parameters + ---------- + con: pyomo.core.base.sos.SOSConstraintData + The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) + + def get_quadratic_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi quadratic constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def set_gurobi_param(self, param, val): + """ + Set a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to set. Options include any gurobi parameter. + Please see the Gurobi documentation for options. + val: any + The value to set the parameter to. See Gurobi documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_gurobi_param_info(self, param): + """ + Get information about a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to get info for. See Gurobi documentation for possible options. + + Returns + ------- + six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. + """ + return self._solver_model.getParamInfo(param) + + def _intermediate_callback(self): + def f(gurobi_model, where): + self._callback_func(self._pyomo_model, self, where) + + return f + + def set_callback(self, func=None): + """ + Specify a callback for gurobi to use. + + Parameters + ---------- + func: function + The function to call. The function should have three arguments. The first will be the pyomo model being + solved. The second will be the GurobiPersistent instance. The third will be an enum member of + gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For + example, suppose we want to solve + + .. math:: + + min 2*x + y + + s.t. + + y >= (x-2)**2 + + 0 <= x <= 4 + + y >= 0 + + y integer + + as an MILP using extended cutting planes in callbacks. + + >>> from gurobipy import GRB # doctest:+SKIP + >>> import pyomo.environ as pyo + >>> from pyomo.core.expr.taylor_series import taylor_series_expansion + >>> from pyomo.contrib import appsi + >>> + >>> m = pyo.ConcreteModel() + >>> m.x = pyo.Var(bounds=(0, 4)) + >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) + >>> m.obj = pyo.Objective(expr=2*m.x + m.y) + >>> m.cons = pyo.ConstraintList() # for the cutting planes + >>> + >>> def _add_cut(xval): + ... # a function to generate the cut + ... m.x.value = xval + ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) + ... + >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x + >>> _c = _add_cut(4) # this is an arbitrary choice + >>> + >>> opt = appsi.solvers.Gurobi() + >>> opt.config.stream_solver = True + >>> opt.set_instance(m) # doctest:+SKIP + >>> opt.gurobi_options['PreCrush'] = 1 + >>> opt.gurobi_options['LazyConstraints'] = 1 + >>> + >>> def my_callback(cb_m, cb_opt, cb_where): + ... if cb_where == GRB.Callback.MIPSOL: + ... cb_opt.cbGetSolution(variables=[m.x, m.y]) + ... if m.y.value < (m.x.value - 2)**2 - 1e-6: + ... cb_opt.cbLazy(_add_cut(m.x.value)) + ... + >>> opt.set_callback(my_callback) + >>> res = opt.solve(m) # doctest:+SKIP + + """ + if func is not None: + self._callback_func = func + self._callback = self._intermediate_callback() + else: + self._callback = None + self._callback_func = None + + def cbCut(self, con): + """ + Add a cut within a callback. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The cut to add + """ + if not con.active: + raise ValueError('cbCut expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbCut expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbCut.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbGet(self, what): + return self._solver_model.cbGet(what) + + def cbGetNodeRel(self, variables): + """ + Parameters + ---------- + variables: Var or iterable of Var + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetNodeRel(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbGetSolution(self, variables): + """ + Parameters + ---------- + variables: iterable of vars + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetSolution(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbLazy(self, con): + """ + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The lazy constraint to add + """ + if not con.active: + raise ValueError('cbLazy expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbLazy expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbLazy.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbSetSolution(self, variables, solution): + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + self._solver_model.cbSetSolution(gurobi_vars, solution) + + def cbUseSolution(self): + return self._solver_model.cbUseSolution() + + def reset(self): + self._solver_model.reset() + + def add_variables(self, variables): + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj): + self._change_detector.set_objective(obj) + + def remove_constrains(self, cons): + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + self._change_detector.update_parameters(params) \ No newline at end of file diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 748e0127151..a0d87835e13 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,9 +30,8 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -48,14 +47,14 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), @@ -64,13 +63,13 @@ ('ipopt', Ipopt), ] qcp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ] nl_solvers = [('ipopt', Ipopt)] From 862c387a8e6478d9b9c3f176b0059046d02f1198 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:18:04 -0600 Subject: [PATCH 35/97] bugs --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index f29c4f61c4e..fed739232ad 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -35,7 +35,7 @@ def load(): name='gurobi_direct_quadratic', legacy_name='gurobi_direct_quadratic_v2', doc='Direct interface to Gurobi', - )(GurobiDirect) + )(GurobiDirectQuadratic) SolverFactory.register( - name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' + name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 844502ca476..b8a8f46d1f6 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -279,7 +279,7 @@ def get_updated_expression(self): incremental_coef_value = ( current_coef_value - self.last_linear_coef_values[ndx] ) - gurobi_expr += incremental_coef_value * coef.var + gurobi_expr += incremental_coef_value * coef.gurobi_var self.last_linear_coef_values[ndx] = current_coef_value for ndx, coef in enumerate(self.quadratic_coefs): current_coef_value = value(coef.expr) @@ -324,7 +324,7 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: def __init__(self, expr, v1id, v2id, var_map): - self.expr = None + self.expr = expr self.var_map = var_map self.v1id = v1id self.v2id = v2id @@ -860,6 +860,7 @@ def _remove_variables(self, variables: List[VarData]): solver_var = self._pyomo_var_to_solver_var_map[v_id] self._solver_model.remove(solver_var) del self._pyomo_var_to_solver_var_map[v_id] + del self._vars[v_id] self._mutable_bounds.pop(v_id, None) self._needs_updated = True From 8f7a61ed3bf8eafc8eee12544755f61097a2b7be Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:35:33 -0600 Subject: [PATCH 36/97] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 7 +- .../solvers/gurobi/gurobi_persistent.py | 6 +- pyomo/contrib/solver/solvers/gurobi_direct.py | 470 ------ .../solver/solvers/gurobi_persistent.py | 1409 ----------------- .../tests/solvers/test_gurobi_persistent.py | 49 +- 5 files changed, 25 insertions(+), 1916 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/gurobi_direct.py delete mode 100644 pyomo/contrib/solver/solvers/gurobi_persistent.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 8f7238c2ee9..bd905e1c61d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -361,12 +361,15 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index b8a8f46d1f6..e91381f41a3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -39,10 +39,6 @@ SolutionStatus, ) from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, @@ -1328,7 +1324,7 @@ def add_sos_constraints(self, cons): def set_objective(self, obj): self._change_detector.set_objective(obj) - def remove_constrains(self, cons): + def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) def remove_sos_constraints(self, cons): diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py deleted file mode 100644 index 45ea9dcc873..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ /dev/null @@ -1,470 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import datetime -import io -import math -import operator -import os - -from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler - -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase - - -gurobipy, gurobipy_available = attempt_import('gurobipy') - - -class GurobiConfigMixin: - """ - Mixin class for Gurobi-specific configurations - """ - - def __init__(self): - self.use_mipstart: bool = self.declare( - 'use_mipstart', - ConfigValue( - default=False, - domain=bool, - description="If True, the current values of the integer variables " - "will be passed to Gurobi.", - ), - ) - - -class GurobiConfig(BranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - BranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyo_obj): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - self._pyo_obj = pyo_obj - GurobiDirect._register_env_client() - - def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None - self._pyo_obj = None - # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirect._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - -class GurobiSolverMixin: - """ - gurobi_direct and gurobi_persistent check availability and set versions - in the same way. This moves the logic to a central location to reduce - duplicate code. - """ - - _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available - - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) - try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( - gurobipy.GRB.VERSION_MAJOR, - gurobipy.GRB.VERSION_MINOR, - gurobipy.GRB.VERSION_TECHNICAL, - ) - return version - - -class GurobiDirect(GurobiSolverMixin, SolverBase): - """ - Interface to Gurobi using gurobipy - """ - - CONFIG = GurobiConfig() - - _tc_map = None - - def __init__(self, **kwds): - super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - - timer.start('compile_model') - repn = LinearStandardFormCompiler().write( - model, mixed_form=True, set_sense=None - ) - timer.stop('compile_model') - - if len(repn.objectives) > 1: - raise IncompatibleModelError( - f"The {self.__class__.__name__} solver only supports models " - f"with zero or one objectives (received {len(repn.objectives)})." - ) - - timer.start('prepare_matrices') - inf = float('inf') - ninf = -inf - bounds = list(map(operator.attrgetter('bounds'), repn.columns)) - lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] - ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] - CON = gurobipy.GRB.CONTINUOUS - BIN = gurobipy.GRB.BINARY - INT = gurobipy.GRB.INTEGER - vtype = [ - ( - CON - if v.is_continuous() - else BIN if v.is_binary() else INT if v.is_integer() else '?' - ) - for v in repn.columns - ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 - sense = [sense_type[r[1]] for r in repn.rows] - timer.stop('prepare_matrices') - - ostreams = [io.StringIO()] + config.tee - res = Results() - - orig_cwd = os.getcwd() - try: - if config.working_dir: - os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, - ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') - finally: - os.chdir(orig_cwd) - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns, repn.objectives - ), - ) - - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res - - def _postsolve(self, timer: HierarchicalTimer, config, loader): - grb_model = loader._grb_model - status = grb_model.Status - - results = Results() - results.solution_loader = loader - results.timing_info.gurobi_time = grb_model.Runtime - - if grb_model.SolCount > 0: - if status == gurobipy.GRB.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - results.termination_condition = self._get_tc_map().get( - status, TerminationCondition.unknown - ) - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - if loader._pyo_obj: - try: - if math.isfinite(grb_model.ObjVal): - results.incumbent_objective = grb_model.ObjVal - else: - results.incumbent_objective = None - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = grb_model.ObjBound - except (gurobipy.GurobiError, AttributeError): - if grb_model.ModelSense == ObjectiveSense.minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - else: - results.incumbent_objective = None - results.objective_bound = None - - results.iteration_count = grb_model.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if grb_model.SolCount > 0: - results.solution_loader.load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _get_tc_map(self): - if GurobiDirect._tc_map is None: - grb = gurobipy.GRB - tc = TerminationCondition - GurobiDirect._tc_map = { - grb.LOADED: tc.unknown, # problem is loaded, but no solution - grb.OPTIMAL: tc.convergenceCriteriaSatisfied, - grb.INFEASIBLE: tc.provenInfeasible, - grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, - grb.UNBOUNDED: tc.unbounded, - grb.CUTOFF: tc.objectiveLimit, - grb.ITERATION_LIMIT: tc.iterationLimit, - grb.NODE_LIMIT: tc.iterationLimit, - grb.TIME_LIMIT: tc.maxTimeLimit, - grb.SOLUTION_LIMIT: tc.unknown, - grb.INTERRUPTED: tc.interrupted, - grb.NUMERIC: tc.unknown, - grb.SUBOPTIMAL: tc.unknown, - grb.USER_OBJ_LIMIT: tc.objectiveLimit, - } - return GurobiDirect._tc_map diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py deleted file mode 100644 index ea3693c1c70..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ /dev/null @@ -1,1409 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import io -import logging -import math -from typing import List, Optional -from collections.abc import Iterable - -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler -from pyomo.core.base.var import VarData -from pyomo.core.base.constraint import ConstraintData -from pyomo.core.base.sos import SOSConstraintData -from pyomo.core.base.param import ParamData -from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types -from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader -from pyomo.core.staleflag import StaleFlagManager - - -logger = logging.getLogger(__name__) - - -def _import_gurobipy(): - try: - import gurobipy - except ImportError: - GurobiPersistent._available = Availability.NotFound - raise - if gurobipy.GRB.VERSION_MAJOR < 7: - GurobiPersistent._available = Availability.BadVersion - raise ImportError('The Persistent Gurobi interface requires gurobipy>=7.0.0') - return gurobipy - - -gurobipy, gurobipy_available = attempt_import('gurobipy', importer=_import_gurobipy) - - -class GurobiConfig(PersistentBranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - PersistentBranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - def get_primals(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - -class _MutableLowerBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('lb', value(self.expr)) - - -class _MutableUpperBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('ub', value(self.expr)) - - -class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None - - def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) - - -class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None - - def update(self): - rhs_val = value(self.rhs_expr) - lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val - slack = self.gurobi_model.getVarByName(self.slack_name) - slack.ub = rhs_val - lhs_val - - -class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None - - def update(self): - self.con.rhs = value(self.expr) - - -class _MutableQuadraticConstraint: - def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs - ): - self.con = gurobi_con - self.gurobi_model = gurobi_model - self.constant = constant - self.last_constant_value = value(self.constant.expr) - self.linear_coefs = linear_coefs - self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) - for ndx, coef in enumerate(self.linear_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_linear_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var - self.last_linear_coef_values[ndx] = current_coef_value - for ndx, coef in enumerate(self.quadratic_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - def get_updated_rhs(self): - return value(self.constant.expr) - - -class _MutableObjective: - def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): - self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) - self.gurobi_model.ObjCon = value(self.constant.expr) - - gurobi_expr = None - for ndx, coef in enumerate(self.quadratic_coefs): - if value(coef.expr) != self.last_quadratic_coef_values[ndx]: - if gurobi_expr is None: - self.gurobi_model.update() - gurobi_expr = self.gurobi_model.getObjective() - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - -class _MutableQuadraticCoefficient: - def __init__(self): - self.expr = None - self.var1 = None - self.var2 = None - - -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - CONFIG = GurobiConfig() - _gurobipy_available = gurobipy_available - - def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - self._register_env_client() - self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None - - def release_license(self): - self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - @property - def symbol_map(self): - return self._symbol_map - - def _solve(self): - config = self._active_config - timer = config.timer - ostreams = [io.StringIO()] + config.tee - - with capture_output(TeeStream(*ostreams), capture_fd=False): - options = config.solver_options - - self._solver_model.setParam('LogToConsole', 1) - - if config.threads is not None: - self._solver_model.setParam('Threads', config.threads) - if config.time_limit is not None: - self._solver_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - self._solver_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - self._solver_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) - - for key, option in options.items(): - self._solver_model.setParam(key, option) - - timer.start('optimize') - self._solver_model.optimize(self._callback) - timer.stop('optimize') - - self._needs_updated = False - res = self._postsolve(timer) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - return res - - def _process_domain_and_bounds( - self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var - ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] - lb, ub, step = _domain_interval - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if _fixed: - lb = _value - ub = _value - else: - if _lb is not None: - if not is_constant(_lb): - mutable_bound = _MutableLowerBound(NPV_MaxExpression((_lb, lb))) - if gurobipy_var is None: - mutable_lbs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'lb'] = (var, mutable_bound) - lb = max(value(_lb), lb) - if _ub is not None: - if not is_constant(_ub): - mutable_bound = _MutableUpperBound(NPV_MinExpression((_ub, ub))) - if gurobipy_var is None: - mutable_ubs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'ub'] = (var, mutable_bound) - ub = min(value(_ub), ub) - - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - var_names = [] - vtypes = [] - lbs = [] - ubs = [] - mutable_lbs = {} - mutable_ubs = {} - for ndx, var in enumerate(variables): - varname = self._symbol_map.getSymbol(var, self._labeler) - lb, ub, vtype = self._process_domain_and_bounds( - var, id(var), mutable_lbs, mutable_ubs, ndx, None - ) - var_names.append(varname) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes, name=var_names - ) - - for ndx, pyomo_var in enumerate(variables): - gurobi_var = gurobi_vars[ndx] - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - for ndx, mutable_bound in mutable_lbs.items(): - mutable_bound.var = gurobi_vars[ndx] - for ndx, mutable_bound in mutable_ubs.items(): - mutable_bound.var = gurobi_vars[ndx] - self._vars_added_since_update.update(variables) - self._needs_updated = True - - def _add_parameters(self, params: List[ParamData]): - pass - - def _reinit(self): - saved_config = self.config - saved_tmp_config = self._active_config - self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() - self.config = saved_config - self._active_config = saved_tmp_config - - def set_instance(self, model): - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - self._reinit() - self._model = model - - if self.config.symbolic_solver_labels: - self._labeler = TextLabeler() - else: - self._labeler = NumericLabeler('x') - - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) - - self.add_block(model) - if self._objective is None: - self.set_objective(None) - - def _get_expr_from_pyomo_expr(self, expr): - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - repn = generate_standard_repn(expr, quadratic=True, compute_values=False) - - degree = repn.polynomial_degree() - if (degree is None) or (degree > 2): - raise IncompatibleModelError( - f'GurobiAuto does not support expressions of degree {degree}.' - ) - - if len(repn.linear_vars) > 0: - linear_coef_vals = [] - for ndx, coef in enumerate(repn.linear_coefs): - if not is_constant(coef): - mutable_linear_coefficient = _MutableLinearCoefficient() - mutable_linear_coefficient.expr = coef - mutable_linear_coefficient.var = self._pyomo_var_to_solver_var_map[ - id(repn.linear_vars[ndx]) - ] - mutable_linear_coefficients.append(mutable_linear_coefficient) - linear_coef_vals.append(value(coef)) - new_expr = gurobipy.LinExpr( - linear_coef_vals, - [self._pyomo_var_to_solver_var_map[id(i)] for i in repn.linear_vars], - ) - else: - new_expr = 0.0 - - for ndx, v in enumerate(repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - coef = repn.quadratic_coefs[ndx] - if not is_constant(coef): - mutable_quadratic_coefficient = _MutableQuadraticCoefficient() - mutable_quadratic_coefficient.expr = coef - mutable_quadratic_coefficient.var1 = gurobi_x - mutable_quadratic_coefficient.var2 = gurobi_y - mutable_quadratic_coefficients.append(mutable_quadratic_coefficient) - coef_val = value(coef) - new_expr += coef_val * gurobi_x * gurobi_y - - return ( - new_expr, - repn.constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - - def _add_constraints(self, cons: List[ConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if ( - gurobi_expr.__class__ in {gurobipy.LinExpr, gurobipy.Var} - or gurobi_expr.__class__ in native_numeric_types - ): - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_lb() and con.has_ub(): - lhs_expr = con.lower - repn_constant - rhs_expr = con.upper - repn_constant - lhs_val = value(lhs_expr) - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addRange( - gurobi_expr, lhs_val, rhs_val, name=conname - ) - self._range_constraints.add(con) - if not is_constant(lhs_expr) or not is_constant(rhs_expr): - mutable_range_constant = _MutableRangeConstant() - mutable_range_constant.lhs_expr = lhs_expr - mutable_range_constant.rhs_expr = rhs_expr - mutable_range_constant.con = gurobipy_con - mutable_range_constant.slack_name = 'Rg' + conname - mutable_range_constant.gurobi_model = self._solver_model - self._mutable_helpers[con] = [mutable_range_constant] - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - for tmp in mutable_linear_coefficients: - tmp.con = gurobipy_con - tmp.gurobi_model = self._solver_model - if len(mutable_linear_coefficients) > 0: - if con not in self._mutable_helpers: - self._mutable_helpers[con] = mutable_linear_coefficients - else: - self._mutable_helpers[con].extend(mutable_linear_coefficients) - elif gurobi_expr.__class__ is gurobipy.QuadExpr: - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - elif con.has_lb() and con.has_ub(): - raise NotImplementedError( - 'Quadratic range constraints are not supported' - ) - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - if ( - len(mutable_linear_coefficients) > 0 - or len(mutable_quadratic_coefficients) > 0 - or not is_constant(repn_constant) - ): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_quadratic_constraint = _MutableQuadraticConstraint( - self._solver_model, - gurobipy_con, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_quadratic_helpers[con] = mutable_quadratic_constraint - else: - raise ValueError( - f'Unrecognized Gurobi expression type: {str(gurobi_expr.__class__)}' - ) - - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _remove_constraints(self, cons: List[ConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_con = self._pyomo_con_to_solver_con_map[con] - self._solver_model.remove(solver_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_con_to_solver_con_map[con] - del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) - self._mutable_helpers.pop(con, None) - self._mutable_quadratic_helpers.pop(con, None) - self._needs_updated = True - - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] - self._solver_model.remove(solver_sos_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_sos_to_solver_sos_map[con] - self._needs_updated = True - - def _remove_variables(self, variables: List[VarData]): - for var in variables: - v_id = id(var) - if var in self._vars_added_since_update: - self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] - self._solver_model.remove(solver_var) - self._symbol_map.removeSymbol(var) - del self._pyomo_var_to_solver_var_map[v_id] - self._mutable_bounds.pop(v_id, None) - self._needs_updated = True - - def _remove_parameters(self, params: List[ParamData]): - pass - - def _update_variables(self, variables: List[VarData]): - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds( - var, var_id, None, None, None, gurobipy_var - ) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - self._needs_updated = True - - def update_parameters(self): - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() - - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: - self._update_gurobi_model() - gurobi_con = helper.con - new_gurobi_expr = helper.get_updated_expression() - new_rhs = helper.get_updated_rhs() - new_sense = gurobi_con.qcsense - pyomo_con = self._solver_con_to_pyomo_con_map[id(gurobi_con)] - name = self._symbol_map.getSymbol(pyomo_con, self._labeler) - self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, name=name - ) - self._pyomo_con_to_solver_con_map[id(pyomo_con)] = new_con - del self._solver_con_to_pyomo_con_map[id(gurobi_con)] - self._solver_con_to_pyomo_con_map[id(new_con)] = pyomo_con - helper.con = new_con - self._constraints_added_since_update.add(con) - - helper = self._mutable_objective - pyomo_obj = self._objective - new_gurobi_expr = helper.get_updated_expression() - if new_gurobi_expr is not None: - if pyomo_obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - else: - sense = gurobipy.GRB.MAXIMIZE - self._solver_model.setObjective(new_gurobi_expr, sense=sense) - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(obj.expr) - - mutable_constant = _MutableConstant() - mutable_constant.expr = repn_constant - mutable_objective = _MutableObjective( - self._solver_model, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_objective = mutable_objective - - # These two lines are needed as a workaround - # see PR #2454 - self._solver_model.setObjective(0) - self._solver_model.update() - - self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) - self._needs_updated = True - - def _postsolve(self, timer: HierarchicalTimer): - config = self._active_config - - gprob = self._solver_model - grb = gurobipy.GRB - status = gprob.Status - - results = Results() - results.solution_loader = GurobiSolutionLoader(self) - results.timing_info.gurobi_time = gprob.Runtime - - if gprob.SolCount > 0: - if status == grb.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - if status == grb.LOADED: # problem is loaded, but no solution - results.termination_condition = TerminationCondition.unknown - elif status == grb.OPTIMAL: # optimal - results.termination_condition = ( - TerminationCondition.convergenceCriteriaSatisfied - ) - elif status == grb.INFEASIBLE: - results.termination_condition = TerminationCondition.provenInfeasible - elif status == grb.INF_OR_UNBD: - results.termination_condition = TerminationCondition.infeasibleOrUnbounded - elif status == grb.UNBOUNDED: - results.termination_condition = TerminationCondition.unbounded - elif status == grb.CUTOFF: - results.termination_condition = TerminationCondition.objectiveLimit - elif status == grb.ITERATION_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.NODE_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.TIME_LIMIT: - results.termination_condition = TerminationCondition.maxTimeLimit - elif status == grb.SOLUTION_LIMIT: - results.termination_condition = TerminationCondition.unknown - elif status == grb.INTERRUPTED: - results.termination_condition = TerminationCondition.interrupted - elif status == grb.NUMERIC: - results.termination_condition = TerminationCondition.unknown - elif status == grb.SUBOPTIMAL: - results.termination_condition = TerminationCondition.unknown - elif status == grb.USER_OBJ_LIMIT: - results.termination_condition = TerminationCondition.objectiveLimit - else: - results.termination_condition = TerminationCondition.unknown - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - results.incumbent_objective = None - results.objective_bound = None - if self._objective is not None: - try: - results.incumbent_objective = gprob.ObjVal - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = gprob.ObjBound - except (gurobipy.GurobiError, AttributeError): - if self._objective.sense == minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - - if results.incumbent_objective is not None and not math.isfinite( - results.incumbent_objective - ): - results.incumbent_objective = None - - results.iteration_count = gprob.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if gprob.SolCount > 0: - self._load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _load_suboptimal_mip_solution(self, vars_to_load, solution_number): - if ( - self.get_model_attr('NumIntVars') == 0 - and self.get_model_attr('NumBinVars') == 0 - ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - original_solution_number = self.get_gurobi_param_info('SolutionNumber')[2] - self.set_gurobi_param('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = self._solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - self.set_gurobi_param('SolutionNumber', original_solution_number) - return res - - def _load_vars(self, vars_to_load=None, solution_number=0): - for v, val in self._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def _get_primals(self, vars_to_load=None, solution_number=0): - if self._needs_updated: - self._update_gurobi_model() # this is needed to ensure that solutions cannot be loaded after the model has been changed - - if self._solver_model.SolCount == 0: - raise NoSolutionError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - if solution_number != 0: - return self._load_suboptimal_mip_solution( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - return res - - def _get_reduced_costs(self, vars_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("Rc", gurobi_vars_to_load) - - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - - return res - - def _get_duals(self, cons_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = {} - - if cons_to_load is None: - linear_cons_to_load = self._solver_model.getConstrs() - quadratic_cons_to_load = self._solver_model.getQConstrs() - else: - gurobi_cons_to_load = OrderedSet( - [con_map[pyomo_con] for pyomo_con in cons_to_load] - ) - linear_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getConstrs()) - ) - ) - quadratic_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getQConstrs()) - ) - ) - linear_vals = self._solver_model.getAttr("Pi", linear_cons_to_load) - quadratic_vals = self._solver_model.getAttr("QCPi", quadratic_cons_to_load) - - for gurobi_con, val in zip(linear_cons_to_load, linear_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - for gurobi_con, val in zip(quadratic_cons_to_load, quadratic_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - - return dual - - def update(self, timer: HierarchicalTimer = None): - if self._needs_updated: - self._update_gurobi_model() - super().update(timer=timer) - self._update_gurobi_model() - - def _update_gurobi_model(self): - self._solver_model.update() - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def get_model_attr(self, attr): - """ - Get the value of an attribute on the Gurobi model. - - Parameters - ---------- - attr: str - The attribute to get. See Gurobi documentation for descriptions of the attributes. - """ - if self._needs_updated: - self._update_gurobi_model() - return self._solver_model.getAttr(attr) - - def write(self, filename): - """ - Write the model to a file (e.g., and lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - """ - self._solver_model.write(filename) - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def set_linear_constraint_attr(self, con, attr, val): - """ - Set the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be modified. - attr: str - The attribute to be modified. Options are: - CBasis - DStart - Lazy - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'Sense', 'RHS', 'ConstrName'}: - raise ValueError( - f'Linear constraint attr {attr} cannot be set with' - ' the set_linear_constraint_attr method. Please use' - ' the remove_constraint and add_constraint methods.' - ) - self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) - self._needs_updated = True - - def set_var_attr(self, var, attr, val): - """ - Set the value of an attribute on a gurobi variable. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be modified. - attr: str - The attribute to be modified. Options are: - Start - VarHintVal - VarHintPri - BranchPriority - VBasis - PStart - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'LB', 'UB', 'VType', 'VarName'}: - raise ValueError( - f'Var attr {attr} cannot be set with' - ' the set_var_attr method. Please use' - ' the update_var method.' - ) - if attr == 'Obj': - raise ValueError( - 'Var attr Obj cannot be set with' - ' the set_var_attr method. Please use' - ' the set_objective method.' - ) - self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) - self._needs_updated = True - - def get_var_attr(self, var, attr): - """ - Get the value of an attribute on a gurobi var. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be retrieved. - attr: str - The attribute to get. See gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) - - def get_linear_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def get_sos_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi sos constraint. - - Parameters - ---------- - con: pyomo.core.base.sos.SOSConstraintData - The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) - - def get_quadratic_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi quadratic constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def set_gurobi_param(self, param, val): - """ - Set a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to set. Options include any gurobi parameter. - Please see the Gurobi documentation for options. - val: any - The value to set the parameter to. See Gurobi documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_gurobi_param_info(self, param): - """ - Get information about a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to get info for. See Gurobi documentation for possible options. - - Returns - ------- - six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. - """ - return self._solver_model.getParamInfo(param) - - def _intermediate_callback(self): - def f(gurobi_model, where): - self._callback_func(self._model, self, where) - - return f - - def set_callback(self, func=None): - """ - Specify a callback for gurobi to use. - - Parameters - ---------- - func: function - The function to call. The function should have three arguments. The first will be the pyomo model being - solved. The second will be the GurobiPersistent instance. The third will be an enum member of - gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For - example, suppose we want to solve - - .. math:: - - min 2*x + y - - s.t. - - y >= (x-2)**2 - - 0 <= x <= 4 - - y >= 0 - - y integer - - as an MILP using extended cutting planes in callbacks. - - >>> from gurobipy import GRB # doctest:+SKIP - >>> import pyomo.environ as pyo - >>> from pyomo.core.expr.taylor_series import taylor_series_expansion - >>> from pyomo.contrib import appsi - >>> - >>> m = pyo.ConcreteModel() - >>> m.x = pyo.Var(bounds=(0, 4)) - >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) - >>> m.obj = pyo.Objective(expr=2*m.x + m.y) - >>> m.cons = pyo.ConstraintList() # for the cutting planes - >>> - >>> def _add_cut(xval): - ... # a function to generate the cut - ... m.x.value = xval - ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) - ... - >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x - >>> _c = _add_cut(4) # this is an arbitrary choice - >>> - >>> opt = appsi.solvers.Gurobi() - >>> opt.config.stream_solver = True - >>> opt.set_instance(m) # doctest:+SKIP - >>> opt.gurobi_options['PreCrush'] = 1 - >>> opt.gurobi_options['LazyConstraints'] = 1 - >>> - >>> def my_callback(cb_m, cb_opt, cb_where): - ... if cb_where == GRB.Callback.MIPSOL: - ... cb_opt.cbGetSolution(variables=[m.x, m.y]) - ... if m.y.value < (m.x.value - 2)**2 - 1e-6: - ... cb_opt.cbLazy(_add_cut(m.x.value)) - ... - >>> opt.set_callback(my_callback) - >>> res = opt.solve(m) # doctest:+SKIP - - """ - if func is not None: - self._callback_func = func - self._callback = self._intermediate_callback() - else: - self._callback = None - self._callback_func = None - - def cbCut(self, con): - """ - Add a cut within a callback. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The cut to add - """ - if not con.active: - raise ValueError('cbCut expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbCut expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbCut.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbGet(self, what): - return self._solver_model.cbGet(what) - - def cbGetNodeRel(self, variables): - """ - Parameters - ---------- - variables: Var or iterable of Var - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetNodeRel(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbGetSolution(self, variables): - """ - Parameters - ---------- - variables: iterable of vars - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetSolution(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbLazy(self, con): - """ - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The lazy constraint to add - """ - if not con.active: - raise ValueError('cbLazy expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbLazy expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbLazy.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbSetSolution(self, variables, solution): - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - self._solver_model.cbSetSolution(gurobi_vars, solution) - - def cbUseSolution(self): - return self._solver_model.cbUseSolution() - - def reset(self): - self._solver_model.reset() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 8703ae9edff..96cd1498956 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -11,7 +11,7 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.core.expr.taylor_series import taylor_series_expansion @@ -471,11 +471,11 @@ def test_solution_number(self): res = opt.solve(m) num_solutions = opt.get_model_attr('SolCount') self.assertEqual(num_solutions, 3) - res.solution_loader.load_vars(solution_number=0) + res.solution_loader.load_vars(solution_id=0) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.431184939357673) - res.solution_loader.load_vars(solution_number=1) + res.solution_loader.load_vars(solution_id=1) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.584793218502477) - res.solution_loader.load_vars(solution_number=2) + res.solution_loader.load_vars(solution_id=2) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.592304628123309) def test_zero_time_limit(self): @@ -496,16 +496,14 @@ def test_zero_time_limit(self): self.assertIsNone(res.incumbent_objective) -class TestManualModel(unittest.TestCase): +class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.config.auto_updates.check_for_new_or_removed_params = False - opt.config.auto_updates.check_for_new_or_removed_vars = False - opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.update_parameters = False - opt.config.auto_updates.update_vars = False - opt.config.auto_updates.update_constraints = False - opt.config.auto_updates.update_named_expressions = False + opt.auto_updates.check_for_new_or_removed_constraints = False + opt.auto_updates.update_parameters = False + opt.auto_updates.update_vars = False + opt.auto_updates.update_constraints = False + opt.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): @@ -603,16 +601,13 @@ def test_update1(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) opt.remove_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) opt.add_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) def test_update2(self): m = pyo.ConcreteModel() @@ -625,16 +620,13 @@ def test_update2(self): opt = self.opt opt.config.symbolic_solver_labels = True opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) opt.remove_constraints([m.c2]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumConstrs'), 0) opt.add_constraints([m.c2]) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) def test_update3(self): m = pyo.ConcreteModel() @@ -684,16 +676,13 @@ def test_update5(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) opt.remove_sos_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) + self.assertEqual(opt.get_model_attr('NumSOS'), 0) opt.add_sos_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) def test_update6(self): m = pyo.ConcreteModel() From 92fa4f5c72a4d26c5568a40e7ce7726ddd12e991 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:49:29 -0600 Subject: [PATCH 37/97] remove unused imports --- .../solver/solvers/gurobi/gurobi_direct.py | 20 ---------- .../solvers/gurobi/gurobi_direct_base.py | 7 +--- .../solvers/gurobi/gurobi_persistent.py | 38 +++---------------- 3 files changed, 7 insertions(+), 58 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f4a33e2cc54..16c633c7d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -9,40 +9,20 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import datetime -import io -import math import operator -import os from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, NoDualsError, NoReducedCostsError, NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -import logging from .gurobi_direct_base import GurobiDirectBase, gurobipy diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index d26dbf54c83..41bdb244743 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -12,14 +12,13 @@ import datetime import io import math -import operator import os -from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.errors import ApplicationError from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,14 +32,12 @@ NoDualsError, NoReducedCostsError, NoSolutionError, - IncompatibleModelError, ) from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase import logging diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index e91381f41a3..7b0463d2cf1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -10,61 +10,33 @@ # ___________________________________________________________________________ from __future__ import annotations -import io import logging -import math -from typing import Dict, List, NoReturn, Optional, Sequence, Mapping +from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.param import ParamData from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import IncompatibleModelError +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiConfig, GurobiDirectBase, gurobipy, - _load_suboptimal_mip_solution, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector From 8a9fc46b802dcc181a2b00a131ab089dfc8c59a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:50:15 -0600 Subject: [PATCH 38/97] run black --- .../solvers/gurobi/gurobi_direct_base.py | 29 +-- .../solvers/gurobi/gurobi_persistent.py | 213 +++++++++++------- .../solver/tests/solvers/test_solvers.py | 11 +- 3 files changed, 153 insertions(+), 100 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 41bdb244743..df6bb8b5327 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -87,9 +87,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ solver_model.getAttr('NumIntVars') == 0 and solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] @@ -112,7 +110,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): for v, val in _get_primals( solver_model=solver_model, var_map=var_map, - vars_to_load=vars_to_load, + vars_to_load=vars_to_load, solution_number=solution_number, ).items(): v.set_value(val, skip_validation=True) @@ -177,7 +175,7 @@ def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_loa """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) @@ -293,7 +291,7 @@ def _create_solver_model(self, pyomo_model): def _pyomo_gurobi_var_iter(self): # generator of tuples (pyomo_var, gurobi_var) raise NotImplementedError('should be implemented by derived classes') - + def _mipstart(self): for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): if pyomo_var.is_integer() and pyomo_var.value is not None: @@ -304,11 +302,8 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - config = self.config( - value=kwds, - preserve_implicit=True, - ) - + config = self.config(value=kwds, preserve_implicit=True) + # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = config object.__setattr__(self, 'config', config) @@ -329,7 +324,9 @@ def solve(self, model, **kwds) -> Results: if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model) + gurobi_model, solution_loader, has_obj = self._create_solver_model( + model + ) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -354,9 +351,7 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._postsolve( - grb_model=gurobi_model, - solution_loader=solution_loader, - has_obj=has_obj, + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) finally: os.chdir(orig_cwd) @@ -450,9 +445,9 @@ def _postsolve(self, grb_model, solution_loader, has_obj): raise NoFeasibleSolutionError() self.config.timer.stop('load solution') - # self.config gets copied a the beginning of + # self.config gets copied a the beginning of # solve and restored at the end, so modifying - # results.solver_config will not actually + # results.solver_config will not actually # modify self.config results.solver_config = self.config results.solver_name = self.name diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 7b0463d2cf1..6628f001421 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -29,8 +29,8 @@ from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiDirectBase, - gurobipy, + GurobiDirectBase, + gurobipy, _load_vars, _get_primals, _get_duals, @@ -45,13 +45,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - linear_cons, - quadratic_cons, + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons ) -> None: super().__init__() self._solver_model = solver_model @@ -62,9 +56,7 @@ def __init__( self._quadratic_cons = quadratic_cons def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -76,9 +68,7 @@ def load_vars( ) def get_primals( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -88,10 +78,9 @@ def get_primals( vars_to_load=vars_to_load, solution_number=solution_id, ) - + def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, + self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -100,10 +89,9 @@ def get_reduced_costs( var_map=self._var_map, vars_to_load=vars_to_load, ) - + def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -124,8 +112,12 @@ def get_duals( class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + def __init__( + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) -> None: + super().__init__( + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) self._valid = True def invalidate(self): @@ -135,19 +127,27 @@ def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - - def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + + def get_primals( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None + ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) @@ -194,7 +194,9 @@ def update(self): class _MutableRangeConstant: - def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + def __init__( + self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model + ): self.lhs_expr = lhs_expr self.rhs_expr = rhs_expr self.pyomo_con = pyomo_con @@ -268,7 +270,9 @@ def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.constant: _MutableConstant = constant self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs - self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] + self.last_quadratic_coef_values: List[float] = [ + value(i.expr) for i in self.quadratic_coefs + ] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): @@ -300,7 +304,7 @@ def __init__(self, expr, v1id, v2id, var_map): @property def var1(self): return self.var_map[self.v1id] - + @property def var2(self): return self.var_map[self.v2id] @@ -325,13 +329,21 @@ def _create_solver_model(self, pyomo_model): self._clear() self._solver_model = gurobipy.Model(env=self.env()) timer.start('collect constraints') - cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + cons = list( + pyomo_model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) timer.stop('collect constraints') timer.start('translate constraints') self._add_constraints(cons) timer.stop('translate constraints') timer.start('sos') - sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + sos = list( + pyomo_model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) self._add_sos_constraints(sos) timer.stop('sos') timer.start('get objective') @@ -351,7 +363,7 @@ def _create_solver_model(self, pyomo_model): ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj - + def _clear(self): self._solver_model = None self._vars = {} @@ -379,9 +391,7 @@ def _process_domain_and_bounds(self, var): else: vtype = gurobipy.GRB.INTEGER else: - raise ValueError( - f'Unrecognized domain: {var.domain}' - ) + raise ValueError(f'Unrecognized domain: {var.domain}') if var.fixed: lb = var.value ub = lb @@ -415,16 +425,13 @@ def _get_expr_from_pyomo_repn(self, repn): raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' ) - + if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr( - coef_list, - vlist, - ) + new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 @@ -455,8 +462,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) @@ -465,9 +471,14 @@ def _add_constraints(self, cons: List[ConstraintData]): elif lb == ub: gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) + gurobi_expr_list.append( + gurobi_expr + == [float(lb - repn.constant), float(ub - repn.constant)] + ) - gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() + gurobi_cons = self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) + ).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) def _add_sos_constraints(self, cons: List[SOSConstraintData]): @@ -485,7 +496,9 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } self._add_variables(list(missing_vars.values())) for v, w in con.get_items(): @@ -608,7 +621,7 @@ def _create_solver_model(self, pyomo_model): ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj - + def release_license(self): self._clear() self.__class__.release_license() @@ -617,17 +630,21 @@ def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) self._needs_updated = False return res - + def _process_domain_and_bounds(self, var): res = super()._process_domain_and_bounds(var) if not is_constant(var._lb): - mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + mutable_lb = _MutableLowerBound( + id(var), var.lower, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) if not is_constant(var._ub): - mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + mutable_ub = _MutableUpperBound( + id(var), var.upper, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) return res - + def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() super()._add_variables(variables) @@ -673,37 +690,60 @@ def _add_constraints(self, cons: List[ConstraintData]): mutable_constant = None if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: rhs_expr = ub - repn.constant gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif ub is None: rhs_expr = lb - repn.constant gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif con.equality: rhs_expr = lb - repn.constant gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) else: - assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + assert ( + len(repn.quadratic_vars) == 0 + ), "Quadratic range constraints are not supported" lhs_expr = lb - repn.constant rhs_expr = ub - repn.constant - gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + gurobi_expr_list.append( + gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))] + ) if not is_constant(lhs_expr) or not is_constant(rhs_expr): conname = f'c{self._constraint_ndx}[{ndx}]' - mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + mutable_constant = _MutableRangeConstant( + lhs_expr, + rhs_expr, + con, + self._pyomo_con_to_solver_con_map, + 'Rg' + conname, + self._solver_model, + ) mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + con, + self._pyomo_con_to_solver_con_map, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) if len(repn.quadratic_vars) == 0: @@ -715,15 +755,19 @@ def _add_constraints(self, cons: List[ConstraintData]): self._mutable_helpers[con].append(mutable_constant) else: if mutable_constant is None: - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) mqc = _MutableQuadraticConstraint( self._solver_model, - con, + con, self._pyomo_con_to_solver_con_map, mutable_constant, mlc_list, @@ -731,10 +775,12 @@ def _add_constraints(self, cons: List[ConstraintData]): ) self._mutable_quadratic_helpers[con] = mqc - gurobi_cons = list(self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), - name=f'c{self._constraint_ndx}' - ).values()) + gurobi_cons = list( + self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}', + ).values() + ) self._constraint_ndx += 1 self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) self._constraints_added_since_update.update(cons) @@ -761,7 +807,9 @@ def _set_objective(self, obj): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) @@ -770,16 +818,27 @@ def _set_objective(self, obj): mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) - self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) # hack # see PR #2454 @@ -865,9 +924,7 @@ def _update_parameters(self, params: List[ParamData]): new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, - ) + new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) self._pyomo_con_to_solver_con_map[con] = new_con helper.pyomo_con = con self._constraints_added_since_update.add(con) @@ -880,7 +937,7 @@ def _update_parameters(self, params: List[ParamData]): else: sense = gurobipy.GRB.MAXIMIZE # TODO: need a test for when part of the object is linear - # and part of the objective is quadratic, but both + # and part of the objective is quadratic, but both # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) @@ -1309,4 +1366,4 @@ def update_variables(self, variables): self._change_detector.update_variables(variables) def update_parameters(self, params): - self._change_detector.update_parameters(params) \ No newline at end of file + self._change_detector.update_parameters(params) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index a0d87835e13..96e2e7b2c38 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -31,7 +31,10 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( + GurobiDirectQuadratic, + GurobiPersistent, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -59,11 +62,9 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [ - ('ipopt', Ipopt), -] +nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ - ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] From 7249b1941c6b3aad5cd3f7a3bd34f7ab28ef3566 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:57:56 -0600 Subject: [PATCH 39/97] update solution loader --- .../solvers/gurobi/gurobi_persistent.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 6628f001421..05acfef2b4f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize @@ -54,6 +55,25 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._vars = None + self._var_map = None + self._con_map = None + self._linear_cons = None + self._quadratic_cons = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 From ac42345de6f87fe5010af92bbdf2cf7d772d95d4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:41:27 -0600 Subject: [PATCH 40/97] updating solution loader --- pyomo/contrib/solver/common/base.py | 7 +-- .../contrib/solver/common/solution_loader.py | 47 +++++++++++++++++-- .../solver/solvers/gurobi/gurobi_direct.py | 32 +++++++++---- pyomo/contrib/solver/solvers/highs.py | 4 +- pyomo/contrib/solver/solvers/ipopt.py | 36 ++++++-------- pyomo/contrib/solver/solvers/sol_reader.py | 30 +++++++++--- 6 files changed, 107 insertions(+), 49 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 280b80629a3..f935f3d4988 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -567,12 +567,7 @@ def _solution_handler( legacy_results._smap_id = id(symbol_map) delete_legacy_soln = True if load_solutions: - if hasattr(model, 'dual') and model.dual.import_enabled(): - for con, val in results.solution_loader.get_duals().items(): - model.dual[con] = val - if hasattr(model, 'rc') and model.rc.import_enabled(): - for var, val in results.solution_loader.get_reduced_costs().items(): - model.rc[var] = val + results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False for var, val in results.solution_loader.get_primals().items(): diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 065c00185f6..e399d6bea55 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,11 +9,32 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations + from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base.suffix import Suffix + + +def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): + dual_suffix = None + rc_suffix = None + for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): + if not suffix.import_enabled(): + continue + if suffix.local_name == 'dual': + dual_suffix = suffix + elif suffix.local_name == 'rc': + rc_suffix = suffix + if dual_suffix is not None: + for k, v in solution_loader.get_duals(solution_id=solution_id).items(): + dual_suffix[k] = v + if rc_suffix is not None: + for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): + rc_suffix[k] = v class SolutionLoaderBase: @@ -178,29 +199,45 @@ class PersistentSolutionLoader(SolutionLoaderBase): Loader for persistent solvers """ - def __init__(self, solver): + def __init__(self, solver, pyomo_model): self._solver = solver self._valid = True + self._pyomo_model = pyomo_model def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def get_primals(self, vars_to_load=None): + def get_solution_ids(self) -> List[Any]: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load) + return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + def invalidate(self): self._valid = False diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 16c633c7d7c..cca23315b1a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ import operator +from typing import List from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -22,17 +23,18 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from .gurobi_direct_base import GurobiDirectBase, gurobipy class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyomo_model): self._grb_model = grb_model self._grb_cons = grb_cons self._grb_vars = grb_vars self._pyo_cons = pyo_cons self._pyo_vars = pyo_vars + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -44,6 +46,7 @@ def __del__(self): self._grb_vars = None self._pyo_cons = None self._pyo_vars = None + self._pyomo_model = None # explicitly release the model self._grb_model.dispose() self._grb_model = None @@ -52,8 +55,16 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_number_of_solutions(self) -> int: + if self._grb_model.SolCount == 0: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [0] + + def load_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -65,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_number=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,7 +87,8 @@ def get_primals(self, vars_to_load=None, solution_number=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None): + def get_duals(self, cons_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -96,7 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None): + def get_reduced_costs(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() @@ -105,6 +118,9 @@ def get_reduced_costs(self, vars_to_load=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 6eb4afa828a..a1d609c5db6 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -675,7 +675,7 @@ def _postsolve(self): status = highs.getModelStatus() results = Results() - results.solution_loader = PersistentSolutionLoader(self) + results.solution_loader = PersistentSolutionLoader(self, self._model) results.timing_info.highs_time = highs.getRunTime() self._sol = highs.getSolution() @@ -751,7 +751,7 @@ def _postsolve(self): if config.load_solutions: if has_feasible_solution: - self._load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 075fc998ecc..441b8eb5f61 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,13 @@ def _error_check(self): ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._error_check() + if solution_id is not None: + raise ValueError('IpoptSolutionLoader does not support solution_id') if self._nl_info.scaling is None: scale_list = [1] * len(self._nl_info.variables) obj_scale = 1 @@ -430,35 +434,35 @@ def solve(self, model, **kwds) -> Results: if proven_infeasible: results = Results() results.termination_condition = TerminationCondition.provenInfeasible - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) results.iteration_count = 0 results.timing_info.total_seconds = 0 elif len(nl_info.variables) == 0: if len(nl_info.eliminated_vars) == 0: results = Results() results.termination_condition = TerminationCondition.emptyModel - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: results = Results() results.termination_condition = ( TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info) + results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: if os.path.isfile(basename + '.sol'): with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: timer.start('parse_sol') - results = self._parse_solution(sol_file, nl_info) + results = self._parse_solution(sol_file, nl_info, model) timer.stop('parse_sol') else: results = Results() if process.returncode != 0: results.extra_info.return_code = process.returncode results.termination_condition = TerminationCondition.error - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: try: results.iteration_count = parsed_output_data.pop('iters') @@ -490,19 +494,7 @@ def solve(self, model, **kwds) -> Results: if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() - results.solution_loader.load_vars() - if ( - hasattr(model, 'dual') - and isinstance(model.dual, Suffix) - and model.dual.import_enabled() - ): - model.dual.update(results.solution_loader.get_duals()) - if ( - hasattr(model, 'rc') - and isinstance(model.rc, Suffix) - and model.rc.import_enabled() - ): - model.rc.update(results.solution_loader.get_reduced_costs()) + results.solution_loader.load_solution() if ( results.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal} @@ -665,7 +657,7 @@ def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any] return parsed_data def _parse_solution( - self, instream: io.TextIOBase, nl_info: NLWriterInfo + self, instream: io.TextIOBase, nl_info: NLWriterInfo, pyomo_model ) -> Results: results = Results() res, sol_data = parse_sol_file( @@ -673,10 +665,10 @@ def _parse_solution( ) if res.solution_status == SolutionStatus.noSolution: - res.solution_loader = SolSolutionLoader(None, None) + res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index e580e2a72f9..7570a1ffc53 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,7 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes class SolFileData: @@ -49,11 +49,25 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo) -> None: + def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: self._sol_data = sol_data self._nl_info = nl_info + self._pyomo_model = pyomo_model - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_number_of_solutions(self) -> int: + if self._nl_info is None: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [None] + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -78,9 +92,11 @@ def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoRetur StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -115,8 +131,10 @@ def get_primals( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, ) -> Dict[ConstraintData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' From 275d848d2c5eb09db7ac6b519794e1a0065b5869 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:43:11 -0600 Subject: [PATCH 41/97] run black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd905e1c61d..4ab52100376 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -208,10 +208,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -237,13 +234,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -347,7 +346,10 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 70ca6e72d50ac39a3c580f116dfd6c75a9b29e92 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:20:44 -0600 Subject: [PATCH 42/97] updating solution loader --- .../solver/solvers/gurobi/gurobi_direct.py | 2 +- .../solvers/gurobi/gurobi_direct_base.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 44 ++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index cca23315b1a..82b47ccb24b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -193,7 +193,7 @@ def _create_solver_model(self, pyomo_model): self._gurobi_vars = x solution_loader = GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns + gurobi_model, A, x, repn.rows, repn.columns, pyomo_model ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index df6bb8b5327..ae887a52fa5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -440,7 +440,7 @@ def _postsolve(self, grb_model, solution_loader, has_obj): self.config.timer.start('load solution') if self.config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() self.config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..8477d855a02 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,7 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +46,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__() self._solver_model = solver_model @@ -55,6 +55,7 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -75,6 +76,12 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() + def get_number_of_solutions(self) -> int: + return self._solver_model.SolCount + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: @@ -87,7 +94,7 @@ def load_vars( solution_number=solution_id, ) - def get_primals( + def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: @@ -100,7 +107,7 @@ def get_primals( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -111,7 +118,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -130,13 +137,16 @@ def get_duals( quadratic_cons_to_load=quadratic_cons_to_load, ) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) self._valid = True @@ -153,23 +163,35 @@ def load_vars( self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - def get_primals( + def get_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) class _MutableLowerBound: @@ -380,6 +402,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj @@ -638,6 +661,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj From 1788ff371d52448a16b539c868aa40142c416b73 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:23:17 -0600 Subject: [PATCH 43/97] dont free gurobi models twice --- .../solver/solvers/gurobi/gurobi_persistent.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..847ec958bdd 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,18 +58,6 @@ def __init__( GurobiDirectBase._register_env_client() def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._solver_model is not None: - self._vars = None - self._var_map = None - self._con_map = None - self._linear_cons = None - self._quadratic_cons = None - # explicitly release the model - self._solver_model.dispose() - self._solver_model = None # Release the gurobi license if this is the last reference to # the environment (either through a results object or solver # interface) From 2885f42e665a6fc87c854d60984145dffd86547a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 08:07:54 -0600 Subject: [PATCH 44/97] update solution loader --- pyomo/contrib/solver/common/base.py | 2 +- .../solver/solvers/gurobi/gurobi_direct.py | 18 +++++++------- .../solvers/gurobi/gurobi_direct_base.py | 8 +++---- .../solvers/gurobi/gurobi_persistent.py | 22 ++++++++--------- pyomo/contrib/solver/solvers/highs.py | 16 +++++++++---- pyomo/contrib/solver/solvers/ipopt.py | 2 +- .../solver/tests/solvers/test_ipopt.py | 2 +- .../solver/tests/solvers/test_solvers.py | 12 +++++----- .../contrib/solver/tests/unit/test_results.py | 11 +++++---- .../solver/tests/unit/test_solution.py | 24 ++++++++++--------- 10 files changed, 65 insertions(+), 52 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index f935f3d4988..0782e577c43 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -570,7 +570,7 @@ def _solution_handler( results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False - for var, val in results.solution_loader.get_primals().items(): + for var, val in results.solution_loader.get_vars().items(): legacy_soln.variable[symbol_map.getSymbol(var)] = {'Value': val} if hasattr(model, 'dual') and model.dual.import_enabled(): for con, val in results.solution_loader.get_duals().items(): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 82b47ccb24b..fd932f90c15 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,7 +10,7 @@ # ___________________________________________________________________________ import operator -from typing import List +from typing import List, Any from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -63,8 +63,8 @@ def get_number_of_solutions(self) -> int: def get_solution_ids(self) -> List[Any]: return [0] - def load_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def load_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_id=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -87,8 +87,8 @@ def get_vars(self, vars_to_load=None, solution_id=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None, solution_id=0): - assert solution_id == 0 + def get_duals(self, cons_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -108,8 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_reduced_costs(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ae887a52fa5..e99d24025d5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -99,7 +99,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ return res -def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): +def _load_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -107,7 +107,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): vars_to_load: List[VarData] solution_number: int """ - for v, val in _get_primals( + for v, val in _get_vars( solver_model=solver_model, var_map=var_map, vars_to_load=vars_to_load, @@ -117,7 +117,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): StaleFlagManager.mark_all_as_stale(delayed=True) -def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): +def _get_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -128,7 +128,7 @@ def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): if solver_model.SolCount == 0: raise NoSolutionError() - if solution_number != 0: + if solution_number not in {0, None}: return _load_suboptimal_mip_solution( solver_model=solver_model, var_map=var_map, diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 752b8512128..9f19bae307f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -33,7 +33,7 @@ GurobiDirectBase, gurobipy, _load_vars, - _get_primals, + _get_vars, _get_duals, _get_reduced_costs, ) @@ -71,7 +71,7 @@ def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) def load_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -83,11 +83,11 @@ def load_vars( ) def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) - return _get_primals( + return _get_vars( solver_model=self._solver_model, var_map=self._var_map, vars_to_load=vars_to_load, @@ -95,7 +95,7 @@ def get_vars( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -106,7 +106,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -146,25 +146,25 @@ def _assert_solution_still_valid(self): raise RuntimeError('The results in the solver are no longer valid.') def load_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) def get_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() - return super().get_primals(vars_to_load, solution_id) + return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index a1d609c5db6..0abf02813ab 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -758,12 +758,16 @@ def _postsolve(self): return results - def _load_vars(self, vars_to_load=None): + def _load_vars(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def _get_primals(self, vars_to_load=None): + def _get_primals(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -786,7 +790,9 @@ def _get_primals(self, vars_to_load=None): return res - def _get_reduced_costs(self, vars_to_load=None): + def _get_reduced_costs(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -804,7 +810,9 @@ def _get_reduced_costs(self, vars_to_load=None): return res - def _get_duals(self, cons_to_load=None): + def _get_duals(self, cons_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 441b8eb5f61..80f9775a657 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -508,7 +508,7 @@ def solve(self, model, **kwds) -> Results: nl_info.objectives[0].expr, substitution_map={ id(v): val - for v, val in results.solution_loader.get_primals().items() + for v, val in results.solution_loader.get_vars().items() }, descend_into_named_expressions=True, remove_named_expressions=True, diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index d788b66982a..f0049922d55 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -62,7 +62,7 @@ def test_custom_instantiation(self): class TestIpoptSolutionLoader(unittest.TestCase): def test_get_reduced_costs_error(self): - loader = ipopt.IpoptSolutionLoader(None, None) + loader = ipopt.IpoptSolutionLoader(None, None, None) with self.assertRaises(NoSolutionError): loader.get_reduced_costs() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..9c67ed7b1e5 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1646,12 +1646,12 @@ def test_solution_loader( m.y.value = None res.solution_loader.load_vars([m.y]) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) - primals = res.solution_loader.get_primals([m.y]) + primals = res.solution_loader.get_vars([m.y]) self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) @@ -2000,7 +2000,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 1) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertIn(m.z, sol) @@ -2010,7 +2010,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 0) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertNotIn(m.z, sol) @@ -2172,7 +2172,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 1) self.assertAlmostEqual(m.x.value, 1) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) if check_duals: @@ -2188,7 +2188,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 2) self.assertAlmostEqual(m.x.value, 2) self.assertAlmostEqual(m.y.value, 2) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 2) self.assertAlmostEqual(primals[m.y], 2) if check_duals: diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a818f4ff4ad..a4def8f9089 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -49,8 +49,9 @@ def __init__( self._duals = duals self._reduced_costs = reduced_costs - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -66,7 +67,8 @@ def get_primals( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -83,7 +85,8 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index 0453f0e0cb2..a0fc4ac9b2f 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,7 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -29,18 +29,16 @@ def test_member_list(self): def test_solution_loader_base(self): self.instance = SolutionLoaderBase() with self.assertRaises(NotImplementedError): - self.instance.get_primals() - with self.assertRaises(NotImplementedError): - self.instance.get_duals() - with self.assertRaises(NotImplementedError): - self.instance.get_reduced_costs() + self.instance.get_vars() + self.assertEqual(self.instance.get_duals(), NotImplemented) + self.assertEqual(self.instance.get_reduced_costs(), NotImplemented) class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -53,10 +51,14 @@ class TestPersistentSolutionLoader(unittest.TestCase): def test_member_list(self): expected_list = [ 'load_vars', - 'get_primals', + 'get_vars', 'get_duals', 'get_reduced_costs', 'invalidate', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution' ] method_list = [ method @@ -69,12 +71,12 @@ def test_default_initialization(self): # Realistically, a solver object should be passed into this. # However, it works with a string. It'll just error loudly if you # try to run get_primals, etc. - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.assertTrue(self.instance._valid) self.assertEqual(self.instance._solver, 'ipopt') def test_invalid(self): - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.instance.invalidate() with self.assertRaises(RuntimeError): - self.instance.get_primals() + self.instance.get_vars() From d16bee5fd1f27cc7c3b55ff858ab28b7478a6975 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:00:06 -0600 Subject: [PATCH 45/97] adding tests for trivial constraints and fixing bugs --- pyomo/contrib/observer/model_observer.py | 2 +- .../contrib/solver/common/solution_loader.py | 30 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 23 +++++++- .../solvers/gurobi/gurobi_persistent.py | 6 ++- .../solver/tests/solvers/test_solvers.py | 53 +++++++++++++++++++ pyomo/repn/plugins/standard_form.py | 3 +- 6 files changed, 113 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4ab52100376..325e7a8aee6 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -614,7 +614,7 @@ def _check_for_var_changes(self): vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: + elif v.fixed and v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..be0ea7ad00c 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -17,6 +17,7 @@ from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix +from .util import NoSolutionError def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): @@ -194,6 +195,35 @@ def load_import_suffixes(self, solution_id=None): return NotImplemented +class NoSolutionSolutionLoader(SolutionLoaderBase): + def __init__(self) -> None: + pass + + def get_solution_ids(self) -> List[Any]: + return [] + + def get_number_of_solutions(self) -> int: + return 0 + + def load_solution(self, solution_id=None): + raise NoSolutionError() + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + raise NoSolutionError() + + def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + raise NoSolutionError() + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def load_import_suffixes(self, solution_id=None): + raise NoSolutionError() + + class PersistentSolutionLoader(SolutionLoaderBase): """ Loader for persistent solvers diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e99d24025d5..ce77c31c6f7 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -18,7 +18,7 @@ from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,6 +33,7 @@ NoReducedCostsError, NoSolutionError, ) +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, @@ -353,6 +354,8 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) + except InfeasibleConstraintException: + res = self._get_infeasible_results() finally: os.chdir(orig_cwd) @@ -390,6 +393,24 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.gurobi_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..27a61e27916 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -17,6 +17,7 @@ from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer +from pyomo.common.errors import InfeasibleConstraintException from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base.var import VarData @@ -464,7 +465,9 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - new_expr = 0.0 + # this can't just be zero in case the constraint is a + # trivial one + new_expr = gurobipy.LinExpr() if len(repn.quadratic_vars) > 0: missing_vars = {} @@ -714,6 +717,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: self._quadratic_cons.add(con) else: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 9c67ed7b1e5..d154253475c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1053,6 +1054,58 @@ def test_results_infeasible( ): res.solution_loader.get_reduced_costs() + @parameterized.expand(input=_load_tests(all_solvers)) + def test_trivial_constraints( + self, name: str, opt_class: Type[SolverBase], use_presolve: bool + ): + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= m.x) + m.c2 = pyo.Constraint(expr=m.y >= -m.x) + m.c3 = pyo.Constraint(expr=m.x >= 0) + + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0) + + m.x.fix(1) + opt.config.tee = True + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 1) + self.assertAlmostEqual(m.y.value, 1) + + m.x.fix(-1) + with self.assertRaises(NoOptimalSolutionError): + res = opt.solve(m) + + opt.config.load_solutions = False + opt.config.raise_exception_on_nonoptimal_result = False + res = opt.solve(m) + self.assertNotEqual(res.solution_status, SolutionStatus.optimal) + if isinstance(opt, Ipopt): + acceptable_termination_conditions = { + TerminationCondition.locallyInfeasible, + TerminationCondition.unbounded, + TerminationCondition.provenInfeasible, + } + else: + acceptable_termination_conditions = { + TerminationCondition.provenInfeasible, + TerminationCondition.infeasibleOrUnbounded, + } + self.assertIn(res.termination_condition, acceptable_termination_conditions) + self.assertIsNone(res.incumbent_objective) + @parameterized.expand(input=_load_tests(all_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 314a1822e09..59c15910350 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -20,6 +20,7 @@ InEnum, document_kwargs_from_configdict, ) +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.dependencies import scipy, numpy as np from pyomo.common.enums import ObjectiveSense from pyomo.common.gc_manager import PauseGC @@ -462,7 +463,7 @@ def write(self, model): # TODO: add a (configurable) feasibility tolerance if (lb is None or lb <= offset) and (ub is None or ub >= offset): continue - raise InfeasibleError( + raise InfeasibleConstraintException( f"model contains a trivially infeasible constraint, '{con.name}'" ) From a4e2b81410b9edba643ef118f21e4ca9f3cc0b37 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:05:43 -0600 Subject: [PATCH 46/97] run black --- .../contrib/solver/common/solution_loader.py | 69 ++++++++----------- .../solver/solvers/gurobi/gurobi_direct.py | 13 ++-- .../solvers/gurobi/gurobi_persistent.py | 43 +++++++++--- pyomo/contrib/solver/solvers/highs.py | 16 +++-- pyomo/contrib/solver/solvers/ipopt.py | 10 +-- pyomo/contrib/solver/solvers/sol_reader.py | 17 +++-- .../contrib/solver/tests/unit/test_results.py | 9 +-- .../solver/tests/unit/test_solution.py | 24 ++++++- 8 files changed, 125 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..3ad688d937f 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -19,7 +19,9 @@ from pyomo.core.base.suffix import Suffix -def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): +def load_import_suffixes( + pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None +): dual_suffix = None rc_suffix = None for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): @@ -46,10 +48,10 @@ class SolutionLoaderBase: def get_solution_ids(self) -> List[Any]: """ - If there are multiple solutions available, this will return a - list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is - available, this will return [None]. If no solutions + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions are available, this will return None Returns @@ -58,7 +60,7 @@ def get_solution_ids(self) -> List[Any]: The identifiers for multiple solutions """ return NotImplemented - + def get_number_of_solutions(self) -> int: """ Returns @@ -75,7 +77,7 @@ def load_solution(self, solution_id=None): Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ # this should load everything it can @@ -83,36 +85,31 @@ def load_solution(self, solution_id=None): self.load_import_suffixes(solution_id=solution_id) def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: """ - Load the solution of the primal variables into the value attribute + Load the solution of the primal variables into the value attribute of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If - vars_to_load is None, then the solution to all primal variables - will be loaded. Even if vars_to_load is specified, the values of + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of other variables may also be loaded depending on the interface. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ for var, val in self.get_vars( - vars_to_load=vars_to_load, - solution_id=solution_id + vars_to_load=vars_to_load, solution_id=solution_id ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def get_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -123,7 +120,7 @@ def get_vars( A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -136,9 +133,7 @@ def get_vars( ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -149,7 +144,7 @@ def get_duals( A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -160,9 +155,7 @@ def get_duals( return NotImplemented def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -173,7 +166,7 @@ def get_reduced_costs( A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -182,13 +175,13 @@ def get_reduced_costs( Maps variables to reduced costs """ return NotImplemented - + def load_import_suffixes(self, solution_id=None): """ Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ return NotImplemented @@ -211,27 +204,25 @@ def _assert_solution_still_valid(self): def get_solution_ids(self) -> List[Any]: self._assert_solution_still_valid() return super().get_solution_ids() - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_id=solution_id + ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index fd932f90c15..42ab82a1d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,7 +23,10 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from .gurobi_direct_base import GurobiDirectBase, gurobipy @@ -59,7 +62,7 @@ def get_number_of_solutions(self) -> int: if self._grb_model.SolCount == 0: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [0] @@ -118,9 +121,11 @@ def get_reduced_costs(self, vars_to_load=None, solution_id=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) - + def load_import_suffixes(self, solution_id=None): - load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) + load_import_suffixes( + pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id + ) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..e36e1e6db1e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,10 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +49,14 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__() self._solver_model = solver_model @@ -66,7 +76,7 @@ def __del__(self): def get_number_of_solutions(self) -> int: return self._solver_model.SolCount - + def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) @@ -131,10 +141,23 @@ def load_import_suffixes(self, solution_id=None): class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) self._valid = True @@ -158,25 +181,25 @@ def get_vars( return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() - + def get_solution_ids(self) -> List: self._assert_solution_still_valid() return super().get_solution_ids() - + def load_import_suffixes(self, solution_id=None): self._assert_solution_still_valid() super().load_import_suffixes(solution_id) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 0abf02813ab..87796b91b68 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -760,14 +760,18 @@ def _postsolve(self): def _load_vars(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def _get_primals(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -792,7 +796,9 @@ def _get_primals(self, vars_to_load=None, solution_id=None): def _get_reduced_costs(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -812,7 +818,9 @@ def _get_reduced_costs(self, vars_to_load=None, solution_id=None): def _get_duals(self, cons_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 80f9775a657..bce5e9bd867 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,7 @@ def _error_check(self): ) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._error_check() if solution_id is not None: @@ -448,7 +446,9 @@ def solve(self, model, **kwds) -> Results: TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) + results.solution_loader = SolSolutionLoader( + None, nl_info=nl_info, pyomo_model=model + ) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: @@ -668,7 +668,7 @@ def _parse_solution( res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index 7570a1ffc53..f405cc85943 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,10 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) class SolFileData: @@ -49,7 +52,9 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: + def __init__( + self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model + ) -> None: self._sol_data = sol_data self._nl_info = nl_info self._pyomo_model = pyomo_model @@ -58,14 +63,16 @@ def get_number_of_solutions(self) -> int: if self._nl_info is None: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [None] def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> NoReturn: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: @@ -131,7 +138,7 @@ def get_vars( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a4def8f9089..3dad4c523d2 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -50,8 +50,7 @@ def __init__( self._reduced_costs = reduced_costs def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -67,8 +66,7 @@ def get_vars( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -85,8 +83,7 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index a0fc4ac9b2f..79e5b39aaf6 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,16 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -38,7 +47,16 @@ class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -58,7 +76,7 @@ def test_member_list(self): 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', - 'load_solution' + 'load_solution', ] method_list = [ method From d25e7215f4f728ff4541060f84f9b6cd92d09229 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:06:51 -0600 Subject: [PATCH 47/97] run black --- .../contrib/solver/common/solution_loader.py | 30 ++++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 2 +- .../solver/tests/solvers/test_solvers.py | 10 +++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index b0bddeb56ba..666ea66e1e9 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -194,25 +194,33 @@ def __init__(self) -> None: def get_solution_ids(self) -> List[Any]: return [] - + def get_number_of_solutions(self) -> int: return 0 - + def load_solution(self, solution_id=None): raise NoSolutionError() - - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: raise NoSolutionError() - - def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: raise NoSolutionError() - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - + def load_import_suffixes(self, solution_id=None): raise NoSolutionError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index c0211604324..b9ea9a6c8e8 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -488,7 +488,7 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - # this can't just be zero in case the constraint is a + # this can't just be zero in case the constraint is a # trivial one new_expr = gurobipy.LinExpr() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index d154253475c..189b0373780 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,7 +35,11 @@ GurobiDirectQuadratic, GurobiPersistent, ) -from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError +from pyomo.contrib.solver.common.util import ( + NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1073,11 +1077,11 @@ def test_trivial_constraints( m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y >= -m.x) m.c3 = pyo.Constraint(expr=m.x >= 0) - + res = opt.solve(m) self.assertAlmostEqual(m.x.value, 0) self.assertAlmostEqual(m.y.value, 0) - + m.x.fix(1) opt.config.tee = True res = opt.solve(m) From 9e7cd0dc9f4679a7b19b4706b2ffcf963bb7dc3f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:39 -0600 Subject: [PATCH 48/97] moving scip to contrib solvers --- .../solvers => contrib/solver/solvers/scip}/scip_direct.py | 0 .../solvers => contrib/solver/solvers/scip}/scip_persistent.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_direct.py (100%) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_persistent.py (100%) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_direct.py rename to pyomo/contrib/solver/solvers/scip/scip_direct.py diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_persistent.py rename to pyomo/contrib/solver/solvers/scip/scip_persistent.py From bf204bb2b6d21644fa5dd8f0d83521075541cb9f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:57 -0600 Subject: [PATCH 49/97] moving scip to contrib solvers --- pyomo/contrib/solver/solvers/scip/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/scip/__init__.py diff --git a/pyomo/contrib/solver/solvers/scip/__init__.py b/pyomo/contrib/solver/solvers/scip/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From dabf031e34490ed6dcf00e25eaba68abafc7a022 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:36:24 -0600 Subject: [PATCH 50/97] porting scip interface --- .../contrib/solver/common/solution_loader.py | 20 +- .../solvers/gurobi/gurobi_direct_base.py | 1 - .../solver/solvers/scip/scip_direct.py | 1228 ++++++++--------- 3 files changed, 604 insertions(+), 645 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 666ea66e1e9..f8723b6e0f4 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -18,6 +18,10 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix from .util import NoSolutionError +import logging + + +logger = logging.getLogger(__name__) def load_import_suffixes( @@ -33,11 +37,19 @@ def load_import_suffixes( elif suffix.local_name == 'rc': rc_suffix = suffix if dual_suffix is not None: - for k, v in solution_loader.get_duals(solution_id=solution_id).items(): - dual_suffix[k] = v + duals = solution_loader.get_duals(solution_id=solution_id) + if duals is NotImplemented: + logger.warning(f'Cannot load duals into suffix') + else: + for k, v in duals.items(): + dual_suffix[k] = v if rc_suffix is not None: - for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): - rc_suffix[k] = v + rc = solution_loader.get_reduced_costs(solution_id=solution_id) + if rc is NotImplemented: + logger.warning(f'cannot load duals into suffix') + else: + for k, v in rc.items(): + rc_suffix[k] = v class SolutionLoaderBase: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ce77c31c6f7..8989fc5047a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -362,7 +362,6 @@ def solve(self, model, **kwds) -> Results: # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = orig_config object.__setattr__(self, 'config', orig_config) - self.config = orig_config res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index c862d9047c1..b8d4d14a6c1 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,392 +9,578 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import datetime +import io import logging -import sys - -from pyomo.common.collections import ComponentSet, ComponentMap, Bunch -from pyomo.common.tempfiles import TempfileManager -from pyomo.core import Var +from typing import Tuple, List, Optional, Sequence, Mapping, Dict + +from pyomo.common.collections import ComponentMap +from pyomo.common.numeric_types import native_numeric_types +from pyomo.common.errors import InfeasibleConstraintException, ApplicationError +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.base.block import BlockData +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.sos import SOSConstraint, SOSConstraintData +from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( - SumExpression, - ProductExpression, - UnaryFunctionExpression, + NegationExpression, PowExpression, + ProductExpression, + MonomialTermExpression, DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, ) -from pyomo.core.expr.numvalue import is_fixed -from pyomo.core.expr.numvalue import value +from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn import generate_standard_repn -from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver -from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( - DirectOrPersistentSolver, +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.dependencies import attempt_import +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, ) -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.opt.results.results_ import SolverResults -from pyomo.opt.results.solution import Solution, SolutionStatus -from pyomo.opt.results.solver import TerminationCondition, SolverStatus -from pyomo.opt.base import SolverFactory +from pyomo.contrib.solver.common.util import get_objective +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) +from pyomo.common.config import ConfigValue +from pyomo.common.tee import capture_output, TeeStream -logger = logging.getLogger("pyomo.solvers") +logger = logging.getLogger(__name__) -class DegreeError(ValueError): - pass +scip, scip_available = attempt_import('pyscipyopt') -def _is_numeric(x): - try: - float(x) - except ValueError: - return False - return True +class ScipConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Scip.", + ), + ) -@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") -class SCIPDirect(DirectSolver): +def _handle_var(node, data, opt): + if id(node) not in opt._pyomo_var_to_solver_var_map: + scip_var = opt._add_var(node) + else: + scip_var = opt._pyomo_var_to_solver_var_map[id(node)] + return scip_var - def __init__(self, **kwds): - kwds["type"] = "scipdirect" - DirectSolver.__init__(self, **kwds) - self._init() - self._solver_model = None - def _init(self): - try: - import pyscipopt +def _handle_param(node, data, opt): + if not node.mutable: + return node.value + if id(node) not in opt._pyomo_param_to_solver_param_map: + scip_param = opt._add_param(node) + else: + scip_param = opt._pyomo_param_to_solver_param_map[id(node)] + return scip_param - self._scip = pyscipopt - self._python_api_exists = True - self._version = tuple( - int(k) for k in str(self._scip.Model().version()).split(".") - ) - self._version_major = self._version[0] - except ImportError: - self._python_api_exists = False - except Exception as e: - print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") - self._python_api_exists = False - - # Note: Undefined capabilities default to None - self._max_constraint_degree = None - self._max_obj_degree = 1 - self._capabilities.linear = True - self._capabilities.quadratic_objective = False - self._capabilities.quadratic_constraint = True - self._capabilities.integer = True - self._capabilities.sos1 = True - self._capabilities.sos2 = True - self._skip_trivial_constraints = True - - # Dictionary used exclusively for SCIP, as we want the constraint expressions - self._pyomo_var_to_solver_var_expr_map = ComponentMap() - self._pyomo_con_to_solver_con_expr_map = dict() - - def _apply_solver(self): - StaleFlagManager.mark_all_as_stale() - - # Suppress solver output if requested - if self._tee: - self._solver_model.hideOutput(quiet=False) - else: - self._solver_model.hideOutput(quiet=True) - # Redirect solver output to a logfile if requested - if self._keepfiles: - # Only save log file when the user wants to keep it. - self._solver_model.setLogfile(self._log_file) - print(f"Solver log file: {self._log_file}") +def _handle_float(node, data, opt): + return float(node) - # Set user specified parameters - for key, option in self.options.items(): - try: - key_type = type(self._solver_model.getParam(key)) - except KeyError: - raise ValueError(f"Key {key} is an invalid parameter for SCIP") - if key_type == str: - self._solver_model.setParam(key, option) - else: - if not _is_numeric(option): - raise ValueError( - f"Value {option} for parameter {key} is not a string and can't be converted to float" - ) - self._solver_model.setParam(key, float(option)) - - self._solver_model.optimize() - - # TODO: Check if this is even needed, or if it is sufficient to close the open file - # if self._keepfiles: - # self._solver_model.setLogfile(None) - - # FIXME: can we get a return code indicating if SCIP had a significant failure? - return Bunch(rc=None, log=None) - - def _get_expr_from_pyomo_repn(self, repn, max_degree=None): - referenced_vars = ComponentSet() - - degree = repn.polynomial_degree() - if (max_degree is not None) and (degree > max_degree): - raise DegreeError( - "While SCIP supports general non-linear constraints, the objective must be linear. " - "Please reformulate the objective by introducing a new variable. " - "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " - "f(x) is the original non-linear objective." - ) +def _handle_negation(node, data, opt): + return -data[0] - new_expr = repn.constant - if len(repn.linear_vars) > 0: - referenced_vars.update(repn.linear_vars) - new_expr += sum( - repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] - for i, var in enumerate(repn.linear_vars) - ) +def _handle_pow(node, data, opt): + return data[0] ** data[1] - for i, v in enumerate(repn.quadratic_vars): - x, y = v - new_expr += ( - repn.quadratic_coefs[i] - * self._pyomo_var_to_solver_var_expr_map[x] - * self._pyomo_var_to_solver_var_expr_map[y] - ) - referenced_vars.add(x) - referenced_vars.add(y) - - if repn.nonlinear_expr is not None: - - def get_nl_expr_recursively(pyomo_expr): - if not hasattr(pyomo_expr, "args"): - if not isinstance(pyomo_expr, Var): - return float(pyomo_expr) - else: - referenced_vars.add(pyomo_expr) - return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] - scip_expr_list = [0 for i in range(pyomo_expr.nargs())] - for i in range(pyomo_expr.nargs()): - scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) - if isinstance(pyomo_expr, PowExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"PowExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] ** (scip_expr_list[1]) - elif isinstance(pyomo_expr, ProductExpression): - return self._scip.quickprod(scip_expr_list) - elif isinstance(pyomo_expr, SumExpression): - return self._scip.quicksum(scip_expr_list) - elif isinstance(pyomo_expr, DivisionExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] / scip_expr_list[1] - elif isinstance(pyomo_expr, UnaryFunctionExpression): - if len(scip_expr_list) != 1: - raise ValueError( - f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" - ) - if pyomo_expr.name == "sin": - return self._scip.sin(scip_expr_list[0]) - elif pyomo_expr.name == "cos": - return self._scip.cos(scip_expr_list[0]) - elif pyomo_expr.name == "exp": - return self._scip.exp(scip_expr_list[0]) - elif pyomo_expr.name == "log": - return self._scip.log(scip_expr_list[0]) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" - ) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" - ) - new_expr += get_nl_expr_recursively(repn.nonlinear_expr) +def _handle_product(node, data, opt): + assert len(data) == 2 + return data[0] * data[1] + + +def _handle_division(node, data, opt): + return data[0] / data[1] + + +def _handle_sum(node, data, opt): + return sum(data) + + +def _handle_exp(node, data, opt): + return scip.exp(data[0]) + + +def _handle_log(node, data, opt): + return scip.log(data[0]) + + +def _handle_sin(node, data, opt): + return scip.sin(data[0]) + + +def _handle_cos(node, data, opt): + return scip.cos(data[0]) + + +def _handle_sqrt(node, data, opt): + return scip.sqrt(data[0]) + + +def _handle_abs(node, data, opt): + return abs(data[0]) + - return new_expr, referenced_vars +def _handle_tan(node, data, opt): + return scip.sin(data[0]) / scip.cos(data[0]) - def _get_expr_from_pyomo_expr(self, expr, max_degree=None): - if max_degree is None or max_degree >= 2: - repn = generate_standard_repn(expr, quadratic=True) + +_unary_map = { + 'exp': _handle_exp, + 'log': _handle_log, + 'sin': _handle_sin, + 'cos': _handle_cos, + 'sqrt': _handle_sqrt, + 'abs': _handle_abs, + 'tan': _handle_tan, +} + + +def _handle_unary(node, data, opt): + if node.getname() in _unary_map: + return _unary_map[node.getname()](node, data, opt) + else: + raise NotImplementedError(f'unable to handle unary expression: {str(node)}') + + +def _handle_equality(node, data, opt): + return data[0] == data[1] + + +def _handle_ranged(node, data, opt): + return data[0] <= (data[1] <= data[2]) + + +def _handle_inequality(node, data, opt): + return data[0] <= data[1] + + +def _handle_named_expression(node, data, opt): + return data[0] + + +_operator_map = { + NegationExpression: _handle_negation, + PowExpression: _handle_pow, + ProductExpression: _handle_product, + MonomialTermExpression: _handle_product, + DivisionExpression: _handle_division, + SumExpression: _handle_sum, + LinearExpression: _handle_sum, + UnaryFunctionExpression: _handle_unary, + NPV_NegationExpression: _handle_negation, + NPV_PowExpression: _handle_pow, + NPV_ProductExpression: _handle_product, + NPV_DivisionExpression: _handle_division, + NPV_SumExpression: _handle_sum, + NPV_UnaryFunctionExpression: _handle_unary, + EqualityExpression: _handle_equality, + RangedExpression: _handle_ranged, + InequalityExpression: _handle_inequality, + ScalarExpression: _handle_named_expression, + ExpressionData: _handle_named_expression, + VarData: _handle_var, + ScalarVar: _handle_var, + ParamData: _handle_param, + ScalarParam: _handle_param, + float: _handle_float, + int: _handle_float, +} + + +class _PyomoToScipVisitor(StreamBasedExpressionVisitor): + def __init__(self, solver, **kwds): + super().__init__(**kwds) + self.solver = solver + + def exitNode(self, node, data): + nt = type(node) + if nt in _operator_map: + return _operator_map[nt](node, data, self.solver) + elif nt in native_numeric_types: + _operator_map[nt] = _handle_float + return _handle_float(node, data, self.solver) else: - repn = generate_standard_repn(expr, quadratic=False) + raise NotImplementedError(f'unrecognized expression type: {nt}') + + +logger = logging.getLogger("pyomo.solvers") + + +class ScipDirectSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._pyomo_model = pyomo_model + # make sure the scip model does not get freed until the solution loader is garbage collected + self._opt = opt + + def get_number_of_solutions(self) -> int: + return self._solver_model.getNSols() + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> None: + for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + v.value = val + + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + if solution_id is None: + solution_id = 0 + sol = self._solver_model.getSols()[solution_id] + res = ComponentMap() + for v in vars_to_load: + sv = self._var_map[id(v)] + res[v] = sol[sv] + return res + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + return NotImplemented + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None + ) -> Dict[ConstraintData, float]: + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + +class SCIPDirect(SolverBase): - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + _available = None + _tc_map = None + _minimum_version = (5, 5, 0) # this is probably conservative - return scip_expr, referenced_vars + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # var id to var + self._params = {} # param id to param + self._pyomo_var_to_solver_var_map = {} # var id to scip var + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_sos_to_solver_sos_map = {} + self._expr_visitor = _PyomoToScipVisitor(self) + self._objective = None # pyomo objective + self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_con = None # a scip constraint (obj_var >= obj_expr) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._params = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._objective = None + self._obj_var = None + self._obj_con = None + + def available(self) -> Availability: + if self._available is not None: + return self._available + + if not scip_available: + SCIPDirect._available = Availability.NotFound + elif self.version() < self._minimum_version: + SCIPDirect._available = Availability.BadVersion + else: + SCIPDirect._available = Availability.FullLicense + + return self._available + + def version(self) -> Tuple: + return tuple(int(i) for i in scip.__version__) + + def solve(self, model: BlockData, **kwargs) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + orig_config = self.config + if not self.available(): + raise ApplicationError( + f'{self.name} is not available: {self.available()}' + ) + try: + config = self.config(value=kwds, preserve_implicit=True) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + + StaleFlagManager.mark_all_as_stale() + + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + ostreams = [io.StringIO()] + config.tee + + scip_model, solution_loader, has_obj = self._create_solver_model(model) + + scip_model.hideOutput(quiet=False) + if config.threads is not None: + scip_model.setParam('lp/threads', config.threads) + if config.time_limit is not None: + scip_model.setParam('limits/time', config.time_limit) + if config.rel_gap is not None: + scip_model.setParam('limits/gap', config.rel_gap) + if config.abs_gap is not None: + scip_model.setParam('limits/absgap', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in config.solver_options.items(): + scip_model.setParam(key, option) + + timer.start('optimize') + with capture_output(TeeStream(*ostreams), capture_fd=False): + scip_model.optimize() + timer.stop('optimize') + + results = self._postsolve(scip_model, solution_loader, has_obj) + except InfeasibleConstraintException: + results = self._get_infeasible_results() + finally: + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) + + results.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + results.timing_info.start_timestamp = start_timestamp + results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.timer = timer + return results + + def _get_tc_map(self): + if SCIPDirect._tc_map is None: + tc = TerminationCondition + SCIPDirect._tc_map = { + "unknown": tc.unknown, + "userinterrupt": tc.interrupted, + "nodelimit": tc.iterationLimit, + "totalnodelimit": tc.iterationLimit, + "stallnodelimit": tc.iterationLimit, + "timelimit": tc.maxTimeLimit, + "memlimit": tc.unknown, + "gaplimit": tc.convergenceCriteriaSatisfied, # TODO: check this + "primallimit": tc.objectiveLimit, + "duallimit": tc.objectiveLimit, + "sollimit": tc.unknown, + "bestsollimit": tc.unknown, + "restartlimit": tc.unknown, + "optimal": tc.convergenceCriteriaSatisfied, + "infeasible": tc.provenInfeasible, + "unbounded": tc.unbounded, + "inforunbd": tc.infeasibleOrUnbounded, + "terminate": tc.unknown, + } + return SCIPDirect._tc_map + + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.scip_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res def _scip_lb_ub_from_var(self, var): if var.is_fixed(): val = var.value return val, val - if var.has_lb(): - lb = value(var.lb) - else: + + lb, ub = var.bounds() + + if lb is None: lb = -self._solver_model.infinity() - if var.has_ub(): - ub = value(var.ub) - else: + if ub is None: ub = self._solver_model.infinity() return lb, ub def _add_var(self, var): - varname = self._symbol_map.getSymbol(var, self._labeler) vtype = self._scip_vtype_from_var(var) lb, ub = self._scip_lb_ub_from_var(var) - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._pyomo_var_to_solver_var_map[var] = scip_var.name - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = 0 - - def close(self): + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + + self._vars[id(var)] = var + self._pyomo_var_to_solver_var_map[id(var)] = scip_var + return scip_var + + def _add_param(self, p): + vtype = "C" + lb = ub = p.value + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + self._params[id(p)] = p + self._pyomo_param_to_solver_param_map[id(p)] = scip_var + return scip_var + + def __del__(self): """Frees SCIP resources used by this solver instance.""" - if self._solver_model is not None: self._solver_model.freeProb() self._solver_model = None - def __exit__(self, t, v, traceback): - super().__exit__(t, v, traceback) - self.close() - - def _set_instance(self, model, kwds={}): - DirectOrPersistentSolver._set_instance(self, model, kwds) - self.available() - try: - self._solver_model = self._scip.Model() - except Exception: - e = sys.exc_info()[1] - msg = ( - "Unable to create SCIP model. " - f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" + def _add_constraints(self, cons: List[ConstraintData]): + for con in cons: + self._add_constraint(con) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for on in cons: + self._add_sos_constraint(con) + + def _create_solver_model(self, model): + timer = self.config.timer + timer.start('create scip model') + self._clear() + self._solver_model = scip.Model() + timer.start('collect constraints') + cons = list( + model.component_data_objects( + Constraint, descend_into=True, active=True ) - raise Exception(msg) - - self._add_block(model) - - for var, n_ref in self._referenced_variables.items(): - if n_ref != 0: - if var.fixed: - if not self._output_fixed_variable_bounds: - raise ValueError( - f"Encountered a fixed variable {var.name} inside " - "an active objective or constraint " - f"expression on model {self._pyomo_model.name}, which is usually " - "indicative of a preprocessing error. Use " - "the IO-option 'output_fixed_variable_bounds=True' " - "to suppress this error and fix the variable " - "by overwriting its bounds in the SCIP instance." - ) - - def _add_block(self, block): - DirectOrPersistentSolver._add_block(self, block) - - def _add_constraint(self, con): - if not con.active: - return None - - if is_fixed(con.body) and self._skip_trivial_constraints: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) - - if con._linear_canonical_form: - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( - con.canonical_form(), self._max_constraint_degree - ) - else: - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - con.body, self._max_constraint_degree - ) - - if con.has_lb(): - if not is_fixed(con.lower): - raise ValueError(f"Lower bound of constraint {con} is not constant.") - con_lower = value(con.lower) - if type(con_lower) != float and type(con_lower) != int: - logger.warning( - f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as SCIP fails otherwise." - ) - con_lower = float(con_lower) - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f"Upper bound of constraint {con} is not constant.") - con_upper = value(con.upper) - - if con.equality: - scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) - elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - rhs = con_upper - if hasattr(con.body, "constant"): - con_constant = value(con.body.constant) - if not isinstance(con_constant, (float, int)): - con_constant = float(con_constant) - rhs -= con_constant - self._solver_model.chgRhs(scip_cons, rhs) - elif con.has_lb(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - elif con.has_ub(): - scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) - else: - raise ValueError( - f"Constraint does not have a lower or an upper bound: {con} \n" + ) + timer.stop('collect constraints') + timer.start('translate constraints') + self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') + sos = list( + model.component_data_objects( + SOSConstraint, descend_into=True, active=True ) + ) + self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') + obj = get_objective(model) + timer.stop('get objective') + timer.start('translate objective') + self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = ScipDirectSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + timer.stop('create scip model') + return self._solver_model, solution_loader, has_obj - for var in referenced_vars: - self._referenced_variables[var] += 1 - self._vars_referenced_by_con[con] = referenced_vars - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + def _add_constraint(self, con): + scip_expr = self._expr_visitor.walk_expression(con.expr) + scip_con = self._solver_model.addCons(scip_expr) + self._pyomo_con_to_solver_con_map[con] = scip_con def _add_sos_constraint(self, con): - if not con.active: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) level = con.level if level not in [1, 2]: - raise ValueError(f"Solver does not support SOS level {level} constraints") + raise ValueError(f"{self.name} does not support SOS level {level} constraints") scip_vars = [] weights = [] - self._vars_referenced_by_con[con] = ComponentSet() - - if hasattr(con, "get_items"): - # aml sos constraint - sos_items = list(con.get_items()) - else: - # kernel sos constraint - sos_items = list(con.items()) - - for v, w in sos_items: - self._vars_referenced_by_con[con].add(v) - scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) - self._referenced_variables[v] += 1 + for v, w in con.get_items(): + vid = id(v) + if vid not in self._pyomo_var_to_solver_var_map: + self._add_var(v) + scip_vars.append(self._pyomo_var_to_solver_var_map[vid]) weights.append(w) if level == 1: scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) else: scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): """ @@ -421,342 +607,104 @@ def _scip_vtype_from_var(self, var): return vtype def _set_objective(self, obj): + if self._obj_var is None: + self._obj_var = self._solver_model.addVar( + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C" + ) + if self._objective is not None: - for var in self._vars_referenced_by_obj: - self._referenced_variables[var] -= 1 - self._vars_referenced_by_obj = ComponentSet() - self._objective = None + self._solver_model.delCons(self._obj_con) - if obj.active is False: - raise ValueError("Cannot add inactive objective to solver.") + if obj is None: + scip_expr = 0 + else: + scip_expr = self._expr_visitor.walk_expression(obj.expr) if obj.sense == minimize: sense = "minimize" + self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) elif obj.sense == maximize: sense = "maximize" + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: raise ValueError(f"Objective sense is not recognized: {obj.sense}") - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - obj.expr, self._max_obj_degree - ) - - for var in referenced_vars: - self._referenced_variables[var] += 1 - - self._solver_model.setObjective(scip_expr, sense=sense) + self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj - self._vars_referenced_by_obj = referenced_vars - - def _get_solver_solution_status(self, scip, soln): - """ """ - # Get the status of the SCIP Model currently - status = scip.getStatus() - - # Go through each potential case and update appropriately - if scip.getStage() == 1: # SCIP Model is created but not yet optimized - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Model is loaded, but no solution information is available." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.unknown - elif status == "optimal": # optimal - self.results.solver.status = SolverStatus.ok - self.results.solver.termination_message = ( - "Model was solved to optimality (subject to tolerances), " - "and an optimal solution is available." - ) - self.results.solver.termination_condition = TerminationCondition.optimal - soln.status = SolutionStatus.optimal - elif status == "infeasible": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be infeasible" - ) - self.results.solver.termination_condition = TerminationCondition.infeasible - soln.status = SolutionStatus.infeasible - elif status == "inforunbd": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Problem proven to be infeasible or unbounded." - ) - self.results.solver.termination_condition = ( - TerminationCondition.infeasibleOrUnbounded - ) - soln.status = SolutionStatus.unsure - elif status == "unbounded": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be unbounded." - ) - self.results.solver.termination_condition = TerminationCondition.unbounded - soln.status = SolutionStatus.unbounded - elif status == "gaplimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the gap dropped below " - "the value specified in the " - "limits/gap parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "stallnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the stalling node limit " - "exceeded the value specified in the " - "limits/stallnodes parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "restartlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the total number of restarts " - "exceeded the value specified in the " - "limits/restarts parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "nodelimit" or status == "totalnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of " - "branch-and-cut nodes explored exceeded the limits specified " - "in the limits/nodes or limits/totalnodes parameter" - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxEvaluations - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "timelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the time expended exceeded " - "the value specified in the limits/time parameter." - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxTimeLimit - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "sollimit" or status == "bestsollimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of solutions found " - "reached the value specified in the limits/solutions or" - "limits/bestsol parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "memlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the memory used exceeded " - "the value specified in the limits/memory parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "userinterrupt": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization was terminated by the user." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - else: - self.results.solver.status = SolverStatus.error - self.results.solver.termination_message = ( - f"Unhandled SCIP status ({str(status)})" - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - return soln - - def _postsolve(self): - # Constraint duals and variable - # reduced-costs were removed as in SCIP they contain - # too many caveats. Slacks were removed as later - # planned interfaces do not intend to support. - # Scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - for suffix in self._suffixes: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model - status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() - - soln = self._get_solver_solution_status(scip, soln) - self.results.problem.name = scip.getProbName() - - self.results.problem.upper_bound = None - self.results.problem.lower_bound = None - if scip.getNSols() > 0: - scip_has_sol = True - else: - scip_has_sol = False - if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): - pass + def _postsolve( + self, + scip_model, + solution_loader: ScipDirectSolutionLoader, + has_obj + ): + + results = Results() + results.solution_loader = solution_loader + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) + + if solution_loader.get_number_of_solutions() > 0: + if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible else: - if n_bin_vars + n_int_vars == 0: - self.results.problem.upper_bound = scip.getObjVal() - self.results.problem.lower_bound = scip.getObjVal() - elif scip.getObjectiveSense() == "minimize": # minimizing - if scip_has_sol: - self.results.problem.upper_bound = scip.getObjVal() - else: - self.results.problem.upper_bound = scip.infinity() - self.results.problem.lower_bound = scip.getDualbound() - else: # maximizing - self.results.problem.upper_bound = scip.getDualbound() - if scip_has_sol: - self.results.problem.lower_bound = scip.getObjVal() + results.solution_status = SolutionStatus.noSolution + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and self.config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if scip_model.getObjVal() < scip_model.infinity(): + results.incumbent_objective = scip_model.getObjVal() else: - self.results.problem.lower_bound = -scip.infinity() - + results.incumbent_objective = None + except: + results.incumbent_objective = None try: - soln.gap = ( - self.results.problem.upper_bound - self.results.problem.lower_bound - ) - except TypeError: - soln.gap = None - - self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - self.results.problem.number_of_variables = scip.getNVars(transformed=False) - self.results.problem.number_of_binary_variables = n_bin_vars - self.results.problem.number_of_integer_variables = n_int_vars - self.results.problem.number_of_continuous_variables = n_con_vars - self.results.problem.number_of_objectives = 1 - self.results.problem.number_of_solutions = scip.getNSols() - - # if a solve was stopped by a limit, we still need to check to - # see if there is a solution available - this may not always - # be the case, both in LP and MIP contexts. - if self._save_results: - """ - This code in this if statement is only needed for backwards compatibility. It is more efficient to set - _save_results to False and use load_vars, load_duals, etc. - """ - - if scip.getNSols() > 0: - soln_variables = soln.variable - - scip_vars = scip.getVars() - scip_var_names = [scip_var.name for scip_var in scip_vars] - var_names = set(self._solver_var_to_pyomo_var_map.keys()) - assert set(scip_var_names) == var_names - var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] - - for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name] = {"Value": val} - - elif self._load_solutions: - if scip.getNSols() > 0: - self.load_vars() - - self.results.solution.insert(soln) - - # finally, clean any temporary files registered with the temp file - # manager, created populated *directly* by this plugin. - TempfileManager.pop(remove=not self._keepfiles) - - return DirectOrPersistentSolver._postsolve(self) - - def warm_start_capable(self): - return True - - def _warm_start(self): - partial_sol = False - for pyomo_var in self._pyomo_var_to_solver_var_expr_map: - if pyomo_var.value is None: - partial_sol = True - break - if partial_sol: - scip_sol = self._solver_model.createPartialSol() - else: - scip_sol = self._solver_model.createSol() - for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): - if pyomo_var.value is not None: - scip_sol[scip_var] = value(pyomo_var) - if partial_sol: - self._solver_model.addSol(scip_sol) + results.objective_bound = scip_model.getDualbound() + if results.objective_bound <= -scip_model.infinity(): + results.objective_bound = -math.inf + if results.objective_bound >= scip_model.infinity(): + results.objective_bound = math.inf + except: + if self._objective.sense == minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf else: - feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) - if feasible: - self._solver_model.addSol(scip_sol) - else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - - def _load_vars(self, vars_to_load=None): - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - var.set_value(val, skip_validation=True) - - def _load_rc(self, vars_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support reduced cost loading." - ) + results.incumbent_objective = None + results.objective_bound = None - def _load_duals(self, cons_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support dual solution loading" - ) - - def _load_slacks(self, cons_to_load=None): - raise NotImplementedError("SCIP via Pyomo does not support slack loading") - - def load_duals(self, cons_to_load=None): - """ - Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_duals(cons_to_load) - - def load_rc(self, vars_to_load): - """ - Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. - - Parameters - ---------- - vars_to_load: list of Var - """ - self._load_rc(vars_to_load) - - def load_slacks(self, cons_to_load=None): - """ - Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent - model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_slacks(cons_to_load) + self.config.timer.start('load solution') + if self.config.load_solutions: + if solution_loader.get_number_of_solutions() > 0: + solution_loader.load_solution() + else: + raise NoFeasibleSolutionError() + self.config.timer.stop('load solution') + + results.iteration_count = scip_model.getNNodes() + results.solver_config = self.config + results.solver_name = self.name + results.solver_version = self.version() + + return results + + def _mipstart(self): + # TODO: it is also possible to specify continuous variables, but + # I think we should have a differnt option for that + sol = self._solver_model.createPartialSol() + for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[vid] + if pyomo_var.is_integer(): + sol[scip_var] = pyomo_var.value + self._solver_model.addSol(sol) From 96017680cd9329626ae18bb2bdf7993f97940f74 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:47:44 -0600 Subject: [PATCH 51/97] porting scip interface --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 8 ++++---- pyomo/solvers/plugins/solvers/__init__.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index b8d4d14a6c1..5ea3391eecd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -69,7 +69,7 @@ logger = logging.getLogger(__name__) -scip, scip_available = attempt_import('pyscipyopt') +scip, scip_available = attempt_import('pyscipopt') class ScipConfig(BranchAndBoundConfig): @@ -360,9 +360,9 @@ def available(self) -> Availability: return self._available def version(self) -> Tuple: - return tuple(int(i) for i in scip.__version__) + return tuple(int(i) for i in scip.__version__.split('.')) - def solve(self, model: BlockData, **kwargs) -> Results: + def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): @@ -470,7 +470,7 @@ def _scip_lb_ub_from_var(self, var): val = var.value return val, val - lb, ub = var.bounds() + lb, ub = var.bounds if lb is None: lb = -self._solver_model.infinity() diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 55baaab9de8..cf10af15186 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -31,8 +31,6 @@ mosek_persistent, xpress_direct, xpress_persistent, - scip_direct, - scip_persistent, SAS, KNITROAMPL, ) From 9d2f22ab5ba29157c7765c82b0fd0e67d34a8e22 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:55:03 -0600 Subject: [PATCH 52/97] porting scip interface --- pyomo/contrib/solver/plugins.py | 6 ++++++ pyomo/contrib/solver/tests/solvers/test_solvers.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index fed739232ad..a4d6f5f9004 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,6 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs +from .solvers.scip.scip_direct import SCIPDirect def load(): @@ -39,3 +40,8 @@ def load(): SolverFactory.register( name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) + SolverFactory.register( + name='scip_direct', + legacy_name='scip_direct_v2', + doc='Direct interface pyscipopt', + ) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 189b0373780..f4988ca5c8b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect from pyomo.contrib.solver.common.util import ( NoSolutionError, NoFeasibleSolutionError, @@ -60,23 +61,30 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), + ('scip_direct', SCIPDirect), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), + ('scip_direct', SCIPDirect), +] +nlp_solvers = [ + ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] -nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('scip_direct', SCIPDirect), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From b4837295c57ee4005643a19f783f25dff3530be9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 00:49:53 -0600 Subject: [PATCH 53/97] bugs and tests --- .../solver/solvers/scip/scip_direct.py | 25 ++- .../solver/tests/solvers/test_solvers.py | 180 ++++++++++-------- 2 files changed, 120 insertions(+), 85 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 5ea3391eecd..1ff470223bd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -12,6 +12,7 @@ import datetime import io import logging +import math from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap @@ -40,6 +41,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager @@ -50,6 +52,7 @@ from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, + NoSolutionError, ) from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader @@ -109,6 +112,8 @@ def _handle_var(node, data, opt): def _handle_param(node, data, opt): + if not opt.is_persistent(): + return node.value if not node.mutable: return node.value if id(node) not in opt._pyomo_param_to_solver_param_map: @@ -231,6 +236,7 @@ def _handle_named_expression(node, data, opt): ScalarParam: _handle_param, float: _handle_float, int: _handle_float, + AutoLinkedBinaryVar: _handle_var, } @@ -287,6 +293,8 @@ def load_vars( def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if self.get_number_of_solutions() == 0: + raise NoSolutionError() if vars_to_load is None: vars_to_load = list(self._vars.values()) if solution_id is None: @@ -403,7 +411,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') @@ -619,17 +627,20 @@ def _set_objective(self, obj): if obj is None: scip_expr = 0 + sense = "minimize" else: scip_expr = self._expr_visitor.walk_expression(obj.expr) + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError(f"Objective sense is not recognized: {obj.sense}") - if obj.sense == minimize: - sense = "minimize" + if sense == "minimize": self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) - elif obj.sense == maximize: - sense = "maximize" - self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: - raise ValueError(f"Objective sense is not recognized: {obj.sense}") + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index f4988ca5c8b..1b6f122219c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -88,6 +88,13 @@ ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} +dual_solvers = [ + ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), + ('highs', Highs), +] def _load_tests(solver_list): @@ -114,7 +121,7 @@ def test_all_solvers_list(): class TestDualSignConvention(unittest.TestCase): - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -166,7 +173,7 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -228,7 +235,7 @@ def test_inequality( self.assertAlmostEqual(duals[m.c1], 0.5) self.assertAlmostEqual(duals[m.c2], 0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -283,7 +290,7 @@ def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -335,7 +342,7 @@ def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -389,7 +396,7 @@ def test_equality_max( self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -451,7 +458,7 @@ def test_inequality_max( self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -508,7 +515,7 @@ def test_bounds_max( rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -664,16 +671,18 @@ def test_range_constraint( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, -1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) m.obj.sense = pyo.maximize res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, 1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -702,7 +711,7 @@ def test_reduced_costs( self.assertAlmostEqual(rc[m.x], -3) self.assertAlmostEqual(rc[m.y], -4) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -768,9 +777,10 @@ def test_param_changes( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_immutable_param( @@ -815,9 +825,10 @@ def test_immutable_param( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -831,6 +842,8 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo check_duals = False else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -922,6 +935,8 @@ def test_no_objective( opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -983,9 +998,10 @@ def test_add_remove_cons( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) m.c3 = pyo.Constraint(expr=m.y >= a3 * m.x + b3) res = opt.solve(m) @@ -994,10 +1010,11 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b3 - b1) / (a1 - a3) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) - self.assertAlmostEqual(duals[m.c2], 0) - self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) + self.assertAlmostEqual(duals[m.c2], 0) + self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) del m.c3 res = opt.solve(m) @@ -1006,9 +1023,10 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_results_infeasible( @@ -1057,14 +1075,15 @@ def test_results_infeasible( NoSolutionError, '.*does not currently have a valid solution.*' ): res.solution_loader.load_vars() - with self.assertRaisesRegex( - NoDualsError, '.*does not currently have valid duals.*' - ): - res.solution_loader.get_duals() - with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' - ): - res.solution_loader.get_reduced_costs() + if (name, opt_class) in dual_solvers: + with self.assertRaisesRegex( + NoDualsError, '.*does not currently have valid duals.*' + ): + res.solution_loader.get_duals() + with self.assertRaisesRegex( + NoReducedCostsError, '.*does not currently have valid reduced costs.*' + ): + res.solution_loader.get_reduced_costs() @parameterized.expand(input=_load_tests(all_solvers)) def test_trivial_constraints( @@ -1118,7 +1137,7 @@ def test_trivial_constraints( self.assertIn(res.termination_condition, acceptable_termination_conditions) self.assertIsNone(res.incumbent_objective) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -1167,13 +1186,13 @@ def test_mutable_quadratic_coefficient( m.c = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.41024548525899274, 4) - self.assertAlmostEqual(m.y.value, 0.34781038127030117, 4) + self.assertAlmostEqual(m.x.value, 0.41024548525899274, 3) + self.assertAlmostEqual(m.y.value, 0.34781038127030117, 3) m.a.value = 2 m.b.value = -0.5 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.10256137418973625, 4) - self.assertAlmostEqual(m.y.value, 0.0869525991355825, 4) + self.assertAlmostEqual(m.x.value, 0.10256137418973625, 3) + self.assertAlmostEqual(m.y.value, 0.0869525991355825, 3) @parameterized.expand(input=_load_tests(qcp_solvers)) def test_mutable_quadratic_objective_qcp( @@ -1198,14 +1217,14 @@ def test_mutable_quadratic_objective_qcp( m.ccon = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.2719178742733325, 4) - self.assertAlmostEqual(m.y.value, 0.5301035741688002, 4) + self.assertAlmostEqual(m.x.value, 0.2719178742733325, 3) + self.assertAlmostEqual(m.y.value, 0.5301035741688002, 3) m.c.value = 3.5 m.d.value = -1 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6962249634573562, 4) - self.assertAlmostEqual(m.y.value, 0.09227926676152151, 4) + self.assertAlmostEqual(m.x.value, 0.6962249634573562, 3) + self.assertAlmostEqual(m.y.value, 0.09227926676152151, 3) @parameterized.expand(input=_load_tests(qp_solvers)) def test_mutable_quadratic_objective_qp( @@ -1412,7 +1431,7 @@ def test_fixed_vars_4( else: opt.config.writer_config.linear_presolve = False m = pyo.ConcreteModel() - m.x = pyo.Var() + m.x = pyo.Var(bounds=(0, None)) m.y = pyo.Var() m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.x == 2 / m.y) @@ -1421,8 +1440,8 @@ def test_fixed_vars_4( self.assertAlmostEqual(m.x.value, 2) m.y.unfix() res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2**0.5) - self.assertAlmostEqual(m.y.value, 2**0.5) + self.assertAlmostEqual(m.x.value, 2**0.5, 3) + self.assertAlmostEqual(m.y.value, 2**0.5, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_mutable_param_with_range( @@ -1506,9 +1525,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound <= m.y.value + 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) else: self.assertAlmostEqual(m.x.value, (c2 - c1) / (a1 - a2), 6) self.assertAlmostEqual(m.y.value, a1 * (c2 - c1) / (a1 - a2) + c1, 6) @@ -1517,9 +1537,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound >= m.y.value - 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) @parameterized.expand(input=_load_tests(all_solvers)) def test_add_and_remove_vars( @@ -1590,8 +1611,8 @@ def test_exp(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, -0.42630274815985264) - self.assertAlmostEqual(m.y.value, 0.6529186341994245) + self.assertAlmostEqual(m.x.value, -0.42630274815985264, 4) + self.assertAlmostEqual(m.y.value, 0.6529186341994245, 4) @parameterized.expand(input=_load_tests(nlp_solvers)) def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -1609,8 +1630,8 @@ def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y <= pyo.log(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6529186341994245) - self.assertAlmostEqual(m.y.value, -0.42630274815985264) + self.assertAlmostEqual(m.x.value, 0.6529186341994245, 3) + self.assertAlmostEqual(m.y.value, -0.42630274815985264, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_with_numpy( @@ -1720,24 +1741,25 @@ def test_solution_loader( self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) - reduced_costs = res.solution_loader.get_reduced_costs() - self.assertIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.x], 1) - self.assertAlmostEqual(reduced_costs[m.y], 0) - reduced_costs = res.solution_loader.get_reduced_costs([m.y]) - self.assertNotIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.y], 0) - duals = res.solution_loader.get_duals() - self.assertIn(m.c1, duals) - self.assertIn(m.c2, duals) - self.assertAlmostEqual(duals[m.c1], 1) - self.assertAlmostEqual(duals[m.c2], 0) - duals = res.solution_loader.get_duals([m.c1]) - self.assertNotIn(m.c2, duals) - self.assertIn(m.c1, duals) - self.assertAlmostEqual(duals[m.c1], 1) + if (name, opt_class) in dual_solvers: + reduced_costs = res.solution_loader.get_reduced_costs() + self.assertIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.x], 1) + self.assertAlmostEqual(reduced_costs[m.y], 0) + reduced_costs = res.solution_loader.get_reduced_costs([m.y]) + self.assertNotIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.y], 0) + duals = res.solution_loader.get_duals() + self.assertIn(m.c1, duals) + self.assertIn(m.c2, duals) + self.assertAlmostEqual(duals[m.c1], 1) + self.assertAlmostEqual(duals[m.c2], 0) + duals = res.solution_loader.get_duals([m.c1]) + self.assertNotIn(m.c2, duals) + self.assertIn(m.c1, duals) + self.assertAlmostEqual(duals[m.c1], 1) @parameterized.expand(input=_load_tests(all_solvers)) def test_time_limit( @@ -2219,6 +2241,8 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() From 37a31a7fc7b7fe8cb14179e4dc18df4dbdd8bc87 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 01:50:18 -0600 Subject: [PATCH 54/97] scip direct --- pyomo/contrib/solver/plugins.py | 2 +- .../contrib/solver/solvers/scip/scip_direct.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index a4d6f5f9004..81e3677b19e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - ) + )(SCIPDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1ff470223bd..3926dd25a1c 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -16,6 +16,7 @@ from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap +from pyomo.core.expr.numvalue import is_constant from pyomo.common.numeric_types import native_numeric_types from pyomo.common.errors import InfeasibleConstraintException, ApplicationError from pyomo.common.timing import HierarchicalTimer @@ -67,6 +68,8 @@ ) from pyomo.common.config import ConfigValue from pyomo.common.tee import capture_output, TeeStream +from pyomo.core.base.units_container import _PyomoUnit +from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr logger = logging.getLogger(__name__) @@ -132,7 +135,15 @@ def _handle_negation(node, data, opt): def _handle_pow(node, data, opt): - return data[0] ** data[1] + x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) + if is_constant(node.args[1]): + return x**y + else: + xlb, xub = compute_bounds_on_expr(node.args[0]) + if xlb > 0: + return scip.exp(y*scip.log(x)) + else: + return x**y # scip will probably raise an error here def _handle_product(node, data, opt): @@ -210,6 +221,10 @@ def _handle_named_expression(node, data, opt): return data[0] +def _handle_unit(node, data, opt): + return node.value + + _operator_map = { NegationExpression: _handle_negation, PowExpression: _handle_pow, @@ -237,6 +252,7 @@ def _handle_named_expression(node, data, opt): float: _handle_float, int: _handle_float, AutoLinkedBinaryVar: _handle_var, + _PyomoUnit: _handle_unit, } From a43a38bacc66aafa6af6c7305448e3f55a3fe263 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 09:30:56 -0600 Subject: [PATCH 55/97] forgot to inherit from PersistentSolverBase --- .../solvers/gurobi/gurobi_persistent.py | 3 +- .../solver/tests/solvers/test_solvers.py | 139 ++++++++---------- 2 files changed, 64 insertions(+), 78 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 847ec958bdd..8d16bb9082e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -28,6 +28,7 @@ from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -575,7 +576,7 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) -class GurobiPersistent(GurobiDirectQuadratic): +class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) def __init__(self, **kwds): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..6965147d167 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1221,55 +1221,50 @@ def test_mutable_quadratic_objective_qp( def test_fixed_vars( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): - for treat_fixed_vars_as_params in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=treat_fixed_vars_as_params) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - m = pyo.ConcreteModel() - m.x = pyo.Var() - m.x.fix(0) - m.y = pyo.Var() - a1 = 1 - a2 = -1 - b1 = 1 - b2 = 2 - m.obj = pyo.Objective(expr=m.y) - m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) - m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.unfix() - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) - self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - m.x.fix(0) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.value = 2 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2) - self.assertAlmostEqual(m.y.value, 3) - m.x.value = 0 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.x.fix(0) + m.y = pyo.Var() + a1 = 1 + a2 = -1 + b1 = 1 + b2 = 2 + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) + m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.unfix() + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) + self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) + m.x.fix(0) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.value = 2 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 2) + self.assertAlmostEqual(m.y.value, 3) + m.x.value = 0 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) @parameterized.expand(input=_load_tests(all_solvers)) def test_fixed_vars_2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1313,8 +1308,6 @@ def test_fixed_vars_3( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1337,8 +1330,6 @@ def test_fixed_vars_4( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1892,10 +1883,7 @@ def test_fixed_binaries( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 1) - if opt.is_persistent(): - opt: SolverBase = opt_class(treat_fixed_vars_as_params=False) - else: - opt = opt_class() + opt = opt_class() m.x.fix(0) res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 0) @@ -2049,33 +2037,30 @@ def test_bug_2(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) This test is for a bug where an objective containing a fixed variable does not get updated properly when the variable is unfixed. """ - for fixed_var_option in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=fixed_var_option) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - - m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-10, 10)) - m.y = pyo.Var() - m.obj = pyo.Objective(expr=3 * m.y - m.x) - m.c = pyo.Constraint(expr=m.y >= m.x) - - m.x.fix(1) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, 2, 5) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False - m.x.unfix() - m.x.setlb(-9) - m.x.setub(9) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, -18, 5) + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(-10, 10)) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=3 * m.y - m.x) + m.c = pyo.Constraint(expr=m.y >= m.x) + + m.x.fix(1) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, 2, 5) + + m.x.unfix() + m.x.setlb(-9) + m.x.setub(9) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, -18, 5) @parameterized.expand(input=_load_tests(nl_solvers)) def test_presolve_with_zero_coef( From e76baae7b130aa3d845e053c802a93ca5e76de26 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 12:25:39 -0600 Subject: [PATCH 56/97] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8d16bb9082e..8145777ffb9 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -595,6 +595,7 @@ def __init__(self, **kwds): self._observer = _GurobiObserver(self) self._change_detector = ModelChangeDetector(observers=[self._observer]) self._constraint_ndx = 0 + self._should_update_parameters = False @property def auto_updates(self): @@ -683,6 +684,8 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) + if self._should_update_parameters: + self._update_parameters([]) timer.stop('update') def _add_constraints(self, cons: List[ConstraintData]): @@ -915,6 +918,8 @@ def _update_variables(self, variables: List[VarData]): gurobipy_var.setAttr('lb', lb) gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) + if var.fixed: + self._should_update_parameters = True self._needs_updated = True def _update_parameters(self, params: List[ParamData]): @@ -950,6 +955,8 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) + self._should_update_parameters = False + def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() From 0b84dcc77e826022c705ae95d187dc29e0b24e1f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 04:57:27 -0600 Subject: [PATCH 57/97] more expression types for scip --- .../solver/solvers/scip/scip_direct.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 3926dd25a1c..d72ce47ef02 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -42,6 +42,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression @@ -126,6 +127,10 @@ def _handle_param(node, data, opt): return scip_param +def _handle_constant(node, data, opt): + return node.value + + def _handle_float(node, data, opt): return float(node) @@ -167,6 +172,10 @@ def _handle_log(node, data, opt): return scip.log(data[0]) +def _handle_log10(node, data, opt): + return scip.log(data[0]) / math.log(10) + + def _handle_sin(node, data, opt): return scip.sin(data[0]) @@ -187,6 +196,12 @@ def _handle_tan(node, data, opt): return scip.sin(data[0]) / scip.cos(data[0]) +def _handle_tanh(node, data, opt): + x = data[0] + _exp = scip.exp + return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) + + _unary_map = { 'exp': _handle_exp, 'log': _handle_log, @@ -195,6 +210,8 @@ def _handle_tan(node, data, opt): 'sqrt': _handle_sqrt, 'abs': _handle_abs, 'tan': _handle_tan, + 'log10': _handle_log10, + 'tanh': _handle_tanh, } @@ -253,6 +270,7 @@ def _handle_unit(node, data, opt): int: _handle_float, AutoLinkedBinaryVar: _handle_var, _PyomoUnit: _handle_unit, + NumericConstant: _handle_constant, } @@ -427,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=True): + with capture_output(TeeStream(*ostreams), capture_fd=False): scip_model.optimize() timer.stop('optimize') @@ -690,7 +708,7 @@ def _postsolve( if has_obj: try: - if scip_model.getObjVal() < scip_model.infinity(): + if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None From c2a01777d43d47d7ea072c5b93ddc020e8b787ee Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 09:13:12 -0600 Subject: [PATCH 58/97] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8145777ffb9..603e8e21800 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -432,7 +432,7 @@ def _add_variables(self, variables: List[VarData]): def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' ) if len(repn.linear_vars) > 0: From 3180462477261e7c1eaa63e349b35abf080c6f05 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 20 Aug 2025 09:08:52 -0600 Subject: [PATCH 59/97] capture_fd for scip --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index d72ce47ef02..8deca600b40 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -445,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') From 72912e0d9e5397a3403061aff3daf32a4e9b281c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 22 Aug 2025 08:49:40 -0600 Subject: [PATCH 60/97] working on persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 161 +++++++++++++++++- 1 file changed, 153 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 8deca600b40..344b1741552 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import datetime import io import logging @@ -49,7 +50,7 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -71,6 +72,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -354,7 +356,72 @@ def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) -class SCIPDirect(SolverBase): +class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__( + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) + self._valid = False + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: + self._assert_solution_still_valid() + return super().load_vars(vars_to_load, solution_id) + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_vars(vars_to_load, solution_id) + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: + self._assert_solution_still_valid() + return super().get_duals(cons_to_load) + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) + + + +class ScipDirect(SolverBase): _available = None _tc_map = None @@ -393,11 +460,11 @@ def available(self) -> Availability: return self._available if not scip_available: - SCIPDirect._available = Availability.NotFound + ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: - SCIPDirect._available = Availability.BadVersion + ScipDirect._available = Availability.BadVersion else: - SCIPDirect._available = Availability.FullLicense + ScipDirect._available = Availability.FullLicense return self._available @@ -465,9 +532,9 @@ def solve(self, model: BlockData, **kwds) -> Results: return results def _get_tc_map(self): - if SCIPDirect._tc_map is None: + if ScipDirect._tc_map is None: tc = TerminationCondition - SCIPDirect._tc_map = { + ScipDirect._tc_map = { "unknown": tc.unknown, "userinterrupt": tc.interrupted, "nodelimit": tc.iterationLimit, @@ -487,7 +554,7 @@ def _get_tc_map(self): "inforunbd": tc.infeasibleOrUnbounded, "terminate": tc.unknown, } - return SCIPDirect._tc_map + return ScipDirect._tc_map def _get_infeasible_results(self): res = Results() @@ -753,3 +820,81 @@ def _mipstart(self): if pyomo_var.is_integer(): sol[scip_var] = pyomo_var.value self._solver_model.addSol(sol) + + +class _SCIPObserver(Observer): + def __init__(self, opt: ScipPersistent) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class ScipPersistent(ScipDirect, PersistentSolverBase): + _minimum_version = (5, 5, 0) # this is probably conservative + + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._observer = _SCIPObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + + def _create_solver_model(self, model): + if model is self._pyomo_model: + self.update() + else: + self.set_instance(model=model) + + solution_loader = ScipPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + + has_obj = self._objective is not None: + return self._solver_model, solution_loader, has_obj \ No newline at end of file From cfa1e9108f6edd05d238056e6f071122a3164d39 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 1 Sep 2025 17:10:19 -0600 Subject: [PATCH 61/97] minor fixes --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 81e3677b19e..895b6387725 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import SCIPDirect +from .solvers.scip.scip_direct import ScipDirect def load(): @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - )(SCIPDirect) + )(ScipDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 344b1741552..1032affd597 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -896,5 +896,5 @@ def _create_solver_model(self, model): opt=self, ) - has_obj = self._objective is not None: + has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj \ No newline at end of file From 576a21768c7c8acbdfb63d863c69fe603017a6c3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 07:22:54 -0600 Subject: [PATCH 62/97] run black --- pyomo/contrib/solver/tests/solvers/test_solvers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 192f1536800..74d3b7ccdbc 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,10 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent, GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi_persistent import ( + GurobiPersistent, + GurobiDirectQuadratic, +) from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect from pyomo.contrib.solver.solvers.highs import Highs from pyomo.contrib.solver.solvers.ipopt import Ipopt From ce99fb2597611449874d78d02678d35fb21eaf47 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:11 -0600 Subject: [PATCH 63/97] observer improvements --- pyomo/contrib/observer/component_collector.py | 7 +- pyomo/contrib/observer/model_observer.py | 166 +++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 166 ++++++++++++------ .../tests/solvers/test_gurobi_persistent.py | 10 +- .../solver/tests/solvers/test_solvers.py | 10 +- 5 files changed, 229 insertions(+), 130 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d30bb128758..22e66aa6c80 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -33,6 +33,7 @@ from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.repn.util import ExitNodeDispatcher +from pyomo.common.numeric_types import native_numeric_types def handle_var(node, collector): @@ -79,8 +80,6 @@ def handle_skip(node, collector): collector_handlers[RangedExpression] = handle_skip collector_handlers[InequalityExpression] = handle_skip collector_handlers[EqualityExpression] = handle_skip -collector_handlers[int] = handle_skip -collector_handlers[float] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): @@ -92,6 +91,10 @@ def __init__(self, **kwds): super().__init__(**kwds) def exitNode(self, node, data): + if type(node) in native_numeric_types: + # we need this here to handle numpy + # (we can't put numpy in the dispatcher?) + return None return collector_handlers[node.__class__](node, self) def beforeChild(self, node, child, child_idx): diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b0a5b07fdfd..71c1a7c5460 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -16,16 +16,18 @@ from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData +from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.objective import ObjectiveData, Objective -from pyomo.core.base.block import BlockData +from pyomo.core.base.block import BlockData, Block from pyomo.core.base.component import ActiveComponent +from pyomo.core.base.suffix import Suffix from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types import gc +import warnings """ @@ -49,6 +51,9 @@ """ +_param_types = {ParamData, ScalarParam} + + @document_configdict() class AutoUpdateConfig(ConfigDict): """ @@ -492,7 +497,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): observers: Sequence[Observer] The objects to notify when changes are made to the model """ - self._known_active_ctypes = {Constraint, SOSConstraint, Objective} + self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} self._observers: List[Observer] = list(observers) self._active_constraints = {} # maps constraint to expression self._active_sos = {} @@ -520,12 +525,14 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], + # dict[var_id, None], # ) self._referenced_params = {} self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} + self._params_referenced_by_var = {} # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -536,11 +543,13 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._set_instance() def add_variables(self, variables: List[VarData]): + params_to_check = {} for v in variables: - if id(v) in self._referenced_variables: + vid = id(v) + if vid in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = ({}, {}, {}) - self._vars[id(v)] = ( + self._referenced_variables[vid] = ({}, {}, {}) + self._vars[vid] = ( v, v._lb, v._ub, @@ -548,6 +557,31 @@ def add_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) + ref_params = set() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + if named_exprs: + raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + if external_functions: + raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + params_to_check.update((id(p), p) for p in parameters) + if vid not in self._params_referenced_by_var: + self._params_referenced_by_var[vid] = [] + self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + ref_params.update(id(p) for p in parameters) + self._check_for_new_params(list(params_to_check.values())) + for v in variables: + if id(v) not in self._params_referenced_by_var: + continue + parameters = self._params_referenced_by_var[id(v)] + for p in parameters: + self._referenced_params[id(p)][3][id(v)] = None for obs in self._observers: obs.add_variables(variables) @@ -556,46 +590,46 @@ def add_parameters(self, params: List[ParamData]): pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = ({}, {}, {}) + self._referenced_params[pid] = ({}, {}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = [] + new_vars = {} for v in variables: if id(v) not in self._referenced_variables: - new_vars.append(v) - self.add_variables(new_vars) + new_vars[id(v)] = v + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = [] + vars_to_remove = {} for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if not ref_cons and not ref_sos and not ref_obj: - vars_to_remove.append(v) - self._remove_variables(vars_to_remove) + vars_to_remove[v_id] = v + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): - new_params = [] + new_params = {} for p in params: if id(p) not in self._referenced_params: - new_params.append(p) - self.add_parameters(new_params) + new_params[id(p)] = p + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = [] + params_to_remove = {} for p in params: p_id = id(p) - ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if not ref_cons and not ref_sos and not ref_obj: - params_to_remove.append(p) - self._remove_parameters(params_to_remove) + ref_cons, ref_sos, ref_obj, ref_vars = self._referenced_params[p_id] + if not ref_cons and not ref_sos and not ref_obj and not ref_vars: + params_to_remove[p_id] = p + self.remove_parameters(list(params_to_remove.values())) def add_constraints(self, cons: List[ConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') @@ -603,16 +637,16 @@ def add_constraints(self, cons: List[ConstraintData]): (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] parameters = self._params_referenced_by_con[con] @@ -624,8 +658,8 @@ def add_constraints(self, cons: List[ConstraintData]): obs.add_constraints(cons) def add_sos_constraints(self, cons: List[SOSConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') @@ -642,12 +676,12 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - vars_to_check.extend(variables) - params_to_check.extend(params) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] params = self._params_referenced_by_con[con] @@ -659,24 +693,24 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): obs.add_sos_constraints(cons) def add_objectives(self, objs: List[ObjectiveData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for obj in objs: obj_id = id(obj) variables = self._vars_referenced_by_obj[obj_id] @@ -692,8 +726,8 @@ def remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: obs.remove_objectives(objs) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) if obj_id not in self._objectives: @@ -704,15 +738,15 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) - params_to_check.extend(self._params_referenced_by_obj[obj_id]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(): @@ -723,9 +757,12 @@ def _check_for_unknown_active_components(self): for comp in self._model.component_data_objects( ctype, active=True, descend_into=True ): + if isinstance(comp, Suffix): + warnings.warn('ModelChangeDetector does not detect changes to suffixes') + continue raise NotImplementedError( - f'ModelChangeDetector does not know how to ' - 'handle components with ctype {ctype}' + 'ModelChangeDetector does not know how to ' + f'handle components with ctype {ctype}' ) def _set_instance(self): @@ -764,8 +801,8 @@ def _set_instance(self): def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_constraints: raise ValueError( @@ -775,21 +812,21 @@ def remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_sos: raise ValueError( @@ -799,23 +836,29 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) + params_to_check = {} for v in variables: v_id = id(v) if v_id not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) + if v_id in self._params_referenced_by_var: + for p in self._params_referenced_by_var[v_id]: + self._referenced_params[id(p)][3].pop(v_id) + params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: raise ValueError( @@ -823,6 +866,7 @@ def remove_variables(self, variables: List[VarData]): ) del self._referenced_variables[v_id] del self._vars[v_id] + self._check_to_remove_params(list(params_to_check.values())) def remove_parameters(self, params: List[ParamData]): for obs in self._observers: @@ -833,8 +877,8 @@ def remove_parameters(self, params: List[ParamData]): raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) - cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or obj_using: + cons_using, sos_using, obj_using, vars_using = self._referenced_params[p_id] + if cons_using or sos_using or obj_using or vars_using: raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 603e8e21800..74952af2b49 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData @@ -33,13 +34,14 @@ from .gurobi_direct_base import ( GurobiDirectBase, gurobipy, + GurobiConfig, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -539,7 +541,7 @@ def _set_objective(self, obj): class _GurobiObserver(Observer): - def __init__(self, opt: GurobiPersistentQuadratic) -> None: + def __init__(self, opt: GurobiPersistent) -> None: self.opt = opt def add_variables(self, variables: List[VarData]): @@ -554,8 +556,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -576,8 +581,31 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) +class GurobiPersistentConfig(GurobiConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + GurobiConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare( + 'auto_updates', AutoUpdateConfig() + ) + + class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) + CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) @@ -592,15 +620,11 @@ def __init__(self, **kwds): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object: Optional[Results] = None - self._observer = _GurobiObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._observer = None + self._change_detector = None self._constraint_ndx = 0 self._should_update_parameters = False - @property - def auto_updates(self): - return self._change_detector.config - def _clear(self): super()._clear() self._pyomo_model = None @@ -669,8 +693,10 @@ def set_instance(self, pyomo_model): self._clear() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) + self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector.set_instance(pyomo_model) + self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector.config = self.config.auto_updates timer.stop('set_instance') def update(self): @@ -804,59 +830,83 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._constraints_added_since_update.update(cons) self._needs_updated = True - def _set_objective(self, obj): - self._invalidate_last_results() - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - self._mutable_objective = None - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' \ + f'{str(obj)}' + ) else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._invalidate_last_results() + self._solver_model.setObjective(0, sense=gurobipy.GRB.MINIMIZE) + # see PR #2454 + self._solver_model.update() + self._objective = None + self._needs_updated = False + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + 'only supports single-objective problems; tried to add ' \ + f'an objective ({str(obj)}), but there is already an ' \ + f'active objective ({str(self._objective)})' ) - repn_constant = value(repn.constant) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - mutable_constant = _MutableConstant(repn.constant, None, None) + self._invalidate_last_results() - mlc_list = [] - for c, v in zip(repn.linear_coefs, repn.linear_vars): - if not is_constant(c): - mlc = _MutableLinearCoefficient( - c, - None, - None, - id(v), - self._pyomo_var_to_solver_var_map, - self._solver_model, - ) - mlc_list.append(mlc) + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') - mqc_list = [] - for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - if not is_constant(coef): - mqc = _MutableQuadraticCoefficient( - coef, id(x), id(y), self._pyomo_var_to_solver_var_map - ) - mqc_list.append(mqc) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) - self._mutable_objective = _MutableObjective( - self._solver_model, mutable_constant, mlc_list, mqc_list - ) + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) + mlc_list.append(mlc) - # hack - # see PR #2454 - if self._objective is not None: - self._solver_model.setObjective(0) - self._solver_model.update() + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._objective = obj @@ -1366,8 +1416,8 @@ def add_constraints(self, cons): def add_sos_constraints(self, cons): self._change_detector.add_sos_constraints(cons) - def set_objective(self, obj): - self._change_detector.set_objective(obj) + def set_objective(self, obj: ObjectiveData): + self._change_detector.add_objectives([obj]) def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 96cd1498956..24b53a19f2b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -499,11 +499,11 @@ def test_zero_time_limit(self): class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.auto_updates.check_for_new_or_removed_constraints = False - opt.auto_updates.update_parameters = False - opt.auto_updates.update_vars = False - opt.auto_updates.update_constraints = False - opt.auto_updates.update_named_expressions = False + opt.config.auto_updates.check_for_new_or_removed_constraints = False + opt.config.auto_updates.update_parameters = False + opt.config.auto_updates.update_vars = False + opt.config.auto_updates.update_constraints = False + opt.config.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 74d3b7ccdbc..e7ff00f7f41 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,7 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( GurobiPersistent, GurobiDirectQuadratic, ) @@ -1537,9 +1537,7 @@ def test_add_and_remove_vars( opt.config.auto_updates.update_vars = False opt.config.auto_updates.update_constraints = False opt.config.auto_updates.update_named_expressions = False - opt.config.auto_updates.check_for_new_or_removed_params = False opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.check_for_new_or_removed_vars = False opt.config.load_solutions = False res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) @@ -1864,7 +1862,11 @@ def test_objective_changes( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 3) if opt.is_persistent(): - opt.config.auto_updates.check_for_new_objective = False + # hack until we get everything ported to the observer + try: + opt.config.auto_updates.check_for_new_or_removed_objectives = False + except: + opt.config.auto_updates.check_for_new_objective = False m.e.expr = 4 res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 4) From 066e4fdf80f401194d63355495ba43f10da073e8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:58 -0600 Subject: [PATCH 64/97] run black --- pyomo/contrib/observer/model_observer.py | 44 ++++++++++++++----- .../solvers/gurobi/gurobi_persistent.py | 30 +++++++------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 71c1a7c5460..2da340aab4f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -532,7 +532,9 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} - self._params_referenced_by_var = {} # for when parameters show up in variable bounds + self._params_referenced_by_var = ( + {} + ) # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -565,15 +567,23 @@ def add_variables(self, variables: List[VarData]): collect_components_from_expr(bnd) ) if _vars: - raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) if named_exprs: - raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) if external_functions: - raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) params_to_check.update((id(p), p) for p in parameters) if vid not in self._params_referenced_by_var: self._params_referenced_by_var[vid] = [] - self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + self._params_referenced_by_var[vid].extend( + p for p in parameters if id(p) not in ref_params + ) ref_params.update(id(p) for p in parameters) self._check_for_new_params(list(params_to_check.values())) for v in variables: @@ -738,8 +748,12 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) + vars_to_check.update( + (id(v), v) for v in self._vars_referenced_by_obj[obj_id] + ) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_obj[obj_id] + ) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) @@ -758,7 +772,9 @@ def _check_for_unknown_active_components(self): ctype, active=True, descend_into=True ): if isinstance(comp, Suffix): - warnings.warn('ModelChangeDetector does not detect changes to suffixes') + warnings.warn( + 'ModelChangeDetector does not detect changes to suffixes' + ) continue raise NotImplementedError( 'ModelChangeDetector does not know how to ' @@ -813,7 +829,9 @@ def remove_constraints(self, cons: List[ConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) @@ -837,7 +855,9 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] @@ -857,7 +877,9 @@ def remove_variables(self, variables: List[VarData]): if v_id in self._params_referenced_by_var: for p in self._params_referenced_by_var[v_id]: self._referenced_params[id(p)][3].pop(v_id) - params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_var[v_id] + ) self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 74952af2b49..315d8c6dc4a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -41,7 +41,11 @@ _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -598,9 +602,7 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.auto_updates: bool = self.declare( - 'auto_updates', AutoUpdateConfig() - ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): @@ -695,7 +697,11 @@ def set_instance(self, pyomo_model): self._solver_model = gurobipy.Model(env=self.env()) self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) self._change_detector.config = self.config.auto_updates timer.stop('set_instance') @@ -834,7 +840,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): for obj in objs: if obj is not self._objective: raise RuntimeError( - 'tried to remove an objective that has not been added: ' \ + 'tried to remove an objective that has not been added: ' f'{str(obj)}' ) else: @@ -848,7 +854,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): def _add_objectives(self, objs: List[ObjectiveData]): if len(objs) > 1: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ + 'the persistent interface to gurobi currently ' f'only supports single-objective problems; got {len(objs)}: ' f'{[str(i) for i in objs]}' ) @@ -860,9 +866,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): if self._objective is not None: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ - 'only supports single-objective problems; tried to add ' \ - f'an objective ({str(obj)}), but there is already an ' \ + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' f'active objective ({str(self._objective)})' ) @@ -875,9 +881,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False - ) + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) From e6331dfa67f60224a075f0fb1a8c4bc57baf43df Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:28:15 -0600 Subject: [PATCH 65/97] persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 223 ++++++++++++++++-- 1 file changed, 204 insertions(+), 19 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1032affd597..cf0c71606f8 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -25,6 +25,7 @@ from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.sos import SOSConstraint, SOSConstraintData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( @@ -72,7 +73,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -420,7 +421,6 @@ def load_import_suffixes(self, solution_id=None): super().load_import_suffixes(solution_id) - class ScipDirect(SolverBase): _available = None @@ -822,7 +822,27 @@ def _mipstart(self): self._solver_model.addSol(sol) -class _SCIPObserver(Observer): +class ScipPersistentConfig(ScipConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + ScipConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) + + +class _ScipObserver(Observer): def __init__(self, opt: ScipPersistent) -> None: self.opt = opt @@ -838,8 +858,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -862,39 +885,201 @@ def update_parameters(self, params: List[ParamData]): class ScipPersistent(ScipDirect, PersistentSolverBase): _minimum_version = (5, 5, 0) # this is probably conservative - - CONFIG = ScipConfig() + CONFIG = ScipPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) self._pyomo_model = None - self._objective = None - self._observer = _SCIPObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) - - @property - def auto_updates(self): - return self._change_detector.config + self._observer = None + self._change_detector = None + self._last_results_object: Optional[Results] = None def _clear(self): super()._clear() self._pyomo_model = None self._objective = None + self._observer = None + self._change_detector = None - def _create_solver_model(self, model): - if model is self._pyomo_model: + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: self.update() else: - self.set_instance(model=model) + self.set_instance(pyomo_model=pyomo_model) solution_loader = ScipPersistentSolutionLoader( solver_model=self._solver_model, var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, - pyomo_model=model, + pyomo_model=pyomo_model, opt=self, ) has_obj = self._objective is not None - return self._solver_model, solution_loader, has_obj \ No newline at end of file + return self._solver_model, solution_loader, has_obj + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + return res + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + self._change_detector.update(timer=timer) + timer.stop('update') + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = scip.Model() + self._observer = _ScipObserver(self) + timer.start('set_instance') + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) + self._change_detector.config = self.config.auto_updates + timer.stop('set_instance') + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for v in variables: + self._add_var(v) + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + super()._add_constraints(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + return super()._add_sos_constraints(cons) + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] + + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' + f'active objective ({str(self._objective)})' + ) + + self._invalidate_last_results() + self._set_objective(obj) + + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' + f'{str(obj)}' + ) + else: + self._invalidate_last_results() + self._set_objective(None) + + def _remove_constraints(self, cons: List[ConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map.pop(vid) + self._solver_model.delVar(scip_var) + self._vars.pop(vid) + + def _update_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map[vid] + vtype = self._scip_vtype_from_var(v) + lb, ub = self._scip_lb_ub_from_var(v) + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def _update_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map[pid] + lb = ub = p.value + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + + def add_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_objectives([obj]) + + def remove_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_parameters(params) From 98e2c9a9904b688eb4cb735474bbfd0ea092370f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:40:47 -0600 Subject: [PATCH 66/97] update docs --- .../reference/topical/solvers/scip_persistent.rst | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst b/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst deleted file mode 100644 index 63ed55b74e3..00000000000 --- a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst +++ /dev/null @@ -1,7 +0,0 @@ -SCIPPersistent -================ - -.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent - :members: - :inherited-members: - :show-inheritance: \ No newline at end of file From f0be4ffd5a9cfed6b51d8aa22453358426d873cd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:41:07 -0600 Subject: [PATCH 67/97] update docs --- doc/OnlineDocs/reference/topical/solvers/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/OnlineDocs/reference/topical/solvers/index.rst b/doc/OnlineDocs/reference/topical/solvers/index.rst index 628f9cfdab0..400032df076 100644 --- a/doc/OnlineDocs/reference/topical/solvers/index.rst +++ b/doc/OnlineDocs/reference/topical/solvers/index.rst @@ -9,4 +9,3 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst - scip_persistent.rst From 7ec95a8f4f2c4a9e678cc61aa97c71be20411171 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:46:55 -0600 Subject: [PATCH 68/97] persistent interface to scip --- pyomo/contrib/solver/plugins.py | 7 ++++++- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 2c7bab3bf03..4ac74ecf560 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import ScipDirect +from .solvers.scip.scip_direct import ScipDirect, ScipPersistent def load(): @@ -45,3 +45,8 @@ def load(): legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) + SolverFactory.register( + name='scip_persistent', + legacy_name='scip_persistent_v2', + doc='Persistent interface pyscipopt', + )(ScipPersistent) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index b49d80baa37..6bd7d01e679 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -34,7 +34,7 @@ SolutionStatus, Results, ) -from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect +from pyomo.contrib.solver.solvers.scip.scip_direct import ScipDirect, ScipPersistent from pyomo.contrib.solver.common.util import ( NoDualsError, NoOptimalSolutionError, @@ -60,30 +60,35 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nlp_solvers = [ ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 75903d6cbbfbcd19f96bb709535e6b2a12b0086c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 08:44:16 -0600 Subject: [PATCH 69/97] persistent interface to scip --- pyomo/contrib/observer/model_observer.py | 9 ++ .../solver/solvers/scip/scip_direct.py | 135 +++++++++++++----- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 2da340aab4f..77356ac1b57 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1202,3 +1202,12 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): finally: if is_gc_enabled: gc.enable() + + def get_variables_impacted_by_param(self, p: ParamData): + return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] + + def get_constraints_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[id(p)][0]) + + def get_constraints_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index cf0c71606f8..99de1d80125 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -110,7 +110,7 @@ def __init__( ) -def _handle_var(node, data, opt): +def _handle_var(node, data, opt, visitor): if id(node) not in opt._pyomo_var_to_solver_var_map: scip_var = opt._add_var(node) else: @@ -118,7 +118,13 @@ def _handle_var(node, data, opt): return scip_var -def _handle_param(node, data, opt): +def _handle_param(node, data, opt, visitor): + # for the persistent interface, we create scip variables in place + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the + # lower and upper parts of range constraints + if visitor.in_range: + return node.value if not opt.is_persistent(): return node.value if not node.mutable: @@ -130,19 +136,19 @@ def _handle_param(node, data, opt): return scip_param -def _handle_constant(node, data, opt): +def _handle_constant(node, data, opt, visitor): return node.value -def _handle_float(node, data, opt): +def _handle_float(node, data, opt, visitor): return float(node) -def _handle_negation(node, data, opt): +def _handle_negation(node, data, opt, visitor): return -data[0] -def _handle_pow(node, data, opt): +def _handle_pow(node, data, opt, visitor): x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) if is_constant(node.args[1]): return x**y @@ -154,52 +160,52 @@ def _handle_pow(node, data, opt): return x**y # scip will probably raise an error here -def _handle_product(node, data, opt): +def _handle_product(node, data, opt, visitor): assert len(data) == 2 return data[0] * data[1] -def _handle_division(node, data, opt): +def _handle_division(node, data, opt, visitor): return data[0] / data[1] -def _handle_sum(node, data, opt): +def _handle_sum(node, data, opt, visitor): return sum(data) -def _handle_exp(node, data, opt): +def _handle_exp(node, data, opt, visitor): return scip.exp(data[0]) -def _handle_log(node, data, opt): +def _handle_log(node, data, opt, visitor): return scip.log(data[0]) -def _handle_log10(node, data, opt): +def _handle_log10(node, data, opt, visitor): return scip.log(data[0]) / math.log(10) -def _handle_sin(node, data, opt): +def _handle_sin(node, data, opt, visitor): return scip.sin(data[0]) -def _handle_cos(node, data, opt): +def _handle_cos(node, data, opt, visitor): return scip.cos(data[0]) -def _handle_sqrt(node, data, opt): +def _handle_sqrt(node, data, opt, visitor): return scip.sqrt(data[0]) -def _handle_abs(node, data, opt): +def _handle_abs(node, data, opt, visitor): return abs(data[0]) -def _handle_tan(node, data, opt): +def _handle_tan(node, data, opt, visitor): return scip.sin(data[0]) / scip.cos(data[0]) -def _handle_tanh(node, data, opt): +def _handle_tanh(node, data, opt, visitor): x = data[0] _exp = scip.exp return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) @@ -218,30 +224,32 @@ def _handle_tanh(node, data, opt): } -def _handle_unary(node, data, opt): +def _handle_unary(node, data, opt, visitor): if node.getname() in _unary_map: - return _unary_map[node.getname()](node, data, opt) + return _unary_map[node.getname()](node, data, opt, visitor) else: raise NotImplementedError(f'unable to handle unary expression: {str(node)}') -def _handle_equality(node, data, opt): +def _handle_equality(node, data, opt, visitor): return data[0] == data[1] -def _handle_ranged(node, data, opt): +def _handle_ranged(node, data, opt, visitor): + # note that the lower and upper parts of the + # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) -def _handle_inequality(node, data, opt): +def _handle_inequality(node, data, opt, visitor): return data[0] <= data[1] -def _handle_named_expression(node, data, opt): +def _handle_named_expression(node, data, opt, visitor): return data[0] -def _handle_unit(node, data, opt): +def _handle_unit(node, data, opt, visitor): return node.value @@ -281,16 +289,26 @@ class _PyomoToScipVisitor(StreamBasedExpressionVisitor): def __init__(self, solver, **kwds): super().__init__(**kwds) self.solver = solver + self.in_range = False + + def initializeWalker(self, expr): + self.in_range = False + return True, None def exitNode(self, node, data): nt = type(node) if nt in _operator_map: - return _operator_map[nt](node, data, self.solver) + return _operator_map[nt](node, data, self.solver, self) elif nt in native_numeric_types: _operator_map[nt] = _handle_float - return _handle_float(node, data, self.solver) + return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') + + def enterNode(self, node): + if type(node) is RangedExpression: + self.in_range = True + return None, [] logger = logging.getLogger("pyomo.solvers") @@ -375,7 +393,7 @@ def __init__( pyomo_model, opt, ) - self._valid = False + self._valid = True def invalidate(self): self._valid = False @@ -513,6 +531,7 @@ def solve(self, model: BlockData, **kwds) -> Results: timer.start('optimize') with capture_output(TeeStream(*ostreams), capture_fd=True): + # scip_model.writeProblem(filename='foo.lp') scip_model.optimize() timer.stop('optimize') @@ -723,7 +742,7 @@ def _set_objective(self, obj): vtype="C" ) - if self._objective is not None: + if self._obj_con is not None: self._solver_model.delCons(self._obj_con) if obj is None: @@ -850,7 +869,7 @@ def add_variables(self, variables: List[VarData]): self.opt._add_variables(variables) def add_parameters(self, params: List[ParamData]): - pass + self.opt._add_parameters(params) def add_constraints(self, cons: List[ConstraintData]): self.opt._add_constraints(cons) @@ -874,7 +893,7 @@ def remove_variables(self, variables: List[VarData]): self.opt._remove_variables(variables) def remove_parameters(self, params: List[ParamData]): - pass + self.opt._remove_parameters(params) def update_variables(self, variables: List[VarData]): self.opt._update_variables(variables) @@ -893,13 +912,22 @@ def __init__(self, **kwds): self._observer = None self._change_detector = None self._last_results_object: Optional[Results] = None - + self._needs_reopt = False + self._range_constraints = set() + def _clear(self): super()._clear() self._pyomo_model = None self._objective = None self._observer = None self._change_detector = None + self._needs_reopt = False + + def _check_reopt(self): + if self._needs_reopt: + # self._solver_model.freeReoptSolve() # when is it safe to use this one??? + self._solver_model.freeTransform() + self._needs_reopt = False def _create_solver_model(self, pyomo_model): if pyomo_model is self._pyomo_model: @@ -921,6 +949,7 @@ def _create_solver_model(self, pyomo_model): def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) + self._needs_reopt = True return res def update(self): @@ -957,19 +986,32 @@ def _invalidate_last_results(self): self._last_results_object.solution_loader.invalidate() def _add_variables(self, variables: List[VarData]): + self._check_reopt() self._invalidate_last_results() for v in variables: self._add_var(v) + def _add_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + self._add_param(p) + def _add_constraints(self, cons: List[ConstraintData]): + self._check_reopt() self._invalidate_last_results() + for con in cons: + if type(con.expr) is RangedExpression: + self._range_constraints.add(con) super()._add_constraints(cons) def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) def _add_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() if len(objs) > 1: raise NotImplementedError( 'the persistent interface to gurobi currently ' @@ -994,6 +1036,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): self._set_objective(obj) def _remove_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() for obj in objs: if obj is not self._objective: raise RuntimeError( @@ -1005,23 +1048,41 @@ def _remove_objectives(self, objs: List[ObjectiveData]): self._set_objective(None) def _remove_constraints(self, cons: List[ConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) + self._range_constraints.discard(con) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) def _remove_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map.pop(vid) self._solver_model.delVar(scip_var) self._vars.pop(vid) + def _remove_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map.pop(pid) + self._solver_model.delVar(scip_var) + self._params.pop(pid) + def _update_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map[vid] @@ -1032,12 +1093,22 @@ def _update_variables(self, variables: List[VarData]): self._solver_model.chgVarType(scip_var, vtype) def _update_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() for p in params: pid = id(p) scip_var = self._pyomo_param_to_solver_param_map[pid] lb = ub = p.value self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) + impacted_vars = self._change_detector.get_variables_impacted_by_param(p) + if impacted_vars: + self._update_variables(impacted_vars) + impacted_cons = self._change_detector.get_constraints_impacted_by_param(p) + for con in impacted_cons: + if con in self._range_constraints: + self._remove_constraints([con]) + self._add_constraints([con]) def add_variables(self, variables): if self._change_detector is None: From 0051024e7a44ab7d4c3df43c96890c49669e3b69 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:46:14 -0600 Subject: [PATCH 70/97] updating tests --- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 6bd7d01e679..3665de4521a 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -2381,7 +2381,8 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=(0, m.y - m.a1 * m.x - m.b1, None)) m.c2 = pyo.Constraint(expr=(None, -m.y + m.a2 * m.x + m.b2, 0)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) params_to_test = [(1, -1, 2, 1), (1, -2, 2, 1), (1, -1, 3, 1)] for a1, a2, b1, b2 in params_to_test: @@ -2393,8 +2394,9 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): pyo.assert_optimal_termination(res) self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=all_solvers) def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): @@ -2405,11 +2407,14 @@ def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): m.x = pyo.Var() m.obj = pyo.Objective(expr=m.x) m.c = pyo.Constraint(expr=(-1, m.x, 1)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) res = opt.solve(m, load_solutions=False) pyo.assert_optimal_termination(res) self.assertIsNone(m.x.value) - self.assertNotIn(m.c, m.dual) + if (name, opt_class) in dual_solvers: + self.assertNotIn(m.c, m.dual) m.solutions.load_from(res) self.assertAlmostEqual(m.x.value, -1) - self.assertAlmostEqual(m.dual[m.c], 1) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c], 1) From b037b9c356cd9caac3bf04924bece2c247962f91 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:57:00 -0600 Subject: [PATCH 71/97] forgot to delete/revert some files --- .../solver/solvers/scip/scip_persistent.py | 192 ----------- pyomo/solvers/tests/checks/test_SCIPDirect.py | 310 ----------------- .../tests/checks/test_SCIPPersistent.py | 318 ------------------ pyomo/solvers/tests/solvers.py | 15 - pyomo/solvers/tests/testcases.py | 9 - 5 files changed, 844 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/scip/scip_persistent.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/contrib/solver/solvers/scip/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py deleted file mode 100644 index bc64edc28a8..00000000000 --- a/pyomo/contrib/solver/solvers/scip/scip_persistent.py +++ /dev/null @@ -1,192 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ -from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect -from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver -from pyomo.opt.base import SolverFactory - - -@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") -class SCIPPersistent(PersistentSolver, SCIPDirect): - """ - A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. - Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces - are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes - to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible - for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. - - Keyword Arguments - ----------------- - model: ConcreteModel - Passing a model to the constructor is equivalent to calling the set_instance method. - type: str - String indicating the class type of the solver instance. - name: str - String representing either the class type of the solver instance or an assigned name. - doc: str - Documentation for the solver - options: dict - Dictionary of solver options - """ - - def __init__(self, **kwds): - kwds["type"] = "scip_persistent" - PersistentSolver.__init__(self, **kwds) - SCIPDirect._init(self) - - self._pyomo_model = kwds.pop("model", None) - if self._pyomo_model is not None: - self.set_instance(self._pyomo_model, **kwds) - - def _remove_constraint(self, solver_conname): - con = self._solver_con_to_pyomo_con_map[solver_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_sos_constraint(self, solver_sos_conname): - con = self._solver_con_to_pyomo_con_map[solver_sos_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_var(self, solver_varname): - var = self._solver_var_to_pyomo_var_map[solver_varname] - scip_var = self._pyomo_var_to_solver_var_expr_map[var] - self._solver_model.delVar(scip_var) - del self._pyomo_var_to_solver_var_expr_map[var] - - def _warm_start(self): - SCIPDirect._warm_start(self) - - def update_var(self, var): - """Update a single variable in the solver's model. - - This will update bounds, fix/unfix the variable as needed, and - update the variable type. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - - """ - # see PR #366 for discussion about handling indexed - # objects and keeping compatibility with the - # pyomo.kernel objects - # if var.is_indexed(): - # for child_var in var.values(): - # self.compile_var(child_var) - # return - if var not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f"The Var provided to compile_var needs to be added first: {var}" - ) - scip_var = self._pyomo_var_to_solver_var_map[var] - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - self._solver_model.chgVarLb(scip_var, lb) - self._solver_model.chgVarUb(scip_var, ub) - self._solver_model.chgVarType(scip_var, vtype) - - def write(self, filename, filetype=""): - """ - Write the model to a file (e.g., an lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - filetype: str - The file type (e.g., lp). - """ - self._solver_model.writeProblem(filename + filetype) - - def set_scip_param(self, param, val): - """ - Set a SCIP parameter. - - Parameters - ---------- - param: str - The SCIP parameter to set. Options include any SCIP parameter. - Please see the SCIP documentation for options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - val: any - The value to set the parameter to. See SCIP documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_scip_param(self, param): - """ - Get the value of the SCIP parameter. - - Parameters - ---------- - param: str or int or float - The SCIP parameter to get the value of. See SCIP documentation for possible options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - """ - return self._solver_model.getParam(param) - - def _add_column(self, var, obj_coef, constraints, coefficients): - """Add a column to the solver's model - - This will add the Pyomo variable var to the solver's - model, and put the coefficients on the associated - constraints in the solver model. If the obj_coef is - not zero, it will add obj_coef*var to the objective - of the solver's model. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - obj_coef: float - constraints: list of solver constraints - coefficients: list of coefficients to put on var in the associated constraint - """ - - # Set-up add var - varname = self._symbol_map.getSymbol(var, self._labeler) - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - # Add the variable to the model and then to all the constraints - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = len(coefficients) - - # Get the SCIP cons by passing through two dictionaries - pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_con] - for pyomo_con in pyomo_cons - ] - - for i, scip_con in enumerate(scip_cons): - if not scip_con.isLinear(): - raise ValueError( - "_add_column functionality not supported for non-linear constraints" - ) - self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) - con = self._solver_con_to_pyomo_con_map[scip_con.name] - self._vars_referenced_by_con[con].add(var) - - sense = self._solver_model.getObjectiveSense() - self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) - - def reset(self): - """This function is necessary to call before making any changes to the - SCIP model after optimizing. It frees solution run specific information - that is not automatically done when changes to an already solved model - are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called.""" - self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py deleted file mode 100644 index 186de0eaf58..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ /dev/null @@ -1,310 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import sys - -import pyomo.common.unittest as unittest - -from pyomo.environ import ( - ConcreteModel, - AbstractModel, - Var, - Objective, - Block, - Constraint, - Suffix, - NonNegativeIntegers, - NonNegativeReals, - Integers, - Binary, - value, -) -from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class SCIPDirectTests(unittest.TestCase): - def setUp(self): - self.stderr = sys.stderr - sys.stderr = None - - def tearDown(self): - sys.stderr = self.stderr - - def test_infeasible_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var() - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - def test_infeasible_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = AbstractModel() - model.X = Var(within=Integers) - model.O = Objective(expr=model.X) - - instance = model.create_instance() - results = opt.solve(instance) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddVar(unittest.TestCase): - def test_add_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_var(model.X) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_multiple_variables(self): - """Test that: - - The variable is added correctly to `solver_model` - - Fixed variable bounds are set correctly - """ - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X1 = Var(within=Binary) - model.X2 = Var(within=NonNegativeReals) - model.X3 = Var(within=NonNegativeIntegers) - - model.X3.fix(5) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 3) - scip_vars = opt._solver_model.getVars() - vtypes = [scip_var.vtype() for scip_var in scip_vars] - assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes - lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] - ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] - assert 0 in lbs and 5 in lbs - assert ( - 1 in ubs - and 5 in ubs - and any([opt._solver_model.isInfinity(ub) for ub in ubs]) - ) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddCon(unittest.TestCase): - def test_add_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.C = Constraint(expr=model.X == 1) - - opt._add_constraint(model.C) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C = Constraint(expr=model.X == 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_multiple_constraints(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C1 = Constraint(expr=model.X == 1) - model.B.C2 = Constraint(expr=model.X <= 1) - model.B.C3 = Constraint(expr=model.X >= 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 3) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestLoadVars(unittest.TestCase): - def setUp(self): - opt = SolverFactory("scip_direct", solver_io="python") - model = ConcreteModel() - model.X = Var(within=NonNegativeReals, initialize=0) - model.Y = Var(within=NonNegativeReals, initialize=0) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - opt.solve(model, load_solutions=False, save_results=False) - - self._model = model - self._opt = opt - - def test_all_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars() - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - def test_only_specified_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.X]) - - self.assertFalse(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.Y]) - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py deleted file mode 100644 index 61cf7385352..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ /dev/null @@ -1,318 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import pyomo.environ -import pyomo.common.unittest as unittest - -from pyomo.core import ( - ConcreteModel, - Var, - Objective, - Constraint, - NonNegativeReals, - NonNegativeIntegers, - Reals, - Binary, - SOSConstraint, - Set, - sin, - cos, - exp, - log, -) -from pyomo.opt import SolverFactory - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestQuadraticObjective(unittest.TestCase): - def test_quadratic_objective_linear_surrogate_is_set(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.Y = Var(bounds=(-2, 2)) - m.Z = Var(within=Reals) - m.O = Objective(expr=m.Z) - m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) - m.C2 = Constraint(expr=m.Y >= -m.X + 2) - m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 1, places=3) - - opt.reset() - - opt.remove_constraint(m.C3) - del m.C3 - m.C3 = Constraint(expr=m.Z >= m.X**2) - opt.add_constraint(m.C3) - opt.solve() - self.assertAlmostEqual(m.X.value, 0, places=3) - self.assertAlmostEqual(m.Y.value, 2, places=3) - - def test_add_and_remove_sos(self): - m = ConcreteModel() - m.I = Set(initialize=[1, 2, 3]) - m.X = Var(m.I, bounds=(-2, 2)) - - m.C = SOSConstraint(var=m.X, sos=1) - - m.O = Objective(expr=m.X[1] + m.X[2]) - - opt = SolverFactory("scip_persistent") - - opt.set_instance(m) - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 2 - - opt.reset() - - opt.remove_sos_constraint(m.C) - del m.C - - m.C = SOSConstraint(var=m.X, sos=2) - opt.add_sos_constraint(m.C) - - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 1 - - def test_get_and_set_param(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.O = Objective(expr=m.X) - m.C3 = Constraint(expr=m.X <= 2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.set_scip_param("limits/time", 60) - - assert opt.get_scip_param("limits/time") == 60 - - def test_non_linear(self): - - PI = 3.141592653589793238462643 - NWIRES = 11 - DIAMETERS = [ - 0.207, - 0.225, - 0.244, - 0.263, - 0.283, - 0.307, - 0.331, - 0.362, - 0.394, - 0.4375, - 0.500, - ] - PRELOAD = 300.0 - MAXWORKLOAD = 1000.0 - MAXDEFLECT = 6.0 - DEFLECTPRELOAD = 1.25 - MAXFREELEN = 14.0 - MAXCOILDIAM = 3.0 - MAXSHEARSTRESS = 189000.0 - SHEARMOD = 11500000.0 - - m = ConcreteModel() - m.coil = Var(within=NonNegativeReals) - m.wire = Var(within=NonNegativeReals) - m.defl = Var( - bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) - ) - m.ncoils = Var(within=NonNegativeIntegers) - m.const1 = Var(within=NonNegativeReals) - m.const2 = Var(within=NonNegativeReals) - m.volume = Var(within=NonNegativeReals) - m.I = Set(initialize=[i for i in range(NWIRES)]) - m.y = Var(m.I, within=Binary) - - m.O = Objective(expr=m.volume) - - m.c1 = Constraint( - expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 - ) - - m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) - - m.c3 = Constraint( - expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 - == 0 - ) - - m.c4 = Constraint( - expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 - - MAXSHEARSTRESS * m.wire**2 - <= 0 - ) - - m.c5 = Constraint( - expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 - ) - - m.c6 = Constraint( - expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire - <= MAXFREELEN - ) - - m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) - - m.c8 = Constraint( - expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 - ) - - m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - - self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) - - def test_non_linear_unary_expressions(self): - - m = ConcreteModel() - m.X = Var(bounds=(1, 2)) - m.Y = Var(within=Reals) - - m.O = Objective(expr=m.Y) - - m.C = Constraint(expr=exp(m.X) == m.Y) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, exp(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=log(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 0, places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=sin(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, sin(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=cos(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 2, places=3) - self.assertAlmostEqual(m.Y.value, cos(2), places=3) - - def test_add_column(self): - m = ConcreteModel() - m.x = Var(within=NonNegativeReals) - m.c = Constraint(expr=(0, m.x, 1)) - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - self.assertAlmostEqual(m.x.value, 1) - - m.y = Var(within=NonNegativeReals) - - opt.reset() - - opt.add_column(m, m.y, -3, [m.c], [2]) - opt.solve() - - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 0.5) - - def test_add_column_exceptions(self): - m = ConcreteModel() - m.x = Var() - m.c = Constraint(expr=(0, m.x, 1)) - m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) - m.cd = Constraint(expr=(0, -m.x, 1)) - m.cd.deactivate() - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - - # set_instance not called - self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) - - opt.set_instance(m) - - m2 = ConcreteModel() - m2.y = Var() - m2.c = Constraint(expr=(0, m.x, 1)) - - # different model than attached to opt - self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) - # pyomo var attached to different model - self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) - - z = Var() - # pyomo var floating - self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) - - m.y = Var() - # len(coefficients) == len(constraints) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) - - # add indexed constraint - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) - # add something not a _ConstraintData - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) - - # constraint not on solver model - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) - - # inactive constraint - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) - - opt.add_var(m.y) - # var already in solver model - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index e9967cd1ce2..e5058e8894b 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -369,21 +369,6 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) - # - # SCIP PERSISTENT - # - - _scip_persistent_capabilities = set( - ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] - ) - - _test_solver_cases["scip_persistent", "python"] = initialize( - name="scip_persistent", - io="python", - capabilities=_scip_persistent_capabilities, - import_suffixes=[], - ) - # # CONOPT # diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index c1725bedee7..696936ddf05 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,15 +248,6 @@ "inside NL files. A ticket has been filed.", ) -# -# SCIP Persistent -# - -ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( - lambda v: v <= _trunk_version, - "SCIP does not allow empty constraints with no variables to be added to the Model.", -) - # # BARON # From c200e2e996920c0eccb2e12d3b74376847e3b448 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 15:09:16 -0600 Subject: [PATCH 72/97] run black --- pyomo/contrib/observer/model_observer.py | 2 +- pyomo/contrib/solver/plugins.py | 8 +- .../solver/solvers/scip/scip_direct.py | 138 +++++++++--------- .../solver/tests/solvers/test_solvers.py | 3 +- 4 files changed, 75 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 77356ac1b57..9bb9c917fc5 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1208,6 +1208,6 @@ def get_variables_impacted_by_param(self, p: ParamData): def get_constraints_impacted_by_param(self, p: ParamData): return list(self._referenced_params[id(p)][0]) - + def get_constraints_impacted_by_var(self, v: VarData): return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 4ac74ecf560..ff24148dd73 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -41,12 +41,12 @@ def load(): name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) SolverFactory.register( - name='scip_direct', - legacy_name='scip_direct_v2', + name='scip_direct', + legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) SolverFactory.register( - name='scip_persistent', - legacy_name='scip_persistent_v2', + name='scip_persistent', + legacy_name='scip_persistent_v2', doc='Persistent interface pyscipopt', )(ScipPersistent) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 99de1d80125..7e39d6e8595 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -47,11 +47,19 @@ from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression -from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, +) from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase +from pyomo.contrib.solver.common.base import ( + SolverBase, + Availability, + PersistentSolverBase, +) from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -73,7 +81,11 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -120,8 +132,8 @@ def _handle_var(node, data, opt, visitor): def _handle_param(node, data, opt, visitor): # for the persistent interface, we create scip variables in place - # of parameters. However, this makes things complicated for range - # constraints because scip does not allow variables in the + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the # lower and upper parts of range constraints if visitor.in_range: return node.value @@ -155,7 +167,7 @@ def _handle_pow(node, data, opt, visitor): else: xlb, xub = compute_bounds_on_expr(node.args[0]) if xlb > 0: - return scip.exp(y*scip.log(x)) + return scip.exp(y * scip.log(x)) else: return x**y # scip will probably raise an error here @@ -236,7 +248,7 @@ def _handle_equality(node, data, opt, visitor): def _handle_ranged(node, data, opt, visitor): - # note that the lower and upper parts of the + # note that the lower and upper parts of the # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) @@ -304,7 +316,7 @@ def exitNode(self, node, data): return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') - + def enterNode(self, node): if type(node) is RangedExpression: self.in_range = True @@ -316,13 +328,7 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: super().__init__() self._solver_model = solver_model @@ -342,7 +348,9 @@ def get_solution_ids(self) -> List: def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: - for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + for v, val in self.get_vars( + vars_to_load=vars_to_load, solution_id=solution_id + ).items(): v.value = val def get_vars( @@ -377,22 +385,9 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: - super().__init__( - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, - ) + super().__init__(solver_model, var_id_map, var_map, con_map, pyomo_model, opt) self._valid = True def invalidate(self): @@ -401,7 +396,7 @@ def invalidate(self): def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - + def load_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: @@ -454,11 +449,15 @@ def __init__(self, **kwds): self._params = {} # param id to param self._pyomo_var_to_solver_var_map = {} # var id to scip var self._pyomo_con_to_solver_con_map = {} - self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_param_to_solver_param_map = ( + {} + ) # param id to scip var with equal bounds self._pyomo_sos_to_solver_sos_map = {} self._expr_visitor = _PyomoToScipVisitor(self) self._objective = None # pyomo objective - self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_var = ( + None # a scip variable because the objective cannot be nonlinear + ) self._obj_con = None # a scip constraint (obj_var >= obj_expr) def _clear(self): @@ -476,7 +475,7 @@ def _clear(self): def available(self) -> Availability: if self._available is not None: return self._available - + if not scip_available: ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: @@ -485,7 +484,7 @@ def available(self) -> Availability: ScipDirect._available = Availability.FullLicense return self._available - + def version(self) -> Tuple: return tuple(int(i) for i in scip.__version__.split('.')) @@ -493,9 +492,7 @@ def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): - raise ApplicationError( - f'{self.name} is not available: {self.available()}' - ) + raise ApplicationError(f'{self.name} is not available: {self.available()}') try: config = self.config(value=kwds, preserve_implicit=True) @@ -546,7 +543,9 @@ def solve(self, model: BlockData, **kwds) -> Results: results.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) results.timing_info.start_timestamp = start_timestamp - results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.wall_time = ( + end_timestamp - start_timestamp + ).total_seconds() results.timing_info.timer = timer return results @@ -616,7 +615,7 @@ def _add_var(self, var): self._vars[id(var)] = var self._pyomo_var_to_solver_var_map[id(var)] = scip_var return scip_var - + def _add_param(self, p): vtype = "C" lb = ub = p.value @@ -646,9 +645,7 @@ def _create_solver_model(self, model): self._solver_model = scip.Model() timer.start('collect constraints') cons = list( - model.component_data_objects( - Constraint, descend_into=True, active=True - ) + model.component_data_objects(Constraint, descend_into=True, active=True) ) timer.stop('collect constraints') timer.start('translate constraints') @@ -656,9 +653,7 @@ def _create_solver_model(self, model): timer.stop('translate constraints') timer.start('sos') sos = list( - model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) + model.component_data_objects(SOSConstraint, descend_into=True, active=True) ) self._add_sos_constraints(sos) timer.stop('sos') @@ -688,7 +683,9 @@ def _add_constraint(self, con): def _add_sos_constraint(self, con): level = con.level if level not in [1, 2]: - raise ValueError(f"{self.name} does not support SOS level {level} constraints") + raise ValueError( + f"{self.name} does not support SOS level {level} constraints" + ) scip_vars = [] weights = [] @@ -701,13 +698,9 @@ def _add_sos_constraint(self, con): weights.append(w) if level == 1: - scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS1(scip_vars, weights=weights) else: - scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS2(scip_vars, weights=weights) self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): @@ -737,9 +730,9 @@ def _scip_vtype_from_var(self, var): def _set_objective(self, obj): if self._obj_var is None: self._obj_var = self._solver_model.addVar( - lb=-self._solver_model.infinity(), - ub=self._solver_model.infinity(), - vtype="C" + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C", ) if self._obj_con is not None: @@ -766,19 +759,21 @@ def _set_objective(self, obj): self._objective = obj def _postsolve( - self, - scip_model, - solution_loader: ScipDirectSolutionLoader, - has_obj + self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj ): results = Results() results.solution_loader = solution_loader - results.timing_info.scip_time = scip_model.getSolvingTime() - results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) - + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get( + scip_model.getStatus(), TerminationCondition.unknown + ) + if solution_loader.get_number_of_solutions() > 0: - if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + if ( + results.termination_condition + == TerminationCondition.convergenceCriteriaSatisfied + ): results.solution_status = SolutionStatus.optimal else: results.solution_status = SolutionStatus.feasible @@ -786,15 +781,18 @@ def _postsolve( results.solution_status = SolutionStatus.noSolution if ( - results.termination_condition + results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() - + if has_obj: try: - if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): + if ( + scip_model.getNSols() > 0 + and scip_model.getObjVal() < scip_model.infinity() + ): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None @@ -831,7 +829,7 @@ def _postsolve( return results def _mipstart(self): - # TODO: it is also possible to specify continuous variables, but + # TODO: it is also possible to specify continuous variables, but # I think we should have a differnt option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): @@ -1009,7 +1007,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) - + def _add_objectives(self, objs: List[ObjectiveData]): self._check_reopt() if len(objs) > 1: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 3665de4521a..e6686266028 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1152,7 +1152,8 @@ def test_results_infeasible( ): res.solution_loader.get_duals() with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' + NoReducedCostsError, + '.*does not currently have valid reduced costs.*', ): res.solution_loader.get_reduced_costs() From 960c531bba4c9bc9ea65cb652c699c5a131a8382 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 17:06:23 -0600 Subject: [PATCH 73/97] typos --- pyomo/contrib/solver/common/solution_loader.py | 2 +- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index f8723b6e0f4..6be23b63c77 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -63,7 +63,7 @@ def get_solution_ids(self) -> List[Any]: """ If there are multiple solutions available, this will return a list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is + methods like `load_solution`. If only one solution is available, this will return [None]. If no solutions are available, this will return None diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 7e39d6e8595..05f39b0cb16 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -830,7 +830,7 @@ def _postsolve( def _mipstart(self): # TODO: it is also possible to specify continuous variables, but - # I think we should have a differnt option for that + # I think we should have a different option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): pyomo_var = self._vars[vid] From cf000a1f663cae02c4c0171179fd8de51fe5dfda Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 08:28:10 -0600 Subject: [PATCH 74/97] directory for all gurobi interfaces --- pyomo/contrib/solver/solvers/{ => gurobi}/gurobi_direct_minlp.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pyomo/contrib/solver/solvers/{ => gurobi}/gurobi_direct_minlp.py (100%) diff --git a/pyomo/contrib/solver/solvers/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py similarity index 100% rename from pyomo/contrib/solver/solvers/gurobi_direct_minlp.py rename to pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py From 7b20095c4a39a0ca45a837a1be6aac4714ef50ad Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 11:29:49 -0600 Subject: [PATCH 75/97] clean up gurobi interfaces --- pyomo/contrib/solver/plugins.py | 7 +- .../contrib/solver/solvers/gurobi/__init__.py | 3 +- .../solver/solvers/gurobi/gurobi_direct.py | 93 +--- .../solvers/gurobi/gurobi_direct_base.py | 135 ++++-- .../solvers/gurobi/gurobi_direct_minlp.py | 84 +--- .../solvers/gurobi/gurobi_persistent.py | 425 +++++------------- .../solver/tests/solvers/test_gurobi_minlp.py | 2 +- .../tests/solvers/test_gurobi_minlp_walker.py | 2 +- .../tests/solvers/test_gurobi_minlp_writer.py | 2 +- .../solver/tests/solvers/test_solvers.py | 5 - 10 files changed, 250 insertions(+), 508 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 99c6f4fb612..430207a736c 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,7 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt, LegacyIpoptSolver from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from .solvers.gurobi.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from .solvers.highs import Highs from .solvers.knitro.direct import KnitroDirectSolver @@ -33,11 +33,6 @@ def load(): legacy_name="gurobi_direct_v2", doc="Direct (scipy-based) interface to Gurobi", )(GurobiDirect) - SolverFactory.register( - name='gurobi_direct_quadratic', - legacy_name='gurobi_direct_quadratic_v2', - doc='Direct interface to Gurobi', - )(GurobiDirectQuadratic) SolverFactory.register( name='gurobi_direct_minlp', legacy_name='gurobi_direct_minlp', diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py index 0ef0c8c9908..0809846ebc3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/__init__.py +++ b/pyomo/contrib/solver/solvers/gurobi/__init__.py @@ -1,2 +1,3 @@ from .gurobi_direct import GurobiDirect -from .gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from .gurobi_persistent import GurobiPersistent +from .gurobi_direct_minlp import GurobiDirectMINLP diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 16c633c7d7c..5ee8ad54f0e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,88 +23,25 @@ IncompatibleModelError, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -from .gurobi_direct_base import GurobiDirectBase, gurobipy +from .gurobi_direct_base import GurobiDirectBase, gurobipy, GurobiDirectSolutionLoaderBase +import logging -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - GurobiDirectBase._register_env_client() +logger = logging.getLogger(__name__) + +class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): def __del__(self): + super().__del__() if python_is_shutting_down(): return # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None + if self._solver_model is not None: + self._var_map = None + self._con_map = None # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirectBase._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) + self._solver_model.dispose() + self._solver_model = None class GurobiDirect(GurobiDirectBase): @@ -116,7 +53,7 @@ def __init__(self, **kwds): self._pyomo_vars = None def _pyomo_gurobi_var_iter(self): - return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + return zip(self._pyomo_vars, self._gurobi_vars) def _create_solver_model(self, pyomo_model): timer = self.config.timer @@ -174,10 +111,12 @@ def _create_solver_model(self, pyomo_model): timer.stop('transfer_model') self._pyomo_vars = repn.columns - self._gurobi_vars = x + self._gurobi_vars = x.tolist() + var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) + con_map = dict(zip([i.constraint for i in repn.rows], A.tolist())) solution_loader = GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns + solver_model=gurobi_model, var_map=var_map, con_map=con_map, ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 489f0b5fe71..506279ee37e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -14,6 +14,7 @@ import math import os import logging +from typing import Mapping, Optional, Sequence, Dict from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue @@ -24,6 +25,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base import VarData, ConstraintData from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -39,10 +41,8 @@ SolutionStatus, TerminationCondition, ) -import logging - +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -logger = logging.getLogger(__name__) logger = logging.getLogger(__name__) @@ -80,8 +80,8 @@ def __init__( def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -92,11 +92,9 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var, val in zip(vars_to_load, vals): - res[var] = val + res = ComponentMap(zip(vars_to_load, vals)) solver_model.setParam('SolutionNumber', original_solution_number) return res @@ -104,8 +102,8 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -122,8 +120,8 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[Vardata, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] solution_number: int """ @@ -138,59 +136,122 @@ def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): solution_number=solution_number, ) - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("X", gurobi_vars_to_load) - res = ComponentMap() - for var, val in zip(vars_to_load, vals): - res[var] = val + res = ComponentMap(zip(vars_to_load, vals)) return res def _get_reduced_costs(solver_model, var_map, vars_to_load): """ solver_model: gurobipy.Model - var_map: Dict[int, gurobipy.Var] - Maps the id of the pyomo variable to the gurobipy variable + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable vars_to_load: List[VarData] """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() - res = ComponentMap() - gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Rc", gurobi_vars_to_load) - for var, val in zip(vars_to_load, vals): - res[var] = val - + res = ComponentMap(zip(vars_to_load, vals)) return res -def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): +def _get_duals(solver_model, con_map, cons_to_load): """ solver_model: gurobipy.Model con_map: Dict[ConstraintData, gurobipy.Constr] Maps the pyomo constraint to the gurobipy constraint - linear_cons_to_load: List[ConstraintData] - quadratic_cons_to_load: List[ConstraintData] + cons_to_load: List[ConstraintData] """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - - linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] - quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] - linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) - quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) - + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + qcons = set(solver_model.getQConstrs()) + duals = {} - for c, val in zip(linear_cons_to_load, linear_vals): - duals[c] = val - for c, val in zip(quadratic_cons_to_load, quadratic_vals): - duals[c] = val + for c in cons_to_load: + gurobi_con = con_map[c] + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi + return duals +class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): + def __init__( + self, solver_model, var_map, con_map, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._var_map = var_map + self._con_map = con_map + GurobiDirectBase._register_env_client() + + def __del__(self): + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> None: + if vars_to_load is None: + vars_to_load = self._var_map + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = self._con_map + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + cons_to_load=cons_to_load, + ) + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index bc7b8362aea..1fdbf27c018 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -24,10 +24,8 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.util import NoSolutionError -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiDirect, - GurobiDirectSolutionLoader, -) +from .gurobi_direct_base import GurobiDirectBase +from .gurobi_direct import GurobiDirectSolutionLoader from pyomo.core.base import ( Binary, @@ -584,27 +582,18 @@ def write(self, model, **options): doc='Direct interface to Gurobi version 12 and up ' 'supporting general nonlinear expressions', ) -class GurobiDirectMINLP(GurobiDirect): - def solve(self, model, **kwds): - """Solve the model. +class GurobiDirectMINLP(GurobiDirectBase): + _minimum_version = (12, 0, 0) - Args: - model (Block): a Pyomo model or Block to be solved - """ - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer + def __init__(self, **kwds): + super().__init__(**kwds) + self._var_map = None - StaleFlagManager.mark_all_as_stale() + def _pyomo_gurobi_var_iter(self): + return self._var_map.items() + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') writer = GurobiMINLPWriter() @@ -614,50 +603,11 @@ def solve(self, model, **kwds): timer.stop('compile_model') - ostreams = [io.StringIO()] + config.tee - - # set options - options = config.solver_options - - grb_model.setParam('LogToConsole', 1) - - if config.threads is not None: - grb_model.setParam('Threads', config.threads) - if config.time_limit is not None: - grb_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - grb_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - grb_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - grb_model.setParam(key, option) - - grbsol = grb_model.optimize() - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - grb_model, - grb_cons=grb_cons, - grb_vars=var_map.values(), - pyo_cons=pyo_cons, - pyo_vars=var_map.keys(), - pyo_obj=pyo_obj, - ), - ) + self._var_map = var_map + con_map = dict(zip(pyo_cons, grb_cons)) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() + solution_loader = GurobiDirectSolutionLoader( + solver_model=grb_model, var_map=var_map, con_map=con_map, + ) - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res + return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 315d8c6dc4a..3462f50b437 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -14,7 +14,7 @@ from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.collections import ComponentSet, OrderedSet, ComponentMap from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer @@ -28,18 +28,15 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, gurobipy, GurobiConfig, - _load_vars, - _get_primals, - _get_duals, - _get_reduced_costs, + GurobiDirectSolutionLoaderBase, ) +from .gurobi_direct import GurobiDirectSolutionLoader from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.model_observer import ( Observer, @@ -51,87 +48,12 @@ logger = logging.getLogger(__name__) -class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): +class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons - ) -> None: - super().__init__() - self._solver_model = solver_model - self._vars = var_id_map - self._var_map = var_map - self._con_map = con_map - self._linear_cons = linear_cons - self._quadratic_cons = quadratic_cons - GurobiDirectBase._register_env_client() - - def __del__(self): - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirectBase._release_env_client() - - def load_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 - ) -> None: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - _load_vars( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) - - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 - ) -> Mapping[VarData, float]: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - return _get_primals( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) - - def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None - ) -> Mapping[VarData, float]: - if vars_to_load is None: - vars_to_load = list(self._vars.values()) - return _get_reduced_costs( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - ) - - def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None - ) -> Dict[ConstraintData, float]: - if cons_to_load is None: - cons_to_load = list(self._con_map.keys()) - linear_cons_to_load = [] - quadratic_cons_to_load = [] - for c in cons_to_load: - if c in self._linear_cons: - linear_cons_to_load.append(c) - else: - assert c in self._quadratic_cons - quadratic_cons_to_load.append(c) - return _get_duals( - solver_model=self._solver_model, - con_map=self._con_map, - linear_cons_to_load=linear_cons_to_load, - quadratic_cons_to_load=quadratic_cons_to_load, - ) - - -class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_map, con_map, ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + solver_model, var_map, con_map, ) self._valid = True @@ -325,225 +247,6 @@ def var2(self): return self.var_map[self.v2id] -class GurobiDirectQuadratic(GurobiDirectBase): - _minimum_version = (7, 0, 0) - - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_model = None - self._vars = {} # from id(v) to v - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._linear_cons = set() - self._quadratic_cons = set() - self._pyomo_sos_to_solver_sos_map = {} - - def _create_solver_model(self, pyomo_model): - timer = self.config.timer - timer.start('create gurobipy model') - self._clear() - self._solver_model = gurobipy.Model(env=self.env()) - timer.start('collect constraints') - cons = list( - pyomo_model.component_data_objects( - Constraint, descend_into=True, active=True - ) - ) - timer.stop('collect constraints') - timer.start('translate constraints') - self._add_constraints(cons) - timer.stop('translate constraints') - timer.start('sos') - sos = list( - pyomo_model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) - ) - self._add_sos_constraints(sos) - timer.stop('sos') - timer.start('get objective') - obj = get_objective(pyomo_model) - timer.stop('get objective') - timer.start('translate objective') - self._set_objective(obj) - timer.stop('translate objective') - has_obj = obj is not None - solution_loader = GurobiDirectQuadraticSolutionLoader( - solver_model=self._solver_model, - var_id_map=self._vars, - var_map=self._pyomo_var_to_solver_var_map, - con_map=self._pyomo_con_to_solver_con_map, - linear_cons=self._linear_cons, - quadratic_cons=self._quadratic_cons, - ) - timer.stop('create gurobipy model') - return self._solver_model, solution_loader, has_obj - - def _clear(self): - self._solver_model = None - self._vars = {} - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._linear_cons = set() - self._quadratic_cons = set() - self._pyomo_sos_to_solver_sos_map = {} - - def _pyomo_gurobi_var_iter(self): - for vid, v in self._vars.items(): - yield v, self._pyomo_var_to_solver_var_map[vid] - - def _process_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError(f'Unrecognized domain: {var.domain}') - if var.fixed: - lb = var.value - ub = lb - else: - if var._lb is not None: - lb = max(lb, value(var._lb)) - if var._ub is not None: - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - vtypes = [] - lbs = [] - ubs = [] - for ndx, var in enumerate(variables): - self._vars[id(var)] = var - lb, ub, vtype = self._process_domain_and_bounds(var) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes - ).values() - - for pyomo_var, gurobi_var in zip(variables, gurobi_vars): - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - - def _get_expr_from_pyomo_repn(self, repn): - if repn.nonlinear_expr is not None: - raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' - ) - - if len(repn.linear_vars) > 0: - missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - self._add_variables(missing_vars) - coef_list = [value(i) for i in repn.linear_coefs] - vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr(coef_list, vlist) - else: - new_expr = 0.0 - - if len(repn.quadratic_vars) > 0: - missing_vars = {} - for x, y in repn.quadratic_vars: - for v in [x, y]: - vid = id(v) - if vid not in self._vars: - missing_vars[vid] = v - self._add_variables(list(missing_vars.values())) - for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += value(coef) * gurobi_x * gurobi_y - - return new_expr - - def _add_constraints(self, cons: List[ConstraintData]): - gurobi_expr_list = [] - for con in cons: - lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - repn = generate_standard_repn(body, quadratic=True, compute_values=True) - if len(repn.quadratic_vars) > 0: - self._quadratic_cons.add(con) - else: - self._linear_cons.add(con) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - if lb is None and ub is None: - raise ValueError( - "Constraint does not have a lower " f"or an upper bound: {con} \n" - ) - elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) - elif ub is None: - gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) - elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) - else: - gurobi_expr_list.append( - gurobi_expr - == [float(lb - repn.constant), float(ub - repn.constant)] - ) - - gurobi_cons = self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) - ).values() - self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - missing_vars = { - id(v): v for v, w in con.get_items() if id(v) not in self._vars - } - self._add_variables(list(missing_vars.values())) - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - repn_constant = repn.constant - - self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - - class _GurobiObserver(Observer): def __init__(self, opt: GurobiPersistent) -> None: self.opt = opt @@ -605,12 +308,16 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): +class GurobiPersistent(GurobiDirectBase, PersistentSolverBase): _minimum_version = (7, 0, 0) CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) + self._solver_model = None + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} self._pyomo_model = None self._objective = None self._mutable_helpers = {} @@ -628,7 +335,10 @@ def __init__(self, **kwds): self._should_update_parameters = False def _clear(self): - super()._clear() + self._solver_model = None + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} self._pyomo_model = None self._objective = None self._mutable_helpers = {} @@ -649,15 +359,15 @@ def _create_solver_model(self, pyomo_model): solution_loader = GurobiPersistentSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, - linear_cons=self._linear_cons, - quadratic_cons=self._quadratic_cons, ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj + def _pyomo_gurobi_var_iter(self): + return self._pyomo_var_to_solver_var_map.items() + def release_license(self): self._clear() self.__class__.release_license() @@ -668,7 +378,28 @@ def solve(self, model, **kwds) -> Results: return res def _process_domain_and_bounds(self, var): - res = super()._process_domain_and_bounds(var) + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError(f'Unrecognized domain: {var.domain}') + if var.fixed: + lb = var.value + ub = lb + else: + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) if not is_constant(var._lb): mutable_lb = _MutableLowerBound( id(var), var.lower, self._pyomo_var_to_solver_var_map @@ -679,11 +410,26 @@ def _process_domain_and_bounds(self, var): id(var), var.upper, self._pyomo_var_to_solver_var_map ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) - return res + return lb, ub, vtype def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() - super()._add_variables(variables) + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ).values() + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var self._vars_added_since_update.update(variables) self._needs_updated = True @@ -720,6 +466,36 @@ def update(self): self._update_parameters([]) timer.stop('update') + def _get_expr_from_pyomo_repn(self, repn): + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiPersistent only supports linear and quadratic expressions: {repn}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] + new_expr = gurobipy.LinExpr(coef_list, vlist) + else: + new_expr = 0.0 + + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += value(coef) * gurobi_x * gurobi_y + + return new_expr + def _add_constraints(self, cons: List[ConstraintData]): self._invalidate_last_results() gurobi_expr_list = [] @@ -832,7 +608,32 @@ def _add_constraints(self, cons: List[ConstraintData]): def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._invalidate_last_results() - super()._add_sos_constraints(cons) + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con self._constraints_added_since_update.update(cons) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py index 30ddd7eca4b..31d90770aca 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py @@ -13,7 +13,7 @@ import pyomo.common.unittest as unittest from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.results import TerminationCondition, SolutionStatus -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from pyomo.core.base.constraint import Constraint from pyomo.environ import ( diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index 14eab91f09c..eeae8a7d96e 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,7 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiMINLPVisitor +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index f86ebd975c4..ab68aae046c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -40,7 +40,7 @@ ) from pyomo.gdp import Disjunction from pyomo.opt import WriterFactory -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import ( GurobiDirectMINLP, GurobiMINLPVisitor, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 92232118b54..5eaa0791e73 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -33,7 +33,6 @@ ) from pyomo.contrib.solver.solvers.gurobi import ( GurobiDirect, - GurobiDirectQuadratic, GurobiPersistent, GurobiDirectMINLP, ) @@ -55,7 +54,6 @@ all_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('gurobi_direct_minlp', GurobiDirectMINLP), ('ipopt', Ipopt), ('highs', Highs), @@ -64,7 +62,6 @@ mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('gurobi_direct_minlp', GurobiDirectMINLP), ('highs', Highs), ('knitro_direct', KnitroDirectSolver), @@ -77,7 +74,6 @@ qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_minlp', GurobiDirectMINLP), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('knitro_direct', KnitroDirectSolver), ] @@ -85,7 +81,6 @@ miqcqp_solvers = [ ('gurobi_direct_minlp', GurobiDirectMINLP), ('gurobi_persistent', GurobiPersistent), - ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('knitro_direct', KnitroDirectSolver), ] nl_solvers = [('ipopt', Ipopt)] From 5073ba0eb840e2fa7b5a65e90776239fb2e6ad58 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 22:26:23 -0600 Subject: [PATCH 76/97] update gurobi persistent to use observer --- pyomo/contrib/observer/model_observer.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 311 ++++++++++++------ .../solver/tests/solvers/test_solvers.py | 8 +- 3 files changed, 213 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 97910eb2bf1..8fa6b597cd2 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -910,7 +910,7 @@ def _update_variables(self, variables: Optional[Collection[VarData]] = None): reason = Reason.no_change if _fixed != fixed: reason |= Reason.fixed - elif _fixed and (value != _value): + elif (_fixed or fixed) and (value != _value): reason |= Reason.value if lb is not _lb or ub is not _ub: reason |= Reason.bounds diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3462f50b437..af4d845fbed 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -42,6 +42,7 @@ Observer, ModelChangeDetector, AutoUpdateConfig, + Reason, ) @@ -247,47 +248,6 @@ def var2(self): return self.var_map[self.v2id] -class _GurobiObserver(Observer): - def __init__(self, opt: GurobiPersistent) -> None: - self.opt = opt - - def add_variables(self, variables: List[VarData]): - self.opt._add_variables(variables) - - def add_parameters(self, params: List[ParamData]): - pass - - def add_constraints(self, cons: List[ConstraintData]): - self.opt._add_constraints(cons) - - def add_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._add_sos_constraints(cons) - - def add_objectives(self, objs: List[ObjectiveData]): - self.opt._add_objectives(objs) - - def remove_objectives(self, objs: List[ObjectiveData]): - self.opt._remove_objectives(objs) - - def remove_constraints(self, cons: List[ConstraintData]): - self.opt._remove_constraints(cons) - - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._remove_sos_constraints(cons) - - def remove_variables(self, variables: List[VarData]): - self.opt._remove_variables(variables) - - def remove_parameters(self, params: List[ParamData]): - pass - - def update_variables(self, variables: List[VarData]): - self.opt._update_variables(variables) - - def update_parameters(self, params: List[ParamData]): - self.opt._update_parameters(params) - - class GurobiPersistentConfig(GurobiConfig): def __init__( self, @@ -308,12 +268,15 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class GurobiPersistent(GurobiDirectBase, PersistentSolverBase): +class GurobiPersistent(GurobiDirectBase, PersistentSolverBase, Observer): _minimum_version = (7, 0, 0) CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) + # we actually want to only grab the license when + # set_instance is called + self._release_env_client() self._solver_model = None self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} @@ -329,13 +292,16 @@ def __init__(self, **kwds): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object: Optional[Results] = None - self._observer = None self._change_detector = None self._constraint_ndx = 0 - self._should_update_parameters = False def _clear(self): + release = False + if self._solver_model is not None: + release = True self._solver_model = None + if release: + self._release_env_client() self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} self._pyomo_sos_to_solver_sos_map = {} @@ -349,6 +315,7 @@ def _clear(self): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object = None + self._change_detector = None self._constraint_ndx = 0 def _create_solver_model(self, pyomo_model): @@ -370,7 +337,7 @@ def _pyomo_gurobi_var_iter(self): def release_license(self): self._clear() - self.__class__.release_license() + super().release_license() def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) @@ -418,7 +385,6 @@ def _add_variables(self, variables: List[VarData]): lbs = [] ubs = [] for ndx, var in enumerate(variables): - self._vars[id(var)] = var lb, ub, vtype = self._process_domain_and_bounds(var) vtypes.append(vtype) lbs.append(lb) @@ -429,7 +395,7 @@ def _add_variables(self, variables: List[VarData]): ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + self._pyomo_var_to_solver_var_map[pyomo_var] = gurobi_var self._vars_added_since_update.update(variables) self._needs_updated = True @@ -439,13 +405,13 @@ def set_instance(self, pyomo_model): else: timer = self.config.timer self._clear() + self._register_env_client() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) - self._observer = _GurobiObserver(self) timer.start('set_instance') self._change_detector = ModelChangeDetector( model=self._pyomo_model, - observers=[self._observer], + observers=[self], **dict(self.config.auto_updates), ) self._change_detector.config = self.config.auto_updates @@ -462,8 +428,6 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) - if self._should_update_parameters: - self._update_parameters([]) timer.stop('update') def _get_expr_from_pyomo_repn(self, repn): @@ -473,25 +437,25 @@ def _get_expr_from_pyomo_repn(self, repn): ) if len(repn.linear_vars) > 0: - missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - self._add_variables(missing_vars) + #missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + #self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] - vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] + vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 if len(repn.quadratic_vars) > 0: - missing_vars = {} - for x, y in repn.quadratic_vars: - for v in [x, y]: - vid = id(v) - if vid not in self._vars: - missing_vars[vid] = v - self._add_variables(list(missing_vars.values())) + # missing_vars = {} + # for x, y in repn.quadratic_vars: + # for v in [x, y]: + # vid = id(v) + # if vid not in self._vars: + # missing_vars[vid] = v + # self._add_variables(list(missing_vars.values())) for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + gurobi_x = self._pyomo_var_to_solver_var_map[x] + gurobi_y = self._pyomo_var_to_solver_var_map[y] new_expr += value(coef) * gurobi_x * gurobi_y return new_expr @@ -502,10 +466,6 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) - if len(repn.quadratic_vars) > 0: - self._quadratic_cons.add(con) - else: - self._linear_cons.add(con) gurobi_expr = self._get_expr_from_pyomo_repn(repn) mutable_constant = None if lb is None and ub is None: @@ -751,53 +711,164 @@ def _remove_variables(self, variables: List[VarData]): v_id = id(var) if var in self._vars_added_since_update: self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] + solver_var = self._pyomo_var_to_solver_var_map.pop(var) self._solver_model.remove(solver_var) - del self._pyomo_var_to_solver_var_map[v_id] - del self._vars[v_id] - self._mutable_bounds.pop(v_id, None) + self._mutable_bounds.pop((v_id, 'lb'), None) + self._mutable_bounds.pop((v_id, 'ub'), None) self._needs_updated = True - def _update_variables(self, variables: List[VarData]): + def _update_variables(self, variables: Mapping[VarData, Reason]): self._invalidate_last_results() - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds(var) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - if var.fixed: - self._should_update_parameters = True + new_vars = [] + old_vars = [] + mod_vars = [] + for v, reason in variables.items(): + if reason & Reason.added: + new_vars.append(v) + elif reason & Reason.removed: + old_vars.append(v) + else: + mod_vars.append(v) + + if new_vars: + self._add_variables(new_vars) + if old_vars: + self._remove_variables(old_vars) + + cons_to_reprocess = OrderedSet() + cons_to_update = OrderedSet() + reprocess_obj = False + update_obj = False + + for v in mod_vars: + reason = variables[v] + if reason & (Reason.bounds | Reason.domain | Reason.fixed | Reason.value): + var_id = id(v) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[v] + lb, ub, vtype = self._process_domain_and_bounds(v) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + if reason & Reason.fixed: + cons_to_reprocess.update(self._change_detector.get_constraints_impacted_by_var(v)) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + reprocess_obj = True + elif (reason & Reason.value) and v.fixed: + cons_to_update.update(self._change_detector.get_constraints_impacted_by_var(v)) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + + self._remove_constraints(cons_to_reprocess) + self._add_constraints(cons_to_reprocess) + cons_to_update -= cons_to_reprocess + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if reprocess_obj: + self._remove_objectives([self._objective]) + self._add_objectives([self._objective]) + elif update_obj: + self._mutable_objective_update() + self._needs_updated = True - def _update_parameters(self, params: List[ParamData]): + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): self._invalidate_last_results() - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.expr: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + self._needs_updated = True - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + self._invalidate_last_results() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.sos_items: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_sos_constraints(old_cons) + if new_cons: + self._add_sos_constraints(new_cons) + self._needs_updated = True + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + self._invalidate_last_results() + new_objs = [] + old_objs = [] + new_sense = [] + for obj, reason in objs.items(): + if reason & Reason.added: + new_objs.append(obj) + elif reason & Reason.removed: + old_objs.append(obj) + elif reason & Reason.expr: + old_objs.append(obj) + new_objs.append(obj) + elif reason & Reason.sense: + new_sense.append(obj) + + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + if new_sense: + assert len(new_sense) == 1 + obj = new_sense[0] + assert obj is self._objective + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._solver_model.ModelSense = sense + + def _update_quadratic_constraint(self, c: ConstraintData): + if c in self._mutable_quadratic_helpers: + if c in self._constraints_added_since_update: self._update_gurobi_model() + helper = self._mutable_quadratic_helpers[c] gurobi_con = helper.gurobi_con new_gurobi_expr = helper.get_updated_expression() new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) - self._pyomo_con_to_solver_con_map[con] = new_con - helper.pyomo_con = con - self._constraints_added_since_update.add(con) + self._pyomo_con_to_solver_con_map[c] = new_con + assert helper.pyomo_con is c + self._constraints_added_since_update.add(c) + def _mutable_objective_update(self): if self._mutable_objective is not None: new_gurobi_expr = self._mutable_objective.get_updated_expression() if new_gurobi_expr is not None: @@ -810,7 +881,43 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) - self._should_update_parameters = False + def _update_parameters(self, params: Mapping[ParamData, Reason]): + self._invalidate_last_results() + + cons_to_update = OrderedSet() + update_obj = False + vars_to_update = ComponentSet() + for p, reason in params.items(): + if reason & Reason.added: + continue + if reason & Reason.removed: + continue + if reason & Reason.value: + cons_to_update.update(self._change_detector.get_constraints_impacted_by_param(p)) + objs = self._change_detector.get_objectives_impacted_by_param(p) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + vars_to_update.update(self._change_detector.get_variables_impacted_by_param(p)) + + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if update_obj: + self._mutable_objective_update() + + for v in vars_to_update: + vid = id(v) + if (vid, 'lb') in self._mutable_bounds: + self._mutable_bounds[(vid, 'lb')][1].update() + if (vid, 'ub') in self._mutable_bounds: + self._mutable_bounds[(vid, 'ub')][1].update() + + self._needs_updated = True def _invalidate_last_results(self): if self._last_results_object is not None: @@ -1212,9 +1319,6 @@ def cbUseSolution(self): def reset(self): self._solver_model.reset() - def add_variables(self, variables): - self._change_detector.add_variables(variables) - def add_constraints(self, cons): self._change_detector.add_constraints(cons) @@ -1230,9 +1334,6 @@ def remove_constraints(self, cons): def remove_sos_constraints(self, cons): self._change_detector.remove_sos_constraints(cons) - def remove_variables(self, variables): - self._change_detector.remove_variables(variables) - def update_variables(self, variables): self._change_detector.update_variables(variables) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5eaa0791e73..76c7cb4be87 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,6 +30,8 @@ NoDualsError, NoReducedCostsError, NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, ) from pyomo.contrib.solver.solvers.gurobi import ( GurobiDirect, @@ -1089,10 +1091,12 @@ def test_results_infeasible( m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y <= m.x - 1) - with self.assertRaises(Exception): + with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) - opt.config.load_solutions = False opt.config.raise_exception_on_nonoptimal_result = False + with self.assertRaises(NoFeasibleSolutionError): + res = opt.solve(m) + opt.config.load_solutions = False res = opt.solve(m) self.assertNotEqual(res.solution_status, SolutionStatus.optimal) if isinstance(opt, Ipopt): From 5e280dc49c23cf0db3996bbb244cd01c928f94ad Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 23:14:18 -0600 Subject: [PATCH 77/97] gurobi refactor: bugs --- pyomo/contrib/solver/common/base.py | 32 ------------------- .../solver/solvers/gurobi/gurobi_direct.py | 9 +++++- .../solvers/gurobi/gurobi_direct_base.py | 16 ++++++++-- .../solvers/gurobi/gurobi_persistent.py | 22 +++---------- .../tests/solvers/test_gurobi_persistent.py | 7 ---- pyomo/contrib/solver/tests/unit/test_base.py | 12 ------- 6 files changed, 25 insertions(+), 73 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 63c1e97ffd6..9f6171b7773 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -321,22 +321,6 @@ def set_objective(self, obj: ObjectiveData): f"Derived class {self.__class__.__name__} failed to implement required method 'set_objective'." ) - def add_variables(self, variables: List[VarData]): - """ - Add variables to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_variables'." - ) - - def add_parameters(self, params: List[ParamData]): - """ - Add parameters to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_parameters'." - ) - def add_constraints(self, cons: List[ConstraintData]): """ Add constraints to the model. @@ -353,22 +337,6 @@ def add_block(self, block: BlockData): f"Derived class {self.__class__.__name__} failed to implement required method 'add_block'." ) - def remove_variables(self, variables: List[VarData]): - """ - Remove variables from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_variables'." - ) - - def remove_parameters(self, params: List[ParamData]): - """ - Remove parameters from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_parameters'." - ) - def remove_constraints(self, cons: List[ConstraintData]): """ Remove constraints from the model. diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5ee8ad54f0e..bd3388b7fc3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -114,7 +114,14 @@ def _create_solver_model(self, pyomo_model): self._gurobi_vars = x.tolist() var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) - con_map = dict(zip([i.constraint for i in repn.rows], A.tolist())) + con_map = {} + for row, gc in zip(repn.rows, A.tolist()): + pc = row.constraint + if pc in con_map: + # range constraint + con_map[pc] = (con_map[pc], gc) + else: + con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( solver_model=gurobi_model, var_map=var_map, con_map=con_map, ) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 506279ee37e..e156dbdca01 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -181,10 +181,20 @@ def _get_duals(solver_model, con_map, cons_to_load): duals = {} for c in cons_to_load: gurobi_con = con_map[c] - if gurobi_con in qcons: - duals[c] = gurobi_con.QCPi + if type(gurobi_con) is tuple: + # only linear range constraints are supported + gc1, gc2 = gurobi_con + d1 = gc1.Pi + d2 = gc2.Pi + if abs(d1) > abs(d2): + duals[c] = d1 + else: + duals[c] = d2 else: - duals[c] = gurobi_con.Pi + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi return duals diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index af4d845fbed..4950cd18919 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -437,8 +437,6 @@ def _get_expr_from_pyomo_repn(self, repn): ) if len(repn.linear_vars) > 0: - #missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] - #self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) @@ -446,13 +444,6 @@ def _get_expr_from_pyomo_repn(self, repn): new_expr = 0.0 if len(repn.quadratic_vars) > 0: - # missing_vars = {} - # for x, y in repn.quadratic_vars: - # for v in [x, y]: - # vid = id(v) - # if vid not in self._vars: - # missing_vars[vid] = v - # self._add_variables(list(missing_vars.values())) for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[x] gurobi_y = self._pyomo_var_to_solver_var_map[y] @@ -582,14 +573,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = { - id(v): v for v, w in con.get_items() if id(v) not in self._vars - } - self._add_variables(list(missing_vars.values())) - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v]) weights.append(w) gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) @@ -776,8 +761,9 @@ def _update_variables(self, variables: Mapping[VarData, Reason]): self._update_quadratic_constraint(c) if reprocess_obj: - self._remove_objectives([self._objective]) - self._add_objectives([self._objective]) + obj = self._objective + self._remove_objectives([obj]) + self._add_objectives([obj]) elif update_obj: self._mutable_objective_update() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 24b53a19f2b..98df7236f25 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -584,13 +584,6 @@ def test_basics(self): self.assertEqual(opt.get_model_attr('NumConstrs'), 1) self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) - m.z = pyo.Var() - opt.add_variables([m.z]) - self.assertEqual(opt.get_model_attr('NumVars'), 3) - opt.remove_variables([m.z]) - del m.z - self.assertEqual(opt.get_model_attr('NumVars'), 2) - def test_update1(self): m = pyo.ConcreteModel() m.x = pyo.Var() diff --git a/pyomo/contrib/solver/tests/unit/test_base.py b/pyomo/contrib/solver/tests/unit/test_base.py index 217b02b9999..08245d37f5e 100644 --- a/pyomo/contrib/solver/tests/unit/test_base.py +++ b/pyomo/contrib/solver/tests/unit/test_base.py @@ -74,15 +74,11 @@ def test_class_method_list(self): '_load_vars', 'add_block', 'add_constraints', - 'add_parameters', - 'add_variables', 'api_version', 'available', 'is_persistent', 'remove_block', 'remove_constraints', - 'remove_parameters', - 'remove_variables', 'set_instance', 'set_objective', 'solve', @@ -103,18 +99,10 @@ def test_init(self): self.assertEqual(instance.api_version(), SolverAPIVersion.V2) with self.assertRaises(NotImplementedError): self.assertEqual(instance.set_instance(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_constraints(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_block(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.remove_constraints(None), None) with self.assertRaises(NotImplementedError): From 8bff218837bb82ad6924ac59394df243b3f11645 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 1 Nov 2025 23:17:49 -0600 Subject: [PATCH 78/97] run black --- .../solver/solvers/gurobi/gurobi_direct.py | 8 +++-- .../solvers/gurobi/gurobi_direct_base.py | 8 ++--- .../solvers/gurobi/gurobi_direct_minlp.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 30 ++++++++++--------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index bd3388b7fc3..f17497c4901 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,7 +23,11 @@ IncompatibleModelError, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -from .gurobi_direct_base import GurobiDirectBase, gurobipy, GurobiDirectSolutionLoaderBase +from .gurobi_direct_base import ( + GurobiDirectBase, + gurobipy, + GurobiDirectSolutionLoaderBase, +) import logging @@ -123,7 +127,7 @@ def _create_solver_model(self, pyomo_model): else: con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, var_map=var_map, con_map=con_map, + solver_model=gurobi_model, var_map=var_map, con_map=con_map ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e156dbdca01..da0a109a6d4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -175,9 +175,9 @@ def _get_duals(solver_model, con_map, cons_to_load): if solver_model.IsMIP: # this will also return True for continuous, nonconvex models raise NoDualsError() - + qcons = set(solver_model.getQConstrs()) - + duals = {} for c in cons_to_load: gurobi_con = con_map[c] @@ -200,9 +200,7 @@ def _get_duals(solver_model, con_map, cons_to_load): class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, - ) -> None: + def __init__(self, solver_model, var_map, con_map) -> None: super().__init__() self._solver_model = solver_model self._var_map = var_map diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 1fdbf27c018..2544e0acd40 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -607,7 +607,7 @@ def _create_solver_model(self, pyomo_model): con_map = dict(zip(pyo_cons, grb_cons)) solution_loader = GurobiDirectSolutionLoader( - solver_model=grb_model, var_map=var_map, con_map=con_map, + solver_model=grb_model, var_map=var_map, con_map=con_map ) return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 4950cd18919..1b4b58f3c02 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -50,12 +50,8 @@ class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, - ) -> None: - super().__init__( - solver_model, var_map, con_map, - ) + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__(solver_model, var_map, con_map) self._valid = True def invalidate(self): @@ -410,9 +406,7 @@ def set_instance(self, pyomo_model): self._solver_model = gurobipy.Model(env=self.env()) timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, - observers=[self], - **dict(self.config.auto_updates), + model=self._pyomo_model, observers=[self], **dict(self.config.auto_updates) ) self._change_detector.config = self.config.auto_updates timer.stop('set_instance') @@ -737,14 +731,18 @@ def _update_variables(self, variables: Mapping[VarData, Reason]): gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) if reason & Reason.fixed: - cons_to_reprocess.update(self._change_detector.get_constraints_impacted_by_var(v)) + cons_to_reprocess.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) objs = self._change_detector.get_objectives_impacted_by_var(v) if objs: assert len(objs) == 1 assert objs[0] is self._objective reprocess_obj = True elif (reason & Reason.value) and v.fixed: - cons_to_update.update(self._change_detector.get_constraints_impacted_by_var(v)) + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) objs = self._change_detector.get_objectives_impacted_by_var(v) if objs: assert len(objs) == 1 @@ -879,14 +877,18 @@ def _update_parameters(self, params: Mapping[ParamData, Reason]): if reason & Reason.removed: continue if reason & Reason.value: - cons_to_update.update(self._change_detector.get_constraints_impacted_by_param(p)) + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_param(p) + ) objs = self._change_detector.get_objectives_impacted_by_param(p) if objs: assert len(objs) == 1 assert objs[0] is self._objective update_obj = True - vars_to_update.update(self._change_detector.get_variables_impacted_by_param(p)) - + vars_to_update.update( + self._change_detector.get_variables_impacted_by_param(p) + ) + for c in cons_to_update: if c in self._mutable_helpers: for i in self._mutable_helpers[c]: From d9dc14db49a38f8766fbcd40814c56ef1963ff49 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 5 Nov 2025 09:13:46 -0700 Subject: [PATCH 79/97] typo --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 2544e0acd40..50da9977c2b 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -598,7 +598,7 @@ def _create_solver_model(self, pyomo_model): writer = GurobiMINLPWriter() grb_model, var_map, pyo_obj, grb_cons, pyo_cons = writer.write( - model, symbolic_solver_labels=config.symbolic_solver_labels + pyomo_model, symbolic_solver_labels=self.config.symbolic_solver_labels ) timer.stop('compile_model') From 1c26ae321890c03672d3196fdc10514546ed771c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 5 Nov 2025 09:17:55 -0700 Subject: [PATCH 80/97] contrib.solvers: bug in gurobi refactor --- .../contrib/solver/solvers/gurobi/gurobi_direct_minlp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 50da9977c2b..593ee0c1daf 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -604,7 +604,13 @@ def _create_solver_model(self, pyomo_model): timer.stop('compile_model') self._var_map = var_map - con_map = dict(zip(pyo_cons, grb_cons)) + con_map = {} + for pc, gc in zip(pyo_cons, grb_cons): + if pc in con_map: + # range constraint + con_map[pc] = (con_map[pc], gc) + else: + con_map[pc] = gc solution_loader = GurobiDirectSolutionLoader( solver_model=grb_model, var_map=var_map, con_map=con_map From a4858caf86b50f5712181df8406944b14916c971 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 6 Nov 2025 10:54:43 -0700 Subject: [PATCH 81/97] contrib.solver: update tests --- pyomo/contrib/solver/solvers/ipopt.py | 1 + .../contrib/solver/tests/solvers/test_gurobi_minlp_walker.py | 4 +++- .../contrib/solver/tests/solvers/test_gurobi_minlp_writer.py | 2 ++ pyomo/contrib/solver/tests/solvers/test_solvers.py | 2 -- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 1bf1fdb7bf9..4ad9729f4a2 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -530,6 +530,7 @@ def solve(self, model, **kwds) -> Results: results.solver_version = self.version(config) if config.load_solutions: + logger.error(f'solution_status: {results.solution_status}') if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() results.solution_loader.load_vars() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index eeae8a7d96e..3c33f273121 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,7 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor, GurobiDirectMINLP from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) @@ -39,6 +39,8 @@ if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): + gurobipy_available = False class CommonTest(unittest.TestCase): diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index ab68aae046c..c114f3e12ca 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -51,6 +51,8 @@ gurobipy, gurobipy_available = attempt_import('gurobipy', minimum_version='12.0.0') if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): + gurobipy_available = False def make_model(): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 76c7cb4be87..21d80c247ce 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1094,8 +1094,6 @@ def test_results_infeasible( with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) opt.config.raise_exception_on_nonoptimal_result = False - with self.assertRaises(NoFeasibleSolutionError): - res = opt.solve(m) opt.config.load_solutions = False res = opt.solve(m) self.assertNotEqual(res.solution_status, SolutionStatus.optimal) From 122511b56759a002c5d9b022ff5b4fe132bf7419 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 7 Nov 2025 10:20:12 -0700 Subject: [PATCH 82/97] run black --- .../solver/tests/solvers/test_gurobi_minlp_walker.py | 6 +++++- .../solver/tests/solvers/test_gurobi_minlp_writer.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index 3c33f273121..03350ed794d 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,10 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor, GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import ( + GurobiMINLPVisitor, + GurobiDirectMINLP, +) from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) @@ -39,6 +42,7 @@ if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): gurobipy_available = False diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index c114f3e12ca..f2b37b88e22 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -51,6 +51,7 @@ gurobipy, gurobipy_available = attempt_import('gurobipy', minimum_version='12.0.0') if gurobipy_available: from gurobipy import GRB + if not GurobiDirectMINLP().available(): gurobipy_available = False From 71963f14569f15f8ffb562603e006c343aba4426 Mon Sep 17 00:00:00 2001 From: Emma Johnson <12833636+emma58@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:31:18 -0700 Subject: [PATCH 83/97] Changing the config option name for 'use_mipstart' to be 'warmstart_discrete_vars' --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index da0a109a6d4..3e362cff9fa 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -66,8 +66,8 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.use_mipstart: bool = self.declare( - 'use_mipstart', + self.warmstart_discrete_vars: bool = self.declare( + 'warmstart_discrete_vars', ConfigValue( default=False, domain=bool, @@ -411,7 +411,7 @@ def solve(self, model, **kwds) -> Results: if config.abs_gap is not None: gurobi_model.setParam('MIPGapAbs', config.abs_gap) - if config.use_mipstart: + if config.warmstart_discrete_vars: self._mipstart() for key, option in options.items(): From 87128848750bdcc6d453fcb566d44c1482c571af Mon Sep 17 00:00:00 2001 From: Emma Johnson <12833636+emma58@users.noreply.github.com> Date: Fri, 7 Nov 2025 16:31:39 -0700 Subject: [PATCH 84/97] Adding tests for Gurobi warmstarts in all the interfaces --- .../tests/solvers/test_gurobi_warm_start.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py new file mode 100644 index 00000000000..93ef0eab347 --- /dev/null +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py @@ -0,0 +1,113 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +from pyomo.common.log import LoggingIntercept +import pyomo.common.unittest as unittest +from pyomo.contrib.solver.common.factory import SolverFactory + +from pyomo.environ import ( + ConcreteModel, + Var, + Constraint, + value, + Binary, + NonNegativeReals, + Objective, + Set, +) + +gurobi_direct = SolverFactory('gurobi_direct') +gurobi_direct_minlp = SolverFactory('gurobi_direct_minlp') +gurobi_persistent = SolverFactory('gurobi_persistent') + + +class TestGurobiWarmStart(unittest.TestCase): + def make_model(self): + m = ConcreteModel() + m.S = Set(initialize=[1, 2, 3, 4, 5]) + m.y = Var(m.S, domain=Binary) + m.x = Var(m.S, domain=NonNegativeReals) + m.obj = Objective(expr=sum(m.x[i] for i in m.S)) + + @m.Constraint(m.S) + def cons(m, i): + if i % 2 == 0: + return m.x[i] + i * m.y[i] >= 3 * i + else: + return m.x[i] - i * m.y[i] >= 3 * i + + # define a suboptimal MIP start + for i in m.S: + m.y[i] = 1 + # objective will be 4 + 4 + 12 + 8 + 20 = 48 + + return m + + def check_optimal_soln(self, m): + # check that we got the optimal solution: + # y[1] = 0, x[1] = 3 + # y[2] = 1, x[2] = 4 + # y[3] = 0, x[3] = 9 + # y[4] = 1, x[4] = 8 + # y[5] = 0, x[5] = 15 + x = {1: 3, 2: 4, 3: 9, 4: 8, 5: 15} + self.assertEqual(value(m.obj), 39) + for i in m.S: + if i % 2 == 0: + self.assertEqual(value(m.y[i]), 1) + else: + self.assertEqual(value(m.y[i]), 0) + self.assertEqual(value(m.x[i]), x[i]) + + @unittest.skipUnless(gurobi_direct.available(), "needs Gurobi Direct interface") + def test_gurobi_direct_warm_start(self): + m = self.make_model() + + gurobi_direct.config.warmstart_discrete_vars = True + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_direct.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) + + @unittest.skipUnless( + gurobi_direct_minlp.available(), "needs Gurobi Direct MINLP interface" + ) + def test_gurobi_minlp_warmstart(self): + m = self.make_model() + + gurobi_direct_minlp.config.warmstart_discrete_vars = True + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_direct_minlp.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) + + @unittest.skipUnless( + gurobi_persistent.available(), "needs Gurobi persistent interface" + ) + def test_gurobi_persistent_warmstart(self): + m = self.make_model() + + gurobi_persistent.config.warmstart_discrete_vars = True + gurobi_persistent.set_instance(m) + logger = logging.getLogger('tee') + with LoggingIntercept(module='tee', level=logging.INFO) as LOG: + gurobi_persistent.solve(m, tee=logger) + self.assertIn( + "User MIP start produced solution with objective 48", LOG.getvalue() + ) + self.check_optimal_soln(m) From 7069f898c7e2940caa7f6729400dbf130644fc13 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 11 Nov 2025 15:04:43 -0700 Subject: [PATCH 85/97] revert modification to ipopt interface --- pyomo/contrib/solver/solvers/ipopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 4ad9729f4a2..1bf1fdb7bf9 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -530,7 +530,6 @@ def solve(self, model, **kwds) -> Results: results.solver_version = self.version(config) if config.load_solutions: - logger.error(f'solution_status: {results.solution_status}') if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() results.solution_loader.load_vars() From 9bfad144fc7fdf4565577ae0a99d36bfa68af360 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 16 Nov 2025 07:12:47 -0700 Subject: [PATCH 86/97] contrib.solver.gurobi: better handling of temporary config options --- pyomo/contrib/solver/plugins.py | 2 +- .../solver/solvers/gurobi/gurobi_direct.py | 4 +- .../solvers/gurobi/gurobi_direct_base.py | 43 ++++++------------- .../solvers/gurobi/gurobi_direct_minlp.py | 6 +-- .../solvers/gurobi/gurobi_persistent.py | 25 +++++------ 5 files changed, 32 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 430207a736c..3bc1340544f 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -39,7 +39,7 @@ def load(): doc='Direct interface to Gurobi accommodating general MINLP', )(GurobiDirectMINLP) SolverFactory.register( - name="highs", legacy_name="highs_v2", doc="Persistent interface to HiGHS" + name="highs", legacy_name="highs", doc="Persistent interface to HiGHS" )(Highs) SolverFactory.register( name="knitro_direct", diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f17497c4901..da7442d738f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -59,8 +59,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars) - def _create_solver_model(self, pyomo_model): - timer = self.config.timer + def _create_solver_model(self, pyomo_model, config): + timer = config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 3e362cff9fa..587362b658b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -348,6 +348,8 @@ def _check_license(self): model.dispose() def version(self): + if not gurobipy_available: + return None version = ( gurobipy.GRB.VERSION_MAJOR, gurobipy.GRB.VERSION_MINOR, @@ -355,7 +357,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -370,21 +372,10 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - orig_config = self.config orig_cwd = os.getcwd() try: config = self.config(value=kwds, preserve_implicit=True) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = config - object.__setattr__(self, 'config', config) - - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) if config.timer is None: config.timer = HierarchicalTimer() timer = config.timer @@ -396,7 +387,8 @@ def solve(self, model, **kwds) -> Results: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): gurobi_model, solution_loader, has_obj = self._create_solver_model( - model + model, + config, ) options = config.solver_options @@ -421,17 +413,12 @@ def solve(self, model, **kwds) -> Results: gurobi_model.optimize(self._callback) timer.stop('optimize') - res = self._postsolve( - grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj + res = self._populate_results( + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj, config=config ) finally: os.chdir(orig_cwd) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = orig_config - object.__setattr__(self, 'config', orig_config) - self.config = orig_config - res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -461,7 +448,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, solution_loader, has_obj): + def _populate_results(self, grb_model, solution_loader, has_obj, config): status = grb_model.Status results = Results() @@ -483,7 +470,7 @@ def _postsolve(self, grb_model, solution_loader, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and self.config.raise_exception_on_nonoptimal_result + and config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -510,19 +497,15 @@ def _postsolve(self, grb_model, solution_loader, has_obj): results.extra_info.BarIterCount = grb_model.getAttr('BarIterCount') results.extra_info.NodeCount = grb_model.getAttr('NodeCount') - self.config.timer.start('load solution') - if self.config.load_solutions: + config.timer.start('load solution') + if config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - self.config.timer.stop('load solution') + config.timer.stop('load solution') - # self.config gets copied a the beginning of - # solve and restored at the end, so modifying - # results.solver_config will not actually - # modify self.config - results.solver_config = self.config + results.solver_config = config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 593ee0c1daf..56ca484b35b 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -592,13 +592,13 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return self._var_map.items() - def _create_solver_model(self, pyomo_model): - timer = self.config.timer + def _create_solver_model(self, pyomo_model, config): + timer = config.timer timer.start('compile_model') writer = GurobiMINLPWriter() grb_model, var_map, pyo_obj, grb_cons, pyo_cons = writer.write( - pyomo_model, symbolic_solver_labels=self.config.symbolic_solver_labels + pyomo_model, symbolic_solver_labels=config.symbolic_solver_labels ) timer.stop('compile_model') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 1b4b58f3c02..d467cfc3d53 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -314,11 +314,11 @@ def _clear(self): self._change_detector = None self._constraint_ndx = 0 - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): if pyomo_model is self._pyomo_model: - self.update() + self.update(**config) else: - self.set_instance(pyomo_model) + self.set_instance(pyomo_model, **config) solution_loader = GurobiPersistentSolutionLoader( solver_model=self._solver_model, @@ -395,33 +395,34 @@ def _add_variables(self, variables: List[VarData]): self._vars_added_since_update.update(variables) self._needs_updated = True - def set_instance(self, pyomo_model): - if self.config.timer is None: + def set_instance(self, pyomo_model, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer self._clear() self._register_env_client() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, observers=[self], **dict(self.config.auto_updates) + model=self._pyomo_model, observers=[self], **config.auto_updates ) - self._change_detector.config = self.config.auto_updates timer.stop('set_instance') - def update(self): - if self.config.timer is None: + def update(self, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer if self._pyomo_model is None: raise RuntimeError('must call set_instance or solve before update') timer.start('update') if self._needs_updated: self._update_gurobi_model() - self._change_detector.update(timer=timer) + self._change_detector.update(timer=timer, **config.auto_updates) timer.stop('update') def _get_expr_from_pyomo_repn(self, repn): From 2ab061a9ca3da1f075dc90f6859c32d1eefe374f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 16 Nov 2025 07:16:46 -0700 Subject: [PATCH 87/97] fix error --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 587362b658b..033d183a7e0 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -154,7 +154,7 @@ def _get_reduced_costs(solver_model, var_map, vars_to_load): raise NoReducedCostsError() if solver_model.IsMIP: # this will also return True for continuous, nonconvex models - raise NoDualsError() + raise NoReducedCostsError() gurobi_vars_to_load = [var_map[v] for v in vars_to_load] vals = solver_model.getAttr("Rc", gurobi_vars_to_load) From f3d7f3a959fa8b5d68d4cc3139931176cdaec3fb Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:08:33 -0700 Subject: [PATCH 88/97] contrib.solvers.gurobi: reworking the solution loader --- .../contrib/solver/common/solution_loader.py | 4 +- .../solver/solvers/gurobi/gurobi_direct.py | 24 +- .../solvers/gurobi/gurobi_direct_base.py | 337 ++++++++++-------- .../solvers/gurobi/gurobi_direct_minlp.py | 34 +- 4 files changed, 240 insertions(+), 159 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 911d8bee50d..c83874397d4 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,7 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, NoReturn +from typing import Sequence, Dict, Optional, Mapping from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -23,7 +23,7 @@ class SolutionLoaderBase: Intent of this class and its children is to load the solution back into the model. """ - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> None: """ Load the solution of the primal variables into the value attribute of the variables. diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index da7442d738f..74c042e6485 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -35,6 +35,21 @@ class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): + def __init__(self, solver_model, pyomo_vars, gurobi_vars, con_map) -> None: + super().__init__(solver_model) + self._pyomo_vars = pyomo_vars + self._gurobi_vars = gurobi_vars + self._con_map = con_map + + def _var_pair_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars) + + def _get_var_map(self): + return ComponentMap(self._var_pair_iter()) + + def _get_con_map(self): + return self._con_map + def __del__(self): super().__del__() if python_is_shutting_down(): @@ -115,9 +130,12 @@ def _create_solver_model(self, pyomo_model, config): timer.stop('transfer_model') self._pyomo_vars = repn.columns + timer.start('tolist') self._gurobi_vars = x.tolist() + timer.stop('tolist') - var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) + timer.start('create maps') + timer.start('con map') con_map = {} for row, gc in zip(repn.rows, A.tolist()): pc = row.constraint @@ -126,8 +144,10 @@ def _create_solver_model(self, pyomo_model, config): con_map[pc] = (con_map[pc], gc) else: con_map[pc] = gc + timer.stop('con map') + timer.stop('create maps') solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, var_map=var_map, con_map=con_map + solver_model=gurobi_model, pyomo_vars=self._pyomo_vars, gurobi_vars=self._gurobi_vars, con_map=con_map ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 033d183a7e0..7b15582f48d 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -42,6 +42,7 @@ TerminationCondition, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import time logger = logging.getLogger(__name__) @@ -77,135 +78,24 @@ def __init__( ) -def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - if ( - solver_model.getAttr('NumIntVars') == 0 - and solver_model.getAttr('NumBinVars') == 0 - ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') - original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] - solver_model.setParam('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap(zip(vars_to_load, vals)) - solver_model.setParam('SolutionNumber', original_solution_number) - return res - - -def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - for v, val in _get_primals( - solver_model=solver_model, - var_map=var_map, - vars_to_load=vars_to_load, - solution_number=solution_number, - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - -def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): - """ - solver_model: gurobipy.Model - var_map: Mapping[Vardata, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - solution_number: int - """ - if solver_model.SolCount == 0: - raise NoSolutionError() - - if solution_number != 0: - return _load_suboptimal_mip_solution( - solver_model=solver_model, - var_map=var_map, - vars_to_load=vars_to_load, - solution_number=solution_number, - ) - - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap(zip(vars_to_load, vals)) - return res - - -def _get_reduced_costs(solver_model, var_map, vars_to_load): - """ - solver_model: gurobipy.Model - var_map: Mapping[VarData, gurobipy.Var] - Maps the pyomo variable to the gurobipy variable - vars_to_load: List[VarData] - """ - if solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - if solver_model.IsMIP: - # this will also return True for continuous, nonconvex models - raise NoReducedCostsError() - - gurobi_vars_to_load = [var_map[v] for v in vars_to_load] - vals = solver_model.getAttr("Rc", gurobi_vars_to_load) - - res = ComponentMap(zip(vars_to_load, vals)) - return res - - -def _get_duals(solver_model, con_map, cons_to_load): - """ - solver_model: gurobipy.Model - con_map: Dict[ConstraintData, gurobipy.Constr] - Maps the pyomo constraint to the gurobipy constraint - cons_to_load: List[ConstraintData] - """ - if solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - if solver_model.IsMIP: - # this will also return True for continuous, nonconvex models - raise NoDualsError() - - qcons = set(solver_model.getQConstrs()) - - duals = {} - for c in cons_to_load: - gurobi_con = con_map[c] - if type(gurobi_con) is tuple: - # only linear range constraints are supported - gc1, gc2 = gurobi_con - d1 = gc1.Pi - d2 = gc2.Pi - if abs(d1) > abs(d2): - duals[c] = d1 - else: - duals[c] = d2 - else: - if gurobi_con in qcons: - duals[c] = gurobi_con.QCPi - else: - duals[c] = gurobi_con.Pi - - return duals - - class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): - def __init__(self, solver_model, var_map, con_map) -> None: + def __init__(self, solver_model) -> None: super().__init__() self._solver_model = solver_model - self._var_map = var_map - self._con_map = con_map GurobiDirectBase._register_env_client() + self.timer = HierarchicalTimer() + + def _var_pair_iter(self): + """ + Should iterate over pairs of (pyomo var, gurobipy var) + """ + raise NotImplementedError('should be implemented by derived classes') + + def _get_var_map(self): + raise NotImplementedError('should be implemented by derived classes') + + def _get_con_map(self): + raise NotImplementedError('should be implemented by derived classes') def __del__(self): # Release the gurobi license if this is the last reference to @@ -213,51 +103,194 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() + def _load_all_vars_solution_0(self): + self.timer.start('vars to load') + gvars = [j for i, j in self._var_pair_iter()] + self.timer.stop('vars to load') + self.timer.start('getAttr') + vals = self._solver_model.getAttr("X", gvars) + self.timer.stop('getAttr') + self.timer.start('set_value') + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self.timer.stop('set_value') + + def _load_subset_vars_solution_0(self, vars_to_load): + var_map = self._get_var_map() + self.timer.start('vars_to_load') + gvars = [var_map[i] for i in vars_to_load] + self.timer.stop('vars_to_load') + self.timer.start('getAttr') + vals = self._solver_model.getAttr("X", gvars) + self.timer.stop('getAttr') + self.timer.start('set_value') + for pv, val in zip(vars_to_load, vals): + pv.set_value(val, skip_validation=True) + self.timer.stop('set_value') + + def _load_all_vars_solution_N(self, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Xn", gvars) + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self._solver_model.setParam('SolutionNumber', original_solution_number) + + def _load_subset_vars_solution_N(self, vars_to_load, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Xn", gvars) + for (pv, _), val in zip(self._var_pair_iter(), vals): + pv.set_value(val, skip_validation=True) + self._solver_model.setParam('SolutionNumber', original_solution_number) + + def _get_all_vars_solution_0(self): + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("X", gvars) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_subset_vars_solution_0(self, vars_to_load): + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("X", gvars) + return ComponentMap(zip(vars_to_load, vals)) + + def _get_all_vars_solution_N(self, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Xn", gvars) + self._solver_model.setParam('SolutionNumber', original_solution_number) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_subset_vars_solution_N(self, vars_to_load, solution_number): + assert solution_number != 0 + if ( + self._solver_model.getAttr('NumIntVars') == 0 + and self._solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] + self._solver_model.setParam('SolutionNumber', solution_number) + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Xn", gvars) + self._solver_model.setParam('SolutionNumber', original_solution_number) + return ComponentMap(zip(vars_to_load, vals)) + def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: + if self._solver_model.SolCount == 0: + raise NoSolutionError() if vars_to_load is None: - vars_to_load = self._var_map - _load_vars( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) + if solution_id == 0: + self._load_all_vars_solution_0() + else: + self._load_all_vars_solution_N(solution_number=solution_id) + else: + if solution_id == 0: + self._load_subset_vars_solution_0(vars_to_load=vars_to_load) + else: + self._load_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + StaleFlagManager.mark_all_as_stale(delayed=True) def get_primals( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: + if self._solver_model.SolCount == 0: + raise NoSolutionError() if vars_to_load is None: - vars_to_load = self._var_map - return _get_primals( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - solution_number=solution_id, - ) + if solution_id == 0: + res = self._get_all_vars_solution_0() + else: + res = self._get_all_vars_solution_N(solution_number=solution_id) + else: + if solution_id == 0: + res = self._get_subset_vars_solution_0(vars_to_load=vars_to_load) + else: + res = self._get_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + return res + + def _get_rc_all_vars(self): + gvars = [j for i, j in self._var_pair_iter()] + vals = self._solver_model.getAttr("Rc", gvars) + return ComponentMap((i[0], val) for i, val in zip(self._var_pair_iter(), vals)) + + def _get_rc_subset_vars(self, vars_to_load): + var_map = self._get_var_map() + gvars = [var_map[i] for i in vars_to_load] + vals = self._solver_model.getAttr("Rc", gvars) + return ComponentMap(zip(vars_to_load, vals)) def get_reduced_costs( self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: + if self._solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + if self._solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoReducedCostsError() if vars_to_load is None: - vars_to_load = self._var_map - return _get_reduced_costs( - solver_model=self._solver_model, - var_map=self._var_map, - vars_to_load=vars_to_load, - ) + res = self._get_rc_all_vars() + else: + res = self._get_rc_subset_vars(vars_to_load=vars_to_load) + return res def get_duals( self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: + if self._solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + if self._solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + qcons = set(self._solver_model.getQConstrs()) + con_map = self._get_con_map() if cons_to_load is None: - cons_to_load = self._con_map - return _get_duals( - solver_model=self._solver_model, - con_map=self._con_map, - cons_to_load=cons_to_load, - ) + cons_to_load = con_map.keys() + + duals = {} + for c in cons_to_load: + gurobi_con = con_map[c] + if type(gurobi_con) is tuple: + # only linear range constraints are supported + gc1, gc2 = gurobi_con + d1 = gc1.Pi + d2 = gc2.Pi + if abs(d1) > abs(d2): + duals[c] = d1 + else: + duals[c] = d2 + else: + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi + + return duals class GurobiDirectBase(SolverBase): @@ -500,7 +533,7 @@ def _populate_results(self, grb_model, solution_loader, has_obj, config): config.timer.start('load solution') if config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars() + results.solution_loader.load_vars(timer=config.timer) else: raise NoFeasibleSolutionError() config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 56ca484b35b..217b489d635 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -19,13 +19,13 @@ from pyomo.common.config import ConfigDict, ConfigValue from pyomo.common.errors import InvalidValueError from pyomo.common.numeric_types import native_complex_types +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.util import NoSolutionError -from .gurobi_direct_base import GurobiDirectBase -from .gurobi_direct import GurobiDirectSolutionLoader +from .gurobi_direct_base import GurobiDirectBase, GurobiDirectSolutionLoaderBase from pyomo.core.base import ( Binary, @@ -577,6 +577,34 @@ def write(self, model, **options): return grb_model, visitor.var_map, pyo_obj, grb_cons, pyo_cons +class GurobiDirectMINLPSolutionLoader(GurobiDirectSolutionLoaderBase): + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__(solver_model) + self._var_map = var_map + self._con_map = con_map + + def _var_pair_iter(self): + return self._var_map.items() + + def _get_var_map(self): + return self._var_map + + def _get_con_map(self): + return self._con_map + + def __del__(self): + super().__del__() + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._var_map = None + self._con_map = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + + @SolverFactory.register( 'gurobi_direct_minlp', doc='Direct interface to Gurobi version 12 and up ' @@ -612,7 +640,7 @@ def _create_solver_model(self, pyomo_model, config): else: con_map[pc] = gc - solution_loader = GurobiDirectSolutionLoader( + solution_loader = GurobiDirectMINLPSolutionLoader( solver_model=grb_model, var_map=var_map, con_map=con_map ) From d08d9932c4874fda00bf5c5a2dc44c092915e3cf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:25:06 -0700 Subject: [PATCH 89/97] contrib.solvers.gurobi: reworking the solution loader --- .../solver/solvers/gurobi/gurobi_direct_base.py | 2 +- .../solver/solvers/gurobi/gurobi_persistent.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 7b15582f48d..05848e1a9cc 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -533,7 +533,7 @@ def _populate_results(self, grb_model, solution_loader, has_obj, config): config.timer.start('load solution') if config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars(timer=config.timer) + results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index d467cfc3d53..a9e5f095b3b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -51,9 +51,20 @@ class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): def __init__(self, solver_model, var_map, con_map) -> None: - super().__init__(solver_model, var_map, con_map) + super().__init__(solver_model) + self._var_map = var_map + self._con_map = con_map self._valid = True + def _var_pair_iter(self): + return self._var_map.items() + + def _get_var_map(self): + return self._var_map + + def _get_con_map(self): + return self._con_map + def invalidate(self): self._valid = False From 3ee4e995dead895f249db1033351bb1fd966062e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 14:25:32 -0700 Subject: [PATCH 90/97] run black --- .../solver/solvers/gurobi/gurobi_direct.py | 5 ++- .../solvers/gurobi/gurobi_direct_base.py | 36 +++++++++++++------ .../solvers/gurobi/gurobi_persistent.py | 4 +-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 74c042e6485..f7547b582f1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -147,7 +147,10 @@ def _create_solver_model(self, pyomo_model, config): timer.stop('con map') timer.stop('create maps') solution_loader = GurobiDirectSolutionLoader( - solver_model=gurobi_model, pyomo_vars=self._pyomo_vars, gurobi_vars=self._gurobi_vars, con_map=con_map + solver_model=gurobi_model, + pyomo_vars=self._pyomo_vars, + gurobi_vars=self._gurobi_vars, + con_map=con_map, ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 05848e1a9cc..acd881dd0c4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -90,10 +90,10 @@ def _var_pair_iter(self): Should iterate over pairs of (pyomo var, gurobipy var) """ raise NotImplementedError('should be implemented by derived classes') - + def _get_var_map(self): raise NotImplementedError('should be implemented by derived classes') - + def _get_con_map(self): raise NotImplementedError('should be implemented by derived classes') @@ -134,7 +134,9 @@ def _load_all_vars_solution_N(self, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) gvars = [j for i, j in self._var_pair_iter()] @@ -149,7 +151,9 @@ def _load_subset_vars_solution_N(self, vars_to_load, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) var_map = self._get_var_map() @@ -176,7 +180,9 @@ def _get_all_vars_solution_N(self, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) gvars = [j for i, j in self._var_pair_iter()] @@ -190,7 +196,9 @@ def _get_subset_vars_solution_N(self, vars_to_load, solution_number): self._solver_model.getAttr('NumIntVars') == 0 and self._solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) original_solution_number = self._solver_model.getParamInfo('SolutionNumber')[2] self._solver_model.setParam('SolutionNumber', solution_number) var_map = self._get_var_map() @@ -213,7 +221,9 @@ def load_vars( if solution_id == 0: self._load_subset_vars_solution_0(vars_to_load=vars_to_load) else: - self._load_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + self._load_subset_vars_solution_N( + vars_to_load=vars_to_load, solution_number=solution_id + ) StaleFlagManager.mark_all_as_stale(delayed=True) def get_primals( @@ -230,7 +240,9 @@ def get_primals( if solution_id == 0: res = self._get_subset_vars_solution_0(vars_to_load=vars_to_load) else: - res = self._get_subset_vars_solution_N(vars_to_load=vars_to_load, solution_number=solution_id) + res = self._get_subset_vars_solution_N( + vars_to_load=vars_to_load, solution_number=solution_id + ) return res def _get_rc_all_vars(self): @@ -420,8 +432,7 @@ def solve(self, model, **kwds) -> Results: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): gurobi_model, solution_loader, has_obj = self._create_solver_model( - model, - config, + model, config ) options = config.solver_options @@ -447,7 +458,10 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._populate_results( - grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj, config=config + grb_model=gurobi_model, + solution_loader=solution_loader, + has_obj=has_obj, + config=config, ) finally: os.chdir(orig_cwd) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index a9e5f095b3b..76479c1c3c4 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,10 +58,10 @@ def __init__(self, solver_model, var_map, con_map) -> None: def _var_pair_iter(self): return self._var_map.items() - + def _get_var_map(self): return self._var_map - + def _get_con_map(self): return self._con_map From b9ca201a3d0c392fdd549951c528861f6afee9d2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 18 Nov 2025 16:32:54 -0700 Subject: [PATCH 91/97] remove some timing statements --- .../solver/solvers/gurobi/gurobi_direct_base.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index acd881dd0c4..81c6a72c199 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -83,7 +83,6 @@ def __init__(self, solver_model) -> None: super().__init__() self._solver_model = solver_model GurobiDirectBase._register_env_client() - self.timer = HierarchicalTimer() def _var_pair_iter(self): """ @@ -104,29 +103,17 @@ def __del__(self): GurobiDirectBase._release_env_client() def _load_all_vars_solution_0(self): - self.timer.start('vars to load') gvars = [j for i, j in self._var_pair_iter()] - self.timer.stop('vars to load') - self.timer.start('getAttr') vals = self._solver_model.getAttr("X", gvars) - self.timer.stop('getAttr') - self.timer.start('set_value') for (pv, _), val in zip(self._var_pair_iter(), vals): pv.set_value(val, skip_validation=True) - self.timer.stop('set_value') def _load_subset_vars_solution_0(self, vars_to_load): var_map = self._get_var_map() - self.timer.start('vars_to_load') gvars = [var_map[i] for i in vars_to_load] - self.timer.stop('vars_to_load') - self.timer.start('getAttr') vals = self._solver_model.getAttr("X", gvars) - self.timer.stop('getAttr') - self.timer.start('set_value') for pv, val in zip(vars_to_load, vals): pv.set_value(val, skip_validation=True) - self.timer.stop('set_value') def _load_all_vars_solution_N(self, solution_number): assert solution_number != 0 From 52d91f5c7eac785e81de12880c39c930763d34c9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 12 Dec 2025 04:43:00 -0700 Subject: [PATCH 92/97] use absolute paths for imports --- pyomo/contrib/solver/solvers/gurobi/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py index 0809846ebc3..bc2245d2a0e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/__init__.py +++ b/pyomo/contrib/solver/solvers/gurobi/__init__.py @@ -1,3 +1,3 @@ -from .gurobi_direct import GurobiDirect -from .gurobi_persistent import GurobiPersistent -from .gurobi_direct_minlp import GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP From c3f2d4821083efec9716eea7010320a42706700c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 12 Dec 2025 09:43:26 -0700 Subject: [PATCH 93/97] solution loader updates --- pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py | 6 +++--- pyomo/contrib/solver/tests/solvers/test_sol_reader.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 217b489d635..514c2b91ad7 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -578,8 +578,8 @@ def write(self, model, **options): class GurobiDirectMINLPSolutionLoader(GurobiDirectSolutionLoaderBase): - def __init__(self, solver_model, var_map, con_map) -> None: - super().__init__(solver_model) + def __init__(self, solver_model, pyomo_model, var_map, con_map) -> None: + super().__init__(solver_model, pyomo_model) self._var_map = var_map self._con_map = con_map @@ -641,7 +641,7 @@ def _create_solver_model(self, pyomo_model, config): con_map[pc] = gc solution_loader = GurobiDirectMINLPSolutionLoader( - solver_model=grb_model, var_map=var_map, con_map=con_map + solver_model=grb_model, pyomo_model=pyomo_model, var_map=var_map, con_map=con_map ) return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py index 62d77341f65..687f0acba67 100644 --- a/pyomo/contrib/solver/tests/solvers/test_sol_reader.py +++ b/pyomo/contrib/solver/tests/solvers/test_sol_reader.py @@ -84,7 +84,7 @@ def test_get_duals_no_objective_returns_zeros(self): # solver returned some (non-zero) duals, but we should zero them out sol_data = self._FakeSolData(duals=[123.0, -7.5]) - loader = SolSolutionLoader(sol_data, nl_info) + loader = SolSolutionLoader(sol_data, nl_info, m) duals = loader.get_duals() self.assertEqual(duals[m.c1], 0.0) self.assertEqual(duals[m.c2], 0.0) From ba4b29c0f51ff58523b95d28658fd777765288db Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 12 Dec 2025 09:45:24 -0700 Subject: [PATCH 94/97] run black --- .../contrib/solver/solvers/gurobi/gurobi_direct.py | 4 +++- .../solver/solvers/gurobi/gurobi_direct_base.py | 13 ++++++------- .../solver/solvers/gurobi/gurobi_direct_minlp.py | 5 ++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 410f44414e4..84f0ad7868e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -36,7 +36,9 @@ class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): - def __init__(self, solver_model, pyomo_model, pyomo_vars, gurobi_vars, con_map) -> None: + def __init__( + self, solver_model, pyomo_model, pyomo_vars, gurobi_vars, con_map + ) -> None: super().__init__(solver_model, pyomo_model) self._pyomo_vars = pyomo_vars self._gurobi_vars = gurobi_vars diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e2f98f9e942..bc710b5bff2 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -41,7 +41,10 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) import time @@ -255,9 +258,7 @@ def _get_rc_subset_vars(self, vars_to_load): return ComponentMap(zip(vars_to_load, vals)) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if solution_id is not None and solution_id != 0: raise NoReducedCostsError('Can only get reduced costs for solution_id = 0') @@ -273,9 +274,7 @@ def get_reduced_costs( return res def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if solution_id is not None and solution_id != 0: raise NoDualsError('Can only get duals for solution_id = 0') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index 514c2b91ad7..a048e4fb4aa 100755 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -641,7 +641,10 @@ def _create_solver_model(self, pyomo_model, config): con_map[pc] = gc solution_loader = GurobiDirectMINLPSolutionLoader( - solver_model=grb_model, pyomo_model=pyomo_model, var_map=var_map, con_map=con_map + solver_model=grb_model, + pyomo_model=pyomo_model, + var_map=var_map, + con_map=con_map, ) return grb_model, solution_loader, bool(pyo_obj) From 07928001d994c4a40c52a4867498981c1ef0b0c0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Dec 2025 09:32:29 -0700 Subject: [PATCH 95/97] fix typo --- pyomo/contrib/solver/common/solution_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 3ad688d937f..70d497fd9f0 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -50,7 +50,7 @@ def get_solution_ids(self) -> List[Any]: """ If there are multiple solutions available, this will return a list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is + methods like `load_solution`. If only one solution is available, this will return [None]. If no solutions are available, this will return None From 879ed3aea700887204873d1dd42b2063d1327f85 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Dec 2025 11:21:10 -0700 Subject: [PATCH 96/97] update scip interface to use observer --- pyomo/contrib/observer/model_observer.py | 3 + .../solver/solvers/scip/scip_direct.py | 273 +++++++++--------- 2 files changed, 145 insertions(+), 131 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b1eda200a9a..9a4e8f27563 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -741,6 +741,9 @@ def remove_objectives(self, objs: Collection[ObjectiveData]): def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): + if not issubclass(ctype, ActiveComponent): + # strangly, this is needed to skip things like Param + continue if ctype in self._known_active_ctypes: continue if ctype is Suffix: diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 05f39b0cb16..2d820e92028 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -85,6 +85,7 @@ Observer, ModelChangeDetector, AutoUpdateConfig, + Reason, ) @@ -111,8 +112,8 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.use_mipstart: bool = self.declare( - 'use_mipstart', + self.warmstart_discrete_vars: bool = self.declare( + 'warmstart_discrete_vars', ConfigValue( default=False, domain=bool, @@ -328,11 +329,10 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt + self, solver_model, var_map, con_map, pyomo_model, opt ) -> None: super().__init__() self._solver_model = solver_model - self._vars = var_id_map self._var_map = var_map self._con_map = con_map self._pyomo_model = pyomo_model @@ -359,13 +359,13 @@ def get_vars( if self.get_number_of_solutions() == 0: raise NoSolutionError() if vars_to_load is None: - vars_to_load = list(self._vars.values()) + vars_to_load = list(self._var_map.keys()) if solution_id is None: solution_id = 0 sol = self._solver_model.getSols()[solution_id] res = ComponentMap() for v in vars_to_load: - sv = self._var_map[id(v)] + sv = self._var_map[v] res[v] = sol[sv] return res @@ -385,9 +385,9 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt + self, solver_model, var_map, con_map, pyomo_model, opt ) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, pyomo_model, opt) + super().__init__(solver_model, var_map, con_map, pyomo_model, opt) self._valid = True def invalidate(self): @@ -445,13 +445,11 @@ class ScipDirect(SolverBase): def __init__(self, **kwds): super().__init__(**kwds) self._solver_model = None - self._vars = {} # var id to var - self._params = {} # param id to param - self._pyomo_var_to_solver_var_map = {} # var id to scip var + self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} self._pyomo_param_to_solver_param_map = ( - {} - ) # param id to scip var with equal bounds + ComponentMap() + ) # param to scip var with equal bounds self._pyomo_sos_to_solver_sos_map = {} self._expr_visitor = _PyomoToScipVisitor(self) self._objective = None # pyomo objective @@ -462,11 +460,9 @@ def __init__(self, **kwds): def _clear(self): self._solver_model = None - self._vars = {} - self._params = {} - self._pyomo_var_to_solver_var_map = {} + self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} - self._pyomo_param_to_solver_param_map = {} + self._pyomo_param_to_solver_param_map = ComponentMap() self._pyomo_sos_to_solver_sos_map = {} self._objective = None self._obj_var = None @@ -490,16 +486,9 @@ def version(self) -> Tuple: def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - orig_config = self.config - if not self.available(): - raise ApplicationError(f'{self.name} is not available: {self.available()}') try: config = self.config(value=kwds, preserve_implicit=True) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = config - object.__setattr__(self, 'config', config) - StaleFlagManager.mark_all_as_stale() if config.timer is None: @@ -508,7 +497,7 @@ def solve(self, model: BlockData, **kwds) -> Results: ostreams = [io.StringIO()] + config.tee - scip_model, solution_loader, has_obj = self._create_solver_model(model) + scip_model, solution_loader, has_obj = self._create_solver_model(model, config) scip_model.hideOutput(quiet=False) if config.threads is not None: @@ -520,7 +509,7 @@ def solve(self, model: BlockData, **kwds) -> Results: if config.abs_gap is not None: scip_model.setParam('limits/absgap', config.abs_gap) - if config.use_mipstart: + if config.warmstart_discrete_vars: self._mipstart() for key, option in config.solver_options.items(): @@ -532,13 +521,10 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.optimize() timer.stop('optimize') - results = self._postsolve(scip_model, solution_loader, has_obj) + results = self._populate_results(scip_model, solution_loader, has_obj, config) except InfeasibleConstraintException: + # is it possible to hit this? results = self._get_infeasible_results() - finally: - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = orig_config - object.__setattr__(self, 'config', orig_config) results.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) @@ -612,16 +598,14 @@ def _add_var(self, var): scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) - self._vars[id(var)] = var - self._pyomo_var_to_solver_var_map[id(var)] = scip_var + self._pyomo_var_to_solver_var_map[var] = scip_var return scip_var def _add_param(self, p): vtype = "C" lb = ub = p.value scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) - self._params[id(p)] = p - self._pyomo_param_to_solver_param_map[id(p)] = scip_var + self._pyomo_param_to_solver_param_map[p] = scip_var return scip_var def __del__(self): @@ -638,8 +622,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for on in cons: self._add_sos_constraint(con) - def _create_solver_model(self, model): - timer = self.config.timer + def _create_solver_model(self, model, config): + timer = config.timer timer.start('create scip model') self._clear() self._solver_model = scip.Model() @@ -666,7 +650,6 @@ def _create_solver_model(self, model): has_obj = obj is not None solution_loader = ScipDirectSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, pyomo_model=model, @@ -758,8 +741,8 @@ def _set_objective(self, obj): self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj - def _postsolve( - self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj + def _populate_results( + self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj, config ): results = Results() @@ -783,7 +766,7 @@ def _postsolve( if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and self.config.raise_exception_on_nonoptimal_result + and config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -813,16 +796,16 @@ def _postsolve( results.incumbent_objective = None results.objective_bound = None - self.config.timer.start('load solution') - if self.config.load_solutions: + config.timer.start('load solution') + if config.load_solutions: if solution_loader.get_number_of_solutions() > 0: solution_loader.load_solution() else: raise NoFeasibleSolutionError() - self.config.timer.stop('load solution') + config.timer.stop('load solution') - results.iteration_count = scip_model.getNNodes() - results.solver_config = self.config + results.extra_info['NNodes'] = scip_model.getNNodes() + results.solver_config = config results.solver_name = self.name results.solver_version = self.version() @@ -832,8 +815,7 @@ def _mipstart(self): # TODO: it is also possible to specify continuous variables, but # I think we should have a different option for that sol = self._solver_model.createPartialSol() - for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[vid] + for pyomo_var, scip_var in self._pyomo_var_to_solver_var_map.items(): if pyomo_var.is_integer(): sol[scip_var] = pyomo_var.value self._solver_model.addSol(sol) @@ -859,55 +841,13 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class _ScipObserver(Observer): - def __init__(self, opt: ScipPersistent) -> None: - self.opt = opt - - def add_variables(self, variables: List[VarData]): - self.opt._add_variables(variables) - - def add_parameters(self, params: List[ParamData]): - self.opt._add_parameters(params) - - def add_constraints(self, cons: List[ConstraintData]): - self.opt._add_constraints(cons) - - def add_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._add_sos_constraints(cons) - - def add_objectives(self, objs: List[ObjectiveData]): - self.opt._add_objectives(objs) - - def remove_objectives(self, objs: List[ObjectiveData]): - self.opt._remove_objectives(objs) - - def remove_constraints(self, cons: List[ConstraintData]): - self.opt._remove_constraints(cons) - - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._remove_sos_constraints(cons) - - def remove_variables(self, variables: List[VarData]): - self.opt._remove_variables(variables) - - def remove_parameters(self, params: List[ParamData]): - self.opt._remove_parameters(params) - - def update_variables(self, variables: List[VarData]): - self.opt._update_variables(variables) - - def update_parameters(self, params: List[ParamData]): - self.opt._update_parameters(params) - - -class ScipPersistent(ScipDirect, PersistentSolverBase): +class ScipPersistent(ScipDirect, PersistentSolverBase, Observer): _minimum_version = (5, 5, 0) # this is probably conservative CONFIG = ScipPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) self._pyomo_model = None - self._observer = None self._change_detector = None self._last_results_object: Optional[Results] = None self._needs_reopt = False @@ -916,10 +856,9 @@ def __init__(self, **kwds): def _clear(self): super()._clear() self._pyomo_model = None - self._objective = None - self._observer = None self._change_detector = None self._needs_reopt = False + self._range_constraints = set() def _check_reopt(self): if self._needs_reopt: @@ -927,15 +866,14 @@ def _check_reopt(self): self._solver_model.freeTransform() self._needs_reopt = False - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): if pyomo_model is self._pyomo_model: - self.update() + self.update(**config) else: - self.set_instance(pyomo_model=pyomo_model) + self.set_instance(pyomo_model, **config) solution_loader = ScipPersistentSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, pyomo_model=pyomo_model, @@ -950,39 +888,128 @@ def solve(self, model, **kwds) -> Results: self._needs_reopt = True return res - def update(self): - if self.config.timer is None: + def update(self, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer if self._pyomo_model is None: raise RuntimeError('must call set_instance or solve before update') timer.start('update') - self._change_detector.update(timer=timer) + self._change_detector.update(timer=timer, **config.auto_updates) timer.stop('update') - def set_instance(self, pyomo_model): - if self.config.timer is None: + def set_instance(self, pyomo_model, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer self._clear() self._pyomo_model = pyomo_model self._solver_model = scip.Model() - self._observer = _ScipObserver(self) timer.start('set_instance') self._change_detector = ModelChangeDetector( model=self._pyomo_model, - observers=[self._observer], - **dict(self.config.auto_updates), + observers=[self], + **config.auto_updates, ) - self._change_detector.config = self.config.auto_updates timer.stop('set_instance') def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() + def _update_variables(self, variables: Mapping[VarData, Reason]): + new_vars = [] + old_vars = [] + mod_vars = [] + for v, reason in variables.items(): + if reason & Reason.added: + new_vars.append(v) + elif reason & Reason.removed: + old_vars.append(v) + else: + mod_vars.append(v) + + if new_vars: + self._add_variables(new_vars) + if old_vars: + self._remove_variables(old_vars) + if mod_vars: + self._update_vars_for_real(mod_vars) + + def _update_parameters(self, params: Mapping[ParamData, Reason]): + new_params = [] + old_params = [] + mod_params = [] + for p, reason in params.items(): + if reason & Reason.added: + new_params.append(p) + elif reason & Reason.removed: + old_params.append(p) + else: + mod_params.append(p) + + if new_params: + self._add_parameters(new_params) + if old_params: + self._remove_parameters(old_params) + if mod_params: + self._update_params_for_real(mod_params) + + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.expr: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.sos_items: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_sos_constraints(old_cons) + if new_cons: + self._add_sos_constraints(new_cons) + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + new_objs = [] + old_objs = [] + for obj, reason in objs.items(): + if reason & Reason.added: + new_objs.append(obj) + elif reason & Reason.removed: + old_objs.append(obj) + elif reason & (Reason.expr | Reason.sense): + old_objs.append(obj) + new_objs.append(obj) + + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + def _add_variables(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() @@ -1024,7 +1051,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): if self._objective is not None: raise NotImplementedError( - 'the persistent interface to gurobi currently ' + 'the persistent interface to scip currently ' 'only supports single-objective problems; tried to add ' f'an objective ({str(obj)}), but there is already an ' f'active objective ({str(self._objective)})' @@ -1064,38 +1091,32 @@ def _remove_variables(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() for v in variables: - vid = id(v) - scip_var = self._pyomo_var_to_solver_var_map.pop(vid) + scip_var = self._pyomo_var_to_solver_var_map.pop(v) self._solver_model.delVar(scip_var) - self._vars.pop(vid) def _remove_parameters(self, params: List[ParamData]): self._check_reopt() self._invalidate_last_results() for p in params: - pid = id(p) - scip_var = self._pyomo_param_to_solver_param_map.pop(pid) + scip_var = self._pyomo_param_to_solver_param_map.pop(p) self._solver_model.delVar(scip_var) - self._params.pop(pid) - def _update_variables(self, variables: List[VarData]): + def _update_vars_for_real(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() for v in variables: - vid = id(v) - scip_var = self._pyomo_var_to_solver_var_map[vid] + scip_var = self._pyomo_var_to_solver_var_map[v] vtype = self._scip_vtype_from_var(v) lb, ub = self._scip_lb_ub_from_var(v) self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) self._solver_model.chgVarType(scip_var, vtype) - def _update_parameters(self, params: List[ParamData]): + def _update_params_for_real(self, params: List[ParamData]): self._check_reopt() self._invalidate_last_results() for p in params: - pid = id(p) - scip_var = self._pyomo_param_to_solver_param_map[pid] + scip_var = self._pyomo_param_to_solver_param_map[p] lb = ub = p.value self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) @@ -1108,11 +1129,6 @@ def _update_parameters(self, params: List[ParamData]): self._remove_constraints([con]) self._add_constraints([con]) - def add_variables(self, variables): - if self._change_detector is None: - raise RuntimeError('call set_instance first') - self._change_detector.add_variables(variables) - def add_constraints(self, cons): if self._change_detector is None: raise RuntimeError('call set_instance first') @@ -1138,11 +1154,6 @@ def remove_sos_constraints(self, cons): raise RuntimeError('call set_instance first') self._change_detector.remove_sos_constraints(cons) - def remove_variables(self, variables): - if self._change_detector is None: - raise RuntimeError('call set_instance first') - self._change_detector.remove_variables(variables) - def update_variables(self, variables): if self._change_detector is None: raise RuntimeError('call set_instance first') From 3bfa5cbd9e5ba3856e8b4c3d73a26d79c0e623c2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 8 Jan 2026 14:40:23 -0700 Subject: [PATCH 97/97] run black --- .../solver/solvers/scip/scip_direct.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 2d820e92028..b5cb1a6946a 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -328,9 +328,7 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, pyomo_model, opt - ) -> None: + def __init__(self, solver_model, var_map, con_map, pyomo_model, opt) -> None: super().__init__() self._solver_model = solver_model self._var_map = var_map @@ -384,9 +382,7 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): - def __init__( - self, solver_model, var_map, con_map, pyomo_model, opt - ) -> None: + def __init__(self, solver_model, var_map, con_map, pyomo_model, opt) -> None: super().__init__(solver_model, var_map, con_map, pyomo_model, opt) self._valid = True @@ -497,7 +493,9 @@ def solve(self, model: BlockData, **kwds) -> Results: ostreams = [io.StringIO()] + config.tee - scip_model, solution_loader, has_obj = self._create_solver_model(model, config) + scip_model, solution_loader, has_obj = self._create_solver_model( + model, config + ) scip_model.hideOutput(quiet=False) if config.threads is not None: @@ -521,7 +519,9 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.optimize() timer.stop('optimize') - results = self._populate_results(scip_model, solution_loader, has_obj, config) + results = self._populate_results( + scip_model, solution_loader, has_obj, config + ) except InfeasibleConstraintException: # is it possible to hit this? results = self._get_infeasible_results() @@ -911,9 +911,7 @@ def set_instance(self, pyomo_model, **kwds): self._solver_model = scip.Model() timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, - observers=[self], - **config.auto_updates, + model=self._pyomo_model, observers=[self], **config.auto_updates ) timer.stop('set_instance') @@ -951,7 +949,7 @@ def _update_parameters(self, params: Mapping[ParamData, Reason]): old_params.append(p) else: mod_params.append(p) - + if new_params: self._add_parameters(new_params) if old_params: