# ___________________________________________________________________________
#
# Pyomo: Python Optimization Modeling Objects
# Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
# Under the terms of Contract DE-NA0003525 with National Technology and
# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
# rights in this software.
# This software is distributed under the 3-clause BSD License.
# ___________________________________________________________________________
import os
import copy
import six
import pickle
import shelve
import json
from pyutilib.misc import ArchiveReaderFactory
from pyomo.common.plugin import implements, alias, SingletonPlugin
from pyomo.pysp import phextension
from pyomo.pysp.phutils import indexToString
from pyomo.pysp.phsolverserverutils import TransmitType
import pyomo.solvers.plugins.smanager
from six import iteritems
bytes_cast = lambda x:x
if six.PY3:
bytes_cast = lambda x: x.encode()
# TODO:
# - snapshot node _fix_queue
def extract_convergence(ph):
metric = ph._convergers[0].lastMetric() \
if len(ph._convergers[0]._metric_history) \
else None
convergence = {'metric':metric,
'fixed variable counts':\
{'continuous':ph._total_fixed_continuous_vars,
'discrete':ph._total_fixed_discrete_vars},
'blended variable counts':\
{'continuous':ph._total_continuous_vars,
'discrete':ph._total_discrete_vars}}
return convergence
def extract_scenario_tree_structure(scenario_tree):
scenario_tree_structure = {}
scenario_tree_structure['scenarios'] = {}
for scenario in scenario_tree._scenarios:
scenario_structure = \
scenario_tree_structure['scenarios'][scenario._name] = {}
scenario_structure['name'] = scenario._name
scenario_structure['probability'] = scenario._probability
scenario_structure['nodes'] = \
[node._name for node in scenario._node_list]
scenario_tree_structure['stages'] = {}
for stage_order, stage in enumerate(scenario_tree._stages):
stage_structure = scenario_tree_structure['stages'][stage._name] = {}
stage_structure['name'] = stage._name
stage_structure['nodes'] = [node._name for node in stage._tree_nodes]
stage_structure['order'] = stage_order
scenario_tree_structure['nodes'] = {}
for tree_node in scenario_tree._tree_nodes:
node_structure = \
scenario_tree_structure['nodes'][tree_node._name] = {}
parent = tree_node._parent
node_structure['name'] = tree_node._name
node_structure['parent'] = \
parent._name if (parent is not None) else None
node_structure['children'] = \
[child_node._name for child_node in tree_node._children]
node_structure['stage'] = tree_node._stage._name
node_structure['conditional probability'] = \
tree_node._conditional_probability
node_structure['probability'] = tree_node._probability
node_structure['scenarios'] = \
[node_scenario._name for node_scenario in tree_node._scenarios]
return scenario_tree_structure
def extract_scenario_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_leaf_stage_vars=True):
scenario_solutions = {}
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name] = {}
variable_sol = scenario_sol['variables'] = {}
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode or include_leaf_stage_vars:
if isNotLeafNode and include_ph_objective_parameters:
weight_values = scenario._w[tree_node._name]
rho_values = scenario._rho[tree_node._name]
x_values = scenario._x[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label] = {}
varsol['value'] = x_values.get(variable_id)
varsol['fixed'] = scenario.is_variable_fixed(tree_node,
variable_id)
varsol['stale'] = scenario.is_variable_stale(tree_node,
variable_id)
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
varsol['rho'] = rho_values[variable_id] \
if (isNotLeafNode) \
else None
varsol['weight'] = weight_values[variable_id] \
if (isNotLeafNode) \
else None
else:
varsol['rho'] = None
varsol['weight'] = None
scenario_sol['objective'] = scenario._objective
scenario_sol['cost'] = scenario._cost
if include_ph_objective_parameters:
scenario_sol['ph weight term'] = scenario._weight_term_cost
scenario_sol['ph proximal term'] = scenario._proximal_term_cost
scenario_sol['stage costs'] = copy.deepcopy(scenario._stage_costs)
return scenario_solutions
def extract_node_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_variable_statistics=False,
include_leaf_stage_vars=True):
scenario_tree.snapshotSolutionFromScenarios()
node_solutions = {}
stages = None
if include_leaf_stage_vars:
stages = scenario_tree._stages
else:
stages = scenario_tree._stages[:-1]
for stage in stages:
for tree_node in stage._tree_nodes:
isNotLeafNode = not tree_node.is_leaf_node()
node_sol = node_solutions[tree_node._name] = {}
variable_sol = node_sol['variables'] = {}
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
sol = variable_sol[name_label] = {}
sol['solution'] = tree_node._solution[variable_id]
sol['fixed'] = tree_node.is_variable_fixed(variable_id)
sol['derived'] = \
bool(variable_id in tree_node._derived_variable_ids)
if include_variable_statistics:
if isNotLeafNode:
sol['minimum'] = tree_node._minimums[variable_id]
sol['average'] = tree_node._averages[variable_id]
sol['maximum'] = tree_node._maximums[variable_id]
else:
sol['minimum'] = None
sol['average'] = None
sol['maximum'] = None
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
sol['xbar'] = tree_node._xbars[variable_id]
sol['wbar'] = tree_node._wbars[variable_id]
else:
sol['xbar'] = None
sol['wbar'] = None
node_sol['expected cost'] = tree_node.computeExpectedNodeCost()
return node_solutions
#
# A PH warmstart consists of values for W and XBAR
# (nothing more)
#
def load_ph_warmstart(ph,
scenariotree_solution):
scenario_tree = ph._scenario_tree
scenario_solutions = scenariotree_solution['scenario solutions']
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name]
variable_sol = scenario_sol['variables']
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode:
scenario._w[tree_node.name].clear()
scenario_w = scenario._w[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label]
if variable_id in tree_node._standard_variable_ids:
if 'weight' in varsol:
scenario_w[variable_id] = varsol['weight']
node_solutions = scenariotree_solution['node solutions']
for stage in scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
variable_sol = node_solutions[tree_node._name]['variables']
for variable_id in tree_node._standard_variable_ids:
var_name, index = tree_node._variable_ids[variable_id]
sol = variable_sol[str(var_name)+str(indexToString(index))]
tree_node._xbars[variable_id] = sol['xbar']
def _dump_to_history(filename,
data,
key,
last=False,
first=False,
use_json=False):
assert not (first and last)
if use_json:
file_string = 'wb' if first else \
'ab+'
with open(filename, file_string) as f:
if first:
f.write(bytes_cast('{\n'))
else:
# make sure we are at the end of the file
f.seek(0,2)
# overwrite the previous \n}\n
f.truncate(f.tell()-3)
f.write(bytes_cast(',\n'))
f.write(bytes_cast(' "'+key+'":\n'))
f.write(bytes_cast(json.dumps(data,indent=2)))
f.write(bytes_cast('\n}\n'))
else:
if first:
flag = 'n'
else:
flag = 'c'
d = shelve.open(filename,
flag=flag,
protocol=pickle.HIGHEST_PROTOCOL)
d[key] = data
if first:
d['results keys'] = []
if key != 'scenario tree':
d['results keys'] += [key]
d.close()
class phhistoryextension(SingletonPlugin):
implements(phextension.IPHExtension)
# the below is a hack to get this extension into the
# set of IPHExtension objects, so it can be queried
# automagically by PH.
alias("PHHistoryExtension")
def __init__(self):
self._history_started = False
self._ph_history_filename = "ph_history"
self._use_json = int(os.environ.get("PHHISTORYEXTENSION_USE_JSON",0))
if self._use_json:
self._ph_history_filename += ".json"
else:
self._ph_history_filename += ".db"
self._history_offset = 0
def reset(self, ph):
self.__init__()
def pre_ph_initialization(self,ph):
pass
def post_instance_creation(self,ph):
pass
def post_ph_initialization(self, ph):
# TODO: Add a print statement notifying the user of this change
# Make sure we transmit at least all the ph variables on the
# scenario tree (including leaf nodes). If the default
# has already been set to transmit more, then we are fine.
# (hence the |=)
if isinstance(ph._solver_manager,
pyomo.solvers.plugins.\
smanager.phpyro.SolverManager_PHPyro):
print("Overriding default variable transmission settings "
"for PHPyro to transmit leaf-stage variable values "
"at intermediate PH iterations.")
ph._phpyro_variable_transmission_flags |= \
TransmitType.all_stages
ph._phpyro_variable_transmission_flags |= \
TransmitType.blended
ph._phpyro_variable_transmission_flags |= \
TransmitType.derived
ph._phpyro_variable_transmission_flags |= \
TransmitType.fixed
ph._phpyro_variable_transmission_flags |= \
TransmitType.stale
def post_iteration_0_solves(self, ph):
pass
def post_iteration_0(self, ph):
pass
def _prepare_history_file(self, ph):
if not self._history_started:
if (ph._ph_warmstarted) and \
(ph._ph_warmstart_file is not None):
assert ph._ph_warmstart_index is not None
self._ph_history_file = ph._ph_warmstart_file
self._history_offset = int(ph._ph_warmstart_index) + 1
print("Detected PH warmstart file. Appending to "
"content and storing new iterations with offset. "
"First new iteration will be saved with index: %s\n"
% (self._history_offset))
self._history_started = True
else:
data = extract_scenario_tree_structure(ph._scenario_tree)
_dump_to_history(self._ph_history_filename,
data,
'scenario tree',
first=True,
use_json=self._use_json)
self._history_started = True
def _snapshot_all(self, ph):
data = {}
data['convergence'] = extract_convergence(ph)
data['scenario solutions'] = \
extract_scenario_solutions(ph._scenario_tree, True)
data['node solutions'] = \
extract_node_solutions(ph._scenario_tree, True, True)
return data
def pre_iteration_k_solves(self, ph):
self._prepare_history_file(ph)
key = str(ph._current_iteration - 1 + self._history_offset)
data = self._snapshot_all(ph)
_dump_to_history(self._ph_history_filename,
data,
key,
use_json=self._use_json)
def post_iteration_k_solves(self, ph):
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
self._prepare_history_file(ph)
key = str(ph._current_iteration + \
self._history_offset)
data = self._snapshot_all(ph)
_dump_to_history(self._ph_history_filename,
data,
key,
last=True,
use_json=self._use_json)
print("PH algorithm history written to file="
+self._ph_history_filename)
def load_history(filename):
with ArchiveReaderFactory(filename) as archive:
outf = archive.extract()
history = None
try:
with open(outf) as f:
history = json.load(f)
except:
history = None
try:
history = shelve.open(outf,
flag='r')
except:
history = None
if history is None:
raise RuntimeError("Unable to open ph history file as JSON "
"or python Shelve DB format")
scenario_tree_dict = history['scenario tree']
try:
iter_keys = history['results keys']
except KeyError:
# we are using json format (which loads the entire file
# anyway)
iter_keys = list(history.keys())
iter_keys.remove('scenario tree')
iterations = sorted(int(k) for k in iter_keys)
iterations = [str(k) for k in iterations]
return scenario_tree_dict, history, iterations
def load_solution(filename):
with ArchiveReaderFactory(filename) as archive:
outf = archive.extract()
solution = None
try:
with open(outf) as f:
solution = json.load(f)
except:
solution = None
try:
solution = shelve.open(outf,
flag='r')
except:
solution = None
if solution is None:
raise RuntimeError("Unable to open ph solution file as JSON "
"or python Shelve DB format")
scenario_tree_dict = solution['scenario tree']
solution.pop('scenario tree')
return scenario_tree_dict, solution