B _@sddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z ddlmZmZmZddlmZmZddlmZmZddlmZmZdd lmZdd lmZmZm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+dd l,m-Z-m.Z.m/Z/m0Z0m1Z1dd lm2Z2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ej9:e;ZddZ?ddZ@d!ddZAd"ddZBddZCd#ddZDeEdkreDejFddd dS)$N) WriterFactory) ComponentMap)value as_numeric)Block _BlockDataSortComponents)Var_VarData) Constraint_ConstraintData) TextLabelerNumericLabeler)InvocationType)locate_annotations$StochasticConstraintBoundsAnnotation"StochasticConstraintBodyAnnotationStochasticObjectiveAnnotation"StochasticVariableBoundsAnnotation)map_variable_stagesmap_constraint_stages build_repns_safe_remove_file_no_negative_zero_deterministic_check_value_deterministic_check_constant ProblemStats)PySPConfigValuePySPConfigBlocksafe_register_common_optionsafe_register_unique_option_domain_must_be_str)ScenarioTreeManagerClientSerialScenarioTreeManagerClientPyro)launch_command) iteritems itervaluesz pyomo.pyspc Os|j}g}xp|jdddD]^}i}t|dr:|j|d<|`t|drR|j|d<|`t|drj|j|d<|`|||fqWz6yt||f||St d|j YnXWdxX|D]P\}}t|dr|`t|dr|`t|dr|`x|D]}t ||||qWqWXdS)NT)active descend_into _gen_obj_repn _gen_con_repn_repnz|;=d1t>|<dk rx|D]\}.}?t?|.t@st|st-|.|krtd2|.j&tjf|t-|.,|.}@|@Astd3|.j&|@j"}Ad|@_"|Adkrd4}A|6|.}-t|-dkstx|-D]}B|BBd5s.|BBd6r|?dksJ|?ddksJt|9d7}9|:=|=|B|;=|>t>t!|.jCt!|A|.jC|7|.<tD|._E|BBd5 r|.jF|8|.<tD|._Gq|BBd7 r|?dks|?ddk r|9d7}9|:=|=|B|;=|>t>t!|.jCt!|A|.jC|7|.<tD|._En|BBd8 r|?dk sD|?ddk sDt|9d7}9|:=|=|B|;=|>t>t!|.jFt!|A|.jF|8|.<tD|._Gnv|BBd9 r|?dk s|?ddk r|9d7}9|:=|=|B|;=|>t>t!|.jFt!|A|.jF|8|.<tD|._Gn dstqWqfWWdQRXWdQRXd}Ct3tj%||j&d:d*}Dt3tj%||j&d;d*}E|j<}d d t |@j#}Ix|HD]}'t?|'tI sxt|'jJ rtd}Jx|-D]6}B|Cd7}C|D=|F|B|Nf|E=|Gt>t!|J qW qdWt|I|@_# qWWdQRXWdQRXd}Ot3tj%||j&d@d*6}Pt3tj%||j&dAd*}Qd/}Rd0}S|Q=d=|dk r|j r|}Tt|Tdk stt|Tdk rtdBtj|Td\}U\}V}W|U| k s$tn |jM\}V}W|"A st!|J qWt|I|"_#|Wr|"j"}YtL|"_"|Ydkrd4}Y|Od7}O|P=|RdF|Q=|St>|YWdQRXWdQRX|j&}ZdG|_Ntj%||j&dH}[t'd*}|||[dId |\}}||[ks tWdQRX|Z|_Nx tO|7D]\}.}\tP|\|._Eq$Wx tO|8D]\}.}]tP|]|._GqFW|#|$|)|*|O|9|CfS)JNrcss|] }|VqdS)Nr:).0conr:r:r; {sz:_convert_external_setup_without_cleanup..T)r'r()Z max_allowedFcs|ddS)NrT)getname)x)constraint_name_bufferr:r;z9_convert_external_setup_without_cleanup..)keyzxThe %s annotation was declared with external entries but no active Constraint objects were recovered from those entries.c3s|]}|jfVqdS)N)default)r>r?)stochastic_rhsr:r;r@scs|ddS)NrT)rA)rB)rCr:r;rDrEc3s|]}|jfVqdS)N)rG)r>r?)stochastic_matrixr:r;r@scss|]\}}t|VqdS)N)id)r>r?_r:r:r;r@scss|]\}}t|VqdS)N)rJ)r>r?rKr:r:r;r@szcThe DDSIP writer does not currently support stochastic variable bounds. Invalid annotation type: %sz|No stochastic annotations found. DDSIP conversion requires at least one of the following annotation types: - %s - %s - %sr+cSsg|] }t|qSr:)r)r>cr:r:r; sz;_convert_external_setup_without_cleanup..cSsg|] }t|qSr:)r)r>rLr:r:r;rMsz .lp.setuplp column_order row_ordercSsdS)NTr:)rBr:r:r;rDrE)!enforce_derived_nonanticipativitycss|]\}}}t|VqdS)N)rJ)r>symbolvarscenario_tree_idr:r:r;r@scss|]\}}}t|VqdS)N)rJ)r>rRrSrTr:r:r;r@scss|]\}}t|VqdS)N)rJ)r>symbolsr?r:r:r;r@ssymbolic_solver_labelsrBcs&|t|trt|kr dnS)N) isinstancer rJ)rB)_labelerfirststage_var_suffixfirststage_variable_idsr:r;rDVs z.lpz .lp.symbolslabelerZforce_objective_constantcSsdS)NTr:)rBr:r:r;rDnrEwz%s %s c3s(|] }|D]\}}||fVqqdS)Nr:)r>Z stage_namerUr?)StageToConstraintMapr:r;r@sz.rhs.sc.structz.rhs.scz %s z %.17g z scen %.17g zThe constraint %s has been declared in the %s annotation but it was not identified as a second-stage constraint. To correct this issue, remove the constraint from this annotation.zaOnly linear constraints are accepted for conversion to DDSIP format. Constraint %s is not linear.gZc_e_Zc_l_Zr_l_Zc_u_Zr_u_z.matrix.sc.structz .matrix.scz %s %s zscen cs|S)Nr:)_v)rOr:r;rD5rEzThe coefficient for variable %s has been marked as stochastic in constraint %s using the %s annotation, but the variable does not appear in the canonical constraint expression.z.cost.sc.structz.cost.sczwThe %s annotation was declared with external entries but no active Objective objects were recovered from those entries.zjOnly linear stochastic objectives are accepted for conversion to DDSIP format. Objective %s is not linear.cs|S)Nr:)r`)rOr:r;rD~rEzThe coefficient for variable %s has been marked as stochastic in objective %s using the %s annotation, but the variable does not appear in the canonical objective expression.ZONE_VAR_CONSTANTZZeroStochasticDataz.lp.detcSsdS)NTr:)rBr:r:r;rDrE)Q pyomo.environospathexistsAssertionErrordict scenario_treer,Z findRootNodeZstageslistZcomponent_data_objectsr rrlenZhas_declarationsZexpand_entriessort RuntimeError__name__tuplersetupdaterr ValueErrorr-rr)r*valuesrZconstantZ linear_coefsZquadratic_coefsjoinr2rrrrZ_instance_objectiveZ parent_blockrJr enumeratepopr ropenZ_ScenarioTreeSymbolMapsortedZ _variable_idsZbySymbolZis_expression_typeZbyObjectr. writelinesZ probabilitywriterrYr Z is_linear startswithlowerrZ_lowerupperZ_upperZ linear_varsr ZfixedziprrG_namer%r)^r4r5output_directoryr[rQ io_optionspyomorgr8rootnodeZ firststageZ secondstageZobjective_name_bufferZvariable_name_bufferZall_constraintsZstochastic_rhs_entriesZempty_rhs_annotationZstochastic_matrix_entriesZempty_matrix_annotationZstochastic_constraint_idsZstochastic_objectiveZstochastic_varboundsZ repn_cacheZ block_repnsZrepnZoutput_filenamewriterZ output_fnameZ symbol_mapZStageToVariableMapZsecondstage_variable_idsZsecondstage_constraint_idsZobjective_objectZobjective_blockZobjective_repnfirststage_variable_countsecondstage_variable_countZ column_indexrRrSrTfirststage_constraint_countsecondstage_constraint_countrPZ row_indexrUr?r]Zsymbols_filenamefZ st_symbol_maplinesZid_Zlp_labelZconstraint_symbolsZmodified_constraint_lbZmodified_constraint_ubstochastic_rhs_countZ f_rhs_structZf_rhsZscenario_probabilityZrhs_struct_templateZ rhs_templateZ include_boundZconstraint_repnZ body_constantZ con_labelstochastic_matrix_countZ f_mat_structZf_matZmatrix_struct_templateZmatrix_templateZvar_listZ new_coefsZvar_coefiZ_varZcoefZ var_labelstochastic_cost_countZ f_obj_structZf_objZobj_struct_templateZ obj_templateZ sorted_valuesobjZobjective_variablesZinclude_constantZstochastic_objective_labelZ obj_constantZreference_model_nameZdet_output_filenamerzr{r:)r_rZrOrCr[r\rIrHr;r/fsH                                                                  *                 $                 &   r/Fc+ Cs8ddl}ddl}|dkri}tj|s,t|j} | rBtdtj |d} tj| sft | |j dt t j| |||fd} | jd} | j} | | \}}}}}}}i}tj |d}t|tj | | d}t||||d<tj |d }t|tj | | d }t||||d <xnd D]d}|d }|dkrT|dkr|q,n(|dkrn|dkr|q,n|dks|ttj ||}|||<t|t|d}|dkr|dks|d| | jdksttj | | jd|d}tj|stt|d}t||WdQRXx\| jD]R} tj | | jd|}tj|s^tt|d}t||WdQRXq0WWdQRXq,W|r,tdtdtj|tdtdtd|td|tdtd|td|td|td|td td!||s|rLtd"| td#d$}!ytd%sdd&}!nd$}!Wnd$}!YnX|rtd'xd(D]}tj | | jd|}"xr| jD]h} tj | | jd|}#|!rtd)|#d*|"}$ntj|#|"d$d+ }$|$rtd,|"| j|#fqWqW|r4td-tj | | jd.}%xn| jD]d} tj | | jd.}&|!rtd)|&d*|%}$ntj|&|%d$d+ }$|$rPtd/|%| j|&fqPW|sD|rtd0xR| jD]H} x@d1D]8}tj | | jd|}'tj|'s tt|'qWqWtt| dkrRtj| d&d2n|rRtd3tj |d4}(t|(d})|)d5|)d6t|d7|)d8t|d7|)d9t|d7|)d:t|d7|)d;|d7|)dt|d7|)d?t|d7|)d@WdQRX|(|dA<tj |dB}*t|*d})|)tj|d|d7|)tj|dA|d7d|kst|)tj|d|d7dC|kr|)tj|dC|d7dD|kr"|)tj|dD|d7WdQRX|*|dE<|S)FNrz,DDSIP conversion does not yet handle bundlesZscenario_filesr<)Zinvocation_typeZ function_argszcore.lpz.lpcorezcore.lp.symbolsz .lp.symbolsrU)rhscostmatrixz.sczcost.scz matrix.sczrhs.scr^rzNames .z.structrz DDSIP Conversion CompletezOutput Saved To: zBasic Problem Information:z - Variables:z - First Stage: %dz - Second Stage: %dz - Constraints:z - Stoch. RHS Entries: %dz - Stoch. Matrix Entries: %dz - Objective:z - Stoch. Cost Entries: %dzS Starting scenario structure consistency checks across scenario files stored in %s.zThis may take some time. If this test is prohibitively slow or can not be executed on your system, disable it by activating the disable_consistency_check option.Fzdiff --help > /dev/nullTz, - Checking structure in stochastic files...)z rhs.sc.structzcost.sc.structzmatrix.sc.structzdiff -q  )ZshallowaThe structure indicated in file '%s' does not match that for scenario %s indicated in file '%s'. This suggests one or more locations of stachastic data have not been annotated. If you feel this message is in error, please report this issue to the PySP developers.z> - Checking deterministic sections in the core problem file...z.lp.detaYOne or more deterministic parts of the problem found in file '%s' do not match those for scenario %s found in file %s. This suggests that one or more locations of stochastic data have not been been annotated on the reference Pyomo model. If this seems like a tolerance issue or a developer error, please report this issue to the PySP developers.z%Cleaning temporary per-scenario files) rNzlp.detz lp.symbolsz matrix.sczmatrix.sc.structzcost.sczcost.sc.structzrhs.scz rhs.sc.struct) ignore_errorszHTemporary per-scenario files are retained in scenario_files subdirectoryz ddsip.configz BEGIN z FIRSTCON  z FIRSTVAR zSECCON zSECVAR zPOSTFIX zSCENAR zSTOCRHS z STOCCOST zSTOCMAT z END configz ddsip.stdinrrscript) raZ%pyomo.solvers.plugins.smanager.phpyrorbrcrdrergZcontains_bundlesrprrmkdirZinvoke_functionthisfilerZ PerScenarioZ scenariosr2rshutilcopy2rurx copyfileobjprintrelpathsystemfilecmpcmprilistdirrmtreestr)+r~r[scenario_tree_managerrQrdisable_consistency_checkskeep_scenario_filesverboserrgZscenario_directorycountsZreference_scenarioZreference_scenario_namerrrrrrrZ input_filesZlp_dstZlp_srcZ symbols_dstZ symbols_src_typebasenamedstfdstsrcfsrcr5Zhas_diffZreference_struct_filenameZscenario_struct_filenamercZreference_lp_det_filenameZscenario_lp_det_filenameZscenario_filenameZconfig_filenamerscript_filenamer:r:r;convert_externalsf                   ,                  &rc Cs|dkrt}t|dt|dt|dt|dt|dt|dtdtddd d t|d td td dd d t|dtdtddd d t|dtdtddd d t|dtdtddd d t|dt|t||S)N disable_gcprofile tracebackrrWr~rzFThe directory in which all DDSIP files will be stored. Default is '.'.r)domain descriptiondocZ visibilityfirst_stage_suffixZ__DDSIP_FIRSTSTAGEzPThe suffix used to identify first-stage variables. Default: '__DDSIP_FIRSTSTAGE'rQFzAdds nonanticipativity constraints for variables flagged as derived within their respective time stage (except for the final time stage). The default behavior behavior is to treat derived variables as belonging to the final time stage.raLDisables consistency checks that attempt to find issues with the DDSIP conversion. By default, these checks are run after conversion takes place and leave behind a temporary directory with per-scenario output files if the checks fail. This option is not recommended, but can be used if the consistency checks are prohibitively slow.rzKeeps around the per-scenario DDSIP files created for testing whether a conversion is valid (whether or not the validation checks are performed). These files can be useful for debugging purposes.r) rrr rr!boolr"Zregister_optionsr#)optionsr:r:r;convertddsip_register_optionssp             rc Csddl}tj|js"t|jt}d|ji}|jr>t d}|j dkrRt }n|j dkr`t }||2}| t|j|j||j||j|j|jd}WdQRXt}tdtd||dS)NrrWserialZpyro)rQrrrrrXz!Total execution time=%.2f seconds)rarbrcrdr~makedirstimerWcompile_scenario_instancesrerr"r#Z initializerrrQrrrr)rr start_timerZ manager_classrfilesZend_timer:r:r;run_convertddsip+s6     rc Csddl}t}t||dj}d|d_y8tjdd}||||d_||j |dWn$t k r}z|j Sd}~XYnXt t |d|j|j|jdS)Nrrzpyomo.pysp.convert.ddsip)prog)r6zpyomo.pysp.convert.ddsip: )Z error_labelrZ profile_countr)rarrgetZ _argparseargparseArgumentParserZinitialize_argparseZimport_argparse parse_args SystemExitcoder$rrrr)r6rrZ argparse_valZapZ_excr:r:r;mainVs$     r__main__r=)r6)FNFFF)N)N)GrbrsysrrrloggingZ pyomo.optrZpyomo.common.collectionsrZpyomo.core.base.numvaluerrZpyomo.core.base.blockrrrZpyomo.core.base.varr r Zpyomo.core.base.constraintr r Zpyomo.core.baser rZpyomo.pysp.scenariotree.managerrZpyomo.pysp.annotationsrrrrrZpyomo.pysp.convert.smpsrrrrrrrrZpyomo.pysp.util.configrrrr r!r"r#Zpyomo.pysp.util.miscr$sixr%r&rcabspath__file__r getLoggerr0r<r/rrrrrlargvr:r:r:r; sP   (   -h  W+ -