Skip to content
Snippets Groups Projects
Unverified Commit b665b95b authored by Lavinia Baumstark's avatar Lavinia Baumstark Committed by GitHub
Browse files

Merge pull request #102 from dklein-pik/develop 

Added option to restart runs
parents fd2f9273 ca744b14
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
config/scenario_config_SSPSDP.csv
+ 4
− 4
View file @ b665b95b
......@@ -2,14 +2,14 @@ title;start;CES_parameters;cm_bioenergy_tax;cm_rcp_scen;cm_iterative_target_adj;
SSP1-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;2;1.025;0;2;1;none;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;off;SSP1;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SSP1-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;2;1.025;0;2;1;none;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;off;SSP1;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SSP1-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NPi2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NPi2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;60;2045;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SDP-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.12;2;1.025;4;5;1;none;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;off;SDP;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SDP-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.12;2;1.025;4;5;1;none;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;off;SDP;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SDP-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NPi2018;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NPi2018;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;200;2045;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg1000;1;load;1.75;rcp20;9;1;1;1000;diffCurvPhaseIn2Lin;150;2045;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
......@@ -17,14 +17,14 @@ SDP-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9
SSP2-calibrate;0;calibrate;1.5;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;3;1.05;0;1;1;none;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;off;SSP2;2;0;forcing_SSP2;SSP2;2005;./config/input.gdx;;
SSP2-Base;1;load;1.5;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;3;1.05;0;1;1;none;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;off;SSP2;2;0;forcing_SSP2;SSP2;2005;./config/input.gdx;;
SSP2-NDC;1;load;1.5;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2015;./config/input.gdx;SSP2-Base;SSP2-Base
SSP2-NPi;0;load;1.5;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NPi2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-NPi;1;load;1.5;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NPi2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg900;1;load;1.5;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;200;2045;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg1100;1;load;1.5;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg1300;1;load;1.5;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP5-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;1;1.05;0;3;2;none;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;off;SSP5;1;1;forcing_SSP5;SSP5;2005;./config/input.gdx;;
SSP5-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;1;1.05;0;3;2;none;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;off;SSP5;1;1;forcing_SSP5;SSP5;2005;./config/input.gdx;;
SSP5-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NPi2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NPi2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;300;2045;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;150;2080;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;80;2100;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
output.R
+ 0
− 7
View file @ b665b95b
......@@ -241,13 +241,6 @@ if (comp==TRUE) {
#included as source (instead of a load from command line)
source_include <- TRUE
if (file.exists(path(outputdir,"output.RData"))) {
data_workspace<-path(outputdir,"output.RData")
} else {
tmp<-strsplit(outputdir,"/")[[1]]
data_workspace<-paste0(outputdir,"/",tmp[length(tmp)],".RData")
}
print(data_workspace)
cat(paste("\nStarting output generation for",outputdir,"\n\n"))
###################################################################################
......
scripts/start/prepare_and_run.R
+ 141
− 102
View file @ b665b95b
......@@ -2,6 +2,10 @@ library(lucode, quietly = TRUE,warn.conflicts =FALSE)
library(dplyr, quietly = TRUE,warn.conflicts =FALSE)
require(gdx)
##################################################################################################
# function: getReportData #
##################################################################################################
getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="costs") {
require(lucode, quietly = TRUE,warn.conflicts =FALSE)
require(magclass, quietly = TRUE,warn.conflicts =FALSE)
......@@ -138,13 +142,13 @@ getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="c
#.agriculture_tradebal(mag)
}
###############################################################################
###############################################################################
###############################################################################
##################################################################################################
# function: prepare #
##################################################################################################
prepare_and_run <- function() {
start_time <- Sys.time()
prepare <- function() {
timePrepareStart <- Sys.time()
# Load libraries
require(lucode, quietly = TRUE,warn.conflicts =FALSE)
......@@ -200,10 +204,6 @@ prepare_and_run <- function() {
if (on_cluster)
system("find ./core/magicc/ -type f | xargs dos2unix -q")
# Set source_include so that loaded scripts know they are included as
# source (instead of being executed from the command line)
source_include <- TRUE
################## M O D E L L O C K ###################################
# Lock the directory for other instances of the start scritps
lock_id <- model_lock(timeout1 = 1, oncluster=on_cluster)
......@@ -393,10 +393,9 @@ prepare_and_run <- function() {
################## M O D E L U N L O C K ###################################
# After full.gms was produced remind folders have to be unlocked to allow setting up the next run
model_unlock(lock_id, oncluster=on_cluster)
# Prevent model_unlock from being executed again at the end
# Reset on.exit: Prevent model_unlock from being executed again at the end
# and remove "setwd(cfg$results_folder)" from on.exit, becaue we change to it in the next line
on.exit()
# Repeat command since on.exit was cleared
on.exit(setwd(cfg$results_folder))
################## M O D E L U N L O C K ###################################
setwd(cfg$results_folder)
......@@ -563,14 +562,6 @@ prepare_and_run <- function() {
manipulateFile("full.gms", full_manipulateThis)
}
# Set value source_include so that loaded scripts know, that they are
# included as source (instead a load from command line)
source_include <- TRUE
# unzip all .gz files
system("gzip -d -f *.gz")
#AJS set MAGCFG file
magcfgFile = paste0('./magicc/MAGCFG_STORE/','MAGCFG_USER_',toupper(cfg$gms$cm_magicc_config),'.CFG')
if(!file.exists(magcfgFile)){
......@@ -582,12 +573,32 @@ prepare_and_run <- function() {
if ( cfg$gms$cm_startyear > 2005 & (!file.exists("levs.gms.gz") | !file.exists("levs.gms"))) {
create_fixing_files(cfg = cfg, input_ref_file = "input_ref.gdx")
}
# Print message
cat("\nStarting REMIND...\n")
timePrepareEnd <- Sys.time()
# Save run statistics to local file
cat("Saving timePrepareStart and timePrepareEnd to runstatistics.rda\n")
lucode::runstatistics(file = paste0("runstatistics.rda"),
timePrepareStart = timePrepareStart,
timePrepareEnd = timePrepareEnd)
# on.exit sets working directory to results folder
} # end of function "prepare"
##################################################################################################
# function: run #
##################################################################################################
run <- function(start_subsequent_runs = TRUE) {
load("config.Rdata")
on.exit(setwd(cfg$results_folder))
# Save start time
begin <- Sys.time()
timeGAMSStart <- Sys.time()
# Print message
cat("\nStarting REMIND...\n")
# Call GAMS
if (cfg$gms$CES_parameters == "load") {
......@@ -669,8 +680,10 @@ prepare_and_run <- function() {
stop("unknown realisation of 29_CES_parameters")
}
# Calculate run time
gams_runtime <- Sys.time() - begin
# Calculate run time statistics
timeGAMSEnd <- Sys.time()
gams_runtime <- timeGAMSEnd - timeGAMSStart
timeOutputStart <- Sys.time()
# If REMIND actually did run
if (cfg$action == "ce" && cfg$gms$c_skip_output != "on") {
......@@ -701,111 +714,137 @@ prepare_and_run <- function() {
if (cfg$gms$cm_startyear > 2005)
system("gzip -f levs.gms margs.gms fixings.gms")
# go up to the main folder, where the cfg files for subsequent runs are stored
# go up to the main folder, where the cfg files for subsequent runs are stored and the output scripts are executed from
setwd(cfg$remind_folder)
#====================== Subsequent runs ===========================
if (start_subsequent_runs) {
# 1. Save the path to the fulldata.gdx of the current run to the cfg files
# of the runs that use it as 'input_bau.gdx'
# 1. Save the path to the fulldata.gdx of the current run to the cfg files
# of the runs that use it as 'input_bau.gdx'
# Use the name to check whether it is a coupled run (TRUE if the name ends with "-rem-xx")
coupled_run <- grepl("-rem-[0-9]{1,2}$",cfg$title)
# Use the name to check whether it is a coupled run (TRUE if the name ends with "-rem-xx")
coupled_run <- grepl("-rem-[0-9]{1,2}$",cfg$title)
no_ref_runs <- identical(cfg$RunsUsingTHISgdxAsBAU,character(0)) | all(is.na(cfg$RunsUsingTHISgdxAsBAU)) | coupled_run
no_ref_runs <- identical(cfg$RunsUsingTHISgdxAsBAU,character(0)) | all(is.na(cfg$RunsUsingTHISgdxAsBAU)) | coupled_run
if(!no_ref_runs) {
source("scripts/start/submit.R")
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
for(run in seq(1,length(cfg_main$RunsUsingTHISgdxAsBAU))){
# for each of the runs that use this gdx as bau, read in the cfg, ...
cat("Writing the path for input_bau.gdx to ",paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"),"\n")
load(paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
# ...change the path_gdx_bau field of the subsequent run to the fulldata gdx of the current run ...
cfg$files2export$start['input_bau.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
if(!no_ref_runs) {
source("scripts/start/submit.R")
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
for(run in seq(1,length(cfg_main$RunsUsingTHISgdxAsBAU))){
# for each of the runs that use this gdx as bau, read in the cfg, ...
cat("Writing the path for input_bau.gdx to ",paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"),"\n")
load(paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
# ...change the path_gdx_bau field of the subsequent run to the fulldata gdx of the current run ...
cfg$files2export$start['input_bau.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
}
# Set cfg back to original
cfg <- cfg_main
}
# Set cfg back to original
cfg <- cfg_main
}
# 2. Save the path to the fulldata.gdx of the current run to the cfg files
# of the subsequent runs that use it as 'input_ref.gdx' and start these runs
# 2. Save the path to the fulldata.gdx of the current run to the cfg files
# of the subsequent runs that use it as 'input_ref.gdx' and start these runs
no_subsequent_runs <- identical(cfg$subsequentruns,character(0)) | identical(cfg$subsequentruns,NULL) | coupled_run
no_subsequent_runs <- identical(cfg$subsequentruns,character(0)) | identical(cfg$subsequentruns,NULL) | coupled_run
if(no_subsequent_runs){
cat('\nNo subsequent run was set for this scenario\n')
} else {
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
source("scripts/start/submit.R")
for(run in seq(1,length(cfg_main$subsequentruns))){
# for each of the subsequent runs, read in the cfg, ...
cat("Writing the path for input_ref.gdx to ",paste0(cfg_main$subsequentruns[run],".RData"),"\n")
load(paste0(cfg_main$subsequentruns[run],".RData"))
# ...change the path_gdx_ref field of the subsequent run to the fulldata gdx of the current (preceding) run ...
cfg$files2export$start['input_ref.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$subsequentruns[run],".RData"))
if(no_subsequent_runs){
cat('\nNo subsequent run was set for this scenario\n')
} else {
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
source("scripts/start/submit.R")
# Subsequent runs will be started in submit.R using the RData files written above
# after the current run has finished.
cat("Starting subsequent run ",cfg_main$subsequentruns[run],"\n")
submit(cfg)
for(run in seq(1,length(cfg_main$subsequentruns))){
# for each of the subsequent runs, read in the cfg, ...
cat("Writing the path for input_ref.gdx to ",paste0(cfg_main$subsequentruns[run],".RData"),"\n")
load(paste0(cfg_main$subsequentruns[run],".RData"))
# ...change the path_gdx_ref field of the subsequent run to the fulldata gdx of the current (preceding) run ...
cfg$files2export$start['input_ref.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$subsequentruns[run],".RData"))
# Subsequent runs will be started in submit.R using the RData files written above
# after the current run has finished.
cat("Starting subsequent run ",cfg_main$subsequentruns[run],"\n")
submit(cfg)
}
# Set cfg back to original
cfg <- cfg_main
}
# Set cfg back to original
cfg <- cfg_main
}
# 3. Create script file that can be used later to restart the subsequent runs manually.
# In case there are no subsequent runs (or it's coupled runs), the file contains only
# a small message.
# 3. Create script file that can be used later to restart the subsequent runs manually.
# In case there are no subsequent runs (or it's coupled runs), the file contains only
# a small message.
subseq_start_file <- paste0(cfg$results_folder,"/start_subsequentruns_manually.R")
subseq_start_file <- paste0(cfg$results_folder,"/start_subsequentruns_manually.R")
if(no_subsequent_runs){
write("cat('\nNo subsequent run was set for this scenario\n')",file=subseq_start_file)
} else {
# go up to the main folder, where the cfg. files for subsequent runs are stored
filetext <- paste0("setwd('",cfg$remind_folder,"')\n")
filetext <- paste0(filetext,"source('scripts/start/submit.R')\n")
for(run in seq(1,length(cfg$subsequentruns))){
filetext <- paste0(filetext,"\n")
filetext <- paste0(filetext,"load('",cfg$subsequentruns[run],".RData')\n")
#filetext <- paste0(filetext,"cfg$results_folder <- 'output/:title::date:'\n")
filetext <- paste0(filetext,"cat('",cfg$subsequentruns[run],"')\n")
filetext <- paste0(filetext,"submit(cfg)\n")
if(no_subsequent_runs){
write("cat('\nNo subsequent run was set for this scenario\n')",file=subseq_start_file)
} else {
# go up to the main folder, where the cfg. files for subsequent runs are stored
filetext <- paste0("setwd('",cfg$remind_folder,"')\n")
filetext <- paste0(filetext,"source('scripts/start/submit.R')\n")
for(run in seq(1,length(cfg$subsequentruns))){
filetext <- paste0(filetext,"\n")
filetext <- paste0(filetext,"load('",cfg$subsequentruns[run],".RData')\n")
#filetext <- paste0(filetext,"cfg$results_folder <- 'output/:title::date:'\n")
filetext <- paste0(filetext,"cat('",cfg$subsequentruns[run],"')\n")
filetext <- paste0(filetext,"submit(cfg)\n")
}
# Write the text to the file
write(filetext,file=subseq_start_file)
}
# Write the text to the file
write(filetext,file=subseq_start_file)
}
#=================== END - Subsequent runs ========================
# Copy important files into output_folder (after REMIND execution)
for (file in cfg$files2export$end)
file.copy(file, cfg$results_folder, overwrite = TRUE)
# Set source_include so that loaded scripts know they are included as
# source (instead of being executed from the command line)
source_include <- TRUE
# Postprocessing / Output Generation
output <- cfg$output
outputdir <- cfg$results_folder
sys.source("output.R",envir=new.env())
end_time <- Sys.time()
# get runtime for output
timeOutputEnd <- Sys.time()
# Save run statistics to local file
cat("Saving start_time and end_time to runstatistics.rda\n")
lucode::runstatistics(file = paste0(cfg$results_folder,"/runstatistics.rda"),
starttime = start_time,
endtime = end_time)
cat("Saving timeGAMSStart, timeGAMSEnd, timeOutputStart and timeOutputStart to runstatistics.rda\n")
lucode::runstatistics(file = paste0(cfg$results_folder, "/runstatistics.rda"),
timeGAMSStart = timeGAMSStart,
timeGAMSEnd = timeGAMSEnd,
timeOutputStart = timeOutputStart,
timeOutputEnd = timeOutputEnd)
return(cfg$results_folder)
# on.exit sets working directory back to results folder
} # end of function "run"
##################################################################################################
# script #
##################################################################################################
# Call prepare and run without cfg, because cfg is read from results folder, where it has been
# copied to by submit(cfg)
if (!file.exists("fulldata.gdx")) {
# If no "fulldata.gdx" exists, the script assumes that REMIND did not run before and
# prepares all inputs before starting the run.
prepare()
start_subsequent_runs <- TRUE
} else {
# If "fulldata.gdx" exists, the script assumes that REMIND did run before and you want
# to restart REMIND in the same folder using the gdx that it previously produced.
file.copy("fulldata.gdx", "input.gdx", overwrite = TRUE)
start_subsequent_runs <- FALSE
}
# call prepare and run (always without cfg, because cfg is always read from results folder, where it has been copied by submit(cfg))
prepare_and_run()
\ No newline at end of file
# Run REMIND, start subsequent runs (if applicable), and produce output.
run(start_subsequent_runs)
scripts/start/submit.R
+ 33
− 31
View file @ b665b95b
......@@ -13,40 +13,42 @@
############## Define function: runsubmit #########################
submit <- function(cfg) {
submit <- function(cfg, restart = FALSE) {
# Generate name of output folder and create the folder
date <- format(Sys.time(), "_%Y-%m-%d_%H.%M.%S")
cfg$results_folder <- gsub(":date:", date, cfg$results_folder, fixed = TRUE)
cfg$results_folder <- gsub(":title:", cfg$title, cfg$results_folder, fixed = TRUE)
# Create output folder
cat(" Creating results folder",cfg$results_folder,"\n")
if (!file.exists(cfg$results_folder)) {
dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
} else if (!cfg$force_replace) {
stop(paste0("Results folder ",cfg$results_folder," could not be created because it already exists."))
} else {
cat(" Deleting results folder because it alreay exists:",cfg$results_folder,"\n")
unlink(cfg$results_folder, recursive = TRUE)
dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
if(!restart) {
# Generate name of output folder and create the folder
date <- format(Sys.time(), "_%Y-%m-%d_%H.%M.%S")
cfg$results_folder <- gsub(":date:", date, cfg$results_folder, fixed = TRUE)
cfg$results_folder <- gsub(":title:", cfg$title, cfg$results_folder, fixed = TRUE)
# Create output folder
cat(" Creating results folder",cfg$results_folder,"\n")
if (!file.exists(cfg$results_folder)) {
dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
} else if (!cfg$force_replace) {
stop(paste0("Results folder ",cfg$results_folder," could not be created because it already exists."))
} else {
cat(" Deleting results folder because it alreay exists:",cfg$results_folder,"\n")
unlink(cfg$results_folder, recursive = TRUE)
dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
}
# remember main folder
cfg$remind_folder <- getwd()
# Save the cfg (with the updated name of the result folder) into the results folder.
# Do not save the new name of the results folder to the .RData file in REMINDs main folder, because it
# might be needed to restart subsequent runs manually and should not contain the time stamp in this case.
filename <- paste0(cfg$results_folder,"/config.Rdata")
cat(" Writing cfg to file",filename,"\n")
save(cfg,file=filename)
# Copy files required to confiugre and start a run
filelist <- c("prepare_and_run.R" = "scripts/start/prepare_and_run.R")
.copy.fromlist(filelist,cfg$results_folder)
# Do not remove .RData files from REMIND main folder because they are needed in case you need to manually restart subsequent runs.
}
# remember main folder
cfg$remind_folder <- getwd()
# Save the cfg (with the updated name of the result folder) into the results folder.
# Do not save the new name of the results folder to the .RData file in REMINDs main folder, because it
# might be needed to restart subsequent runs manually and should not contain the time stamp in this case.
filename <- paste0(cfg$results_folder,"/config.Rdata")
cat(" Writing cfg to file",filename,"\n")
save(cfg,file=filename)
# Copy files required to confiugre and start a run
filelist <- c("prepare_and_run.R" = "scripts/start/prepare_and_run.R")
.copy.fromlist(filelist,cfg$results_folder)
# Do not remove .RData files from REMIND main folder because they are needed in case you need to manually restart subsequent runs.
# Change to run folder
setwd(cfg$results_folder)
on.exit(setwd(cfg$remind_folder))
......
start.R
+ 153
− 63
View file @ b665b95b
......@@ -4,6 +4,70 @@ library(lucode)
source("scripts/start/submit.R")
source("scripts/start/choose_slurmConfig.R")
############## Define function: get_line ##############################
get_line <- function(){
# gets characters (line) from the terminal of from a connection
# and stores it in the return object
if(interactive()){
s <- readline()
} else {
con <- file("stdin")
s <- readLines(con, 1, warn=FALSE)
on.exit(close(con))
}
return(s);
}
############## Define function: choose_folder #########################
choose_folder <- function(folder,title="Please choose a folder") {
dirs <- NULL
# Detect all output folders containing fulldata.gdx
# For coupled runs please use the outcommented text block below
dirs <- sub("/fulldata.gdx","",sub("./output/","",Sys.glob(file.path(folder,"*","fulldata.gdx"))))
# DK: The following outcommented lines are specially made for listing results of coupled runs
#runs <- findCoupledruns(folder)
#dirs <- findIterations(runs,modelpath=folder,latest=TRUE)
#dirs <- sub("./output/","",dirs)
dirs <- c("all",dirs)
cat("\n\n",title,":\n\n")
cat(paste(1:length(dirs), dirs, sep=": " ),sep="\n")
cat(paste(length(dirs)+1, "Search by the pattern.\n", sep=": "))
cat("\nNumber: ")
identifier <- get_line()
identifier <- strsplit(identifier,",")[[1]]
tmp <- NULL
for (i in 1:length(identifier)) {
if (length(strsplit(identifier,":")[[i]]) > 1) tmp <- c(tmp,as.numeric(strsplit(identifier,":")[[i]])[1]:as.numeric(strsplit(identifier,":")[[i]])[2])
else tmp <- c(tmp,as.numeric(identifier[i]))
}
identifier <- tmp
# PATTERN
if(length(identifier==1) && identifier==(length(dirs)+1)){
cat("\nInsert the search pattern or the regular expression: ")
pattern <- get_line()
id <- grep(pattern=pattern, dirs[-1])
# lists all chosen directories and ask for the confirmation of the made choice
cat("\n\nYou have chosen the following directories:\n")
cat(paste(1:length(id), dirs[id+1], sep=": "), sep="\n")
cat("\nAre you sure these are the right directories?(y/n): ")
answer <- get_line()
if(answer=="y"){
return(dirs[id+1])
} else choose_folder(folder,title)
#
} else if(any(dirs[identifier] == "all")){
identifier <- 2:length(dirs)
return(dirs[identifier])
} else return(dirs[identifier])
}
############## Define function: configure_cfg #########################
configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
......@@ -87,84 +151,110 @@ configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
return(icfg)
}
###################### Choose submission type #########################
slurmConfig <- choose_slurmConfig()
###################### Load csv if provided ##########################
# If testOneRegi was selected, set up a testOneRegi run. IF a
# scenario_config.csv file was provided, set cfg according to it (copy from
# start_bundle).
# check command-line arguments for testOneRegi and scenario_config file
argv <- commandArgs(trailingOnly = TRUE)
config.file <- argv[1]
if ('--testOneRegi' %in% argv) {
testOneRegi <- TRUE
config.file <- NA
} else {
testOneRegi <- FALSE
}
if (!is.na(config.file)) {
cat(paste("\nReading config file", config.file, "\n"))
# define arguments that are accepted
accepted <- c('--restart','--testOneRegi')
# Read-in the switches table, use first column as row names
settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
# Select scenarios that are flagged to start
scenarios <- settings[settings$start==1,]
if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
} else {
# if no csv was provided create dummy list with default as the only scenario
scenarios <- data.frame("default" = "default",row.names = "default")
# check if user provided any unknown arguments or config files that do not exist
known <- argv %in% accepted
if (!all(known)) {
file_exists <- file.exists(argv[!known])
if (!all(file_exists)) stop("Unknown paramter provided: ",paste(argv[!known][!file_exists]," "))
}
###################### Loop over csv ###############################
# Modify and save cfg for all runs
for (scen in rownames(scenarios)) {
#source cfg file for each scenario to avoid duplication of gdx entries in files2export
source("config/default.cfg")
###################### Choose submission type #########################
slurmConfig <- choose_slurmConfig()
# Have the log output written in a file (not on the screen)
cfg$slurmConfig <- slurmConfig
cfg$logoption <- 2
start_now <- TRUE
# Restart REMIND in existing results folder (if required by user)
if ('--restart' %in% argv) {
# choose results folder from list
outputdirs <- choose_folder("./output","Please choose the runs to be restarted")
for (outputdir in outputdirs) {
cat("Restarting",outputdir,"\n")
load(paste0("output/",outputdir,"/config.Rdata")) # read config.Rdata from results folder
submit(cfg, restart = TRUE)
#cat(paste0("output/",outputdir,"/config.Rdata"),"\n")
}
} else {
# testOneRegi settings
if (testOneRegi) {
cfg$title <- 'testOneRegi'
cfg$gms$optimization <- 'testOneRegi'
cfg$output <- NA
cfg$results_folder <- 'output/testOneRegi'
# If testOneRegi was selected, set up a testOneRegi run.
# delete existing Results directory
cfg$force_replace <- TRUE
if ('--testOneRegi' %in% argv) {
testOneRegi <- TRUE
config.file <- NA
} else {
testOneRegi <- FALSE
}
cat("\n",scen,"\n")
###################### Load csv if provided ##########################
# If a scenario_config.csv file was provided, set cfg according to it.
# configure cfg based on settings from csv if provided
if (!is.na(config.file)) {
cfg <- configure_cfg(cfg, scen, scenarios, settings)
# Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx"
| is.na(scenarios[scen,"path_gdx_ref"]))
}
# save the cfg data for later start of subsequent runs (after preceding run finished)
filename <- paste0(scen,".RData")
cat(" Writing cfg to file",filename,"\n")
save(cfg,file=filename)
if (start_now){
# Create results folder and start run
submit(cfg)
} else {
cat(" Waiting for", scenarios[scen,'path_gdx_ref'] ,"\n")
cat(paste("\nReading config file", config.file, "\n"))
# Read-in the switches table, use first column as row names
settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
# Select scenarios that are flagged to start
scenarios <- settings[settings$start==1,]
if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
} else {
# if no csv was provided create dummy list with default as the only scenario
scenarios <- data.frame("default" = "default",row.names = "default")
}
if (!identical(cfg$subsequentruns,character(0))) cat(" Subsequent runs:",cfg$subsequentruns,"\n")
###################### Loop over scenarios ###############################
# Modify and save cfg for all runs
for (scen in rownames(scenarios)) {
#source cfg file for each scenario to avoid duplication of gdx entries in files2export
source("config/default.cfg")
# Have the log output written in a file (not on the screen)
cfg$slurmConfig <- slurmConfig
cfg$logoption <- 2
start_now <- TRUE
# testOneRegi settings
if (testOneRegi) {
cfg$title <- 'testOneRegi'
cfg$gms$optimization <- 'testOneRegi'
cfg$output <- NA
cfg$results_folder <- 'output/testOneRegi'
# delete existing Results directory
cfg$force_replace <- TRUE
}
cat("\n",scen,"\n")
# configure cfg based on settings from csv if provided
if (!is.na(config.file)) {
cfg <- configure_cfg(cfg, scen, scenarios, settings)
# Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx"
| is.na(scenarios[scen,"path_gdx_ref"]))
}
# save the cfg data for later start of subsequent runs (after preceding run finished)
filename <- paste0(scen,".RData")
cat(" Writing cfg to file",filename,"\n")
save(cfg,file=filename)
if (start_now){
# Create results folder and start run
submit(cfg)
} else {
cat(" Waiting for", scenarios[scen,'path_gdx_ref'] ,"\n")
}
if (!identical(cfg$subsequentruns,character(0))) cat(" Subsequent runs:",cfg$subsequentruns,"\n")
}
}
tutorials/2_RunningREMIND.md
+ 12
− 2
View file @ b665b95b
......@@ -104,7 +104,7 @@ Rscript start.R config/scenario_config_XYZ.csv
Note: Please do not make changes to the REMIND code until the last run has stared running GAMS (including subsequent runs).
A message similar to following confirms that your runs has been submitted to the cluster: `The job "cwsa.iplex.pik-potsdam.de.65539" has been submitted.`
A message similar to following confirms that your runs has been submitted to the cluster: `Submitted batch job 15489230`
You can check if the run has been accepted by the cluster just by using the command
......@@ -113,7 +113,7 @@ sq
```
in the terminal.
To see how far your run is or whether it was stopped due to some problems, go to the `Output` folder and type
To see how far your run is or whether it was stopped due to some problems, go to the `output` folder and type
``` bash
rs
......@@ -125,6 +125,16 @@ NOTE: A few words on the scripts that we currently use to start runs. The script
- they create the full.gms file and compile the needed files to start a run in a subfolder of the output folder
Restarting runs
-----------------------
Sometimes you want to restart a run in its already existing results folder whithout creating a new results folder and without compiling a new full.gms., e.g. you want a nash run to perform additional nash iterations because you are not satisfied with the convergence so far. Adding the parameter `--restart` displays a list of existing runs and lets you choose the run(s) you want to restart:
``` bash
Rscript start.R --restart
```
This will use the result of the previous optimization (fulldata.gdx) as input for the restart.
2. What happens during a REMIND run?
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment