diff --git a/config/scenario_config_SSPSDP.csv b/config/scenario_config_SSPSDP.csv
index 92159777de62b614fdfc48ca765843578561dc26..09f25744f9bfb3c7993b872130255531dd340c15 100644
--- a/config/scenario_config_SSPSDP.csv
+++ b/config/scenario_config_SSPSDP.csv
@@ -2,14 +2,14 @@ title;start;CES_parameters;cm_bioenergy_tax;cm_rcp_scen;cm_iterative_target_adj;
SSP1-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;2;1.025;0;2;1;none;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;off;SSP1;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SSP1-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;2;1.025;0;2;1;none;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;off;SSP1;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SSP1-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
-SSP1-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NPi2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
+SSP1-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;2;1.025;0;2;1;NPi2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;3;off;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;60;2045;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SSP1-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9;0.09;2;1.025;0;2;1;NDC2018;pop_SSP1;gdp_SSP1;lowOil;lowGas;lowCoal;2;2;4;2;on;SSP1;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SSP1-Base
SDP-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.12;2;1.025;4;5;1;none;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;off;SDP;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SDP-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.12;2;1.025;4;5;1;none;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;off;SDP;1;1;forcing_SSP1;SSP1;2005;./config/input.gdx;;
SDP-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
-SDP-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NPi2018;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
+SDP-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.12;2;1.025;4;5;1;NPi2018;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;3;off;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;200;2045;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg1000;1;load;1.75;rcp20;9;1;1;1000;diffCurvPhaseIn2Lin;150;2045;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
SDP-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.12;2;1.025;4;5;1;NDC2018plus;pop_SDP;gdp_SDP;lowOil;lowGas;lowCoal;2;2;4;2;on;SDP;1;1;forcing_SSP1;SSP1;2025;./config/input.gdx;SSP2-NDC;SDP-Base
@@ -17,14 +17,14 @@ SDP-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9
SSP2-calibrate;0;calibrate;1.5;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;3;1.05;0;1;1;none;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;off;SSP2;2;0;forcing_SSP2;SSP2;2005;./config/input.gdx;;
SSP2-Base;1;load;1.5;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;3;1.05;0;1;1;none;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;off;SSP2;2;0;forcing_SSP2;SSP2;2005;./config/input.gdx;;
SSP2-NDC;1;load;1.5;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2015;./config/input.gdx;SSP2-Base;SSP2-Base
-SSP2-NPi;0;load;1.5;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NPi2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
+SSP2-NPi;1;load;1.5;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;3;1.05;0;1;1;NPi2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;3;off;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg900;1;load;1.5;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;200;2045;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg1100;1;load;1.5;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;100;2080;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP2-PkBudg1300;1;load;1.5;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;60;2100;3;2050;9;0.09;3;1.05;0;1;1;NDC2018;pop_SSP2;gdp_SSP2;medOil;medGas;medCoal;1;2;1;2;on;SSP2;2;0;forcing_SSP2;SSP2;2025;./config/input.gdx;SSP2-NDC;SSP2-Base
SSP5-calibrate;0;calibrate;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;1;1.05;0;3;2;none;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;off;SSP5;1;1;forcing_SSP5;SSP5;2005;./config/input.gdx;;
SSP5-Base;1;load;1.75;none;0;1;1;0;none;-1;2100;3;2050;1;0.09;1;1.05;0;3;2;none;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;off;SSP5;1;1;forcing_SSP5;SSP5;2005;./config/input.gdx;;
SSP5-NDC;1;load;1.75;rcp45;3;1;1;0;NDC2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
-SSP5-NPi;0;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NPi2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
+SSP5-NPi;1;load;1.75;rcp45;3;1;1;0;NPi2018;1;2100;3;2050;9;0.09;1;1.05;0;3;2;NPi2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;3;off;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg900;1;load;1.75;rcp20;9;1;1;900;diffCurvPhaseIn2Lin;300;2045;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg1100;1;load;1.75;rcp26;9;1;1;1100;diffCurvPhaseIn2Lin;150;2080;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
SSP5-PkBudg1300;1;load;1.75;rcp26;9;1;1;1300;diffCurvPhaseIn2Lin;80;2100;3;2050;9;0.09;1;1.05;0;3;2;NDC2018;pop_SSP5;gdp_SSP5;highOil;highGas;highCoal;3;6;4;2;on;SSP5;1;1;forcing_SSP5;SSP5;2025;./config/input.gdx;SSP2-NDC;SSP5-Base
diff --git a/output.R b/output.R
index 22448342e613d9b5bcde3290a964cc2afcd86f67..b5cac1212a7e20dc7cb9f600f476bdcdce230ae6 100644
--- a/output.R
+++ b/output.R
@@ -241,13 +241,6 @@ if (comp==TRUE) {
#included as source (instead of a load from command line)
source_include <- TRUE
- if (file.exists(path(outputdir,"output.RData"))) {
- data_workspace<-path(outputdir,"output.RData")
- } else {
- tmp<-strsplit(outputdir,"/")[[1]]
- data_workspace<-paste0(outputdir,"/",tmp[length(tmp)],".RData")
- }
- print(data_workspace)
cat(paste("\nStarting output generation for",outputdir,"\n\n"))
###################################################################################
diff --git a/scripts/start/prepare_and_run.R b/scripts/start/prepare_and_run.R
index 62e2ce5f70df88dd36021758341389e72e49ecf0..6539f01e16730c6d1bbf83e5e73fb9406bcbbb63 100644
--- a/scripts/start/prepare_and_run.R
+++ b/scripts/start/prepare_and_run.R
@@ -2,6 +2,10 @@ library(lucode, quietly = TRUE,warn.conflicts =FALSE)
library(dplyr, quietly = TRUE,warn.conflicts =FALSE)
require(gdx)
+##################################################################################################
+# function: getReportData #
+##################################################################################################
+
getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="costs") {
require(lucode, quietly = TRUE,warn.conflicts =FALSE)
require(magclass, quietly = TRUE,warn.conflicts =FALSE)
@@ -138,13 +142,13 @@ getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="c
#.agriculture_tradebal(mag)
}
-###############################################################################
-###############################################################################
-###############################################################################
+##################################################################################################
+# function: prepare #
+##################################################################################################
-prepare_and_run <- function() {
-
- start_time <- Sys.time()
+prepare <- function() {
+
+ timePrepareStart <- Sys.time()
# Load libraries
require(lucode, quietly = TRUE,warn.conflicts =FALSE)
@@ -200,10 +204,6 @@ prepare_and_run <- function() {
if (on_cluster)
system("find ./core/magicc/ -type f | xargs dos2unix -q")
- # Set source_include so that loaded scripts know they are included as
- # source (instead of being executed from the command line)
- source_include <- TRUE
-
################## M O D E L L O C K ###################################
# Lock the directory for other instances of the start scritps
lock_id <- model_lock(timeout1 = 1, oncluster=on_cluster)
@@ -393,10 +393,9 @@ prepare_and_run <- function() {
################## M O D E L U N L O C K ###################################
# After full.gms was produced remind folders have to be unlocked to allow setting up the next run
model_unlock(lock_id, oncluster=on_cluster)
- # Prevent model_unlock from being executed again at the end
+ # Reset on.exit: Prevent model_unlock from being executed again at the end
+ # and remove "setwd(cfg$results_folder)" from on.exit, becaue we change to it in the next line
on.exit()
- # Repeat command since on.exit was cleared
- on.exit(setwd(cfg$results_folder))
################## M O D E L U N L O C K ###################################
setwd(cfg$results_folder)
@@ -563,14 +562,6 @@ prepare_and_run <- function() {
manipulateFile("full.gms", full_manipulateThis)
}
-
- # Set value source_include so that loaded scripts know, that they are
- # included as source (instead a load from command line)
- source_include <- TRUE
-
- # unzip all .gz files
- system("gzip -d -f *.gz")
-
#AJS set MAGCFG file
magcfgFile = paste0('./magicc/MAGCFG_STORE/','MAGCFG_USER_',toupper(cfg$gms$cm_magicc_config),'.CFG')
if(!file.exists(magcfgFile)){
@@ -582,12 +573,32 @@ prepare_and_run <- function() {
if ( cfg$gms$cm_startyear > 2005 & (!file.exists("levs.gms.gz") | !file.exists("levs.gms"))) {
create_fixing_files(cfg = cfg, input_ref_file = "input_ref.gdx")
}
-
- # Print message
- cat("\nStarting REMIND...\n")
+
+ timePrepareEnd <- Sys.time()
+ # Save run statistics to local file
+ cat("Saving timePrepareStart and timePrepareEnd to runstatistics.rda\n")
+ lucode::runstatistics(file = paste0("runstatistics.rda"),
+ timePrepareStart = timePrepareStart,
+ timePrepareEnd = timePrepareEnd)
+
+ # on.exit sets working directory to results folder
+
+} # end of function "prepare"
+
+##################################################################################################
+# function: run #
+##################################################################################################
+run <- function(start_subsequent_runs = TRUE) {
+
+ load("config.Rdata")
+ on.exit(setwd(cfg$results_folder))
+
# Save start time
- begin <- Sys.time()
+ timeGAMSStart <- Sys.time()
+
+ # Print message
+ cat("\nStarting REMIND...\n")
# Call GAMS
if (cfg$gms$CES_parameters == "load") {
@@ -669,8 +680,10 @@ prepare_and_run <- function() {
stop("unknown realisation of 29_CES_parameters")
}
- # Calculate run time
- gams_runtime <- Sys.time() - begin
+ # Calculate run time statistics
+ timeGAMSEnd <- Sys.time()
+ gams_runtime <- timeGAMSEnd - timeGAMSStart
+ timeOutputStart <- Sys.time()
# If REMIND actually did run
if (cfg$action == "ce" && cfg$gms$c_skip_output != "on") {
@@ -701,111 +714,137 @@ prepare_and_run <- function() {
if (cfg$gms$cm_startyear > 2005)
system("gzip -f levs.gms margs.gms fixings.gms")
- # go up to the main folder, where the cfg files for subsequent runs are stored
+ # go up to the main folder, where the cfg files for subsequent runs are stored and the output scripts are executed from
setwd(cfg$remind_folder)
#====================== Subsequent runs ===========================
+ if (start_subsequent_runs) {
+ # 1. Save the path to the fulldata.gdx of the current run to the cfg files
+ # of the runs that use it as 'input_bau.gdx'
- # 1. Save the path to the fulldata.gdx of the current run to the cfg files
- # of the runs that use it as 'input_bau.gdx'
-
- # Use the name to check whether it is a coupled run (TRUE if the name ends with "-rem-xx")
- coupled_run <- grepl("-rem-[0-9]{1,2}$",cfg$title)
+ # Use the name to check whether it is a coupled run (TRUE if the name ends with "-rem-xx")
+ coupled_run <- grepl("-rem-[0-9]{1,2}$",cfg$title)
- no_ref_runs <- identical(cfg$RunsUsingTHISgdxAsBAU,character(0)) | all(is.na(cfg$RunsUsingTHISgdxAsBAU)) | coupled_run
+ no_ref_runs <- identical(cfg$RunsUsingTHISgdxAsBAU,character(0)) | all(is.na(cfg$RunsUsingTHISgdxAsBAU)) | coupled_run
- if(!no_ref_runs) {
- source("scripts/start/submit.R")
- # Save the current cfg settings into a different data object, so that they are not overwritten
- cfg_main <- cfg
-
- for(run in seq(1,length(cfg_main$RunsUsingTHISgdxAsBAU))){
- # for each of the runs that use this gdx as bau, read in the cfg, ...
- cat("Writing the path for input_bau.gdx to ",paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"),"\n")
- load(paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
- # ...change the path_gdx_bau field of the subsequent run to the fulldata gdx of the current run ...
- cfg$files2export$start['input_bau.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
- save(cfg, file = paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
+ if(!no_ref_runs) {
+ source("scripts/start/submit.R")
+ # Save the current cfg settings into a different data object, so that they are not overwritten
+ cfg_main <- cfg
+
+ for(run in seq(1,length(cfg_main$RunsUsingTHISgdxAsBAU))){
+ # for each of the runs that use this gdx as bau, read in the cfg, ...
+ cat("Writing the path for input_bau.gdx to ",paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"),"\n")
+ load(paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
+ # ...change the path_gdx_bau field of the subsequent run to the fulldata gdx of the current run ...
+ cfg$files2export$start['input_bau.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
+ save(cfg, file = paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
+ }
+ # Set cfg back to original
+ cfg <- cfg_main
}
- # Set cfg back to original
- cfg <- cfg_main
- }
- # 2. Save the path to the fulldata.gdx of the current run to the cfg files
- # of the subsequent runs that use it as 'input_ref.gdx' and start these runs
+ # 2. Save the path to the fulldata.gdx of the current run to the cfg files
+ # of the subsequent runs that use it as 'input_ref.gdx' and start these runs
- no_subsequent_runs <- identical(cfg$subsequentruns,character(0)) | identical(cfg$subsequentruns,NULL) | coupled_run
+ no_subsequent_runs <- identical(cfg$subsequentruns,character(0)) | identical(cfg$subsequentruns,NULL) | coupled_run
- if(no_subsequent_runs){
- cat('\nNo subsequent run was set for this scenario\n')
- } else {
- # Save the current cfg settings into a different data object, so that they are not overwritten
- cfg_main <- cfg
- source("scripts/start/submit.R")
-
- for(run in seq(1,length(cfg_main$subsequentruns))){
- # for each of the subsequent runs, read in the cfg, ...
- cat("Writing the path for input_ref.gdx to ",paste0(cfg_main$subsequentruns[run],".RData"),"\n")
- load(paste0(cfg_main$subsequentruns[run],".RData"))
- # ...change the path_gdx_ref field of the subsequent run to the fulldata gdx of the current (preceding) run ...
- cfg$files2export$start['input_ref.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
- save(cfg, file = paste0(cfg_main$subsequentruns[run],".RData"))
+ if(no_subsequent_runs){
+ cat('\nNo subsequent run was set for this scenario\n')
+ } else {
+ # Save the current cfg settings into a different data object, so that they are not overwritten
+ cfg_main <- cfg
+ source("scripts/start/submit.R")
- # Subsequent runs will be started in submit.R using the RData files written above
- # after the current run has finished.
- cat("Starting subsequent run ",cfg_main$subsequentruns[run],"\n")
- submit(cfg)
+ for(run in seq(1,length(cfg_main$subsequentruns))){
+ # for each of the subsequent runs, read in the cfg, ...
+ cat("Writing the path for input_ref.gdx to ",paste0(cfg_main$subsequentruns[run],".RData"),"\n")
+ load(paste0(cfg_main$subsequentruns[run],".RData"))
+ # ...change the path_gdx_ref field of the subsequent run to the fulldata gdx of the current (preceding) run ...
+ cfg$files2export$start['input_ref.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
+ save(cfg, file = paste0(cfg_main$subsequentruns[run],".RData"))
+
+ # Subsequent runs will be started in submit.R using the RData files written above
+ # after the current run has finished.
+ cat("Starting subsequent run ",cfg_main$subsequentruns[run],"\n")
+ submit(cfg)
+ }
+ # Set cfg back to original
+ cfg <- cfg_main
}
- # Set cfg back to original
- cfg <- cfg_main
- }
- # 3. Create script file that can be used later to restart the subsequent runs manually.
- # In case there are no subsequent runs (or it's coupled runs), the file contains only
- # a small message.
+ # 3. Create script file that can be used later to restart the subsequent runs manually.
+ # In case there are no subsequent runs (or it's coupled runs), the file contains only
+ # a small message.
- subseq_start_file <- paste0(cfg$results_folder,"/start_subsequentruns_manually.R")
+ subseq_start_file <- paste0(cfg$results_folder,"/start_subsequentruns_manually.R")
- if(no_subsequent_runs){
- write("cat('\nNo subsequent run was set for this scenario\n')",file=subseq_start_file)
- } else {
- # go up to the main folder, where the cfg. files for subsequent runs are stored
- filetext <- paste0("setwd('",cfg$remind_folder,"')\n")
- filetext <- paste0(filetext,"source('scripts/start/submit.R')\n")
- for(run in seq(1,length(cfg$subsequentruns))){
- filetext <- paste0(filetext,"\n")
- filetext <- paste0(filetext,"load('",cfg$subsequentruns[run],".RData')\n")
- #filetext <- paste0(filetext,"cfg$results_folder <- 'output/:title::date:'\n")
- filetext <- paste0(filetext,"cat('",cfg$subsequentruns[run],"')\n")
- filetext <- paste0(filetext,"submit(cfg)\n")
+ if(no_subsequent_runs){
+ write("cat('\nNo subsequent run was set for this scenario\n')",file=subseq_start_file)
+ } else {
+ # go up to the main folder, where the cfg. files for subsequent runs are stored
+ filetext <- paste0("setwd('",cfg$remind_folder,"')\n")
+ filetext <- paste0(filetext,"source('scripts/start/submit.R')\n")
+ for(run in seq(1,length(cfg$subsequentruns))){
+ filetext <- paste0(filetext,"\n")
+ filetext <- paste0(filetext,"load('",cfg$subsequentruns[run],".RData')\n")
+ #filetext <- paste0(filetext,"cfg$results_folder <- 'output/:title::date:'\n")
+ filetext <- paste0(filetext,"cat('",cfg$subsequentruns[run],"')\n")
+ filetext <- paste0(filetext,"submit(cfg)\n")
+ }
+ # Write the text to the file
+ write(filetext,file=subseq_start_file)
}
- # Write the text to the file
- write(filetext,file=subseq_start_file)
}
-
#=================== END - Subsequent runs ========================
-
+
# Copy important files into output_folder (after REMIND execution)
for (file in cfg$files2export$end)
file.copy(file, cfg$results_folder, overwrite = TRUE)
+ # Set source_include so that loaded scripts know they are included as
+ # source (instead of being executed from the command line)
+ source_include <- TRUE
+
# Postprocessing / Output Generation
output <- cfg$output
outputdir <- cfg$results_folder
sys.source("output.R",envir=new.env())
-
- end_time <- Sys.time()
+ # get runtime for output
+ timeOutputEnd <- Sys.time()
+
# Save run statistics to local file
- cat("Saving start_time and end_time to runstatistics.rda\n")
- lucode::runstatistics(file = paste0(cfg$results_folder,"/runstatistics.rda"),
- starttime = start_time,
- endtime = end_time)
+ cat("Saving timeGAMSStart, timeGAMSEnd, timeOutputStart and timeOutputStart to runstatistics.rda\n")
+ lucode::runstatistics(file = paste0(cfg$results_folder, "/runstatistics.rda"),
+ timeGAMSStart = timeGAMSStart,
+ timeGAMSEnd = timeGAMSEnd,
+ timeOutputStart = timeOutputStart,
+ timeOutputEnd = timeOutputEnd)
return(cfg$results_folder)
-
# on.exit sets working directory back to results folder
+} # end of function "run"
+
+
+##################################################################################################
+# script #
+##################################################################################################
+
+# Call prepare and run without cfg, because cfg is read from results folder, where it has been
+# copied to by submit(cfg)
+
+if (!file.exists("fulldata.gdx")) {
+ # If no "fulldata.gdx" exists, the script assumes that REMIND did not run before and
+ # prepares all inputs before starting the run.
+ prepare()
+ start_subsequent_runs <- TRUE
+} else {
+ # If "fulldata.gdx" exists, the script assumes that REMIND did run before and you want
+ # to restart REMIND in the same folder using the gdx that it previously produced.
+ file.copy("fulldata.gdx", "input.gdx", overwrite = TRUE)
+ start_subsequent_runs <- FALSE
}
-# call prepare and run (always without cfg, because cfg is always read from results folder, where it has been copied by submit(cfg))
-prepare_and_run()
\ No newline at end of file
+# Run REMIND, start subsequent runs (if applicable), and produce output.
+run(start_subsequent_runs)
diff --git a/scripts/start/submit.R b/scripts/start/submit.R
index ccc81c54aa367247828ef357ae059a3ec0cb8b2f..abf4cb31f3d8152566441c84019ce871090b26e1 100644
--- a/scripts/start/submit.R
+++ b/scripts/start/submit.R
@@ -13,40 +13,42 @@
############## Define function: runsubmit #########################
-submit <- function(cfg) {
+submit <- function(cfg, restart = FALSE) {
- # Generate name of output folder and create the folder
- date <- format(Sys.time(), "_%Y-%m-%d_%H.%M.%S")
- cfg$results_folder <- gsub(":date:", date, cfg$results_folder, fixed = TRUE)
- cfg$results_folder <- gsub(":title:", cfg$title, cfg$results_folder, fixed = TRUE)
- # Create output folder
- cat(" Creating results folder",cfg$results_folder,"\n")
- if (!file.exists(cfg$results_folder)) {
- dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
- } else if (!cfg$force_replace) {
- stop(paste0("Results folder ",cfg$results_folder," could not be created because it already exists."))
- } else {
- cat(" Deleting results folder because it alreay exists:",cfg$results_folder,"\n")
- unlink(cfg$results_folder, recursive = TRUE)
- dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
+ if(!restart) {
+ # Generate name of output folder and create the folder
+ date <- format(Sys.time(), "_%Y-%m-%d_%H.%M.%S")
+ cfg$results_folder <- gsub(":date:", date, cfg$results_folder, fixed = TRUE)
+ cfg$results_folder <- gsub(":title:", cfg$title, cfg$results_folder, fixed = TRUE)
+ # Create output folder
+ cat(" Creating results folder",cfg$results_folder,"\n")
+ if (!file.exists(cfg$results_folder)) {
+ dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
+ } else if (!cfg$force_replace) {
+ stop(paste0("Results folder ",cfg$results_folder," could not be created because it already exists."))
+ } else {
+ cat(" Deleting results folder because it alreay exists:",cfg$results_folder,"\n")
+ unlink(cfg$results_folder, recursive = TRUE)
+ dir.create(cfg$results_folder, recursive = TRUE, showWarnings = FALSE)
+ }
+
+ # remember main folder
+ cfg$remind_folder <- getwd()
+
+ # Save the cfg (with the updated name of the result folder) into the results folder.
+ # Do not save the new name of the results folder to the .RData file in REMINDs main folder, because it
+ # might be needed to restart subsequent runs manually and should not contain the time stamp in this case.
+ filename <- paste0(cfg$results_folder,"/config.Rdata")
+ cat(" Writing cfg to file",filename,"\n")
+ save(cfg,file=filename)
+
+ # Copy files required to confiugre and start a run
+ filelist <- c("prepare_and_run.R" = "scripts/start/prepare_and_run.R")
+ .copy.fromlist(filelist,cfg$results_folder)
+
+ # Do not remove .RData files from REMIND main folder because they are needed in case you need to manually restart subsequent runs.
}
- # remember main folder
- cfg$remind_folder <- getwd()
-
- # Save the cfg (with the updated name of the result folder) into the results folder.
- # Do not save the new name of the results folder to the .RData file in REMINDs main folder, because it
- # might be needed to restart subsequent runs manually and should not contain the time stamp in this case.
- filename <- paste0(cfg$results_folder,"/config.Rdata")
- cat(" Writing cfg to file",filename,"\n")
- save(cfg,file=filename)
-
- # Copy files required to confiugre and start a run
- filelist <- c("prepare_and_run.R" = "scripts/start/prepare_and_run.R")
- .copy.fromlist(filelist,cfg$results_folder)
-
- # Do not remove .RData files from REMIND main folder because they are needed in case you need to manually restart subsequent runs.
-
# Change to run folder
setwd(cfg$results_folder)
on.exit(setwd(cfg$remind_folder))
diff --git a/start.R b/start.R
index 8f15d27a407511d2cccbc174ed14475dcf63f890..8419ebb4ab23d21a7de99ee0d37bf476070e6033 100755
--- a/start.R
+++ b/start.R
@@ -4,6 +4,70 @@ library(lucode)
source("scripts/start/submit.R")
source("scripts/start/choose_slurmConfig.R")
+############## Define function: get_line ##############################
+
+get_line <- function(){
+ # gets characters (line) from the terminal of from a connection
+ # and stores it in the return object
+ if(interactive()){
+ s <- readline()
+ } else {
+ con <- file("stdin")
+ s <- readLines(con, 1, warn=FALSE)
+ on.exit(close(con))
+ }
+ return(s);
+}
+
+############## Define function: choose_folder #########################
+
+choose_folder <- function(folder,title="Please choose a folder") {
+ dirs <- NULL
+
+ # Detect all output folders containing fulldata.gdx
+ # For coupled runs please use the outcommented text block below
+
+ dirs <- sub("/fulldata.gdx","",sub("./output/","",Sys.glob(file.path(folder,"*","fulldata.gdx"))))
+
+ # DK: The following outcommented lines are specially made for listing results of coupled runs
+ #runs <- findCoupledruns(folder)
+ #dirs <- findIterations(runs,modelpath=folder,latest=TRUE)
+ #dirs <- sub("./output/","",dirs)
+
+ dirs <- c("all",dirs)
+ cat("\n\n",title,":\n\n")
+ cat(paste(1:length(dirs), dirs, sep=": " ),sep="\n")
+ cat(paste(length(dirs)+1, "Search by the pattern.\n", sep=": "))
+ cat("\nNumber: ")
+ identifier <- get_line()
+ identifier <- strsplit(identifier,",")[[1]]
+ tmp <- NULL
+ for (i in 1:length(identifier)) {
+ if (length(strsplit(identifier,":")[[i]]) > 1) tmp <- c(tmp,as.numeric(strsplit(identifier,":")[[i]])[1]:as.numeric(strsplit(identifier,":")[[i]])[2])
+ else tmp <- c(tmp,as.numeric(identifier[i]))
+ }
+ identifier <- tmp
+ # PATTERN
+ if(length(identifier==1) && identifier==(length(dirs)+1)){
+ cat("\nInsert the search pattern or the regular expression: ")
+ pattern <- get_line()
+ id <- grep(pattern=pattern, dirs[-1])
+ # lists all chosen directories and ask for the confirmation of the made choice
+ cat("\n\nYou have chosen the following directories:\n")
+ cat(paste(1:length(id), dirs[id+1], sep=": "), sep="\n")
+ cat("\nAre you sure these are the right directories?(y/n): ")
+ answer <- get_line()
+ if(answer=="y"){
+ return(dirs[id+1])
+ } else choose_folder(folder,title)
+ #
+ } else if(any(dirs[identifier] == "all")){
+ identifier <- 2:length(dirs)
+ return(dirs[identifier])
+ } else return(dirs[identifier])
+}
+
+
############## Define function: configure_cfg #########################
configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
@@ -87,84 +151,110 @@ configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
return(icfg)
}
-###################### Choose submission type #########################
-slurmConfig <- choose_slurmConfig()
-
-###################### Load csv if provided ##########################
-
-# If testOneRegi was selected, set up a testOneRegi run. IF a
-# scenario_config.csv file was provided, set cfg according to it (copy from
-# start_bundle).
# check command-line arguments for testOneRegi and scenario_config file
argv <- commandArgs(trailingOnly = TRUE)
config.file <- argv[1]
-if ('--testOneRegi' %in% argv) {
- testOneRegi <- TRUE
- config.file <- NA
-} else {
- testOneRegi <- FALSE
-}
-if (!is.na(config.file)) {
- cat(paste("\nReading config file", config.file, "\n"))
+# define arguments that are accepted
+accepted <- c('--restart','--testOneRegi')
- # Read-in the switches table, use first column as row names
- settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
-
- # Select scenarios that are flagged to start
- scenarios <- settings[settings$start==1,]
- if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
-} else {
- # if no csv was provided create dummy list with default as the only scenario
- scenarios <- data.frame("default" = "default",row.names = "default")
+# check if user provided any unknown arguments or config files that do not exist
+known <- argv %in% accepted
+if (!all(known)) {
+ file_exists <- file.exists(argv[!known])
+ if (!all(file_exists)) stop("Unknown paramter provided: ",paste(argv[!known][!file_exists]," "))
}
-###################### Loop over csv ###############################
-
-# Modify and save cfg for all runs
-for (scen in rownames(scenarios)) {
- #source cfg file for each scenario to avoid duplication of gdx entries in files2export
- source("config/default.cfg")
+###################### Choose submission type #########################
+slurmConfig <- choose_slurmConfig()
- # Have the log output written in a file (not on the screen)
- cfg$slurmConfig <- slurmConfig
- cfg$logoption <- 2
- start_now <- TRUE
+# Restart REMIND in existing results folder (if required by user)
+if ('--restart' %in% argv) {
+ # choose results folder from list
+ outputdirs <- choose_folder("./output","Please choose the runs to be restarted")
+ for (outputdir in outputdirs) {
+ cat("Restarting",outputdir,"\n")
+ load(paste0("output/",outputdir,"/config.Rdata")) # read config.Rdata from results folder
+ submit(cfg, restart = TRUE)
+ #cat(paste0("output/",outputdir,"/config.Rdata"),"\n")
+ }
+
+} else {
- # testOneRegi settings
- if (testOneRegi) {
- cfg$title <- 'testOneRegi'
- cfg$gms$optimization <- 'testOneRegi'
- cfg$output <- NA
- cfg$results_folder <- 'output/testOneRegi'
+ # If testOneRegi was selected, set up a testOneRegi run.
- # delete existing Results directory
- cfg$force_replace <- TRUE
+ if ('--testOneRegi' %in% argv) {
+ testOneRegi <- TRUE
+ config.file <- NA
+ } else {
+ testOneRegi <- FALSE
}
- cat("\n",scen,"\n")
+ ###################### Load csv if provided ##########################
+
+ # If a scenario_config.csv file was provided, set cfg according to it.
- # configure cfg based on settings from csv if provided
if (!is.na(config.file)) {
- cfg <- configure_cfg(cfg, scen, scenarios, settings)
- # Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
- start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx"
- | is.na(scenarios[scen,"path_gdx_ref"]))
- }
-
- # save the cfg data for later start of subsequent runs (after preceding run finished)
- filename <- paste0(scen,".RData")
- cat(" Writing cfg to file",filename,"\n")
- save(cfg,file=filename)
-
- if (start_now){
- # Create results folder and start run
- submit(cfg)
- } else {
- cat(" Waiting for", scenarios[scen,'path_gdx_ref'] ,"\n")
+ cat(paste("\nReading config file", config.file, "\n"))
+
+ # Read-in the switches table, use first column as row names
+ settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
+
+ # Select scenarios that are flagged to start
+ scenarios <- settings[settings$start==1,]
+ if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
+ } else {
+ # if no csv was provided create dummy list with default as the only scenario
+ scenarios <- data.frame("default" = "default",row.names = "default")
}
- if (!identical(cfg$subsequentruns,character(0))) cat(" Subsequent runs:",cfg$subsequentruns,"\n")
-
+ ###################### Loop over scenarios ###############################
+
+ # Modify and save cfg for all runs
+ for (scen in rownames(scenarios)) {
+ #source cfg file for each scenario to avoid duplication of gdx entries in files2export
+ source("config/default.cfg")
+
+ # Have the log output written in a file (not on the screen)
+ cfg$slurmConfig <- slurmConfig
+ cfg$logoption <- 2
+ start_now <- TRUE
+
+ # testOneRegi settings
+ if (testOneRegi) {
+ cfg$title <- 'testOneRegi'
+ cfg$gms$optimization <- 'testOneRegi'
+ cfg$output <- NA
+ cfg$results_folder <- 'output/testOneRegi'
+
+ # delete existing Results directory
+ cfg$force_replace <- TRUE
+ }
+
+ cat("\n",scen,"\n")
+
+ # configure cfg based on settings from csv if provided
+ if (!is.na(config.file)) {
+ cfg <- configure_cfg(cfg, scen, scenarios, settings)
+ # Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
+ start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx"
+ | is.na(scenarios[scen,"path_gdx_ref"]))
+ }
+
+ # save the cfg data for later start of subsequent runs (after preceding run finished)
+ filename <- paste0(scen,".RData")
+ cat(" Writing cfg to file",filename,"\n")
+ save(cfg,file=filename)
+
+ if (start_now){
+ # Create results folder and start run
+ submit(cfg)
+ } else {
+ cat(" Waiting for", scenarios[scen,'path_gdx_ref'] ,"\n")
+ }
+
+ if (!identical(cfg$subsequentruns,character(0))) cat(" Subsequent runs:",cfg$subsequentruns,"\n")
+
+ }
}
diff --git a/tutorials/2_RunningREMIND.md b/tutorials/2_RunningREMIND.md
index b384fae6ce3b2066170be4767412308942255817..2875c4182d0bdb6fd59ca484671eb94dbb6e86b6 100644
--- a/tutorials/2_RunningREMIND.md
+++ b/tutorials/2_RunningREMIND.md
@@ -104,7 +104,7 @@ Rscript start.R config/scenario_config_XYZ.csv
Note: Please do not make changes to the REMIND code until the last run has stared running GAMS (including subsequent runs).
-A message similar to following confirms that your runs has been submitted to the cluster: `The job "cwsa.iplex.pik-potsdam.de.65539" has been submitted.`
+A message similar to following confirms that your runs has been submitted to the cluster: `Submitted batch job 15489230`
You can check if the run has been accepted by the cluster just by using the command
@@ -113,7 +113,7 @@ sq
```
in the terminal.
-To see how far your run is or whether it was stopped due to some problems, go to the `Output` folder and type
+To see how far your run is or whether it was stopped due to some problems, go to the `output` folder and type
``` bash
rs
@@ -125,6 +125,16 @@ NOTE: A few words on the scripts that we currently use to start runs. The script
- they create the full.gms file and compile the needed files to start a run in a subfolder of the output folder
+Restarting runs
+-----------------------
+
+Sometimes you want to restart a run in its already existing results folder whithout creating a new results folder and without compiling a new full.gms., e.g. you want a nash run to perform additional nash iterations because you are not satisfied with the convergence so far. Adding the parameter `--restart` displays a list of existing runs and lets you choose the run(s) you want to restart:
+
+``` bash
+Rscript start.R --restart
+```
+
+This will use the result of the previous optimization (fulldata.gdx) as input for the restart.
2. What happens during a REMIND run?