Refactoring: -part 3 (start scripts)

scripts/run_submit/prepare_and_run.R

-  library(lucode, quietly = TRUE,warn.conflicts =FALSE)
+library(lucode, quietly = TRUE,warn.conflicts =FALSE)
 library(dplyr, quietly = TRUE,warn.conflicts =FALSE)
 require(gdx)
 
-
-
 getReportData <- function(rep,inputpath_mag="magpie",inputpath_acc="costs") {
 	require(lucode, quietly = TRUE,warn.conflicts =FALSE)
   require(magclass, quietly = TRUE,warn.conflicts =FALSE)
@@ -144,7 +142,7 @@ getReportData <- function(rep,inputpath_mag="magpie",inputpath_acc="costs") {
 ###############################################################################
 ###############################################################################
 
-prepare_and_run <- function(cfg) {
+prepare_and_run <- function() {
   
   # Load libraries
   require(lucode, quietly = TRUE,warn.conflicts =FALSE)
@@ -379,6 +377,7 @@ prepare_and_run <- function(cfg) {
   save(cfg, file = path(cfg$results_folder, "config.Rdata"))
 
   # Merge GAMS files
+  cat("Creating full.gms\n")
   singleGAMSfile(mainfile=cfg$model,output = path(cfg$results_folder, "full.gms"))
   
   # Collect run statistics (will be saved to central database in submit.R)
@@ -815,3 +814,5 @@ prepare_and_run <- function(cfg) {
   
 }
 
+# call prepare and run (always without cfg, because cfg is always read from results folder, where it has been copied by submit_run(cfg))
+prepare_and_run()
\ No newline at end of file

scripts/start/submit_run.R

@@ -38,12 +38,12 @@ submit_run <- function(cfg) {
   save(cfg,file=paste0(cfg$title,".RData"))
   
   # Copy files required to confiugre and start a run
-  filelist <- c(paste0(cfg$title,".Rdata")            = "config.Rdata",
-                "scripts/run_submit/prepare_and_run.R" = "prepare_and_run.R")
+  filelist <- c("config.Rdata" = paste0(cfg$title,".RData"),
+                "prepare_and_run.R" = "scripts/run_submit/prepare_and_run.R")
   .copy.fromlist(filelist,cfg$results_folder)
   
   # remove config in main folder (after copying into results folder)
-  if(!file.remove(paste0(cfg$title,".Rdata"))
+  file.remove(paste0(cfg$title,".RData"))
 
   # change to run folder
   setwd(cfg$results_folder)
@@ -51,7 +51,7 @@ submit_run <- function(cfg) {
 
   # send prepare_and_run.R to cluster 
   cat("Executing prepare_and_run.R for",cfg$title,"\n")
-  sbatch_command <- paste0("sbatch --job-name=",cfg$title," --output=",cfg$title,"-%j.out --mail-type=END --comment=REMIND --wrap=\"Rscript prepare_and_run.R ",cfg$title,"\"")
+  sbatch_command <- paste0("sbatch --job-name=",cfg$title," --output=",cfg$title,"-%j.out --mail-type=END --comment=REMIND --wrap=\"Rscript prepare_and_run.R \"")
   if(cfg$slurmConfig=="direct") {
     log <- format(Sys.time(), paste0(cfg$title,"-%Y-%H-%M-%S-%OS3.log"))
     system("Rscript prepare_and_run.R ",cfg$title, stderr = log, stdout = log, wait=FALSE)
@@ -59,8 +59,8 @@ submit_run <- function(cfg) {
     system(paste(sbatch_command,"--qos=priority"))
     Sys.sleep(1)
   } else if(cfg$slurmConfig=="standby") {
-    #tmp <- paste(sbatch_command,"--qos=standby")
-    #print(tmp)
+    tmp <- paste(sbatch_command,"--qos=standby")
+    print(tmp)
     system(paste(sbatch_command,"--qos=standby"))
     Sys.sleep(1)
   } else if(cfg$slurmConfig=="short") {

start_new.R

@@ -173,18 +173,22 @@ for (scen in rownames(scenarios)) {
   # Have the log output written in a file (not on the screen)
   cfg$slurmConfig <- slurmConfig
   cfg$logoption   <- 2
+  start_now <- TRUE
   
   # configure cfg based on settings from csv if provided
-  if (!is.na(config.file)) cfg <- configure_cfg(cfg, scen, scenarios, settings)
+  if (!is.na(config.file)) {
+  
+  cfg <- configure_cfg(cfg, scen, scenarios, settings)
+  
+  # Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
+  start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx" 
+               | is.na(scenarios[scen,"path_gdx_ref"]))
+  }
   
   # save the cfg data for later start of subsequent runs (after preceding run finished)
   cat("Writing cfg to file\n")
   save(cfg,file=paste0(scen,".RData"))
   
-  # Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
-  start_now <- substr(settings[scen,"path_gdx_ref"], nchar(settings[scen,"path_gdx_ref"])-3, nchar(settings[scen,"path_gdx_ref"])) == ".gdx" 
-               | is.na(settings[scen,"path_gdx_ref"])
-  
   if (start_now){
    cat("Creating and starting: ",cfg$title,"\n")
    submit_run(cfg)