Refactoring start scripts: adapt coupling scripts, call SLURM directly without...

Refactoring start scripts: adapt coupling scripts, call SLURM directly without the detour via cmd files

cluster_start_coupled.cmd

-#!/bin/bash
-
-#--- New Cluster Job Submission parameters ------
-#SBATCH --qos=priority
-#SBATCH --job-name=start_coupled
-#SBATCH --output=log.txt
-#SBATCH --nodes=1
-#SBATCH --tasks-per-node=1
-#SBATCH --mail-type=END
-#------------------------------------------------
-
-Rscript start_coupled.R coupled_config=SSP1_RCP45_nash.RData

scripts/start/prepare_and_run.R

@@ -2,7 +2,7 @@ library(lucode, quietly = TRUE,warn.conflicts =FALSE)
 library(dplyr, quietly = TRUE,warn.conflicts =FALSE)
 require(gdx)
 
-getReportData <- function(rep,inputpath_mag="magpie",inputpath_acc="costs") {
+getReportData <- function(path_to_report,inputpath_mag="magpie",inputpath_acc="costs") {
 	require(lucode, quietly = TRUE,warn.conflicts =FALSE)
   require(magclass, quietly = TRUE,warn.conflicts =FALSE)
   .bioenergy_price <- function(mag){
@@ -124,7 +124,7 @@ getReportData <- function(rep,inputpath_mag="magpie",inputpath_acc="costs") {
     write.magpie(out[notGLO,,],paste0("./modules/26_agCosts/",inputpath_acc,"/input/trade_bal_reg.rem.csv"),file_type="csvr")
   }
   
-  rep <- read.report(rep,as.list=FALSE)
+  rep <- read.report(path_to_report,as.list=FALSE)
   if (length(getNames(rep,dim="scenario"))!=1) stop("getReportData: MAgPIE data contains more or less than 1 scenario.")
   rep <- collapseNames(rep) # get rid of scenrio and model dimension if they exist
   years <- 2000+5*(1:30)

start_bundle_coupled.R

@@ -95,7 +95,7 @@ if (!identical(common,character(0))) {
 for(scen in common){
   cat(paste0("\n################################\nPreparing run ",scen,"\n"))
   
-  prefix_runname <- strsplit(path_remind,"/")[[1]][length(strsplit(path_remind,"/")[[1]])]
+  prefix_runname <- "C" #strsplit(path_remind,"/")[[1]][length(strsplit(path_remind,"/")[[1]])]
   prefix_runname <- paste0(prefix_runname,"_")
   
   runname      <- paste0(prefix_runname,scen)            # name of the run that is used for the folder names
@@ -293,26 +293,16 @@ for(scen in common){
   cat("path_report : ",ifelse(file.exists(path_report),green,red), path_report, NC, "\n",sep="")
   cat("LU_pricing  :",LU_pricing,"\n")
  
-  # create cluster_start_coupled_scen.cmd file
-  # 1. copy general cluster_start_coupled file
-  #system(paste0("cp cluster_start_coupled.cmd cluster_start_coupled_",scen,".cmd"))
-  # 2. modify accordingly
-  #manipulateConfig(paste0("cluster_start_coupled_",scen,".cmd"),coupled_config=paste0(runname,".RData"),line_endings = "NOTwin")
-  #manipulateConfig(paste0("cluster_start_coupled_",scen,".cmd"),"--job-name"=runname,line_endings = "NOTwin")
-  #manipulateConfig(paste0("cluster_start_coupled_",scen,".cmd"),"--output"=paste0(runname,".log"),line_endings = "NOTwin")
   if (cfg_rem$gms$optimization == "nash" && cfg_rem$gms$cm_nash_mode == "parallel") {
     # for nash: set the number of CPUs per node to number of regions + 1
     nr_of_regions <- length(levels(read.csv2(cfg_rem$regionmapping)$RegionCode)) + 1 
-    #manipulateConfig(paste0("cluster_start_coupled_",scen,".cmd"),"--tasks-per-node"=nr_of_regions,line_endings = "NOTwin")
   } else {
     # for negishi: use only one CPU
     nr_of_regions <- 1
-    #manipulateConfig(paste0("cluster_start_coupled_",scen,".cmd"),"--tasks-per-node"=1,line_endings = "NOTwin")
   }
 
   if (start_now){
-      #if (!exists("test")) system(paste0("sbatch cluster_start_coupled_",scen,".cmd"))
-      if (!exists("test")) system(paste0("sbatch --qos=standby --job-name=",runname," --output=",runname,".log --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=",nr_of_regions," --wrap=\"Rscript start_coupled.R \" coupled_config=",runname,".RData"))
+      if (!exists("test")) system(paste0("sbatch --qos=standby --job-name=",runname," --output=",runname,".log --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=",nr_of_regions," --wrap=\"Rscript start_coupled.R coupled_config=",runname,".RData\""))
       else cat("Test mode: run NOT submitted to the cluster\n")
   } else {
      cat(paste0("Run ",runname," will start after preceding run ",prefix_runname,settings_remind[scen,"path_gdx_ref"]," has finished\n"))

start_coupled.R

@@ -52,7 +52,7 @@ start_coupled <- function(path_remind,path_magpie,cfg_rem,cfg_mag,runname,max_it
     cat("### COUPLING ### Set working directory from",getwd());
     setwd(path_remind)
     cat(" to",getwd(),"\n")
-    source("scripts/start_functions.R") # provide source of "get_magpie_data" and "start_run"
+    source("scripts/start/submit.R") # provide source of "get_magpie_data" and "start_run"
     
     cfg_rem$results_folder <- paste0("output/",runname,"-rem-",i)
     cfg_rem$title          <- paste0(runname,"-rem-",i)
@@ -212,11 +212,10 @@ start_coupled <- function(path_remind,path_magpie,cfg_rem,cfg_mag,runname,max_it
     nr_of_regions <- 1
   }
 
-  #start subsequent runs via cmd scripts created at the end of start_bundle_coupled.R
+  #start subsequent runs via sbatch
   for(run in cfg_rem$subsequentruns){
     cat("Submitting subsequent run",run,"\n")
-    #system(paste0("sbatch cluster_start_coupled_",run,".cmd"))
-    system(paste0("sbatch --qos=standby --job-name=",run," --output=",run,".log --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=13",nr_of_regions," --wrap=\"Rscript start_coupled.R \" coupled_config=",run,".RData"))
+    system(paste0("sbatch --qos=standby --job-name=",run," --output=",run,".log --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=13",nr_of_regions," --wrap=\"Rscript start_coupled.R coupled_config=",run,".RData\""))
   }
   
   # Read runtime of ALL coupled runs (not just the current scenario) and produce comparison pdf