-
David Klein authoredDavid Klein authored
start.R 11.06 KiB
#!/usr/bin/env Rscript
library(lucode)
#' Usage:
#' Rscript start.R [options]
#' Rscript start.R file
#'
#' Without additional arguments this starts a single REMIND runs using the settings
#' from `config/default.cfg`.
#'
#' Control the script's behavior by providing additional arguments:
#'
#' --testOneRegi: Starting a single REMIND run in OneRegi mode using the
#' settings from `config/default.cfg`
#'
#' --restart: Restart a run.
#'
#' Starting a bundle of REMIND runs using the settings from a scenario_config_XYZ.csv:
#'
#' Rscript start.R config/scenario_config_XYZ.csv
#'
source("scripts/start/submit.R")
source("scripts/start/choose_slurmConfig.R")
############## Define function: get_line ##############################
get_line <- function(){
# gets characters (line) from the terminal of from a connection
# and stores it in the return object
if(interactive()){
s <- readline()
} else {
con <- file("stdin")
s <- readLines(con, 1, warn=FALSE)
on.exit(close(con))
}
return(s);
}
############## Define function: choose_folder #########################
choose_folder <- function(folder,title="Please choose a folder") {
dirs <- NULL
# Detect all output folders containing fulldata.gdx or non_optimal.gdx
# For coupled runs please use the outcommented text block below
dirs <- sub("/full.gms","",sub("./output/","",Sys.glob(file.path(folder,"*","full.gms"))))
# DK: The following outcommented lines are specially made for listing results of coupled runs
#runs <- findCoupledruns(folder)
#dirs <- findIterations(runs,modelpath=folder,latest=TRUE)
#dirs <- sub("./output/","",dirs)
dirs <- c("all",dirs)
cat("\n\n",title,":\n\n")
cat(paste(1:length(dirs), dirs, sep=": " ),sep="\n")
cat(paste(length(dirs)+1, "Search by the pattern.\n", sep=": "))
cat("\nNumber: ")
identifier <- get_line()
identifier <- strsplit(identifier,",")[[1]]
tmp <- NULL
for (i in 1:length(identifier)) {
if (length(strsplit(identifier,":")[[i]]) > 1) tmp <- c(tmp,as.numeric(strsplit(identifier,":")[[i]])[1]:as.numeric(strsplit(identifier,":")[[i]])[2])
else tmp <- c(tmp,as.numeric(identifier[i]))
}
identifier <- tmp
# PATTERN if(length(identifier==1) && identifier==(length(dirs)+1)){
cat("\nInsert the search pattern or the regular expression: ")
pattern <- get_line()
id <- grep(pattern=pattern, dirs[-1])
# lists all chosen directories and ask for the confirmation of the made choice
cat("\n\nYou have chosen the following directories:\n")
cat(paste(1:length(id), dirs[id+1], sep=": "), sep="\n")
cat("\nAre you sure these are the right directories?(y/n): ")
answer <- get_line()
if(answer=="y"){
return(dirs[id+1])
} else choose_folder(folder,title)
#
} else if(any(dirs[identifier] == "all")){
identifier <- 2:length(dirs)
return(dirs[identifier])
} else return(dirs[identifier])
}
############## Define function: configure_cfg #########################
configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
.setgdxcopy <- function(needle, stack, new) {
# delete entries in stack that contain needle and append new
out <- c(stack[-grep(needle, stack)], new)
return(out)
}
# Edit run title
icfg$title <- iscen
cat(" Configuring cfg for", iscen,"\n")
# Edit main file of model
if( "model" %in% names(iscenarios)){
icfg$model <- iscenarios[iscen,"model"]
}
# Edit regional aggregation
if( "regionmapping" %in% names(iscenarios)){
icfg$regionmapping <- iscenarios[iscen,"regionmapping"]
}
# Edit input data revision
if( "revision" %in% names(iscenarios)){
icfg$revision <- iscenarios[iscen,"revision"]
}
# Edit switches in default.cfg according to the values given in the scenarios table
for (switchname in intersect(names(icfg$gms), names(iscenarios))) {
icfg$gms[[switchname]] <- iscenarios[iscen,switchname]
}
# Set reporting script
if( "output" %in% names(iscenarios)){
icfg$output <- paste0("c(\"",gsub(",","\",\"",gsub(", ",",",iscenarios[iscen,"output"])),"\")")
}
# check if full input.gdx path is provided and, if not, search for correct path
if (!substr(isettings[iscen,"path_gdx"], nchar(isettings[iscen,"path_gdx"])-3, nchar(isettings[iscen,"path_gdx"])) == ".gdx"){
#if there is no correct scenario folder within the output folder path provided, take the config/input.gdx
if(length(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))==0){
isettings[iscen,"path_gdx"] <- "config/input.gdx"
#if there is only one instance of an output folder with that name, take the fulldata.gdx from this
} else if (length(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))==1){
isettings[iscen,"path_gdx"] <- paste0(isettings[iscen,"path_gdx"],"/",
grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),"/fulldata.gdx")
} else {
#if there are multiple instances, take the newest one isettings[iscen,"path_gdx"] <- paste0(isettings[iscen,"path_gdx"],"/",
substr(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),1,
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))-19)[1],
max(substr(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))-18,
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T)))),"/fulldata.gdx")
}
}
# if the above has not created a path to a valid gdx, take config/input.gdx
if (!file.exists(isettings[iscen,"path_gdx"])){
isettings[iscen,"path_gdx"] <- "config/input.gdx"
#if even this is not existent, stop
if (!file.exists(isettings[iscen,"path_gdx"])){
stop("Cant find a gdx under path_gdx, please specify full path to gdx or else location of output folder that contains previous run")
}
}
# Define path where the GDXs will be taken from
gdxlist <- c(input.gdx = isettings[iscen, "path_gdx"],
input_ref.gdx = isettings[iscen, "path_gdx_ref"],
input_bau.gdx = isettings[iscen, "path_gdx_bau"])
# Remove potential elements that contain ".gdx" and append gdxlist
icfg$files2export$start <- .setgdxcopy(".gdx", icfg$files2export$start, gdxlist)
# add gdx information for subsequent runs
icfg$subsequentruns <- rownames(isettings[isettings$path_gdx_ref == iscen & !is.na(isettings$path_gdx_ref) & isettings$start == 1,])
icfg$RunsUsingTHISgdxAsBAU <- rownames(isettings[isettings$path_gdx_bau == iscen & !is.na(isettings$path_gdx_bau) & isettings$start == 1,])
return(icfg)
}
# check command-line arguments for testOneRegi and scenario_config file
argv <- commandArgs(trailingOnly = TRUE)
config.file <- argv[1]
# define arguments that are accepted
accepted <- c('--restart','--testOneRegi')
# check if user provided any unknown arguments or config files that do not exist
known <- argv %in% accepted
if (!all(known)) {
file_exists <- file.exists(argv[!known])
if (!all(file_exists)) stop("Unknown argument provided: ",paste(argv[!known][!file_exists]," \nAccepted arguments are '--testOneRegi', '--restart' or a path to an existing scenario_config.csv"))
}
###################### Choose submission type #########################
slurmConfig <- choose_slurmConfig()
# Restart REMIND in existing results folder (if required by user)
if ('--restart' %in% argv) {
# choose results folder from list
outputdirs <- choose_folder("./output","Please choose the runs to be restarted")
for (outputdir in outputdirs) {
cat("Restarting",outputdir,"\n")
load(paste0("output/",outputdir,"/config.Rdata")) # read config.Rdata from results folder
cfg$slurmConfig <- combine_slurmConfig(cfg$slurmConfig,slurmConfig) # update the slurmConfig setting to what the user just chose (it was being ignored before)
cfg$results_folder <- paste0("output/",outputdir) # overwrite results_folder in cfg with name of the folder the user wants to restart, because user might have renamed the folder before restarting
submit(cfg, restart = TRUE)
#cat(paste0("output/",outputdir,"/config.Rdata"),"\n")
}
} else {
# If testOneRegi was selected, set up a testOneRegi run.
if ('--testOneRegi' %in% argv) {
testOneRegi <- TRUE config.file <- NA
} else {
testOneRegi <- FALSE
}
###################### Load csv if provided ##########################
# If a scenario_config.csv file was provided, set cfg according to it.
if (!is.na(config.file)) {
cat(paste("\nReading config file", config.file, "\n"))
# Read-in the switches table, use first column as row names
settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
# Select scenarios that are flagged to start
scenarios <- settings[settings$start==1,]
if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
} else {
# if no csv was provided create dummy list with default as the only scenario
scenarios <- data.frame("default" = "default",row.names = "default")
}
###################### Loop over scenarios ###############################
# Modify and save cfg for all runs
for (scen in rownames(scenarios)) {
#source cfg file for each scenario to avoid duplication of gdx entries in files2export
source("config/default.cfg")
# Have the log output written in a file (not on the screen)
cfg$slurmConfig <- combine_slurmConfig(cfg$slurmConfig,slurmConfig)
cfg$logoption <- 2
start_now <- TRUE
# testOneRegi settings
if (testOneRegi) {
cfg$title <- 'testOneRegi'
cfg$gms$optimization <- 'testOneRegi'
cfg$output <- NA
cfg$results_folder <- 'output/testOneRegi'
# delete existing Results directory
cfg$force_replace <- TRUE
}
cat("\n",scen,"\n")
# configure cfg based on settings from csv if provided
if (!is.na(config.file)) {
cfg <- configure_cfg(cfg, scen, scenarios, settings)
# Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
start_now <- (substr(scenarios[scen,"path_gdx_ref"], nchar(scenarios[scen,"path_gdx_ref"])-3, nchar(scenarios[scen,"path_gdx_ref"])) == ".gdx"
| is.na(scenarios[scen,"path_gdx_ref"]))
}
# save the cfg data for later start of subsequent runs (after preceding run finished)
filename <- paste0(scen,".RData")
cat(" Writing cfg to file",filename,"\n")
save(cfg,file=filename)
if (start_now){
# Create results folder and start run
submit(cfg)
} else {
cat(" Waiting for", scenarios[scen,'path_gdx_ref'] ,"\n")
}
if (!identical(cfg$subsequentruns,character(0))) cat(" Subsequent runs:",cfg$subsequentruns,"\n")
}
}