Newer
Older
library(lucode)
David Klein
committed
source("scripts/start/submit_run.R")
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#######################################################################
############### Select slurm partitiion ###############################
#######################################################################
get_line <- function(){
# gets characters (line) from the terminal or from a connection
# and returns it
if(interactive()){
s <- readline()
} else {
con <- file("stdin")
s <- readLines(con, 1, warn=FALSE)
on.exit(close(con))
}
return(s);
}
choose_submit <- function(title="Please choose run submission type") {
# xxx add REMIND specific combinations of qos and number of nodes
modes <- c("SLURM priority (recommended)",
"SLURM standby (recommended)",
"SLURM short",
"SLURM medium",
"SLURM long")
cat("\nCurrent cluster utilization:\n")
system("sclass")
cat("\n")
cat("\n",title,":\n", sep="")
cat(paste(1:length(modes), modes, sep=": " ),sep="\n")
cat("Number: ")
identifier <- get_line()
identifier <- as.numeric(strsplit(identifier,",")[[1]])
comp <- switch(identifier,
"1" = "priority",
"2" = "standby",
"3" = "short",
"4" = "medium",
"5" = "long")
if(is.null(comp)) stop("This type is invalid. Please choose a valid type")
return(comp)
}
# Choose submission type
slurm <- TRUE #suppressWarnings(ifelse(system2("srun",stdout=FALSE,stderr=FALSE) != 127, TRUE, FALSE))
if (slurm) {
David Klein
committed
slurmConfig <- choose_submit("Choose submission type")
} else {
David Klein
committed
slurmConfig <- "direct"
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
}
#######################################################################
######################## Submit run ###################################
#######################################################################
############## Define function: configure_cfg #########################
configure_cfg <- function(icfg, iscen, iscenarios, isettings) {
.setgdxcopy <- function(needle, stack, new) {
# delete entries in stack that contain needle and append new
out <- c(stack[-grep(needle, stack)], new)
return(out)
}
# Edit run title
icfg$title <- iscen
cat("\n", iscen, "\n")
# Edit main file of model
if( "model" %in% names(iscenarios)){
icfg$model <- iscenarios[iscen,"model"]
}
# Edit regional aggregation
if( "regionmapping" %in% names(iscenarios)){
icfg$regionmapping <- iscenarios[iscen,"regionmapping"]
}
# Edit input data revision
if( "revision" %in% names(iscenarios)){
icfg$revision <- iscenarios[iscen,"revision"]
}
# Edit switches in default.cfg according to the values given in the scenarios table
for (switchname in intersect(names(icfg$gms), names(iscenarios))) {
icfg$gms[[switchname]] <- iscenarios[iscen,switchname]
}
# Set reporting script
if( "output" %in% names(iscenarios)){
icfg$output <- paste0("c(\"",gsub(",","\",\"",gsub(", ",",",iscenarios[iscen,"output"])),"\")")
}
# check if full input.gdx path is provided and, if not, search for correct path
if (!substr(isettings[iscen,"path_gdx"], nchar(isettings[iscen,"path_gdx"])-3, nchar(isettings[iscen,"path_gdx"])) == ".gdx"){
#if there is no correct scenario folder within the output folder path provided, take the config/input.gdx
if(length(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))==0){
isettings[iscen,"path_gdx"] <- "config/input.gdx"
#if there is only one instance of an output folder with that name, take the fulldata.gdx from this
} else if (length(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))==1){
isettings[iscen,"path_gdx"] <- paste0(isettings[iscen,"path_gdx"],"/",
grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),"/fulldata.gdx")
} else {
#if there are multiple instances, take the newest one
isettings[iscen,"path_gdx"] <- paste0(isettings[iscen,"path_gdx"],"/",
substr(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),1,
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))-19)[1],
max(substr(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T),
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T))-18,
nchar(grep(iscen,list.files(path=isettings[iscen,"path_gdx"]),value=T)))),"/fulldata.gdx")
}
}
# if the above has not created a path to a valid gdx, take config/input.gdx
if (!file.exists(isettings[iscen,"path_gdx"])){
isettings[iscen,"path_gdx"] <- "config/input.gdx"
#if even this is not existent, stop
if (!file.exists(isettings[iscen,"path_gdx"])){
stop("Cant find a gdx under path_gdx, please specify full path to gdx or else location of output folder that contains previous run")
}
}
# Define path where the GDXs will be taken from
gdxlist <- c(input.gdx = isettings[iscen, "path_gdx"],
input_ref.gdx = isettings[iscen, "path_gdx_ref"],
input_bau.gdx = isettings[iscen, "path_gdx_bau"])
# Remove potential elements that contain ".gdx" and append gdxlist
icfg$files2export$start <- .setgdxcopy(".gdx", icfg$files2export$start, gdxlist)
# add gdx information for subsequent runs
icfg$subsequentruns <- rownames(isettings[isettings$path_gdx_ref == iscen & !is.na(isettings$path_gdx_ref) & isettings$start == 1,])
icfg$RunsUsingTHISgdxAsBAU <- rownames(isettings[isettings$path_gdx_bau == iscen & !is.na(isettings$path_gdx_bau) & isettings$start == 1,])
return(icfg)
}
###################### Load csv if provided ##########################
David Klein
committed
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
# If scenario_config.csv was provided from command line, set cfg according to it (copy from start_bundle)
# check for config file parameter
config.file <- commandArgs(trailingOnly = TRUE)[1]
if (!is.na(config.file)) {
cat(paste("reading config file", config.file, "\n"))
# Read-in the switches table, use first column as row names
settings <- read.csv2(config.file, stringsAsFactors = FALSE, row.names = 1, comment.char = "#", na.strings = "")
# Select scenarios that are flagged to start
scenarios <- settings[settings$start==1,]
if (length(grep("\\.",rownames(scenarios))) > 0) stop("One or more titles contain dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ")
} else {
# if no csv was provided create dummy list with default as the only scenario
scenarios <- data.frame("default" = "default",row.names = "default")
}
###################### Loop over csv ###############################
# Modify and save cfg for all runs
for (scen in rownames(scenarios)) {
#source cfg file for each scenario to avoid duplication of gdx entries in files2export
source("config/default.cfg")
# Have the log output written in a file (not on the screen)
David Klein
committed
cfg$slurmConfig <- slurmConfig
cfg$logoption <- 2
# configure cfg based on settings from csv if provided
if (!is.na(config.file)) cfg <- configure_cfg(cfg, scen, scenarios, settings)
# save the cfg data for later start of subsequent runs (after preceding run finished)
cat("Writing cfg to file\n")
save(cfg,file=paste0(scen,".RData"))
# Directly start runs that have a gdx file location given as path_gdx_ref or where this field is empty
start_now <- substr(settings[scen,"path_gdx_ref"], nchar(settings[scen,"path_gdx_ref"])-3, nchar(settings[scen,"path_gdx_ref"])) == ".gdx"
| is.na(settings[scen,"path_gdx_ref"])
if (start_now){
cat("Creating and starting: ",cfg$title,"\n")
submit_run(cfg)
}
}