Commit 67bcc0a7 authored by Jan Philipp Dietrich's avatar Jan Philipp Dietrich
Browse files

added "submit" argument in start_preprocessing for parallel submissions

parent 7984996d
start_preprocessing <- function(cfg, debug=FALSE){
runsubmit <- function(runscripts, submit) {
allowedSubmissionTypes <- c("direct", "background", "slurmstandard", "slurmpriority","debug")
if(!(submit %in% allowedSubmissionTypes)) {
stop("Unknown submission type \"",submit,"\"! Allowed types are: ", paste(allowedSubmissionTypes, collapse=", "))
}
for(rout in runscripts){
name <- paste0("./start/",rout,".R")
if(!file.exists(name)) {
warning("Script ",name, " could not be found. Skip execution!")
next
}
cat("Executing",name,"\n")
sbatch_command <- paste0("sbatch --job-name=",rout," --output=",rout,"-%j.out --mail-type=END --wrap=\"Rscript ",name,"\"")
if(submit=="direct") {
tmp.env <- new.env()
tmp.error <- try(sys.source(name,envir=tmp.env))
if(!is.null(tmp.error)) warning("Script ",name," was stopped by an error and not executed properly!")
rm(tmp.env)
} else if(submit=="background") {
log <- format(Sys.time(), paste0(rout,"-%Y-%H-%M-%S-%OS3.log"))
system2("Rscript",name, stderr = log, stdout = log, wait=FALSE)
} else if(submit=="slurmstandard") {
system(paste(sbatch_command,"--qos=standby --mem=32000"))
} else if(submit=="slurmpriority") {
system(paste(sbatch_command,"--qos=priority --mem=32000"))
} else if(submit=="debug") {
tmp.env <- new.env()
sys.source(name,envir=tmp.env)
rm(tmp.env)
} else {
stop("Unknown submission type")
}
}
}
start_preprocessing <- function(cfg, debug=FALSE, submit = NULL){
require(gms)
require(madrat)
require(mrland)
......@@ -10,6 +49,20 @@ start_preprocessing <- function(cfg, debug=FALSE){
if(!grepl(".csv$",cfg$regionmapping)) cfg$regionmapping <- paste0("regionmapping", cfg$regionmapping, ".csv")
if(!file.exists(cfg$regionmapping)) cfg$regionmapping <- paste0("config/",cfg$regionmapping)
if(is.null(cfg$dev)) cfg$dev <- ""
if(!is.null(submit)) {
tmpStartFolder <- "./start/tmp"
if(!dir.exists(tmpStartFolder)) dir.create(tmpStartFolder)
hash <- digest::digest(cfg, algo="xxhash32")
saveRDS(cfg, file = file.path(tmpStartFolder,paste0(hash,".rds")))
writeLines(c("# This script has been created by preprocessing.R for parallel submissions",
"source(\"preprocessing.R\")",
paste0("cfg <- readRDS(\"",file.path(tmpStartFolder,hash),".rds\")"),
paste0("start_preprocessing(cfg, debug = ",debug,", submit = NULL)")),
con = paste0(file.path(tmpStartFolder,hash),".R"))
runsubmit(runscripts = file.path("tmp",hash), submit = submit)
return(NULL)
}
madrat::setConfig(regionmapping=cfg$regionmapping, nocores=cfg$nocores, debug=debug)
message(paste0("Start preprocessing for \n climatescenario: ",cfg$climatetype, "\n rev: ",cfg$revision,
......@@ -26,3 +79,4 @@ start_preprocessing <- function(cfg, debug=FALSE){
emu_id = cfg$emu_id, maxLengthLogMessage = 500)
madrat::retrieveData(model="Validation", rev=cfg$revision, dev=cfg$dev, cachetype="def")
}
......@@ -9,6 +9,7 @@
##########################################################
library(lucode2)
source("preprocessing.R")
runOutputs <- function(runscripts=NULL, submit=NULL) {
......@@ -25,7 +26,6 @@ runOutputs <- function(runscripts=NULL, submit=NULL) {
return(s);
}
choose_module <- function(Rfolder,title="Please choose an outputmodule") {
forder <- paste0(Rfolder,"/order.cfg")
if(file.exists(forder)) {
......@@ -76,41 +76,6 @@ runOutputs <- function(runscripts=NULL, submit=NULL) {
return(comp)
}
runsubmit <- function(runscripts, submit) {
for(rout in runscripts){
name <- paste0("./start/",rout,".R")
if(!file.exists(name)) {
warning("Script ",name, " could not be found. Skip execution!")
next
}
cat("Executing",name,"\n")
sbatch_command <- paste0("sbatch --job-name=",rout," --output=",rout,"-%j.out --mail-type=END --wrap=\"Rscript ",name,"\"")
if(submit=="direct") {
tmp.env <- new.env()
tmp.error <- try(sys.source(name,envir=tmp.env))
if(!is.null(tmp.error)) warning("Script ",name," was stopped by an error and not executed properly!")
rm(tmp.env)
} else if(submit=="background") {
log <- format(Sys.time(), paste0(rout,"-%Y-%H-%M-%S-%OS3.log"))
system2("Rscript",name, stderr = log, stdout = log, wait=FALSE)
} else if(submit=="slurmstandard") {
system(paste(sbatch_command,"--qos=standby --mem=32000"))
} else if(submit=="slurmpriority") {
system(paste(sbatch_command,"--qos=priority --mem=32000"))
} else if(submit=="debug") {
tmp.env <- new.env()
sys.source(name,envir=tmp.env)
rm(tmp.env)
} else {
stop("Unknown submission type")
}
}
}
if(is.null(runscripts)) runscripts <- choose_module("./start",
"Choose start script")
if(is.null(submit)) submit <- choose_submit("Choose submission type")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment