Skip to content
Snippets Groups Projects
Commit 1044c8e6 authored by Alois Dirnaichner's avatar Alois Dirnaichner
Browse files

Merge branch 'REMIND_new_transp_esm' into tobemerged

parents 5b1a7e18 20f0c823
No related branches found
No related tags found
1 merge request!67Request to merge the new transport module EDGE-T
Showing
with 675 additions and 1075 deletions
......@@ -12,6 +12,6 @@ vm_shSynSe(ttot,all_regi) "Share of synthetic liquids in all fossil liquids. V
Equations
q35_shBioFe(ttot,all_regi) "Define the share of biofuels in all final energies for transportation."
q35_shSynSe(ttot,all_regi) "Define share of synthetic liquids in all fossil liquids."
*q35_shSynSe(ttot,all_regi) "Define share of synthetic liquids in all fossil liquids."
;
*** EOF ./modules/35_transport/edge_esm/declarations.gms
......@@ -4,8 +4,13 @@
*** | AGPL-3.0, you are granted additional permissions described in the
*** | REMIND License Exception, version 1.0 (see LICENSE file).
*** | Contact: remind@pik-potsdam.de
*** SOF ./modules/50_damages/BurkeLike/declarations.gms
parameters
p50_damageFuncCoef1 "coef1 of damamge function",
p50_damageFuncCoef2 "coef2 of damamge function"
;
positive variable
vm_damageFactor(ttot,all_regi) "damage factor reducing GDP"
;
*** EOF ./modules/50_damages/BurkeLike/declarations.gms
......@@ -4,7 +4,13 @@
*** | AGPL-3.0, you are granted additional permissions described in the
*** | REMIND License Exception, version 1.0 (see LICENSE file).
*** | Contact: remind@pik-potsdam.de
*** SOF ./modules/50_damages/DiceLike/declarations.gms
parameters
p50_damageFuncCoef1 "damage function coefficient, linear in temperature"
p50_damageFuncCoef2 "damage function coefficient, quadratic in temperture"
;
positive variable
vm_damageFactor(ttot,all_regi) "damage factor reducing GDP"
;
*** EOF ./modules/50_damages/DiceLike/declarations.gms
*** | (C) 2006-2019 Potsdam Institute for Climate Impact Research (PIK)
*** | authors, and contributors see CITATION.cff file. This file is part
*** | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of
*** | AGPL-3.0, you are granted additional permissions described in the
*** | REMIND License Exception, version 1.0 (see LICENSE file).
*** | Contact: remind@pik-potsdam.de
*** SOF ./modules/50_damages/off/declarations.gms
positive variable
vm_damageFactor(ttot,all_regi) "damage factor reducing GDP"
;
*** EOF ./modules/50_damages/off/declarations.gms
......@@ -9,6 +9,7 @@
*' @description The off-realization of the damage module sets the damage factor on output to 1, meaning no damage.
*####################### R SECTION START (PHASES) ##############################
$Ifi "%phase%" == "declarations" $include "./modules/50_damages/off/declarations.gms"
$Ifi "%phase%" == "bounds" $include "./modules/50_damages/off/bounds.gms"
*######################## R SECTION END (PHASES) ###############################
*** EOF ./modules/50_damages/off.gms
......@@ -9,15 +9,12 @@
pm_pvp(ttot,trade)$(ttot.val ge 2005) = 1;
p80_trade(ttot,regi,trade)$(ttot.val ge 2005) = 0;
$ifthen.cm_compile_main %cm_compile_main% == "FALSE"
if (cm_emiscen eq 1,
Execute_Loadpoint "./input.gdx", p80_currentaccount_bau = p80_curracc;
else
Execute_Loadpoint "./input_ref.gdx", p80_currentaccount_bau = p80_curracc;
);
$endif.cm_compile_main
p80_defic_sum("1") = 1;
......
......@@ -24,20 +24,20 @@ if(o_modelstat eq 2,
IF(o_modelstat eq 2,
*AJS*2013-05* Default Negishi convergence scheme:
if (ord(iteration) eq 1, s80_cnptfile = 2);
if (ord(iteration) eq 2, s80_cnptfile = 3);
if (ord(iteration) eq 3, s80_cnptfile = 4);
if (ord(iteration) eq 4, s80_cnptfile = 5);
if (ord(iteration) eq 1, s80_cnptfile = 2); !! rtredg = 1.d-6
if (ord(iteration) eq 2, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 3, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 4, s80_cnptfile = 3); !! rtredg = 1.d-7
*RP* Slower convergence scheme
$IFTHEN.cm_SlowConvergence %cm_SlowConvergence% == "on"
if (ord(iteration) eq 1, s80_cnptfile = 1);
if (ord(iteration) eq 2, s80_cnptfile = 2);
if (ord(iteration) eq 3, s80_cnptfile = 2);
if (ord(iteration) eq 4, s80_cnptfile = 3);
if (ord(iteration) eq 5, s80_cnptfile = 3);
if (ord(iteration) eq 6, s80_cnptfile = 4);
if (ord(iteration) eq 7, s80_cnptfile = 4);
if (ord(iteration) eq 8, s80_cnptfile = 5);
if (ord(iteration) eq 1, s80_cnptfile = 1); !! rtredg = 1.d-5
if (ord(iteration) eq 2, s80_cnptfile = 2); !! rtredg = 1.d-6
if (ord(iteration) eq 3, s80_cnptfile = 2); !! rtredg = 1.d-6
if (ord(iteration) eq 4, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 5, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 6, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 7, s80_cnptfile = 3); !! rtredg = 1.d-7
if (ord(iteration) eq 8, s80_cnptfile = 3); !! rtredg = 1.d-7
$ENDIF.cm_SlowConvergence
);
*** EOF ./modules/80_optimization/negishi/solve.gms
......@@ -6,8 +6,7 @@
*** | Contact: remind@pik-potsdam.de
*** SOF ./modules/80_optimization/testOneRegi/solve.gms
$IFI %optimization% == "testOneRegi" s80_cnptfile = 9;
hybrid.optfile = s80_cnptfile;
hybrid.optfile = 9;
***reduce the problem to one region
regi(all_regi) = NO;
......
# | (C) 2006-2019 Potsdam Institute for Climate Impact Research (PIK)
# | authors, and contributors see CITATION.cff file. This file is part
# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of
# | AGPL-3.0, you are granted additional permissions described in the
# | REMIND License Exception, version 1.0 (see LICENSE file).
# | Contact: remind@pik-potsdam.de
#! /usr/bin/Rscript
##########################################################
#### Script to start a REMIND run ####
##########################################################
library(lucode, quietly = TRUE, warn.conflicts = FALSE)
library(magclass, quietly = TRUE, warn.conflicts = FALSE)
#Here the function start_run(cfg) is loaded which is needed to start REMIND runs
#The function needs information about the configuration of the run. This can be either supplied as a list of settings or as a file name of a config file
source("scripts/start_functions.R")
if (file.exists("./output/testOneRegi/"))
unlink("./output/testOneRegi/", recursive = TRUE)
#Load config-file
cfg_REMIND <- "oneRegi.cfg"
readArgs("cfg")
# start REMIND run
start_run(cfg_REMIND)
#rep = read.report("coupling.mif")
#start_reportrun(rep)
......@@ -52,11 +52,12 @@ start_comp <- function(outputdirs,shortTerm,outfilename) {
cat("Starting ",jobname,"\n")
on_cluster <- file.exists("/p/projects/")
if (on_cluster) {
system(paste0("srun --qos=standby --job-name=",jobname," --output=",jobname,".out --error=",jobname,".err --mail-type=END --time=200 --mem-per-cpu=8000 Rscript scripts/run_submit/run_compareScenarios.R outputdirs=",paste(outputdirs,collapse=",")," shortTerm=",shortTerm," outfilename=",jobname," &"))
clcom <- paste0("sbatch --qos=standby --job-name=",jobname," --output=",jobname,".out --error=",jobname,".err --mail-type=END --time=200 --mem-per-cpu=8000 --wrap=\"Rscript scripts/utils/run_compareScenarios.R outputdirs=",paste(outputdirs,collapse=",")," shortTerm=",shortTerm," outfilename=",jobname,"\"")
system(clcom)
} else {
outfilename <- jobname
tmp.env <- new.env()
script <- "scripts/run_submit/run_compareScenarios.R"
script <- "scripts/utils/run_compareScenarios.R"
tmp.error <- try(sys.source(script,envir=tmp.env))
if(!is.null(tmp.error)) warning("Script ",script," was stopped by an error and not executed properly!")
rm(tmp.env)
......
......@@ -66,7 +66,7 @@ compareScenTable <- function(listofruns){
if(system("hash sbatch 2>/dev/null") == 0){
cat("Submitting comparison Jobs:\n")
system("sbatch scripts/run_submit/submit_compare.cmd")
system(paste0("sbatch --job-name=rem-compare --output=log-%j.out --mail-type=END --cpus-per-task=2 --qos=priority --wrap=\"Rscript scripts/utils/compareParallel.R \""))
}else{
source("scripts/utils/compareParallel.R")
}
......
# | (C) 2006-2019 Potsdam Institute for Climate Impact Research (PIK)
# | authors, and contributors see CITATION.cff file. This file is part
# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of
# | AGPL-3.0, you are granted additional permissions described in the
# | REMIND License Exception, version 1.0 (see LICENSE file).
# | Contact: remind@pik-potsdam.de
source("scripts/start_functions.R")
performance_start <- function(cfg="default.cfg",modulepath="modules/",id="performance",sequential=NULL) {
require(lucode)
if(!is.list(cfg)) {
if(is.character(cfg)) {
source(path("config",cfg),local=TRUE)
if(!is.list(cfg)) stop("Wrong input file format: config file does not contain a cfg list!")
} else {
stop("Wrong input format: cfg is neither a list nor a character!")
}
}
cfg$results_folder <- "output/:title:"
cfg$sequential <- sequential
cfg$logoption <- 2
#LB# funzt erstmal nicht in REMIND
#svn <- system("svn info",intern=TRUE)
#save(svn,file="svn.rda")
#cfg$files2export$start <- c(cfg$files2export$start,"svn.rda")
#start default run
cfg$title <- paste(id,"default",sep="__")
cat(cfg$title,"\n")
start_run(cfg)
m <- getModules(modulepath)
for(i in 1:dim(m)[1]) {
#for(i in c(1,3,4)) {
default <- cfg$gms[[m[i,"name"]]]
r <- strsplit(m[i,"realizations"],",")[[1]]
r <- r[r!=default] #remove default case
for(j in r) {
cfg$gms[[m[i,"name"]]] <- j
cfg$title <- paste(id,m[i,"name"],j,sep="__")
cat(cfg$title,"\n")
start_functions(cfg) # for REMIND
# start_run(cfg) # for MAGPIE
}
cfg$gms[[m[i,"name"]]] <- default
}
}
performance_collect <- function(id="performance",results_folder="output/",plot=TRUE) {
require(magpie)
require(lucode)
maindir <- getwd()
cat("\n wir sind in ", maindir,"\n")
on.exit(setwd(maindir))
setwd(results_folder)
cat("\n wir sind in ",results_folder,"\n")
folders <- grep(paste("^",id,"__",sep=""),list.dirs(),value=TRUE)
tmp <- grep(paste("^",id,"__default",sep=""),folders)
default <- folders[tmp]
if(length(default)==0) stop("No default folder found which fits to the given id (",id,")")
folders <- folders[-tmp]
if(length(folders)==0) stop("No folders found which fit to the given id (",id,")")
.modelstats <- function(f,colMeans=TRUE) {
# logfile <- path(f,"log.txt") # for MAGPIE
logfile <- path(f,"full.log") # for REMIND
tmp <- readLines(logfile)
l1 <- grep("rows",tmp,value=TRUE)
l2 <- grep("nl-code",tmp,value=TRUE)
if(length(l1)==0 & length(l2)==0) return(NA)
p1 <- "--- ([^ ]*) rows ([^ ]*) columns ([^ ]*) non-zeroes"
rows <- as.integer(gsub(",","",gsub(p1,"\\1",l1)))
columns <- as.integer(gsub(",","",gsub(p1,"\\2",l1)))
nonzeroes <- as.integer(gsub(",","",gsub(p1,"\\3",l1)))
p2 <- "--- ([^ ]*) nl-code ([^ ]*) nl-non-zeroes"
nlcode <- as.integer(gsub(",","",gsub(p2,"\\1",l2)))
nlnonzeroes <- as.integer(gsub(",","",gsub(p2,"\\2",l2)))
out <- cbind(rows,columns,nonzeroes,nlcode,nlnonzeroes)
rownames(out) <- paste("t",1:dim(out)[1],sep="")
if(colMeans) out <- colMeans(out)
return(out)
}
.infescheck <- function(gdx) {
if(!file.exists(gdx)) return(3)
tmp <- modelstat(gdx)
if(any(tmp!=2 & tmp!=7)) {
x <- 2
} else if(any(tmp!=2 & tmp==7)) {
x <- 1
} else {
x <- 0
}
return(x)
}
.gettime <- function(rdata) {
load(rdata)
# tmp <- as.double(validation$technical$time$magpie.gms,unit="mins") # for MAGPIE
tmp <- as.double(validation$technical$time$full.gms,unit="mins") # for REMIND
return(tmp)
}
results <- NULL
infes <- list()
cat("\n folgende folder werden gleich bearbeitet ",folders,"\n")
for(f in folders){
tmp <- strsplit(f,"__")[[1]]
ms <- .modelstats(f,colMeans=TRUE)
cat("\n module ",tmp[2],"\n")
cat("\n realization ",tmp[3],"\n")
# for MAGPIE
#tmp2 <- data.frame(module=tmp[2],realization=tmp[3],default=FALSE,runtime=.gettime(path(f,f,ftype="RData")),infes=.infescheck(path(f,"fulldata.gdx")),
# rows=ms["rows"],columns=ms["columns"],nonzeroes=ms["nonzeroes"],nlcode=ms["nlcode"],nlnonzeroes=ms["nlnonzeroes"])
# for REMIND
tmp2 <- data.frame(module=tmp[2],realization=tmp[3],default=FALSE,runtime=.gettime(path(f,f,ftype="RData")),infes=.infescheck(path(f,"optim.gdx")),
rows=ms["rows"],columns=ms["columns"],nonzeroes=ms["nonzeroes"],nlcode=ms["nlcode"],nlnonzeroes=ms["nlnonzeroes"])
results <- rbind(results,tmp2)
}
load(path(default,"config.Rdata"))
for(n in unique(results$module)) {
ms <- .modelstats(default,colMeans=TRUE)
# for MAGPIE
# tmp <- data.frame(module=n,realization=cfg$gms[[n]],default=TRUE,runtime=.gettime(path(default,default,ftype="RData")),infes=.infescheck(path(default,"fulldata.gdx")),
# rows=ms["rows"],columns=ms["columns"],nonzeroes=ms["nonzeroes"],nlcode=ms["nlcode"],nlnonzeroes=ms["nlnonzeroes"])
# for REMIND
tmp <- data.frame(module=n,realization=cfg$gms[[n]],default=TRUE,runtime=.gettime(path(default,default,ftype="RData")),infes=.infescheck(path(default,"optim.gdx")),
rows=ms["rows"],columns=ms["columns"],nonzeroes=ms["nonzeroes"],nlcode=ms["nlcode"],nlnonzeroes=ms["nlnonzeroes"])
results <- rbind(results,tmp)
}
results$info <- paste(results$module,results$realization,sep=": ")
results$relative_runtime <- results$runtime/results[results$default,"runtime"][1] - 1
results$more_info[results$default] <- "default"
results$more_info[results$infes==0 & !results$default] <- paste(round(results$relative_runtime[results$infes==0 & !results$default]*100),"%",sep="")
results$more_info[results$infes==1] <- "non-optimal solution"
results$more_info[results$infes==2] <- "infeasible solution"
results$more_info[results$infes==3] <- "compilation error"
results$relative_rows <- results$rows/results[results$default,"rows"][1] - 1
results$relative_columns <- results$columns/results[results$default,"columns"][1] - 1
results$relative_nonzeroes <- results$nonzeroes/results[results$default,"nonzeroes"][1] - 1
results$relative_nlcode <- results$nlcode/results[results$default,"nlcode"][1] - 1
results$relative_nlnonzeroes <- results$nlnonzeroes/results[results$default,"nlnonzeroes"][1] - 1
attr(results,"default_cfg") <- cfg
attr(results,"id") <- id
#if(file.exists(path(default,"svn.rda"))) {
# load(path(default,"svn.rda"))
# attr(results,"svn") <- svn
#}
setwd(maindir)
if(plot) performance_plot(results)
return(results)
}
performance_plot <- function(x) {
require(ggplot2)
require(lusweave)
.create_plot <- function(x,weight="relative_runtime",label="runtime/default_runtime2") {
p <- ggplot(x, aes_string(x="info",weight=weight,fill="module")) + geom_bar(position="dodge")+coord_flip()+labs(y=label,x="",size=2)
p <- p + geom_text(aes(y=0,label=more_info), hjust=0, size = 3) + theme(legend.position="none")
print(p)
}
.tmptable <- function(sw,tmp) {
nrow <- ceiling(length(tmp)/2)
if(nrow*2 > length(tmp)) tmp <- c(tmp,"")
swtable(sw,matrix(tmp,nrow),align="l",include.colnames=FALSE,include.rownames=FALSE,vert.lines=0,hor.lines=0)
return(sw)
}
sw_option <- "width=11,height=12"
sw <- swopen(paste("performance_check_",attr(x,"id"),".pdf",sep=""))
swlatex(sw,c("\\title{Performance test results}","\\author{Model Operations Group}","\\maketitle","\\tableofcontents"))
swlatex(sw,"\\newpage")
swlatex(sw,"\\section{General information}")
swlatex(sw,"\\subsection{Settings default run}")
sw <- .tmptable(sw,x$info[x$more_info =="default"])
swlatex(sw,"\\subsection{Run information default run}")
tmp <- x[x$more_info=="default",][1,c("runtime","rows","columns","nonzeroes","nlcode","nlnonzeroes")]
tmp2 <- cbind(format(round(as.vector(as.matrix(tmp)),2),nsmall=2),names(tmp))
rownames(tmp2) <- names(tmp)
colnames(tmp2) <- c("data","description")
tmp2[,"description"] <- c(" minutes (total runtime)",
" (average over all optimizations)",
" (average over all optimizations)",
" (average over all optimizations)",
" (average over all optimizations)",
" (average over all optimizations)")
swtable(sw,tmp2,align="r")
swlatex(sw,"\\newpage")
#remove default run from results
x <- x[x$more_info!="default",]
tmp <- x$info[x$more_info =="compilation error"]
if(length(tmp)>0){
swlatex(sw,"\\subsection{Compilation errors}")
sw <- .tmptable(sw,tmp)
#remove runs with compilation error from results
x <- x[x$more_info!="compilation error",]
}
tmp <- x$info[x$more_info =="infeasible solution"]
if(length(tmp)>0){
swlatex(sw,"\\subsection{Infeasible solutions}")
sw <- .tmptable(sw,tmp)
}
.tmp <- function(sw,x) {
swlatex(sw,"\\subsection{relative runtime}")
swfigure(sw,.create_plot,x,weight="relative_runtime",label="runtime relative to default run",sw_option=sw_option)
swlatex(sw,"\\subsection{relative number of rows}")
swfigure(sw,.create_plot,x,weight="relative_rows",label="No. of rows relative to default",sw_option=sw_option)
swlatex(sw,"\\subsection{relative number of columns}")
swfigure(sw,.create_plot,x,weight="relative_columns",label="No. of columns relative to default",sw_option=sw_option)
swlatex(sw,"\\subsection{relative number of nonzeroes}")
swfigure(sw,.create_plot,x,weight="relative_nonzeroes",label="No. of nonzeroes relative to default",sw_option=sw_option)
swlatex(sw,"\\subsection{relative number of nl code}")
swfigure(sw,.create_plot,x,weight="relative_nlcode",label="Lines of NL code relative to default",sw_option=sw_option)
swlatex(sw,"\\subsection{relative number of nl nonzeroes}")
swfigure(sw,.create_plot,x,weight="relative_nlnonzeroes",label="No. of NL nonzeroes relative to default",sw_option=sw_option)
return(sw)
}
swlatex(sw,"\\section{Results sorted by module}")
sw <- .tmp(sw,x)
x <- x[order(x$runtime,decreasing=TRUE),]
x$info <- factor(x$info,x$info)
swlatex(sw,"\\section{Results sorted by runtime}")
sw <- .tmp(sw,x)
if(!is.null(attr(x,"svn"))){
swlatex(sw,"\\section{SVN info}")
sw <- .tmptable(sw,c(attr(x,"svn"),rep("",length(attr(x,"svn")))))
}
swlatex(sw,"\\newpage\\section{Full default config}")
tmp <- unlist(attr(x,"default_cfg"))
n <- 35
while(length(tmp)>n) {
sw <- .tmptable(sw,c(names(tmp)[1:n],tmp[1:n]))
tmp <- tmp[-(1:n)]
}
sw <- .tmptable(sw,c(names(tmp),tmp))
swclose(sw)
}
# | (C) 2006-2019 Potsdam Institute for Climate Impact Research (PIK)
# | authors, and contributors see CITATION.cff file. This file is part
# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of
# | AGPL-3.0, you are granted additional permissions described in the
# | REMIND License Exception, version 1.0 (see LICENSE file).
# | Contact: remind@pik-potsdam.de
library(lucode, quietly = TRUE,warn.conflicts =FALSE)
library(dplyr, quietly = TRUE,warn.conflicts =FALSE)
require(gdx)
# Function to create the levs.gms, fixings.gms, and margs.gms files, used in
# delay scenarios.
create_fixing_files <- function(cfg, input_ref_file = "input_ref.gdx") {
# Start the clock.
begin <- Sys.time()
# Extract data from input_ref.gdx file and store in levs_margs_ref.gms.
system(paste("gdxdump",
input_ref_file,
"Format=gamsbas Delim=comma FilterDef=N Output=levs_margs_ref.gms",
sep = " "))
# Read data from levs_margs_ref.gms.
ref_gdx_data <- suppressWarnings(readLines("levs_margs_ref.gms"))
# Create fixing files.
cat("\n")
create_standard_fixings(cfg, ref_gdx_data)
# Stop the clock.
cat("Time it took to create the fixing files: ")
manipulate_runtime <- Sys.time()-begin
print(manipulate_runtime)
cat("\n")
# Delete file.
file.remove("levs_margs_ref.gms")
}
# Function to create the levs.gms, fixings.gms, and margs.gms files, used in
# the standard (i.e. the non-macro stand-alone) delay scenarios.
create_standard_fixings <- function(cfg, ref_gdx_data) {
# Declare empty lists to hold the strings for the 'manipulateFile' functions.
full_manipulateThis <- NULL
levs_manipulateThis <- NULL
fixings_manipulateThis <- NULL
margs_manipulateThis <- NULL
str_years <- c()
no_years <- (cfg$gms$cm_startyear - 2005) / 5
# Write level values to file
levs <- c()
for (i in 1:no_years) {
str_years[i] <- paste("L \\('", 2000 + i * 5, sep = "")
levs <- c(levs, grep(str_years[i], ref_gdx_data, value = TRUE))
}
writeLines(levs, "levs.gms")
# Replace fixing.gms with level values
file.copy("levs.gms", "fixings.gms", overwrite = TRUE)
fixings_manipulateThis <- c(fixings_manipulateThis, list(c(".L ", ".FX ")))
#cb q_co2eq is only "static" equation to be active before cm_startyear, as multigasscen could be different from a scenario to another that is fixed on the first
#cb therefore, vm_co2eq cannot be fixed, otherwise infeasibilities would result. vm_co2eq.M is meaningless, is never used in the code (a manipulateFile delete line command would be even better)
# manipulateFile("fixings.gms", list(c("vm_co2eq.FX ", "vm_co2eq.M ")))
# Write marginal values to file
margs <- c()
str_years <- c()
for (i in 1:no_years) {
str_years[i] <- paste("M \\('", 2000 + i * 5, sep = "")
margs <- c(margs, grep(str_years[i], ref_gdx_data, value = TRUE))
}
writeLines(margs, "margs.gms")
# temporary fix so that you can use older gdx for fixings - will become obsolete in the future and can be deleted once the next variable name change is done
margs_manipulateThis <- c(margs_manipulateThis, list(c("q_taxrev","q21_taxrev")))
# fixing for SPA runs based on ModPol input data
margs_manipulateThis <- c(margs_manipulateThis,
list(c("q41_emitrade_restr_mp.M", "!!q41_emitrade_restr_mp.M")),
list(c("q41_emitrade_restr_mp2.M", "!!q41_emitrade_restr_mp2.M")))
#AJS this symbol is not known and crashes the run - is it depreciated? TODO
levs_manipulateThis <- c(levs_manipulateThis,
list(c("vm_pebiolc_price_base.L", "!!vm_pebiolc_price_base.L")))
#AJS filter out nash marginals in negishi case, as they would lead to a crash when trying to fix on them:
if(cfg$gms$optimization == 'negishi'){
margs_manipulateThis <- c(margs_manipulateThis, list(c("q80_costAdjNash.M", "!!q80_costAdjNash.M")))
}
if(cfg$gms$subsidizeLearning == 'off'){
levs_manipulateThis <- c(levs_manipulateThis,
list(c("v22_costSubsidizeLearningForeign.L",
"!!v22_costSubsidizeLearningForeign.L")))
margs_manipulateThis <- c(margs_manipulateThis,
list(c("q22_costSubsidizeLearning.M", "!!q22_costSubsidizeLearning.M")),
list(c("v22_costSubsidizeLearningForeign.M",
"!!v22_costSubsidizeLearningForeign.M")),
list(c("q22_costSubsidizeLearningForeign.M",
"!!q22_costSubsidizeLearningForeign.M")))
fixings_manipulateThis <- c(fixings_manipulateThis,
list(c("v22_costSubsidizeLearningForeign.FX",
"!!v22_costSubsidizeLearningForeign.FX")))
}
#JH filter out negishi marginals in nash case, as they would lead to a crash when trying to fix on them:
if(cfg$gms$optimization == 'nash'){
margs_manipulateThis <- c(margs_manipulateThis,
list(c("q80_balTrade.M", "!!q80_balTrade.M")),
list(c("q80_budget_helper.M", "!!q80_budget_helper.M")))
}
#RP filter out module 40 techpol fixings
if(cfg$gms$techpol == 'none'){
margs_manipulateThis <- c(margs_manipulateThis,
list(c("q40_NewRenBound.M", "!!q40_NewRenBound.M")),
list(c("q40_CoalBound.M", "!!q40_CoalBound.M")),
list(c("q40_LowCarbonBound.M", "!!q40_LowCarbonBound.M")),
list(c("q40_FE_RenShare.M", "!!q40_FE_RenShare.M")),
list(c("q40_trp_bound.M", "!!q40_trp_bound.M")),
list(c("q40_TechBound.M", "!!q40_TechBound.M")),
list(c("q40_ElecBioBound.M", "!!q40_ElecBioBound.M")),
list(c("q40_PEBound.M", "!!q40_PEBound.M")),
list(c("q40_PEcoalBound.M", "!!q40_PEcoalBound.M")),
list(c("q40_PEgasBound.M", "!!q40_PEgasBound.M")),
list(c("q40_PElowcarbonBound.M", "!!q40_PElowcarbonBound.M")),
list(c("q40_EV_share.M", "!!q40_EV_share.M")),
list(c("q40_TrpEnergyRed.M", "!!q40_TrpEnergyRed.M")),
list(c("q40_El_RenShare.M", "!!q40_El_RenShare.M")),
list(c("q40_BioFuelBound.M", "!!q40_BioFuelBound.M")))
}
if(cfg$gms$techpol == 'NPi2018'){
margs_manipulateThis <- c(margs_manipulateThis,
list(c("q40_El_RenShare.M", "!!q40_El_RenShare.M")),
list(c("q40_CoalBound.M", "!!q40_CoalBound.M")))
}
# Include fixings (levels) and marginals in full.gms at predefined position
# in core/loop.gms.
full_manipulateThis <- c(full_manipulateThis,
list(c("cb20150605readinpositionforlevelfile",
paste("first offlisting inclusion of levs.gms so that level value can be accessed",
"$offlisting",
"$include \"levs.gms\";",
"$onlisting", sep = "\n"))))
full_manipulateThis <- c(full_manipulateThis, list(c("cb20140305readinpositionforfinxingfiles",
paste("offlisting inclusion of levs.gms, fixings.gms, and margs.gms",
"$offlisting",
"$include \"levs.gms\";",
"$include \"fixings.gms\";",
"$include \"margs.gms\";",
"$onlisting", sep = "\n"))))
# Perform actual manipulation on levs.gms, fixings.gms, and margs.gms in
# single, respective, parses of the texts.
manipulateFile("levs.gms", levs_manipulateThis)
manipulateFile("fixings.gms", fixings_manipulateThis)
manipulateFile("margs.gms", margs_manipulateThis)
# Perform actual manipulation on full.gms, in single parse of the text.
manipulateFile("full.gms", full_manipulateThis)
}
# Set value source_include so that loaded scripts know, that they are
# included as source (instead a load from command line)
source_include <- TRUE
# unzip all .gz files
system("gzip -d -f *.gz")
# Load REMIND run configuration
load("config.Rdata")
#AJS set MAGCFG file
magcfgFile = paste0('./magicc/MAGCFG_STORE/','MAGCFG_USER_',toupper(cfg$gms$cm_magicc_config),'.CFG')
if(!file.exists(magcfgFile)){
stop(paste('ERROR in MAGGICC configuration: Could not find file ',magcfgFile))
}
system(paste0('cp ',magcfgFile,' ','./magicc/MAGCFG_USER.CFG'))
# Change flag "cm_compile_main" from TRUE to FALSE since we are not compiling
# main.gms but executing full.gms and therefore want to load some data from the
# input.gdx files.
manipulateFile("full.gms", list(c("\\$setglobal cm_compile_main *TRUE",
"\\$setglobal cm_compile_main FALSE")))
# Prepare the files containing the fixings for delay scenarios (for fixed runs)
if ( cfg$gms$cm_startyear > 2005 & (!file.exists("levs.gms.gz") | !file.exists("levs.gms"))) {
create_fixing_files(cfg = cfg, input_ref_file = "input_ref.gdx")
}
# Store REMIND directory and output file names
maindir <- cfg$remind_folder
REMIND_mif_name <- paste("REMIND_generic_", cfg$title, ".mif", sep = "")
# Print message
cat("\nStarting REMIND...\n")
# Save start time
begin <- Sys.time()
# Call GAMS
if (cfg$gms$CES_parameters == "load") {
system(paste0(cfg$gamsv, " full.gms -errmsg=1 -a=", cfg$action,
" -ps=0 -pw=185 -gdxcompress=1 -logoption=", cfg$logoption))
} else if (cfg$gms$CES_parameters == "calibrate") {
# Remember file modification time of fulldata.gdx to see if it changed
fulldata_m_time <- Sys.time();
# Save original input
file.copy("input.gdx", "input_00.gdx", overwrite = TRUE)
# Iterate calibration algorithm
for (cal_itr in 1:cfg$gms$c_CES_calibration_iterations) {
cat("CES calibration iteration: ", cal_itr, "\n")
# Update calibration iteration in GAMS file
system(paste0("sed -i 's/^\\(\\$setglobal c_CES_calibration_iteration ",
"\\).*/\\1", cal_itr, "/' full.gms"))
system(paste0(cfg$gamsv, " full.gms -errmsg=1 -a=", cfg$action,
" -ps=0 -pw=185 -gdxcompress=1 -logoption=", cfg$logoption))
# If GAMS found a solution
if ( file.exists("fulldata.gdx")
&& file.info("fulldata.gdx")$mtime > fulldata_m_time) {
#create the file to be used in the load mode
getLoadFile <- function(){
file_name = paste0(cfg$gms$cm_CES_configuration,"_ITERATION_",cal_itr,".inc")
ces_in = system("gdxdump fulldata.gdx symb=in NoHeader Format=CSV", intern = TRUE) %>% gsub("\"","",.) #" This comment is just to obtain correct syntax highlighting
expr_ces_in = paste0("(",paste(ces_in, collapse = "|") ,")")
tmp = system("gdxdump fulldata.gdx symb=pm_cesdata", intern = TRUE)[-(1:2)] %>%
grep("(quantity|price|eff|effgr|xi|rho|offset_quantity|compl_coef)", x = ., value = TRUE)
tmp = tmp %>% grep(expr_ces_in,x = ., value = T)
tmp %>%
sub("'([^']*)'.'([^']*)'.'([^']*)'.'([^']*)' (.*)[ ,][ /];?",
"pm_cesdata(\"\\1\",\"\\2\",\"\\3\",\"\\4\") = \\5;", x = .) %>%
write(file_name)
pm_cesdata_putty = system("gdxdump fulldata.gdx symb=pm_cesdata_putty", intern = TRUE)
if (length(pm_cesdata_putty) == 2){
tmp_putty = gsub("^Parameter *([A-z_(,)])+cesParameters\\).*$",'\\1"quantity") = 0;', pm_cesdata_putty[2])
} else {
tmp_putty = pm_cesdata_putty[-(1:2)] %>%
grep("quantity", x = ., value = TRUE) %>%
grep(expr_ces_in,x = ., value = T)
}
tmp_putty %>%
sub("'([^']*)'.'([^']*)'.'([^']*)'.'([^']*)' (.*)[ ,][ /];?",
"pm_cesdata_putty(\"\\1\",\"\\2\",\"\\3\",\"\\4\") = \\5;", x = .)%>% write(file_name,append =T)
}
getLoadFile()
# Store all the interesting output
file.copy("full.lst", sprintf("full_%02i.lst", cal_itr), overwrite = TRUE)
file.copy("full.log", sprintf("full_%02i.log", cal_itr), overwrite = TRUE)
file.copy("fulldata.gdx", "input.gdx", overwrite = TRUE)
file.copy("fulldata.gdx", sprintf("input_%02i.gdx", cal_itr),
overwrite = TRUE)
# Update file modification time
fulldata_m_time <- file.info("fulldata.gdx")$mtime
} else {
break
}
}
} else {
stop("unknown realisation of 29_CES_parameters")
}
# Calculate run time
gams_runtime <- Sys.time() - begin
# If REMIND actually did run
if (cfg$action == "ce" && cfg$gms$c_skip_output != "on") {
# Print Message
cat("\nREMIND run finished!\n")
# Create solution report for Nash runs
if (cfg$gms$optimization == "nash" && cfg$gms$cm_nash_mode != "debug" && file.exists("fulldata.gdx")) {
system("gdxdump fulldata.gdx Format=gamsbas Delim=comma Output=output_nash.gms")
file.append("full.lst", "output_nash.gms")
file.remove("output_nash.gms")
}
}
# Collect and submit run statistics to central data base
lucode::runstatistics(file = "runstatistics.rda",
modelstat = readGDX(gdx="fulldata.gdx","o_modelstat", format="first_found"),
config = cfg,
runtime = gams_runtime,
setup_info = lucode::setup_info(),
submit = cfg$runstatistics)
# Compress files with the fixing-information
if (cfg$gms$cm_startyear > 2005)
system("gzip -f levs.gms margs.gms fixings.gms")
# go up to the main folder, where the cfg files for subsequent runs are stored
setwd(cfg$remind_folder)
#====================== Subsequent runs ===========================
# 1. Save the path to the fulldata.gdx of the current run to the cfg files
# of the runs that use it as 'input_bau.gdx'
# Use the name to check whether it is a coupled run (TRUE if the name ends with "-rem-xx")
coupled_run <- grepl("-rem-[0-9]{1,2}$",cfg$title)
no_ref_runs <- identical(cfg$RunsUsingTHISgdxAsBAU,character(0)) | all(is.na(cfg$RunsUsingTHISgdxAsBAU)) | coupled_run
if(!no_ref_runs) {
source("scripts/start_functions.R")
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
for(run in seq(1,length(cfg_main$RunsUsingTHISgdxAsBAU))){
# for each of the runs that use this gdx as bau, read in the cfg, ...
cat("Writing the path for input_bau.gdx to ",paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"),"\n")
load(paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
# ...change the path_gdx_bau field of the subsequent run to the fulldata gdx of the current run ...
cfg$files2export$start['input_bau.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$RunsUsingTHISgdxAsBAU[run],".RData"))
}
# Set cfg back to original
cfg <- cfg_main
}
# 2. Save the path to the fulldata.gdx of the current run to the cfg files
# of the subsequent runs that use it as 'input_ref.gdx' and start these runs
no_subsequent_runs <- identical(cfg$subsequentruns,character(0)) | identical(cfg$subsequentruns,NULL) | coupled_run
if(no_subsequent_runs){
cat('\nNo subsequent run was set for this scenario\n')
} else {
# Save the current cfg settings into a different data object, so that they are not overwritten
cfg_main <- cfg
source("scripts/start_functions.R")
for(run in seq(1,length(cfg_main$subsequentruns))){
# for each of the subsequent runs, read in the cfg, ...
cat("Writing the path for input_ref.gdx to ",paste0(cfg_main$subsequentruns[run],".RData"),"\n")
load(paste0(cfg_main$subsequentruns[run],".RData"))
# ...change the path_gdx_ref field of the subsequent run to the fulldata gdx of the current (preceding) run ...
cfg$files2export$start['input_ref.gdx'] <- paste0(cfg_main$remind_folder,"/",cfg_main$results_folder,"/fulldata.gdx")
save(cfg, file = paste0(cfg_main$subsequentruns[run],".RData"))
# Subsequent runs will be started in submit.R using the RData files written above
# after the current run has finished.
cat("Starting subsequent run ",cfg_main$subsequentruns[run],"\n")
start_run(cfg)
}
# Set cfg back to original
cfg <- cfg_main
}
# 3. Create script file that can be used later to restart the subsequent runs manually.
# In case there are no subsequent runs (or it's coupled runs), the file contains only
# a small message.
subseq_start_file <- paste0(cfg$results_folder,"/start_subsequentruns.R")
if(no_subsequent_runs){
write("cat('\nNo subsequent run was set for this scenario\n')",file=subseq_start_file)
} else {
# go up to the main folder, where the cfg. files for subsequent runs are stored
filetext <- paste0("setwd('",cfg$remind_folder,"')\n")
filetext <- paste0(filetext,"source('scripts/start_functions.R')\n")
for(run in seq(1,length(cfg$subsequentruns))){
filetext <- paste0(filetext,"\n")
filetext <- paste0(filetext,"load('",cfg$subsequentruns[run],".RData')\n")
filetext <- paste0(filetext,"cat('",cfg$subsequentruns[run],"')\n")
filetext <- paste0(filetext,"start_run(cfg)\n")
}
# Write the text to the file
write(filetext,file=subseq_start_file)
}
#=================== END - Subsequent runs ========================
# Print REMIND runtime
cat("\n gams_runtime is ", gams_runtime, "\n")
# Copy important files into output_folder (after REMIND execution)
for (file in cfg$files2export$end)
file.copy(file, cfg$results_folder, overwrite = TRUE)
# Postprocessing / Output Generation
output <- cfg$output
outputdir <- cfg$results_folder
sys.source("output.R",envir=new.env())
#!/bin/bash
#--- Job Submission parameters ------
#SBATCH --qos=medium
#SBATCH --job-name=__JOB_NAME__
#SBATCH --output=log.txt
#SBATCH --tasks-per-node=2
#SBATCH --mail-type=END
#------------------------------------------------
# report git revision info and changes in the files
git rev-parse --short HEAD
git status
# start gams job
Rscript submit.R
#!/bin/bash
#SBATCH --qos=priority
#SBATCH --job-name=rem-compare
#SBATCH --output=log-%j.out
#SBATCH --mail-type=END
#SBATCH --mem=32000
#SBATCH --cpus-per-task=2
Rscript scripts/utils/compareParallel.R
#!/bin/bash
#--- Job Submission parameters ------
#SBATCH --qos=short
#SBATCH --job-name=__JOB_NAME__
#SBATCH --output=log.txt
#SBATCH --nodes=1
#SBATCH --tasks-per-node=12
#SBATCH --mail-type=END
#------------------------------------------------
# report git revision info and changes in the files
git rev-parse --short HEAD
git status
# start gams job
Rscript submit.R
#!/bin/bash
#--- Job Submission parameters ------
#SBATCH --qos=short
#SBATCH --job-name=__JOB_NAME__
#SBATCH --output=log.txt
#SBATCH --nodes=1
#SBATCH --tasks-per-node=16
#SBATCH --mail-type=END
#------------------------------------------------
# report git revision info and changes in the files
git rev-parse --short HEAD
git status
# start gams job
Rscript submit.R
#!/bin/bash
#--- Job Submission parameters ------
#SBATCH --qos=short
#SBATCH --job-name=__JOB_NAME__
#SBATCH --output=log.txt
#SBATCH --mail-type=END
#------------------------------------------------
# report git revision info and changes in the files
git rev-parse --short HEAD
git status
# start gams job
Rscript submit.R
#######################################################################
############### Select slurm partitiion ###############################
#######################################################################
get_line <- function(){
# gets characters (line) from the terminal or from a connection
# and returns it
if(interactive()){
s <- readline()
} else {
con <- file("stdin")
s <- readLines(con, 1, warn=FALSE)
on.exit(close(con))
}
return(s);
}
choose_slurmConfig <- function() {
slurm <- suppressWarnings(ifelse(system2("srun",stdout=FALSE,stderr=FALSE) != 127, TRUE, FALSE))
if (slurm) {
modes <- c(" SLURM standby - task per node: 12 (nash H12) [recommended]",
" SLURM standby - task per node: 13 (nash H12 coupled)",
" SLURM standby - task per node: 16 (nash H12+)",
" SLURM standby - task per node: 1 (nash debug, test one regi)",
" SLURM priority - task per node: 12 (nash H12) [recommended]",
" SLURM priority - task per node: 13 (nash H12 coupled)",
" SLURM priority - task per node: 16 (nash H12+)",
" SLURM priority - task per node: 1 (nash debug, test one regi)",
" SLURM short - task per node: 12 (nash H12)",
"SLURM short - task per node: 16 (nash H12+)",
"SLURM short - task per node: 1 (nash debug, test one regi)",
"SLURM medium - task per node: 1 (negishi)",
"SLURM long - task per node: 1 (negishi)")
cat("\nCurrent cluster utilization:\n")
system("sclass")
cat("\n")
cat("\nPlease choose run submission type:\n")
cat(paste(1:length(modes), modes, sep=": " ),sep="\n")
cat("Number: ")
identifier <- get_line()
identifier <- as.numeric(strsplit(identifier,",")[[1]])
comp <- switch(identifier,
"1" = "--qos=standby --nodes=1 --tasks-per-node=12" , # SLURM standby - task per node: 12 (nash H12) [recommended]
"2" = "--qos=standby --nodes=1 --tasks-per-node=13" , # SLURM standby - task per node: 13 (nash H12 coupled)
"3" = "--qos=standby --nodes=1 --tasks-per-node=16" , # SLURM standby - task per node: 16 (nash H12+)
"4" = "--qos=standby --nodes=1 --tasks-per-node=1" , # SLURM standby - task per node: 1 (nash debug, test one regi)
"5" = "--qos=priority --nodes=1 --tasks-per-node=12" , # SLURM priority - task per node: 12 (nash H12) [recommended]
"6" = "--qos=priority --nodes=1 --tasks-per-node=13" , # SLURM priority - task per node: 13 (nash H12 coupled)
"7" = "--qos=priority --nodes=1 --tasks-per-node=16" , # SLURM priority - task per node: 16 (nash H12+)
"8" = "--qos=priority --nodes=1 --tasks-per-node=1" , # SLURM priority - task per node: 1 (nash debug, test one regi)
"9" = "--qos=short --nodes=1 --tasks-per-node=12" , # SLURM short - task per node: 12 (nash H12)
"10" = "--qos=short --nodes=1 --tasks-per-node=16" , # SLURM short - task per node: 16 (nash H12+)
"11" = "--qos=short --nodes=1 --tasks-per-node=1" , # SLURM short - task per node: 1 (nash debug, test one regi)
"12" = "--qos=medium --nodes=1 --tasks-per-node=1" , # SLURM medium - task per node: 1 (negishi)
"13" = "--qos=long --nodes=1 --tasks-per-node=1" ) # SLURM long - task per node: 1 (negishi)
if(is.null(comp)) stop("This type is invalid. Please choose a valid type")
} else {
comp <- "direct"
}
return(comp)
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment