logging and config changed

.flake8

+[flake8]
+max-line-length = 122
+select = C,E,F,W,B,B9
+ignore = E203, E501, W503
+exclude = __init__.py, venv/, config/
\ No newline at end of file

.gitignore

 .idea
 config/
 p.py
-__pycache__
\ No newline at end of file
+__pycache__
+/hypercorn.access
+/hypercorn.error
+.pyc

README.md

@@ -10,52 +10,36 @@ It also serves the web frontend.
 virtualenv venv
 source venv/bin/activate
 pip install -r requirements.txt
-``` 
-
-At the moment, there is a breaking bug in tap, so you have to edit `tap/utils.py` in line 182 after installation:
-```python
-def get_class_column(obj: type) -> int:
-    """Determines the column number for class variables in a class."""
-    first_line = 1
-    for token_type, token, (start_line, start_column), (end_line, end_column), line in tokenize_source(obj):
-        if token.strip() == '@':
-            first_line += 1
-        if start_line <= first_line or token.strip() == '':
-            continue
-
-        return start_column
 ```
-Keep track of https://github.com/swansonk14/typed-argument-parser/issues/80
 
 For development, it is advised to install `nacsos-data` locally (not from git) via
 ```bash
 pip install -e ../nacsos-data/
 ```
-(assuming both projects live side-by-side)
+(assuming both projects reside side-by-side, otherwise adapt path accordingly)
+
+There needs to be a change to the hypercorn code as per https://gitlab.com/pgjones/hypercorn/-/merge_requests/70/diffs
 
 ## Running the database with docker
 Start up the database by running docker (or use your local instance)
 ```bash
 sudo systemctl start docker
-docker-compose up
+docker-compose up -d
 ```
 
 ## Starting the server
 ```bash
-python main.py
-
-# optionally, you can specify the config file directly
-NACSOS_CONFIG=config/local.toml python main.py
+# set this in case you want to use a different config (optional)
+export NACSOS_CONFIG=config/default.env
 
-# additionally, you can always override all exposed config variables directly
-# for more info, check
-python main.py -h
+# for development, using the --reload option is helpful
+hypercorn --config=config/hypercorn.toml --reload main:app 
 ```
 
 The configuration is read in the following order (and overridden by consecutive steps):
-- `@dataclasses` in `server/util/config.py`
-- TOML config file (either `config/default.toml` or whatever is in `NACSOS_CONFIG`)
-- Command line arguments
+1. Classes in `server/util/config.py`
+2. .env config file (whatever is in `NACSOS_CONFIG`; defaulting to `config/default.env`)
+3. Environment variables
 
 The default config is set up to work with a locally running docker instance with its respective default config.
 It should never be changed, always make a local copy and never commit it to the repository!
\ No newline at end of file

config/default.toml

-[server]
-host = "localhost"
-port = 8080
-hosts = ['0.0.0.0', 'localhost']
-
-[db]
-host = "nacsos_postgres"
-port = 5432
-user = "root"
-pw = "root"
-database = "nacsos_core"
\ No newline at end of file

docker-compose.yml

+version: '3.8'
+
+services:
+  db:
+    container_name: nacsos_postgres
+    image: postgres:14.2-bullseye
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: nacsos_core
+    ports:
+      - "5432:5432"
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+  pgadmin:
+    container_name: pgadmin4_container
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@admin.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+volumes:
+  pg_data:
\ No newline at end of file

main.py

 #!/usr/bin/env python3
 
+from server.util.config import settings
+from server.util.logging import get_logger
 
-def run(args=None):
-    import logging
+logger = get_logger('nacsos.main')
+logger.info('Starting up server')
 
-    from server.util.log import init_logging
-    init_logging()
 
-    logger = logging.getLogger('nacsos.main')
-    logger.info('Starting up uvicorn')
-
-    # this should be imported here to ensure config gets initialised first
-    from server.util.config import conf
-
-    # import asyncio
-    # from hypercorn.config import Config
-    # from hypercorn.asyncio import serve
-
-    import uvicorn
+def get_app():
     from server.api.server import Server
-    from server.data.database import init_db
 
     server = Server()
 
-    uvicorn.run(server.app, host=conf.server.host, port=conf.server.port)
+    return server.app
 
-    init_db(server.app)
 
-    return server.app
+app = get_app()
+
+# config = Config()
+# config.bind = f'{settings.SERVER.HOST}:{settings.SERVER.PORT}'
+# config.debug = settings.SERVER.DEBUG_MODE
+# config.accesslog = get_logger('hypercorn.access')
+# config.errorlog = get_logger('hypercorn.error')
+# config.logconfig_dict = settings.LOGGING_CONF
+# config.access_log_format = '%(s)s | "%(R)s" |  Size: %(b)s | Referrer: "%(f)s"'
 
-if __name__ == '__main__':
-    run()

requirements.txt

-fastapi==0.75.0
+fastapi==0.75.1
 hypercorn==0.13.2
-typed-argument-parser==1.7.2
-toml==0.10.2
\ No newline at end of file
+toml==0.10.2
+email-validator==1.1.3
+python-dotenv==0.20.0
\ No newline at end of file

server/api/middlewares.py

 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
 from starlette.requests import Request
 from starlette.responses import Response
-import logging
+from server.util.logging import get_logger
 import time
 
-logger = logging.getLogger('nacsos.server.middlewares')
+logger = get_logger('nacsos.server.middlewares')
 try:
     from resource import getrusage, RUSAGE_SELF
 except ImportError as e:

server/api/routes/admin/users.py

+from fastapi import APIRouter
+from fastapi.responses import PlainTextResponse
+from server.util.logging import get_logger
+from nacsos_data.models.users import User
+logger = get_logger('nacsos.api.route.admin.users')
+router = APIRouter()
+
+
+@router.get('/', response_class=PlainTextResponse)
+async def get_users() -> str:
+    return 'pong'
+

server/api/routes/ping.py

 from fastapi import APIRouter
 from fastapi.responses import PlainTextResponse
-import logging
+from server.util.logging import get_logger
 
-logger = logging.getLogger('nacsos.api.route.ping')
+logger = get_logger('nacsos.api.route.ping')
 router = APIRouter()
 
 logger.debug('Setup nacsos.api.route.ping router')

server/api/server.py

@@ -5,14 +5,15 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.middleware.cors import CORSMiddleware
 from server.api.middlewares import TimingMiddleware
 
-from server.util.config import conf
+from server.util.config import settings
+from server.util.logging import get_logger
 from server.api.routes import ping
-import logging
+from server.api.routes.admin import users
 import mimetypes
 
 mimetypes.init()
 
-logger = logging.getLogger('nacsos.server')
+logger = get_logger('nacsos.server')
 
 try:
     from resource import getrusage, RUSAGE_SELF
@@ -21,6 +22,7 @@ except ImportError as e:
 
     RUSAGE_SELF = None
 
+
     def getrusage(*args):
         return 0.0, 0.0
 
@@ -30,7 +32,7 @@ class APISubRouter:
         self.router = APIRouter()
         self.paths = {
             '/ping': ping,
-            # '/platforms': platforms,
+            '/admin/users': users,
             # '/graph': graph
         }
         for path, router in self.paths.items():
@@ -44,17 +46,16 @@ class Server:
         logger.debug('Setting up server and middlewares')
         mimetypes.add_type('application/javascript', '.js')
 
-        if conf.server.header_trusted_host:
-            self.app.add_middleware(TrustedHostMiddleware, allowed_hosts=conf.server.hosts)
-        if conf.server.header_cors:
-            self.app.add_middleware(CORSMiddleware, allow_origins=conf.server.hosts,
-                                    allow_methods=['GET', 'POST', 'DELETE'])
+        if settings.SERVER.HEADER_TRUSTED_HOST:
+            self.app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.SERVER.CORS_ORIGINS)
+        if settings.SERVER.HEADER_CORS:
+            self.app.add_middleware(CORSMiddleware, allow_origins=settings.SERVER.CORS_ORIGINS,
+                                    allow_methods=['GET', 'POST', 'DELETE', 'POST'])
         self.app.add_middleware(GZipMiddleware, minimum_size=1000)
-        self.app.add_middleware(TimingMiddleware)
+        # self.app.add_middleware(TimingMiddleware)
 
         logger.debug('Setup routers')
         self.api_router = APISubRouter()
         self.app.include_router(self.api_router.router, prefix='/api')
 
-        self.app.mount('/', StaticFiles(directory=conf.server.static_files, html=True), name='static')
-
+        self.app.mount('/', StaticFiles(directory=settings.SERVER.STATIC_FILES, html=True), name='static')

server/util/config.py

-from tap import Tap
-from tap.utils import get_class_variables
-from typing import get_type_hints, Any
-from dataclasses import dataclass, field
-from collections import OrderedDict
+from typing import Any, Dict, List, Optional, Union
+import secrets
+import json
+import yaml
 import os
-import toml
-
-NestedConfigDict = dict[str, dict[str, Any]]
-
-
-@dataclass
-class ServerConfig:
-    host: str = 'localhost'  # host to run this server on
-    port: int = 8080  # port for this serve to listen at
-    debug_mode: bool = False  # set this to true in order to get more detailed logs
-    hosts: list[str] = field(default_factory=lambda: ['0.0.0.0', 'localhost'])  # list of trusted hosts
-    header_trusted_host: bool = False  # set to true to allow hosts from any origin
-    header_cors: bool = False  # set to true to allow CORS
-    workers: int = 2  # number of worker processes
-    static_files: str = '../nacsos-web/dist/'  # path to the static files to be served
-
-
-@dataclass
-class DatabaseConfig:
-    pw: str  # password for the database user
-    user: str = 'nacsos'  # username for the database
-    database: str = 'nacsos_core'  # name of the database
-    host: str = 'localhost'  # host of the db server
-    port: int = 5432  # port of the db server
-
-
-class Config:
-    _sub_configs = {
-        'server': ServerConfig,
-        'db': DatabaseConfig,
-    }
-
-    DEFAULT_CONFIG_FILE = 'config/default.toml'
-
-    def __init__(self, cli_args: list[str] = None):
-        stored_config = self._read_config_file()
-        shlex_config = dict2shlex(stored_config)
-        config = self._read_cli_args(shlex_config, cli_args)
-
-        self.server: ServerConfig = ServerConfig(**config['server'])
-        self.db: DatabaseConfig = DatabaseConfig(**config['db'])
-
-    def _read_config_file(self) -> NestedConfigDict:
-        conf_file = os.environ.get('NACSOS_CONF', self.DEFAULT_CONFIG_FILE)
-        with open(conf_file, 'r') as f:
-            return toml.load(f)
-
-    def _read_cli_args(self, shlex_config: list[str], cli_args: list[str]):
-        """
-        This method generates a Tap (typed-argument-parser) instance from the config classes
-        and exposes the variables including help (comments) and types to the command line.
-        It then parses all CLI arguments and returns a nested dictionary.
-        :return:
-        """
-
-        # create a typed argument parser by gathering all sub-configs (as argument prefixes)
-        # to expose all config attributes to the command line
-        class ProgrammaticArgumentParser(Tap):
-            def configure(self):
-                class_variables = []
-                annotations = []
-                for cls_name, cls in Config._sub_configs.items():
-                    # append all class attributes, including annotations (e.g. comments)
-                    class_variables += [(f'{cls_name}_{var}', data)
-                                        for var, data in get_class_variables(cls).items()]
-
-                    # append all annotations (e.g. type hints)
-                    annotations += [(f'{cls_name}_{var}', data)
-                                    for var, data in get_type_hints(cls).items()]
-
-                    # transfer default parameters to this instance
-                    for var in get_type_hints(cls).keys():
-                        try:
-                            setattr(self, f'{cls_name}_{var}', getattr(cls, var))
-                        except AttributeError:
-                            # this attribute has no default value
-                            pass
-
-                # inject the gathered class variables and annotations to the tap instance
-                self.class_variables = OrderedDict(class_variables)
-                self._annotations = dict(annotations)
-                self.args_from_configs = shlex_config
-
-        # parse command line arguments
-        args = ProgrammaticArgumentParser(underscores_to_dashes=True).parse_args(cli_args)
-
-        config = {}
-        for arg, value in args.as_dict().items():
-            parts = arg.split('_')
-            if parts[0] not in config:
-                config[parts[0]] = {}
-            config[parts[0]]['_'.join(parts[1:])] = value
-
-        return config
-
-
-def dict2shlex(config: NestedConfigDict) -> list[str]:
-    ret = []
-    for cls_name, attrs in config.items():
-        for attr, value in attrs.items():
-            ret.append(f'--{cls_name}-{attr.replace("_", "-")}')
-            ret.append(f'"{value}"')
-    return ret
-
-
-conf = Config()
-
-__all__ = ['Config', 'conf']
-
-# if __name__ == '__main__':
-#     from hypercorn.config import Config as HyperConfig
-#
-#     conf = init_config()
-#     config = HyperConfig()
-#     config.workers = conf.server.workers
-#     config.server_names = conf.server.hosts
-#     config.bind = f'{conf.server.host}:{conf.server.port}'
-#     print('test')
+
+from pydantic import BaseSettings, BaseModel, PostgresDsn, AnyHttpUrl, EmailStr, validator
+
+
+# For more information how BaseSettings work, check the documentation:
+# https://pydantic-docs.helpmanual.io/usage/settings/
+
+# This is inspired by
+# https://github.com/tiangolo/full-stack-fastapi-postgresql/blob/490c554e23343eec0736b06e59b2108fdd057fdc/%7B%7Bcookiecutter.project_slug%7D%7D/backend/app/app/core/config.py
+
+
+class ServerConfig(BaseModel):
+    HOST: str = 'localhost'  # host to run this server on
+    PORT: int = 8080  # port for this serve to listen at
+    DEBUG_MODE: bool = False  # set this to true in order to get more detailed logs
+    WORKERS: int = 2  # number of worker processes
+    STATIC_FILES: str = '../nacsos-web/dist/'  # path to the static files to be served
+
+    SECRET_KEY: str = secrets.token_urlsafe(32)
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8  # = 8 days
+
+    HEADER_CORS: bool = False  # set to true to allow CORS
+    HEADER_TRUSTED_HOST: bool = False  # set to true to allow hosts from any origin
+    CORS_ORIGINS: List[AnyHttpUrl] = []  # list of trusted hosts
+
+    @validator("CORS_ORIGINS", pre=True)
+    def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
+        if isinstance(v, str) and not v.startswith('['):
+            return [i.strip() for i in v.split(',')]
+        if isinstance(v, str) and v.startswith('['):
+            return json.loads(v)
+        elif isinstance(v, (list, str)):
+            return v
+        raise ValueError(v)
+
+
+class DatabaseConfig(BaseModel):
+    HOST: str = 'localhost'  # host of the db server
+    PORT: int = 5432  # port of the db server
+    USER: str = 'nacsos'  # username for the database
+    PASSWORD: str  # password for the database user
+    DATABASE: str = 'nacsos_core'  # name of the database
+
+    CONNECTION_STR: Optional[PostgresDsn] = None
+
+    @validator('CONNECTION_STR', pre=True)
+    def build_connection_string(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
+        if isinstance(v, str):
+            return v
+        return PostgresDsn.build(
+            scheme="postgresql",
+            user=values.get('USER'),
+            password=values.get('PASSWORD'),
+            host=values.get('HOST'),
+            path=f'/{values.get("DATABASE", "")}',
+        )
+
+
+class EmailConfig(BaseModel):
+    SMTP_TLS: bool = True
+    SMTP_PORT: Optional[int] = None
+    SMTP_HOST: Optional[str] = None
+    SMTP_USER: Optional[str] = None
+    SMTP_PASSWORD: Optional[str] = None
+    SENDER_ADDRESS: Optional[EmailStr] = None
+    SENDER_NAME: Optional[str] = 'NACSOS'
+    ENABLED: bool = False
+
+    @validator("ENABLED", pre=True)
+    def get_emails_enabled(cls, v: bool, values: dict[str, Any]) -> bool:
+        return bool(
+            values.get('SMTP_HOST')
+            and values.get('SMTP_PORT')
+            and values.get('SENDER_ADDRESS')
+        )
+
+    TEST_USER: EmailStr = 'test@nacsos.eu'
+
+
+class UserConfig(BaseModel):
+    FIRST_SUPERUSER: EmailStr
+    FIRST_SUPERUSER_PASSWORD: str
+    REGISTRATION_ENABLED: bool = False
+
+
+class Settings(BaseSettings):
+    SERVER: ServerConfig
+    DB: DatabaseConfig
+    # EMAIL: EmailConfig
+
+    LOG_CONF_FILE: str = 'config/logging.conf'
+    LOGGING_CONF: Optional[dict] = None
+
+    @validator('LOGGING_CONF', pre=True)
+    def read_logging_config(cls, v: dict, values: dict[str, Any]) -> dict:
+        if isinstance(v, dict):
+            return v
+        with open(values.get('LOG_CONF_FILE'), 'r') as f:
+            return yaml.safe_load(f.read())
+
+    class Config:
+        case_sensitive = True
+        env_nested_delimiter = '__'
+
+
+conf_file = os.environ.get('NACSOS_CONFIG', 'config/default.env')
+settings = Settings(_env_file=conf_file, _env_file_encoding='utf-8')
+
+__all__ = ['settings']

server/util/email.py

+# TODO integrate this somehow in a sensible way
+# https://github.com/tiangolo/full-stack-fastapi-postgresql/blob/490c554e23343eec0736b06e59b2108fdd057fdc/%7B%7Bcookiecutter.project_slug%7D%7D/backend/app/app/utils.py
\ No newline at end of file

server/util/log.py → server/util/logging.py

-from server.util.config import conf
 import math
 import traceback
-from uvicorn.logging import AccessFormatter, DefaultFormatter
-import yaml
-import os
 import logging
 import logging.config
 
+from uvicorn.logging import DefaultFormatter
+from hypercorn.logging import Logger
 
-def init_logging():
-    with open(os.environ.get('LOGGING_CONF', 'config/logging.conf'), 'r') as f:
-        config = yaml.safe_load(f.read())
-        logging.config.dictConfig(config)
+from server.util.config import settings
 
 
-class AccessLogFormatter(AccessFormatter):
-    def formatMessage(self, record):
-        try:
-            record.__dict__.update({
-                'wall_time': record.__dict__['scope']['timing_stats']['wall_time'],
-                'cpu_time': record.__dict__['scope']['timing_stats']['cpu_time']
-            })
-        except KeyError:
-            record.__dict__.update({
-                'wall_time': '0.0?s',
-                'cpu_time': '0.0?s'
-            })
-        return super().formatMessage(record)
+def get_logger(name=None):
+    logging.config.dictConfig(settings.LOGGING_CONF)
+    return logging.getLogger(name)
 
 
 class ColourFormatter(DefaultFormatter):
@@ -42,7 +27,7 @@ class ColourFormatter(DefaultFormatter):
 
 
 def except2str(e, logger=None):
-    if conf.server.debug_mode:
+    if settings.SERVER.DEBUG_MODE:
         tb = traceback.format_exc()
         if logger:
             logger.error(tb)

server/util/security.py

+# TODO check how this can be integrated and what it does
+# https://github.com/tiangolo/full-stack-fastapi-postgresql/blob/490c554e23343eec0736b06e59b2108fdd057fdc/%7B%7Bcookiecutter.project_slug%7D%7D/backend/app/app/core/security.py
\ No newline at end of file