Merge branch 'main' into 'production'

Main

See merge request !104

.gitignore

@@ -9,4 +9,5 @@ __pycache__
 server.md
 dumps/
 scratch/
-.tasks/
+.tasks/
\ No newline at end of file
+volumes
\ No newline at end of file

README.md

@@ -84,3 +84,16 @@ FinalKillSignal=SIGKILL
 [Install]
 WantedBy=multi-user.target
 ```
+
+## Testing database things in the console
+```
+from sqlalchemy import select
+from nacsos_data.db import get_engine
+from nacsos_data.db.schemas.users import User
+engine = get_engine('config/remote.env')
+with engine.session() as session:
+    users = session.execute(select(User.email, User.full_name, User.username)
+                                       .where(User.setting_newsletter == False,
+                                              User.is_active == True)).mappings().all()
+    print(users[0])
+```
\ No newline at end of file

requirements.txt

 -r requirements_base.txt
-nacsos_data[utils,scripts] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.16.5
+nacsos_data[utils,scripts] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.17.0
\ No newline at end of file

server/api/routes/annotations.py

@@ -137,7 +137,7 @@ async def put_annotation_scheme(annotation_scheme: AnnotationSchemeModel,
 @router.delete('/schemes/definition/{scheme_id}')
 async def remove_annotation_scheme(annotation_scheme_id: str,
                                    permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
-    await delete_annotation_scheme(annotation_scheme_id=annotation_scheme_id, db_engine=db_engine)
+    await delete_annotation_scheme(annotation_scheme_id=annotation_scheme_id, db_engine=db_engine, use_commit=True)
 
 
 @router.get('/schemes/list/{project_id}', response_model=list[AnnotationSchemeModel])
@@ -272,7 +272,7 @@ async def put_assignment_scope(assignment_scope: AssignmentScopeModel,
 async def remove_assignment_scope(assignment_scope_id: str,
                                   permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
     try:
-        await delete_assignment_scope(assignment_scope_id=assignment_scope_id, db_engine=db_engine)
+        await delete_assignment_scope(assignment_scope_id=assignment_scope_id, db_engine=db_engine, use_commit=True)
     except ValueError as e:
         raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST,
                             detail=str(e))
@@ -409,7 +409,7 @@ async def make_assignments(payload: MakeAssignmentsRequestModel,
                             detail=f'Method "{payload.config.config_type}" is unknown.')
 
     if payload.save:
-        await store_assignments(assignments=assignments, db_engine=db_engine)
+        await store_assignments(assignments=assignments, db_engine=db_engine, use_commit=True)
 
     return assignments
 
@@ -441,6 +441,7 @@ async def clear_empty_assignments(scope_id: str,
                 WHERE cnt = 0
         );''')
         await session.execute(stmt, {'scope_id': scope_id})
+        await session.commit()
 
 
 class AssignmentEditInfo(BaseModel):
@@ -593,7 +594,7 @@ async def save_resolved_annotations(settings: BotMetaResolveBase,
                                     assignment_scope_id: str,
                                     annotation_scheme_id: str,
                                     permissions=Depends(UserPermissionChecker('annotations_edit'))):
-    meta_id = await store_resolved_bot_annotations(db_engine=db_engine,
+    meta_id = await store_resolved_bot_annotations(db_engine=db_engine, use_commit=True,
                                                    project_id=permissions.permissions.project_id,
                                                    assignment_scope_id=assignment_scope_id,
                                                    annotation_scheme_id=annotation_scheme_id,
@@ -612,7 +613,7 @@ async def update_resolved_annotations(bot_annotation_metadata_id: str,
                                       permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
     # TODO: allow update of filters and settings?
     await update_resolved_bot_annotations(bot_annotation_metadata_id=bot_annotation_metadata_id,
-                                          name=name, matrix=matrix, db_engine=db_engine)
+                                          name=name, matrix=matrix, db_engine=db_engine, use_commit=True)
 
 
 @router.get('/config/resolved-list/', response_model=list[BotAnnotationMetaDataBaseModel])
@@ -649,6 +650,7 @@ async def delete_saved_resolved_annotations(bot_annotation_metadata_id: str,
             .scalars().one_or_none()
         if meta is not None:
             await session.delete(meta)
+            await session.commit()
         # TODO: do we need to commit?
         # TODO: ensure bot_annotations are deleted via cascade
 

server/api/routes/evaluation.py

@@ -140,7 +140,7 @@ async def update_tracker(tracker_id: str,
 
         # Update labels
         tracker.labels = batched_sequence
-        await session.flush()
+        await session.commit()
 
         # We are not handing over the existing tracker ORM, because the session is not persistent
         background_tasks.add_task(bg_populate_tracker, tracker_id, tracker.batch_size, diff)
@@ -186,6 +186,7 @@ async def bg_populate_tracker(tracker_id: str, batch_size: int | None = None, la
                 tracker.buscar = tracker.buscar + [(x, y)]
                 # save after each step, so the user can refresh the page and get data as it becomes available
                 await session.flush()
+        await session.commit()
 
 
 @router.get('/quality/load/{assignment_scope_id}', response_model=list[AnnotationQualityModel])

server/api/routes/imports.py

@@ -49,7 +49,7 @@ async def put_import_details(import_details: ImportModel,
                              permissions: UserPermissions = Depends(UserPermissionChecker('imports_edit'))) -> str:
     if str(import_details.project_id) == str(permissions.permissions.project_id):
         logger.debug(import_details)
-        key = await upsert_import(import_model=import_details, engine=db_engine)
+        key = await upsert_import(import_model=import_details, engine=db_engine, use_commit=True)
         return str(key)
 
     raise InsufficientPermissions('You do not have permission to edit this data import.')
@@ -75,7 +75,7 @@ async def delete_import_details(import_id: str,
 
     # First, make sure the user trying to delete this import is actually authorised to delete this specific import
     if import_details is not None and str(import_details.project_id) == str(permissions.permissions.project_id):
-        await delete_import(import_id=import_id, engine=db_engine)
+        await delete_import(import_id=import_id, engine=db_engine, use_commit=True)
         return str(import_id)
 
     raise InsufficientPermissions('You do not have permission to delete this data import.')

server/api/routes/mailing.py

@@ -37,19 +37,20 @@ async def reset_password(username: str,
         await auth_helper.clear_tokens_by_user(username=username)
         # Create new token
         token = await auth_helper.refresh_or_create_token(username=username,
-                                                          token_lifetime_minutes=3 * 60)
+                                                          token_lifetime_minutes=24 * 60)
         try:
             background_tasks.add_task(
                 send_message,
                 sender=None,
                 recipients=[user.email],
+                bcc=[],
                 subject='[NACSOS] Reset password',
                 message=f'Dear {user.full_name},\n'
                         f'You are receiving this message because you or someone else '
                         f'tried to reset your password.\n'
                         f'We closed all your active sessions, so you will have to log in again.\n'
                         f'\n'
-                        f'You can use the following link within the next 3h to reset your password:\n'
+                        f'You can use the following link within the next 24h to reset your password:\n'
                         f'{settings.SERVER.WEB_URL}/#/password-reset/{token.token_id}\n'
                         f'\n'
                         f'Sincerely,\n'
@@ -73,11 +74,12 @@ async def welcome_mail(username: str,
     if user is not None and user.email is not None:
         # Create new token
         token = await auth_helper.refresh_or_create_token(username=username,
-                                                          token_lifetime_minutes=3 * 60)
+                                                          token_lifetime_minutes=24 * 60)
         background_tasks.add_task(
             send_message,
             sender=None,
             recipients=[user.email],
+            bcc=[],
             subject='[NACSOS] Welcome to the platform',
             message=f'Dear {user.full_name},\n'
                     f'I created an account on our scoping platform for you.\n '
@@ -88,7 +90,7 @@ async def welcome_mail(username: str,
                     f'You can change your password after logging in by opening the user menu at '
                     f'the top right and clicking "edit profile".\n'
                     f'\n'
-                    f'Alternatively, you can use the following link within the next 3h to reset your password:\n'
+                    f'Alternatively, you can use the following link within the next 24h to reset your password:\n'
                     f'{settings.SERVER.WEB_URL}/#/password-reset/{token.token_id}\n'
                     f'\n'
                     f'We are working on expanding the documentation for the platform here:\n'
@@ -138,6 +140,7 @@ async def remind_users_assigment(assignment_scope_id: str,
                     send_message,
                     sender=None,
                     recipients=[res['email']],
+                    bcc=[],
                     subject='[NACSOS] Assignments waiting for you',
                     message=f'Dear {res["full_name"]},\n'
                             f'In the project "{info["project_name"]}", in the scope "{info["scope_name"]}", '
@@ -157,3 +160,49 @@ async def remind_users_assigment(assignment_scope_id: str,
             else:
                 logger.debug(f'Not reminding {res}')
         return reminded_users
+
+
+@router.post('/news')
+async def news_mail(subject: str,
+                    body: str,
+                    background_tasks: BackgroundTasks,
+                    superuser: UserModel = Depends(get_current_active_superuser)) -> list[str]:
+    reminded_users: list[str] = []
+
+    session: AsyncSession
+    async with db_engine.session() as session:
+        users = (await session.execute(select(User.email, User.full_name, User.username)
+                                       .where(User.setting_newsletter is True,  # type: ignore[arg-type]
+                                              User.is_active is True))).mappings().all()  # type: ignore[arg-type]
+
+        for user in users:
+            try:
+                logger.debug(f'Trying to remind {user["username"]}')
+                background_tasks.add_task(
+                    send_message,
+                    sender=None,
+                    recipients=[user['email']],
+                    bcc=[],
+                    subject=f'[NACSOS] -NEWS- | {subject}',
+                    message=f'Dear {user["full_name"]},\n'
+                            f'\n'
+                            f'The following message was sent to you by the platform:\n'
+                            f'\n'
+                            f'------------------------------------------------------\n'
+                            f'{body}\n'
+                            f'------------------------------------------------------\n'
+                            f' / end of message\n'
+                            f'\n'
+                            f'If you do not want to receive any more emails like this, please log in and '
+                            f'edit your user profile.\n'
+                            f'We created a guide in the documentation: https://apsis.mcc-berlin.net/nacsos-docs/user/issues/\n'
+                            f'There is also a high-level changelog: https://apsis.mcc-berlin.net/nacsos-docs/news/\n'
+                            f'\n'
+                            f'Sincerely,\n'
+                            f'The Platform'
+                )
+                reminded_users.append(user['username'])
+            except Exception as e:
+                logger.exception(e)
+
+        return reminded_users

server/api/routes/ping.py

 from fastapi import APIRouter
 from fastapi.responses import PlainTextResponse
 
+from sqlalchemy import select, func as F
+from nacsos_data.db.schemas.projects import Project
+
 from server.pipelines import tasks
 from server.util.logging import get_logger
 from server.util.security import InsufficientPermissions
+from server.data import db_engine
 
 logger = get_logger('nacsos.api.route.ping')
 router = APIRouter()
@@ -57,6 +61,15 @@ async def perm():
     raise InsufficientPermissions('You do not have permission to edit this data import.')
 
 
+@router.get('/database')
+async def db_test():
+    async with db_engine.engine.connect() as session:
+        rslt = (await session.execute(select(F.count(Project.project_id)))).scalar()
+        logger.debug(f'There are {rslt:,} projects on the platform')
+        await session.close()
+        return rslt
+
+
 @router.post('/{name}', response_class=PlainTextResponse)
 async def _ping(name: str) -> str:
     return f'Hello {name}'

server/api/routes/project/__init__.py

@@ -32,7 +32,7 @@ async def get_project(permission=Depends(UserPermissionChecker())) -> ProjectMod
 async def save_project(project_info: ProjectModel,
                        permission=Depends(UserPermissionChecker('owner'))) -> str:
     pkey = await upsert_orm(upsert_model=project_info, Schema=Project, primary_key='project_id',
-                            skip_update=['project_id'], db_engine=db_engine)
+                            skip_update=['project_id'], db_engine=db_engine, use_commit=True)
     return str(pkey)
 
 

server/api/routes/search.py

@@ -3,9 +3,9 @@ from pydantic import BaseModel
 from fastapi import APIRouter, Depends
 import sqlalchemy.sql.functions as func
 from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.asyncio import AsyncSession  # noqa: F401
 
-from nacsos_data.db.engine import ensure_session
+from nacsos_data.db.engine import ensure_session, DBSession
 from nacsos_data.db.schemas import Project, ItemType
 from nacsos_data.util.nql import NQLQuery, NQLFilter
 from nacsos_data.util.academic.readers.openalex import query_async, SearchResult
@@ -90,7 +90,7 @@ class QueryResult(BaseModel):
 
 
 @ensure_session
-async def _get_query(session: AsyncSession, query: NQLFilter, project_id: str) -> NQLQuery:
+async def _get_query(session: DBSession, query: NQLFilter, project_id: str) -> NQLQuery:
     project_type: ItemType | None = (
         await session.scalar(select(Project.type).where(Project.project_id == project_id)))
 

server/pipelines/actor.py

@@ -79,7 +79,7 @@ class NacsosActor(Actor[P, R]):
                         params=params, fingerprint=fingerprint, comment=comment, message_id=message.message_id,
                         rec_expunge=self.rec_expunge, status=TaskStatus.PENDING)
             session.add(task)
-            session.flush()
+            session.commit()
             self.logger.info('Wrote task info to database.')
 
         return message

server/util/email.py

@@ -39,6 +39,7 @@ class EmailNotSentError(Exception):
 
 
 def construct_email(recipients: list[str],
+                    bcc: list[str],
                     subject: str,
                     message: str,
                     sender: str | None = None) -> EmailMessage:
@@ -50,21 +51,24 @@ def construct_email(recipients: list[str],
     email['Subject'] = subject
     email['From'] = sender  # type: ignore[assignment]
     email['To'] = ', '.join(recipients)
+    email['Bcc'] = ', '.join(bcc)
     return email
 
 
 async def send_message(recipients: list[str],
+                       bcc: list[str],
                        subject: str,
                        message: str,
                        sender: str | None = None) -> bool:
-    email = construct_email(sender=sender, recipients=recipients, subject=subject, message=message)
+    email = construct_email(sender=sender, recipients=recipients, bcc=bcc, subject=subject, message=message)
     return await send_email(email)
 
 
 async def send_email(email: EmailMessage) -> bool:
     if not settings.EMAIL.ENABLED:
         raise EmailNotSentError(f'Mailing system inactive, '
-                                f'email with subject "{email["Subject"]}" not sent to {email["To"]}')
+                                f'email with subject "{email["Subject"]}" '
+                                f'not sent to {email["To"]} (Bcc: {email["Bcc"]})')
 
     if email['From'] is None:
         del email['From']
@@ -89,18 +93,20 @@ async def send_email(email: EmailMessage) -> bool:
     except (SMTPRecipientsRefused, SMTPResponseException, ValueError, SMTPException, SMTPTimeoutError,
             SMTPAuthenticationError, SMTPNotSupported, SMTPConnectTimeoutError, SMTPConnectError,
             SMTPConnectResponseError, SMTPServerDisconnected, SMTPHeloError, SMTPSenderRefused) as e:
-        logger.warning(f'Failed sending email to {email["To"]} with subject "{email["Subject"]}"')
+        logger.warning(f'Failed sending email to {email["To"]} (Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
         logger.error(e)
         await client.quit()
 
-        raise EmailNotSentError(f'Email with subject "{email["Subject"]}" not sent to {email["To"]} because of "{e}"')
+        raise EmailNotSentError(f'Email with subject "{email["Subject"]}" '
+                                f'not sent to {email["To"]} (Bcc: {email["Bcc"]}) because of "{e}"')
 
 
 def send_message_sync(recipients: list[str],
+                      bcc: list[str],
                       subject: str,
                       message: str,
                       sender: str | None = None) -> bool:
-    email = construct_email(sender=sender, recipients=recipients, subject=subject, message=message)
+    email = construct_email(sender=sender, recipients=recipients, bcc=bcc, subject=subject, message=message)
     return send_email_sync(email)
 
 
@@ -130,15 +136,18 @@ def send_email_sync(email: EmailMessage) -> bool:
                 smtp.login(user=user, password=password)
 
             smtp.connect()
-            logger.info(f'Trying to send email to {email["To"]} with subject "{email["Subject"]}"')
+            logger.info(f'Trying to send email to {email["To"]} '
+                        f'(Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
             status = smtp.send_message(email)
             logger.debug(status)
-            logger.info(f'Successfully sent email to {email["To"]} with subject "{email["Subject"]}"')
+            logger.info(f'Successfully sent email to {email["To"]} '
+                        f'(Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
 
             return True
 
     except (SMTPHeloErrorOrig, SMTPRecipientsRefusedOrig, SMTPSenderRefusedOrig,
             SMTPDataError, SMTPNotSupportedError, SMTPExceptionOrig) as e:
-        logger.warning(f'Failed sending email to {email["To"]} with subject "{email["Subject"]}"')
+        logger.warning(f'Failed sending email to {email["To"]} (Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
         logger.error(e)
-        raise EmailNotSentError(f'Email with subject "{email["Subject"]}" not sent to {email["To"]} because of "{e}"')
+        raise EmailNotSentError(f'Email with subject "{email["Subject"]}" '
+                                f'not sent to {email["To"]} (Bcc: {email["Bcc"]}) because of "{e}"')