From 7751e1a79dfcb7e2c0670bc008a1299dbccc5f55 Mon Sep 17 00:00:00 2001
From: Tim Repke <repke@mcc-berlin.net>
Date: Mon, 2 Sep 2024 21:18:35 +0200
Subject: [PATCH] session updates and news endoint
---
.gitignore | 3 +-
README.md | 13 ++++++
requirements.txt | 2 +-
server/api/routes/annotations.py | 12 +++---
server/api/routes/evaluation.py | 3 +-
server/api/routes/imports.py | 4 +-
server/api/routes/mailing.py | 57 +++++++++++++++++++++++++--
server/api/routes/project/__init__.py | 2 +-
server/api/routes/search.py | 6 +--
server/pipelines/actor.py | 2 +-
server/util/email.py | 27 ++++++++-----
11 files changed, 103 insertions(+), 28 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3d8aad4..371dd99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,5 @@ __pycache__
server.md
dumps/
scratch/
-.tasks/
\ No newline at end of file
+.tasks/
+volumes
\ No newline at end of file
diff --git a/README.md b/README.md
index 62f2d55..8d8e9f5 100644
--- a/README.md
+++ b/README.md
@@ -84,3 +84,16 @@ FinalKillSignal=SIGKILL
[Install]
WantedBy=multi-user.target
```
+
+## Testing database things in the console
+```
+from sqlalchemy import select
+from nacsos_data.db import get_engine
+from nacsos_data.db.schemas.users import User
+engine = get_engine('config/remote.env')
+with engine.session() as session:
+ users = session.execute(select(User.email, User.full_name, User.username)
+ .where(User.setting_newsletter == False,
+ User.is_active == True)).mappings().all()
+ print(users[0])
+```
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0fa3349..cb448e2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
-r requirements_base.txt
-nacsos_data[utils,scripts] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.16.5
\ No newline at end of file
+nacsos_data[utils,scripts] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.17.0
\ No newline at end of file
diff --git a/server/api/routes/annotations.py b/server/api/routes/annotations.py
index e71d3e9..a242d71 100644
--- a/server/api/routes/annotations.py
+++ b/server/api/routes/annotations.py
@@ -137,7 +137,7 @@ async def put_annotation_scheme(annotation_scheme: AnnotationSchemeModel,
@router.delete('/schemes/definition/{scheme_id}')
async def remove_annotation_scheme(annotation_scheme_id: str,
permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
- await delete_annotation_scheme(annotation_scheme_id=annotation_scheme_id, db_engine=db_engine)
+ await delete_annotation_scheme(annotation_scheme_id=annotation_scheme_id, db_engine=db_engine, use_commit=True)
@router.get('/schemes/list/{project_id}', response_model=list[AnnotationSchemeModel])
@@ -272,7 +272,7 @@ async def put_assignment_scope(assignment_scope: AssignmentScopeModel,
async def remove_assignment_scope(assignment_scope_id: str,
permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
try:
- await delete_assignment_scope(assignment_scope_id=assignment_scope_id, db_engine=db_engine)
+ await delete_assignment_scope(assignment_scope_id=assignment_scope_id, db_engine=db_engine, use_commit=True)
except ValueError as e:
raise HTTPException(status_code=http_status.HTTP_400_BAD_REQUEST,
detail=str(e))
@@ -409,7 +409,7 @@ async def make_assignments(payload: MakeAssignmentsRequestModel,
detail=f'Method "{payload.config.config_type}" is unknown.')
if payload.save:
- await store_assignments(assignments=assignments, db_engine=db_engine)
+ await store_assignments(assignments=assignments, db_engine=db_engine, use_commit=True)
return assignments
@@ -441,6 +441,7 @@ async def clear_empty_assignments(scope_id: str,
WHERE cnt = 0
);''')
await session.execute(stmt, {'scope_id': scope_id})
+ await session.commit()
class AssignmentEditInfo(BaseModel):
@@ -593,7 +594,7 @@ async def save_resolved_annotations(settings: BotMetaResolveBase,
assignment_scope_id: str,
annotation_scheme_id: str,
permissions=Depends(UserPermissionChecker('annotations_edit'))):
- meta_id = await store_resolved_bot_annotations(db_engine=db_engine,
+ meta_id = await store_resolved_bot_annotations(db_engine=db_engine, use_commit=True,
project_id=permissions.permissions.project_id,
assignment_scope_id=assignment_scope_id,
annotation_scheme_id=annotation_scheme_id,
@@ -612,7 +613,7 @@ async def update_resolved_annotations(bot_annotation_metadata_id: str,
permissions=Depends(UserPermissionChecker('annotations_edit'))) -> None:
# TODO: allow update of filters and settings?
await update_resolved_bot_annotations(bot_annotation_metadata_id=bot_annotation_metadata_id,
- name=name, matrix=matrix, db_engine=db_engine)
+ name=name, matrix=matrix, db_engine=db_engine, use_commit=True)
@router.get('/config/resolved-list/', response_model=list[BotAnnotationMetaDataBaseModel])
@@ -649,6 +650,7 @@ async def delete_saved_resolved_annotations(bot_annotation_metadata_id: str,
.scalars().one_or_none()
if meta is not None:
await session.delete(meta)
+ await session.commit()
# TODO: do we need to commit?
# TODO: ensure bot_annotations are deleted via cascade
diff --git a/server/api/routes/evaluation.py b/server/api/routes/evaluation.py
index 2983433..d23718d 100644
--- a/server/api/routes/evaluation.py
+++ b/server/api/routes/evaluation.py
@@ -140,7 +140,7 @@ async def update_tracker(tracker_id: str,
# Update labels
tracker.labels = batched_sequence
- await session.flush()
+ await session.commit()
# We are not handing over the existing tracker ORM, because the session is not persistent
background_tasks.add_task(bg_populate_tracker, tracker_id, tracker.batch_size, diff)
@@ -186,6 +186,7 @@ async def bg_populate_tracker(tracker_id: str, batch_size: int | None = None, la
tracker.buscar = tracker.buscar + [(x, y)]
# save after each step, so the user can refresh the page and get data as it becomes available
await session.flush()
+ await session.commit()
@router.get('/quality/load/{assignment_scope_id}', response_model=list[AnnotationQualityModel])
diff --git a/server/api/routes/imports.py b/server/api/routes/imports.py
index 244de7c..45d6b05 100644
--- a/server/api/routes/imports.py
+++ b/server/api/routes/imports.py
@@ -49,7 +49,7 @@ async def put_import_details(import_details: ImportModel,
permissions: UserPermissions = Depends(UserPermissionChecker('imports_edit'))) -> str:
if str(import_details.project_id) == str(permissions.permissions.project_id):
logger.debug(import_details)
- key = await upsert_import(import_model=import_details, engine=db_engine)
+ key = await upsert_import(import_model=import_details, engine=db_engine, use_commit=True)
return str(key)
raise InsufficientPermissions('You do not have permission to edit this data import.')
@@ -75,7 +75,7 @@ async def delete_import_details(import_id: str,
# First, make sure the user trying to delete this import is actually authorised to delete this specific import
if import_details is not None and str(import_details.project_id) == str(permissions.permissions.project_id):
- await delete_import(import_id=import_id, engine=db_engine)
+ await delete_import(import_id=import_id, engine=db_engine, use_commit=True)
return str(import_id)
raise InsufficientPermissions('You do not have permission to delete this data import.')
diff --git a/server/api/routes/mailing.py b/server/api/routes/mailing.py
index 3255224..db33ed6 100644
--- a/server/api/routes/mailing.py
+++ b/server/api/routes/mailing.py
@@ -37,19 +37,20 @@ async def reset_password(username: str,
await auth_helper.clear_tokens_by_user(username=username)
# Create new token
token = await auth_helper.refresh_or_create_token(username=username,
- token_lifetime_minutes=3 * 60)
+ token_lifetime_minutes=24 * 60)
try:
background_tasks.add_task(
send_message,
sender=None,
recipients=[user.email],
+ bcc=[],
subject='[NACSOS] Reset password',
message=f'Dear {user.full_name},\n'
f'You are receiving this message because you or someone else '
f'tried to reset your password.\n'
f'We closed all your active sessions, so you will have to log in again.\n'
f'\n'
- f'You can use the following link within the next 3h to reset your password:\n'
+ f'You can use the following link within the next 24h to reset your password:\n'
f'{settings.SERVER.WEB_URL}/#/password-reset/{token.token_id}\n'
f'\n'
f'Sincerely,\n'
@@ -73,11 +74,12 @@ async def welcome_mail(username: str,
if user is not None and user.email is not None:
# Create new token
token = await auth_helper.refresh_or_create_token(username=username,
- token_lifetime_minutes=3 * 60)
+ token_lifetime_minutes=24 * 60)
background_tasks.add_task(
send_message,
sender=None,
recipients=[user.email],
+ bcc=[],
subject='[NACSOS] Welcome to the platform',
message=f'Dear {user.full_name},\n'
f'I created an account on our scoping platform for you.\n '
@@ -88,7 +90,7 @@ async def welcome_mail(username: str,
f'You can change your password after logging in by opening the user menu at '
f'the top right and clicking "edit profile".\n'
f'\n'
- f'Alternatively, you can use the following link within the next 3h to reset your password:\n'
+ f'Alternatively, you can use the following link within the next 24h to reset your password:\n'
f'{settings.SERVER.WEB_URL}/#/password-reset/{token.token_id}\n'
f'\n'
f'We are working on expanding the documentation for the platform here:\n'
@@ -138,6 +140,7 @@ async def remind_users_assigment(assignment_scope_id: str,
send_message,
sender=None,
recipients=[res['email']],
+ bcc=[],
subject='[NACSOS] Assignments waiting for you',
message=f'Dear {res["full_name"]},\n'
f'In the project "{info["project_name"]}", in the scope "{info["scope_name"]}", '
@@ -157,3 +160,49 @@ async def remind_users_assigment(assignment_scope_id: str,
else:
logger.debug(f'Not reminding {res}')
return reminded_users
+
+
+@router.post('/news')
+async def news_mail(subject: str,
+ body: str,
+ background_tasks: BackgroundTasks,
+ superuser: UserModel = Depends(get_current_active_superuser)) -> list[str]:
+ reminded_users: list[str] = []
+
+ session: AsyncSession
+ async with db_engine.session() as session:
+ users = (await session.execute(select(User.email, User.full_name, User.username)
+ .where(User.setting_newsletter is True, # type: ignore[arg-type]
+ User.is_active is True))).mappings().all() # type: ignore[arg-type]
+
+ for user in users:
+ try:
+ logger.debug(f'Trying to remind {user["username"]}')
+ background_tasks.add_task(
+ send_message,
+ sender=None,
+ recipients=[user['email']],
+ bcc=[],
+ subject=f'[NACSOS] -NEWS- | {subject}',
+ message=f'Dear {user["full_name"]},\n'
+ f'\n'
+ f'The following message was sent to you by the platform:\n'
+ f'\n'
+ f'------------------------------------------------------\n'
+ f'{body}\n'
+ f'------------------------------------------------------\n'
+ f' / end of message\n'
+ f'\n'
+ f'If you do not want to receive any more emails like this, please log in and '
+ f'edit your user profile.\n'
+ f'We created a guide in the documentation: https://apsis.mcc-berlin.net/nacsos-docs/user/issues/\n'
+ f'There is also a high-level changelog: https://apsis.mcc-berlin.net/nacsos-docs/news/\n'
+ f'\n'
+ f'Sincerely,\n'
+ f'The Platform'
+ )
+ reminded_users.append(user['username'])
+ except Exception as e:
+ logger.exception(e)
+
+ return reminded_users
diff --git a/server/api/routes/project/__init__.py b/server/api/routes/project/__init__.py
index fa9816b..a878017 100644
--- a/server/api/routes/project/__init__.py
+++ b/server/api/routes/project/__init__.py
@@ -32,7 +32,7 @@ async def get_project(permission=Depends(UserPermissionChecker())) -> ProjectMod
async def save_project(project_info: ProjectModel,
permission=Depends(UserPermissionChecker('owner'))) -> str:
pkey = await upsert_orm(upsert_model=project_info, Schema=Project, primary_key='project_id',
- skip_update=['project_id'], db_engine=db_engine)
+ skip_update=['project_id'], db_engine=db_engine, use_commit=True)
return str(pkey)
diff --git a/server/api/routes/search.py b/server/api/routes/search.py
index 8be3660..0b49db0 100644
--- a/server/api/routes/search.py
+++ b/server/api/routes/search.py
@@ -3,9 +3,9 @@ from pydantic import BaseModel
from fastapi import APIRouter, Depends
import sqlalchemy.sql.functions as func
from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.asyncio import AsyncSession # noqa: F401
-from nacsos_data.db.engine import ensure_session
+from nacsos_data.db.engine import ensure_session, DBSession
from nacsos_data.db.schemas import Project, ItemType
from nacsos_data.util.nql import NQLQuery, NQLFilter
from nacsos_data.util.academic.readers.openalex import query_async, SearchResult
@@ -90,7 +90,7 @@ class QueryResult(BaseModel):
@ensure_session
-async def _get_query(session: AsyncSession, query: NQLFilter, project_id: str) -> NQLQuery:
+async def _get_query(session: DBSession, query: NQLFilter, project_id: str) -> NQLQuery:
project_type: ItemType | None = (
await session.scalar(select(Project.type).where(Project.project_id == project_id)))
diff --git a/server/pipelines/actor.py b/server/pipelines/actor.py
index 9e31441..1d4a569 100644
--- a/server/pipelines/actor.py
+++ b/server/pipelines/actor.py
@@ -79,7 +79,7 @@ class NacsosActor(Actor[P, R]):
params=params, fingerprint=fingerprint, comment=comment, message_id=message.message_id,
rec_expunge=self.rec_expunge, status=TaskStatus.PENDING)
session.add(task)
- session.flush()
+ session.commit()
self.logger.info('Wrote task info to database.')
return message
diff --git a/server/util/email.py b/server/util/email.py
index 11eeebd..c422c40 100644
--- a/server/util/email.py
+++ b/server/util/email.py
@@ -39,6 +39,7 @@ class EmailNotSentError(Exception):
def construct_email(recipients: list[str],
+ bcc: list[str],
subject: str,
message: str,
sender: str | None = None) -> EmailMessage:
@@ -50,21 +51,24 @@ def construct_email(recipients: list[str],
email['Subject'] = subject
email['From'] = sender # type: ignore[assignment]
email['To'] = ', '.join(recipients)
+ email['Bcc'] = ', '.join(bcc)
return email
async def send_message(recipients: list[str],
+ bcc: list[str],
subject: str,
message: str,
sender: str | None = None) -> bool:
- email = construct_email(sender=sender, recipients=recipients, subject=subject, message=message)
+ email = construct_email(sender=sender, recipients=recipients, bcc=bcc, subject=subject, message=message)
return await send_email(email)
async def send_email(email: EmailMessage) -> bool:
if not settings.EMAIL.ENABLED:
raise EmailNotSentError(f'Mailing system inactive, '
- f'email with subject "{email["Subject"]}" not sent to {email["To"]}')
+ f'email with subject "{email["Subject"]}" '
+ f'not sent to {email["To"]} (Bcc: {email["Bcc"]})')
if email['From'] is None:
del email['From']
@@ -89,18 +93,20 @@ async def send_email(email: EmailMessage) -> bool:
except (SMTPRecipientsRefused, SMTPResponseException, ValueError, SMTPException, SMTPTimeoutError,
SMTPAuthenticationError, SMTPNotSupported, SMTPConnectTimeoutError, SMTPConnectError,
SMTPConnectResponseError, SMTPServerDisconnected, SMTPHeloError, SMTPSenderRefused) as e:
- logger.warning(f'Failed sending email to {email["To"]} with subject "{email["Subject"]}"')
+ logger.warning(f'Failed sending email to {email["To"]} (Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
logger.error(e)
await client.quit()
- raise EmailNotSentError(f'Email with subject "{email["Subject"]}" not sent to {email["To"]} because of "{e}"')
+ raise EmailNotSentError(f'Email with subject "{email["Subject"]}" '
+ f'not sent to {email["To"]} (Bcc: {email["Bcc"]}) because of "{e}"')
def send_message_sync(recipients: list[str],
+ bcc: list[str],
subject: str,
message: str,
sender: str | None = None) -> bool:
- email = construct_email(sender=sender, recipients=recipients, subject=subject, message=message)
+ email = construct_email(sender=sender, recipients=recipients, bcc=bcc, subject=subject, message=message)
return send_email_sync(email)
@@ -130,15 +136,18 @@ def send_email_sync(email: EmailMessage) -> bool:
smtp.login(user=user, password=password)
smtp.connect()
- logger.info(f'Trying to send email to {email["To"]} with subject "{email["Subject"]}"')
+ logger.info(f'Trying to send email to {email["To"]} '
+ f'(Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
status = smtp.send_message(email)
logger.debug(status)
- logger.info(f'Successfully sent email to {email["To"]} with subject "{email["Subject"]}"')
+ logger.info(f'Successfully sent email to {email["To"]} '
+ f'(Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
return True
except (SMTPHeloErrorOrig, SMTPRecipientsRefusedOrig, SMTPSenderRefusedOrig,
SMTPDataError, SMTPNotSupportedError, SMTPExceptionOrig) as e:
- logger.warning(f'Failed sending email to {email["To"]} with subject "{email["Subject"]}"')
+ logger.warning(f'Failed sending email to {email["To"]} (Bcc: {email["Bcc"]}) with subject "{email["Subject"]}"')
logger.error(e)
- raise EmailNotSentError(f'Email with subject "{email["Subject"]}" not sent to {email["To"]} because of "{e}"')
+ raise EmailNotSentError(f'Email with subject "{email["Subject"]}" '
+ f'not sent to {email["To"]} (Bcc: {email["Bcc"]}) because of "{e}"')
--
GitLab