add quality endpoints

requirements.txt

@@ -7,4 +7,4 @@ passlib[bcrypt]==1.7.4
 pymitter==0.4.2
 uvicorn==0.23.2
 python-multipart==0.0.6
-nacsos_data[scripts,server,utils] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.11.2
+nacsos_data[scripts,server,utils] @ git+ssh://git@gitlab.pik-potsdam.de/mcc-apsis/nacsos/nacsos-data.git@v0.12.0

requirements_dev.txt

 flake8==6.1.0
-tox==4.6.4
-pytest==7.4.2
+tox==4.11.3
+pytest==7.4.3
 pytest-cov==4.1.0
-mypy==1.5.1
+mypy==1.6.1
 types-toml==0.10.8.7
-types-PyYAML==6.0.12.11
+types-PyYAML==6.0.12.12
\ No newline at end of file

server/api/routes/evaluation.py

@@ -3,17 +3,20 @@ from typing import Literal
 
 from fastapi import APIRouter, BackgroundTasks, Depends
 from nacsos_data.db.crud import upsert_orm
-from nacsos_data.db.schemas import AnnotationTracker, AssignmentScope, AnnotationScheme, BotAnnotationMetaData
+from nacsos_data.db.schemas import AnnotationTracker, AssignmentScope, AnnotationScheme, BotAnnotationMetaData, \
+    AnnotationQuality
+from nacsos_data.models.annotation_quality import AnnotationQualityModel
 from nacsos_data.models.annotation_tracker import AnnotationTrackerModel, DehydratedAnnotationTracker
 from nacsos_data.util.annotations.evaluation import get_new_label_batches
 from nacsos_data.util.annotations.evaluation.buscar import (
     calculate_h0s_for_batches,
     compute_recall,
     calculate_h0s)
+from nacsos_data.util.annotations.evaluation.irr import compute_irr_scores
 from nacsos_data.util.annotations.label_transform import annotations_to_sequence, get_annotations
 from nacsos_data.util.auth import UserPermissions
 from pydantic import BaseModel
-from sqlalchemy import select, String, literal
+from sqlalchemy import select, String, literal, delete
 
 from server.data import db_engine
 from server.api.errors import DataNotFoundWarning
@@ -172,3 +175,36 @@ async def bg_populate_tracker(tracker_id: str, batch_size: int | None = None, la
                 tracker.buscar = tracker.buscar + [(x, y)]
                 # save after each step, so the user can refresh the page and get data as it becomes available
                 await session.flush()
+
+
+@router.get('/quality/load/{assignment_scope_id}', response_model=list[AnnotationQualityModel])
+async def get_irr(assignment_scope_id: str,
+                  permissions: UserPermissions = Depends(UserPermissionChecker('annotations_read'))) \
+        -> list[AnnotationQualityModel]:
+    async with db_engine.session() as session:  # type: AsyncSession
+        results = (
+            await session.execute(select(AnnotationQuality)
+                                  .where(AnnotationQuality.assignment_scope_id == assignment_scope_id))
+        ).scalars().all()
+
+        return [AnnotationQualityModel(**r.__dict__) for r in results]
+
+
+@router.get('/quality/compute/{assignment_scope_id}', response_model=list[AnnotationQualityModel])
+async def recompute_irr(assignment_scope_id: str,
+                        permissions: UserPermissions = Depends(UserPermissionChecker('annotations_read'))) \
+        -> list[AnnotationQualityModel]:
+    async with db_engine.session() as session:  # type: AsyncSession
+        # Delete existing metrics
+        await session.execute(delete(AnnotationQuality)
+                              .where(AnnotationQuality.assignment_scope_id == assignment_scope_id))
+        # Compute new metrics
+        metrics = await compute_irr_scores(session=session,
+                                           assignment_scope_id=assignment_scope_id,
+                                           project_id=permissions.permissions.project_id)
+
+        metrics_orm = [AnnotationQuality(**metric.model_dump()) for metric in metrics]
+        session.add_all(metrics_orm)
+        await session.commit()
+
+    return await get_irr(assignment_scope_id, permissions)

server/api/routes/users.py

 from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, Query
+from sqlalchemy import select, asc
 
-from nacsos_data.models.users import UserModel, UserInDBModel, UserBaseModel
 from nacsos_data.util.auth import UserPermissions
-from nacsos_data.db.schemas import User
-from nacsos_data.db.crud.users import \
-    read_users, \
-    read_user_by_id, \
-    read_users_by_ids, \
-    create_or_update_user, get_password_hash
-from sqlalchemy import select
+from nacsos_data.models.users import UserModel, UserInDBModel, UserBaseModel
+from nacsos_data.db.schemas import User, AssignmentScope, AnnotationScheme, Assignment
+from nacsos_data.db.crud.users import (
+    read_users,
+    read_user_by_id,
+    read_users_by_ids,
+    create_or_update_user,
+    get_password_hash
+)
 
 from server.data import db_engine
 from server.api.errors import DataNotFoundWarning, UserNotFoundError, UserPermissionError
@@ -33,6 +35,26 @@ async def get_all_users(current_user: UserModel = Depends(get_current_active_use
     return result
 
 
+@router.get('/list/project/annotators/{project_id}', response_model=dict[str, UserBaseModel])
+async def get_project_annotator_users(project_id: str,
+                                      permissions: UserPermissions = Depends(UserPermissionChecker())) \
+        -> dict[str, UserBaseModel]:
+    session: AsyncSession
+    async with db_engine.session() as session:
+        stmt = (select(User)
+                .join(Assignment, User.user_id == Assignment.user_id)
+                .join(AssignmentScope, Assignment.assignment_scope_id == AssignmentScope.assignment_scope_id)
+                .join(AnnotationScheme, AssignmentScope.annotation_scheme_id == AnnotationScheme.annotation_scheme_id)
+                .where(AnnotationScheme.project_id == project_id)
+                .order_by(asc(User.username)))
+
+        result = (await session.scalars(stmt)).all()
+        return {
+            str(res.user_id): UserBaseModel(**res.__dict__)
+            for res in result
+        }
+
+
 @router.get('/list/project/{project_id}', response_model=list[UserBaseModel])
 async def get_project_users(project_id: str,
                             permissions: UserPermissions = Depends(UserPermissionChecker())) \