Merge branch 'import' into 'main'

new deduplication logic for imports

See merge request !98

server/api/routes/evaluation.py

@@ -116,7 +116,6 @@ async def save_tracker(tracker: AnnotationTrackerModel,
 @router.post('/tracking/refresh', response_model=AnnotationTrackerModel)
 async def update_tracker(tracker_id: str,
                          background_tasks: BackgroundTasks,
-                         batch_size: int | None = None,
                          reset: bool = False,
                          permissions: UserPermissions = Depends(UserPermissionChecker('annotations_edit'))) \
         -> AnnotationTrackerModel:
@@ -144,7 +143,7 @@ async def update_tracker(tracker_id: str,
         await session.flush()
 
         # We are not handing over the existing tracker ORM, because the session is not persistent
-        background_tasks.add_task(bg_populate_tracker, tracker_id, batch_size, diff)
+        background_tasks.add_task(bg_populate_tracker, tracker_id, tracker.batch_size, diff)
 
         return AnnotationTrackerModel.model_validate(tracker.__dict__)
 

server/api/routes/search.py

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from nacsos_data.db.engine import ensure_session
 from nacsos_data.db.schemas import Project, ItemType
 from nacsos_data.util.nql import NQLQuery, NQLFilter
-from nacsos_data.util.academic.openalex import query_async, SearchResult
+from nacsos_data.util.academic.readers.openalex import query_async, SearchResult
 from nacsos_data.models.items import AcademicItemModel, FullLexisNexisItemModel, GenericItemModel
 from nacsos_data.models.openalex.solr import SearchField, DefType, OpType
 

server/pipelines/tasks/imports.py

@@ -7,7 +7,7 @@ from nacsos_data.db import get_engine_async
 from nacsos_data.db.schemas import Import
 from nacsos_data.models.imports import ImportConfig, ImportModel
 from nacsos_data.util import ensure_values
-from nacsos_data.util.academic.importing import (
+from nacsos_data.util.academic.importer import (
     import_wos_files,
     import_openalex_files,
     import_academic_db,

server/util/config.py

@@ -154,7 +154,7 @@ class Settings(BaseSettings):
 
     EMAIL: EmailConfig
 
-    LOG_CONF_FILE: str = 'config/logging.conf'
+    LOG_CONF_FILE: str = 'config/logging.toml'
     LOGGING_CONF: dict[str, Any] | None = None
 
     @field_validator('LOGGING_CONF', mode='before')
@@ -165,6 +165,7 @@ class Settings(BaseSettings):
         if isinstance(v, dict):
             return v
         filename = info.data.get('LOG_CONF_FILE', None)
+
         if filename is not None:
             with open(filename, 'r') as f:
                 ret = toml.loads(f.read())