feat: unify scope settings and catalog table interactions

backend/app/catalog/research_metadata.py

@@ -30,7 +30,7 @@ def setting_rows(data):
     try:
         children = data["actions"]["POST"]["settings"]["children"]
 
-        def choices(key, instrument=None, region=None):
+        def choices(key, instrument=None, region=None, delay=None, universe=None):
             value = children[key]["choices"]
             if isinstance(value, dict) and "instrumentType" in value:
                 value = value["instrumentType"][instrument]
@@ -38,6 +38,14 @@ def setting_rows(data):
                 value = value[instrument]
             if isinstance(value, dict) and "region" in value:
                 value = value["region"][region]
+            for dimension, selected in (("delay", delay), ("universe", universe)):
+                if isinstance(value, dict) and dimension in value:
+                    branches = value[dimension]
+                    if not isinstance(branches, dict):
+                        raise ValueError()
+                    value = branches.get(str(selected), branches.get(selected))
+            if not isinstance(value, list):
+                raise ValueError()
             return [item["value"] for item in value]
 
         rows = []
@@ -48,12 +56,29 @@ def setting_rows(data):
                 for delay in choices("delay", instrument, region):
                     if type(delay) is not int or delay not in (0, 1):
                         continue
-                    for universe in choices("universe", instrument, region):
+                    for universe in choices("universe", instrument, region, delay):
                         neutralizations = (
-                            choices("neutralization", instrument, region)
+                            choices("neutralization", instrument, region, delay, universe)
                             if "neutralization" in children
                             else []
                         )
+                        fields = {}
+                        for key in (
+                            "decay", "truncation", "pasteurization", "unitHandling",
+                            "nanHandling", "language", "visualization", "maxTrade", "maxPosition",
+                        ):
+                            definition = children.get(key)
+                            if not isinstance(definition, dict):
+                                continue
+                            field = {}
+                            if "choices" in definition:
+                                field["choices"] = choices(key, instrument, region, delay, universe)
+                                if any(type(v) not in (str, int, float, bool) for v in field["choices"]):
+                                    raise ValueError()
+                            for source, target in (("min_value", "minimum"), ("max_value", "maximum")):
+                                if type(definition.get(source)) in (int, float):
+                                    field[target] = definition[source]
+                            fields[key] = field
                         rows.append(
                             {
                                 "instrument_type": instrument,
@@ -61,6 +86,7 @@ def setting_rows(data):
                                 "universe": universe,
                                 "delay": delay,
                                 "neutralizations": neutralizations,
+                                "fields": fields,
                             }
                         )
         if not rows:

backend/app/preparations/routes.py

@@ -1,6 +1,6 @@
 """Authenticated preparation and field-directory endpoints."""
 
-from typing import Annotated
+from typing import Annotated, Literal
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request
 from sqlalchemy import delete, select
@@ -55,9 +55,11 @@ async def preparations(
     scope_key: str | None = None,
     limit: int = Query(25, ge=1, le=100),
     offset: int = Query(0, ge=0),
+    sort: Literal["name", "created_at", "updated_at"] = "updated_at",
+    direction: Literal["asc", "desc"] = "desc",
 ):
     async with request.app.state.sessions() as db:
-        return await Preparations(db).list(q, scope_key, limit, offset)
+        return await Preparations(db).list(q, scope_key, limit, offset, sort, direction)
 
 
 @router.post("/data-preparations", status_code=201)

backend/app/preparations/service.py

@@ -138,9 +138,12 @@ class Preparations:
     async def online_fields(self, filters):
         text_filters = (filters.q, filters.dataset_id, filters.field_type)
         numeric_filters = (
-            filters.coverage_min, filters.coverage_max,
-            filters.user_count_min, filters.user_count_max,
-            filters.alpha_count_min, filters.alpha_count_max,
+            filters.coverage_min,
+            filters.coverage_max,
+            filters.user_count_min,
+            filters.user_count_max,
+            filters.alpha_count_min,
+            filters.alpha_count_max,
         )
         if not any(value and value.strip() for value in text_filters) and not any(
             value is not None for value in numeric_filters
@@ -304,7 +307,7 @@ class Preparations:
             )
         )
 
-    async def list(self, q="", scope_key=None, limit=25, offset=0):
+    async def list(self, q="", scope_key=None, limit=25, offset=0, sort="updated_at", direction="desc"):
         query = select(DataPreparation)
         if scope_key:
             query = query.where(DataPreparation.scope_key == scope_key)
@@ -316,9 +319,13 @@ class Preparations:
                 )
             )
         total = await self.db.scalar(select(func.count()).select_from(query.subquery()))
-        rows = await self.db.scalars(
-            query.order_by(DataPreparation.updated_at.desc(), DataPreparation.id).limit(limit).offset(offset)
-        )
+        column = {
+            "name": DataPreparation.name,
+            "created_at": DataPreparation.created_at,
+            "updated_at": DataPreparation.updated_at,
+        }[sort]
+        order = column.asc() if direction == "asc" else column.desc()
+        rows = await self.db.scalars(query.order_by(order, DataPreparation.id).limit(limit).offset(offset))
         return page([await self.output(row) for row in rows], total, limit, offset)
 
     async def members(self, preparation_id, q="", dataset_id=None, limit=25, offset=0):

backend/tests/test_preparation_sort.py

@@ -0,0 +1,33 @@
+"""Preparation sorting applies before pagination and rejects unsupported keys."""
+
+
+async def test_preparation_sort_pagination(logged_in):
+    client = logged_in
+    scope = {"instrument_type": "EQUITY", "region": "USA", "universe": "TOP3000", "delay": 1}
+    for name in ["sort-C", "sort-A", "sort-B", "unrelated"]:
+        response = await client.post("/api/v1/data-preparations", json={"name": name, "scope": scope})
+        assert response.status_code == 201
+    for direction, expected in [
+        ("asc", ["sort-A", "sort-B", "sort-C"]),
+        ("desc", ["sort-C", "sort-B", "sort-A"]),
+    ]:
+        names = []
+        for offset in range(3):
+            response = await client.get(
+                "/api/v1/data-preparations",
+                params={
+                    "q": "sort-",
+                    "sort": "name",
+                    "direction": direction,
+                    "limit": 1,
+                    "offset": offset,
+                },
+            )
+            assert response.status_code == 200
+            assert response.json()["total"] == 3
+            names.append(response.json()["items"][0]["name"])
+        assert names == expected
+    for params in [{"sort": "field_count"}, {"direction": "sideways"}]:
+        assert (await client.get("/api/v1/data-preparations", params=params)).status_code == 422
+    default = (await client.get("/api/v1/data-preparations")).json()
+    assert default["items"][0]["name"] == "unrelated"

backend/tests/test_settings_metadata.py

@@ -0,0 +1,56 @@
+"""Settings metadata retains platform choices, numeric bounds and scope dependencies."""
+
+import pytest
+from fastapi import HTTPException
+
+from app.catalog.research_metadata import ResearchMetadata, setting_rows
+from tests.catalog_fake import platform_response
+
+
+def settings_response():
+    data = platform_response()
+    children = data["actions"]["POST"]["settings"]["children"]
+    children["neutralization"] = {"choices": [{"value": "INDUSTRY"}, {"value": "NONE"}]}
+    children["decay"] = {"min_value": 0, "max_value": 500}
+    children["truncation"] = {"choices": [{"value": 0.05}, {"value": 0.08}]}
+    children["visualization"] = {"choices": [{"value": False}, {"value": True}]}
+    children["nanHandling"] = {"choices": {"instrumentType": {"EQUITY": {"region": {
+        region: {"delay": {str(delay): [{"value": "OFF" if delay == 0 else "ON"}] for delay in [0, 1]}}
+        for region in ["USA", "CHN", "IND"]
+    }}}}}
+    return data
+
+
+def test_metadata_preserves_types_and_resolves_delay():
+    rows = setting_rows(settings_response())
+    first = next(r for r in rows if r["region"] == "USA" and r["delay"] == 0)
+    second = next(r for r in rows if r["region"] == "USA" and r["delay"] == 1)
+    assert first["fields"]["decay"] == {"minimum": 0, "maximum": 500}
+    assert first["fields"]["truncation"]["choices"] == [0.05, 0.08]
+    assert first["fields"]["visualization"]["choices"] == [False, True]
+    assert first["fields"]["nanHandling"]["choices"] == ["OFF"]
+    assert second["fields"]["nanHandling"]["choices"] == ["ON"]
+    assert first["neutralizations"] == ["INDUSTRY", "NONE"]
+
+
+async def test_refresh_publishes_fields_and_keeps_previous_snapshot_on_invalid_options(app):
+    class Client:
+        async def research_setting_options(self):
+            return settings_response()
+
+    async with app.state.sessions.begin() as db:
+        service = ResearchMetadata(db, Client())
+        previous = await service.refresh_settings()
+        assert previous["content"]["items"][0]["fields"]["visualization"]["choices"] == [False, True]
+
+        class InvalidClient:
+            async def research_setting_options(self):
+                data = settings_response()
+                data["actions"]["POST"]["settings"]["children"]["nanHandling"]["choices"] = {"unsupported": []}
+                return data
+
+        service.client = InvalidClient()
+        with pytest.raises(HTTPException) as error:
+            await service.refresh_settings()
+        assert error.value.status_code == 502
+        assert (await service.get("settings"))["content"] == previous["content"]