my-home/worldquant-alpha-system
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore stage-based Alpha check classification

Browse Source
This commit is contained in:
yuxuanhui
2026-09-12 22:34:14 +08:00
parent 45eb4c3a17
commit c18960946b
13 changed files with 253 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_alpha_list_fields.py
+5 -5
View File
@@ -37,10 +37,10 @@ def checks(failures):
(None, "PENDING"),
([None], "PENDING"),
([{}], "PENDING"),
([{"name": "LOW_SHARPE", "result": "WARNING"}], "PENDING"),
([{"name": "LOW_SHARPE", "result": "WARNING"}], "PRE_CHECK"),
([{"name": "LOW_SHARPE", "result": "PASS"}], "PRE_CHECK"),
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PENDING"),
(checks(0), "PASS"),
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PRE_CHECK"),
(checks(0), "PRE_CHECK"),
(checks(1), "FAIL_1"),
(checks(2), "FAIL_2"),
(checks(3), "FAIL_2"),
@@ -58,7 +58,7 @@ async def test_checks_filter_before_pagination_and_share_export_scope(app, logge
for check_type, expected in [
("FAIL_1", ["failed1"]),
("FAIL_2", ["failed2", "failed3"]),
("PASS", ["failed0"]),
("PRE_CHECK", ["failed0"]),
("PENDING", ["unknown"]),
]:
response = await logged_in.get(
@@ -218,7 +218,7 @@ def test_migration_backfills_multiple_batches_and_preserves_research(tmp_path, m
rows = db.execute(sa.select(alphas).order_by(alphas.c.id)).mappings().all()
assert len(rows) == 503
for i, row in enumerate(rows):
expected = snapshot_columns(row["settings"], row["is_metrics"], checks(i % 4))
expected = snapshot_columns(row["settings"], row["is_metrics"], checks(i % 4), checked=True)
assert {key: row[key] for key in expected} == expected
record = db.execute(sa.select(research)).mappings().one()
assert record["tags"] == ["PPAC"] and record["note"] == "keep" and record["version"] == 7
backend/tests/test_check_stage_migration.py
+64
View File
@@ -0,0 +1,64 @@
"""Historical stage recovery requires a persisted /check newer than the latest sync."""
from datetime import datetime, timezone
from pathlib import Path
import sqlalchemy as sa
from alembic import command
from alembic.config import Config
from cryptography.fernet import Fernet
def test_stage_backfill_preserves_evidence_and_uses_checkpoints(tmp_path, monkeypatch):
path = tmp_path / "stages.db"
monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{path}")
monkeypatch.setenv("ADMIN_PASSWORD", "migration-test-only")
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
monkeypatch.setenv("WQ_EMAIL", "")
monkeypatch.setenv("WQ_PASSWORD", "")
root = Path(__file__).resolve().parents[1]
config = Config(str(root / "alembic.ini"))
config.set_main_option("script_location", str(root / "migrations"))
command.upgrade(config, "0018")
engine = sa.create_engine(f"sqlite:///{path}")
alphas = sa.Table("alphas", sa.MetaData(), autoload_with=engine)
jobs = sa.Table("sync_jobs", sa.MetaData(), autoload_with=engine)
synced = datetime(2026, 9, 12, 0, tzinfo=timezone.utc)
pending = [{"name": "PROD_CORRELATION", "result": "PENDING"},
{"name": "REGULAR_SUBMISSION", "result": "FAIL"}]
passed = [{"name": "PROD_CORRELATION", "result": "PASS"}]
patterns = [
(pending, "PENDING", "PRE_CHECK", None),
(pending, "PENDING", "PASS", {"phase": "checked", "checked_at": "2026-09-12T01:00:00+00:00"}),
(pending, "PENDING", "PRE_CHECK", {"phase": "checked", "checked_at": "2026-09-11T23:00:00+00:00"}),
(pending, "PENDING", "PRE_CHECK", {"phase": "check"}),
(passed, "PASS", "PRE_CHECK", None),
([{"name": "LOW_SHARPE", "result": "FAIL"}], "FAIL_1", "FAIL_1", None),
([{}], "PENDING", "PENDING", {"phase": "checked", "checked_at": "2026-09-12T01:00:00Z"}),
]
with engine.begin() as db:
db.execute(alphas.insert(), [
{"id": f"stage{i:04}", "hidden": False, "settings": {}, "os_metrics": {},
"is_metrics": {"checks": patterns[i % 7][0]}, "checks": patterns[i % 7][0],
"check_type": patterns[i % 7][1], "synced_at": synced,
"raw": {"is": {"checks": patterns[i % 7][0]}}}
for i in range(503)
])
db.execute(jobs.insert(), [
{"id": f"job{i}", "kind": "submission_check", "status": "completed",
"payload": {"alpha_ids": [f"stage{i:04}"]},
"checkpoint": {**patterns[i % 7][3], "alpha_id": f"stage{i:04}"},
"processed": 1, "failed": 0, "total": 1, "cancel_requested": False,
"created_at": synced, "updated_at": synced}
for i in range(503) if patterns[i % 7][3]
])
for target, expected_index in [("0019", 2), ("0018", 1), ("0019", 2)]:
(command.upgrade if target == "0019" else command.downgrade)(config, target)
with engine.connect() as db:
rows = db.execute(sa.select(alphas).order_by(alphas.c.id)).mappings().all()
assert len(rows) == 503
for i, row in enumerate(rows):
assert row["check_type"] == patterns[i % 7][expected_index]
assert row["checks"] == row["raw"]["is"]["checks"] == row["is_metrics"]["checks"] == patterns[i % 7][0]
command.check(config)
engine.dispose()
backend/tests/test_check_stages.py
+56
View File
@@ -0,0 +1,56 @@
"""The same checks have different meanings at sync and explicit /check stages."""
import pytest
from app.alphas import snapshot_columns
@pytest.mark.parametrize("checks", [
[{"name": "LOW_SHARPE", "result": "PASS"}],
[{"name": "PROD_CORRELATION", "result": "PASS"}],
[{"name": "PROD_CORRELATION", "result": "PENDING"}],
[{"name": "MATCHES_THEMES", "result": "WARNING"}],
])
def test_stage_not_correlation_presence_decides_pass(checks):
assert snapshot_columns({}, {}, checks)["check_type"] == "PRE_CHECK"
assert snapshot_columns({}, {}, checks, checked=True)["check_type"] == "PASS"
@pytest.mark.parametrize("checked", [False, True])
@pytest.mark.parametrize("checks,expected", [
([], "PENDING"),
([None], "PENDING"),
([{}], "PENDING"),
([{"name": "UNKNOWN", "result": "OTHER"}], "PENDING"),
([{"name": "REGULAR_SUBMISSION", "result": "FAIL"}], "PENDING"),
([{"name": "LOW_SHARPE", "result": "fail"}, {"name": "REGULAR_SUBMISSION", "result": "FAIL"}], "FAIL_1"),
([{"name": "LOW_SHARPE", "result": "FAIL"}, {"name": "LOW_FITNESS", "result": "Fail"}], "FAIL_2"),
])
def test_stage_preserves_failures_and_missing_evidence(checked, checks, expected):
assert snapshot_columns({}, {}, checks, checked=checked)["check_type"] == expected
async def test_sync_check_and_resync_use_distinct_stages(app, logged_in, monkeypatch):
from app.alphas import upsert_alpha
from tests import test_submission
from tests.conftest import alpha
checks = [{"name": "LOW_SHARPE", "result": "PASS"},
{"name": "PROD_CORRELATION", "result": "PENDING"},
{"name": "MATCHES_THEMES", "result": "WARNING"},
{"name": "REGULAR_SUBMISSION", "result": "FAIL"}]
monkeypatch.setattr(test_submission, "CHECKS", checks)
await test_submission.setup(app, alpha(**{"is": {"checks": checks}}))
endpoint = "/api/v1/alphas/alpha1/submission"
assert (await logged_in.get(endpoint)).json()["check_summary"]["check_type"] == "PRE_CHECK"
response = await test_submission.enqueue(logged_in)
assert response.status_code == 202
await app.state.runner.execute(response.json()["id"])
state = (await logged_in.get(endpoint)).json()
assert state["job"]["status"] == "completed"
assert state["check_summary"]["check_type"] == "PASS"
assert state["check_summary"]["submission_limits"]["status"] == "blocked"
assert (await logged_in.get("/api/v1/alphas/alpha1")).json()["check_type"] == "PASS"
async with app.state.sessions.begin() as db:
await upsert_alpha(db, alpha(**{"is": {"checks": checks}}))
assert (await logged_in.get(endpoint)).json()["check_summary"]["check_type"] == "PRE_CHECK"
backend/tests/test_mcp_submission.py
+2 -2
View File
@@ -11,7 +11,7 @@ from tests.test_submission import FIELDS, Description, setup
@pytest.mark.parametrize("kind", ["REGULAR", "SUPER"])
@pytest.mark.parametrize("result", ["PASS", "FAIL"])
@pytest.mark.parametrize("result", ["PASS", "FAIL", "PENDING", "WARNING"])
@pytest.mark.parametrize("limited", [False, True])
async def test_mcp_check_never_submits(mcp_app, kind, result, limited, monkeypatch):
from tests import test_submission
@@ -38,7 +38,7 @@ async def test_mcp_check_never_submits(mcp_app, kind, result, limited, monkeypat
assert job["status"] == "completed", job
data = await invoke(mcp_app, principal, "get_submission_check", {"alpha_id": "alpha1"})
assert data["checks"] == checks and data["checked_at"]
assert data["check_summary"]["check_type"] == ("PASS" if result == "PASS" else "FAIL_1")
assert data["check_summary"]["check_type"] == ("FAIL_1" if result == "FAIL" else "PASS")
assert data["check_summary"]["submission_limits"]["status"] == ("blocked" if limited else "unknown")
assert data["production_submission"] is False
assert platform.patches == [{s: {"description": args["descriptions"][s]} for s in sections}]
backend/tests/test_submission_limit_migration.py
+1
View File
@@ -47,5 +47,6 @@ def test_limit_migration_is_reversible_and_preserves_snapshots(tmp_path, monkeyp
for i, row in enumerate(rows):
assert row["check_type"] == patterns[i % 5][position]
assert row["checks"] == row["is_metrics"]["checks"] == row["raw"]["is"]["checks"] == patterns[i % 5][0]
command.upgrade(config, "head")
command.check(config)
engine.dispose()
backend/tests/test_submission_limits.py
+3 -3
View File
@@ -8,13 +8,13 @@ LIMIT = {"name": "REGULAR_SUBMISSION", "result": "FAIL", "value": 4, "limit": 4}
@pytest.mark.parametrize("checks,expected", [
([{"name": "PROD_CORRELATION", "result": "PASS"}], "PASS"),
([{"name": "PROD_CORRELATION", "result": "PASS"}], "PRE_CHECK"),
([{"name": "LOW_SHARPE", "result": "PASS"}], "PRE_CHECK"),
([{"name": "LOW_SHARPE", "result": "FAIL"}], "FAIL_1"),
([], "PENDING"),
([None], "PENDING"),
([{"name": "UNKNOWN_CHECK", "result": "FAIL"}], "FAIL_1"),
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PENDING"),
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PRE_CHECK"),
])
def test_limit_does_not_change_alpha_verdict(checks, expected):
original = [*checks, LIMIT]
@@ -28,7 +28,7 @@ def test_all_limit_states_are_separate(result):
from app.alphas import check_summary
checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, {**LIMIT, "result": result}]
summary = check_summary(checks)
summary = check_summary(checks, check_type="PASS")
assert summary["check_type"] == "PASS"
assert summary["submission_limits"]["status"] == ("not_blocked" if result == "PASS" else "unknown")