worldquant-alpha-system/backend/tests/test_submission_limits.py

"""Submission limits must not disqualify an otherwise passing Alpha."""

import pytest

from app.alphas import failed_checks, snapshot_columns

LIMIT = {"name": "REGULAR_SUBMISSION", "result": "FAIL", "value": 4, "limit": 4}


@pytest.mark.parametrize("checks,expected", [
    ([{"name": "PROD_CORRELATION", "result": "PASS"}], "PRE_CHECK"),
    ([{"name": "LOW_SHARPE", "result": "PASS"}], "PRE_CHECK"),
    ([{"name": "LOW_SHARPE", "result": "FAIL"}], "FAIL_1"),
    ([], "PENDING"),
    ([None], "PENDING"),
    ([{"name": "UNKNOWN_CHECK", "result": "FAIL"}], "FAIL_1"),
    ([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PRE_CHECK"),
])
def test_limit_does_not_change_alpha_verdict(checks, expected):
    original = [*checks, LIMIT]
    assert snapshot_columns({}, {}, original)["check_type"] == expected
    assert "REGULAR_SUBMISSION" not in failed_checks(original)
    assert original[-1] == LIMIT


@pytest.mark.parametrize("result", ["PASS", "PENDING", "WARNING", "UNRECOGNIZED"])
def test_all_limit_states_are_separate(result):
    from app.alphas import check_summary

    checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, {**LIMIT, "result": result}]
    summary = check_summary(checks, check_type="PASS")
    assert summary["check_type"] == "PASS"
    assert summary["submission_limits"]["status"] == ("not_blocked" if result == "PASS" else "unknown")


def test_research_and_mcp_evidence_keep_limits_separate():
    from types import SimpleNamespace

    from app.research.evaluations import assess
    from app.research_access.queries import checks_summary

    snapshot = {"is": {"sharpe": 2, "fitness": 2, "turnover": 0.1,
                       "checks": [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]}}
    rules = SimpleNamespace(sharpe_min=1, fitness_min=1, turnover_max=0.5)
    finding = assess(snapshot, rules)
    assert finding["verdict"] == "pass" and finding["failed"] == []
    assert finding["existing_platform_checks"] == snapshot["is"]["checks"]
    assert finding["submission_limits"]["status"] == "blocked"
    summary = checks_summary(snapshot)
    assert summary["counts"]["FAIL"] == 0 and summary["total"] == 1
    assert summary["non_pass"] == []
    assert summary["submission_limits"] == [{"section": "is", "raw": LIMIT}]
    snapshot["is"]["checks"] = [LIMIT]
    assert assess(snapshot, rules)["verdict"] == "review"
    assert checks_summary(snapshot)["status"] == "unknown"


async def test_http_check_persists_limit_without_failing_alpha(app, logged_in, monkeypatch):
    from app.models import Alpha
    from tests import test_submission

    checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]
    monkeypatch.setattr(test_submission, "CHECKS", checks)
    await test_submission.setup(app)
    response = await test_submission.enqueue(logged_in)
    assert response.status_code == 202
    await app.state.runner.execute(response.json()["id"])
    async with app.state.sessions() as db:
        item = await db.get(Alpha, "alpha1")
        assert item.check_type == "PASS"
        assert item.checks == item.is_metrics["checks"] == item.raw["is"]["checks"] == checks
    state = (await logged_in.get("/api/v1/alphas/alpha1/submission")).json()
    assert state["job"]["status"] == "completed"
    assert state["check_summary"]["submission_limits"]["status"] == "blocked"
    rows = (await logged_in.get("/api/v1/alphas?check_type=PASS")).json()["items"]
    assert rows[0]["id"] == "alpha1" and rows[0]["failed_checks"] == []