feat: 分离 Alpha 检查与提交限制,优化检查统计与前端展示

This commit is contained in:
yuxuanhui
2026-09-12 22:11:11 +08:00
parent 5eb6008ef7
commit 45eb4c3a17
13 changed files with 300 additions and 21 deletions
+6 -1
View File
@@ -12,10 +12,13 @@ from tests.test_submission import FIELDS, Description, setup
@pytest.mark.parametrize("kind", ["REGULAR", "SUPER"])
@pytest.mark.parametrize("result", ["PASS", "FAIL"])
async def test_mcp_check_never_submits(mcp_app, kind, result, monkeypatch):
@pytest.mark.parametrize("limited", [False, True])
async def test_mcp_check_never_submits(mcp_app, kind, result, limited, monkeypatch):
from tests import test_submission
checks = [{"name": "PROD_CORRELATION", "result": result}]
if limited:
checks.append({"name": "REGULAR_SUBMISSION", "result": "FAIL"})
monkeypatch.setattr(test_submission, "CHECKS", checks)
sections = ["regular"] if kind == "REGULAR" else ["selection", "combo"]
platform = await setup(mcp_app, alpha(type=kind, **{s: {"code": "rank(close)"} for s in sections}))
@@ -35,6 +38,8 @@ async def test_mcp_check_never_submits(mcp_app, kind, result, monkeypatch):
assert job["status"] == "completed", job
data = await invoke(mcp_app, principal, "get_submission_check", {"alpha_id": "alpha1"})
assert data["checks"] == checks and data["checked_at"]
assert data["check_summary"]["check_type"] == ("PASS" if result == "PASS" else "FAIL_1")
assert data["check_summary"]["submission_limits"]["status"] == ("blocked" if limited else "unknown")
assert data["production_submission"] is False
assert platform.patches == [{s: {"description": args["descriptions"][s]} for s in sections}]
assert platform.calls.count(("GET", "/alphas/alpha1/check")) == 2
@@ -0,0 +1,51 @@
"""Upgrade cached classifications across batches while preserving platform evidence."""
from pathlib import Path
import sqlalchemy as sa
from alembic import command
from alembic.config import Config
from cryptography.fernet import Fernet
from app.models import now
def test_limit_migration_is_reversible_and_preserves_snapshots(tmp_path, monkeypatch):
path = tmp_path / "limits.db"
monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{path}")
monkeypatch.setenv("ADMIN_PASSWORD", "migration-test-only")
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
monkeypatch.setenv("WQ_EMAIL", "")
monkeypatch.setenv("WQ_PASSWORD", "")
root = Path(__file__).resolve().parents[1]
config = Config(str(root / "alembic.ini"))
config.set_main_option("script_location", str(root / "migrations"))
command.upgrade(config, "0017")
engine = sa.create_engine(f"sqlite:///{path}")
table = sa.Table("alphas", sa.MetaData(), autoload_with=engine)
limit = {"name": "REGULAR_SUBMISSION", "result": "FAIL"}
patterns = [
([{"name": "PROD_CORRELATION", "result": "PASS"}, limit], "FAIL_1", "PASS"),
([{"name": "LOW_SHARPE", "result": "FAIL"}, limit], "FAIL_2", "FAIL_1"),
([limit], "FAIL_1", "PENDING"),
([{"name": "LOW_SHARPE", "result": "PASS"}, limit], "FAIL_1", "PRE_CHECK"),
([{"name": "UNKNOWN", "result": "FAIL"}], "FAIL_1", "FAIL_1"),
]
with engine.begin() as db:
db.execute(table.insert(), [
{"id": f"old{i:04}", "hidden": False, "settings": {}, "os_metrics": {},
"is_metrics": {"checks": patterns[i % 5][0]}, "checks": patterns[i % 5][0],
"check_type": patterns[i % 5][1], "synced_at": now(),
"raw": {"is": {"checks": patterns[i % 5][0]}}}
for i in range(503)
])
for version, position in [("0018", 2), ("0017", 1), ("0018", 2)]:
(command.upgrade if version == "0018" else command.downgrade)(config, version)
with engine.connect() as db:
rows = db.execute(sa.select(table).order_by(table.c.id)).mappings().all()
assert len(rows) == 503
for i, row in enumerate(rows):
assert row["check_type"] == patterns[i % 5][position]
assert row["checks"] == row["is_metrics"]["checks"] == row["raw"]["is"]["checks"] == patterns[i % 5][0]
command.check(config)
engine.dispose()
+76
View File
@@ -0,0 +1,76 @@
"""Submission limits must not disqualify an otherwise passing Alpha."""
import pytest
from app.alphas import failed_checks, snapshot_columns
LIMIT = {"name": "REGULAR_SUBMISSION", "result": "FAIL", "value": 4, "limit": 4}
@pytest.mark.parametrize("checks,expected", [
([{"name": "PROD_CORRELATION", "result": "PASS"}], "PASS"),
([{"name": "LOW_SHARPE", "result": "PASS"}], "PRE_CHECK"),
([{"name": "LOW_SHARPE", "result": "FAIL"}], "FAIL_1"),
([], "PENDING"),
([None], "PENDING"),
([{"name": "UNKNOWN_CHECK", "result": "FAIL"}], "FAIL_1"),
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PENDING"),
])
def test_limit_does_not_change_alpha_verdict(checks, expected):
original = [*checks, LIMIT]
assert snapshot_columns({}, {}, original)["check_type"] == expected
assert "REGULAR_SUBMISSION" not in failed_checks(original)
assert original[-1] == LIMIT
@pytest.mark.parametrize("result", ["PASS", "PENDING", "WARNING", "UNRECOGNIZED"])
def test_all_limit_states_are_separate(result):
from app.alphas import check_summary
checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, {**LIMIT, "result": result}]
summary = check_summary(checks)
assert summary["check_type"] == "PASS"
assert summary["submission_limits"]["status"] == ("not_blocked" if result == "PASS" else "unknown")
def test_research_and_mcp_evidence_keep_limits_separate():
from types import SimpleNamespace
from app.research.evaluations import assess
from app.research_access.queries import checks_summary
snapshot = {"is": {"sharpe": 2, "fitness": 2, "turnover": 0.1,
"checks": [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]}}
rules = SimpleNamespace(sharpe_min=1, fitness_min=1, turnover_max=0.5)
finding = assess(snapshot, rules)
assert finding["verdict"] == "pass" and finding["failed"] == []
assert finding["existing_platform_checks"] == snapshot["is"]["checks"]
assert finding["submission_limits"]["status"] == "blocked"
summary = checks_summary(snapshot)
assert summary["counts"]["FAIL"] == 0 and summary["total"] == 1
assert summary["non_pass"] == []
assert summary["submission_limits"] == [{"section": "is", "raw": LIMIT}]
snapshot["is"]["checks"] = [LIMIT]
assert assess(snapshot, rules)["verdict"] == "review"
assert checks_summary(snapshot)["status"] == "unknown"
async def test_http_check_persists_limit_without_failing_alpha(app, logged_in, monkeypatch):
from app.models import Alpha
from tests import test_submission
checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]
monkeypatch.setattr(test_submission, "CHECKS", checks)
await test_submission.setup(app)
response = await test_submission.enqueue(logged_in)
assert response.status_code == 202
await app.state.runner.execute(response.json()["id"])
async with app.state.sessions() as db:
item = await db.get(Alpha, "alpha1")
assert item.check_type == "PASS"
assert item.checks == item.is_metrics["checks"] == item.raw["is"]["checks"] == checks
state = (await logged_in.get("/api/v1/alphas/alpha1/submission")).json()
assert state["job"]["status"] == "completed"
assert state["check_summary"]["submission_limits"]["status"] == "blocked"
rows = (await logged_in.get("/api/v1/alphas?check_type=PASS")).json()["items"]
assert rows[0]["id"] == "alpha1" and rows[0]["failed_checks"] == []