feat: 分离 Alpha 检查与提交限制,优化检查统计与前端展示
This commit is contained in:
@@ -12,10 +12,13 @@ from tests.test_submission import FIELDS, Description, setup
|
||||
|
||||
@pytest.mark.parametrize("kind", ["REGULAR", "SUPER"])
|
||||
@pytest.mark.parametrize("result", ["PASS", "FAIL"])
|
||||
async def test_mcp_check_never_submits(mcp_app, kind, result, monkeypatch):
|
||||
@pytest.mark.parametrize("limited", [False, True])
|
||||
async def test_mcp_check_never_submits(mcp_app, kind, result, limited, monkeypatch):
|
||||
from tests import test_submission
|
||||
|
||||
checks = [{"name": "PROD_CORRELATION", "result": result}]
|
||||
if limited:
|
||||
checks.append({"name": "REGULAR_SUBMISSION", "result": "FAIL"})
|
||||
monkeypatch.setattr(test_submission, "CHECKS", checks)
|
||||
sections = ["regular"] if kind == "REGULAR" else ["selection", "combo"]
|
||||
platform = await setup(mcp_app, alpha(type=kind, **{s: {"code": "rank(close)"} for s in sections}))
|
||||
@@ -35,6 +38,8 @@ async def test_mcp_check_never_submits(mcp_app, kind, result, monkeypatch):
|
||||
assert job["status"] == "completed", job
|
||||
data = await invoke(mcp_app, principal, "get_submission_check", {"alpha_id": "alpha1"})
|
||||
assert data["checks"] == checks and data["checked_at"]
|
||||
assert data["check_summary"]["check_type"] == ("PASS" if result == "PASS" else "FAIL_1")
|
||||
assert data["check_summary"]["submission_limits"]["status"] == ("blocked" if limited else "unknown")
|
||||
assert data["production_submission"] is False
|
||||
assert platform.patches == [{s: {"description": args["descriptions"][s]} for s in sections}]
|
||||
assert platform.calls.count(("GET", "/alphas/alpha1/check")) == 2
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Upgrade cached classifications across batches while preserving platform evidence."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app.models import now
|
||||
|
||||
|
||||
def test_limit_migration_is_reversible_and_preserves_snapshots(tmp_path, monkeypatch):
|
||||
path = tmp_path / "limits.db"
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{path}")
|
||||
monkeypatch.setenv("ADMIN_PASSWORD", "migration-test-only")
|
||||
monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
|
||||
monkeypatch.setenv("WQ_EMAIL", "")
|
||||
monkeypatch.setenv("WQ_PASSWORD", "")
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
config = Config(str(root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(root / "migrations"))
|
||||
command.upgrade(config, "0017")
|
||||
engine = sa.create_engine(f"sqlite:///{path}")
|
||||
table = sa.Table("alphas", sa.MetaData(), autoload_with=engine)
|
||||
limit = {"name": "REGULAR_SUBMISSION", "result": "FAIL"}
|
||||
patterns = [
|
||||
([{"name": "PROD_CORRELATION", "result": "PASS"}, limit], "FAIL_1", "PASS"),
|
||||
([{"name": "LOW_SHARPE", "result": "FAIL"}, limit], "FAIL_2", "FAIL_1"),
|
||||
([limit], "FAIL_1", "PENDING"),
|
||||
([{"name": "LOW_SHARPE", "result": "PASS"}, limit], "FAIL_1", "PRE_CHECK"),
|
||||
([{"name": "UNKNOWN", "result": "FAIL"}], "FAIL_1", "FAIL_1"),
|
||||
]
|
||||
with engine.begin() as db:
|
||||
db.execute(table.insert(), [
|
||||
{"id": f"old{i:04}", "hidden": False, "settings": {}, "os_metrics": {},
|
||||
"is_metrics": {"checks": patterns[i % 5][0]}, "checks": patterns[i % 5][0],
|
||||
"check_type": patterns[i % 5][1], "synced_at": now(),
|
||||
"raw": {"is": {"checks": patterns[i % 5][0]}}}
|
||||
for i in range(503)
|
||||
])
|
||||
for version, position in [("0018", 2), ("0017", 1), ("0018", 2)]:
|
||||
(command.upgrade if version == "0018" else command.downgrade)(config, version)
|
||||
with engine.connect() as db:
|
||||
rows = db.execute(sa.select(table).order_by(table.c.id)).mappings().all()
|
||||
assert len(rows) == 503
|
||||
for i, row in enumerate(rows):
|
||||
assert row["check_type"] == patterns[i % 5][position]
|
||||
assert row["checks"] == row["is_metrics"]["checks"] == row["raw"]["is"]["checks"] == patterns[i % 5][0]
|
||||
command.check(config)
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Submission limits must not disqualify an otherwise passing Alpha."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.alphas import failed_checks, snapshot_columns
|
||||
|
||||
LIMIT = {"name": "REGULAR_SUBMISSION", "result": "FAIL", "value": 4, "limit": 4}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checks,expected", [
|
||||
([{"name": "PROD_CORRELATION", "result": "PASS"}], "PASS"),
|
||||
([{"name": "LOW_SHARPE", "result": "PASS"}], "PRE_CHECK"),
|
||||
([{"name": "LOW_SHARPE", "result": "FAIL"}], "FAIL_1"),
|
||||
([], "PENDING"),
|
||||
([None], "PENDING"),
|
||||
([{"name": "UNKNOWN_CHECK", "result": "FAIL"}], "FAIL_1"),
|
||||
([{"name": "PROD_CORRELATION", "result": "PENDING"}], "PENDING"),
|
||||
])
|
||||
def test_limit_does_not_change_alpha_verdict(checks, expected):
|
||||
original = [*checks, LIMIT]
|
||||
assert snapshot_columns({}, {}, original)["check_type"] == expected
|
||||
assert "REGULAR_SUBMISSION" not in failed_checks(original)
|
||||
assert original[-1] == LIMIT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", ["PASS", "PENDING", "WARNING", "UNRECOGNIZED"])
|
||||
def test_all_limit_states_are_separate(result):
|
||||
from app.alphas import check_summary
|
||||
|
||||
checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, {**LIMIT, "result": result}]
|
||||
summary = check_summary(checks)
|
||||
assert summary["check_type"] == "PASS"
|
||||
assert summary["submission_limits"]["status"] == ("not_blocked" if result == "PASS" else "unknown")
|
||||
|
||||
|
||||
def test_research_and_mcp_evidence_keep_limits_separate():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.research.evaluations import assess
|
||||
from app.research_access.queries import checks_summary
|
||||
|
||||
snapshot = {"is": {"sharpe": 2, "fitness": 2, "turnover": 0.1,
|
||||
"checks": [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]}}
|
||||
rules = SimpleNamespace(sharpe_min=1, fitness_min=1, turnover_max=0.5)
|
||||
finding = assess(snapshot, rules)
|
||||
assert finding["verdict"] == "pass" and finding["failed"] == []
|
||||
assert finding["existing_platform_checks"] == snapshot["is"]["checks"]
|
||||
assert finding["submission_limits"]["status"] == "blocked"
|
||||
summary = checks_summary(snapshot)
|
||||
assert summary["counts"]["FAIL"] == 0 and summary["total"] == 1
|
||||
assert summary["non_pass"] == []
|
||||
assert summary["submission_limits"] == [{"section": "is", "raw": LIMIT}]
|
||||
snapshot["is"]["checks"] = [LIMIT]
|
||||
assert assess(snapshot, rules)["verdict"] == "review"
|
||||
assert checks_summary(snapshot)["status"] == "unknown"
|
||||
|
||||
|
||||
async def test_http_check_persists_limit_without_failing_alpha(app, logged_in, monkeypatch):
|
||||
from app.models import Alpha
|
||||
from tests import test_submission
|
||||
|
||||
checks = [{"name": "PROD_CORRELATION", "result": "PASS"}, LIMIT]
|
||||
monkeypatch.setattr(test_submission, "CHECKS", checks)
|
||||
await test_submission.setup(app)
|
||||
response = await test_submission.enqueue(logged_in)
|
||||
assert response.status_code == 202
|
||||
await app.state.runner.execute(response.json()["id"])
|
||||
async with app.state.sessions() as db:
|
||||
item = await db.get(Alpha, "alpha1")
|
||||
assert item.check_type == "PASS"
|
||||
assert item.checks == item.is_metrics["checks"] == item.raw["is"]["checks"] == checks
|
||||
state = (await logged_in.get("/api/v1/alphas/alpha1/submission")).json()
|
||||
assert state["job"]["status"] == "completed"
|
||||
assert state["check_summary"]["submission_limits"]["status"] == "blocked"
|
||||
rows = (await logged_in.get("/api/v1/alphas?check_type=PASS")).json()["items"]
|
||||
assert rows[0]["id"] == "alpha1" and rows[0]["failed_checks"] == []
|
||||
Reference in New Issue
Block a user