211 lines
14 KiB
Python
211 lines
14 KiB
Python
|
"""SUPER public HTTP/MCP acceptance using isolated persistence and synthetic upstream."""
|
||
|
|
import asyncio
|
||
|
|
from copy import deepcopy
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
from fastapi import HTTPException
|
||
|
|
from sqlalchemy import func, select
|
||
|
|
|
||
|
|
from app.backtests.contracts import Candidate
|
||
|
|
from app.models import SimulationAttempt, SuperSelectionSnapshot
|
||
|
|
from app.superalpha.evidence import parse_components
|
||
|
|
from tests.test_backtests import PARAMS, candidate, execute, preview, setup, start
|
||
|
|
from tests.test_mcp import credentials, invoke, mcp_app # noqa: F401
|
||
|
|
|
||
|
|
SETTINGS = PARAMS | {"selectionHandling": "POSITIVE", "selectionLimit": 100, "componentActivation": "IS"}
|
||
|
|
|
||
|
|
|
||
|
|
def super_candidate(index=0, **changes):
|
||
|
|
return {"client_item_id": f"super-{index}", "alpha_type": "SUPER", "selection": f"turnover < {0.1 + index / 10}", "combo": "alpha", "settings": SETTINGS} | changes
|
||
|
|
|
||
|
|
|
||
|
|
def plan(**changes):
|
||
|
|
return {"name": "Super 研究", "hypothesis": "降低组件换手", "selection": "turnover < {threshold}", "combo": "alpha", "variables": {"threshold": {"kind": "number", "values": [0.1, 0.2]}}, "settings": SETTINGS, "include_baseline": True} | changes
|
||
|
|
|
||
|
|
|
||
|
|
async def test_plan_selection_build_versions_and_generic_run(app, logged_in):
|
||
|
|
platform, lane = await setup(app)
|
||
|
|
body = {"plan": plan(), "idempotency_key": "save1"}
|
||
|
|
saved = (await logged_in.post("/api/v1/superalpha/plans", json=body)).json()
|
||
|
|
assert saved["version"] == 1, saved
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/plans", json=body)).json() == saved
|
||
|
|
selection = {"selection": "turnover < 0.1", "settings": SETTINGS, "plan_id": saved["id"], "version": 1}
|
||
|
|
job = (await logged_in.post("/api/v1/superalpha/selections", json=selection)).json()
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/selections", json=selection)).json()["job_id"] == job["job_id"]
|
||
|
|
await app.state.runner.run_next()
|
||
|
|
snap = (await logged_in.get(f"/api/v1/superalpha/selections?job_id={job['job_id']}&limit=1")).json()
|
||
|
|
assert snap["complete"] and snap["has_more"] and snap["total"] == 2, snap
|
||
|
|
assert len(platform.selection_reads) == 1 and not platform.posts
|
||
|
|
assert set(platform.selection_reads[0]) == {"selection", "instrumentType", "region", "delay", "selectionLimit", "selectionHandling"}
|
||
|
|
build = {"plan_id": saved["id"], "version": 1, "selection_snapshot_ids": [snap["snapshot_id"]], "idempotency_key": "build1"}
|
||
|
|
exp = (await logged_in.post("/api/v1/superalpha/candidates", json=build)).json()
|
||
|
|
assert exp["total"] == 4, exp
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/candidates", json=build)).json() == exp
|
||
|
|
# A subset keeps the original experiment's evidence without claiming it applies to every candidate.
|
||
|
|
p = await logged_in.post(f"/api/v1/superalpha/experiments/{exp['id']}/preview", json={"candidate_ids": ["super-2", "super-2-baseline"]})
|
||
|
|
assert p.status_code == 200, p.text
|
||
|
|
assert p.json()["batch_count"] == 2 and not platform.posts
|
||
|
|
rid = (await start(logged_in, p.json()))["backtest_run_id"]
|
||
|
|
await execute(app, lane, rid)
|
||
|
|
result = (await logged_in.get(f"/api/v1/backtests/runs/{rid}/results")).json()
|
||
|
|
assert all(i["persistence_status"] == "saved" and i["alpha_type"] == "SUPER" for i in result["items"]), result
|
||
|
|
assert len(platform.posts) == 2 and all(len(p) == 1 for p in platform.posts)
|
||
|
|
item = result["items"][0]
|
||
|
|
actual = (await logged_in.get(f"/api/v1/backtests/items/{item['id']}/artifact?kind=components")).json()
|
||
|
|
assert actual["complete"] and actual["source"] == "actual" and actual["snapshot_id"] != snap["snapshot_id"]
|
||
|
|
assert actual["component_hash"] == snap["component_hash"]
|
||
|
|
alpha = (await logged_in.get(f"/api/v1/superalpha/alphas/{item['alpha_id']}")).json()
|
||
|
|
assert alpha["descriptions"]["combo"] == "Combo rationale"
|
||
|
|
assert alpha["sources"]["items"][0]["source"]["superalpha_plan_version"] == 1
|
||
|
|
changed = body | {"plan": plan(name="修订"), "plan_id": saved["id"], "version": 1, "idempotency_key": "save2"}
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/plans", json=changed)).json()["version"] == 2
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/plans", json=changed | {"idempotency_key": "save3"})).status_code == 409
|
||
|
|
assert (await logged_in.delete(f"/api/v1/superalpha/plans/{saved['id']}?version=2")).status_code == 200
|
||
|
|
assert (await logged_in.get(f"/api/v1/superalpha/plans/{saved['id']}?version=1")).json()["content"]["name"] == "Super 研究"
|
||
|
|
rows = (await logged_in.get(f"/api/v1/superalpha/experiments/{exp['id']}/results")).json()
|
||
|
|
assert rows["total"] == 2 and rows["items"][0]["pnl_fetched_at"] is None
|
||
|
|
async with app.state.sessions() as db:
|
||
|
|
assert await db.scalar(select(func.count()).select_from(SuperSelectionSnapshot)) == 3
|
||
|
|
|
||
|
|
|
||
|
|
async def test_mixed_run_isolation_facets_export_and_strict_match(app, logged_in):
|
||
|
|
platform, lane = await setup(app)
|
||
|
|
p = await preview(logged_in, [candidate(0), candidate(1), super_candidate(), super_candidate(1)])
|
||
|
|
assert p["batch_count"] == 3
|
||
|
|
rid = (await start(logged_in, p))["backtest_run_id"]
|
||
|
|
await execute(app, lane, rid)
|
||
|
|
assert len(platform.posts) == 3
|
||
|
|
results = (await logged_in.get(f"/api/v1/backtests/runs/{rid}/results")).json()["items"]
|
||
|
|
assert all(i["persistence_status"] == "saved" for i in results), results
|
||
|
|
for scope, kind in (("super", "SUPER"), ("non_super", "REGULAR")):
|
||
|
|
page = (await logged_in.get(f"/api/v1/alphas?management_scope={scope}")).json()
|
||
|
|
assert page["total"] == 2 and all(i["alpha_type"] == kind for i in page["items"])
|
||
|
|
facets = (await logged_in.get(f"/api/v1/alphas/facets?management_scope={scope}")).json()
|
||
|
|
assert facets["alpha_type"] == [kind], facets
|
||
|
|
export = await logged_in.get(f"/api/v1/alphas/export?management_scope={scope}")
|
||
|
|
assert export.status_code == 200
|
||
|
|
assert all(i["alpha_id"] in export.text for i in results if i["alpha_type"] == kind)
|
||
|
|
assert all(i["alpha_id"] not in export.text for i in results if i["alpha_type"] != kind)
|
||
|
|
rid2 = (await start(logged_in, await preview(logged_in, [super_candidate(9)]), "mismatch"))["backtest_run_id"]
|
||
|
|
async with app.state.sessions() as db:
|
||
|
|
aid = await db.scalar(select(SimulationAttempt.id).where(SimulationAttempt.run_id == rid2))
|
||
|
|
await lane.step(aid)
|
||
|
|
next(reversed(platform.alphas.values()))["combo"]["code"] = "WRONG"
|
||
|
|
await lane.step(aid)
|
||
|
|
row = (await logged_in.get(f"/api/v1/backtests/runs/{rid2}/results")).json()["items"][0]
|
||
|
|
assert row["persistence_status"] != "saved"
|
||
|
|
assert (await logged_in.get("/api/v1/backtests/runs?alpha_type=SUPER")).json()["total"] == 2
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("reject", ["unknown", "missing_location", "rate"])
|
||
|
|
async def test_super_reliability_no_unknown_resubmission(app, logged_in, reject):
|
||
|
|
platform, lane = await setup(app)
|
||
|
|
platform.reject = reject
|
||
|
|
rid = (await start(logged_in, await preview(logged_in, [super_candidate()])))["backtest_run_id"]
|
||
|
|
async with app.state.sessions() as db:
|
||
|
|
aid = await db.scalar(select(SimulationAttempt.id).where(SimulationAttempt.run_id == rid))
|
||
|
|
for _ in range(app.state.settings.retry_attempts + 1):
|
||
|
|
await lane.step(aid)
|
||
|
|
await asyncio.sleep(0.02)
|
||
|
|
if reject == "rate":
|
||
|
|
assert len(platform.posts) == app.state.settings.retry_attempts
|
||
|
|
else:
|
||
|
|
await lane.start()
|
||
|
|
await lane.stop()
|
||
|
|
await logged_in.post(f"/api/v1/backtests/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||
|
|
await lane.step(aid)
|
||
|
|
assert len(platform.posts) == 1
|
||
|
|
|
||
|
|
|
||
|
|
async def test_unknown_components_and_recovery(app, logged_in):
|
||
|
|
platform, lane = await setup(app)
|
||
|
|
platform.actual_components = None
|
||
|
|
platform.detail_fail = True
|
||
|
|
rid = (await start(logged_in, await preview(logged_in, [super_candidate()])))["backtest_run_id"]
|
||
|
|
ids = await execute(app, lane, rid)
|
||
|
|
platform.detail_fail = False
|
||
|
|
await logged_in.post(f"/api/v1/backtests/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||
|
|
await lane.step(ids[0])
|
||
|
|
item = (await logged_in.get(f"/api/v1/backtests/runs/{rid}/results")).json()["items"][0]
|
||
|
|
assert item["persistence_status"] == "saved" and len(platform.posts) == 1
|
||
|
|
components = (await logged_in.get(f"/api/v1/backtests/items/{item['id']}/artifact?kind=components")).json()
|
||
|
|
assert components["complete"] is False and components["component_hash"] is None
|
||
|
|
|
||
|
|
|
||
|
|
async def test_sampling_validation_and_component_evidence(app, logged_in):
|
||
|
|
await setup(app)
|
||
|
|
for invalid in (super_candidate(selection=" "), super_candidate(combo=""), super_candidate(expression="close"), super_candidate(settings=PARAMS)):
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
Candidate.model_validate(invalid)
|
||
|
|
assert (await logged_in.get("/api/v1/superalpha/selections")).status_code == 422
|
||
|
|
assert parse_components({"count": 0, "results": []})["complete"]
|
||
|
|
for raw in (["a"], {"count": 2, "results": ["a"]}, {"count": 2, "results": ["a", "a"]}, {"count": 1, "results": ["a"], "next": "next"}):
|
||
|
|
assert not parse_components(raw)["complete"]
|
||
|
|
complete = parse_components({"count": 1, "results": ["a"], "warnings": ["synthetic warning"]})
|
||
|
|
assert complete["complete"] and complete["warnings"] == ["synthetic warning"]
|
||
|
|
args = {"plan": plan(variables={"threshold": {"kind": "number", "values": list(range(100))}}), "mode": "random", "limit": 5, "seed": 42, "idempotency_key": "random1"}
|
||
|
|
a = (await logged_in.post("/api/v1/superalpha/candidates", json=args)).json()
|
||
|
|
b = (await logged_in.post("/api/v1/superalpha/candidates", json=args | {"idempotency_key": "random2"})).json()
|
||
|
|
assert a["candidates"] == b["candidates"] and a["total"] == 10
|
||
|
|
bad = deepcopy(args)
|
||
|
|
bad["plan"]["setting_variants"] = {"selectionLimit": [0]}
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/candidates", json=bad)).status_code == 422
|
||
|
|
|
||
|
|
|
||
|
|
async def test_mcp_interop_permissions_inline_and_sources(mcp_app): # noqa: F811
|
||
|
|
app = mcp_app
|
||
|
|
principal, _ = await credentials(app, {"research:read", "research:write", "research:refresh", "backtests:execute"})
|
||
|
|
saved = await invoke(app, principal, "save_superalpha_plan", {"plan": plan(), "idempotency_key": "mcp-plan"})
|
||
|
|
built = await invoke(app, principal, "build_superalpha_candidates", {"plan_id": saved["id"], "version": 1, "idempotency_key": "mcp-build"})
|
||
|
|
assert not built["starts_backtests"] and built["total"] == 4
|
||
|
|
assert (await invoke(app, principal, "get_superalpha_plan", {"experiment_id": built["id"], "limit": 1}))["has_more"]
|
||
|
|
submit = {"name": "MCP SUPER", "candidates": built["candidates"][:1], "source": {"research_id": built["id"], "superalpha_plan_id": saved["id"], "superalpha_plan_version": 1}, "duplicate_policy": "rerun", "idempotency_key": "mcp-run"}
|
||
|
|
result = await invoke(app, principal, "submit_backtests", submit)
|
||
|
|
assert result == await invoke(app, principal, "submit_backtests", submit)
|
||
|
|
rid = result["backtest_run_id"]
|
||
|
|
await execute(app, app.state.runner.backtests, rid)
|
||
|
|
results = await invoke(app, principal, "get_backtest_results", {"run_id": rid})
|
||
|
|
item = results["items"][0]
|
||
|
|
assert item["persistence_status"] == "saved", results
|
||
|
|
alpha = await invoke(app, principal, "get_superalpha", {"alpha_id": item["alpha_id"]})
|
||
|
|
assert alpha["alpha_type"] == "SUPER" and "#superalphas?" in alpha["web_url"]
|
||
|
|
forged = deepcopy(submit)
|
||
|
|
forged["idempotency_key"] = "forged"
|
||
|
|
forged["candidates"][0]["combo"] = "WRONG"
|
||
|
|
assert (await app.state.mcp.invoke(principal, "submit_backtests", forged)).is_error
|
||
|
|
direct = submit | {"source": {}, "candidates": [super_candidate(9)], "idempotency_key": "direct"}
|
||
|
|
assert (await invoke(app, principal, "submit_backtests", direct))["source"]["superalpha_plan_id"] is None
|
||
|
|
readonly, _ = await credentials(app, {"research:read"})
|
||
|
|
with pytest.raises(HTTPException) as exc:
|
||
|
|
await app.state.mcp.invoke(readonly, "save_superalpha_plan", {"plan": plan(), "idempotency_key": "denied"})
|
||
|
|
assert exc.value.status_code == 403
|
||
|
|
|
||
|
|
|
||
|
|
async def test_metadata_stage_constraints_selection_recovery_and_cancel(app, logged_in):
|
||
|
|
from app.catalog.research_metadata import ResearchMetadata
|
||
|
|
from app.models import Job
|
||
|
|
from app.superalpha.jobs import run_selection
|
||
|
|
platform, _ = await setup(app)
|
||
|
|
async with app.state.sessions.begin() as db:
|
||
|
|
await ResearchMetadata(db).publish("settings", "settings", {"items": [{"instrument_type": "EQUITY", "region": "USA", "delay": 1, "universe": "TOP3000", "neutralizations": ["INDUSTRY"], "fields": {"selectionLimit": {"maximum": 50}}}]})
|
||
|
|
await ResearchMetadata(db).publish("operators", "operators", {"items": [
|
||
|
|
{"name": "combo_a", "category": "Combo", "scope": ["COMBO"]},
|
||
|
|
{"name": "not_combo", "category": "Other", "scope": ["UNKNOWN_COMBO"]},
|
||
|
|
{"name": "unspecified", "category": "Other", "scope": None}]})
|
||
|
|
assert (await logged_in.post("/api/v1/superalpha/plans", json={"plan": plan(), "idempotency_key": "bad-settings"})).status_code == 422
|
||
|
|
p = await logged_in.post("/api/v1/backtests/previews", json={"inline": {"name": "bad", "candidates": [super_candidate()]}})
|
||
|
|
assert p.status_code == 422
|
||
|
|
assert [o["name"] for o in (await logged_in.get("/api/v1/catalog/operators?stage=COMBO")).json()["items"]] == ["combo_a"]
|
||
|
|
payload = {"selection": "turnover", "settings": SETTINGS | {"selectionLimit": 50}}
|
||
|
|
job = (await logged_in.post("/api/v1/superalpha/selections", json=payload)).json()
|
||
|
|
await app.state.runner.run_next()
|
||
|
|
await run_selection(app.state.runner, job["job_id"], payload) # Restart after commit preserves evidence.
|
||
|
|
assert len(platform.selection_reads) == 1
|
||
|
|
cancelled = (await logged_in.post("/api/v1/superalpha/selections", json=payload)).json()
|
||
|
|
async with app.state.sessions.begin() as db:
|
||
|
|
(await db.get(Job, cancelled["job_id"])).cancel_requested = True
|
||
|
|
await app.state.runner.run_next()
|
||
|
|
async with app.state.sessions() as db:
|
||
|
|
assert (await db.get(Job, cancelled["job_id"])).status == "cancelled"
|
||
|
|
assert await db.scalar(select(func.count()).select_from(SuperSelectionSnapshot)) == 1
|