yuxuanhui
192 lines
8.3 KiB
Python
192 lines
8.3 KiB
Python
"""Stage-two acceptance: immutable plans, views, lineage and evidence-backed decisions."""
|
|
|
|
import pytest
|
|
|
|
from app.alphas import upsert_alpha
|
|
from app.models import Research
|
|
from tests.conftest import alpha
|
|
from tests.test_research_workspace import catalog, expansion, research_input, template
|
|
|
|
__all__ = ["catalog", "research_input"]
|
|
|
|
|
|
async def test_feature_conversion_keeps_original_version_through_experiment(logged_in, research_input):
|
|
content = {
|
|
"name": "特征方案",
|
|
"hypothesis": "排序稳定性",
|
|
"input_ids": [research_input["id"]],
|
|
"steps": [{"name": "排序", "rationale": "降低异常值影响", "expression": "rank(TEST_FIN_001)"}],
|
|
"template": template(),
|
|
}
|
|
saved = (
|
|
await logged_in.post("/api/v1/research/assets", json={"kind": "feature", "content": content})
|
|
).json()
|
|
converted = await logged_in.post(f"/api/v1/research/features/{saved['id']}/template", json={"version": 1})
|
|
assert converted.status_code == 201, converted.text
|
|
result = converted.json()
|
|
changed = await logged_in.put(
|
|
f"/api/v1/research/assets/{saved['id']}",
|
|
json={"kind": "feature", "version": 1, "content": {**content, "hypothesis": "新假设"}},
|
|
)
|
|
assert changed.status_code == 200
|
|
body = expansion(research_input["id"], asset_id=result["id"], version=1)
|
|
body.pop("template")
|
|
experiment = (await logged_in.post("/api/v1/research/experiments", json=body)).json()
|
|
source = experiment["evidence"]["template"]["provenance"]["feature"]
|
|
assert source["version"] == 1 and source["content"]["hypothesis"] == "排序稳定性"
|
|
assert source["content"]["input_ids"] == [research_input["id"]]
|
|
|
|
|
|
async def test_saved_views_validate_and_retain_sort_columns(logged_in):
|
|
content = {
|
|
"name": "低换手队列",
|
|
"filters": {
|
|
"turnover_max": "0.15",
|
|
"sort": "fitness",
|
|
"direction": "desc",
|
|
"submission": "UNSUBMITTED",
|
|
},
|
|
"columns": ["name", "fitness"],
|
|
}
|
|
response = await logged_in.post("/api/v1/research/assets", json={"kind": "view", "content": content})
|
|
assert response.status_code == 201, response.text
|
|
item = response.json()
|
|
assert item["content"]["filters"]["turnover_max"] == 0.15
|
|
assert item["content"]["filters"]["sort"] == "fitness"
|
|
assert item["content"]["columns"] == ["name", "fitness"]
|
|
bad = await logged_in.post(
|
|
"/api/v1/research/assets",
|
|
json={"kind": "view", "content": {**content, "filters": {"sort": "arbitrary_sql"}}},
|
|
)
|
|
assert bad.status_code == 422
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"metrics,verdict,missing",
|
|
[
|
|
({"sharpe": None, "fitness": 1, "turnover": 0.1}, "review", "sharpe"),
|
|
({"sharpe": 2, "fitness": 1, "turnover": 0.1}, "review", "platform_checks"),
|
|
(
|
|
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "FAIL"}]},
|
|
"block",
|
|
None,
|
|
),
|
|
(
|
|
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]},
|
|
"pass",
|
|
None,
|
|
),
|
|
],
|
|
)
|
|
async def test_evaluations_keep_missing_and_failures(app, logged_in, metrics, verdict, missing):
|
|
async with app.state.sessions.begin() as db:
|
|
await upsert_alpha(db, alpha("seed", **{"is": metrics}))
|
|
response = await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})
|
|
assert response.status_code == 201, response.text
|
|
report = response.json()["report"]
|
|
assert report["verdict"] == verdict
|
|
if missing:
|
|
assert missing in report["records"][0]["missing"]
|
|
async with app.state.sessions() as db:
|
|
assert (await db.get(Research, "seed")).state == "inbox"
|
|
|
|
|
|
async def test_sync_and_model_advice_do_not_rewrite_report(app, logged_in):
|
|
from app.research.evaluations import Evaluations
|
|
|
|
async with app.state.sessions.begin() as db:
|
|
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": -1}}))
|
|
original = (await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})).json()
|
|
async with app.state.sessions.begin() as db:
|
|
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": 9}}))
|
|
advised = await Evaluations(db).add_advice(
|
|
original["id"],
|
|
{"summary": "建议通过", "risks": [], "suggestions": []},
|
|
{"model": "fixture", "revision": 1},
|
|
)
|
|
assert advised["id"] != original["id"]
|
|
assert advised["report"]["verdict"] == "block"
|
|
assert advised["report"]["records"][0]["snapshot"]["is"]["sharpe"] == -1
|
|
assert (await logged_in.get(f"/api/v1/research/evaluations/{original['id']}")).json() == original
|
|
|
|
|
|
async def test_lineage_retains_multiple_parents_and_descendants(app, logged_in, research_input):
|
|
async with app.state.sessions.begin() as db:
|
|
await upsert_alpha(db, alpha("seed"))
|
|
|
|
async def create(parents=None, seeds=None):
|
|
result = await logged_in.post(
|
|
"/api/v1/research/experiments",
|
|
json=expansion(
|
|
research_input["id"], parent_experiment_ids=parents or [], parent_alpha_ids=seeds or []
|
|
),
|
|
)
|
|
assert result.status_code == 201, result.text
|
|
return result.json()["id"]
|
|
|
|
first, second = await create(seeds=["seed"]), await create(seeds=["seed"])
|
|
child = await create([first, second])
|
|
grandchild = await create([child])
|
|
graph = (await logged_in.get("/api/v1/research/lineage?alpha_id=seed")).json()
|
|
assert {r["id"] for r in graph["items"]} == {first, second, child, grandchild}
|
|
assert len(graph["edges"]) == 5 and not graph["truncated"]
|
|
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={grandchild}")).json()
|
|
assert len(graph["items"]) == 4
|
|
|
|
|
|
async def test_evaluate_backtest_keeps_experiment_and_incomplete_evidence(app, logged_in, research_input):
|
|
from sqlalchemy import select
|
|
|
|
from app.models import BacktestResult
|
|
from tests.test_backtests import execute, setup, start
|
|
|
|
experiment = (
|
|
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
|
).json()
|
|
preview = (
|
|
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
|
).json()
|
|
_, lane = await setup(app)
|
|
run = await start(logged_in, preview, "stage2-evidence")
|
|
await execute(app, lane, run["backtest_run_id"])
|
|
async with app.state.sessions.begin() as db:
|
|
result = await db.scalar(select(BacktestResult))
|
|
result.complete = False
|
|
result.snapshot = {
|
|
"is": {"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]}
|
|
}
|
|
response = await logged_in.post(
|
|
"/api/v1/research/evaluations",
|
|
json={"experiment_id": experiment["id"], "backtest_run_id": run["backtest_run_id"]},
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
report = response.json()["report"]
|
|
assert report["experiment"]["inputs"][0]["id"] == research_input["id"]
|
|
incomplete = next(r for r in report["records"] if not r["complete"])
|
|
assert incomplete["verdict"] == "review" and "backtest_error" in incomplete["missing"]
|
|
assert incomplete["settings"] and incomplete["expression"]
|
|
|
|
|
|
async def test_lineage_crosses_saved_alpha_into_next_research(app, logged_in, research_input):
|
|
from tests.test_backtests import execute, setup, start
|
|
|
|
experiment = (
|
|
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
|
).json()
|
|
preview = (
|
|
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
|
).json()
|
|
_, lane = await setup(app)
|
|
run = await start(logged_in, preview, "stage2-alpha-lineage")
|
|
await execute(app, lane, run["backtest_run_id"])
|
|
result = (await logged_in.get(f"/api/v1/backtests/runs/{run['backtest_run_id']}/results")).json()
|
|
aid = result["items"][0]["alpha_id"]
|
|
child = (
|
|
await logged_in.post(
|
|
"/api/v1/research/experiments", json=expansion(research_input["id"], parent_alpha_ids=[aid])
|
|
)
|
|
).json()
|
|
for eid in [experiment["id"], child["id"]]:
|
|
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={eid}")).json()
|
|
assert {r["id"] for r in graph["items"]} == {experiment["id"], child["id"]}
|