feat: persist feature research and versioned alpha assessments
This commit is contained in:
yuxuanhui
@@ -0,0 +1,191 @@
|
||||
"""Stage-two acceptance: immutable plans, views, lineage and evidence-backed decisions."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.alphas import upsert_alpha
|
||||
from app.models import Research
|
||||
from tests.conftest import alpha
|
||||
from tests.test_research_workspace import catalog, expansion, research_input, template
|
||||
|
||||
__all__ = ["catalog", "research_input"]
|
||||
|
||||
|
||||
async def test_feature_conversion_keeps_original_version_through_experiment(logged_in, research_input):
|
||||
content = {
|
||||
"name": "特征方案",
|
||||
"hypothesis": "排序稳定性",
|
||||
"input_ids": [research_input["id"]],
|
||||
"steps": [{"name": "排序", "rationale": "降低异常值影响", "expression": "rank(TEST_FIN_001)"}],
|
||||
"template": template(),
|
||||
}
|
||||
saved = (
|
||||
await logged_in.post("/api/v1/research/assets", json={"kind": "feature", "content": content})
|
||||
).json()
|
||||
converted = await logged_in.post(f"/api/v1/research/features/{saved['id']}/template", json={"version": 1})
|
||||
assert converted.status_code == 201, converted.text
|
||||
result = converted.json()
|
||||
changed = await logged_in.put(
|
||||
f"/api/v1/research/assets/{saved['id']}",
|
||||
json={"kind": "feature", "version": 1, "content": {**content, "hypothesis": "新假设"}},
|
||||
)
|
||||
assert changed.status_code == 200
|
||||
body = expansion(research_input["id"], asset_id=result["id"], version=1)
|
||||
body.pop("template")
|
||||
experiment = (await logged_in.post("/api/v1/research/experiments", json=body)).json()
|
||||
source = experiment["evidence"]["template"]["provenance"]["feature"]
|
||||
assert source["version"] == 1 and source["content"]["hypothesis"] == "排序稳定性"
|
||||
assert source["content"]["input_ids"] == [research_input["id"]]
|
||||
|
||||
|
||||
async def test_saved_views_validate_and_retain_sort_columns(logged_in):
|
||||
content = {
|
||||
"name": "低换手队列",
|
||||
"filters": {
|
||||
"turnover_max": "0.15",
|
||||
"sort": "fitness",
|
||||
"direction": "desc",
|
||||
"submission": "UNSUBMITTED",
|
||||
},
|
||||
"columns": ["name", "fitness"],
|
||||
}
|
||||
response = await logged_in.post("/api/v1/research/assets", json={"kind": "view", "content": content})
|
||||
assert response.status_code == 201, response.text
|
||||
item = response.json()
|
||||
assert item["content"]["filters"]["turnover_max"] == 0.15
|
||||
assert item["content"]["filters"]["sort"] == "fitness"
|
||||
assert item["content"]["columns"] == ["name", "fitness"]
|
||||
bad = await logged_in.post(
|
||||
"/api/v1/research/assets",
|
||||
json={"kind": "view", "content": {**content, "filters": {"sort": "arbitrary_sql"}}},
|
||||
)
|
||||
assert bad.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metrics,verdict,missing",
|
||||
[
|
||||
({"sharpe": None, "fitness": 1, "turnover": 0.1}, "review", "sharpe"),
|
||||
({"sharpe": 2, "fitness": 1, "turnover": 0.1}, "review", "platform_checks"),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "FAIL"}]},
|
||||
"block",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]},
|
||||
"pass",
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_evaluations_keep_missing_and_failures(app, logged_in, metrics, verdict, missing):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": metrics}))
|
||||
response = await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["verdict"] == verdict
|
||||
if missing:
|
||||
assert missing in report["records"][0]["missing"]
|
||||
async with app.state.sessions() as db:
|
||||
assert (await db.get(Research, "seed")).state == "inbox"
|
||||
|
||||
|
||||
async def test_sync_and_model_advice_do_not_rewrite_report(app, logged_in):
|
||||
from app.research.evaluations import Evaluations
|
||||
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": -1}}))
|
||||
original = (await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})).json()
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": 9}}))
|
||||
advised = await Evaluations(db).add_advice(
|
||||
original["id"],
|
||||
{"summary": "建议通过", "risks": [], "suggestions": []},
|
||||
{"model": "fixture", "revision": 1},
|
||||
)
|
||||
assert advised["id"] != original["id"]
|
||||
assert advised["report"]["verdict"] == "block"
|
||||
assert advised["report"]["records"][0]["snapshot"]["is"]["sharpe"] == -1
|
||||
assert (await logged_in.get(f"/api/v1/research/evaluations/{original['id']}")).json() == original
|
||||
|
||||
|
||||
async def test_lineage_retains_multiple_parents_and_descendants(app, logged_in, research_input):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed"))
|
||||
|
||||
async def create(parents=None, seeds=None):
|
||||
result = await logged_in.post(
|
||||
"/api/v1/research/experiments",
|
||||
json=expansion(
|
||||
research_input["id"], parent_experiment_ids=parents or [], parent_alpha_ids=seeds or []
|
||||
),
|
||||
)
|
||||
assert result.status_code == 201, result.text
|
||||
return result.json()["id"]
|
||||
|
||||
first, second = await create(seeds=["seed"]), await create(seeds=["seed"])
|
||||
child = await create([first, second])
|
||||
grandchild = await create([child])
|
||||
graph = (await logged_in.get("/api/v1/research/lineage?alpha_id=seed")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {first, second, child, grandchild}
|
||||
assert len(graph["edges"]) == 5 and not graph["truncated"]
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={grandchild}")).json()
|
||||
assert len(graph["items"]) == 4
|
||||
|
||||
|
||||
async def test_evaluate_backtest_keeps_experiment_and_incomplete_evidence(app, logged_in, research_input):
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import BacktestResult
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-evidence")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
async with app.state.sessions.begin() as db:
|
||||
result = await db.scalar(select(BacktestResult))
|
||||
result.complete = False
|
||||
result.snapshot = {
|
||||
"is": {"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]}
|
||||
}
|
||||
response = await logged_in.post(
|
||||
"/api/v1/research/evaluations",
|
||||
json={"experiment_id": experiment["id"], "backtest_run_id": run["backtest_run_id"]},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["experiment"]["inputs"][0]["id"] == research_input["id"]
|
||||
incomplete = next(r for r in report["records"] if not r["complete"])
|
||||
assert incomplete["verdict"] == "review" and "backtest_error" in incomplete["missing"]
|
||||
assert incomplete["settings"] and incomplete["expression"]
|
||||
|
||||
|
||||
async def test_lineage_crosses_saved_alpha_into_next_research(app, logged_in, research_input):
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-alpha-lineage")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
result = (await logged_in.get(f"/api/v1/backtests/runs/{run['backtest_run_id']}/results")).json()
|
||||
aid = result["items"][0]["alpha_id"]
|
||||
child = (
|
||||
await logged_in.post(
|
||||
"/api/v1/research/experiments", json=expansion(research_input["id"], parent_alpha_ids=[aid])
|
||||
)
|
||||
).json()
|
||||
for eid in [experiment["id"], child["id"]]:
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={eid}")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {experiment["id"], child["id"]}
|
||||
Reference in New Issue
Block a user