feat: persist feature research and versioned alpha assessments
This commit is contained in:
yuxuanhui
@@ -0,0 +1,80 @@
|
||||
"""Stage-two PostgreSQL acceptance in dedicated databases only."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
NAME = "wq_research_stage2_test"
|
||||
RESTORE = "wq_research_restore_stage2"
|
||||
os.environ.update(
|
||||
DATABASE_URL=f"postgresql+asyncpg://postgres:research-test-only@127.0.0.1:18436/{NAME}",
|
||||
ADMIN_PASSWORD="research-acceptance-only",
|
||||
ENCRYPTION_KEY=Fernet.generate_key().decode(),
|
||||
)
|
||||
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
return subprocess.run(["docker", "exec", "-i", "wq-research-acceptance-pg", *args], check=True, **kwargs)
|
||||
|
||||
|
||||
async def acceptance():
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import Settings
|
||||
from app.main import create_app
|
||||
from app.models import ResearchExperiment, ResearchParent, TemplateInput
|
||||
from tests.test_research_outcomes import (
|
||||
test_feature_conversion_keeps_original_version_through_experiment,
|
||||
test_lineage_retains_multiple_parents_and_descendants,
|
||||
test_saved_views_validate_and_retain_sort_columns,
|
||||
test_sync_and_model_advice_do_not_rewrite_report,
|
||||
)
|
||||
|
||||
app = create_app(Settings(_env_file=None, enable_runner=False, public_origin="http://testserver"))
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app), base_url="http://testserver", headers={"X-WQ-Request": "1"}
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/api/v1/auth/login", json={"username": "admin", "password": "research-acceptance-only"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
async with app.state.sessions() as db:
|
||||
fixed = await db.scalar(select(TemplateInput))
|
||||
for experiment in await db.scalars(select(ResearchExperiment)):
|
||||
for parent in experiment.parents:
|
||||
assert await db.get(ResearchParent, (experiment.id, parent["kind"], parent["id"]))
|
||||
await test_feature_conversion_keeps_original_version_through_experiment(client, {"id": fixed.id})
|
||||
await test_saved_views_validate_and_retain_sort_columns(client)
|
||||
await test_sync_and_model_advice_do_not_rewrite_report(app, client)
|
||||
await test_lineage_retains_multiple_parents_and_descendants(app, client, {"id": fixed.id})
|
||||
print("PASS PostgreSQL: feature revisions, saved views, immutable evaluations and multi-parent traversal")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
docker("createdb", "-U", "postgres", NAME)
|
||||
with Path("/tmp/wq-research-stage1.dump").open("rb") as source:
|
||||
docker("pg_restore", "-U", "postgres", "-d", NAME, stdin=source)
|
||||
config = Config("alembic.ini")
|
||||
command.upgrade(config, "0007")
|
||||
command.check(config)
|
||||
asyncio.run(acceptance())
|
||||
dump = Path("/tmp/wq-research-stage2.dump")
|
||||
with dump.open("wb") as output:
|
||||
docker("pg_dump", "-U", "postgres", "-Fc", NAME, stdout=output)
|
||||
docker("createdb", "-U", "postgres", RESTORE)
|
||||
with dump.open("rb") as source:
|
||||
docker("pg_restore", "-U", "postgres", "-d", RESTORE, stdin=source)
|
||||
query = "SELECT (SELECT count(*) FROM research_revisions),(SELECT count(*) FROM research_evaluations),(SELECT count(*) FROM research_parents),(SELECT note FROM research WHERE alpha_id='OLD_RESEARCH')"
|
||||
a = docker("psql", "-U", "postgres", "-d", NAME, "-Atc", query, capture_output=True).stdout
|
||||
b = docker("psql", "-U", "postgres", "-d", RESTORE, "-Atc", query, capture_output=True).stdout
|
||||
assert a == b
|
||||
print(
|
||||
"PASS PostgreSQL 17: 0006 → 0007 and pg_dump/pg_restore preserve history, graph and old research notes"
|
||||
)
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Stage-two acceptance: immutable plans, views, lineage and evidence-backed decisions."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.alphas import upsert_alpha
|
||||
from app.models import Research
|
||||
from tests.conftest import alpha
|
||||
from tests.test_research_workspace import catalog, expansion, research_input, template
|
||||
|
||||
__all__ = ["catalog", "research_input"]
|
||||
|
||||
|
||||
async def test_feature_conversion_keeps_original_version_through_experiment(logged_in, research_input):
|
||||
content = {
|
||||
"name": "特征方案",
|
||||
"hypothesis": "排序稳定性",
|
||||
"input_ids": [research_input["id"]],
|
||||
"steps": [{"name": "排序", "rationale": "降低异常值影响", "expression": "rank(TEST_FIN_001)"}],
|
||||
"template": template(),
|
||||
}
|
||||
saved = (
|
||||
await logged_in.post("/api/v1/research/assets", json={"kind": "feature", "content": content})
|
||||
).json()
|
||||
converted = await logged_in.post(f"/api/v1/research/features/{saved['id']}/template", json={"version": 1})
|
||||
assert converted.status_code == 201, converted.text
|
||||
result = converted.json()
|
||||
changed = await logged_in.put(
|
||||
f"/api/v1/research/assets/{saved['id']}",
|
||||
json={"kind": "feature", "version": 1, "content": {**content, "hypothesis": "新假设"}},
|
||||
)
|
||||
assert changed.status_code == 200
|
||||
body = expansion(research_input["id"], asset_id=result["id"], version=1)
|
||||
body.pop("template")
|
||||
experiment = (await logged_in.post("/api/v1/research/experiments", json=body)).json()
|
||||
source = experiment["evidence"]["template"]["provenance"]["feature"]
|
||||
assert source["version"] == 1 and source["content"]["hypothesis"] == "排序稳定性"
|
||||
assert source["content"]["input_ids"] == [research_input["id"]]
|
||||
|
||||
|
||||
async def test_saved_views_validate_and_retain_sort_columns(logged_in):
|
||||
content = {
|
||||
"name": "低换手队列",
|
||||
"filters": {
|
||||
"turnover_max": "0.15",
|
||||
"sort": "fitness",
|
||||
"direction": "desc",
|
||||
"submission": "UNSUBMITTED",
|
||||
},
|
||||
"columns": ["name", "fitness"],
|
||||
}
|
||||
response = await logged_in.post("/api/v1/research/assets", json={"kind": "view", "content": content})
|
||||
assert response.status_code == 201, response.text
|
||||
item = response.json()
|
||||
assert item["content"]["filters"]["turnover_max"] == 0.15
|
||||
assert item["content"]["filters"]["sort"] == "fitness"
|
||||
assert item["content"]["columns"] == ["name", "fitness"]
|
||||
bad = await logged_in.post(
|
||||
"/api/v1/research/assets",
|
||||
json={"kind": "view", "content": {**content, "filters": {"sort": "arbitrary_sql"}}},
|
||||
)
|
||||
assert bad.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metrics,verdict,missing",
|
||||
[
|
||||
({"sharpe": None, "fitness": 1, "turnover": 0.1}, "review", "sharpe"),
|
||||
({"sharpe": 2, "fitness": 1, "turnover": 0.1}, "review", "platform_checks"),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "FAIL"}]},
|
||||
"block",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]},
|
||||
"pass",
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_evaluations_keep_missing_and_failures(app, logged_in, metrics, verdict, missing):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": metrics}))
|
||||
response = await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["verdict"] == verdict
|
||||
if missing:
|
||||
assert missing in report["records"][0]["missing"]
|
||||
async with app.state.sessions() as db:
|
||||
assert (await db.get(Research, "seed")).state == "inbox"
|
||||
|
||||
|
||||
async def test_sync_and_model_advice_do_not_rewrite_report(app, logged_in):
|
||||
from app.research.evaluations import Evaluations
|
||||
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": -1}}))
|
||||
original = (await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})).json()
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": 9}}))
|
||||
advised = await Evaluations(db).add_advice(
|
||||
original["id"],
|
||||
{"summary": "建议通过", "risks": [], "suggestions": []},
|
||||
{"model": "fixture", "revision": 1},
|
||||
)
|
||||
assert advised["id"] != original["id"]
|
||||
assert advised["report"]["verdict"] == "block"
|
||||
assert advised["report"]["records"][0]["snapshot"]["is"]["sharpe"] == -1
|
||||
assert (await logged_in.get(f"/api/v1/research/evaluations/{original['id']}")).json() == original
|
||||
|
||||
|
||||
async def test_lineage_retains_multiple_parents_and_descendants(app, logged_in, research_input):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed"))
|
||||
|
||||
async def create(parents=None, seeds=None):
|
||||
result = await logged_in.post(
|
||||
"/api/v1/research/experiments",
|
||||
json=expansion(
|
||||
research_input["id"], parent_experiment_ids=parents or [], parent_alpha_ids=seeds or []
|
||||
),
|
||||
)
|
||||
assert result.status_code == 201, result.text
|
||||
return result.json()["id"]
|
||||
|
||||
first, second = await create(seeds=["seed"]), await create(seeds=["seed"])
|
||||
child = await create([first, second])
|
||||
grandchild = await create([child])
|
||||
graph = (await logged_in.get("/api/v1/research/lineage?alpha_id=seed")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {first, second, child, grandchild}
|
||||
assert len(graph["edges"]) == 5 and not graph["truncated"]
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={grandchild}")).json()
|
||||
assert len(graph["items"]) == 4
|
||||
|
||||
|
||||
async def test_evaluate_backtest_keeps_experiment_and_incomplete_evidence(app, logged_in, research_input):
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import BacktestResult
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-evidence")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
async with app.state.sessions.begin() as db:
|
||||
result = await db.scalar(select(BacktestResult))
|
||||
result.complete = False
|
||||
result.snapshot = {
|
||||
"is": {"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]}
|
||||
}
|
||||
response = await logged_in.post(
|
||||
"/api/v1/research/evaluations",
|
||||
json={"experiment_id": experiment["id"], "backtest_run_id": run["backtest_run_id"]},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["experiment"]["inputs"][0]["id"] == research_input["id"]
|
||||
incomplete = next(r for r in report["records"] if not r["complete"])
|
||||
assert incomplete["verdict"] == "review" and "backtest_error" in incomplete["missing"]
|
||||
assert incomplete["settings"] and incomplete["expression"]
|
||||
|
||||
|
||||
async def test_lineage_crosses_saved_alpha_into_next_research(app, logged_in, research_input):
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-alpha-lineage")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
result = (await logged_in.get(f"/api/v1/backtests/runs/{run['backtest_run_id']}/results")).json()
|
||||
aid = result["items"][0]["alpha_id"]
|
||||
child = (
|
||||
await logged_in.post(
|
||||
"/api/v1/research/experiments", json=expansion(research_input["id"], parent_alpha_ids=[aid])
|
||||
)
|
||||
).json()
|
||||
for eid in [experiment["id"], child["id"]]:
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={eid}")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {experiment["id"], child["id"]}
|
||||
Reference in New Issue
Block a user