feat: persist feature research and versioned alpha assessments
This commit is contained in:
@@ -36,7 +36,7 @@ class ModelSettingsInput(Contract):
|
||||
|
||||
|
||||
class PageContext(Contract):
|
||||
page: Literal["alphas", "account", "datasets", "backtests", "operators", "templates", "variants"] = "alphas"
|
||||
page: Literal["alphas", "account", "datasets", "backtests", "operators", "templates", "features", "variants"] = "alphas"
|
||||
research_asset_id: str | None = Field(default=None, max_length=36)
|
||||
research_experiment_id: str | None = Field(default=None, max_length=36)
|
||||
catalog_scope: Scope | None = None
|
||||
|
||||
@@ -348,6 +348,7 @@ class CatalogDataset(Base):
|
||||
|
||||
class CatalogEntry(Base):
|
||||
"""Immutable published snapshots; staging rows remain invisible until batch completion."""
|
||||
|
||||
__tablename__ = "catalog_entries"
|
||||
batch_id: Mapped[str] = mapped_column(ForeignKey("catalog_batches.id"), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(200), primary_key=True)
|
||||
@@ -389,6 +390,7 @@ class TemplateInput(Base):
|
||||
|
||||
class CatalogResource(Base):
|
||||
"""Read-only upstream metadata snapshots; local annotations live separately."""
|
||||
|
||||
__tablename__ = "catalog_resources"
|
||||
key: Mapped[str] = mapped_column(String(250), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(30), index=True)
|
||||
@@ -406,6 +408,7 @@ class OperatorNote(Base):
|
||||
|
||||
class ResearchAsset(Base):
|
||||
"""Stable identity for typed templates, feature plans, views and workflow definitions."""
|
||||
|
||||
__tablename__ = "research_assets"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(30), index=True)
|
||||
@@ -426,6 +429,7 @@ class ResearchRevision(Base):
|
||||
|
||||
class ResearchExperiment(Base):
|
||||
"""Immutable generated candidates; never masquerade as platform Alpha records."""
|
||||
|
||||
__tablename__ = "research_experiments"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
@@ -436,3 +440,21 @@ class ResearchExperiment(Base):
|
||||
candidates: Mapped[list] = mapped_column(JSON)
|
||||
evidence: Mapped[dict] = mapped_column(JSON)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
|
||||
|
||||
|
||||
class ResearchEvaluation(Base):
|
||||
__tablename__ = "research_evaluations"
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
alpha_id: Mapped[str | None] = mapped_column(ForeignKey("alphas.id"), index=True)
|
||||
experiment_id: Mapped[str | None] = mapped_column(ForeignKey("research_experiments.id"), index=True)
|
||||
report: Mapped[dict] = mapped_column(JSON)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
|
||||
|
||||
|
||||
class ResearchParent(Base):
|
||||
"""Queryable edges; parent snapshots remain authoritative in the experiment."""
|
||||
|
||||
__tablename__ = "research_parents"
|
||||
child_id: Mapped[str] = mapped_column(ForeignKey("research_experiments.id"), primary_key=True)
|
||||
parent_kind: Mapped[str] = mapped_column(String(30), primary_key=True)
|
||||
parent_id: Mapped[str] = mapped_column(String(100), primary_key=True, index=True)
|
||||
|
||||
@@ -65,8 +65,8 @@ class Assets:
|
||||
if body.kind == "feature":
|
||||
from ..catalog.service import Catalog
|
||||
|
||||
for input_id in content["input_ids"]:
|
||||
await Catalog(self.db).input(input_id)
|
||||
snapshots = [await Catalog(self.db).input(input_id) for input_id in content["input_ids"]]
|
||||
provenance = {**(provenance or {}), "inputs": snapshots}
|
||||
if asset_id:
|
||||
if body.version is None:
|
||||
raise HTTPException(422, "更新需要素材版本")
|
||||
@@ -87,7 +87,12 @@ class Assets:
|
||||
self.db.add(ResearchAsset(id=asset_id, kind=body.kind, name=content["name"], version=version))
|
||||
await self.db.flush()
|
||||
self.db.add(
|
||||
ResearchRevision(asset_id=asset_id, version=version, content=content, provenance=provenance or {})
|
||||
ResearchRevision(
|
||||
asset_id=asset_id,
|
||||
version=version,
|
||||
content=content,
|
||||
provenance=jsonable_encoder(provenance or {}),
|
||||
)
|
||||
)
|
||||
await self.db.flush()
|
||||
return await self.get(asset_id, version)
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Snapshot-based research assessment. Model advice cannot replace deterministic findings."""
|
||||
|
||||
import math
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..backtests.service import Backtests, uid
|
||||
from ..models import Alpha, ResearchEvaluation, ResearchExperiment, SelfCorrelation
|
||||
from .experiments import Experiments
|
||||
from .serialization import encode_snapshot as jsonable_encoder
|
||||
|
||||
|
||||
def assess(snapshot, rules):
|
||||
metrics = snapshot.get("is") or {}
|
||||
evidence, missing, failed = [], [], []
|
||||
for key, bound, direction in (
|
||||
("sharpe", rules.sharpe_min, "min"),
|
||||
("fitness", rules.fitness_min, "min"),
|
||||
("turnover", rules.turnover_max, "max"),
|
||||
):
|
||||
value = metrics.get(key)
|
||||
if type(value) not in (int, float) or not math.isfinite(value):
|
||||
missing.append(key)
|
||||
value, status = None, "unknown"
|
||||
else:
|
||||
status = "pass" if (value >= bound if direction == "min" else value <= bound) else "block"
|
||||
if status == "block":
|
||||
failed.append(key)
|
||||
evidence.append(
|
||||
{"metric": key, "value": value, "bound": bound, "direction": direction, "status": status}
|
||||
)
|
||||
checks = metrics.get("checks") or []
|
||||
if not checks:
|
||||
missing.append("platform_checks")
|
||||
for check in checks:
|
||||
if check.get("result") == "FAIL":
|
||||
failed.append(f"platform:{check.get('name', 'unknown')}")
|
||||
unknown_checks = [check for check in checks if check.get("result") not in ("PASS", "FAIL")]
|
||||
if unknown_checks:
|
||||
missing.append("unresolved_platform_checks")
|
||||
return {
|
||||
"verdict": "block" if failed else "review" if missing else "pass",
|
||||
"evidence": evidence,
|
||||
"failed": failed,
|
||||
"missing": missing,
|
||||
"existing_platform_checks": checks,
|
||||
"meaning": "本地研究筛选结果,不是官方提交资格",
|
||||
}
|
||||
|
||||
|
||||
class Evaluations:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def create(self, body):
|
||||
records = []
|
||||
if body.alpha_id:
|
||||
alpha = await self.db.get(Alpha, body.alpha_id)
|
||||
if not alpha:
|
||||
raise HTTPException(404, "Alpha 尚未同步")
|
||||
records.append(
|
||||
{
|
||||
"alpha_id": alpha.id,
|
||||
"snapshot": alpha.raw,
|
||||
"observed_at": alpha.synced_at,
|
||||
"client_item_id": None,
|
||||
}
|
||||
)
|
||||
else:
|
||||
run = await Backtests(self.db).run(body.backtest_run_id)
|
||||
if body.experiment_id and run["source"].get("research_id") != body.experiment_id:
|
||||
raise HTTPException(422, "回测运行不属于指定研究实验")
|
||||
for offset in range(0, run["total"], 100):
|
||||
page = await Backtests(self.db).results(body.backtest_run_id, 100, offset)
|
||||
for item in page["items"]:
|
||||
records.append(
|
||||
{
|
||||
"alpha_id": item["alpha_id"],
|
||||
"snapshot": item["result"]["snapshot"] if item["result"] else {},
|
||||
"observed_at": item["result"]["observed_at"] if item["result"] else None,
|
||||
"client_item_id": item["client_item_id"],
|
||||
"item_id": item["id"],
|
||||
"error": item["error"],
|
||||
"complete": bool(item["result"] and item["result"]["complete"]),
|
||||
"expression": item["expression"],
|
||||
"settings": item["settings"],
|
||||
}
|
||||
)
|
||||
if body.experiment_id and not await self.db.get(ResearchExperiment, body.experiment_id):
|
||||
raise HTTPException(404, "研究实验不存在")
|
||||
findings = []
|
||||
for record in records:
|
||||
correlation = (
|
||||
await self.db.get(SelfCorrelation, record["alpha_id"]) if record["alpha_id"] else None
|
||||
)
|
||||
finding = assess(record["snapshot"], body.rules)
|
||||
if record.get("error") or record.get("complete") is False:
|
||||
finding["missing"].append("backtest_error")
|
||||
if finding["verdict"] == "pass":
|
||||
finding["verdict"] = "review"
|
||||
findings.append(
|
||||
{
|
||||
**record,
|
||||
**finding,
|
||||
"local_correlation": {
|
||||
"result": correlation.result,
|
||||
"stale": correlation.stale,
|
||||
"calculated_at": correlation.calculated_at,
|
||||
}
|
||||
if correlation
|
||||
else None,
|
||||
}
|
||||
)
|
||||
report = jsonable_encoder(
|
||||
{
|
||||
"rules": body.rules.model_dump(),
|
||||
"records": findings,
|
||||
"backtest_run_id": body.backtest_run_id,
|
||||
"experiment": await Experiments(self.db).get(body.experiment_id)
|
||||
if body.experiment_id
|
||||
else None,
|
||||
"model_advice": None,
|
||||
"verdict": "block"
|
||||
if any(r["verdict"] == "block" for r in findings)
|
||||
else "review"
|
||||
if not findings or any(r["verdict"] == "review" for r in findings)
|
||||
else "pass",
|
||||
}
|
||||
)
|
||||
row = ResearchEvaluation(
|
||||
id=uid(), alpha_id=body.alpha_id, experiment_id=body.experiment_id, report=report
|
||||
)
|
||||
self.db.add(row)
|
||||
await self.db.flush()
|
||||
return await self.get(row.id)
|
||||
|
||||
async def get(self, evaluation_id):
|
||||
row = await self.db.get(ResearchEvaluation, evaluation_id)
|
||||
if not row:
|
||||
raise HTTPException(404, "评估报告不存在")
|
||||
return jsonable_encoder(
|
||||
{key: getattr(row, key) for key in ("id", "alpha_id", "experiment_id", "report", "created_at")}
|
||||
)
|
||||
|
||||
async def list(self, alpha_id=None, experiment_id=None, limit=25, offset=0):
|
||||
query = select(ResearchEvaluation)
|
||||
if alpha_id:
|
||||
query = query.where(ResearchEvaluation.alpha_id == alpha_id)
|
||||
if experiment_id:
|
||||
query = query.where(ResearchEvaluation.experiment_id == experiment_id)
|
||||
rows = await self.db.scalars(
|
||||
query.order_by(ResearchEvaluation.created_at.desc()).limit(limit).offset(offset)
|
||||
)
|
||||
return {"items": [await self.get(row.id) for row in rows]}
|
||||
|
||||
async def add_advice(self, evaluation_id, advice, model_evidence):
|
||||
original = await self.get(evaluation_id)
|
||||
report = {
|
||||
**original["report"],
|
||||
"model_advice": advice,
|
||||
"model_evidence": model_evidence,
|
||||
"previous_evaluation_id": evaluation_id,
|
||||
}
|
||||
row = ResearchEvaluation(
|
||||
id=uid(), alpha_id=original["alpha_id"], experiment_id=original["experiment_id"], report=report
|
||||
)
|
||||
self.db.add(row)
|
||||
await self.db.flush()
|
||||
return await self.get(row.id)
|
||||
@@ -204,6 +204,11 @@ class Experiments:
|
||||
)
|
||||
self.db.add(row)
|
||||
await self.db.flush()
|
||||
from ..models import ResearchParent
|
||||
|
||||
for parent_kind, parent_id in {(p["kind"], p["id"]) for p in parents}:
|
||||
self.db.add(ResearchParent(child_id=row.id, parent_kind=parent_kind, parent_id=parent_id))
|
||||
await self.db.flush()
|
||||
return await self.get(row.id)
|
||||
|
||||
async def get(self, experiment_id):
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Feature plans remain versioned inputs to the shared template research path."""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from .assets import Assets
|
||||
from .workspace_contracts import AssetWrite
|
||||
|
||||
|
||||
class Features:
|
||||
def __init__(self, db):
|
||||
self.assets = Assets(db)
|
||||
|
||||
async def to_template(self, asset_id, version):
|
||||
feature = await self.assets.get(asset_id, version, "feature")
|
||||
if not feature["content"].get("template"):
|
||||
raise HTTPException(422, "特征方案尚未定义输出模板")
|
||||
return await self.assets.save(
|
||||
AssetWrite(kind="template", content=feature["content"]["template"]),
|
||||
provenance={"feature": feature},
|
||||
)
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Bounded graph traversal with explicit continuation, retaining every result source."""
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from ..models import Alpha, BacktestResult, BacktestRun, ResearchExperiment, ResearchParent
|
||||
from .experiments import Experiments
|
||||
from .provenance import alpha_sources, saved_sources
|
||||
|
||||
|
||||
async def lineage(db, alpha_id=None, experiment_id=None, limit=25, offset=0):
|
||||
if bool(alpha_id) == bool(experiment_id):
|
||||
raise HTTPException(422, "指定 Alpha 或实验之一")
|
||||
sources = None
|
||||
frontier = set()
|
||||
truncated = False
|
||||
if alpha_id:
|
||||
if not await db.get(Alpha, alpha_id):
|
||||
raise HTTPException(404, "Alpha 尚未同步")
|
||||
sources = await alpha_sources(db, alpha_id, limit, offset)
|
||||
source_ids = list(
|
||||
await db.scalars(
|
||||
saved_sources()
|
||||
.with_only_columns(BacktestRun.source["research_id"].as_string())
|
||||
.where(BacktestResult.alpha_id == alpha_id)
|
||||
.distinct()
|
||||
.limit(101)
|
||||
)
|
||||
)
|
||||
children = list(
|
||||
await db.scalars(
|
||||
select(ResearchParent.child_id)
|
||||
.where(ResearchParent.parent_kind == "alpha", ResearchParent.parent_id == alpha_id)
|
||||
.order_by(ResearchParent.child_id)
|
||||
.limit(101)
|
||||
)
|
||||
)
|
||||
frontier = set(source_ids + children) - {None}
|
||||
truncated = len(frontier) > 100
|
||||
else:
|
||||
await Experiments(db).get(experiment_id)
|
||||
frontier.add(experiment_id)
|
||||
found, edges = {}, {}
|
||||
for _ in range(8):
|
||||
wanted = sorted(frontier - set(found))
|
||||
if not wanted:
|
||||
break
|
||||
remaining = 100 - len(found)
|
||||
if len(wanted) > remaining:
|
||||
truncated = True
|
||||
wanted = wanted[:remaining]
|
||||
if not wanted:
|
||||
break
|
||||
rows = list(await db.scalars(select(ResearchExperiment).where(ResearchExperiment.id.in_(wanted))))
|
||||
for row in rows:
|
||||
found[row.id] = await Experiments(db).get(row.id)
|
||||
produced = list(
|
||||
await db.scalars(
|
||||
saved_sources()
|
||||
.with_only_columns(BacktestResult.alpha_id)
|
||||
.where(BacktestRun.source["research_id"].as_string().in_(wanted))
|
||||
.distinct()
|
||||
.limit(1001)
|
||||
)
|
||||
)
|
||||
truncated |= len(produced) > 1000
|
||||
parent_alphas = {p["id"] for row in rows for p in row.parents if p["kind"] == "alpha"}
|
||||
related_sources = list(
|
||||
await db.scalars(
|
||||
saved_sources()
|
||||
.with_only_columns(BacktestRun.source["research_id"].as_string())
|
||||
.where(BacktestResult.alpha_id.in_(parent_alphas | set(produced[:1000])))
|
||||
.distinct()
|
||||
.limit(101)
|
||||
)
|
||||
)
|
||||
truncated |= len(related_sources) > 100
|
||||
relations = list(
|
||||
await db.scalars(
|
||||
select(ResearchParent)
|
||||
.where(
|
||||
or_(
|
||||
ResearchParent.child_id.in_(wanted),
|
||||
(ResearchParent.parent_kind == "alpha")
|
||||
& ResearchParent.parent_id.in_(produced[:1000]),
|
||||
(ResearchParent.parent_kind == "experiment") & ResearchParent.parent_id.in_(wanted),
|
||||
)
|
||||
)
|
||||
.order_by(ResearchParent.child_id, ResearchParent.parent_kind, ResearchParent.parent_id)
|
||||
.limit(1001)
|
||||
)
|
||||
)
|
||||
truncated |= len(relations) > 1000
|
||||
frontier = set(related_sources[:100]) - {None}
|
||||
for edge in relations[:1000]:
|
||||
edges[(edge.child_id, edge.parent_kind, edge.parent_id)] = {
|
||||
"child_id": edge.child_id,
|
||||
"parent_kind": edge.parent_kind,
|
||||
"parent_id": edge.parent_id,
|
||||
}
|
||||
frontier.add(edge.child_id)
|
||||
if edge.parent_kind == "experiment":
|
||||
frontier.add(edge.parent_id)
|
||||
unresolved = sorted(frontier - set(found))
|
||||
return {
|
||||
"items": list(found.values()),
|
||||
"edges": list(edges.values()),
|
||||
"sources": sources,
|
||||
"truncated": truncated or bool(unresolved),
|
||||
"unresolved_experiment_ids": unresolved,
|
||||
"limit": 100,
|
||||
"max_depth": 8,
|
||||
}
|
||||
@@ -5,13 +5,17 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from ..security import require_auth
|
||||
from .assets import Assets
|
||||
from .comparisons import compare
|
||||
from .evaluations import Evaluations
|
||||
from .experiments import Experiments
|
||||
from .model import request_model
|
||||
from .features import Features
|
||||
from .model import OUTPUTS, Advice, request_model
|
||||
from .workspace_contracts import (
|
||||
AssetWrite,
|
||||
CompareInput,
|
||||
EvaluateInput,
|
||||
Expansion,
|
||||
ExperimentPreview,
|
||||
FeatureConversion,
|
||||
Generation,
|
||||
ImportCommit,
|
||||
ImportPreview,
|
||||
@@ -35,7 +39,7 @@ async def assets(
|
||||
limit: int = Query(25, ge=1, le=100),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
if kind != "template":
|
||||
if kind not in ("template", "feature", "view"):
|
||||
raise HTTPException(422, "当前素材类型尚未开放")
|
||||
async with request.app.state.sessions() as db:
|
||||
return await Assets(db).list(kind, q, limit, offset)
|
||||
@@ -43,7 +47,7 @@ async def assets(
|
||||
|
||||
@router.post("/assets", status_code=201)
|
||||
async def save_asset(body: AssetWrite, request: Request):
|
||||
if body.kind != "template":
|
||||
if body.kind not in ("template", "feature", "view"):
|
||||
raise HTTPException(422, "当前素材类型尚未开放")
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
return await Assets(db).save(body)
|
||||
@@ -57,7 +61,7 @@ async def asset(asset_id: str, request: Request, version: int | None = Query(Non
|
||||
|
||||
@router.put("/assets/{asset_id}")
|
||||
async def update_asset(asset_id: str, body: AssetWrite, request: Request):
|
||||
if body.kind != "template":
|
||||
if body.kind not in ("template", "feature", "view"):
|
||||
raise HTTPException(422, "当前素材类型尚未开放")
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
return await Assets(db).save(body, asset_id)
|
||||
@@ -89,14 +93,17 @@ async def import_commit(body: ImportCommit, request: Request):
|
||||
|
||||
@router.post("/generate", status_code=201)
|
||||
async def generate(body: Generation, request: Request):
|
||||
if body.method == "feature":
|
||||
raise HTTPException(422, "特征方案生成将在特征工程阶段开放")
|
||||
async with request.app.state.sessions() as db:
|
||||
context = await Experiments(db).generation_context(body)
|
||||
result, evidence = await request_model(request.app.state.ai, context)
|
||||
result, evidence = await request_model(request.app.state.ai, context, OUTPUTS[body.method])
|
||||
if body.method == "feature" and set(result.input_ids) != set(body.input_ids):
|
||||
raise HTTPException(422, "模型不能改变已固定的输入范围")
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
asset = await Assets(db).save(
|
||||
AssetWrite(kind="template", content=result.model_dump(mode="json")),
|
||||
AssetWrite(
|
||||
kind="feature" if body.method == "feature" else "template",
|
||||
content=result.model_dump(mode="json"),
|
||||
),
|
||||
provenance={"generation": evidence, "context": context},
|
||||
)
|
||||
return {
|
||||
@@ -149,3 +156,72 @@ async def settings_variants(body: SettingVariants, request: Request):
|
||||
async def comparison(body: CompareInput, request: Request):
|
||||
async with request.app.state.sessions() as db:
|
||||
return await compare(db, body.alpha_ids)
|
||||
|
||||
|
||||
@router.post("/features/{asset_id}/template", status_code=201)
|
||||
async def feature_template(asset_id: str, body: FeatureConversion, request: Request):
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
return await Features(db).to_template(asset_id, body.version)
|
||||
|
||||
|
||||
@router.post("/evaluations", status_code=201)
|
||||
async def evaluate(body: EvaluateInput, request: Request):
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
return await Evaluations(db).create(body)
|
||||
|
||||
|
||||
@router.get("/evaluations")
|
||||
async def evaluations(
|
||||
request: Request,
|
||||
alpha_id: str | None = None,
|
||||
experiment_id: str | None = None,
|
||||
limit: int = Query(25, ge=1, le=100),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
async with request.app.state.sessions() as db:
|
||||
return await Evaluations(db).list(alpha_id, experiment_id, limit, offset)
|
||||
|
||||
|
||||
@router.get("/evaluations/{evaluation_id}")
|
||||
async def evaluation(evaluation_id: str, request: Request):
|
||||
async with request.app.state.sessions() as db:
|
||||
return await Evaluations(db).get(evaluation_id)
|
||||
|
||||
|
||||
@router.post("/evaluations/{evaluation_id}/advice", status_code=201)
|
||||
async def evaluation_advice(evaluation_id: str, request: Request):
|
||||
async with request.app.state.sessions() as db:
|
||||
original = await Evaluations(db).get(evaluation_id)
|
||||
report = original["report"]
|
||||
context = {
|
||||
"evaluation_id": evaluation_id,
|
||||
"rules": report["rules"],
|
||||
"verdict": report["verdict"],
|
||||
"total_records": len(report["records"]),
|
||||
"records": [
|
||||
{
|
||||
key: row.get(key)
|
||||
for key in ("alpha_id", "client_item_id", "evidence", "missing", "failed", "verdict")
|
||||
}
|
||||
for row in report["records"][:20]
|
||||
],
|
||||
"record_limit": 20,
|
||||
}
|
||||
result, evidence = await request_model(request.app.state.ai, context, Advice)
|
||||
evidence["context"] = context
|
||||
async with request.app.state.sessions.begin() as db:
|
||||
return await Evaluations(db).add_advice(evaluation_id, result.model_dump(mode="json"), evidence)
|
||||
|
||||
|
||||
@router.get("/lineage")
|
||||
async def research_lineage(
|
||||
request: Request,
|
||||
alpha_id: str | None = None,
|
||||
experiment_id: str | None = None,
|
||||
limit: int = Query(25, ge=1, le=100),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
from .lineage import lineage
|
||||
|
||||
async with request.app.state.sessions() as db:
|
||||
return await lineage(db, alpha_id, experiment_id, limit, offset)
|
||||
|
||||
@@ -75,6 +75,10 @@ class FeatureSpec(Contract):
|
||||
template: TemplateSpec | None = None
|
||||
|
||||
|
||||
class FeatureConversion(Contract):
|
||||
version: int = Field(ge=1)
|
||||
|
||||
|
||||
class ViewSpec(Contract):
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
filters: dict = Field(default_factory=dict)
|
||||
@@ -85,8 +89,7 @@ class ViewSpec(Contract):
|
||||
def valid_filters(cls, value):
|
||||
from ..schemas import AlphaFilters
|
||||
|
||||
AlphaFilters.model_validate(value)
|
||||
return value
|
||||
return AlphaFilters.model_validate(value).model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
class AssetWrite(Contract):
|
||||
@@ -151,6 +154,8 @@ class EvaluateInput(Contract):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def target(self):
|
||||
if self.alpha_id and self.experiment_id:
|
||||
raise ValueError("实验评估应选择关联回测运行;Alpha 评估单独保存")
|
||||
if bool(self.alpha_id) == bool(self.backtest_run_id):
|
||||
raise ValueError("选择 Alpha 或回测运行之一")
|
||||
return self
|
||||
|
||||
@@ -6,8 +6,10 @@ from ..ai.capabilities import Capability
|
||||
from ..catalog.research_metadata import ResearchMetadata
|
||||
from ..schemas import Contract
|
||||
from .assets import Assets
|
||||
from .evaluations import Evaluations
|
||||
from .experiments import Experiments
|
||||
from .workspace_contracts import Expansion, SettingVariants
|
||||
from .features import Features
|
||||
from .workspace_contracts import AssetWrite, EvaluateInput, Expansion, FeatureSpec, SettingVariants
|
||||
|
||||
|
||||
class AssetQuery(Contract):
|
||||
@@ -21,6 +23,16 @@ class AssetReference(Contract):
|
||||
version: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class FixedAssetReference(AssetReference):
|
||||
version: int = Field(ge=1)
|
||||
|
||||
|
||||
class FeatureWrite(Contract):
|
||||
content: FeatureSpec
|
||||
asset_id: str | None = Field(default=None, max_length=36)
|
||||
version: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
class ExperimentReference(Contract):
|
||||
experiment_id: str = Field(min_length=1, max_length=36)
|
||||
|
||||
@@ -105,3 +117,46 @@ CAPABILITIES = (
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
CAPABILITIES += (
|
||||
Capability(
|
||||
name="search_feature_plans",
|
||||
schema=AssetQuery,
|
||||
description="搜索版本化特征方案。",
|
||||
label="搜索特征方案",
|
||||
renderer="research",
|
||||
effect="query",
|
||||
handler=lambda ctx, args: Assets(ctx.business.db).list("feature", **args.model_dump()),
|
||||
),
|
||||
Capability(
|
||||
name="save_feature_plan",
|
||||
schema=FeatureWrite,
|
||||
description="保存固定输入、处理步骤和经济假设,不执行计算或回测。",
|
||||
label="保存特征方案",
|
||||
renderer="research",
|
||||
effect="prepare",
|
||||
handler=lambda ctx, args: Assets(ctx.business.db).save(
|
||||
AssetWrite(kind="feature", content=args.content.model_dump(mode="json"), version=args.version),
|
||||
args.asset_id,
|
||||
),
|
||||
),
|
||||
Capability(
|
||||
name="feature_to_template",
|
||||
schema=FixedAssetReference,
|
||||
description="将指定特征方案版本转为新模板,保留来源。",
|
||||
label="特征转为模板",
|
||||
renderer="research",
|
||||
effect="prepare",
|
||||
handler=lambda ctx, args: Features(ctx.business.db).to_template(args.asset_id, args.version),
|
||||
),
|
||||
Capability(
|
||||
name="evaluate_research",
|
||||
schema=EvaluateInput,
|
||||
description="保存版本化规则评估,缺失数据不会作为通过,不修改研究状态。",
|
||||
label="保存规则评估",
|
||||
renderer="research",
|
||||
effect="prepare",
|
||||
handler=lambda ctx, args: Evaluations(ctx.business.db).create(args),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Stage two: immutable evaluations; features and saved views reuse asset revisions."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0007"
|
||||
down_revision = "0006"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
edges = op.create_table(
|
||||
"research_parents",
|
||||
sa.Column("child_id", sa.String(36), sa.ForeignKey("research_experiments.id"), primary_key=True),
|
||||
sa.Column("parent_kind", sa.String(30), primary_key=True),
|
||||
sa.Column("parent_id", sa.String(100), primary_key=True),
|
||||
)
|
||||
op.create_index("ix_research_parents_parent_id", "research_parents", ["parent_id"])
|
||||
experiments = sa.table("research_experiments", sa.column("id"), sa.column("parents", sa.JSON()))
|
||||
for row in op.get_bind().execute(sa.select(experiments)).mappings():
|
||||
values = [
|
||||
{"child_id": row["id"], "parent_kind": kind, "parent_id": parent_id}
|
||||
for kind, parent_id in {(p["kind"], p["id"]) for p in row["parents"]}
|
||||
]
|
||||
if values:
|
||||
op.bulk_insert(edges, values)
|
||||
op.create_table(
|
||||
"research_evaluations",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("alpha_id", sa.String(100), sa.ForeignKey("alphas.id"), nullable=True),
|
||||
sa.Column("experiment_id", sa.String(36), sa.ForeignKey("research_experiments.id"), nullable=True),
|
||||
sa.Column("report", sa.JSON(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_research_evaluations_alpha_id", "research_evaluations", ["alpha_id"])
|
||||
op.create_index("ix_research_evaluations_experiment_id", "research_evaluations", ["experiment_id"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("research_evaluations")
|
||||
op.drop_table("research_parents")
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Stage-two PostgreSQL acceptance in dedicated databases only."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
NAME = "wq_research_stage2_test"
|
||||
RESTORE = "wq_research_restore_stage2"
|
||||
os.environ.update(
|
||||
DATABASE_URL=f"postgresql+asyncpg://postgres:research-test-only@127.0.0.1:18436/{NAME}",
|
||||
ADMIN_PASSWORD="research-acceptance-only",
|
||||
ENCRYPTION_KEY=Fernet.generate_key().decode(),
|
||||
)
|
||||
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
return subprocess.run(["docker", "exec", "-i", "wq-research-acceptance-pg", *args], check=True, **kwargs)
|
||||
|
||||
|
||||
async def acceptance():
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import Settings
|
||||
from app.main import create_app
|
||||
from app.models import ResearchExperiment, ResearchParent, TemplateInput
|
||||
from tests.test_research_outcomes import (
|
||||
test_feature_conversion_keeps_original_version_through_experiment,
|
||||
test_lineage_retains_multiple_parents_and_descendants,
|
||||
test_saved_views_validate_and_retain_sort_columns,
|
||||
test_sync_and_model_advice_do_not_rewrite_report,
|
||||
)
|
||||
|
||||
app = create_app(Settings(_env_file=None, enable_runner=False, public_origin="http://testserver"))
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app), base_url="http://testserver", headers={"X-WQ-Request": "1"}
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/api/v1/auth/login", json={"username": "admin", "password": "research-acceptance-only"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
async with app.state.sessions() as db:
|
||||
fixed = await db.scalar(select(TemplateInput))
|
||||
for experiment in await db.scalars(select(ResearchExperiment)):
|
||||
for parent in experiment.parents:
|
||||
assert await db.get(ResearchParent, (experiment.id, parent["kind"], parent["id"]))
|
||||
await test_feature_conversion_keeps_original_version_through_experiment(client, {"id": fixed.id})
|
||||
await test_saved_views_validate_and_retain_sort_columns(client)
|
||||
await test_sync_and_model_advice_do_not_rewrite_report(app, client)
|
||||
await test_lineage_retains_multiple_parents_and_descendants(app, client, {"id": fixed.id})
|
||||
print("PASS PostgreSQL: feature revisions, saved views, immutable evaluations and multi-parent traversal")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
docker("createdb", "-U", "postgres", NAME)
|
||||
with Path("/tmp/wq-research-stage1.dump").open("rb") as source:
|
||||
docker("pg_restore", "-U", "postgres", "-d", NAME, stdin=source)
|
||||
config = Config("alembic.ini")
|
||||
command.upgrade(config, "0007")
|
||||
command.check(config)
|
||||
asyncio.run(acceptance())
|
||||
dump = Path("/tmp/wq-research-stage2.dump")
|
||||
with dump.open("wb") as output:
|
||||
docker("pg_dump", "-U", "postgres", "-Fc", NAME, stdout=output)
|
||||
docker("createdb", "-U", "postgres", RESTORE)
|
||||
with dump.open("rb") as source:
|
||||
docker("pg_restore", "-U", "postgres", "-d", RESTORE, stdin=source)
|
||||
query = "SELECT (SELECT count(*) FROM research_revisions),(SELECT count(*) FROM research_evaluations),(SELECT count(*) FROM research_parents),(SELECT note FROM research WHERE alpha_id='OLD_RESEARCH')"
|
||||
a = docker("psql", "-U", "postgres", "-d", NAME, "-Atc", query, capture_output=True).stdout
|
||||
b = docker("psql", "-U", "postgres", "-d", RESTORE, "-Atc", query, capture_output=True).stdout
|
||||
assert a == b
|
||||
print(
|
||||
"PASS PostgreSQL 17: 0006 → 0007 and pg_dump/pg_restore preserve history, graph and old research notes"
|
||||
)
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Stage-two acceptance: immutable plans, views, lineage and evidence-backed decisions."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.alphas import upsert_alpha
|
||||
from app.models import Research
|
||||
from tests.conftest import alpha
|
||||
from tests.test_research_workspace import catalog, expansion, research_input, template
|
||||
|
||||
__all__ = ["catalog", "research_input"]
|
||||
|
||||
|
||||
async def test_feature_conversion_keeps_original_version_through_experiment(logged_in, research_input):
|
||||
content = {
|
||||
"name": "特征方案",
|
||||
"hypothesis": "排序稳定性",
|
||||
"input_ids": [research_input["id"]],
|
||||
"steps": [{"name": "排序", "rationale": "降低异常值影响", "expression": "rank(TEST_FIN_001)"}],
|
||||
"template": template(),
|
||||
}
|
||||
saved = (
|
||||
await logged_in.post("/api/v1/research/assets", json={"kind": "feature", "content": content})
|
||||
).json()
|
||||
converted = await logged_in.post(f"/api/v1/research/features/{saved['id']}/template", json={"version": 1})
|
||||
assert converted.status_code == 201, converted.text
|
||||
result = converted.json()
|
||||
changed = await logged_in.put(
|
||||
f"/api/v1/research/assets/{saved['id']}",
|
||||
json={"kind": "feature", "version": 1, "content": {**content, "hypothesis": "新假设"}},
|
||||
)
|
||||
assert changed.status_code == 200
|
||||
body = expansion(research_input["id"], asset_id=result["id"], version=1)
|
||||
body.pop("template")
|
||||
experiment = (await logged_in.post("/api/v1/research/experiments", json=body)).json()
|
||||
source = experiment["evidence"]["template"]["provenance"]["feature"]
|
||||
assert source["version"] == 1 and source["content"]["hypothesis"] == "排序稳定性"
|
||||
assert source["content"]["input_ids"] == [research_input["id"]]
|
||||
|
||||
|
||||
async def test_saved_views_validate_and_retain_sort_columns(logged_in):
|
||||
content = {
|
||||
"name": "低换手队列",
|
||||
"filters": {
|
||||
"turnover_max": "0.15",
|
||||
"sort": "fitness",
|
||||
"direction": "desc",
|
||||
"submission": "UNSUBMITTED",
|
||||
},
|
||||
"columns": ["name", "fitness"],
|
||||
}
|
||||
response = await logged_in.post("/api/v1/research/assets", json={"kind": "view", "content": content})
|
||||
assert response.status_code == 201, response.text
|
||||
item = response.json()
|
||||
assert item["content"]["filters"]["turnover_max"] == 0.15
|
||||
assert item["content"]["filters"]["sort"] == "fitness"
|
||||
assert item["content"]["columns"] == ["name", "fitness"]
|
||||
bad = await logged_in.post(
|
||||
"/api/v1/research/assets",
|
||||
json={"kind": "view", "content": {**content, "filters": {"sort": "arbitrary_sql"}}},
|
||||
)
|
||||
assert bad.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metrics,verdict,missing",
|
||||
[
|
||||
({"sharpe": None, "fitness": 1, "turnover": 0.1}, "review", "sharpe"),
|
||||
({"sharpe": 2, "fitness": 1, "turnover": 0.1}, "review", "platform_checks"),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "FAIL"}]},
|
||||
"block",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]},
|
||||
"pass",
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_evaluations_keep_missing_and_failures(app, logged_in, metrics, verdict, missing):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": metrics}))
|
||||
response = await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["verdict"] == verdict
|
||||
if missing:
|
||||
assert missing in report["records"][0]["missing"]
|
||||
async with app.state.sessions() as db:
|
||||
assert (await db.get(Research, "seed")).state == "inbox"
|
||||
|
||||
|
||||
async def test_sync_and_model_advice_do_not_rewrite_report(app, logged_in):
|
||||
from app.research.evaluations import Evaluations
|
||||
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": -1}}))
|
||||
original = (await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})).json()
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": 9}}))
|
||||
advised = await Evaluations(db).add_advice(
|
||||
original["id"],
|
||||
{"summary": "建议通过", "risks": [], "suggestions": []},
|
||||
{"model": "fixture", "revision": 1},
|
||||
)
|
||||
assert advised["id"] != original["id"]
|
||||
assert advised["report"]["verdict"] == "block"
|
||||
assert advised["report"]["records"][0]["snapshot"]["is"]["sharpe"] == -1
|
||||
assert (await logged_in.get(f"/api/v1/research/evaluations/{original['id']}")).json() == original
|
||||
|
||||
|
||||
async def test_lineage_retains_multiple_parents_and_descendants(app, logged_in, research_input):
|
||||
async with app.state.sessions.begin() as db:
|
||||
await upsert_alpha(db, alpha("seed"))
|
||||
|
||||
async def create(parents=None, seeds=None):
|
||||
result = await logged_in.post(
|
||||
"/api/v1/research/experiments",
|
||||
json=expansion(
|
||||
research_input["id"], parent_experiment_ids=parents or [], parent_alpha_ids=seeds or []
|
||||
),
|
||||
)
|
||||
assert result.status_code == 201, result.text
|
||||
return result.json()["id"]
|
||||
|
||||
first, second = await create(seeds=["seed"]), await create(seeds=["seed"])
|
||||
child = await create([first, second])
|
||||
grandchild = await create([child])
|
||||
graph = (await logged_in.get("/api/v1/research/lineage?alpha_id=seed")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {first, second, child, grandchild}
|
||||
assert len(graph["edges"]) == 5 and not graph["truncated"]
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={grandchild}")).json()
|
||||
assert len(graph["items"]) == 4
|
||||
|
||||
|
||||
async def test_evaluate_backtest_keeps_experiment_and_incomplete_evidence(app, logged_in, research_input):
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import BacktestResult
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-evidence")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
async with app.state.sessions.begin() as db:
|
||||
result = await db.scalar(select(BacktestResult))
|
||||
result.complete = False
|
||||
result.snapshot = {
|
||||
"is": {"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]}
|
||||
}
|
||||
response = await logged_in.post(
|
||||
"/api/v1/research/evaluations",
|
||||
json={"experiment_id": experiment["id"], "backtest_run_id": run["backtest_run_id"]},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
report = response.json()["report"]
|
||||
assert report["experiment"]["inputs"][0]["id"] == research_input["id"]
|
||||
incomplete = next(r for r in report["records"] if not r["complete"])
|
||||
assert incomplete["verdict"] == "review" and "backtest_error" in incomplete["missing"]
|
||||
assert incomplete["settings"] and incomplete["expression"]
|
||||
|
||||
|
||||
async def test_lineage_crosses_saved_alpha_into_next_research(app, logged_in, research_input):
|
||||
from tests.test_backtests import execute, setup, start
|
||||
|
||||
experiment = (
|
||||
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
|
||||
).json()
|
||||
preview = (
|
||||
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
|
||||
).json()
|
||||
_, lane = await setup(app)
|
||||
run = await start(logged_in, preview, "stage2-alpha-lineage")
|
||||
await execute(app, lane, run["backtest_run_id"])
|
||||
result = (await logged_in.get(f"/api/v1/backtests/runs/{run['backtest_run_id']}/results")).json()
|
||||
aid = result["items"][0]["alpha_id"]
|
||||
child = (
|
||||
await logged_in.post(
|
||||
"/api/v1/research/experiments", json=expansion(research_input["id"], parent_alpha_ids=[aid])
|
||||
)
|
||||
).json()
|
||||
for eid in [experiment["id"], child["id"]]:
|
||||
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={eid}")).json()
|
||||
assert {r["id"] for r in graph["items"]} == {experiment["id"], child["id"]}
|
||||
Reference in New Issue
Block a user