feat: persist feature research and versioned alpha assessments

This commit is contained in:
yuxuanhui
2026-09-08 21:52:46 +08:00
parent f89ae211d2
commit eb4850a003
32 changed files with 2003 additions and 20 deletions
+1 -1
View File
@@ -36,7 +36,7 @@ class ModelSettingsInput(Contract):
class PageContext(Contract):
page: Literal["alphas", "account", "datasets", "backtests", "operators", "templates", "variants"] = "alphas"
page: Literal["alphas", "account", "datasets", "backtests", "operators", "templates", "features", "variants"] = "alphas"
research_asset_id: str | None = Field(default=None, max_length=36)
research_experiment_id: str | None = Field(default=None, max_length=36)
catalog_scope: Scope | None = None
+22
View File
@@ -348,6 +348,7 @@ class CatalogDataset(Base):
class CatalogEntry(Base):
"""Immutable published snapshots; staging rows remain invisible until batch completion."""
__tablename__ = "catalog_entries"
batch_id: Mapped[str] = mapped_column(ForeignKey("catalog_batches.id"), primary_key=True)
id: Mapped[str] = mapped_column(String(200), primary_key=True)
@@ -389,6 +390,7 @@ class TemplateInput(Base):
class CatalogResource(Base):
"""Read-only upstream metadata snapshots; local annotations live separately."""
__tablename__ = "catalog_resources"
key: Mapped[str] = mapped_column(String(250), primary_key=True)
kind: Mapped[str] = mapped_column(String(30), index=True)
@@ -406,6 +408,7 @@ class OperatorNote(Base):
class ResearchAsset(Base):
"""Stable identity for typed templates, feature plans, views and workflow definitions."""
__tablename__ = "research_assets"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
kind: Mapped[str] = mapped_column(String(30), index=True)
@@ -426,6 +429,7 @@ class ResearchRevision(Base):
class ResearchExperiment(Base):
"""Immutable generated candidates; never masquerade as platform Alpha records."""
__tablename__ = "research_experiments"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(200))
@@ -436,3 +440,21 @@ class ResearchExperiment(Base):
candidates: Mapped[list] = mapped_column(JSON)
evidence: Mapped[dict] = mapped_column(JSON)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
class ResearchEvaluation(Base):
__tablename__ = "research_evaluations"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
alpha_id: Mapped[str | None] = mapped_column(ForeignKey("alphas.id"), index=True)
experiment_id: Mapped[str | None] = mapped_column(ForeignKey("research_experiments.id"), index=True)
report: Mapped[dict] = mapped_column(JSON)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
class ResearchParent(Base):
"""Queryable edges; parent snapshots remain authoritative in the experiment."""
__tablename__ = "research_parents"
child_id: Mapped[str] = mapped_column(ForeignKey("research_experiments.id"), primary_key=True)
parent_kind: Mapped[str] = mapped_column(String(30), primary_key=True)
parent_id: Mapped[str] = mapped_column(String(100), primary_key=True, index=True)
+8 -3
View File
@@ -65,8 +65,8 @@ class Assets:
if body.kind == "feature":
from ..catalog.service import Catalog
for input_id in content["input_ids"]:
await Catalog(self.db).input(input_id)
snapshots = [await Catalog(self.db).input(input_id) for input_id in content["input_ids"]]
provenance = {**(provenance or {}), "inputs": snapshots}
if asset_id:
if body.version is None:
raise HTTPException(422, "更新需要素材版本")
@@ -87,7 +87,12 @@ class Assets:
self.db.add(ResearchAsset(id=asset_id, kind=body.kind, name=content["name"], version=version))
await self.db.flush()
self.db.add(
ResearchRevision(asset_id=asset_id, version=version, content=content, provenance=provenance or {})
ResearchRevision(
asset_id=asset_id,
version=version,
content=content,
provenance=jsonable_encoder(provenance or {}),
)
)
await self.db.flush()
return await self.get(asset_id, version)
+170
View File
@@ -0,0 +1,170 @@
"""Snapshot-based research assessment. Model advice cannot replace deterministic findings."""
import math
from fastapi import HTTPException
from sqlalchemy import select
from ..backtests.service import Backtests, uid
from ..models import Alpha, ResearchEvaluation, ResearchExperiment, SelfCorrelation
from .experiments import Experiments
from .serialization import encode_snapshot as jsonable_encoder
def assess(snapshot, rules):
metrics = snapshot.get("is") or {}
evidence, missing, failed = [], [], []
for key, bound, direction in (
("sharpe", rules.sharpe_min, "min"),
("fitness", rules.fitness_min, "min"),
("turnover", rules.turnover_max, "max"),
):
value = metrics.get(key)
if type(value) not in (int, float) or not math.isfinite(value):
missing.append(key)
value, status = None, "unknown"
else:
status = "pass" if (value >= bound if direction == "min" else value <= bound) else "block"
if status == "block":
failed.append(key)
evidence.append(
{"metric": key, "value": value, "bound": bound, "direction": direction, "status": status}
)
checks = metrics.get("checks") or []
if not checks:
missing.append("platform_checks")
for check in checks:
if check.get("result") == "FAIL":
failed.append(f"platform:{check.get('name', 'unknown')}")
unknown_checks = [check for check in checks if check.get("result") not in ("PASS", "FAIL")]
if unknown_checks:
missing.append("unresolved_platform_checks")
return {
"verdict": "block" if failed else "review" if missing else "pass",
"evidence": evidence,
"failed": failed,
"missing": missing,
"existing_platform_checks": checks,
"meaning": "本地研究筛选结果,不是官方提交资格",
}
class Evaluations:
def __init__(self, db):
self.db = db
async def create(self, body):
records = []
if body.alpha_id:
alpha = await self.db.get(Alpha, body.alpha_id)
if not alpha:
raise HTTPException(404, "Alpha 尚未同步")
records.append(
{
"alpha_id": alpha.id,
"snapshot": alpha.raw,
"observed_at": alpha.synced_at,
"client_item_id": None,
}
)
else:
run = await Backtests(self.db).run(body.backtest_run_id)
if body.experiment_id and run["source"].get("research_id") != body.experiment_id:
raise HTTPException(422, "回测运行不属于指定研究实验")
for offset in range(0, run["total"], 100):
page = await Backtests(self.db).results(body.backtest_run_id, 100, offset)
for item in page["items"]:
records.append(
{
"alpha_id": item["alpha_id"],
"snapshot": item["result"]["snapshot"] if item["result"] else {},
"observed_at": item["result"]["observed_at"] if item["result"] else None,
"client_item_id": item["client_item_id"],
"item_id": item["id"],
"error": item["error"],
"complete": bool(item["result"] and item["result"]["complete"]),
"expression": item["expression"],
"settings": item["settings"],
}
)
if body.experiment_id and not await self.db.get(ResearchExperiment, body.experiment_id):
raise HTTPException(404, "研究实验不存在")
findings = []
for record in records:
correlation = (
await self.db.get(SelfCorrelation, record["alpha_id"]) if record["alpha_id"] else None
)
finding = assess(record["snapshot"], body.rules)
if record.get("error") or record.get("complete") is False:
finding["missing"].append("backtest_error")
if finding["verdict"] == "pass":
finding["verdict"] = "review"
findings.append(
{
**record,
**finding,
"local_correlation": {
"result": correlation.result,
"stale": correlation.stale,
"calculated_at": correlation.calculated_at,
}
if correlation
else None,
}
)
report = jsonable_encoder(
{
"rules": body.rules.model_dump(),
"records": findings,
"backtest_run_id": body.backtest_run_id,
"experiment": await Experiments(self.db).get(body.experiment_id)
if body.experiment_id
else None,
"model_advice": None,
"verdict": "block"
if any(r["verdict"] == "block" for r in findings)
else "review"
if not findings or any(r["verdict"] == "review" for r in findings)
else "pass",
}
)
row = ResearchEvaluation(
id=uid(), alpha_id=body.alpha_id, experiment_id=body.experiment_id, report=report
)
self.db.add(row)
await self.db.flush()
return await self.get(row.id)
async def get(self, evaluation_id):
row = await self.db.get(ResearchEvaluation, evaluation_id)
if not row:
raise HTTPException(404, "评估报告不存在")
return jsonable_encoder(
{key: getattr(row, key) for key in ("id", "alpha_id", "experiment_id", "report", "created_at")}
)
async def list(self, alpha_id=None, experiment_id=None, limit=25, offset=0):
query = select(ResearchEvaluation)
if alpha_id:
query = query.where(ResearchEvaluation.alpha_id == alpha_id)
if experiment_id:
query = query.where(ResearchEvaluation.experiment_id == experiment_id)
rows = await self.db.scalars(
query.order_by(ResearchEvaluation.created_at.desc()).limit(limit).offset(offset)
)
return {"items": [await self.get(row.id) for row in rows]}
async def add_advice(self, evaluation_id, advice, model_evidence):
original = await self.get(evaluation_id)
report = {
**original["report"],
"model_advice": advice,
"model_evidence": model_evidence,
"previous_evaluation_id": evaluation_id,
}
row = ResearchEvaluation(
id=uid(), alpha_id=original["alpha_id"], experiment_id=original["experiment_id"], report=report
)
self.db.add(row)
await self.db.flush()
return await self.get(row.id)
+5
View File
@@ -204,6 +204,11 @@ class Experiments:
)
self.db.add(row)
await self.db.flush()
from ..models import ResearchParent
for parent_kind, parent_id in {(p["kind"], p["id"]) for p in parents}:
self.db.add(ResearchParent(child_id=row.id, parent_kind=parent_kind, parent_id=parent_id))
await self.db.flush()
return await self.get(row.id)
async def get(self, experiment_id):
+20
View File
@@ -0,0 +1,20 @@
"""Feature plans remain versioned inputs to the shared template research path."""
from fastapi import HTTPException
from .assets import Assets
from .workspace_contracts import AssetWrite
class Features:
def __init__(self, db):
self.assets = Assets(db)
async def to_template(self, asset_id, version):
feature = await self.assets.get(asset_id, version, "feature")
if not feature["content"].get("template"):
raise HTTPException(422, "特征方案尚未定义输出模板")
return await self.assets.save(
AssetWrite(kind="template", content=feature["content"]["template"]),
provenance={"feature": feature},
)
+113
View File
@@ -0,0 +1,113 @@
"""Bounded graph traversal with explicit continuation, retaining every result source."""
from fastapi import HTTPException
from sqlalchemy import or_, select
from ..models import Alpha, BacktestResult, BacktestRun, ResearchExperiment, ResearchParent
from .experiments import Experiments
from .provenance import alpha_sources, saved_sources
async def lineage(db, alpha_id=None, experiment_id=None, limit=25, offset=0):
if bool(alpha_id) == bool(experiment_id):
raise HTTPException(422, "指定 Alpha 或实验之一")
sources = None
frontier = set()
truncated = False
if alpha_id:
if not await db.get(Alpha, alpha_id):
raise HTTPException(404, "Alpha 尚未同步")
sources = await alpha_sources(db, alpha_id, limit, offset)
source_ids = list(
await db.scalars(
saved_sources()
.with_only_columns(BacktestRun.source["research_id"].as_string())
.where(BacktestResult.alpha_id == alpha_id)
.distinct()
.limit(101)
)
)
children = list(
await db.scalars(
select(ResearchParent.child_id)
.where(ResearchParent.parent_kind == "alpha", ResearchParent.parent_id == alpha_id)
.order_by(ResearchParent.child_id)
.limit(101)
)
)
frontier = set(source_ids + children) - {None}
truncated = len(frontier) > 100
else:
await Experiments(db).get(experiment_id)
frontier.add(experiment_id)
found, edges = {}, {}
for _ in range(8):
wanted = sorted(frontier - set(found))
if not wanted:
break
remaining = 100 - len(found)
if len(wanted) > remaining:
truncated = True
wanted = wanted[:remaining]
if not wanted:
break
rows = list(await db.scalars(select(ResearchExperiment).where(ResearchExperiment.id.in_(wanted))))
for row in rows:
found[row.id] = await Experiments(db).get(row.id)
produced = list(
await db.scalars(
saved_sources()
.with_only_columns(BacktestResult.alpha_id)
.where(BacktestRun.source["research_id"].as_string().in_(wanted))
.distinct()
.limit(1001)
)
)
truncated |= len(produced) > 1000
parent_alphas = {p["id"] for row in rows for p in row.parents if p["kind"] == "alpha"}
related_sources = list(
await db.scalars(
saved_sources()
.with_only_columns(BacktestRun.source["research_id"].as_string())
.where(BacktestResult.alpha_id.in_(parent_alphas | set(produced[:1000])))
.distinct()
.limit(101)
)
)
truncated |= len(related_sources) > 100
relations = list(
await db.scalars(
select(ResearchParent)
.where(
or_(
ResearchParent.child_id.in_(wanted),
(ResearchParent.parent_kind == "alpha")
& ResearchParent.parent_id.in_(produced[:1000]),
(ResearchParent.parent_kind == "experiment") & ResearchParent.parent_id.in_(wanted),
)
)
.order_by(ResearchParent.child_id, ResearchParent.parent_kind, ResearchParent.parent_id)
.limit(1001)
)
)
truncated |= len(relations) > 1000
frontier = set(related_sources[:100]) - {None}
for edge in relations[:1000]:
edges[(edge.child_id, edge.parent_kind, edge.parent_id)] = {
"child_id": edge.child_id,
"parent_kind": edge.parent_kind,
"parent_id": edge.parent_id,
}
frontier.add(edge.child_id)
if edge.parent_kind == "experiment":
frontier.add(edge.parent_id)
unresolved = sorted(frontier - set(found))
return {
"items": list(found.values()),
"edges": list(edges.values()),
"sources": sources,
"truncated": truncated or bool(unresolved),
"unresolved_experiment_ids": unresolved,
"limit": 100,
"max_depth": 8,
}
+84 -8
View File
@@ -5,13 +5,17 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
from ..security import require_auth
from .assets import Assets
from .comparisons import compare
from .evaluations import Evaluations
from .experiments import Experiments
from .model import request_model
from .features import Features
from .model import OUTPUTS, Advice, request_model
from .workspace_contracts import (
AssetWrite,
CompareInput,
EvaluateInput,
Expansion,
ExperimentPreview,
FeatureConversion,
Generation,
ImportCommit,
ImportPreview,
@@ -35,7 +39,7 @@ async def assets(
limit: int = Query(25, ge=1, le=100),
offset: int = Query(0, ge=0),
):
if kind != "template":
if kind not in ("template", "feature", "view"):
raise HTTPException(422, "当前素材类型尚未开放")
async with request.app.state.sessions() as db:
return await Assets(db).list(kind, q, limit, offset)
@@ -43,7 +47,7 @@ async def assets(
@router.post("/assets", status_code=201)
async def save_asset(body: AssetWrite, request: Request):
if body.kind != "template":
if body.kind not in ("template", "feature", "view"):
raise HTTPException(422, "当前素材类型尚未开放")
async with request.app.state.sessions.begin() as db:
return await Assets(db).save(body)
@@ -57,7 +61,7 @@ async def asset(asset_id: str, request: Request, version: int | None = Query(Non
@router.put("/assets/{asset_id}")
async def update_asset(asset_id: str, body: AssetWrite, request: Request):
if body.kind != "template":
if body.kind not in ("template", "feature", "view"):
raise HTTPException(422, "当前素材类型尚未开放")
async with request.app.state.sessions.begin() as db:
return await Assets(db).save(body, asset_id)
@@ -89,14 +93,17 @@ async def import_commit(body: ImportCommit, request: Request):
@router.post("/generate", status_code=201)
async def generate(body: Generation, request: Request):
if body.method == "feature":
raise HTTPException(422, "特征方案生成将在特征工程阶段开放")
async with request.app.state.sessions() as db:
context = await Experiments(db).generation_context(body)
result, evidence = await request_model(request.app.state.ai, context)
result, evidence = await request_model(request.app.state.ai, context, OUTPUTS[body.method])
if body.method == "feature" and set(result.input_ids) != set(body.input_ids):
raise HTTPException(422, "模型不能改变已固定的输入范围")
async with request.app.state.sessions.begin() as db:
asset = await Assets(db).save(
AssetWrite(kind="template", content=result.model_dump(mode="json")),
AssetWrite(
kind="feature" if body.method == "feature" else "template",
content=result.model_dump(mode="json"),
),
provenance={"generation": evidence, "context": context},
)
return {
@@ -149,3 +156,72 @@ async def settings_variants(body: SettingVariants, request: Request):
async def comparison(body: CompareInput, request: Request):
async with request.app.state.sessions() as db:
return await compare(db, body.alpha_ids)
@router.post("/features/{asset_id}/template", status_code=201)
async def feature_template(asset_id: str, body: FeatureConversion, request: Request):
async with request.app.state.sessions.begin() as db:
return await Features(db).to_template(asset_id, body.version)
@router.post("/evaluations", status_code=201)
async def evaluate(body: EvaluateInput, request: Request):
async with request.app.state.sessions.begin() as db:
return await Evaluations(db).create(body)
@router.get("/evaluations")
async def evaluations(
request: Request,
alpha_id: str | None = None,
experiment_id: str | None = None,
limit: int = Query(25, ge=1, le=100),
offset: int = Query(0, ge=0),
):
async with request.app.state.sessions() as db:
return await Evaluations(db).list(alpha_id, experiment_id, limit, offset)
@router.get("/evaluations/{evaluation_id}")
async def evaluation(evaluation_id: str, request: Request):
async with request.app.state.sessions() as db:
return await Evaluations(db).get(evaluation_id)
@router.post("/evaluations/{evaluation_id}/advice", status_code=201)
async def evaluation_advice(evaluation_id: str, request: Request):
async with request.app.state.sessions() as db:
original = await Evaluations(db).get(evaluation_id)
report = original["report"]
context = {
"evaluation_id": evaluation_id,
"rules": report["rules"],
"verdict": report["verdict"],
"total_records": len(report["records"]),
"records": [
{
key: row.get(key)
for key in ("alpha_id", "client_item_id", "evidence", "missing", "failed", "verdict")
}
for row in report["records"][:20]
],
"record_limit": 20,
}
result, evidence = await request_model(request.app.state.ai, context, Advice)
evidence["context"] = context
async with request.app.state.sessions.begin() as db:
return await Evaluations(db).add_advice(evaluation_id, result.model_dump(mode="json"), evidence)
@router.get("/lineage")
async def research_lineage(
request: Request,
alpha_id: str | None = None,
experiment_id: str | None = None,
limit: int = Query(25, ge=1, le=100),
offset: int = Query(0, ge=0),
):
from .lineage import lineage
async with request.app.state.sessions() as db:
return await lineage(db, alpha_id, experiment_id, limit, offset)
+7 -2
View File
@@ -75,6 +75,10 @@ class FeatureSpec(Contract):
template: TemplateSpec | None = None
class FeatureConversion(Contract):
version: int = Field(ge=1)
class ViewSpec(Contract):
name: str = Field(min_length=1, max_length=200)
filters: dict = Field(default_factory=dict)
@@ -85,8 +89,7 @@ class ViewSpec(Contract):
def valid_filters(cls, value):
from ..schemas import AlphaFilters
AlphaFilters.model_validate(value)
return value
return AlphaFilters.model_validate(value).model_dump(mode="json", exclude_none=True)
class AssetWrite(Contract):
@@ -151,6 +154,8 @@ class EvaluateInput(Contract):
@model_validator(mode="after")
def target(self):
if self.alpha_id and self.experiment_id:
raise ValueError("实验评估应选择关联回测运行;Alpha 评估单独保存")
if bool(self.alpha_id) == bool(self.backtest_run_id):
raise ValueError("选择 Alpha 或回测运行之一")
return self
+56 -1
View File
@@ -6,8 +6,10 @@ from ..ai.capabilities import Capability
from ..catalog.research_metadata import ResearchMetadata
from ..schemas import Contract
from .assets import Assets
from .evaluations import Evaluations
from .experiments import Experiments
from .workspace_contracts import Expansion, SettingVariants
from .features import Features
from .workspace_contracts import AssetWrite, EvaluateInput, Expansion, FeatureSpec, SettingVariants
class AssetQuery(Contract):
@@ -21,6 +23,16 @@ class AssetReference(Contract):
version: int | None = Field(default=None, ge=1)
class FixedAssetReference(AssetReference):
version: int = Field(ge=1)
class FeatureWrite(Contract):
content: FeatureSpec
asset_id: str | None = Field(default=None, max_length=36)
version: int | None = Field(default=None, ge=1)
class ExperimentReference(Contract):
experiment_id: str = Field(min_length=1, max_length=36)
@@ -105,3 +117,46 @@ CAPABILITIES = (
),
),
)
CAPABILITIES += (
Capability(
name="search_feature_plans",
schema=AssetQuery,
description="搜索版本化特征方案。",
label="搜索特征方案",
renderer="research",
effect="query",
handler=lambda ctx, args: Assets(ctx.business.db).list("feature", **args.model_dump()),
),
Capability(
name="save_feature_plan",
schema=FeatureWrite,
description="保存固定输入、处理步骤和经济假设,不执行计算或回测。",
label="保存特征方案",
renderer="research",
effect="prepare",
handler=lambda ctx, args: Assets(ctx.business.db).save(
AssetWrite(kind="feature", content=args.content.model_dump(mode="json"), version=args.version),
args.asset_id,
),
),
Capability(
name="feature_to_template",
schema=FixedAssetReference,
description="将指定特征方案版本转为新模板,保留来源。",
label="特征转为模板",
renderer="research",
effect="prepare",
handler=lambda ctx, args: Features(ctx.business.db).to_template(args.asset_id, args.version),
),
Capability(
name="evaluate_research",
schema=EvaluateInput,
description="保存版本化规则评估,缺失数据不会作为通过,不修改研究状态。",
label="保存规则评估",
renderer="research",
effect="prepare",
handler=lambda ctx, args: Evaluations(ctx.business.db).create(args),
),
)
@@ -0,0 +1,42 @@
"""Stage two: immutable evaluations; features and saved views reuse asset revisions."""
import sqlalchemy as sa
from alembic import op
revision = "0007"
down_revision = "0006"
branch_labels = None
depends_on = None
def upgrade():
edges = op.create_table(
"research_parents",
sa.Column("child_id", sa.String(36), sa.ForeignKey("research_experiments.id"), primary_key=True),
sa.Column("parent_kind", sa.String(30), primary_key=True),
sa.Column("parent_id", sa.String(100), primary_key=True),
)
op.create_index("ix_research_parents_parent_id", "research_parents", ["parent_id"])
experiments = sa.table("research_experiments", sa.column("id"), sa.column("parents", sa.JSON()))
for row in op.get_bind().execute(sa.select(experiments)).mappings():
values = [
{"child_id": row["id"], "parent_kind": kind, "parent_id": parent_id}
for kind, parent_id in {(p["kind"], p["id"]) for p in row["parents"]}
]
if values:
op.bulk_insert(edges, values)
op.create_table(
"research_evaluations",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("alpha_id", sa.String(100), sa.ForeignKey("alphas.id"), nullable=True),
sa.Column("experiment_id", sa.String(36), sa.ForeignKey("research_experiments.id"), nullable=True),
sa.Column("report", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_research_evaluations_alpha_id", "research_evaluations", ["alpha_id"])
op.create_index("ix_research_evaluations_experiment_id", "research_evaluations", ["experiment_id"])
def downgrade():
op.drop_table("research_evaluations")
op.drop_table("research_parents")
@@ -0,0 +1,80 @@
"""Stage-two PostgreSQL acceptance in dedicated databases only."""
import asyncio
import os
import subprocess
from pathlib import Path
from alembic import command
from alembic.config import Config
from cryptography.fernet import Fernet
NAME = "wq_research_stage2_test"
RESTORE = "wq_research_restore_stage2"
os.environ.update(
DATABASE_URL=f"postgresql+asyncpg://postgres:research-test-only@127.0.0.1:18436/{NAME}",
ADMIN_PASSWORD="research-acceptance-only",
ENCRYPTION_KEY=Fernet.generate_key().decode(),
)
def docker(*args, **kwargs):
return subprocess.run(["docker", "exec", "-i", "wq-research-acceptance-pg", *args], check=True, **kwargs)
async def acceptance():
import httpx
from sqlalchemy import select
from app.config import Settings
from app.main import create_app
from app.models import ResearchExperiment, ResearchParent, TemplateInput
from tests.test_research_outcomes import (
test_feature_conversion_keeps_original_version_through_experiment,
test_lineage_retains_multiple_parents_and_descendants,
test_saved_views_validate_and_retain_sort_columns,
test_sync_and_model_advice_do_not_rewrite_report,
)
app = create_app(Settings(_env_file=None, enable_runner=False, public_origin="http://testserver"))
async with app.router.lifespan_context(app):
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app), base_url="http://testserver", headers={"X-WQ-Request": "1"}
) as client:
response = await client.post(
"/api/v1/auth/login", json={"username": "admin", "password": "research-acceptance-only"}
)
assert response.status_code == 200
async with app.state.sessions() as db:
fixed = await db.scalar(select(TemplateInput))
for experiment in await db.scalars(select(ResearchExperiment)):
for parent in experiment.parents:
assert await db.get(ResearchParent, (experiment.id, parent["kind"], parent["id"]))
await test_feature_conversion_keeps_original_version_through_experiment(client, {"id": fixed.id})
await test_saved_views_validate_and_retain_sort_columns(client)
await test_sync_and_model_advice_do_not_rewrite_report(app, client)
await test_lineage_retains_multiple_parents_and_descendants(app, client, {"id": fixed.id})
print("PASS PostgreSQL: feature revisions, saved views, immutable evaluations and multi-parent traversal")
if __name__ == "__main__":
docker("createdb", "-U", "postgres", NAME)
with Path("/tmp/wq-research-stage1.dump").open("rb") as source:
docker("pg_restore", "-U", "postgres", "-d", NAME, stdin=source)
config = Config("alembic.ini")
command.upgrade(config, "0007")
command.check(config)
asyncio.run(acceptance())
dump = Path("/tmp/wq-research-stage2.dump")
with dump.open("wb") as output:
docker("pg_dump", "-U", "postgres", "-Fc", NAME, stdout=output)
docker("createdb", "-U", "postgres", RESTORE)
with dump.open("rb") as source:
docker("pg_restore", "-U", "postgres", "-d", RESTORE, stdin=source)
query = "SELECT (SELECT count(*) FROM research_revisions),(SELECT count(*) FROM research_evaluations),(SELECT count(*) FROM research_parents),(SELECT note FROM research WHERE alpha_id='OLD_RESEARCH')"
a = docker("psql", "-U", "postgres", "-d", NAME, "-Atc", query, capture_output=True).stdout
b = docker("psql", "-U", "postgres", "-d", RESTORE, "-Atc", query, capture_output=True).stdout
assert a == b
print(
"PASS PostgreSQL 17: 0006 → 0007 and pg_dump/pg_restore preserve history, graph and old research notes"
)
+191
View File
@@ -0,0 +1,191 @@
"""Stage-two acceptance: immutable plans, views, lineage and evidence-backed decisions."""
import pytest
from app.alphas import upsert_alpha
from app.models import Research
from tests.conftest import alpha
from tests.test_research_workspace import catalog, expansion, research_input, template
__all__ = ["catalog", "research_input"]
async def test_feature_conversion_keeps_original_version_through_experiment(logged_in, research_input):
content = {
"name": "特征方案",
"hypothesis": "排序稳定性",
"input_ids": [research_input["id"]],
"steps": [{"name": "排序", "rationale": "降低异常值影响", "expression": "rank(TEST_FIN_001)"}],
"template": template(),
}
saved = (
await logged_in.post("/api/v1/research/assets", json={"kind": "feature", "content": content})
).json()
converted = await logged_in.post(f"/api/v1/research/features/{saved['id']}/template", json={"version": 1})
assert converted.status_code == 201, converted.text
result = converted.json()
changed = await logged_in.put(
f"/api/v1/research/assets/{saved['id']}",
json={"kind": "feature", "version": 1, "content": {**content, "hypothesis": "新假设"}},
)
assert changed.status_code == 200
body = expansion(research_input["id"], asset_id=result["id"], version=1)
body.pop("template")
experiment = (await logged_in.post("/api/v1/research/experiments", json=body)).json()
source = experiment["evidence"]["template"]["provenance"]["feature"]
assert source["version"] == 1 and source["content"]["hypothesis"] == "排序稳定性"
assert source["content"]["input_ids"] == [research_input["id"]]
async def test_saved_views_validate_and_retain_sort_columns(logged_in):
content = {
"name": "低换手队列",
"filters": {
"turnover_max": "0.15",
"sort": "fitness",
"direction": "desc",
"submission": "UNSUBMITTED",
},
"columns": ["name", "fitness"],
}
response = await logged_in.post("/api/v1/research/assets", json={"kind": "view", "content": content})
assert response.status_code == 201, response.text
item = response.json()
assert item["content"]["filters"]["turnover_max"] == 0.15
assert item["content"]["filters"]["sort"] == "fitness"
assert item["content"]["columns"] == ["name", "fitness"]
bad = await logged_in.post(
"/api/v1/research/assets",
json={"kind": "view", "content": {**content, "filters": {"sort": "arbitrary_sql"}}},
)
assert bad.status_code == 422
@pytest.mark.parametrize(
"metrics,verdict,missing",
[
({"sharpe": None, "fitness": 1, "turnover": 0.1}, "review", "sharpe"),
({"sharpe": 2, "fitness": 1, "turnover": 0.1}, "review", "platform_checks"),
(
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "FAIL"}]},
"block",
None,
),
(
{"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]},
"pass",
None,
),
],
)
async def test_evaluations_keep_missing_and_failures(app, logged_in, metrics, verdict, missing):
async with app.state.sessions.begin() as db:
await upsert_alpha(db, alpha("seed", **{"is": metrics}))
response = await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})
assert response.status_code == 201, response.text
report = response.json()["report"]
assert report["verdict"] == verdict
if missing:
assert missing in report["records"][0]["missing"]
async with app.state.sessions() as db:
assert (await db.get(Research, "seed")).state == "inbox"
async def test_sync_and_model_advice_do_not_rewrite_report(app, logged_in):
from app.research.evaluations import Evaluations
async with app.state.sessions.begin() as db:
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": -1}}))
original = (await logged_in.post("/api/v1/research/evaluations", json={"alpha_id": "seed"})).json()
async with app.state.sessions.begin() as db:
await upsert_alpha(db, alpha("seed", **{"is": {"sharpe": 9}}))
advised = await Evaluations(db).add_advice(
original["id"],
{"summary": "建议通过", "risks": [], "suggestions": []},
{"model": "fixture", "revision": 1},
)
assert advised["id"] != original["id"]
assert advised["report"]["verdict"] == "block"
assert advised["report"]["records"][0]["snapshot"]["is"]["sharpe"] == -1
assert (await logged_in.get(f"/api/v1/research/evaluations/{original['id']}")).json() == original
async def test_lineage_retains_multiple_parents_and_descendants(app, logged_in, research_input):
async with app.state.sessions.begin() as db:
await upsert_alpha(db, alpha("seed"))
async def create(parents=None, seeds=None):
result = await logged_in.post(
"/api/v1/research/experiments",
json=expansion(
research_input["id"], parent_experiment_ids=parents or [], parent_alpha_ids=seeds or []
),
)
assert result.status_code == 201, result.text
return result.json()["id"]
first, second = await create(seeds=["seed"]), await create(seeds=["seed"])
child = await create([first, second])
grandchild = await create([child])
graph = (await logged_in.get("/api/v1/research/lineage?alpha_id=seed")).json()
assert {r["id"] for r in graph["items"]} == {first, second, child, grandchild}
assert len(graph["edges"]) == 5 and not graph["truncated"]
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={grandchild}")).json()
assert len(graph["items"]) == 4
async def test_evaluate_backtest_keeps_experiment_and_incomplete_evidence(app, logged_in, research_input):
from sqlalchemy import select
from app.models import BacktestResult
from tests.test_backtests import execute, setup, start
experiment = (
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
).json()
preview = (
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
).json()
_, lane = await setup(app)
run = await start(logged_in, preview, "stage2-evidence")
await execute(app, lane, run["backtest_run_id"])
async with app.state.sessions.begin() as db:
result = await db.scalar(select(BacktestResult))
result.complete = False
result.snapshot = {
"is": {"sharpe": 2, "fitness": 1, "turnover": 0.1, "checks": [{"name": "X", "result": "PASS"}]}
}
response = await logged_in.post(
"/api/v1/research/evaluations",
json={"experiment_id": experiment["id"], "backtest_run_id": run["backtest_run_id"]},
)
assert response.status_code == 201, response.text
report = response.json()["report"]
assert report["experiment"]["inputs"][0]["id"] == research_input["id"]
incomplete = next(r for r in report["records"] if not r["complete"])
assert incomplete["verdict"] == "review" and "backtest_error" in incomplete["missing"]
assert incomplete["settings"] and incomplete["expression"]
async def test_lineage_crosses_saved_alpha_into_next_research(app, logged_in, research_input):
from tests.test_backtests import execute, setup, start
experiment = (
await logged_in.post("/api/v1/research/experiments", json=expansion(research_input["id"]))
).json()
preview = (
await logged_in.post(f"/api/v1/research/experiments/{experiment['id']}/preview", json={})
).json()
_, lane = await setup(app)
run = await start(logged_in, preview, "stage2-alpha-lineage")
await execute(app, lane, run["backtest_run_id"])
result = (await logged_in.get(f"/api/v1/backtests/runs/{run['backtest_run_id']}/results")).json()
aid = result["items"][0]["alpha_id"]
child = (
await logged_in.post(
"/api/v1/research/experiments", json=expansion(research_input["id"], parent_alpha_ids=[aid])
)
).json()
for eid in [experiment["id"], child["id"]]:
graph = (await logged_in.get(f"/api/v1/research/lineage?experiment_id={eid}")).json()
assert {r["id"] for r in graph["items"]} == {experiment["id"], child["id"]}