refactor: unify data preparations and research input snapshots

backend/app/research/ai_tools.py

@@ -4,6 +4,8 @@ from pydantic import Field
 
 from ..ai.alpha_tools import AlphaArgs
 from ..ai.capabilities import Capability
+from ..preparations.service import Preparations
+from ..schemas import Contract
 from .contracts import ChatboxResearchInput, InputPageArgs, ResearchInputSelection, ResearchPreviewInput
 
 
@@ -16,14 +18,26 @@ async def prepare(ctx, args):
     return await ctx.business.research_builder.prepare(ResearchPreviewInput(**args.model_dump()))
 
 
-INSTRUCTIONS = "Chatbox 是研究来源，不是 Alpha 备注。新候选由服务端记录会话和生成轮次；引用已有草稿和重跑保留原来源。\n数据集研究先用 search_catalog 读取实际字段/类型/集合版本，或 get_research_input 读取页面提供的固定输入。\n只有明确选出的字段才能 prepare_research_input；不要把一页搜索结果当作整集。页面 unsaved_field_selection 为 true 且无输入引用时，请用户保存选择并点击“用此输入研究”，不能忽略排除项。\n有输入快照时使用 prepare_research_backtest，提供研究假设、具名占位符和实际字段类型，模拟参数须匹配输入范围。模板中的所有数据字段使用绑定占位符。\n字段类型未知时说明限制；VECTOR 处理方式在模板中明确写出，不把 VECTOR 当作 MATRIX。绑定校验不等于 FASTEXPR 语义或平台权限验证。\n无数据集输入的直接表达式研究可以 prepare_backtest；不得声称来自已核验数据集。已有候选草稿先 get_backtest_draft，再按 ID 和版本引用。"
+INSTRUCTIONS = "研究先用 search_data_preparations 查询可编辑集合，读取集合 ID 与 version 后使用 prepare_research_input 固定输入。已有快照使用 get_research_input。所有研究来源保留独立快照，删除集合不影响已有研究。构建回测需明确假设、字段绑定和范围；VECTOR 必须显式处理，不能当作 MATRIX。直接表达式回测不声明数据准备来源。"
+
+
+
+class PreparationSearch(Contract):
+    q: str = Field(default="", max_length=300)
+    scope_key: str | None = None
+    limit: int = Field(default=25, ge=1, le=100)
+    offset: int = Field(default=0, ge=0)
 
 
 CAPABILITIES = (
+    Capability(name="search_data_preparations", schema=PreparationSearch,
+        description="分页搜索数据准备集合，返回 ID、version、范围与字段数；非空集合可固定为研究输入。",
+        label="查询数据准备", renderer="catalog", effect="query",
+        handler=lambda ctx, args: Preparations(ctx.business.db).list(**args.model_dump())),
     Capability(
         name="prepare_research_input",
         schema=ResearchInputSelection,
-        description="把明确选出的 1–100 个字段固定为研究输入快照；必须提供已读取的集合版本。只保存本地输入，不启动同步或回测。已有输入引用时直接读取，不重建。",
+        description="将数据准备集合的明确 ID 和 version 固定为研究快照，保留完整字段与数据集归属。已有快照直接读取。",
         label="固定研究输入",
         renderer="catalog",
         effect="prepare",

backend/app/research/assets.py

@@ -57,7 +57,11 @@ class Assets:
             "view": ViewSpec,
             "workflow": WorkflowSpec,
         }[body.kind]
-        content = schema.model_validate(body.content).model_dump(mode="json")
+        parsed = schema.model_validate(body.content)
+        if body.kind == "feature":
+            from ..preparations.service import Preparations
+            await Preparations(self.db).bind(parsed)
+        content = parsed.model_dump(mode="json")
         if body.kind == "workflow":
             from .workflows import validate_graph
 

backend/app/research/contracts.py

@@ -5,16 +5,13 @@ from typing import Literal
 from pydantic import Field, model_validator
 
 from ..backtests.contracts import SimulationSettings, Source
-from ..catalog.contracts import Scope
+from ..preparations.contracts import PreparationReference
 from ..schemas import Contract
 from .expressions import PLACEHOLDER
 
 
 class ResearchInputSelection(Contract):
-    scope: Scope
-    dataset_id: str = Field(min_length=1, max_length=200)
-    collection_version: str = Field(min_length=1, max_length=36)
-    field_ids: list[str] = Field(min_length=1, max_length=100)
+    items: list[PreparationReference] = Field(min_length=1, max_length=1)
 
 
 class InputPageArgs(Contract):
@@ -50,7 +47,7 @@ class ChatboxResearchInput(Contract):
 
     name: str = Field(min_length=1, max_length=200)
     hypothesis: str = Field(min_length=1, max_length=2000)
-    template_input_id: str = Field(min_length=1, max_length=36)
+    input_snapshot_id: str = Field(min_length=1, max_length=36)
     candidates: list[ResearchCandidate] = Field(min_length=1, max_length=100)
 
     @model_validator(mode="after")

backend/app/research/experiments.py

@@ -11,7 +11,8 @@ from ..backtests.contracts import Candidate, DraftInput, PreviewInput, Simulatio
 from ..backtests.service import Backtests, uid
 from ..catalog.research_metadata import ResearchMetadata
 from ..catalog.service import Catalog
-from ..models import Alpha, BacktestRun, CatalogResource, ResearchExperiment, TemplateInput
+from ..models import Alpha, BacktestRun, CatalogResource, ResearchExperiment
+from ..preparations.service import Preparations
 from .assets import Assets
 from .expressions import GROUPS, analyze, expand
 from .serialization import encode_snapshot as jsonable_encoder
@@ -86,7 +87,7 @@ class Experiments:
                     "candidates": experiment["candidates"],
                     "hypothesis": experiment["hypothesis"],
                     "input_references": [
-                        {k: entry[k] for k in ("id", "collection_version", "scope", "dataset_id")}
+                        {k: entry[k] for k in ("id", "preparation_id", "preparation_version", "scope", "dataset_ids")}
                         for entry in experiment["inputs"]
                     ],
                     "template_reference": {
@@ -133,6 +134,7 @@ class Experiments:
         return validation
 
     async def create(self, body, kind="template", extra_evidence=None, *, parent_snapshots=None):
+        await Preparations(self.db).bind(body)
         asset = await self.assets.get(body.asset_id, body.version, "template") if body.asset_id else None
         template = TemplateSpec.model_validate(asset["content"]) if asset else body.template
         scope = scope_of(body.settings)
@@ -318,7 +320,8 @@ class Experiments:
                         kind=source_kind or experiment["kind"],
                         reference=reference or experiment_id,
                         research_id=experiment_id,
-                        template_input_id=inputs[0]["id"] if len(inputs) == 1 else None,
+                        input_snapshot_ids=[i["id"] for i in inputs],
+                        input_snapshot_id=inputs[0]["id"] if len(inputs) == 1 else None,
                         hypothesis=experiment["hypothesis"][:2000],
                     ),
                     candidates=[
@@ -342,6 +345,7 @@ class Experiments:
         original = parents[0]
         base = seed_settings(original["settings"])
         expression = original["expression"]
+        await Preparations(self.db).bind(body)
         snapshots, _ = await self.inputs(body.input_ids)
         groups = defaultdict(list)
         for snapshot in snapshots:
@@ -405,6 +409,7 @@ class Experiments:
         )
 
     async def generation_context(self, body):
+        await Preparations(self.db).bind(body)
         snapshots, fields = await self.inputs(body.input_ids)
         parents = await self.parents(body.parent_alpha_ids, body.parent_experiment_ids)
         metadata = await ResearchMetadata(self.db).operators(limit=100)
@@ -414,7 +419,7 @@ class Experiments:
             "hypothesis": body.hypothesis,
             "method": body.method,
             "inputs": [
-                {"id": item["id"], "scope": item["scope"], "dataset_id": item["dataset_id"]}
+                {"id": item["id"], "scope": item["scope"], "dataset_ids": item["dataset_ids"], "name": item["name"], "fields": item["fields"][:100]}
                 for item in snapshots
             ],
             "fields": dict(list(fields.items())[:300]),
@@ -424,9 +429,3 @@ class Experiments:
             ],
             "parents": [{**p, "candidates": p.get("candidates", [])[:10]} for p in parents],
         }
-
-    async def available_inputs(self, limit=100):
-        rows = await self.db.scalars(
-            select(TemplateInput).order_by(TemplateInput.created_at.desc()).limit(limit)
-        )
-        return {"items": [await self.catalog.input(row.id) for row in rows]}

backend/app/research/routes.py

@@ -28,12 +28,6 @@ from .workspace_contracts import (
 router = APIRouter(prefix="/api/v1/research", tags=["research"], dependencies=[Depends(require_auth)])
 
 
-@router.get("/inputs")
-async def inputs(request: Request, limit: int = Query(100, ge=1, le=100)):
-    async with request.app.state.sessions() as db:
-        return await Experiments(db).available_inputs(limit)
-
-
 @router.get("/assets")
 async def assets(
     request: Request,
@@ -96,10 +90,10 @@ async def import_commit(body: ImportCommit, request: Request):
 
 @router.post("/generate", status_code=201)
 async def generate(body: Generation, request: Request):
-    async with request.app.state.sessions() as db:
+    async with request.app.state.sessions.begin() as db:
         context = await Experiments(db).generation_context(body)
     result, evidence = await request_model(request.app.state.ai, context, OUTPUTS[body.method])
-    if body.method == "feature" and set(result.input_ids) != set(body.input_ids):
+    if body.method == "feature" and (result.preparation_refs or set(result.input_ids) != set(body.input_ids)):
         raise HTTPException(422, "模型不能改变已固定的输入范围")
     async with request.app.state.sessions.begin() as db:
         asset = await Assets(db).save(

backend/app/research/runtime.py

@@ -476,9 +476,9 @@ class ResearchRuntime:
             step = await db.get(ResearchStepRun, step_id)
             if not step or step.status != "running":
                 return
-            if isinstance(result, FeatureSpec) and set(result.input_ids) != set(
+            if isinstance(result, FeatureSpec) and (result.preparation_refs or set(result.input_ids) != set(
                 [i["id"] for i in step.output["context"]["inputs"]]
-            ):
+            )):
                 raise HTTPException(422, "模型不能改变已固定的输入范围")
             # A paused/stopped run may collect this already-issued model output, but cannot advance.
             asset = await Assets(db).save(

backend/app/research/service.py

@@ -5,12 +5,9 @@ not FASTEXPR operator semantics or the account's current platform permissions.
 """
 
 from fastapi import HTTPException
-from sqlalchemy import select
 
 from ..backtests.contracts import Candidate, DraftInput, PreviewInput, Source
-from ..catalog.contracts import EntryOutput, InputPreparation
 from ..catalog.service import Catalog
-from ..models import CatalogEntry
 from .expressions import analyze, expand
 
 
@@ -21,55 +18,18 @@ class ResearchBuilder:
         self.backtests = backtests
 
     async def select_input(self, body):
-        """Fix explicit fields in one published version; reject missing or stale members."""
-        collection = await self.catalog.collection(body.scope, body.dataset_id)
-        chosen = set(body.field_ids)
-        if len(chosen) != len(body.field_ids) or not chosen.issubset(collection["field_ids"]):
-            raise HTTPException(422, "字段选择含重复、未知或其他数据集字段")
-        saved = await self.catalog.prepare(
-            InputPreparation(
-                scope=body.scope,
-                dataset_id=body.dataset_id,
-                collection_version=body.collection_version,
-                selection="explicit",
-                excluded_ids=[field for field in collection["field_ids"] if field not in chosen],
-            )
-        )
+        from ..preparations.service import Preparations
+        saved = (await Preparations(self.db).freeze(body.items))[0]
         return await self.input_page(saved["id"])
 
     async def input_page(self, input_id, limit=25, offset=0, q="", field_type=None):
-        """Read the saved version, including field descriptions, with explicit pagination."""
         saved = await self.catalog.input(input_id)
-        ids = [
-            field
-            for field in saved["field_ids"]
-            if q.lower() in field.lower()
-            and (field_type is None or saved["field_types"].get(field) == field_type)
-        ]
-        page = ids[offset : offset + limit]
-        entries = {
-            row.id: row
-            for row in await self.db.scalars(
-                select(CatalogEntry).where(
-                    CatalogEntry.batch_id == saved["collection_version"], CatalogEntry.id.in_(page)
-                )
-            )
-        }
-        return {
-            **{
-                k: saved[k]
-                for k in ("id", "scope", "dataset_id", "collection_version", "selection", "created_at")
-            },
-            "field_count": len(saved["field_ids"]),
-            "items": [
-                EntryOutput.model_validate(entries[field], from_attributes=True).model_dump()
-                for field in page
-            ],
-            "total": len(ids),
-            "limit": limit,
-            "offset": offset,
-            "has_more": offset + limit < len(ids),
-        }
+        fields = [f for f in saved["fields"] if (not q or q.lower() in
+                  " ".join(str(f.get(k) or "") for k in ("id", "name", "description", "dataset_id")).lower())
+                  and (not field_type or f["field_type"] == field_type)]
+        return {**{k: v for k, v in saved.items() if k not in ("fields", "field_ids", "field_types")},
+                "field_count": len(saved["fields"]), "items": fields[offset:offset + limit],
+                "total": len(fields), "limit": limit, "offset": offset, "has_more": offset + limit < len(fields)}
 
     async def prepare(self, body):
         """Bind templates against an immutable input, then reuse the fixed-preview interface.
@@ -77,7 +37,7 @@ class ResearchBuilder:
         Raises HTTPException(422) for wrong scope, membership or declared type.
         No expression execution or implicit cleaning/aggregation takes place here.
         """
-        saved = await self.catalog.input(body.template_input_id)
+        saved = await self.catalog.input(body.input_snapshot_id)
         scope = saved["scope"]
         candidates = []
         for item in body.candidates:
@@ -114,7 +74,7 @@ class ResearchBuilder:
         source = Source.model_validate(
             {
                 **body.source.model_dump(),
-                "template_input_id": saved["id"],
+                "input_snapshot_id": saved["id"],
                 "hypothesis": body.hypothesis,
             }
         )

backend/app/research/workflows.py

@@ -181,6 +181,8 @@ class Workflows:
         for node in graph.nodes:
             if node.type == "iterate" and node.config["max_rounds"] > body.budget.max_rounds:
                 raise HTTPException(422, "流程迭代上限超过本次授权轮数")
+        from ..preparations.service import Preparations
+        await Preparations(self.db).bind(body)
         experiments = Experiments(self.db)
         settings_variant = any(
             n.type == "variant" and n.config.get("method") == "settings" for n in graph.nodes

backend/app/research/workspace_contracts.py

@@ -7,6 +7,7 @@ from pydantic import Field, field_validator, model_validator
 
 from ..backtests.contracts import SimulationSettings
 from ..catalog.contracts import Scope
+from ..preparations.contracts import PreparationReference
 from ..schemas import Contract
 from .expressions import IDENTIFIER, PLACEHOLDER, normalize_template
 
@@ -70,7 +71,8 @@ class FeatureStep(Contract):
 class FeatureSpec(Contract):
     name: str = Field(min_length=1, max_length=200)
     hypothesis: str = Field(min_length=1, max_length=10000)
-    input_ids: list[str] = Field(min_length=1, max_length=20)
+    input_ids: list[str] = Field(default_factory=list, max_length=20)
+    preparation_refs: list[PreparationReference] = Field(default_factory=list, max_length=20)
     steps: list[FeatureStep] = Field(default_factory=list, max_length=30)
     template: TemplateSpec | None = None
 
@@ -102,7 +104,8 @@ class Expansion(Contract):
     asset_id: str | None = Field(default=None, max_length=36)
     version: int | None = Field(default=None, ge=1)
     template: TemplateSpec | None = None
-    input_ids: list[str] = Field(min_length=1, max_length=20)
+    input_ids: list[str] = Field(default_factory=list, max_length=20)
+    preparation_refs: list[PreparationReference] = Field(default_factory=list, max_length=20)
     hypothesis: str = Field(min_length=1, max_length=10000)
     settings: SimulationSettings
     mode: Literal["all", "random"] = "all"
@@ -123,7 +126,8 @@ class Expansion(Contract):
 class Generation(Contract):
     name: str = Field(min_length=1, max_length=200)
     hypothesis: str = Field(min_length=1, max_length=10000)
-    input_ids: list[str] = Field(min_length=1, max_length=20)
+    input_ids: list[str] = Field(default_factory=list, max_length=20)
+    preparation_refs: list[PreparationReference] = Field(default_factory=list, max_length=20)
     parent_alpha_ids: list[str] = Field(default_factory=list, max_length=20)
     parent_experiment_ids: list[str] = Field(default_factory=list, max_length=20)
     method: Literal["template", "structure", "feature"] = "template"
@@ -131,7 +135,8 @@ class Generation(Contract):
 
 class SettingVariants(Contract):
     alpha_id: str = Field(min_length=1, max_length=100)
-    input_ids: list[str] = Field(min_length=1, max_length=100)
+    input_ids: list[str] = Field(default_factory=list, max_length=100)
+    preparation_refs: list[PreparationReference] = Field(default_factory=list, max_length=100)
     hypothesis: str = Field(default="保持表达式，比较字段共同支持的市场与设置", max_length=10000)
 
 
@@ -229,7 +234,8 @@ class FlowStart(Contract):
     name: str = Field(min_length=1, max_length=200)
     workflow_id: str | None = None
     workflow_version: int | None = Field(default=None, ge=1)
-    input_ids: list[str] = Field(min_length=1, max_length=20)
+    input_ids: list[str] = Field(default_factory=list, max_length=20)
+    preparation_refs: list[PreparationReference] = Field(default_factory=list, max_length=20)
     hypothesis: str = Field(min_length=1, max_length=10000)
     settings: SimulationSettings
     budget: Budget