fix(research): remove template scope constraints and migrate stored templates

backend/migrations/versions/0023_remove_template_scope.py

@@ -0,0 +1,80 @@
+"""Remove retired template scope metadata without changing execution scopes."""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0023"
+down_revision = "0022"
+branch_labels = None
+depends_on = None
+
+# Only structured research documents carry reusable template definitions. Do not
+# rewrite opaque model messages, audit digests, or already-authorized tool calls.
+DOCUMENTS = (
+    ("research_revisions", ("asset_id", "version"), ("content", "provenance")),
+    ("research_experiments", ("id",), ("evidence", "parents")),
+    ("research_flow_runs", ("id",), ("definition", "authorization")),
+    ("research_step_runs", ("id",), ("output",)),
+    ("research_requests", ("id",), ("response",)),
+)
+
+
+def remove_template_scope(value):
+    """Copy structured templates without scope; preserve input scopes and bindings.
+
+    Match the frozen persisted TemplateSpec shape, including nested feature
+    templates and asset snapshots. Never remove an arbitrary key named scope:
+    input snapshots and scope-named expression variables still require it.
+    """
+    if isinstance(value, list):
+        return [remove_template_scope(item) for item in value]
+    if not isinstance(value, dict):
+        return value
+    is_template = (
+        isinstance(value.get("name"), str)
+        and isinstance(value.get("expression"), str)
+        and isinstance(value.get("variables"), dict)
+    )
+    return {
+        key: remove_template_scope(item)
+        for key, item in value.items()
+        if not (is_template and key == "scope")
+    }
+
+
+def upgrade():
+    """Clean template documents in bounded batches, retaining IDs and versions."""
+    connection = op.get_bind()
+    for name, keys, documents in DOCUMENTS:
+        table = sa.table(name, *(
+            [sa.column(key, sa.Integer() if key == "version" else sa.String()) for key in keys]
+            + [sa.column(column, sa.JSON()) for column in documents]
+        ))
+        last = None
+        while True:
+            query = sa.select(table).order_by(*(table.c[key] for key in keys)).limit(500)
+            if last is not None:
+                query = query.where(sa.or_(*(
+                    sa.and_(*(table.c[keys[j]] == last[j] for j in range(i)), table.c[key] > last[i])
+                    for i, key in enumerate(keys)
+                )))
+            rows = connection.execute(query).mappings().all()
+            if not rows:
+                break
+            for row in rows:
+                changes = {}
+                for column in documents:
+                    cleaned = remove_template_scope(row[column])
+                    if cleaned != row[column]:
+                        changes[column] = cleaned
+                if changes:
+                    connection.execute(table.update().where(
+                        *(table.c[key] == row[key] for key in keys)
+                    ).values(**changes))
+            last = tuple(rows[-1][key] for key in keys)
+
+
+def downgrade():
+    # Previous schemas allow absent template scope; deleted metadata cannot be
+    # reconstructed. Execution scopes and all other research data are unchanged.
+    pass