yuxuanhui
ba0ed9d03f
Deploy production / deploy (push) Successful in 1m12s
- Added new metrics fields: sub_universe_sharpe, robust_universe_sharpe, two_year_sharpe, prod_correlation, pnl, check_type, and neutralization to the Alpha model. - Updated snapshot_columns function to derive new metrics and check types from platform snapshots. - Enhanced API to include failed checks and check types in responses. - Created migration script to backfill existing Alpha records with new metrics and check types. - Updated frontend components to display new metrics and allow editing of custom tags. - Improved filtering and sorting capabilities for new metrics in the Alpha list. - Added tests for new functionality including checks classification and metrics filtering.
289 lines
11 KiB
Python
289 lines
11 KiB
Python
"""Normalize upstream business data, without making platform or research decisions."""
|
|
|
|
import math
|
|
import re
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import or_, select, update
|
|
|
|
from .models import Alpha, Research, ResearchTag, SelfCorrelation, now
|
|
from .research.provenance import source_alpha_ids
|
|
|
|
METRIC_FIELDS = (
|
|
"sharpe", "fitness", "returns", "turnover", "margin", "drawdown",
|
|
"sub_universe_sharpe", "robust_universe_sharpe", "two_year_sharpe", "prod_correlation", "pnl",
|
|
)
|
|
|
|
|
|
def failed_checks(checks):
|
|
"""Return failed platform check names; local correlation never changes this list."""
|
|
return [
|
|
check.get("name") if isinstance(check.get("name"), str) else "未命名检查"
|
|
for check in checks if isinstance(check, dict) and check.get("result") == "FAIL"
|
|
] if isinstance(checks, list) else []
|
|
|
|
|
|
def snapshot_columns(settings, metrics, checks):
|
|
"""Derive list fields from a platform snapshot, preserving missing metrics as null.
|
|
|
|
Only explicit FAIL results count. Empty, malformed and unfinished checks are
|
|
pending; all known checks passing without PROD_CORRELATION is only a pre-check.
|
|
No submission eligibility or activity eligibility is inferred here.
|
|
"""
|
|
settings = settings if isinstance(settings, dict) else {}
|
|
metrics = metrics if isinstance(metrics, dict) else {}
|
|
checks = checks if isinstance(checks, list) else []
|
|
valid = [check for check in checks if isinstance(check, dict)]
|
|
failures = len(failed_checks(checks))
|
|
by_name = {check["name"]: check for check in valid if isinstance(check.get("name"), str)}
|
|
if failures:
|
|
check_type = "FAIL_1" if failures == 1 else "FAIL_2"
|
|
elif not checks or len(valid) != len(checks) or any(check.get("result") != "PASS" for check in valid):
|
|
check_type = "PENDING"
|
|
else:
|
|
check_type = "PASS" if "PROD_CORRELATION" in by_name else "PRE_CHECK"
|
|
neutralization = settings.get("neutralization")
|
|
return {
|
|
"check_type": check_type,
|
|
"neutralization": neutralization if isinstance(neutralization, str) else None,
|
|
"pnl": number(metrics.get("pnl")),
|
|
**{
|
|
field: number(by_name.get(name, {}).get("value"))
|
|
for field, name in (
|
|
("sub_universe_sharpe", "LOW_SUB_UNIVERSE_SHARPE"),
|
|
("robust_universe_sharpe", "LOW_ROBUST_UNIVERSE_SHARPE"),
|
|
("two_year_sharpe", "LOW_2Y_SHARPE"),
|
|
("prod_correlation", "PROD_CORRELATION"),
|
|
)
|
|
},
|
|
}
|
|
|
|
|
|
def submission_condition(submission):
|
|
"""Match the platform list contract; a missing status is never assumed submitted."""
|
|
return Alpha.status == "UNSUBMITTED" if submission == "UNSUBMITTED" else Alpha.status != "UNSUBMITTED"
|
|
|
|
|
|
async def invalidate_correlations(db, alpha_id, regions=()):
|
|
"""A changed baseline or PnL invalidates local conclusions without touching platform checks."""
|
|
await db.execute(
|
|
update(SelfCorrelation)
|
|
.where(or_(SelfCorrelation.alpha_id == alpha_id, SelfCorrelation.region.in_(regions)))
|
|
.values(stale=True)
|
|
)
|
|
|
|
|
|
SENSITIVE_KEYS = {
|
|
"password",
|
|
"token",
|
|
"cookies",
|
|
"cookie",
|
|
"authorization",
|
|
"credentials",
|
|
"secret",
|
|
"accesstoken",
|
|
"refreshtoken",
|
|
"sessiontoken",
|
|
"clientsecret",
|
|
"authorizationheader",
|
|
"setcookie",
|
|
"apikey",
|
|
"csrftoken",
|
|
"xsrftoken",
|
|
"authentication",
|
|
}
|
|
|
|
|
|
def sanitize(value):
|
|
if isinstance(value, dict):
|
|
return {
|
|
k: sanitize(v) for k, v in value.items() if re.sub(r"[^a-z]", "", k.lower()) not in SENSITIVE_KEYS
|
|
}
|
|
if isinstance(value, list):
|
|
return [sanitize(v) for v in value]
|
|
if isinstance(value, float) and not math.isfinite(value):
|
|
return None
|
|
return value
|
|
|
|
|
|
def number(value):
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
result = float(value)
|
|
return result if math.isfinite(result) else None
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def date(value):
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None
|
|
except (ValueError, TypeError, AttributeError):
|
|
return None
|
|
|
|
|
|
def code(value):
|
|
return value.get("code") if isinstance(value, dict) else value if isinstance(value, str) else None
|
|
|
|
|
|
async def upsert_alpha(db, raw: dict):
|
|
alpha_id = raw.get("id")
|
|
if not isinstance(alpha_id, str) or not alpha_id:
|
|
raise ValueError("Alpha 数据缺少 ID")
|
|
item = await db.get(Alpha, alpha_id)
|
|
previous_region = item.region if item else None
|
|
previous_status = item.status if item else None
|
|
if item is None:
|
|
item = Alpha(id=alpha_id)
|
|
db.add(item)
|
|
settings = raw.get("settings") or {}
|
|
metrics = raw.get("is") if isinstance(raw.get("is"), dict) else {}
|
|
item.name = raw.get("name")
|
|
item.expression = code(raw.get("regular"))
|
|
item.selection, item.combo = code(raw.get("selection")), code(raw.get("combo"))
|
|
item.alpha_type, item.language = raw.get("type"), settings.get("language")
|
|
item.stage, item.status, item.hidden = raw.get("stage"), raw.get("status"), raw.get("hidden") is True
|
|
item.region, item.universe = settings.get("region"), settings.get("universe")
|
|
if previous_region != item.region or previous_status != item.status:
|
|
regions = {
|
|
region
|
|
for region, status in ((previous_region, previous_status), (item.region, item.status))
|
|
if region and status and status != "UNSUBMITTED"
|
|
}
|
|
await invalidate_correlations(db, alpha_id, regions)
|
|
item.settings, item.is_metrics = sanitize(settings), sanitize(metrics)
|
|
item.os_metrics = sanitize(raw.get("os")) if isinstance(raw.get("os"), dict) else {}
|
|
item.checks = sanitize(metrics.get("checks") or raw.get("checks") or [])
|
|
for key, value in snapshot_columns(item.settings, item.is_metrics, item.checks).items():
|
|
setattr(item, key, value)
|
|
for key in ("sharpe", "fitness", "returns", "turnover", "margin", "drawdown"):
|
|
setattr(item, key, number(metrics.get(key)))
|
|
item.date_created, item.date_submitted = date(raw.get("dateCreated")), date(raw.get("dateSubmitted"))
|
|
item.synced_at, item.raw = now(), sanitize(raw)
|
|
await db.flush()
|
|
if await db.get(Research, alpha_id) is None:
|
|
db.add(Research(alpha_id=alpha_id))
|
|
return item
|
|
|
|
|
|
def list_statement(filters):
|
|
query = select(Alpha, Research).join(Research, Research.alpha_id == Alpha.id)
|
|
if filters.submission:
|
|
query = query.where(submission_condition(filters.submission))
|
|
source_filters = {k: getattr(filters, k) for k in ("source", "source_reference", "research_id", "backtest_run_id")}
|
|
if any(source_filters.values()):
|
|
query = query.where(Alpha.id.in_(source_alpha_ids(**source_filters)))
|
|
q = filters.q
|
|
if q:
|
|
pattern = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
|
|
query = query.where(
|
|
or_(
|
|
*(
|
|
getattr(Alpha, f).ilike(pattern, escape="\\")
|
|
for f in ("id", "name", "expression", "selection", "combo")
|
|
)
|
|
)
|
|
)
|
|
for name in ("region", "universe", "alpha_type", "language", "status", "stage", "hidden", "check_type", "neutralization"):
|
|
value = getattr(filters, name)
|
|
if value is not None:
|
|
query = query.where(getattr(Alpha, name) == value)
|
|
if filters.research_state:
|
|
query = query.where(Research.state == filters.research_state)
|
|
if filters.favorite is not None:
|
|
query = query.where(Research.favorite == filters.favorite)
|
|
if filters.tag:
|
|
query = query.where(Alpha.id.in_(select(ResearchTag.alpha_id).where(ResearchTag.tag == filters.tag)))
|
|
if filters.created_from:
|
|
query = query.where(Alpha.date_created >= filters.created_from)
|
|
if filters.created_to:
|
|
query = query.where(Alpha.date_created <= filters.created_to)
|
|
for name in METRIC_FIELDS:
|
|
for suffix, compare in (("min", "ge"), ("max", "le")):
|
|
value = getattr(filters, f"{name}_{suffix}")
|
|
if value is not None:
|
|
column = getattr(Alpha, name)
|
|
query = query.where(column >= value if compare == "ge" else column <= value)
|
|
return query
|
|
|
|
|
|
def sorted_statement(query, sort, direction):
|
|
column = getattr(Alpha, sort)
|
|
order = column.desc() if direction == "desc" else column.asc()
|
|
return query.order_by(order.nullslast(), Alpha.id.asc())
|
|
|
|
|
|
def summary(item: Alpha, research: Research):
|
|
keys = (
|
|
"id",
|
|
"name",
|
|
"alpha_type",
|
|
"language",
|
|
"stage",
|
|
"status",
|
|
"hidden",
|
|
"region",
|
|
"universe",
|
|
"sharpe",
|
|
"fitness",
|
|
"returns",
|
|
"turnover",
|
|
"margin",
|
|
"drawdown",
|
|
"date_created",
|
|
"date_submitted",
|
|
"synced_at",
|
|
"check_type",
|
|
"neutralization",
|
|
"sub_universe_sharpe",
|
|
"robust_universe_sharpe",
|
|
"two_year_sharpe",
|
|
"prod_correlation",
|
|
"pnl",
|
|
)
|
|
result = {k: getattr(item, k) for k in keys}
|
|
result["failed_checks"] = failed_checks(item.checks)
|
|
result["expression_preview"] = (item.expression or item.selection or "")[:240]
|
|
result["research"] = {
|
|
k: getattr(research, k) for k in ("note", "tags", "favorite", "state", "updated_at", "version")
|
|
}
|
|
return result
|
|
|
|
|
|
def pnl_points(raw):
|
|
"""Use schema column names, preserving missing values rather than creating zero PnL."""
|
|
records = raw.get("records")
|
|
schema = raw.get("schema") or {}
|
|
properties = schema.get("properties", []) if isinstance(schema, dict) else schema
|
|
if isinstance(properties, dict):
|
|
names = list(properties)
|
|
else:
|
|
names = [p.get("name", "") if isinstance(p, dict) else str(p) for p in properties]
|
|
normalized = [name.lower() for name in names]
|
|
if not isinstance(records, list):
|
|
raise ValueError("PnL 缺少 records")
|
|
points = []
|
|
for row in records:
|
|
if isinstance(row, dict):
|
|
row = {str(k).lower(): v for k, v in row.items()}
|
|
timestamp = next((row[k] for k in ("date", "datetime", "timestamp") if k in row), None)
|
|
value = next((row[k] for k in ("pnl", "value") if k in row), None)
|
|
else:
|
|
date_i = next(
|
|
(i for i, n in enumerate(normalized) if n in ("date", "datetime", "timestamp")), None
|
|
)
|
|
pnl_i = next((i for i, n in enumerate(normalized) if n in ("pnl", "value")), None)
|
|
if date_i is None or pnl_i is None or not isinstance(row, list) or len(row) <= max(date_i, pnl_i):
|
|
raise ValueError("PnL schema 无法识别日期或数值列")
|
|
timestamp, value = row[date_i], row[pnl_i]
|
|
if timestamp is not None:
|
|
if isinstance(timestamp, (int, float)):
|
|
from datetime import timezone
|
|
|
|
timestamp = datetime.fromtimestamp(
|
|
timestamp / 1000 if timestamp > 1e11 else timestamp, tz=timezone.utc
|
|
).isoformat()
|
|
points.append({"date": str(timestamp), "value": number(value)})
|
|
return sorted(points, key=lambda p: p["date"])
|