yuxuanhui
226 lines
8.3 KiB
Python
226 lines
8.3 KiB
Python
"""Normalize upstream business data, without making platform or research decisions."""
|
|
|
|
import math
|
|
import re
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import or_, select, update
|
|
|
|
from .models import Alpha, Research, ResearchTag, SelfCorrelation, now
|
|
|
|
|
|
def submission_condition(submission):
|
|
"""Match the platform list contract; a missing status is never assumed submitted."""
|
|
return Alpha.status == "UNSUBMITTED" if submission == "UNSUBMITTED" else Alpha.status != "UNSUBMITTED"
|
|
|
|
|
|
async def invalidate_correlations(db, alpha_id, regions=()):
|
|
"""A changed baseline or PnL invalidates local conclusions without touching platform checks."""
|
|
await db.execute(
|
|
update(SelfCorrelation)
|
|
.where(or_(SelfCorrelation.alpha_id == alpha_id, SelfCorrelation.region.in_(regions)))
|
|
.values(stale=True)
|
|
)
|
|
|
|
|
|
SENSITIVE_KEYS = {
|
|
"password",
|
|
"token",
|
|
"cookies",
|
|
"cookie",
|
|
"authorization",
|
|
"credentials",
|
|
"secret",
|
|
"accesstoken",
|
|
"refreshtoken",
|
|
"sessiontoken",
|
|
"clientsecret",
|
|
"authorizationheader",
|
|
"setcookie",
|
|
"apikey",
|
|
"csrftoken",
|
|
"xsrftoken",
|
|
"authentication",
|
|
}
|
|
|
|
|
|
def sanitize(value):
|
|
if isinstance(value, dict):
|
|
return {
|
|
k: sanitize(v) for k, v in value.items() if re.sub(r"[^a-z]", "", k.lower()) not in SENSITIVE_KEYS
|
|
}
|
|
if isinstance(value, list):
|
|
return [sanitize(v) for v in value]
|
|
if isinstance(value, float) and not math.isfinite(value):
|
|
return None
|
|
return value
|
|
|
|
|
|
def number(value):
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
try:
|
|
result = float(value)
|
|
return result if math.isfinite(result) else None
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def date(value):
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None
|
|
except (ValueError, TypeError, AttributeError):
|
|
return None
|
|
|
|
|
|
def code(value):
|
|
return value.get("code") if isinstance(value, dict) else value if isinstance(value, str) else None
|
|
|
|
|
|
async def upsert_alpha(db, raw: dict):
|
|
alpha_id = raw.get("id")
|
|
if not isinstance(alpha_id, str) or not alpha_id:
|
|
raise ValueError("Alpha 数据缺少 ID")
|
|
item = await db.get(Alpha, alpha_id)
|
|
previous_region = item.region if item else None
|
|
previous_status = item.status if item else None
|
|
if item is None:
|
|
item = Alpha(id=alpha_id)
|
|
db.add(item)
|
|
settings = raw.get("settings") or {}
|
|
metrics = raw.get("is") if isinstance(raw.get("is"), dict) else {}
|
|
item.name = raw.get("name")
|
|
item.expression = code(raw.get("regular"))
|
|
item.selection, item.combo = code(raw.get("selection")), code(raw.get("combo"))
|
|
item.alpha_type, item.language = raw.get("type"), settings.get("language")
|
|
item.stage, item.status, item.hidden = raw.get("stage"), raw.get("status"), raw.get("hidden") is True
|
|
item.region, item.universe = settings.get("region"), settings.get("universe")
|
|
if previous_region != item.region or previous_status != item.status:
|
|
regions = {
|
|
region
|
|
for region, status in ((previous_region, previous_status), (item.region, item.status))
|
|
if region and status and status != "UNSUBMITTED"
|
|
}
|
|
await invalidate_correlations(db, alpha_id, regions)
|
|
item.settings, item.is_metrics = sanitize(settings), sanitize(metrics)
|
|
item.os_metrics = sanitize(raw.get("os")) if isinstance(raw.get("os"), dict) else {}
|
|
item.checks = sanitize(metrics.get("checks") or raw.get("checks") or [])
|
|
for key in ("sharpe", "fitness", "returns", "turnover", "margin", "drawdown"):
|
|
setattr(item, key, number(metrics.get(key)))
|
|
item.date_created, item.date_submitted = date(raw.get("dateCreated")), date(raw.get("dateSubmitted"))
|
|
item.synced_at, item.raw = now(), sanitize(raw)
|
|
await db.flush()
|
|
if await db.get(Research, alpha_id) is None:
|
|
db.add(Research(alpha_id=alpha_id))
|
|
return item
|
|
|
|
|
|
def list_statement(filters):
|
|
query = select(Alpha, Research).join(Research, Research.alpha_id == Alpha.id)
|
|
if filters.submission:
|
|
query = query.where(submission_condition(filters.submission))
|
|
q = filters.q
|
|
if q:
|
|
pattern = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
|
|
query = query.where(
|
|
or_(
|
|
*(
|
|
getattr(Alpha, f).ilike(pattern, escape="\\")
|
|
for f in ("id", "name", "expression", "selection", "combo")
|
|
)
|
|
)
|
|
)
|
|
for name in ("region", "universe", "alpha_type", "language", "status", "stage", "hidden"):
|
|
value = getattr(filters, name)
|
|
if value is not None:
|
|
query = query.where(getattr(Alpha, name) == value)
|
|
if filters.research_state:
|
|
query = query.where(Research.state == filters.research_state)
|
|
if filters.favorite is not None:
|
|
query = query.where(Research.favorite == filters.favorite)
|
|
if filters.tag:
|
|
query = query.where(Alpha.id.in_(select(ResearchTag.alpha_id).where(ResearchTag.tag == filters.tag)))
|
|
if filters.created_from:
|
|
query = query.where(Alpha.date_created >= filters.created_from)
|
|
if filters.created_to:
|
|
query = query.where(Alpha.date_created <= filters.created_to)
|
|
for name in ("sharpe", "fitness", "returns", "turnover", "margin", "drawdown"):
|
|
for suffix, compare in (("min", "ge"), ("max", "le")):
|
|
value = getattr(filters, f"{name}_{suffix}")
|
|
if value is not None:
|
|
column = getattr(Alpha, name)
|
|
query = query.where(column >= value if compare == "ge" else column <= value)
|
|
return query
|
|
|
|
|
|
def sorted_statement(query, sort, direction):
|
|
column = getattr(Alpha, sort)
|
|
order = column.desc() if direction == "desc" else column.asc()
|
|
return query.order_by(order.nullslast(), Alpha.id.asc())
|
|
|
|
|
|
def summary(item: Alpha, research: Research):
|
|
keys = (
|
|
"id",
|
|
"name",
|
|
"alpha_type",
|
|
"language",
|
|
"stage",
|
|
"status",
|
|
"hidden",
|
|
"region",
|
|
"universe",
|
|
"sharpe",
|
|
"fitness",
|
|
"returns",
|
|
"turnover",
|
|
"margin",
|
|
"drawdown",
|
|
"date_created",
|
|
"date_submitted",
|
|
"synced_at",
|
|
)
|
|
result = {k: getattr(item, k) for k in keys}
|
|
result["expression_preview"] = (item.expression or item.selection or "")[:240]
|
|
result["research"] = {
|
|
k: getattr(research, k) for k in ("note", "tags", "favorite", "state", "updated_at", "version")
|
|
}
|
|
return result
|
|
|
|
|
|
def pnl_points(raw):
|
|
"""Use schema column names, preserving missing values rather than creating zero PnL."""
|
|
records = raw.get("records")
|
|
schema = raw.get("schema") or {}
|
|
properties = schema.get("properties", []) if isinstance(schema, dict) else schema
|
|
if isinstance(properties, dict):
|
|
names = list(properties)
|
|
else:
|
|
names = [p.get("name", "") if isinstance(p, dict) else str(p) for p in properties]
|
|
normalized = [name.lower() for name in names]
|
|
if not isinstance(records, list):
|
|
raise ValueError("PnL 缺少 records")
|
|
points = []
|
|
for row in records:
|
|
if isinstance(row, dict):
|
|
row = {str(k).lower(): v for k, v in row.items()}
|
|
timestamp = next((row[k] for k in ("date", "datetime", "timestamp") if k in row), None)
|
|
value = next((row[k] for k in ("pnl", "value") if k in row), None)
|
|
else:
|
|
date_i = next(
|
|
(i for i, n in enumerate(normalized) if n in ("date", "datetime", "timestamp")), None
|
|
)
|
|
pnl_i = next((i for i, n in enumerate(normalized) if n in ("pnl", "value")), None)
|
|
if date_i is None or pnl_i is None or not isinstance(row, list) or len(row) <= max(date_i, pnl_i):
|
|
raise ValueError("PnL schema 无法识别日期或数值列")
|
|
timestamp, value = row[date_i], row[pnl_i]
|
|
if timestamp is not None:
|
|
if isinstance(timestamp, (int, float)):
|
|
from datetime import timezone
|
|
|
|
timestamp = datetime.fromtimestamp(
|
|
timestamp / 1000 if timestamp > 1e11 else timestamp, tz=timezone.utc
|
|
).isoformat()
|
|
points.append({"date": str(timestamp), "value": number(value)})
|
|
return sorted(points, key=lambda p: p["date"])
|