286 lines
12 KiB
Python
286 lines
12 KiB
Python
|
"""Bounded FASTEXPR syntax analysis and mixed-radix sampling, without execution.
|
|||
|
|
|
|||
|
|
This parser establishes syntax and identifier provenance, not full BRAIN semantics.
|
|||
|
|
Unknown fields/operators must be resolved against snapshots before simulation.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import math
|
|||
|
|
import random
|
|||
|
|
import re
|
|||
|
|
from dataclasses import dataclass
|
|||
|
|
|
|||
|
|
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|||
|
|
LEGACY_PLACEHOLDER = re.compile(r"<([A-Za-z_][A-Za-z0-9_]*)/>")
|
|||
|
|
IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|||
|
|
GROUPS = {"sector", "industry", "subindustry", "market", "country", "exchange"}
|
|||
|
|
CONSTANTS = {"true", "false", "nan", "NaN", "inf"}
|
|||
|
|
TOKEN = re.compile(
|
|||
|
|
r"""\s*(?:(\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?)|([A-Za-z_][A-Za-z0-9_]*)|("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')|(==|!=|<=|>=|&&|\|\||\*\*|[()+\-*/%^<>=!?:,;]))"""
|
|||
|
|
)
|
|||
|
|
PRECEDENCE = {
|
|||
|
|
"||": 1,
|
|||
|
|
"&&": 2,
|
|||
|
|
"==": 3,
|
|||
|
|
"!=": 3,
|
|||
|
|
"<": 4,
|
|||
|
|
">": 4,
|
|||
|
|
"<=": 4,
|
|||
|
|
">=": 4,
|
|||
|
|
"+": 5,
|
|||
|
|
"-": 5,
|
|||
|
|
"*": 6,
|
|||
|
|
"/": 6,
|
|||
|
|
"%": 6,
|
|||
|
|
"^": 7,
|
|||
|
|
"**": 7,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class ExpressionError(ValueError):
|
|||
|
|
message: str
|
|||
|
|
position: int = 0
|
|||
|
|
|
|||
|
|
def __str__(self):
|
|||
|
|
return f"{self.message}（位置 {self.position + 1}）"
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Parser:
|
|||
|
|
def __init__(self, expression):
|
|||
|
|
if not expression.strip() or len(expression) > 20000:
|
|||
|
|
raise ExpressionError("表达式为空或超过 20000 字符")
|
|||
|
|
self.tokens = []
|
|||
|
|
position = 0
|
|||
|
|
while position < len(expression.rstrip()):
|
|||
|
|
match = TOKEN.match(expression, position)
|
|||
|
|
if not match:
|
|||
|
|
raise ExpressionError("无法识别的字符", position)
|
|||
|
|
self.tokens.append((match.lastindex, match.group(match.lastindex), match.start()))
|
|||
|
|
position = match.end()
|
|||
|
|
if len(self.tokens) > 5000:
|
|||
|
|
raise ExpressionError("表达式过于复杂")
|
|||
|
|
self.tokens.append((0, "EOF", len(expression)))
|
|||
|
|
self.i = 0
|
|||
|
|
self.locals = set()
|
|||
|
|
self.fields = set()
|
|||
|
|
self.operators = set()
|
|||
|
|
|
|||
|
|
def peek(self, offset=0):
|
|||
|
|
return self.tokens[min(self.i + offset, len(self.tokens) - 1)][1]
|
|||
|
|
|
|||
|
|
def take(self, expected=None):
|
|||
|
|
token = self.tokens[self.i]
|
|||
|
|
if expected and token[1] != expected:
|
|||
|
|
raise ExpressionError(f"需要 {expected}，实际为 {token[1]}", token[2])
|
|||
|
|
self.i += 1
|
|||
|
|
return token
|
|||
|
|
|
|||
|
|
def expression(self, minimum=0, depth=0):
|
|||
|
|
if depth > 64:
|
|||
|
|
raise ExpressionError("嵌套层数超过 64")
|
|||
|
|
kind, value, pos = self.take()
|
|||
|
|
if value in ("+", "-", "!"):
|
|||
|
|
left = {"kind": "unary", "value": value, "args": [self.expression(7, depth + 1)]}
|
|||
|
|
elif value == "(":
|
|||
|
|
left = self.expression(0, depth + 1)
|
|||
|
|
self.take(")")
|
|||
|
|
elif kind in (1, 3):
|
|||
|
|
if kind == 1 and not math.isfinite(float(value)):
|
|||
|
|
raise ExpressionError("数值必须有限", pos)
|
|||
|
|
left = {"kind": "number" if kind == 1 else "string", "value": value}
|
|||
|
|
elif kind == 2:
|
|||
|
|
if self.peek() == "(":
|
|||
|
|
self.operators.add(value)
|
|||
|
|
self.take("(")
|
|||
|
|
args, keywords = [], set()
|
|||
|
|
if self.peek() != ")":
|
|||
|
|
while True:
|
|||
|
|
keyword = None
|
|||
|
|
if self.tokens[self.i][0] == 2 and self.peek(1) == "=":
|
|||
|
|
keyword = self.take()[1]
|
|||
|
|
self.take("=")
|
|||
|
|
if keyword in keywords:
|
|||
|
|
raise ExpressionError("命名参数重复", pos)
|
|||
|
|
keywords.add(keyword)
|
|||
|
|
elif keywords:
|
|||
|
|
raise ExpressionError("位置参数不能出现在命名参数后", pos)
|
|||
|
|
argument = self.expression(0, depth + 1)
|
|||
|
|
args.append(
|
|||
|
|
{"kind": "keyword", "value": keyword, "args": [argument]} if keyword else argument
|
|||
|
|
)
|
|||
|
|
if self.peek() != ",":
|
|||
|
|
break
|
|||
|
|
self.take(",")
|
|||
|
|
self.take(")")
|
|||
|
|
left = {"kind": "call", "value": value, "args": args}
|
|||
|
|
else:
|
|||
|
|
if value not in self.locals and value not in CONSTANTS:
|
|||
|
|
self.fields.add(value)
|
|||
|
|
left = {"kind": "local" if value in self.locals else "field", "value": value}
|
|||
|
|
else:
|
|||
|
|
raise ExpressionError("需要字段、常量或算子调用", pos)
|
|||
|
|
while self.peek() in PRECEDENCE and PRECEDENCE[self.peek()] >= minimum:
|
|||
|
|
op = self.take()[1]
|
|||
|
|
right = self.expression(PRECEDENCE[op] + (0 if op in ("^", "**") else 1), depth + 1)
|
|||
|
|
left = {"kind": "binary", "value": op, "args": [left, right]}
|
|||
|
|
if minimum == 0 and self.peek() == "?":
|
|||
|
|
self.take("?")
|
|||
|
|
yes = self.expression(0, depth + 1)
|
|||
|
|
self.take(":")
|
|||
|
|
left = {"kind": "conditional", "args": [left, yes, self.expression(0, depth + 1)]}
|
|||
|
|
return left
|
|||
|
|
|
|||
|
|
def parse(self):
|
|||
|
|
statements = []
|
|||
|
|
final_is_assignment = False
|
|||
|
|
while self.peek() != "EOF":
|
|||
|
|
name = None
|
|||
|
|
if self.tokens[self.i][0] == 2 and self.peek(1) == "=":
|
|||
|
|
name = self.take()[1]
|
|||
|
|
self.take("=")
|
|||
|
|
node = self.expression()
|
|||
|
|
if name:
|
|||
|
|
self.locals.add(name)
|
|||
|
|
node = {"kind": "assignment", "value": name, "args": [node]}
|
|||
|
|
final_is_assignment = name is not None
|
|||
|
|
statements.append(node)
|
|||
|
|
if self.peek() != "EOF":
|
|||
|
|
self.take(";")
|
|||
|
|
if final_is_assignment:
|
|||
|
|
raise ExpressionError("最后一项必须是返回表达式")
|
|||
|
|
return {
|
|||
|
|
"ast": statements,
|
|||
|
|
"fields": sorted(self.fields),
|
|||
|
|
"operators": sorted(self.operators),
|
|||
|
|
"locals": sorted(self.locals),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def analyze(expression, fields=None, operators=None):
|
|||
|
|
"""Return separate syntax, type and availability findings; unknown never means valid."""
|
|||
|
|
try:
|
|||
|
|
parsed = Parser(expression).parse()
|
|||
|
|
except (ExpressionError, RecursionError) as exc:
|
|||
|
|
return {
|
|||
|
|
"status": "invalid",
|
|||
|
|
"syntax": [str(exc)],
|
|||
|
|
"types": [],
|
|||
|
|
"availability": [],
|
|||
|
|
"fields": [],
|
|||
|
|
"operators": [],
|
|||
|
|
"locals": [],
|
|||
|
|
}
|
|||
|
|
types, availability = [], []
|
|||
|
|
known = {**{name: "GROUP" for name in GROUPS}, **(fields or {})}
|
|||
|
|
for field in parsed["fields"]:
|
|||
|
|
if field not in known and field not in CONSTANTS:
|
|||
|
|
availability.append(f"字段 {field} 尚未在固定输入中核实")
|
|||
|
|
elif field in known and known[field] not in ("MATRIX", "VECTOR", "GROUP"):
|
|||
|
|
availability.append(f"字段 {field} 的类型尚不支持")
|
|||
|
|
for operator in parsed["operators"]:
|
|||
|
|
if operators is None or operator not in operators:
|
|||
|
|
availability.append(f"算子 {operator} 尚未在算子目录中核实")
|
|||
|
|
local_types = {}
|
|||
|
|
|
|||
|
|
def infer(node):
|
|||
|
|
kind, value = node["kind"], node.get("value")
|
|||
|
|
if kind == "field":
|
|||
|
|
if value in CONSTANTS:
|
|||
|
|
return "SCALAR"
|
|||
|
|
return known.get(value, "UNKNOWN")
|
|||
|
|
if kind in ("number", "string"):
|
|||
|
|
return "SCALAR" if kind == "number" else "STRING"
|
|||
|
|
if kind == "local":
|
|||
|
|
return local_types.get(value, "UNKNOWN")
|
|||
|
|
args = [infer(arg) for arg in node.get("args", [])]
|
|||
|
|
if kind == "assignment":
|
|||
|
|
local_types[value] = args[0]
|
|||
|
|
if kind == "call" and value.startswith("vec_"):
|
|||
|
|
if not args:
|
|||
|
|
types.append(f"{value} 缺少 VECTOR 参数")
|
|||
|
|
if args and args[0] not in ("VECTOR", "UNKNOWN"):
|
|||
|
|
types.append(f"{value} 的首个参数必须是 VECTOR")
|
|||
|
|
return "MATRIX"
|
|||
|
|
if kind == "call" and "VECTOR" in args:
|
|||
|
|
types.append(f"{value} 使用 VECTOR 前需要显式聚合")
|
|||
|
|
if kind == "call" and value in {
|
|||
|
|
"rank",
|
|||
|
|
"ts_rank",
|
|||
|
|
"ts_mean",
|
|||
|
|
"ts_sum",
|
|||
|
|
"ts_delta",
|
|||
|
|
"ts_std_dev",
|
|||
|
|
"zscore",
|
|||
|
|
"group_rank",
|
|||
|
|
"group_neutralize",
|
|||
|
|
}:
|
|||
|
|
minimum = 2 if value.startswith(("ts_", "group_")) else 1
|
|||
|
|
if len(args) < minimum:
|
|||
|
|
types.append(f"{value} 缺少必需参数")
|
|||
|
|
if args and args[0] == "VECTOR":
|
|||
|
|
types.append(f"{value} 不能直接使用 VECTOR，请显式选择聚合方法")
|
|||
|
|
if kind == "call" and value in {"group_rank", "group_neutralize", "group_zscore"}:
|
|||
|
|
if len(args) > 1 and args[1] not in ("GROUP", "UNKNOWN"):
|
|||
|
|
types.append(f"{value} 的分组参数必须是 GROUP")
|
|||
|
|
if kind == "binary" and "VECTOR" in args:
|
|||
|
|
types.append("VECTOR 参与数值运算前需要显式聚合")
|
|||
|
|
if "VECTOR" in args:
|
|||
|
|
return "VECTOR"
|
|||
|
|
return args[0] if kind in ("unary", "keyword", "assignment") and args else "MATRIX"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
result_type = None
|
|||
|
|
for node in parsed.pop("ast"):
|
|||
|
|
result_type = infer(node)
|
|||
|
|
if result_type == "VECTOR":
|
|||
|
|
types.append("最终 Alpha 输出不能直接是 VECTOR，请显式选择聚合方法")
|
|||
|
|
except RecursionError:
|
|||
|
|
types.append("表达式推导过于复杂，请拆分局部变量")
|
|||
|
|
return {
|
|||
|
|
**parsed,
|
|||
|
|
"syntax": [],
|
|||
|
|
"types": list(dict.fromkeys(types)),
|
|||
|
|
"availability": availability,
|
|||
|
|
"status": "invalid" if types else "needs_review" if availability else "valid",
|
|||
|
|
"limitation": "仅验证支持的语法、字段归属及已知类型约束；平台语义与权限以实际模拟为准",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_template(expression):
|
|||
|
|
return LEGACY_PLACEHOLDER.sub(lambda match: "{" + match[1] + "}", expression)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def expand(expression, variables, mode="all", limit=100, seed=0):
|
|||
|
|
"""Sample integer indices in the Cartesian space without materializing that space."""
|
|||
|
|
expression = normalize_template(expression)
|
|||
|
|
names = list(dict.fromkeys(PLACEHOLDER.findall(expression)))
|
|||
|
|
if set(names) != set(variables) or any(not values for values in variables.values()):
|
|||
|
|
raise ValueError("占位符必须与非空变量候选逐一对应")
|
|||
|
|
if "{" in PLACEHOLDER.sub("", expression) or "}" in PLACEHOLDER.sub("", expression):
|
|||
|
|
raise ValueError("占位符格式应为 {name}")
|
|||
|
|
total = math.prod(len(variables[name]) for name in names)
|
|||
|
|
if not 1 <= limit <= 10000:
|
|||
|
|
raise ValueError("生成上限必须在 1–10000 之间")
|
|||
|
|
if mode == "all" and total > limit:
|
|||
|
|
raise ValueError(f"组合数 {total} 超过上限 {limit}，请缩小候选或使用随机采样")
|
|||
|
|
count = min(total, limit)
|
|||
|
|
if mode == "random":
|
|||
|
|
# Floyd sampling supports arbitrary-size integers (random.sample(range(N)) does not).
|
|||
|
|
rng, chosen = random.Random(seed), set()
|
|||
|
|
for j in range(total - count, total):
|
|||
|
|
candidate = rng.randrange(j + 1)
|
|||
|
|
chosen.add(j if candidate in chosen else candidate)
|
|||
|
|
indices = sorted(chosen)
|
|||
|
|
else:
|
|||
|
|
indices = range(count)
|
|||
|
|
results = []
|
|||
|
|
for index in indices:
|
|||
|
|
bindings = {}
|
|||
|
|
for name in reversed(names):
|
|||
|
|
values = variables[name]
|
|||
|
|
index, digit = divmod(index, len(values))
|
|||
|
|
bindings[name] = values[digit]
|
|||
|
|
text = PLACEHOLDER.sub(lambda match: str(bindings[match[1]]), expression)
|
|||
|
|
results.append({"expression": text, "bindings": bindings})
|
|||
|
|
return {"combination_count": str(total), "seed": seed if mode == "random" else None, "items": results}
|