"""Bounded FASTEXPR syntax analysis and mixed-radix sampling, without execution. This parser establishes syntax and identifier provenance, not full BRAIN semantics. Unknown fields/operators must be resolved against snapshots before simulation. """ import math import random import re from dataclasses import dataclass PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}") LEGACY_PLACEHOLDER = re.compile(r"<([A-Za-z_][A-Za-z0-9_]*)/>") IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") GROUPS = {"sector", "industry", "subindustry", "market", "country", "exchange"} CONSTANTS = {"true", "false", "nan", "NaN", "inf"} TOKEN = re.compile( r"""\s*(?:(\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?)|([A-Za-z_][A-Za-z0-9_]*)|("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')|(==|!=|<=|>=|&&|\|\||\*\*|[()+\-*/%^<>=!?:,;]))""" ) PRECEDENCE = { "||": 1, "&&": 2, "==": 3, "!=": 3, "<": 4, ">": 4, "<=": 4, ">=": 4, "+": 5, "-": 5, "*": 6, "/": 6, "%": 6, "^": 7, "**": 7, } @dataclass class ExpressionError(ValueError): message: str position: int = 0 def __str__(self): return f"{self.message}(位置 {self.position + 1})" class Parser: def __init__(self, expression): if not expression.strip() or len(expression) > 20000: raise ExpressionError("表达式为空或超过 20000 字符") self.tokens = [] position = 0 while position < len(expression.rstrip()): match = TOKEN.match(expression, position) if not match: raise ExpressionError("无法识别的字符", position) self.tokens.append((match.lastindex, match.group(match.lastindex), match.start())) position = match.end() if len(self.tokens) > 5000: raise ExpressionError("表达式过于复杂") self.tokens.append((0, "EOF", len(expression))) self.i = 0 self.locals = set() self.fields = set() self.operators = set() def peek(self, offset=0): return self.tokens[min(self.i + offset, len(self.tokens) - 1)][1] def take(self, expected=None): token = self.tokens[self.i] if expected and token[1] != expected: raise ExpressionError(f"需要 {expected},实际为 {token[1]}", token[2]) self.i += 1 return token def expression(self, minimum=0, depth=0): if depth > 64: raise ExpressionError("嵌套层数超过 64") kind, value, pos = self.take() if value in ("+", "-", "!"): left = {"kind": "unary", "value": value, "args": [self.expression(7, depth + 1)]} elif value == "(": left = self.expression(0, depth + 1) self.take(")") elif kind in (1, 3): if kind == 1 and not math.isfinite(float(value)): raise ExpressionError("数值必须有限", pos) left = {"kind": "number" if kind == 1 else "string", "value": value} elif kind == 2: if self.peek() == "(": self.operators.add(value) self.take("(") args, keywords = [], set() if self.peek() != ")": while True: keyword = None if self.tokens[self.i][0] == 2 and self.peek(1) == "=": keyword = self.take()[1] self.take("=") if keyword in keywords: raise ExpressionError("命名参数重复", pos) keywords.add(keyword) elif keywords: raise ExpressionError("位置参数不能出现在命名参数后", pos) argument = self.expression(0, depth + 1) args.append( {"kind": "keyword", "value": keyword, "args": [argument]} if keyword else argument ) if self.peek() != ",": break self.take(",") self.take(")") left = {"kind": "call", "value": value, "args": args} else: if value not in self.locals and value not in CONSTANTS: self.fields.add(value) left = {"kind": "local" if value in self.locals else "field", "value": value} else: raise ExpressionError("需要字段、常量或算子调用", pos) while self.peek() in PRECEDENCE and PRECEDENCE[self.peek()] >= minimum: op = self.take()[1] right = self.expression(PRECEDENCE[op] + (0 if op in ("^", "**") else 1), depth + 1) left = {"kind": "binary", "value": op, "args": [left, right]} if minimum == 0 and self.peek() == "?": self.take("?") yes = self.expression(0, depth + 1) self.take(":") left = {"kind": "conditional", "args": [left, yes, self.expression(0, depth + 1)]} return left def parse(self): statements = [] final_is_assignment = False while self.peek() != "EOF": name = None if self.tokens[self.i][0] == 2 and self.peek(1) == "=": name = self.take()[1] self.take("=") node = self.expression() if name: self.locals.add(name) node = {"kind": "assignment", "value": name, "args": [node]} final_is_assignment = name is not None statements.append(node) if self.peek() != "EOF": self.take(";") if final_is_assignment: raise ExpressionError("最后一项必须是返回表达式") return { "ast": statements, "fields": sorted(self.fields), "operators": sorted(self.operators), "locals": sorted(self.locals), } def analyze(expression, fields=None, operators=None): """Return separate syntax, type and availability findings; unknown never means valid.""" try: parsed = Parser(expression).parse() except (ExpressionError, RecursionError) as exc: return { "status": "invalid", "syntax": [str(exc)], "types": [], "availability": [], "fields": [], "operators": [], "locals": [], } types, availability = [], [] known = {**{name: "GROUP" for name in GROUPS}, **(fields or {})} for field in parsed["fields"]: if field not in known and field not in CONSTANTS: availability.append(f"字段 {field} 尚未在固定输入中核实") elif field in known and known[field] not in ("MATRIX", "VECTOR", "GROUP"): availability.append(f"字段 {field} 的类型尚不支持") for operator in parsed["operators"]: if operators is None or operator not in operators: availability.append(f"算子 {operator} 尚未在算子目录中核实") local_types = {} def infer(node): kind, value = node["kind"], node.get("value") if kind == "field": if value in CONSTANTS: return "SCALAR" return known.get(value, "UNKNOWN") if kind in ("number", "string"): return "SCALAR" if kind == "number" else "STRING" if kind == "local": return local_types.get(value, "UNKNOWN") args = [infer(arg) for arg in node.get("args", [])] if kind == "assignment": local_types[value] = args[0] if kind == "call" and value.startswith("vec_"): if not args: types.append(f"{value} 缺少 VECTOR 参数") if args and args[0] not in ("VECTOR", "UNKNOWN"): types.append(f"{value} 的首个参数必须是 VECTOR") return "MATRIX" if kind == "call" and "VECTOR" in args: types.append(f"{value} 使用 VECTOR 前需要显式聚合") if kind == "call" and value in { "rank", "ts_rank", "ts_mean", "ts_sum", "ts_delta", "ts_std_dev", "zscore", "group_rank", "group_neutralize", }: minimum = 2 if value.startswith(("ts_", "group_")) else 1 if len(args) < minimum: types.append(f"{value} 缺少必需参数") if args and args[0] == "VECTOR": types.append(f"{value} 不能直接使用 VECTOR,请显式选择聚合方法") if kind == "call" and value in {"group_rank", "group_neutralize", "group_zscore"}: if len(args) > 1 and args[1] not in ("GROUP", "UNKNOWN"): types.append(f"{value} 的分组参数必须是 GROUP") if kind == "binary" and "VECTOR" in args: types.append("VECTOR 参与数值运算前需要显式聚合") if "VECTOR" in args: return "VECTOR" return args[0] if kind in ("unary", "keyword", "assignment") and args else "MATRIX" try: result_type = None for node in parsed.pop("ast"): result_type = infer(node) if result_type == "VECTOR": types.append("最终 Alpha 输出不能直接是 VECTOR,请显式选择聚合方法") except RecursionError: types.append("表达式推导过于复杂,请拆分局部变量") return { **parsed, "syntax": [], "types": list(dict.fromkeys(types)), "availability": availability, "status": "invalid" if types else "needs_review" if availability else "valid", "limitation": "仅验证支持的语法、字段归属及已知类型约束;平台语义与权限以实际模拟为准", } def normalize_template(expression): return LEGACY_PLACEHOLDER.sub(lambda match: "{" + match[1] + "}", expression) def expand(expression, variables, mode="all", limit=100, seed=0): """Sample integer indices in the Cartesian space without materializing that space.""" expression = normalize_template(expression) names = list(dict.fromkeys(PLACEHOLDER.findall(expression))) if set(names) != set(variables) or any(not values for values in variables.values()): raise ValueError("占位符必须与非空变量候选逐一对应") if "{" in PLACEHOLDER.sub("", expression) or "}" in PLACEHOLDER.sub("", expression): raise ValueError("占位符格式应为 {name}") total = math.prod(len(variables[name]) for name in names) if not 1 <= limit <= 10000: raise ValueError("生成上限必须在 1–10000 之间") if mode == "all" and total > limit: raise ValueError(f"组合数 {total} 超过上限 {limit},请缩小候选或使用随机采样") count = min(total, limit) if mode == "random": # Floyd sampling supports arbitrary-size integers (random.sample(range(N)) does not). rng, chosen = random.Random(seed), set() for j in range(total - count, total): candidate = rng.randrange(j + 1) chosen.add(j if candidate in chosen else candidate) indices = sorted(chosen) else: indices = range(count) results = [] for index in indices: bindings = {} for name in reversed(names): values = variables[name] index, digit = divmod(index, len(values)) bindings[name] = values[digit] text = PLACEHOLDER.sub(lambda match: str(bindings[match[1]]), expression) results.append({"expression": text, "bindings": bindings}) return {"combination_count": str(total), "seed": seed if mode == "random" else None, "items": results}