feat: add durable WorldQuant backtests with UI and AI confirmation
This commit is contained in:
@@ -17,6 +17,23 @@ async def fake_stream(messages, info):
|
||||
str(p.content) for m in messages[latest:] for p in m.parts if isinstance(p, UserPromptPart)
|
||||
)
|
||||
returns = [p for m in messages[latest:] for p in m.parts if isinstance(p, ToolReturnPart)]
|
||||
if returns and returns[-1].tool_name == "prepare_backtest":
|
||||
content = returns[-1].content
|
||||
content = json.loads(content) if isinstance(content, str) else content
|
||||
yield {
|
||||
0: DeltaToolCall(
|
||||
name="start_backtest",
|
||||
json_args=json.dumps(
|
||||
{
|
||||
"preview_id": content["preview_id"],
|
||||
"version": 1,
|
||||
"idempotency_key": content["preview_id"],
|
||||
}
|
||||
),
|
||||
tool_call_id=uuid4().hex,
|
||||
)
|
||||
}
|
||||
return
|
||||
if returns and "LOOP" not in text:
|
||||
if returns[-1].tool_name == "capability_probe":
|
||||
yield str(returns[-1].content)
|
||||
@@ -35,6 +52,23 @@ async def fake_stream(messages, info):
|
||||
await asyncio.sleep(2)
|
||||
yield ",查询完成。"
|
||||
return
|
||||
elif "回测" in text:
|
||||
name, args = (
|
||||
"prepare_backtest",
|
||||
{
|
||||
"inline": {
|
||||
"name": "AI 固定回测",
|
||||
"source": {"kind": "ai"},
|
||||
"candidates": [
|
||||
{
|
||||
"client_item_id": "ai-1",
|
||||
"expression": "rank(close)",
|
||||
"settings": {"region": "USA", "universe": "TOP3000", "delay": 1},
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
elif "批量" in text:
|
||||
name, args = "bulk_update_research", {"alpha_ids": ["a0000", "a0001"], "add_tags": ["AI"]}
|
||||
elif "修改" in text or "update" in text:
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Synthetic simulation HTTP used by isolated API and browser acceptance."""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class Platform:
|
||||
def __init__(self):
|
||||
self.posts = []
|
||||
self.existing_alpha_ids = None
|
||||
self.simulations = {}
|
||||
self.alphas = {}
|
||||
self.reject = None
|
||||
self.pending = False
|
||||
self.detail_fail = False
|
||||
self.fail_child = None
|
||||
self.missing = False
|
||||
self.secret = "synthetic-platform-secret"
|
||||
|
||||
def __call__(self, request):
|
||||
path = request.url.path
|
||||
if path == "/authentication":
|
||||
return httpx.Response(201, json={})
|
||||
if path == "/simulations" and request.method == "POST":
|
||||
data = json.loads(request.content)
|
||||
data = data if isinstance(data, list) else [data]
|
||||
self.posts.append(data)
|
||||
if self.reject == "unknown":
|
||||
raise httpx.ReadTimeout("synthetic timeout", request=request)
|
||||
if self.reject == "session":
|
||||
self.reject = None
|
||||
return httpx.Response(401)
|
||||
if self.reject == "rate":
|
||||
return httpx.Response(429, headers={"Retry-After": "0.01"})
|
||||
if self.reject == "bad":
|
||||
return httpx.Response(400, json={"error": self.secret})
|
||||
parent = f"p{len(self.posts)}"
|
||||
ids = []
|
||||
for i, item in enumerate(data):
|
||||
child = parent if len(data) == 1 else f"{parent}c{i}"
|
||||
aid = self.existing_alpha_ids[i] if self.existing_alpha_ids else f"alpha{parent}{i}"
|
||||
progress = {
|
||||
"status": "COMPLETE",
|
||||
"alpha": aid,
|
||||
"regular": item["regular"],
|
||||
"settings": item["settings"],
|
||||
}
|
||||
if i == self.fail_child:
|
||||
progress = {
|
||||
"status": "FAILED",
|
||||
"regular": item["regular"],
|
||||
"settings": item["settings"],
|
||||
"message": "invalid expression",
|
||||
}
|
||||
self.simulations[child] = progress
|
||||
self.alphas[aid] = {
|
||||
"id": aid,
|
||||
"regular": {"code": item["regular"]},
|
||||
"type": "REGULAR",
|
||||
"settings": item["settings"],
|
||||
"is": {"sharpe": None, "fitness": 0.8},
|
||||
"status": "UNSUBMITTED",
|
||||
}
|
||||
ids.append(child)
|
||||
if len(data) > 1:
|
||||
self.simulations[parent] = {
|
||||
"status": "COMPLETE",
|
||||
"children": list(reversed(ids[1:] if self.missing else ids)),
|
||||
}
|
||||
if self.reject == "missing_location":
|
||||
return httpx.Response(201)
|
||||
return httpx.Response(
|
||||
201, headers={"Location": f"https://api.worldquantbrain.com/simulations/{parent}"}
|
||||
)
|
||||
if path.startswith("/simulations/"):
|
||||
return httpx.Response(
|
||||
200, json={"status": "PENDING"} if self.pending else self.simulations[path.rsplit("/", 1)[-1]]
|
||||
)
|
||||
if path.startswith("/alphas/"):
|
||||
if self.detail_fail:
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, json=self.alphas[path.rsplit("/", 1)[-1]])
|
||||
if path == "/users/self":
|
||||
return httpx.Response(200, json={"id": "TEST_USER"})
|
||||
if path.startswith("/users/self/"):
|
||||
return httpx.Response(200, json={"results": [], "count": 0})
|
||||
raise AssertionError(f"Unexpected HTTP {request.method} {path}")
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Isolated PostgreSQL migration/concurrency acceptance. Never point at a personal database.
|
||||
|
||||
Run with DATABASE_URL ending in /wq_backtest_test, synthetic ADMIN_PASSWORD and
|
||||
ENCRYPTION_KEY. Uses only mock WorldQuant HTTP and a disposable database.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import httpx
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.alphas import upsert_alpha
|
||||
from app.config import Settings
|
||||
from app.db import create_database
|
||||
from app.main import create_app
|
||||
from app.models import BacktestEvent, BacktestResult, BacktestRun, Research, SimulationAttempt
|
||||
from app.worldquant import WqClient
|
||||
from tests.backtest_fake import Platform
|
||||
from tests.test_backtests import candidate, preview, setup, start, tick
|
||||
|
||||
|
||||
async def seed_old(settings):
|
||||
engine, sessions = create_database(settings.database_url)
|
||||
async with sessions.begin() as db:
|
||||
await upsert_alpha(
|
||||
db,
|
||||
{
|
||||
"id": "MIGRATION_ALPHA",
|
||||
"type": "REGULAR",
|
||||
"regular": {"code": "rank(close) + 0"},
|
||||
"settings": candidate()["settings"],
|
||||
},
|
||||
)
|
||||
await db.flush()
|
||||
research = await db.get(Research, "MIGRATION_ALPHA")
|
||||
research.note = "keep old research across upgrade and simulations"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def acceptance(settings):
|
||||
fake = Platform()
|
||||
app = create_app(settings, WqClient(settings, transport=httpx.MockTransport(fake)))
|
||||
async with app.router.lifespan_context(app):
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app),
|
||||
base_url="http://testserver",
|
||||
headers={"X-WQ-Request": "1"},
|
||||
) as client:
|
||||
assert (
|
||||
await client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": "admin", "password": settings.admin_password.get_secret_value()},
|
||||
)
|
||||
).status_code == 200
|
||||
fake, lane = await setup(app)
|
||||
fake.existing_alpha_ids = ["MIGRATION_ALPHA"]
|
||||
p = await preview(client, [candidate(0), candidate(0) | {"client_item_id": "repeat"}])
|
||||
a, b = await asyncio.gather(
|
||||
start(client, p, "concurrent-confirm"), start(client, p, "concurrent-confirm")
|
||||
)
|
||||
assert a["backtest_run_id"] == b["backtest_run_id"]
|
||||
rid = a["backtest_run_id"]
|
||||
for _ in range(5):
|
||||
await tick(lane)
|
||||
result = (await client.get(f"/api/v1/backtests/runs/{rid}")).json()
|
||||
assert result["status"] == "completed", result
|
||||
assert len(fake.posts) == 2
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestRun)) == 1
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestResult)) == 2
|
||||
note = (await db.get(Research, "MIGRATION_ALPHA")).note
|
||||
assert note == "keep old research across upgrade and simulations"
|
||||
events = list(
|
||||
await db.scalars(
|
||||
select(BacktestEvent.seq)
|
||||
.where(BacktestEvent.run_id == rid)
|
||||
.order_by(BacktestEvent.seq)
|
||||
)
|
||||
)
|
||||
assert events == list(range(1, len(events) + 1))
|
||||
# Leave an accepted run for a new application instance to recover.
|
||||
next_run = await start(client, await preview(client, [candidate(2)]), "restart")
|
||||
async with app.state.sessions() as db:
|
||||
aid = await db.scalar(
|
||||
select(SimulationAttempt.id).where(
|
||||
SimulationAttempt.run_id == next_run["backtest_run_id"]
|
||||
)
|
||||
)
|
||||
await lane.step(aid)
|
||||
await lane.interrupt()
|
||||
replacement = create_app(settings, WqClient(settings, transport=httpx.MockTransport(fake)))
|
||||
async with replacement.router.lifespan_context(replacement):
|
||||
lane = replacement.state.runner.backtests
|
||||
await lane.start()
|
||||
await lane.stop()
|
||||
await lane.step(aid)
|
||||
async with replacement.state.sessions() as db:
|
||||
assert (await db.get(BacktestRun, next_run["backtest_run_id"])).status == "completed"
|
||||
assert len(fake.posts) == 3
|
||||
print(
|
||||
"PASS PostgreSQL: concurrent confirmation creates one run; two attempts share one Alpha safely; contiguous transactional events; research preserved; replacement application resumes accepted simulation without POST"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if not os.environ.get("DATABASE_URL", "").endswith("/wq_backtest_test"):
|
||||
raise SystemExit("Only an isolated wq_backtest_test database is allowed")
|
||||
settings = Settings(_env_file=None, enable_runner=False, public_origin="http://testserver")
|
||||
config = Config("alembic.ini")
|
||||
command.upgrade(config, "0002")
|
||||
asyncio.run(seed_old(settings))
|
||||
command.upgrade(config, "head")
|
||||
command.check(config)
|
||||
asyncio.run(acceptance(settings))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -12,6 +12,7 @@ from app.main import create_app
|
||||
from app.models import Base
|
||||
from app.worldquant import WqClient
|
||||
from tests.ai_fake import fake_model
|
||||
from tests.backtest_fake import Platform
|
||||
|
||||
TEST_PASSWORD = "browser-test-password"
|
||||
|
||||
@@ -78,6 +79,8 @@ def create_test_app():
|
||||
public_origin="http://127.0.0.1:5179",
|
||||
)
|
||||
records = [sample(i) for i in range(620)]
|
||||
simulations = Platform()
|
||||
simulations.existing_alpha_ids = [f"TEST{i:04}" for i in range(1, 100)]
|
||||
|
||||
def upstream(request):
|
||||
path = request.url.path
|
||||
@@ -102,6 +105,10 @@ def create_test_app():
|
||||
},
|
||||
headers={"Set-Cookie": "mock=only; Path=/"},
|
||||
)
|
||||
if path.startswith("/simulations") or (
|
||||
path.startswith("/alphas/") and path.rsplit("/", 1)[-1] in simulations.alphas
|
||||
):
|
||||
return simulations(request)
|
||||
if request.method != "GET":
|
||||
raise AssertionError("Browser acceptance attempted an upstream mutation")
|
||||
if path == "/users/self":
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
"""End-to-end business tests: real persistence/runtime, only the platform HTTP is replaced."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.backtests.contracts import SimulationSettings
|
||||
from app.models import Account, Alpha, BacktestResult, BacktestRun, Research, SimulationAttempt
|
||||
from app.security import cipher
|
||||
from app.worldquant import WqClient
|
||||
from tests.backtest_fake import Platform
|
||||
|
||||
PREFIX = "/api/v1/backtests"
|
||||
PARAMS = SimulationSettings(region="USA", universe="TOP3000", delay=1).model_dump()
|
||||
|
||||
|
||||
def candidate(index=0, **settings):
|
||||
return {
|
||||
"client_item_id": f"item-{index}",
|
||||
"expression": f"rank(close) + {index}",
|
||||
"settings": PARAMS | settings,
|
||||
}
|
||||
|
||||
|
||||
async def setup(app):
|
||||
platform = Platform()
|
||||
runner = app.state.runner
|
||||
await runner.client.close()
|
||||
runner.client = WqClient(app.state.settings, transport=httpx.MockTransport(platform))
|
||||
runner.backtests.client = runner.client
|
||||
runner.backtests.poll_interval = 0
|
||||
async with app.state.sessions.begin() as db:
|
||||
account = await db.get(Account, 1)
|
||||
account.email, account.wq_user_id, account.connection_status = (
|
||||
"synthetic@example.com",
|
||||
"TEST_USER",
|
||||
"connected",
|
||||
)
|
||||
account.password_encrypted = cipher(app.state.settings).encrypt(platform.secret.encode()).decode()
|
||||
return platform, runner.backtests
|
||||
|
||||
|
||||
async def preview(client, candidates=None):
|
||||
response = await client.post(
|
||||
f"{PREFIX}/previews",
|
||||
json={
|
||||
"inline": {
|
||||
"name": "测试研究",
|
||||
"source": {"kind": "test"},
|
||||
"candidates": candidates or [candidate()],
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
return response.json()
|
||||
|
||||
|
||||
async def start(client, p, key="request-1"):
|
||||
response = await client.post(
|
||||
f"{PREFIX}/runs",
|
||||
json={"preview_id": p["preview_id"], "version": p["version"], "idempotency_key": key},
|
||||
)
|
||||
assert response.status_code == 202, response.text
|
||||
return response.json()
|
||||
|
||||
|
||||
async def execute(app, lane, run_id):
|
||||
async with app.state.sessions() as db:
|
||||
ids = list(
|
||||
await db.scalars(
|
||||
select(SimulationAttempt.id)
|
||||
.where(SimulationAttempt.run_id == run_id)
|
||||
.order_by(SimulationAttempt.ordinal)
|
||||
)
|
||||
)
|
||||
for aid in ids:
|
||||
await lane.step(aid)
|
||||
await lane.step(aid)
|
||||
return ids
|
||||
|
||||
|
||||
async def test_fixed_preview_grouping_mapping_and_history(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
p = await preview(logged_in, [candidate(0), candidate(1, universe="TOP1000"), candidate(2, delay=0)])
|
||||
assert p["batch_count"] == 2 and p["total"] == 3
|
||||
run = await start(logged_in, p)
|
||||
again = await start(logged_in, p)
|
||||
assert run["backtest_run_id"] == again["backtest_run_id"]
|
||||
rid = run["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
data = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()
|
||||
assert len(platform.posts) == 2
|
||||
assert all(i["persistence_status"] == "saved" for i in data["items"]), data
|
||||
for item in data["items"]:
|
||||
assert item["result"]["snapshot"]["regular"]["code"] == item["expression"]
|
||||
assert item["result"]["snapshot"]["settings"] == item["settings"]
|
||||
assert item["result"]["snapshot"]["is"]["sharpe"] is None
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
item = data["items"][0]
|
||||
async with app.state.sessions.begin() as db:
|
||||
alpha = await db.get(Alpha, item["alpha_id"])
|
||||
alpha.is_metrics = {"sharpe": 999}
|
||||
assert await db.get(Research, alpha.id)
|
||||
historical = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()
|
||||
assert historical["items"][0]["result"]["snapshot"]["is"]["sharpe"] is None
|
||||
events = (await logged_in.get(f"{PREFIX}/runs/{rid}/events?limit=2")).json()
|
||||
later = (await logged_in.get(f"{PREFIX}/runs/{rid}/events?after={events['next_cursor']}")).json()
|
||||
assert events["has_more"] and later["items"][0]["seq"] > events["next_cursor"]
|
||||
assert (await preview(logged_in))["duplicate_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rejection", ["unknown", "missing_location"])
|
||||
async def test_unknown_submission_never_reposted(app, logged_in, rejection):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = rejection
|
||||
run = await start(logged_in, await preview(logged_in))
|
||||
rid = run["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
# A process crash/recovery must not turn an unknown POST into queued work.
|
||||
await lane.start()
|
||||
await lane.stop()
|
||||
response = await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
assert response.json()["status"] == "needs_review"
|
||||
async with app.state.sessions() as db:
|
||||
assert (await db.get(SimulationAttempt, ids[0])).state == "needs_review"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_partial_failure_and_rerun_only_selected(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.fail_child = 0
|
||||
run = await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)]))
|
||||
rid = run["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
result = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert result[0]["platform_status"] == "failed" and result[1]["persistence_status"] == "saved"
|
||||
rerun = await logged_in.post(f"{PREFIX}/runs/{rid}/rerun-preview", json={"item_ids": [result[0]["id"]]})
|
||||
assert rerun.status_code == 201
|
||||
assert rerun.json()["total"] == 1 and rerun.json()["source"]["parent_run_id"] == rid
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_detail_failure_recovers_without_resubmit(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.detail_fail = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "needs_review"
|
||||
platform.detail_fail = False
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_draft_version_snapshot_and_pause_stop(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
body = {"name": "草稿", "candidates": [candidate(0), candidate(1, delay=0)]}
|
||||
d = (await logged_in.post(f"{PREFIX}/drafts", json=body)).json()
|
||||
p = (await logged_in.post(f"{PREFIX}/previews", json={"draft_id": d["id"], "draft_version": 1})).json()
|
||||
changed = await logged_in.put(
|
||||
f"{PREFIX}/drafts/{d['id']}", json=body | {"version": 1, "candidates": [candidate(9)]}
|
||||
)
|
||||
assert changed.json()["version"] == 2
|
||||
assert (
|
||||
await logged_in.post(f"{PREFIX}/previews", json={"draft_id": d["id"], "draft_version": 1})
|
||||
).status_code == 409
|
||||
run = await start(logged_in, p)
|
||||
rid = run["backtest_run_id"]
|
||||
async with app.state.sessions() as db:
|
||||
ids = list(
|
||||
await db.scalars(
|
||||
select(SimulationAttempt.id)
|
||||
.where(SimulationAttempt.run_id == rid)
|
||||
.order_by(SimulationAttempt.ordinal)
|
||||
)
|
||||
)
|
||||
await lane.step(ids[0])
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "pause", "version": 1})
|
||||
await lane.step(ids[1])
|
||||
await lane.step(ids[0])
|
||||
assert len(platform.posts) == 1
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "stop", "version": 2})
|
||||
r = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert r[0]["persistence_status"] == "saved" and r[1]["platform_status"] == "skipped"
|
||||
assert r[0]["expression"] == candidate(0)["expression"]
|
||||
|
||||
|
||||
async def test_batch_missing_child_does_not_misattribute(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.missing = True
|
||||
rid = (await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)])))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
items = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert items[0]["platform_status"] == "unknown"
|
||||
assert items[1]["persistence_status"] == "saved"
|
||||
platform.simulations["p1"]["children"] = ["p1c1", "p1c0"]
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_validation_auth_and_idempotency_conflict(app, logged_in, client):
|
||||
await setup(app)
|
||||
assert (
|
||||
await logged_in.post(
|
||||
f"{PREFIX}/previews",
|
||||
json={"inline": {"name": "x", "candidates": [candidate() | {"alpha_type": "SUPER"}]}},
|
||||
)
|
||||
).status_code == 422
|
||||
p1, p2 = await preview(logged_in), await preview(logged_in, [candidate(2)])
|
||||
await start(logged_in, p1)
|
||||
assert (
|
||||
await logged_in.post(
|
||||
f"{PREFIX}/runs", json={"preview_id": p2["preview_id"], "idempotency_key": "request-1"}
|
||||
)
|
||||
).status_code == 409
|
||||
assert (await logged_in.get(f"{PREFIX}/runs?limit=101")).status_code == 422
|
||||
await client.post("/api/v1/auth/logout")
|
||||
assert (await client.get(f"{PREFIX}/runs")).status_code == 401
|
||||
|
||||
|
||||
async def tick(lane):
|
||||
await lane.tick()
|
||||
await asyncio.gather(*lane.tasks.values(), return_exceptions=False)
|
||||
|
||||
|
||||
async def test_account_budget_round_robin_and_sync_independence(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
assert (
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 1, "version": 1})
|
||||
).status_code == 200
|
||||
r1 = await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)]), "first")
|
||||
r2 = await start(logged_in, await preview(logged_in, [candidate(2), candidate(3)]), "second")
|
||||
await tick(lane) # one submission, occupied until remote terminal
|
||||
assert len(platform.posts) == 1
|
||||
await tick(lane) # poll first result
|
||||
await tick(lane) # other run gets next slot
|
||||
assert len(platform.posts) == 2
|
||||
assert platform.posts[0][0]["regular"] == candidate(0)["expression"]
|
||||
assert platform.posts[1][0]["regular"] == candidate(2)["expression"]
|
||||
platform.pending = True
|
||||
sync = await logged_in.post("/api/v1/sync-jobs", json={"kind": "full_sync"})
|
||||
await app.state.runner.run_next()
|
||||
assert (await logged_in.get(f"/api/v1/sync-jobs/{sync.json()['id']}")).json()["status"] == "completed"
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 2, "batch_size": 8, "version": 2})
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 3
|
||||
# Batch sizing of both existing runs remains 1 despite config update.
|
||||
assert all(len(p) == 1 for p in platform.posts)
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 8, "version": 3})
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 3
|
||||
await lane.interrupt()
|
||||
assert r1["batch_size"] == r2["batch_size"] == 1
|
||||
|
||||
|
||||
async def test_rate_limit_and_failed_submit_are_bounded(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "rate"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
async with app.state.sessions() as db:
|
||||
aid = await db.scalar(select(SimulationAttempt.id).where(SimulationAttempt.run_id == rid))
|
||||
for _ in range(app.state.settings.retry_attempts):
|
||||
await lane.step(aid)
|
||||
await asyncio.sleep(0.02)
|
||||
assert len(platform.posts) == app.state.settings.retry_attempts
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed_with_errors"
|
||||
assert platform.secret not in (await logged_in.get(f"{PREFIX}/runs/{rid}/attempts")).text
|
||||
|
||||
|
||||
async def test_poll_timeout_and_crash_after_acceptance(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
lane.poll_limit = 1
|
||||
platform.pending = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "needs_review"
|
||||
platform.pending = False
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
# Simulate a crash checkpoint with the Location already persisted.
|
||||
async with app.state.sessions.begin() as db:
|
||||
a = await db.get(SimulationAttempt, ids[0])
|
||||
a.state = "submitting"
|
||||
await lane.start()
|
||||
await lane.stop()
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_result_transaction_failure_recovers_from_saved_receipt(app, logged_in):
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
platform, lane = await setup(app)
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
failed = False
|
||||
|
||||
def fail_once(conn, cursor, statement, parameters, context, executemany):
|
||||
nonlocal failed
|
||||
if "INSERT INTO backtest_results" in statement and not failed:
|
||||
failed = True
|
||||
raise OperationalError("synthetic persistence outage", {}, Exception("synthetic"))
|
||||
|
||||
event.listen(app.state.engine.sync_engine, "before_cursor_execute", fail_once)
|
||||
try:
|
||||
ids = await execute(app, lane, rid)
|
||||
finally:
|
||||
event.remove(app.state.engine.sync_engine, "before_cursor_execute", fail_once)
|
||||
assert failed
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestResult)) == 0
|
||||
assert await db.scalar(select(func.count()).select_from(Alpha)) == 0
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_ai_fixed_set_confirmation_and_duplicate_decision(app, logged_in):
|
||||
from tests.test_ai import configure
|
||||
from tests.test_ai import start as start_ai
|
||||
|
||||
platform, lane = await setup(app)
|
||||
await configure(app, logged_in)
|
||||
_, run, _ = await start_ai(app, logged_in, "回测固定候选")
|
||||
assert run["status"] == "waiting_approval", run
|
||||
approval = next(c for c in run["tools"] if c["name"] == "start_backtest")
|
||||
assert approval["preview"]["backtest"]["total"] == 1
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestRun)) == 0
|
||||
for _ in range(2):
|
||||
response = await logged_in.post(
|
||||
f"/api/v1/ai/approvals/{approval['id']}/decision", json={"approved": True}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
async with app.state.sessions() as db:
|
||||
rows = list(await db.scalars(select(BacktestRun)))
|
||||
assert len(rows) == 1
|
||||
assert rows[0].ai_context["ai_run_id"] == run["id"]
|
||||
await logged_in.post(f"/api/v1/ai/runs/{run['id']}/cancel")
|
||||
await execute(app, lane, rows[0].id)
|
||||
assert len(platform.posts) == 1
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rows[0].id}")).json()["status"] == "completed"
|
||||
|
||||
|
||||
async def test_duplicate_inputs_are_separate_attempts_and_share_alpha_safely(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.existing_alpha_ids = ["shared_alpha"]
|
||||
p = await preview(logged_in, [candidate(0), candidate(0) | {"client_item_id": "other-experiment"}])
|
||||
assert p["batch_count"] == 2 and p["duplicate_count"] == 1
|
||||
rid = (await start(logged_in, p))["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(Alpha)) == 1
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestResult)) == 2
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
|
||||
|
||||
async def test_original_reference_recovery_without_new_post(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "missing_location"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
path = f"{PREFIX}/attempts/{ids[0]}/reference"
|
||||
assert (
|
||||
await logged_in.post(
|
||||
path, json={"progress_url": "https://foreign.example/simulations/p1", "version": 1}
|
||||
)
|
||||
).status_code == 422
|
||||
linked = await logged_in.post(path, json={"progress_url": "/simulations/p1", "version": 1})
|
||||
assert linked.status_code == 200, linked.text
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_preview_subset_uses_whole_snapshot_and_does_not_change_original(app, logged_in):
|
||||
await setup(app)
|
||||
p = await preview(logged_in, [candidate(i) for i in range(40)])
|
||||
subset = await logged_in.post(
|
||||
f"{PREFIX}/previews/{p['preview_id']}/subset", json={"exclude_ids": ["item-30"]}
|
||||
)
|
||||
assert subset.json()["total"] == 39 and subset.json()["preview_id"] != p["preview_id"]
|
||||
assert (await logged_in.get(f"{PREFIX}/previews/{p['preview_id']}")).json()["total"] == 40
|
||||
|
||||
|
||||
async def test_session_reauthentication_does_not_retry_accepted_submission(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "session"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 2 # first explicitly rejected with 401, second accepted
|
||||
|
||||
|
||||
async def test_terminal_detail_failure_releases_slot_but_keeps_platform_success(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 1, "version": 1})
|
||||
platform.detail_fail = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
item = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"][0]
|
||||
assert item["platform_status"] == "completed" and item["collection_status"] == "failed"
|
||||
await start(logged_in, await preview(logged_in, [candidate(2)]), "next")
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 2
|
||||
await lane.interrupt()
|
||||
Reference in New Issue
Block a user