feat: add durable WorldQuant backtests with UI and AI confirmation
This commit is contained in:
yuxuanhui
@@ -0,0 +1,413 @@
|
||||
"""End-to-end business tests: real persistence/runtime, only the platform HTTP is replaced."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.backtests.contracts import SimulationSettings
|
||||
from app.models import Account, Alpha, BacktestResult, BacktestRun, Research, SimulationAttempt
|
||||
from app.security import cipher
|
||||
from app.worldquant import WqClient
|
||||
from tests.backtest_fake import Platform
|
||||
|
||||
PREFIX = "/api/v1/backtests"
|
||||
PARAMS = SimulationSettings(region="USA", universe="TOP3000", delay=1).model_dump()
|
||||
|
||||
|
||||
def candidate(index=0, **settings):
|
||||
return {
|
||||
"client_item_id": f"item-{index}",
|
||||
"expression": f"rank(close) + {index}",
|
||||
"settings": PARAMS | settings,
|
||||
}
|
||||
|
||||
|
||||
async def setup(app):
|
||||
platform = Platform()
|
||||
runner = app.state.runner
|
||||
await runner.client.close()
|
||||
runner.client = WqClient(app.state.settings, transport=httpx.MockTransport(platform))
|
||||
runner.backtests.client = runner.client
|
||||
runner.backtests.poll_interval = 0
|
||||
async with app.state.sessions.begin() as db:
|
||||
account = await db.get(Account, 1)
|
||||
account.email, account.wq_user_id, account.connection_status = (
|
||||
"synthetic@example.com",
|
||||
"TEST_USER",
|
||||
"connected",
|
||||
)
|
||||
account.password_encrypted = cipher(app.state.settings).encrypt(platform.secret.encode()).decode()
|
||||
return platform, runner.backtests
|
||||
|
||||
|
||||
async def preview(client, candidates=None):
|
||||
response = await client.post(
|
||||
f"{PREFIX}/previews",
|
||||
json={
|
||||
"inline": {
|
||||
"name": "测试研究",
|
||||
"source": {"kind": "test"},
|
||||
"candidates": candidates or [candidate()],
|
||||
}
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
return response.json()
|
||||
|
||||
|
||||
async def start(client, p, key="request-1"):
|
||||
response = await client.post(
|
||||
f"{PREFIX}/runs",
|
||||
json={"preview_id": p["preview_id"], "version": p["version"], "idempotency_key": key},
|
||||
)
|
||||
assert response.status_code == 202, response.text
|
||||
return response.json()
|
||||
|
||||
|
||||
async def execute(app, lane, run_id):
|
||||
async with app.state.sessions() as db:
|
||||
ids = list(
|
||||
await db.scalars(
|
||||
select(SimulationAttempt.id)
|
||||
.where(SimulationAttempt.run_id == run_id)
|
||||
.order_by(SimulationAttempt.ordinal)
|
||||
)
|
||||
)
|
||||
for aid in ids:
|
||||
await lane.step(aid)
|
||||
await lane.step(aid)
|
||||
return ids
|
||||
|
||||
|
||||
async def test_fixed_preview_grouping_mapping_and_history(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
p = await preview(logged_in, [candidate(0), candidate(1, universe="TOP1000"), candidate(2, delay=0)])
|
||||
assert p["batch_count"] == 2 and p["total"] == 3
|
||||
run = await start(logged_in, p)
|
||||
again = await start(logged_in, p)
|
||||
assert run["backtest_run_id"] == again["backtest_run_id"]
|
||||
rid = run["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
data = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()
|
||||
assert len(platform.posts) == 2
|
||||
assert all(i["persistence_status"] == "saved" for i in data["items"]), data
|
||||
for item in data["items"]:
|
||||
assert item["result"]["snapshot"]["regular"]["code"] == item["expression"]
|
||||
assert item["result"]["snapshot"]["settings"] == item["settings"]
|
||||
assert item["result"]["snapshot"]["is"]["sharpe"] is None
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
item = data["items"][0]
|
||||
async with app.state.sessions.begin() as db:
|
||||
alpha = await db.get(Alpha, item["alpha_id"])
|
||||
alpha.is_metrics = {"sharpe": 999}
|
||||
assert await db.get(Research, alpha.id)
|
||||
historical = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()
|
||||
assert historical["items"][0]["result"]["snapshot"]["is"]["sharpe"] is None
|
||||
events = (await logged_in.get(f"{PREFIX}/runs/{rid}/events?limit=2")).json()
|
||||
later = (await logged_in.get(f"{PREFIX}/runs/{rid}/events?after={events['next_cursor']}")).json()
|
||||
assert events["has_more"] and later["items"][0]["seq"] > events["next_cursor"]
|
||||
assert (await preview(logged_in))["duplicate_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rejection", ["unknown", "missing_location"])
|
||||
async def test_unknown_submission_never_reposted(app, logged_in, rejection):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = rejection
|
||||
run = await start(logged_in, await preview(logged_in))
|
||||
rid = run["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
# A process crash/recovery must not turn an unknown POST into queued work.
|
||||
await lane.start()
|
||||
await lane.stop()
|
||||
response = await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
assert response.json()["status"] == "needs_review"
|
||||
async with app.state.sessions() as db:
|
||||
assert (await db.get(SimulationAttempt, ids[0])).state == "needs_review"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_partial_failure_and_rerun_only_selected(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.fail_child = 0
|
||||
run = await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)]))
|
||||
rid = run["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
result = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert result[0]["platform_status"] == "failed" and result[1]["persistence_status"] == "saved"
|
||||
rerun = await logged_in.post(f"{PREFIX}/runs/{rid}/rerun-preview", json={"item_ids": [result[0]["id"]]})
|
||||
assert rerun.status_code == 201
|
||||
assert rerun.json()["total"] == 1 and rerun.json()["source"]["parent_run_id"] == rid
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_detail_failure_recovers_without_resubmit(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.detail_fail = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "needs_review"
|
||||
platform.detail_fail = False
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_draft_version_snapshot_and_pause_stop(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
body = {"name": "草稿", "candidates": [candidate(0), candidate(1, delay=0)]}
|
||||
d = (await logged_in.post(f"{PREFIX}/drafts", json=body)).json()
|
||||
p = (await logged_in.post(f"{PREFIX}/previews", json={"draft_id": d["id"], "draft_version": 1})).json()
|
||||
changed = await logged_in.put(
|
||||
f"{PREFIX}/drafts/{d['id']}", json=body | {"version": 1, "candidates": [candidate(9)]}
|
||||
)
|
||||
assert changed.json()["version"] == 2
|
||||
assert (
|
||||
await logged_in.post(f"{PREFIX}/previews", json={"draft_id": d["id"], "draft_version": 1})
|
||||
).status_code == 409
|
||||
run = await start(logged_in, p)
|
||||
rid = run["backtest_run_id"]
|
||||
async with app.state.sessions() as db:
|
||||
ids = list(
|
||||
await db.scalars(
|
||||
select(SimulationAttempt.id)
|
||||
.where(SimulationAttempt.run_id == rid)
|
||||
.order_by(SimulationAttempt.ordinal)
|
||||
)
|
||||
)
|
||||
await lane.step(ids[0])
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "pause", "version": 1})
|
||||
await lane.step(ids[1])
|
||||
await lane.step(ids[0])
|
||||
assert len(platform.posts) == 1
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "stop", "version": 2})
|
||||
r = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert r[0]["persistence_status"] == "saved" and r[1]["platform_status"] == "skipped"
|
||||
assert r[0]["expression"] == candidate(0)["expression"]
|
||||
|
||||
|
||||
async def test_batch_missing_child_does_not_misattribute(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.missing = True
|
||||
rid = (await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)])))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
items = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"]
|
||||
assert items[0]["platform_status"] == "unknown"
|
||||
assert items[1]["persistence_status"] == "saved"
|
||||
platform.simulations["p1"]["children"] = ["p1c1", "p1c0"]
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_validation_auth_and_idempotency_conflict(app, logged_in, client):
|
||||
await setup(app)
|
||||
assert (
|
||||
await logged_in.post(
|
||||
f"{PREFIX}/previews",
|
||||
json={"inline": {"name": "x", "candidates": [candidate() | {"alpha_type": "SUPER"}]}},
|
||||
)
|
||||
).status_code == 422
|
||||
p1, p2 = await preview(logged_in), await preview(logged_in, [candidate(2)])
|
||||
await start(logged_in, p1)
|
||||
assert (
|
||||
await logged_in.post(
|
||||
f"{PREFIX}/runs", json={"preview_id": p2["preview_id"], "idempotency_key": "request-1"}
|
||||
)
|
||||
).status_code == 409
|
||||
assert (await logged_in.get(f"{PREFIX}/runs?limit=101")).status_code == 422
|
||||
await client.post("/api/v1/auth/logout")
|
||||
assert (await client.get(f"{PREFIX}/runs")).status_code == 401
|
||||
|
||||
|
||||
async def tick(lane):
|
||||
await lane.tick()
|
||||
await asyncio.gather(*lane.tasks.values(), return_exceptions=False)
|
||||
|
||||
|
||||
async def test_account_budget_round_robin_and_sync_independence(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
assert (
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 1, "version": 1})
|
||||
).status_code == 200
|
||||
r1 = await start(logged_in, await preview(logged_in, [candidate(0), candidate(1)]), "first")
|
||||
r2 = await start(logged_in, await preview(logged_in, [candidate(2), candidate(3)]), "second")
|
||||
await tick(lane) # one submission, occupied until remote terminal
|
||||
assert len(platform.posts) == 1
|
||||
await tick(lane) # poll first result
|
||||
await tick(lane) # other run gets next slot
|
||||
assert len(platform.posts) == 2
|
||||
assert platform.posts[0][0]["regular"] == candidate(0)["expression"]
|
||||
assert platform.posts[1][0]["regular"] == candidate(2)["expression"]
|
||||
platform.pending = True
|
||||
sync = await logged_in.post("/api/v1/sync-jobs", json={"kind": "full_sync"})
|
||||
await app.state.runner.run_next()
|
||||
assert (await logged_in.get(f"/api/v1/sync-jobs/{sync.json()['id']}")).json()["status"] == "completed"
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 2, "batch_size": 8, "version": 2})
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 3
|
||||
# Batch sizing of both existing runs remains 1 despite config update.
|
||||
assert all(len(p) == 1 for p in platform.posts)
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 8, "version": 3})
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 3
|
||||
await lane.interrupt()
|
||||
assert r1["batch_size"] == r2["batch_size"] == 1
|
||||
|
||||
|
||||
async def test_rate_limit_and_failed_submit_are_bounded(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "rate"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
async with app.state.sessions() as db:
|
||||
aid = await db.scalar(select(SimulationAttempt.id).where(SimulationAttempt.run_id == rid))
|
||||
for _ in range(app.state.settings.retry_attempts):
|
||||
await lane.step(aid)
|
||||
await asyncio.sleep(0.02)
|
||||
assert len(platform.posts) == app.state.settings.retry_attempts
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed_with_errors"
|
||||
assert platform.secret not in (await logged_in.get(f"{PREFIX}/runs/{rid}/attempts")).text
|
||||
|
||||
|
||||
async def test_poll_timeout_and_crash_after_acceptance(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
lane.poll_limit = 1
|
||||
platform.pending = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "needs_review"
|
||||
platform.pending = False
|
||||
await logged_in.post(f"{PREFIX}/runs/{rid}/control", json={"action": "recover", "version": 1})
|
||||
# Simulate a crash checkpoint with the Location already persisted.
|
||||
async with app.state.sessions.begin() as db:
|
||||
a = await db.get(SimulationAttempt, ids[0])
|
||||
a.state = "submitting"
|
||||
await lane.start()
|
||||
await lane.stop()
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_result_transaction_failure_recovers_from_saved_receipt(app, logged_in):
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
platform, lane = await setup(app)
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
failed = False
|
||||
|
||||
def fail_once(conn, cursor, statement, parameters, context, executemany):
|
||||
nonlocal failed
|
||||
if "INSERT INTO backtest_results" in statement and not failed:
|
||||
failed = True
|
||||
raise OperationalError("synthetic persistence outage", {}, Exception("synthetic"))
|
||||
|
||||
event.listen(app.state.engine.sync_engine, "before_cursor_execute", fail_once)
|
||||
try:
|
||||
ids = await execute(app, lane, rid)
|
||||
finally:
|
||||
event.remove(app.state.engine.sync_engine, "before_cursor_execute", fail_once)
|
||||
assert failed
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestResult)) == 0
|
||||
assert await db.scalar(select(func.count()).select_from(Alpha)) == 0
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_ai_fixed_set_confirmation_and_duplicate_decision(app, logged_in):
|
||||
from tests.test_ai import configure
|
||||
from tests.test_ai import start as start_ai
|
||||
|
||||
platform, lane = await setup(app)
|
||||
await configure(app, logged_in)
|
||||
_, run, _ = await start_ai(app, logged_in, "回测固定候选")
|
||||
assert run["status"] == "waiting_approval", run
|
||||
approval = next(c for c in run["tools"] if c["name"] == "start_backtest")
|
||||
assert approval["preview"]["backtest"]["total"] == 1
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestRun)) == 0
|
||||
for _ in range(2):
|
||||
response = await logged_in.post(
|
||||
f"/api/v1/ai/approvals/{approval['id']}/decision", json={"approved": True}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
async with app.state.sessions() as db:
|
||||
rows = list(await db.scalars(select(BacktestRun)))
|
||||
assert len(rows) == 1
|
||||
assert rows[0].ai_context["ai_run_id"] == run["id"]
|
||||
await logged_in.post(f"/api/v1/ai/runs/{run['id']}/cancel")
|
||||
await execute(app, lane, rows[0].id)
|
||||
assert len(platform.posts) == 1
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rows[0].id}")).json()["status"] == "completed"
|
||||
|
||||
|
||||
async def test_duplicate_inputs_are_separate_attempts_and_share_alpha_safely(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.existing_alpha_ids = ["shared_alpha"]
|
||||
p = await preview(logged_in, [candidate(0), candidate(0) | {"client_item_id": "other-experiment"}])
|
||||
assert p["batch_count"] == 2 and p["duplicate_count"] == 1
|
||||
rid = (await start(logged_in, p))["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
async with app.state.sessions() as db:
|
||||
assert await db.scalar(select(func.count()).select_from(Alpha)) == 1
|
||||
assert await db.scalar(select(func.count()).select_from(BacktestResult)) == 2
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
|
||||
|
||||
async def test_original_reference_recovery_without_new_post(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "missing_location"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
path = f"{PREFIX}/attempts/{ids[0]}/reference"
|
||||
assert (
|
||||
await logged_in.post(
|
||||
path, json={"progress_url": "https://foreign.example/simulations/p1", "version": 1}
|
||||
)
|
||||
).status_code == 422
|
||||
linked = await logged_in.post(path, json={"progress_url": "/simulations/p1", "version": 1})
|
||||
assert linked.status_code == 200, linked.text
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 1
|
||||
|
||||
|
||||
async def test_preview_subset_uses_whole_snapshot_and_does_not_change_original(app, logged_in):
|
||||
await setup(app)
|
||||
p = await preview(logged_in, [candidate(i) for i in range(40)])
|
||||
subset = await logged_in.post(
|
||||
f"{PREFIX}/previews/{p['preview_id']}/subset", json={"exclude_ids": ["item-30"]}
|
||||
)
|
||||
assert subset.json()["total"] == 39 and subset.json()["preview_id"] != p["preview_id"]
|
||||
assert (await logged_in.get(f"{PREFIX}/previews/{p['preview_id']}")).json()["total"] == 40
|
||||
|
||||
|
||||
async def test_session_reauthentication_does_not_retry_accepted_submission(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
platform.reject = "session"
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
ids = await execute(app, lane, rid)
|
||||
await lane.step(ids[0])
|
||||
assert (await logged_in.get(f"{PREFIX}/runs/{rid}")).json()["status"] == "completed"
|
||||
assert len(platform.posts) == 2 # first explicitly rejected with 401, second accepted
|
||||
|
||||
|
||||
async def test_terminal_detail_failure_releases_slot_but_keeps_platform_success(app, logged_in):
|
||||
platform, lane = await setup(app)
|
||||
await logged_in.put(f"{PREFIX}/config", json={"concurrency": 1, "batch_size": 1, "version": 1})
|
||||
platform.detail_fail = True
|
||||
rid = (await start(logged_in, await preview(logged_in)))["backtest_run_id"]
|
||||
await execute(app, lane, rid)
|
||||
item = (await logged_in.get(f"{PREFIX}/runs/{rid}/results")).json()["items"][0]
|
||||
assert item["platform_status"] == "completed" and item["collection_status"] == "failed"
|
||||
await start(logged_in, await preview(logged_in, [candidate(2)]), "next")
|
||||
await tick(lane)
|
||||
assert len(platform.posts) == 2
|
||||
await lane.interrupt()
|
||||
Reference in New Issue
Block a user