"""MCP integration uses real database/transport and a synthetic WorldQuant only.""" import asyncio from datetime import timedelta import httpx import pytest from sqlalchemy import func, select from app.mcp_api.auth import SCOPES, authenticate, create_token from app.models import ( BacktestPreview, BacktestResult, BacktestRun, MCPAudit, MCPToken, Pnl, now, ) from tests.test_backtests import candidate, execute, setup ENDPOINT = "/api/v1/mcp/" @pytest.fixture async def mcp_app(settings): from app.main import create_app from app.models import Base from app.worldquant import WqClient settings.mcp_enabled = True def no_network(request): raise AssertionError("No real platform") application = create_app(settings, WqClient(settings, transport=httpx.MockTransport(no_network))) async with application.state.engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) ready, stop = asyncio.Event(), asyncio.Event() async def lifespan(): async with application.router.lifespan_context(application): ready.set() await stop.wait() task = asyncio.create_task(lifespan()) await ready.wait() try: await setup(application) yield application finally: stop.set() await task async def credentials(app, scopes=SCOPES): async with app.state.sessions.begin() as db: row, secret = await create_token(db, "synthetic test", scopes) principal = await authenticate(db, secret) return principal, secret def submission(key="batch-1", items=None, **extra): return {"name": "MCP batch", "candidates": items or [candidate()], "idempotency_key": key, **extra} async def invoke(app, principal, name, arguments=None): result = await app.state.mcp.invoke(principal, name, arguments or {}) assert not result.is_error, result.structured_content return result.structured_content async def test_http_auth_and_discovery(mcp_app): principal, secret = await credentials(mcp_app, {"research:read"}) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=mcp_app), base_url="http://testserver") as client: payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"} assert (await client.post(ENDPOINT, json=payload)).status_code == 401 headers = {"Authorization": f"Bearer {secret}", "Accept": "application/json, text/event-stream"} response = await client.post(ENDPOINT, json=payload, headers=headers) assert response.status_code == 200, response.text names = {t["name"] for t in response.json()["result"]["tools"]} assert "search_backtests" in names and "submit_backtests" not in names denied = {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "submit_backtests", "arguments": submission()}} assert (await client.post(ENDPOINT, json=denied, headers=headers)).status_code == 403 assert (await client.get("/api/v1/account", headers=headers)).status_code == 401 assert (await client.post(ENDPOINT, json=payload, headers=headers | {"Origin": "https://bad.test"})).status_code == 403 assert (await client.post(ENDPOINT, json=payload, headers=headers | {"Host": "bad.test"})).status_code == 403 async with mcp_app.state.sessions.begin() as db: row = await db.get(MCPToken, principal.token_id) row.revoked_at = now() assert (await client.post(ENDPOINT, json=payload, headers=headers)).status_code == 401 async def test_submit_replay_duplicates_and_rotation(mcp_app): principal, _ = await credentials(mcp_app) result = await invoke(mcp_app, principal, "submit_backtests", submission()) second, _ = await credentials(mcp_app) replay = await invoke(mcp_app, second, "submit_backtests", submission()) assert replay == result conflict = await mcp_app.state.mcp.invoke(second, "submit_backtests", submission(items=[candidate(1)])) assert conflict.is_error and conflict.structured_content["error"]["code"] == "IDEMPOTENCY_CONFLICT" duplicate = await mcp_app.state.mcp.invoke(second, "submit_backtests", submission("different")) assert duplicate.structured_content["error"]["code"] == "DUPLICATE_INPUT" rerun = await invoke(mcp_app, second, "submit_backtests", submission("different", duplicate_policy="rerun")) assert rerun["backtest_run_id"] != result["backtest_run_id"] async with mcp_app.state.sessions() as db: assert await db.scalar(select(func.count()).select_from(BacktestRun)) == 2 assert await db.scalar(select(func.count()).select_from(BacktestPreview)) == 2 audits = list(await db.scalars(select(MCPAudit))) assert len(audits) == 5 assert result["source"]["kind"] == "mcp" assert "#backtests?run_id=" in result["web_url"] async def test_validation_and_within_batch_atomic(mcp_app): principal, _ = await credentials(mcp_app) incomplete = candidate() incomplete["settings"] = {"region": "USA", "universe": "TOP3000", "delay": 1} result = await mcp_app.state.mcp.invoke(principal, "submit_backtests", submission(items=[incomplete])) assert result.structured_content["error"]["code"] == "INVALID_INPUT" result = await mcp_app.state.mcp.invoke(principal, "submit_backtests", submission(items=[candidate(), candidate() | {"client_item_id": "second"}])) assert result.structured_content["error"]["code"] == "DUPLICATE_INPUT" # Failed requests did not consume the key or leave previews. await invoke(mcp_app, principal, "submit_backtests", submission()) async with mcp_app.state.sessions() as db: assert await db.scalar(select(func.count()).select_from(BacktestPreview)) == 1 async def test_results_control_and_evidence(mcp_app): principal, _ = await credentials(mcp_app) result = await invoke(mcp_app, principal, "submit_backtests", submission(items=[candidate(0), candidate(1)])) rid = result["backtest_run_id"] control = {"run_id": rid, "action": "pause", "expected_version": 1, "idempotency_key": "pause"} paused = await invoke(mcp_app, principal, "control_backtest", control) assert paused == await invoke(mcp_app, principal, "control_backtest", control) await invoke(mcp_app, principal, "control_backtest", control | {"action": "resume", "expected_version": 2, "idempotency_key": "resume"}) await execute(mcp_app, mcp_app.state.runner.backtests, rid) data = await invoke(mcp_app, principal, "get_backtest_results", {"run_id": rid, "limit": 1}) assert data["has_more"] and data["items"][0]["metrics"]["is"]["sharpe"] is None item = data["items"][0] async with mcp_app.state.sessions.begin() as db: saved = await db.get(BacktestResult, item["id"]) saved.snapshot = {**saved.snapshot, "is": {"checks": [{"name": "a", "result": "FAIL"}, {"name": "b", "result": "NEW_STATUS"}]}} data = await invoke(mcp_app, principal, "get_backtest_results", {"run_id": rid}) assert data["items"][0]["checks"]["counts"]["FAIL"] == 1 assert data["items"][0]["checks"]["counts"]["UNKNOWN"] == 1 pnl = await invoke(mcp_app, principal, "get_backtest_artifact", {"item_id": item["id"], "kind": "pnl"}) assert pnl["status"] == "not_cached" snapshot = await invoke(mcp_app, principal, "get_backtest_artifact", {"item_id": item["id"], "kind": "snapshot", "limit": 1}) assert snapshot["has_more"] history = await invoke(mcp_app, principal, "search_backtests", {"candidates": [candidate()], "source": "mcp"}) assert history["total"] == 1 and history["items"][0]["match_type"] == "exact_input" progress = await invoke(mcp_app, principal, "get_backtest", {"run_id": rid, "after": 0, "event_limit": 1}) assert progress["events"]["has_more"] and progress["submission_counts"]["actual_platform_consumption"] is None async def test_official_sdk_client_and_error_contract(mcp_app): import httpx2 from mcp import ClientSession from mcp.client.streamable_http import streamable_http_client principal, secret = await credentials(mcp_app) async with httpx2.AsyncClient(transport=httpx2.ASGITransport(app=mcp_app), headers={"Authorization": f"Bearer {secret}"}) as http: async with streamable_http_client("http://testserver/api/v1/mcp/", http_client=http) as streams: async with ClientSession(streams[0], streams[1]) as client: await client.initialize() listed = await client.list_tools() assert len(listed.tools) == 11 caps = await client.call_tool("get_research_capabilities", {}) assert caps.structured_content["max_candidates"] == 100 result = await client.call_tool("submit_backtests", submission()) assert not result.is_error, result rid = result.structured_content["backtest_run_id"] await execute(mcp_app, mcp_app.state.runner.backtests, rid) results = await client.call_tool("get_backtest_results", {"run_id": rid}) assert results.structured_content["items"][0]["persistence_status"] == "saved" bad = await client.call_tool("submit_backtests", submission("duplicate")) assert bad.is_error and bad.structured_content["error"]["code"] == "DUPLICATE_INPUT" await client.call_tool("get_backtest", {"run_id": rid}) async def test_expiry_binding_disabled_and_audit_redaction(app, mcp_app): from fastapi import HTTPException from app.models import Account principal, secret = await credentials(mcp_app) await mcp_app.state.mcp.invoke(principal, "get_research_capabilities", {}, request_id=secret) async with mcp_app.state.sessions.begin() as db: audit = await db.scalar(select(MCPAudit)) assert secret not in audit.request_id row = await db.get(MCPToken, principal.token_id) row.expires_at = now() - timedelta(seconds=1) async with mcp_app.state.sessions() as db: with pytest.raises(HTTPException): await authenticate(db, secret) _, another = await credentials(mcp_app) async with mcp_app.state.sessions.begin() as db: account = await db.get(Account, 1) account.wq_user_id = "CHANGED" async with mcp_app.state.sessions() as db: with pytest.raises(HTTPException): await authenticate(db, another) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://testserver") as client: assert (await client.post(ENDPOINT, json={})).status_code == 404 async def test_metadata_refresh_and_pnl_pagination(mcp_app): from app.worldquant import WqClient from tests.research_metadata_fake import response principal, _ = await credentials(mcp_app) caps = await invoke(mcp_app, principal, "get_research_metadata", {"query": {"kind": "settings"}}) assert caps["status"] == "not_cached" await mcp_app.state.runner.client.close() from tests.backtest_fake import Platform platform = Platform() client = WqClient(mcp_app.state.settings, transport=httpx.MockTransport(lambda request: response(request) or platform(request))) await client.authenticate("synthetic@example.com", "synthetic-password") mcp_app.state.runner.client = client mcp_app.state.runner.backtests.client = client operators = await invoke(mcp_app, principal, "refresh_research_data", {"query": {"kind": "operators"}}) assert operators["status"] == "completed" page = await invoke(mcp_app, principal, "get_research_metadata", {"query": {"kind": "operators", "limit": 1}}) assert page["total"] == 2 and page["has_more"] await invoke(mcp_app, principal, "refresh_research_data", {"query": {"kind": "settings"}}) await invoke(mcp_app, principal, "refresh_research_data", {"query": {"kind": "field_availability", "scope": {"region": "USA", "universe": "TOP3000", "delay": 1}, "field_id": "close"}}) invalid = await mcp_app.state.mcp.invoke(principal, "submit_backtests", submission(items=[candidate(region="INVALID")])) assert invalid.structured_content["error"]["code"] == "UNSUPPORTED_SETTINGS" run = await invoke(mcp_app, principal, "submit_backtests", submission()) await execute(mcp_app, mcp_app.state.runner.backtests, run["backtest_run_id"]) result = await invoke(mcp_app, principal, "get_backtest_results", {"run_id": run["backtest_run_id"]}) item = result["items"][0] async with mcp_app.state.sessions.begin() as db: db.add(Pnl(alpha_id=item["alpha_id"], raw={}, points=[{"date": f"2025-01-0{i}", "value": i} for i in range(1, 4)])) pnl = await invoke(mcp_app, principal, "get_backtest_artifact", {"item_id": item["id"], "kind": "pnl", "limit": 1, "date_from": "2025-01-02"}) assert pnl["total"] == 2 and pnl["items"][0]["value"] == 2 and pnl["has_more"] async def test_lost_http_response_replays_without_new_run(mcp_app): import json _, secret = await credentials(mcp_app) class LoseResponse(httpx.ASGITransport): dropped = False async def handle_async_request(self, request): response = await super().handle_async_request(request) if not self.dropped: self.dropped = True await response.aread() raise httpx.ReadError("synthetic lost response", request=request) return response transport = LoseResponse(app=mcp_app) async with httpx.AsyncClient(transport=transport, base_url="http://testserver", headers={"Authorization": f"Bearer {secret}", "Accept": "application/json, text/event-stream"}) as client: payload = {"jsonrpc": "2.0", "id": "lost", "method": "tools/call", "params": {"name": "submit_backtests", "arguments": submission()}} with pytest.raises(httpx.ReadError): await client.post(ENDPOINT, json=payload) response = await client.post(ENDPOINT, json=payload) assert response.status_code == 200 assert not response.json()["result"]["isError"] async with mcp_app.state.sessions() as db: assert await db.scalar(select(func.count()).select_from(BacktestRun)) == 1 audits = list(await db.scalars(select(MCPAudit))) assert all("wqmcp_" not in json.dumps(a.__dict__, default=str) for a in audits) async def test_refresh_job_error_pages_and_kind_isolation(mcp_app): from app.models import Job, JobItem principal, _ = await credentials(mcp_app) async with mcp_app.state.sessions.begin() as db: db.add(Job(id="refresh", kind="pnl_refresh", failed=2)) db.add(Job(id="auth", kind="connect")) await db.flush() db.add_all([JobItem(job_id="refresh", alpha_id=str(i), error="synthetic error") for i in range(2)]) result = await invoke(mcp_app, principal, "get_refresh_job", {"job_id": "refresh", "limit": 1}) assert result["errors"]["total"] == 2 and result["errors"]["has_more"] result = await invoke(mcp_app, principal, "get_refresh_job", {"job_id": "refresh", "offset": 1}) assert result["errors"]["items"][0]["alpha_id"] == "1" error = await mcp_app.state.mcp.invoke(principal, "get_refresh_job", {"job_id": "auth"}) assert error.structured_content["error"]["code"] == "NOT_FOUND"