This commit is contained in:
+9
-1
@@ -3,10 +3,18 @@ ZHIXING_SERVER_PORT=8000
|
||||
ZHIXING_WEB_PORT=5173
|
||||
ZHIXING_APP_ENV=development
|
||||
ZHIXING_LOG_LEVEL=INFO
|
||||
|
||||
# Development: docker-compose.dev.yml starts an independent local PostgreSQL.
|
||||
ZHIXING_POSTGRES_DB=zhixing
|
||||
ZHIXING_POSTGRES_USER=zhixing
|
||||
ZHIXING_POSTGRES_PASSWORD=zhixing
|
||||
ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing
|
||||
# docker-compose.dev.yml uses the same local URL when this variable is unset.
|
||||
# ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing
|
||||
|
||||
# Production: use the existing 1Panel PostgreSQL container on the external
|
||||
# Docker network. Replace <url-encoded-password>; do not use the development URL.
|
||||
# The current 1Panel PostgreSQL has SSL disabled; use its stable network alias.
|
||||
# ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=disable
|
||||
ZHIXING_TUSHARE_TOKEN=
|
||||
ZHIXING_MARKET_DATA_CSV_ROOT=/app/data/market-data
|
||||
API_UPSTREAM=http://server:8000
|
||||
|
||||
@@ -14,17 +14,24 @@ jobs:
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: zhixing-system
|
||||
ZHIXING_WEB_PORT: "8111"
|
||||
ZHIXING_DATABASE_URL: ${{ secrets.ZHIXING_DATABASE_URL }}
|
||||
ZHIXING_TUSHARE_TOKEN: ${{ secrets.ZHIXING_TUSHARE_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Validate production compose
|
||||
run: docker compose -f docker-compose.prod.yml config
|
||||
run: |
|
||||
docker compose -f docker-compose.prod.yml config --quiet
|
||||
docker compose -f docker-compose.prod.yml --profile jobs config --quiet
|
||||
|
||||
- name: Build and deploy services
|
||||
run: docker compose -f docker-compose.prod.yml up -d --build --remove-orphans
|
||||
|
||||
- name: Run database migrations
|
||||
run: docker compose -f docker-compose.prod.yml --profile jobs run --rm --build migrate
|
||||
|
||||
- name: Verify web service
|
||||
run: docker compose -f docker-compose.prod.yml exec -T web wget -q -O - http://127.0.0.1/healthz
|
||||
|
||||
|
||||
@@ -24,6 +24,9 @@
|
||||
- `ZHIXING_MARKET_DATA_COVERAGE_THRESHOLD`,默认 `0.99`
|
||||
- `ZHIXING_MARKET_DATA_MAX_RETRIES`、`ZHIXING_MARKET_DATA_RETRY_BACKOFF_SECONDS`、`ZHIXING_MARKET_DATA_REQUEST_INTERVAL_SECONDS`
|
||||
- `ZHIXING_MARKET_DATA_ADVISORY_LOCK_KEY`
|
||||
- `ZHIXING_DATABASE_URL` 使用普通 `postgresql://...` 形式供 Psycopg 3 直接连接;Alembic/SQLAlchemy 在边界处转换为 `postgresql+psycopg://...`,不得丢失 `sslmode` 等 query 参数。
|
||||
- 开发 Compose 使用项目自带 PostgreSQL;生产 Compose 不声明内部 PostgreSQL 服务,`server`、`migrate` 和 `market-sync` 连接外部 Docker 网络 `1panel-network`。`server` 同时保留 Compose 默认网络以供 web 访问。
|
||||
- 当前生产 PostgreSQL 在 `1panel-network` 上的稳定别名为 `postgresql`,服务端 SSL 为关闭状态,生产连接串使用 `sslmode=disable`;数据库用户必须具备 `public` schema 的 `CREATE` 权限以执行 Alembic 迁移。
|
||||
- 股票 qfq 快照路径为 `bars/<ts_code>.csv`;每日指标路径为 `daily-basic/<YYYY>/<YYYYMMDD>.csv`;当前股票主数据为 `stock-basic/current.csv`。
|
||||
- `market_daily_bar` 的唯一键是 `(ts_code, trade_date)`,`source_adj` 必须是 `qfq`;所有价格、金额和比率使用有限 `NUMERIC`/`Decimal`。
|
||||
- `SyncBatchSummary` 至少返回 `batch_id`、目标交易日、窗口、状态、目标数、有效数、覆盖率、策略资格、插入数、更新数、未变化数和失败列表。
|
||||
|
||||
+15
-25
@@ -1,32 +1,14 @@
|
||||
name: ${COMPOSE_PROJECT_NAME:-zhixing-system}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: ${ZHIXING_POSTGRES_DB:-zhixing}
|
||||
POSTGRES_PASSWORD: ${ZHIXING_POSTGRES_PASSWORD:-zhixing}
|
||||
POSTGRES_USER: ${ZHIXING_POSTGRES_USER:-zhixing}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
server:
|
||||
build:
|
||||
context: ./zhixing-server
|
||||
target: production
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
ZHIXING_APP_ENV: production
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing}
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
|
||||
ZHIXING_LOG_LEVEL: ${ZHIXING_LOG_LEVEL:-INFO}
|
||||
ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data
|
||||
ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-}
|
||||
@@ -45,6 +27,9 @@ services:
|
||||
start_period: 10s
|
||||
volumes:
|
||||
- market-data:/app/data/market-data
|
||||
networks:
|
||||
- default
|
||||
- 1panel-network
|
||||
|
||||
web:
|
||||
build:
|
||||
@@ -66,11 +51,10 @@ services:
|
||||
context: ./zhixing-server
|
||||
target: production
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing}
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
|
||||
networks:
|
||||
- 1panel-network
|
||||
|
||||
market-sync:
|
||||
profiles: ["jobs"]
|
||||
@@ -82,12 +66,18 @@ services:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing}
|
||||
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
|
||||
ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data
|
||||
ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-}
|
||||
volumes:
|
||||
- market-data:/app/data/market-data
|
||||
networks:
|
||||
- 1panel-network
|
||||
|
||||
volumes:
|
||||
market-data:
|
||||
postgres-data:
|
||||
|
||||
networks:
|
||||
1panel-network:
|
||||
name: 1panel-network
|
||||
external: true
|
||||
|
||||
@@ -2,6 +2,44 @@
|
||||
|
||||
`market-data-sync` 是外部调度器触发的一次性任务。FastAPI 进程不包含定时器;PostgreSQL 是策略查询事实源,`market-data` 卷中的 CSV 只作为 qfq 落地快照和恢复介质。
|
||||
|
||||
## 环境与数据库连接
|
||||
|
||||
开发环境由 `docker-compose.dev.yml` 启动项目自带的独立 PostgreSQL,容器内连接串保持:
|
||||
|
||||
```dotenv
|
||||
ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing
|
||||
```
|
||||
|
||||
生产环境不启动项目自带 PostgreSQL。现有 1Panel PostgreSQL 容器通过外部 Docker 网络 `1panel-network` 的稳定别名 `postgresql` 访问,数据库和用户均使用 `zhixing-system`。生产 `.env` 必须显式设置连接串;Compose 不会回退到开发数据库:
|
||||
|
||||
```dotenv
|
||||
ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=disable
|
||||
```
|
||||
|
||||
当前服务器执行 `SHOW ssl` 的结果为 `off`,因此使用 `sslmode=disable` 或省略该参数。若日后在 1Panel/PostgreSQL 中启用 TLS,再将它改成 `sslmode=require`。密码中的 `@`、`:`、`/` 等字符必须 URL 编码。启用 TLS 后的示例如下:
|
||||
|
||||
```dotenv
|
||||
ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=require
|
||||
```
|
||||
|
||||
部署前确认外部网络和数据库容器都在该网络中:
|
||||
|
||||
```bash
|
||||
docker network inspect 1panel-network --format '{{json .Containers}}'
|
||||
docker inspect 1Panel-postgresql-5Fc7 --format '{{json .NetworkSettings.Networks}}'
|
||||
```
|
||||
|
||||
迁移或同步容器连接成功后,可只输出当前连接是否使用 SSL(不会打印连接串):
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.prod.yml --profile jobs run --rm --no-deps market-sync \
|
||||
python -c 'import psycopg; from zhixing_server.bootstrap.config import get_settings; connection=psycopg.connect(get_settings().database_url); print(connection.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]); connection.close()'
|
||||
```
|
||||
|
||||
输出 `t` 表示当前连接使用 SSL,输出 `f` 表示未使用。本服务器当前应输出 `f`,与 `sslmode=disable` 配套;若日后要求 TLS,应先检查 PostgreSQL/1Panel 的 SSL 配置,再改用 `sslmode=require`。
|
||||
|
||||
运行时保留普通 `postgresql://` 形式供 Psycopg 3 使用;Alembic/SQLAlchemy 会自动转换为 `postgresql+psycopg://`,并保留 `sslmode` 等 query 参数。
|
||||
|
||||
## 首次部署
|
||||
|
||||
先准备 `.env`,至少设置 `ZHIXING_TUSHARE_TOKEN`、PostgreSQL 凭据和 `ZHIXING_DATABASE_URL`,然后执行迁移:
|
||||
|
||||
@@ -5,12 +5,13 @@ from logging.config import fileConfig
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from zhixing_server.bootstrap.config import get_settings
|
||||
from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url
|
||||
from zhixing_server.modules.market_data.infrastructure.schema import metadata
|
||||
|
||||
config = context.config
|
||||
settings = get_settings()
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url.replace("%", "%%"))
|
||||
sqlalchemy_url = sqlalchemy_database_url(settings.database_url)
|
||||
config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%"))
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
@@ -21,7 +22,7 @@ def run_migrations_offline() -> None:
|
||||
"""Run migrations without opening a database connection."""
|
||||
|
||||
context.configure(
|
||||
url=settings.database_url,
|
||||
url=sqlalchemy_url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
|
||||
@@ -31,6 +31,31 @@ class Settings(BaseSettings):
|
||||
)
|
||||
|
||||
|
||||
def sqlalchemy_database_url(database_url: str) -> str:
|
||||
"""Add SQLAlchemy's Psycopg 3 driver to a PostgreSQL URL.
|
||||
|
||||
``Settings.database_url`` remains a plain ``postgresql://`` URL because
|
||||
Psycopg 3 accepts that form directly. Alembic and SQLAlchemy need the
|
||||
explicit ``postgresql+psycopg://`` dialect, while the rest of the URL
|
||||
(including query parameters such as ``sslmode``) must remain unchanged.
|
||||
|
||||
Args:
|
||||
database_url: PostgreSQL connection URL from the process environment.
|
||||
|
||||
Returns:
|
||||
The URL with SQLAlchemy's Psycopg 3 dialect prefix.
|
||||
|
||||
Raises:
|
||||
ValueError: If the URL does not use a supported PostgreSQL scheme.
|
||||
"""
|
||||
|
||||
if database_url.startswith("postgresql+psycopg://"):
|
||||
return database_url
|
||||
if database_url.startswith("postgresql://"):
|
||||
return "postgresql+psycopg://" + database_url.removeprefix("postgresql://")
|
||||
raise ValueError("database_url must use postgresql:// or postgresql+psycopg://")
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
"""Return one immutable-by-convention configuration object per process.
|
||||
|
||||
@@ -6,17 +6,24 @@ from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import Engine, create_engine, inspect
|
||||
|
||||
from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_postgres_migration_creates_market_data_contract() -> None:
|
||||
def test_postgres_migration_creates_market_data_contract(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
database_url = os.getenv("ZHIXING_TEST_DATABASE_URL")
|
||||
if not database_url:
|
||||
pytest.skip("set ZHIXING_TEST_DATABASE_URL to run PostgreSQL integration tests")
|
||||
|
||||
monkeypatch.setenv("ZHIXING_DATABASE_URL", database_url)
|
||||
get_settings.cache_clear()
|
||||
server_root = Path(__file__).parents[2]
|
||||
config = Config(str(server_root / "alembic.ini"))
|
||||
config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%"))
|
||||
engine: Engine = create_engine(database_url)
|
||||
sqlalchemy_url = sqlalchemy_database_url(database_url)
|
||||
config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%"))
|
||||
engine: Engine = create_engine(sqlalchemy_url)
|
||||
command.upgrade(config, "head")
|
||||
try:
|
||||
tables = set(inspect(engine).get_table_names())
|
||||
@@ -29,3 +36,4 @@ def test_postgres_migration_creates_market_data_contract() -> None:
|
||||
} <= tables
|
||||
finally:
|
||||
engine.dispose()
|
||||
get_settings.cache_clear()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import pytest
|
||||
|
||||
from zhixing_server.bootstrap.config import sqlalchemy_database_url
|
||||
|
||||
|
||||
def test_sqlalchemy_url_adds_psycopg_driver_and_preserves_query() -> None:
|
||||
database_url = (
|
||||
"postgresql://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system"
|
||||
"?sslmode=require&connect_timeout=5"
|
||||
)
|
||||
|
||||
assert sqlalchemy_database_url(database_url) == (
|
||||
"postgresql+psycopg://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system"
|
||||
"?sslmode=require&connect_timeout=5"
|
||||
)
|
||||
|
||||
|
||||
def test_sqlalchemy_url_does_not_duplicate_psycopg_driver() -> None:
|
||||
database_url = "postgresql+psycopg://user@localhost:5432/db?sslmode=disable"
|
||||
|
||||
assert sqlalchemy_database_url(database_url) == database_url
|
||||
|
||||
|
||||
def test_sqlalchemy_url_rejects_non_postgresql_scheme() -> None:
|
||||
with pytest.raises(ValueError, match="postgresql"):
|
||||
sqlalchemy_database_url("mysql://user@localhost/db")
|
||||
Reference in New Issue
Block a user