feat(deploy): 接入生产密钥与数据库迁移
Deploy Production / deploy (push) Successful in 1m42s

This commit is contained in:
yuxuanhui
2026-08-05 22:45:57 +08:00
parent d7d5ed8979
commit afdc5ca909
9 changed files with 139 additions and 33 deletions
+9 -1
View File
@@ -3,10 +3,18 @@ ZHIXING_SERVER_PORT=8000
ZHIXING_WEB_PORT=5173 ZHIXING_WEB_PORT=5173
ZHIXING_APP_ENV=development ZHIXING_APP_ENV=development
ZHIXING_LOG_LEVEL=INFO ZHIXING_LOG_LEVEL=INFO
# Development: docker-compose.dev.yml starts an independent local PostgreSQL.
ZHIXING_POSTGRES_DB=zhixing ZHIXING_POSTGRES_DB=zhixing
ZHIXING_POSTGRES_USER=zhixing ZHIXING_POSTGRES_USER=zhixing
ZHIXING_POSTGRES_PASSWORD=zhixing ZHIXING_POSTGRES_PASSWORD=zhixing
ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing # docker-compose.dev.yml uses the same local URL when this variable is unset.
# ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing
# Production: use the existing 1Panel PostgreSQL container on the external
# Docker network. Replace <url-encoded-password>; do not use the development URL.
# The current 1Panel PostgreSQL has SSL disabled; use its stable network alias.
# ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=disable
ZHIXING_TUSHARE_TOKEN= ZHIXING_TUSHARE_TOKEN=
ZHIXING_MARKET_DATA_CSV_ROOT=/app/data/market-data ZHIXING_MARKET_DATA_CSV_ROOT=/app/data/market-data
API_UPSTREAM=http://server:8000 API_UPSTREAM=http://server:8000
+8 -1
View File
@@ -14,17 +14,24 @@ jobs:
env: env:
COMPOSE_PROJECT_NAME: zhixing-system COMPOSE_PROJECT_NAME: zhixing-system
ZHIXING_WEB_PORT: "8111" ZHIXING_WEB_PORT: "8111"
ZHIXING_DATABASE_URL: ${{ secrets.ZHIXING_DATABASE_URL }}
ZHIXING_TUSHARE_TOKEN: ${{ secrets.ZHIXING_TUSHARE_TOKEN }}
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Validate production compose - name: Validate production compose
run: docker compose -f docker-compose.prod.yml config run: |
docker compose -f docker-compose.prod.yml config --quiet
docker compose -f docker-compose.prod.yml --profile jobs config --quiet
- name: Build and deploy services - name: Build and deploy services
run: docker compose -f docker-compose.prod.yml up -d --build --remove-orphans run: docker compose -f docker-compose.prod.yml up -d --build --remove-orphans
- name: Run database migrations
run: docker compose -f docker-compose.prod.yml --profile jobs run --rm --build migrate
- name: Verify web service - name: Verify web service
run: docker compose -f docker-compose.prod.yml exec -T web wget -q -O - http://127.0.0.1/healthz run: docker compose -f docker-compose.prod.yml exec -T web wget -q -O - http://127.0.0.1/healthz
@@ -24,6 +24,9 @@
- `ZHIXING_MARKET_DATA_COVERAGE_THRESHOLD`,默认 `0.99` - `ZHIXING_MARKET_DATA_COVERAGE_THRESHOLD`,默认 `0.99`
- `ZHIXING_MARKET_DATA_MAX_RETRIES`、`ZHIXING_MARKET_DATA_RETRY_BACKOFF_SECONDS`、`ZHIXING_MARKET_DATA_REQUEST_INTERVAL_SECONDS` - `ZHIXING_MARKET_DATA_MAX_RETRIES`、`ZHIXING_MARKET_DATA_RETRY_BACKOFF_SECONDS`、`ZHIXING_MARKET_DATA_REQUEST_INTERVAL_SECONDS`
- `ZHIXING_MARKET_DATA_ADVISORY_LOCK_KEY` - `ZHIXING_MARKET_DATA_ADVISORY_LOCK_KEY`
- `ZHIXING_DATABASE_URL` 使用普通 `postgresql://...` 形式供 Psycopg 3 直接连接;Alembic/SQLAlchemy 在边界处转换为 `postgresql+psycopg://...`,不得丢失 `sslmode` 等 query 参数。
- 开发 Compose 使用项目自带 PostgreSQL;生产 Compose 不声明内部 PostgreSQL 服务,`server`、`migrate` 和 `market-sync` 连接外部 Docker 网络 `1panel-network`。`server` 同时保留 Compose 默认网络以供 web 访问。
- 当前生产 PostgreSQL 在 `1panel-network` 上的稳定别名为 `postgresql`,服务端 SSL 为关闭状态,生产连接串使用 `sslmode=disable`;数据库用户必须具备 `public` schema 的 `CREATE` 权限以执行 Alembic 迁移。
- 股票 qfq 快照路径为 `bars/<ts_code>.csv`;每日指标路径为 `daily-basic/<YYYY>/<YYYYMMDD>.csv`;当前股票主数据为 `stock-basic/current.csv`。 - 股票 qfq 快照路径为 `bars/<ts_code>.csv`;每日指标路径为 `daily-basic/<YYYY>/<YYYYMMDD>.csv`;当前股票主数据为 `stock-basic/current.csv`。
- `market_daily_bar` 的唯一键是 `(ts_code, trade_date)`,`source_adj` 必须是 `qfq`;所有价格、金额和比率使用有限 `NUMERIC`/`Decimal`。 - `market_daily_bar` 的唯一键是 `(ts_code, trade_date)`,`source_adj` 必须是 `qfq`;所有价格、金额和比率使用有限 `NUMERIC`/`Decimal`。
- `SyncBatchSummary` 至少返回 `batch_id`、目标交易日、窗口、状态、目标数、有效数、覆盖率、策略资格、插入数、更新数、未变化数和失败列表。 - `SyncBatchSummary` 至少返回 `batch_id`、目标交易日、窗口、状态、目标数、有效数、覆盖率、策略资格、插入数、更新数、未变化数和失败列表。
+15 -25
View File
@@ -1,32 +1,14 @@
name: ${COMPOSE_PROJECT_NAME:-zhixing-system} name: ${COMPOSE_PROJECT_NAME:-zhixing-system}
services: services:
postgres:
image: postgres:16-alpine
restart: unless-stopped
environment:
POSTGRES_DB: ${ZHIXING_POSTGRES_DB:-zhixing}
POSTGRES_PASSWORD: ${ZHIXING_POSTGRES_PASSWORD:-zhixing}
POSTGRES_USER: ${ZHIXING_POSTGRES_USER:-zhixing}
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 10s
timeout: 5s
retries: 5
volumes:
- postgres-data:/var/lib/postgresql/data
server: server:
build: build:
context: ./zhixing-server context: ./zhixing-server
target: production target: production
depends_on:
postgres:
condition: service_healthy
restart: unless-stopped restart: unless-stopped
environment: environment:
ZHIXING_APP_ENV: production ZHIXING_APP_ENV: production
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
ZHIXING_LOG_LEVEL: ${ZHIXING_LOG_LEVEL:-INFO} ZHIXING_LOG_LEVEL: ${ZHIXING_LOG_LEVEL:-INFO}
ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data
ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-} ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-}
@@ -45,6 +27,9 @@ services:
start_period: 10s start_period: 10s
volumes: volumes:
- market-data:/app/data/market-data - market-data:/app/data/market-data
networks:
- default
- 1panel-network
web: web:
build: build:
@@ -66,11 +51,10 @@ services:
context: ./zhixing-server context: ./zhixing-server
target: production target: production
command: ["alembic", "upgrade", "head"] command: ["alembic", "upgrade", "head"]
depends_on:
postgres:
condition: service_healthy
environment: environment:
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
networks:
- 1panel-network
market-sync: market-sync:
profiles: ["jobs"] profiles: ["jobs"]
@@ -82,12 +66,18 @@ services:
migrate: migrate:
condition: service_completed_successfully condition: service_completed_successfully
environment: environment:
ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL}
ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data
ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-} ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-}
volumes: volumes:
- market-data:/app/data/market-data - market-data:/app/data/market-data
networks:
- 1panel-network
volumes: volumes:
market-data: market-data:
postgres-data:
networks:
1panel-network:
name: 1panel-network
external: true
+38
View File
@@ -2,6 +2,44 @@
`market-data-sync` 是外部调度器触发的一次性任务。FastAPI 进程不包含定时器;PostgreSQL 是策略查询事实源,`market-data` 卷中的 CSV 只作为 qfq 落地快照和恢复介质。 `market-data-sync` 是外部调度器触发的一次性任务。FastAPI 进程不包含定时器;PostgreSQL 是策略查询事实源,`market-data` 卷中的 CSV 只作为 qfq 落地快照和恢复介质。
## 环境与数据库连接
开发环境由 `docker-compose.dev.yml` 启动项目自带的独立 PostgreSQL,容器内连接串保持:
```dotenv
ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing
```
生产环境不启动项目自带 PostgreSQL。现有 1Panel PostgreSQL 容器通过外部 Docker 网络 `1panel-network` 的稳定别名 `postgresql` 访问,数据库和用户均使用 `zhixing-system`。生产 `.env` 必须显式设置连接串;Compose 不会回退到开发数据库:
```dotenv
ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=disable
```
当前服务器执行 `SHOW ssl` 的结果为 `off`,因此使用 `sslmode=disable` 或省略该参数。若日后在 1Panel/PostgreSQL 中启用 TLS,再将它改成 `sslmode=require`。密码中的 `@`、`:`、`/` 等字符必须 URL 编码。启用 TLS 后的示例如下:
```dotenv
ZHIXING_DATABASE_URL=postgresql://zhixing-system:<url-encoded-password>@postgresql:5432/zhixing-system?sslmode=require
```
部署前确认外部网络和数据库容器都在该网络中:
```bash
docker network inspect 1panel-network --format '{{json .Containers}}'
docker inspect 1Panel-postgresql-5Fc7 --format '{{json .NetworkSettings.Networks}}'
```
迁移或同步容器连接成功后,可只输出当前连接是否使用 SSL(不会打印连接串):
```bash
docker compose -f docker-compose.prod.yml --profile jobs run --rm --no-deps market-sync \
python -c 'import psycopg; from zhixing_server.bootstrap.config import get_settings; connection=psycopg.connect(get_settings().database_url); print(connection.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]); connection.close()'
```
输出 `t` 表示当前连接使用 SSL,输出 `f` 表示未使用。本服务器当前应输出 `f`,与 `sslmode=disable` 配套;若日后要求 TLS,应先检查 PostgreSQL/1Panel 的 SSL 配置,再改用 `sslmode=require`。
运行时保留普通 `postgresql://` 形式供 Psycopg 3 使用;Alembic/SQLAlchemy 会自动转换为 `postgresql+psycopg://`,并保留 `sslmode` 等 query 参数。
## 首次部署 ## 首次部署
先准备 `.env`,至少设置 `ZHIXING_TUSHARE_TOKEN`、PostgreSQL 凭据和 `ZHIXING_DATABASE_URL`,然后执行迁移: 先准备 `.env`,至少设置 `ZHIXING_TUSHARE_TOKEN`、PostgreSQL 凭据和 `ZHIXING_DATABASE_URL`,然后执行迁移:
+4 -3
View File
@@ -5,12 +5,13 @@ from logging.config import fileConfig
from alembic import context from alembic import context
from sqlalchemy import engine_from_config, pool from sqlalchemy import engine_from_config, pool
from zhixing_server.bootstrap.config import get_settings from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url
from zhixing_server.modules.market_data.infrastructure.schema import metadata from zhixing_server.modules.market_data.infrastructure.schema import metadata
config = context.config config = context.config
settings = get_settings() settings = get_settings()
config.set_main_option("sqlalchemy.url", settings.database_url.replace("%", "%%")) sqlalchemy_url = sqlalchemy_database_url(settings.database_url)
config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%"))
if config.config_file_name is not None: if config.config_file_name is not None:
fileConfig(config.config_file_name) fileConfig(config.config_file_name)
@@ -21,7 +22,7 @@ def run_migrations_offline() -> None:
"""Run migrations without opening a database connection.""" """Run migrations without opening a database connection."""
context.configure( context.configure(
url=settings.database_url, url=sqlalchemy_url,
target_metadata=target_metadata, target_metadata=target_metadata,
literal_binds=True, literal_binds=True,
dialect_opts={"paramstyle": "named"}, dialect_opts={"paramstyle": "named"},
@@ -31,6 +31,31 @@ class Settings(BaseSettings):
) )
def sqlalchemy_database_url(database_url: str) -> str:
"""Add SQLAlchemy's Psycopg 3 driver to a PostgreSQL URL.
``Settings.database_url`` remains a plain ``postgresql://`` URL because
Psycopg 3 accepts that form directly. Alembic and SQLAlchemy need the
explicit ``postgresql+psycopg://`` dialect, while the rest of the URL
(including query parameters such as ``sslmode``) must remain unchanged.
Args:
database_url: PostgreSQL connection URL from the process environment.
Returns:
The URL with SQLAlchemy's Psycopg 3 dialect prefix.
Raises:
ValueError: If the URL does not use a supported PostgreSQL scheme.
"""
if database_url.startswith("postgresql+psycopg://"):
return database_url
if database_url.startswith("postgresql://"):
return "postgresql+psycopg://" + database_url.removeprefix("postgresql://")
raise ValueError("database_url must use postgresql:// or postgresql+psycopg://")
@lru_cache @lru_cache
def get_settings() -> Settings: def get_settings() -> Settings:
"""Return one immutable-by-convention configuration object per process. """Return one immutable-by-convention configuration object per process.
@@ -6,17 +6,24 @@ from alembic import command
from alembic.config import Config from alembic.config import Config
from sqlalchemy import Engine, create_engine, inspect from sqlalchemy import Engine, create_engine, inspect
from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url
@pytest.mark.integration @pytest.mark.integration
def test_postgres_migration_creates_market_data_contract() -> None: def test_postgres_migration_creates_market_data_contract(
monkeypatch: pytest.MonkeyPatch,
) -> None:
database_url = os.getenv("ZHIXING_TEST_DATABASE_URL") database_url = os.getenv("ZHIXING_TEST_DATABASE_URL")
if not database_url: if not database_url:
pytest.skip("set ZHIXING_TEST_DATABASE_URL to run PostgreSQL integration tests") pytest.skip("set ZHIXING_TEST_DATABASE_URL to run PostgreSQL integration tests")
monkeypatch.setenv("ZHIXING_DATABASE_URL", database_url)
get_settings.cache_clear()
server_root = Path(__file__).parents[2] server_root = Path(__file__).parents[2]
config = Config(str(server_root / "alembic.ini")) config = Config(str(server_root / "alembic.ini"))
config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%")) sqlalchemy_url = sqlalchemy_database_url(database_url)
engine: Engine = create_engine(database_url) config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%"))
engine: Engine = create_engine(sqlalchemy_url)
command.upgrade(config, "head") command.upgrade(config, "head")
try: try:
tables = set(inspect(engine).get_table_names()) tables = set(inspect(engine).get_table_names())
@@ -29,3 +36,4 @@ def test_postgres_migration_creates_market_data_contract() -> None:
} <= tables } <= tables
finally: finally:
engine.dispose() engine.dispose()
get_settings.cache_clear()
@@ -0,0 +1,26 @@
import pytest
from zhixing_server.bootstrap.config import sqlalchemy_database_url
def test_sqlalchemy_url_adds_psycopg_driver_and_preserves_query() -> None:
database_url = (
"postgresql://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system"
"?sslmode=require&connect_timeout=5"
)
assert sqlalchemy_database_url(database_url) == (
"postgresql+psycopg://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system"
"?sslmode=require&connect_timeout=5"
)
def test_sqlalchemy_url_does_not_duplicate_psycopg_driver() -> None:
database_url = "postgresql+psycopg://user@localhost:5432/db?sslmode=disable"
assert sqlalchemy_database_url(database_url) == database_url
def test_sqlalchemy_url_rejects_non_postgresql_scheme() -> None:
with pytest.raises(ValueError, match="postgresql"):
sqlalchemy_database_url("mysql://user@localhost/db")