diff --git a/.env.example b/.env.example index 8ba6cd3..d5a4388 100644 --- a/.env.example +++ b/.env.example @@ -3,10 +3,18 @@ ZHIXING_SERVER_PORT=8000 ZHIXING_WEB_PORT=5173 ZHIXING_APP_ENV=development ZHIXING_LOG_LEVEL=INFO + +# Development: docker-compose.dev.yml starts an independent local PostgreSQL. ZHIXING_POSTGRES_DB=zhixing ZHIXING_POSTGRES_USER=zhixing ZHIXING_POSTGRES_PASSWORD=zhixing -ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing +# docker-compose.dev.yml uses the same local URL when this variable is unset. +# ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing + +# Production: use the existing 1Panel PostgreSQL container on the external +# Docker network. Replace ; do not use the development URL. +# The current 1Panel PostgreSQL has SSL disabled; use its stable network alias. +# ZHIXING_DATABASE_URL=postgresql://zhixing-system:@postgresql:5432/zhixing-system?sslmode=disable ZHIXING_TUSHARE_TOKEN= ZHIXING_MARKET_DATA_CSV_ROOT=/app/data/market-data API_UPSTREAM=http://server:8000 diff --git a/.gitea/workflows/deploy-production.yaml b/.gitea/workflows/deploy-production.yaml index a5751a2..938b5b8 100644 --- a/.gitea/workflows/deploy-production.yaml +++ b/.gitea/workflows/deploy-production.yaml @@ -14,17 +14,24 @@ jobs: env: COMPOSE_PROJECT_NAME: zhixing-system ZHIXING_WEB_PORT: "8111" + ZHIXING_DATABASE_URL: ${{ secrets.ZHIXING_DATABASE_URL }} + ZHIXING_TUSHARE_TOKEN: ${{ secrets.ZHIXING_TUSHARE_TOKEN }} steps: - name: Checkout repository uses: actions/checkout@v4 - name: Validate production compose - run: docker compose -f docker-compose.prod.yml config + run: | + docker compose -f docker-compose.prod.yml config --quiet + docker compose -f docker-compose.prod.yml --profile jobs config --quiet - name: Build and deploy services run: docker compose -f docker-compose.prod.yml up -d --build --remove-orphans + - name: Run database migrations + run: docker compose -f docker-compose.prod.yml --profile jobs run --rm --build migrate + - name: Verify web service run: docker compose -f docker-compose.prod.yml exec -T web wget -q -O - http://127.0.0.1/healthz diff --git a/.trellis/spec/backend/market-data-sync.md b/.trellis/spec/backend/market-data-sync.md index a82cb3f..052e544 100644 --- a/.trellis/spec/backend/market-data-sync.md +++ b/.trellis/spec/backend/market-data-sync.md @@ -24,6 +24,9 @@ - `ZHIXING_MARKET_DATA_COVERAGE_THRESHOLD`，默认 `0.99` - `ZHIXING_MARKET_DATA_MAX_RETRIES`、`ZHIXING_MARKET_DATA_RETRY_BACKOFF_SECONDS`、`ZHIXING_MARKET_DATA_REQUEST_INTERVAL_SECONDS` - `ZHIXING_MARKET_DATA_ADVISORY_LOCK_KEY` +- `ZHIXING_DATABASE_URL` 使用普通 `postgresql://...` 形式供 Psycopg 3 直接连接；Alembic/SQLAlchemy 在边界处转换为 `postgresql+psycopg://...`，不得丢失 `sslmode` 等 query 参数。 +- 开发 Compose 使用项目自带 PostgreSQL；生产 Compose 不声明内部 PostgreSQL 服务，`server`、`migrate` 和 `market-sync` 连接外部 Docker 网络 `1panel-network`。`server` 同时保留 Compose 默认网络以供 web 访问。 +- 当前生产 PostgreSQL 在 `1panel-network` 上的稳定别名为 `postgresql`，服务端 SSL 为关闭状态，生产连接串使用 `sslmode=disable`；数据库用户必须具备 `public` schema 的 `CREATE` 权限以执行 Alembic 迁移。 - 股票 qfq 快照路径为 `bars/.csv`；每日指标路径为 `daily-basic//.csv`；当前股票主数据为 `stock-basic/current.csv`。 - `market_daily_bar` 的唯一键是 `(ts_code, trade_date)`，`source_adj` 必须是 `qfq`；所有价格、金额和比率使用有限 `NUMERIC`/`Decimal`。 - `SyncBatchSummary` 至少返回 `batch_id`、目标交易日、窗口、状态、目标数、有效数、覆盖率、策略资格、插入数、更新数、未变化数和失败列表。 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 372e815..40eb0ce 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,32 +1,14 @@ name: ${COMPOSE_PROJECT_NAME:-zhixing-system} services: - postgres: - image: postgres:16-alpine - restart: unless-stopped - environment: - POSTGRES_DB: ${ZHIXING_POSTGRES_DB:-zhixing} - POSTGRES_PASSWORD: ${ZHIXING_POSTGRES_PASSWORD:-zhixing} - POSTGRES_USER: ${ZHIXING_POSTGRES_USER:-zhixing} - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] - interval: 10s - timeout: 5s - retries: 5 - volumes: - - postgres-data:/var/lib/postgresql/data - server: build: context: ./zhixing-server target: production - depends_on: - postgres: - condition: service_healthy restart: unless-stopped environment: ZHIXING_APP_ENV: production - ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} + ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL} ZHIXING_LOG_LEVEL: ${ZHIXING_LOG_LEVEL:-INFO} ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-} @@ -45,6 +27,9 @@ services: start_period: 10s volumes: - market-data:/app/data/market-data + networks: + - default + - 1panel-network web: build: @@ -66,11 +51,10 @@ services: context: ./zhixing-server target: production command: ["alembic", "upgrade", "head"] - depends_on: - postgres: - condition: service_healthy environment: - ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} + ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL} + networks: + - 1panel-network market-sync: profiles: ["jobs"] @@ -82,12 +66,18 @@ services: migrate: condition: service_completed_successfully environment: - ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:-postgresql://zhixing:zhixing@postgres:5432/zhixing} + ZHIXING_DATABASE_URL: ${ZHIXING_DATABASE_URL:?Set ZHIXING_DATABASE_URL to the 1Panel PostgreSQL URL} ZHIXING_MARKET_DATA_CSV_ROOT: /app/data/market-data ZHIXING_TUSHARE_TOKEN: ${ZHIXING_TUSHARE_TOKEN:-} volumes: - market-data:/app/data/market-data + networks: + - 1panel-network volumes: market-data: - postgres-data: + +networks: + 1panel-network: + name: 1panel-network + external: true diff --git a/docs/market-data-sync.md b/docs/market-data-sync.md index efde67a..5f93f31 100644 --- a/docs/market-data-sync.md +++ b/docs/market-data-sync.md @@ -2,6 +2,44 @@ `market-data-sync` 是外部调度器触发的一次性任务。FastAPI 进程不包含定时器；PostgreSQL 是策略查询事实源，`market-data` 卷中的 CSV 只作为 qfq 落地快照和恢复介质。 +## 环境与数据库连接 + +开发环境由 `docker-compose.dev.yml` 启动项目自带的独立 PostgreSQL，容器内连接串保持： + +```dotenv +ZHIXING_DATABASE_URL=postgresql://zhixing:zhixing@postgres:5432/zhixing +``` + +生产环境不启动项目自带 PostgreSQL。现有 1Panel PostgreSQL 容器通过外部 Docker 网络 `1panel-network` 的稳定别名 `postgresql` 访问，数据库和用户均使用 `zhixing-system`。生产 `.env` 必须显式设置连接串；Compose 不会回退到开发数据库： + +```dotenv +ZHIXING_DATABASE_URL=postgresql://zhixing-system:@postgresql:5432/zhixing-system?sslmode=disable +``` + +当前服务器执行 `SHOW ssl` 的结果为 `off`，因此使用 `sslmode=disable` 或省略该参数。若日后在 1Panel/PostgreSQL 中启用 TLS，再将它改成 `sslmode=require`。密码中的 `@`、`:`、`/` 等字符必须 URL 编码。启用 TLS 后的示例如下： + +```dotenv +ZHIXING_DATABASE_URL=postgresql://zhixing-system:@postgresql:5432/zhixing-system?sslmode=require +``` + +部署前确认外部网络和数据库容器都在该网络中： + +```bash +docker network inspect 1panel-network --format '{{json .Containers}}' +docker inspect 1Panel-postgresql-5Fc7 --format '{{json .NetworkSettings.Networks}}' +``` + +迁移或同步容器连接成功后，可只输出当前连接是否使用 SSL（不会打印连接串）： + +```bash +docker compose -f docker-compose.prod.yml --profile jobs run --rm --no-deps market-sync \ + python -c 'import psycopg; from zhixing_server.bootstrap.config import get_settings; connection=psycopg.connect(get_settings().database_url); print(connection.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]); connection.close()' +``` + +输出 `t` 表示当前连接使用 SSL，输出 `f` 表示未使用。本服务器当前应输出 `f`，与 `sslmode=disable` 配套；若日后要求 TLS，应先检查 PostgreSQL/1Panel 的 SSL 配置，再改用 `sslmode=require`。 + +运行时保留普通 `postgresql://` 形式供 Psycopg 3 使用；Alembic/SQLAlchemy 会自动转换为 `postgresql+psycopg://`，并保留 `sslmode` 等 query 参数。 + ## 首次部署 先准备 `.env`，至少设置 `ZHIXING_TUSHARE_TOKEN`、PostgreSQL 凭据和 `ZHIXING_DATABASE_URL`，然后执行迁移： diff --git a/zhixing-server/migrations/env.py b/zhixing-server/migrations/env.py index 1b36969..9f9f726 100644 --- a/zhixing-server/migrations/env.py +++ b/zhixing-server/migrations/env.py @@ -5,12 +5,13 @@ from logging.config import fileConfig from alembic import context from sqlalchemy import engine_from_config, pool -from zhixing_server.bootstrap.config import get_settings +from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url from zhixing_server.modules.market_data.infrastructure.schema import metadata config = context.config settings = get_settings() -config.set_main_option("sqlalchemy.url", settings.database_url.replace("%", "%%")) +sqlalchemy_url = sqlalchemy_database_url(settings.database_url) +config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%")) if config.config_file_name is not None: fileConfig(config.config_file_name) @@ -21,7 +22,7 @@ def run_migrations_offline() -> None: """Run migrations without opening a database connection.""" context.configure( - url=settings.database_url, + url=sqlalchemy_url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, diff --git a/zhixing-server/src/zhixing_server/bootstrap/config.py b/zhixing-server/src/zhixing_server/bootstrap/config.py index 527b8ba..8889338 100644 --- a/zhixing-server/src/zhixing_server/bootstrap/config.py +++ b/zhixing-server/src/zhixing_server/bootstrap/config.py @@ -31,6 +31,31 @@ class Settings(BaseSettings): ) +def sqlalchemy_database_url(database_url: str) -> str: + """Add SQLAlchemy's Psycopg 3 driver to a PostgreSQL URL. + + ``Settings.database_url`` remains a plain ``postgresql://`` URL because + Psycopg 3 accepts that form directly. Alembic and SQLAlchemy need the + explicit ``postgresql+psycopg://`` dialect, while the rest of the URL + (including query parameters such as ``sslmode``) must remain unchanged. + + Args: + database_url: PostgreSQL connection URL from the process environment. + + Returns: + The URL with SQLAlchemy's Psycopg 3 dialect prefix. + + Raises: + ValueError: If the URL does not use a supported PostgreSQL scheme. + """ + + if database_url.startswith("postgresql+psycopg://"): + return database_url + if database_url.startswith("postgresql://"): + return "postgresql+psycopg://" + database_url.removeprefix("postgresql://") + raise ValueError("database_url must use postgresql:// or postgresql+psycopg://") + + @lru_cache def get_settings() -> Settings: """Return one immutable-by-convention configuration object per process. diff --git a/zhixing-server/tests/integration/test_market_data_migration.py b/zhixing-server/tests/integration/test_market_data_migration.py index 1d992b2..ae57f20 100644 --- a/zhixing-server/tests/integration/test_market_data_migration.py +++ b/zhixing-server/tests/integration/test_market_data_migration.py @@ -6,17 +6,24 @@ from alembic import command from alembic.config import Config from sqlalchemy import Engine, create_engine, inspect +from zhixing_server.bootstrap.config import get_settings, sqlalchemy_database_url + @pytest.mark.integration -def test_postgres_migration_creates_market_data_contract() -> None: +def test_postgres_migration_creates_market_data_contract( + monkeypatch: pytest.MonkeyPatch, +) -> None: database_url = os.getenv("ZHIXING_TEST_DATABASE_URL") if not database_url: pytest.skip("set ZHIXING_TEST_DATABASE_URL to run PostgreSQL integration tests") + monkeypatch.setenv("ZHIXING_DATABASE_URL", database_url) + get_settings.cache_clear() server_root = Path(__file__).parents[2] config = Config(str(server_root / "alembic.ini")) - config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%")) - engine: Engine = create_engine(database_url) + sqlalchemy_url = sqlalchemy_database_url(database_url) + config.set_main_option("sqlalchemy.url", sqlalchemy_url.replace("%", "%%")) + engine: Engine = create_engine(sqlalchemy_url) command.upgrade(config, "head") try: tables = set(inspect(engine).get_table_names()) @@ -29,3 +36,4 @@ def test_postgres_migration_creates_market_data_contract() -> None: } <= tables finally: engine.dispose() + get_settings.cache_clear() diff --git a/zhixing-server/tests/unit/test_database_url.py b/zhixing-server/tests/unit/test_database_url.py new file mode 100644 index 0000000..7a03033 --- /dev/null +++ b/zhixing-server/tests/unit/test_database_url.py @@ -0,0 +1,26 @@ +import pytest + +from zhixing_server.bootstrap.config import sqlalchemy_database_url + + +def test_sqlalchemy_url_adds_psycopg_driver_and_preserves_query() -> None: + database_url = ( + "postgresql://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system" + "?sslmode=require&connect_timeout=5" + ) + + assert sqlalchemy_database_url(database_url) == ( + "postgresql+psycopg://zhixing-system@1Panel-postgresql-5Fc7:5432/zhixing-system" + "?sslmode=require&connect_timeout=5" + ) + + +def test_sqlalchemy_url_does_not_duplicate_psycopg_driver() -> None: + database_url = "postgresql+psycopg://user@localhost:5432/db?sslmode=disable" + + assert sqlalchemy_database_url(database_url) == database_url + + +def test_sqlalchemy_url_rejects_non_postgresql_scheme() -> None: + with pytest.raises(ValueError, match="postgresql"): + sqlalchemy_database_url("mysql://user@localhost/db")