Revert "Refactor Gitea production deployment process to utilize SSH for remote operations"

This reverts commit 99bc36439e.

scripts/deploy-production.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
-# Pull and deploy on server B with configuration injected over SSH by Gitea.
-# Returns nonzero on configuration/pull/migration/health failure. Never removes data.
+# Deploy on the target Linux Docker host with configuration injected by Gitea.
+# Returns nonzero on configuration/build/migration/health failure. Never removes data.
 set -Eeuo pipefail
 umask 077
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 # Fail before touching the host; never read a checkout's local .env file.
-for key in IMAGE_PREFIX DEPLOY_TAG REGISTRY_USERNAME REGISTRY_PASSWORD WQ_EMAIL WQ_PASSWORD DATABASE_URL ADMIN_PASSWORD ENCRYPTION_KEY PUBLIC_ORIGIN; do
+for key in WQ_EMAIL WQ_PASSWORD DATABASE_URL ADMIN_PASSWORD ENCRYPTION_KEY DATABASE_NETWORK PUBLIC_ORIGIN; do
   if [[ -z "${!key:-}" ]]; then
     echo "Missing required Gitea configuration: $key" >&2
     exit 1
   fi
 done
+export DEPLOY_TAG="${DEPLOY_TAG:-$(git rev-parse HEAD)}"
 compose=(docker compose --env-file /dev/null -p wq-alpha-production -f compose.production.yaml)
 lock_id=""
-auth_dir=""
 cleanup() {
   local rc=$?
   "${compose[@]}" --profile jobs ps -a || true
@@ -22,34 +22,25 @@ cleanup() {
   if [[ -n "$lock_id" ]]; then
     docker rm "$lock_id" >/dev/null || true
   fi
-  if [[ -n "$auth_dir" ]]; then
-    rm -rf -- "$auth_dir"
-  fi
   exit "$rc"
 }
 trap cleanup EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
-trap 'exit 129' HUP
 
 "${compose[@]}" config --quiet
 "${compose[@]}" --profile jobs config --quiet
-auth_dir=$(mktemp -d)
-export DOCKER_CONFIG="$auth_dir"
-printf '%s' "$REGISTRY_PASSWORD" | docker login "${IMAGE_PREFIX%%/*}" --username "$REGISTRY_USERNAME" --password-stdin
-unset REGISTRY_PASSWORD
-# Download both images before stopping anything; migration uses the backend image.
-"${compose[@]}" pull --policy always backend web
+"${compose[@]}" build backend web
 # Docker enforces unique container names across runner jobs and checkout paths.
 # The lock container is never started and receives no deployment credentials.
 if ! lock_id=$(docker create --name wq-alpha-production-deploy-lock \
   --label "wq.deploy.commit=$DEPLOY_TAG" \
-  --network none --entrypoint /bin/true "$IMAGE_PREFIX-backend:$DEPLOY_TAG"); then
+  --network none --entrypoint /bin/true "wq-alpha-production-backend:$DEPLOY_TAG"); then
   echo 'Cannot acquire deployment lock; check Docker and other active deployments.' >&2
   exit 1
 fi
 # Validate secrets and DB connectivity before interrupting the running version.
-"${compose[@]}" run --rm --no-deps --pull never -T backend python -c '
+"${compose[@]}" run --rm --no-deps backend python -c '
 import asyncio
 from app.config import Settings
 from sqlalchemy import text
@@ -75,6 +66,6 @@ if [[ -n "$previous_images" ]]; then
   docker image inspect --format '{{.Id}} {{json .RepoTags}}' $previous_images
 fi
 "${compose[@]}" stop web backend
-"${compose[@]}" --profile jobs run --rm --no-deps --pull never -T migrate
-"${compose[@]}" up -d --no-build --pull never --remove-orphans --wait --wait-timeout 180 backend web
+"${compose[@]}" --profile jobs run --rm --no-deps migrate
+"${compose[@]}" up -d --no-build --remove-orphans --wait --wait-timeout 180 backend web
 echo "Production is healthy; release $DEPLOY_TAG"

scripts/deploy-remote.sh

@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-# Send a release bundle and environment to server B. No application .env is written.
-# Requires Bash/OpenSSH locally and Bash/tar/base64/Docker Compose on server B.
-# Returns the remote deployment status; credentials never appear in SSH arguments.
-set -Eeuo pipefail
-umask 077
-
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
-for key in PROD_HOST PROD_USER DEPLOY_PATH PROD_SSH_KEY PROD_KNOWN_HOSTS IMAGE_PREFIX DEPLOY_TAG REGISTRY_USERNAME REGISTRY_PASSWORD WQ_EMAIL WQ_PASSWORD DATABASE_URL ADMIN_PASSWORD ENCRYPTION_KEY PUBLIC_ORIGIN; do
-  if [[ -z "${!key:-}" ]]; then
-    echo "Missing required Gitea configuration: $key" >&2
-    exit 1
-  fi
-done
-if [[ "$DEPLOY_PATH" != /* || ! "$DEPLOY_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]]; then
-  echo 'DEPLOY_PATH must be absolute and DEPLOY_TAG must be a valid Docker tag.' >&2
-  exit 1
-fi
-
-ssh_dir=$(mktemp -d)
-trap 'rm -rf -- "$ssh_dir"' EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM
-trap 'exit 129' HUP
-printf '%s\n' "$PROD_SSH_KEY" > "$ssh_dir/id"
-printf '%s\n' "$PROD_KNOWN_HOSTS" > "$ssh_dir/known_hosts"
-unset PROD_SSH_KEY PROD_KNOWN_HOSTS
-ssh_options=(
-  -F /dev/null -T
-  -i "$ssh_dir/id"
-  -p "${PROD_PORT:-22}"
-  -l "$PROD_USER"
-  -o BatchMode=yes
-  -o IdentitiesOnly=yes
-  -o StrictHostKeyChecking=yes
-  -o "UserKnownHostsFile=$ssh_dir/known_hosts"
-  -o GlobalKnownHostsFile=/dev/null
-  -o ConnectTimeout=15
-  -o ServerAliveInterval=15
-  -o ServerAliveCountMax=4
-)
-
-# Bash %q preserves quotes, dollar signs and newlines without evaluating values.
-# Only this allowlist crosses SSH; private keys and the runner's environment stay local.
-{
-  printf 'set -Eeuo pipefail\numask 077\n'
-  for key in DEPLOY_PATH IMAGE_PREFIX DEPLOY_TAG REGISTRY_USERNAME REGISTRY_PASSWORD WQ_EMAIL WQ_PASSWORD DATABASE_URL ADMIN_PASSWORD ENCRYPTION_KEY PUBLIC_ORIGIN ADMIN_USERNAME MCP_ENABLED; do
-    printf 'export %s=%q\n' "$key" "${!key:-}"
-  done
-  cat <<'REMOTE'
-mkdir -p -- "$DEPLOY_PATH/releases"
-release_dir=$(mktemp -d "$DEPLOY_PATH/releases/$DEPLOY_TAG.XXXXXX")
-cd "$release_dir"
-base64 -d <<'WQ_RELEASE_BUNDLE' | tar -xzf -
-REMOTE
-  # A small base64 archive lets one SSH connection carry files and shell-quoted env.
-  # Each attempt gets its own directory, so competing jobs cannot overwrite files.
-  COPYFILE_DISABLE=1 tar -czf - compose.production.yaml scripts/deploy-production.sh | base64
-  printf '\nWQ_RELEASE_BUNDLE\n'
-  # Docker must not consume the SSH script input. B does not need a Git checkout.
-  printf 'exec bash scripts/deploy-production.sh </dev/null\n'
-} | ssh "${ssh_options[@]}" -- "$PROD_HOST" 'bash --noprofile --norc -se'

scripts/tests/test_deployment.py

@@ -1,219 +0,0 @@
-"""Exercise deployment failure boundaries without Docker or network mutations.
-
-Run with: python3 -m unittest discover -s scripts/tests -v
-"""
-
-import json
-import os
-from pathlib import Path
-import subprocess
-import sys
-import tempfile
-import unittest
-
-
-ROOT = Path(__file__).resolve().parents[2]
-RUNTIME_KEYS = (
-    "IMAGE_PREFIX", "DEPLOY_TAG", "REGISTRY_USERNAME", "REGISTRY_PASSWORD",
-    "WQ_EMAIL", "WQ_PASSWORD", "DATABASE_URL", "ADMIN_PASSWORD", "ENCRYPTION_KEY",
-    "PUBLIC_ORIGIN",
-)
-
-DOCKER_STUB = r'''
-import json, os, pathlib, sys
-a = sys.argv[1:]
-if a[0] == "compose":
-    if "config" in a: op = "config"
-    elif "pull" in a: op = "pull"
-    elif "run" in a: op = "migrate" if "migrate" in a else "preflight"
-    elif "images" in a: op = "images"
-    elif "stop" in a: op = "stop"
-    elif "up" in a: op = "up"
-    elif "ps" in a: op = "ps"
-    else: raise SystemExit("Unexpected compose invocation")
-else:
-    op = {"create": "lock", "rm": "unlock", "image": "inspect"}.get(a[0], a[0])
-entry = {"op": op, "args": a, "auth_dir": os.environ.get("DOCKER_CONFIG"),
-         "env": {k: os.environ.get(k) for k in
-                 ("ADMIN_PASSWORD", "WQ_PASSWORD", "DATABASE_URL", "DEPLOY_TAG")}}
-if op == "login":
-    entry["password"] = sys.stdin.read()
-    auth = pathlib.Path(os.environ["DOCKER_CONFIG"])
-    (auth / "config.json").write_text("test credentials")
-with open(os.environ["TEST_DOCKER_LOG"], "a") as out:
-    out.write(json.dumps(entry) + "\n")
-if os.environ.get("TEST_FAIL") == op:
-    raise SystemExit(23)
-if op == "lock": print("owned-lock-id")
-if op == "images": print("sha256:previous-backend\nsha256:previous-web")
-'''
-
-SSH_STUB = r'''
-import json, os, pathlib, stat, subprocess, sys
-a = sys.argv[1:]
-key = pathlib.Path(a[a.index("-i") + 1])
-hosts = pathlib.Path(next(x.split("=", 1)[1] for x in a if x.startswith("UserKnownHostsFile=")))
-pathlib.Path(os.environ["TEST_SSH_LOG"]).write_text(json.dumps({
-    "args": a, "key": key.read_text(), "hosts": hosts.read_text(),
-    "key_mode": stat.S_IMODE(key.stat().st_mode), "key_path": str(key),
-}))
-if os.environ.get("TEST_FAIL") == "ssh":
-    raise SystemExit(255)
-# Simulate a different host: none of the deployment variables are inherited.
-env = {k: v for k, v in os.environ.items() if k.startswith("TEST_") or k in ("PATH", "HOME", "TMPDIR")}
-result = subprocess.run(["bash", "--noprofile", "--norc", "-se"],
-                        input=sys.stdin.buffer.read(), env=env)
-raise SystemExit(result.returncode)
-'''
-
-
-class DeploymentTest(unittest.TestCase):
-    def setUp(self):
-        self.temp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.temp.cleanup)
-        self.directory = Path(self.temp.name)
-        self.bin = self.directory / "bin"
-        self.bin.mkdir()
-        for name, code in (("docker", DOCKER_STUB), ("ssh", SSH_STUB)):
-            path = self.bin / name
-            path.write_text(f"#!{sys.executable}\n" + code)
-            path.chmod(0o700)
-        self.log = self.directory / "docker.jsonl"
-        self.ssh_log = self.directory / "ssh.json"
-        self.env = {
-            "PATH": str(self.bin) + os.pathsep + os.environ["PATH"],
-            "HOME": str(self.directory), "TMPDIR": str(self.directory),
-            "TEST_DOCKER_LOG": str(self.log), "TEST_SSH_LOG": str(self.ssh_log),
-            "IMAGE_PREFIX": "registry.example.test/team/wq", "DEPLOY_TAG": "abc123",
-            "REGISTRY_USERNAME": "publisher", "REGISTRY_PASSWORD": "fake-registry-token",
-            "WQ_EMAIL": "test@example.test", "WQ_PASSWORD": "fake-worldquant-password",
-            "DATABASE_URL": "postgresql+asyncpg://wq:fake@10.0.0.30:5432/wq",
-            "ADMIN_PASSWORD": "fake-admin-password", "ENCRYPTION_KEY": "fake-key",
-            "PUBLIC_ORIGIN": "https://alpha.example.test",
-            "PROD_HOST": "server-b.example.test", "PROD_USER": "deploy", "PROD_PORT": "2222",
-            "PROD_SSH_KEY": "fake-private-key\nsecond-line",
-            "PROD_KNOWN_HOSTS": "[server-b.example.test]:2222 ssh-ed25519 fake-public-key",
-            "DEPLOY_PATH": str(self.directory / "server B's project"),
-        }
-
-    def run_script(self, script="deploy-production.sh", **overrides):
-        self.log.unlink(missing_ok=True)
-        self.ssh_log.unlink(missing_ok=True)
-        return subprocess.run(
-            ["bash", str(ROOT / "scripts" / script)],
-            env=self.env | overrides, capture_output=True, text=True, timeout=20,
-        )
-
-    def calls(self):
-        return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else []
-
-    def assert_auth_cleaned(self):
-        for call in self.calls():
-            if call["auth_dir"]:
-                self.assertFalse(Path(call["auth_dir"]).exists())
-
-    def test_release_pulls_before_stopping_and_uses_only_local_images_afterward(self):
-        result = self.run_script()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        calls = self.calls()
-        ops = [c["op"] for c in calls]
-        important = [op for op in ops if op in {"pull", "lock", "preflight", "stop", "migrate", "up", "unlock"}]
-        self.assertEqual(important, ["pull", "lock", "preflight", "stop", "migrate", "up", "unlock"])
-        for call in calls:
-            args = call["args"]
-            if call["op"] in {"preflight", "migrate", "up"}:
-                self.assertEqual(args[args.index("--pull") + 1], "never")
-                self.assertEqual(call["env"]["DATABASE_URL"], self.env["DATABASE_URL"])
-            self.assertNotIn("build", args)
-            if call["op"] == "unlock":
-                self.assertEqual(args, ["rm", "owned-lock-id"])
-        self.assertIn("Production is healthy", result.stdout)
-        self.assert_auth_cleaned()
-
-    def test_failures_before_switch_do_not_stop_running_services(self):
-        for stage in ("config", "login", "pull", "lock", "preflight"):
-            with self.subTest(stage=stage):
-                result = self.run_script(TEST_FAIL=stage)
-                self.assertNotEqual(result.returncode, 0)
-                ops = [c["op"] for c in self.calls()]
-                self.assertNotIn("stop", ops)
-                self.assertNotIn("migrate", ops)
-                self.assertNotIn("up", ops)
-                self.assertNotIn("Production is healthy", result.stdout)
-                self.assertEqual("unlock" in ops, stage == "preflight")
-                self.assert_auth_cleaned()
-
-    def test_migration_failure_does_not_start_application(self):
-        result = self.run_script(TEST_FAIL="migrate")
-        self.assertEqual(result.returncode, 23)
-        ops = [c["op"] for c in self.calls()]
-        self.assertIn("stop", ops)
-        self.assertNotIn("up", ops)
-        self.assertIn("unlock", ops)
-        self.assertNotIn("Production is healthy", result.stdout)
-        self.assert_auth_cleaned()
-
-    def test_health_failure_is_not_reported_as_success(self):
-        result = self.run_script(TEST_FAIL="up")
-        self.assertEqual(result.returncode, 23)
-        self.assertNotIn("Production is healthy", result.stdout)
-        self.assertIn("unlock", [c["op"] for c in self.calls()])
-        self.assert_auth_cleaned()
-
-    def test_missing_runtime_configuration_fails_before_docker(self):
-        for key in RUNTIME_KEYS:
-            with self.subTest(key=key):
-                result = self.run_script(**{key: ""})
-                self.assertNotEqual(result.returncode, 0)
-                self.assertIn(key, result.stderr)
-                self.assertEqual(self.calls(), [])
-
-    def test_ssh_transmits_literal_secrets_and_only_deployment_files(self):
-        marker = self.directory / "MUST_NOT_EXIST"
-        secret = f"literal $value 'double\" slash\\; $(touch {marker})\nsecond line"
-        result = self.run_script("deploy-remote.sh", ADMIN_PASSWORD=secret, WQ_PASSWORD=secret, REGISTRY_PASSWORD=secret)
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertFalse(marker.exists())
-        self.assertNotIn(secret, result.stdout + result.stderr)
-        login = next(c for c in self.calls() if c["op"] == "login")
-        self.assertEqual(login["password"], secret)
-        preflight = next(c for c in self.calls() if c["op"] == "preflight")
-        self.assertEqual(preflight["env"]["ADMIN_PASSWORD"], secret)
-        self.assertEqual(preflight["env"]["WQ_PASSWORD"], secret)
-        for call in self.calls():
-            if call["op"] in {"preflight", "migrate", "up"}:
-                self.assertEqual(call["env"]["DATABASE_URL"], self.env["DATABASE_URL"])
-        ssh = json.loads(self.ssh_log.read_text())
-        self.assertIn("StrictHostKeyChecking=yes", ssh["args"])
-        self.assertIn("BatchMode=yes", ssh["args"])
-        self.assertEqual(ssh["args"][ssh["args"].index("-p") + 1], "2222")
-        self.assertEqual(ssh["key"], self.env["PROD_SSH_KEY"] + "\n")
-        self.assertEqual(ssh["key_mode"], 0o600)
-        self.assertNotIn(secret, " ".join(ssh["args"]))
-        self.assertFalse(Path(ssh["key_path"]).exists())
-        releases = list((Path(self.env["DEPLOY_PATH"]) / "releases").iterdir())
-        self.assertEqual(len(releases), 1)
-        files = {str(p.relative_to(releases[0])) for p in releases[0].rglob("*") if p.is_file()}
-        self.assertEqual(files, {"compose.production.yaml", "scripts/deploy-production.sh"})
-        self.assert_auth_cleaned()
-
-    def test_ssh_and_remote_failures_propagate_and_clean_key(self):
-        for stage in ("ssh", "pull", "migrate", "up"):
-            with self.subTest(stage=stage):
-                result = self.run_script("deploy-remote.sh", TEST_FAIL=stage)
-                self.assertEqual(result.returncode, 255 if stage == "ssh" else 23, result.stderr)
-                self.assertNotIn("Production is healthy", result.stdout)
-                ssh = json.loads(self.ssh_log.read_text())
-                self.assertFalse(Path(ssh["key_path"]).exists())
-                self.assert_auth_cleaned()
-
-    def test_missing_ssh_configuration_fails_before_connecting(self):
-        for key in ("PROD_HOST", "PROD_USER", "PROD_SSH_KEY", "PROD_KNOWN_HOSTS", "DEPLOY_PATH"):
-            with self.subTest(key=key):
-                result = self.run_script("deploy-remote.sh", **{key: ""})
-                self.assertNotEqual(result.returncode, 0)
-                self.assertFalse(self.ssh_log.exists())
-
-
-if __name__ == "__main__":
-    unittest.main()