Refactor Gitea production deployment process to utilize SSH for remote operations
Deploy production / deploy (push) Failing after 8s
Deploy production / deploy (push) Failing after 8s
- Updated deployment specification to reflect the new architecture involving servers A, B, and C. - Revised README to describe the new deployment method using Gitea Runner and SSH. - Modified `compose.production.yaml` to remove build context and use image tags directly. - Enhanced deployment documentation to clarify configuration steps and environment variable requirements. - Introduced `deploy-remote.sh` script for handling remote deployment tasks over SSH. - Added unit tests for deployment scripts to ensure robustness and error handling. - Updated `deploy-production.sh` to streamline image pulling and deployment processes.
This commit is contained in:
yuxuanhui
@@ -0,0 +1,219 @@
|
||||
"""Exercise deployment failure boundaries without Docker or network mutations.
|
||||
|
||||
Run with: python3 -m unittest discover -s scripts/tests -v
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
RUNTIME_KEYS = (
|
||||
"IMAGE_PREFIX", "DEPLOY_TAG", "REGISTRY_USERNAME", "REGISTRY_PASSWORD",
|
||||
"WQ_EMAIL", "WQ_PASSWORD", "DATABASE_URL", "ADMIN_PASSWORD", "ENCRYPTION_KEY",
|
||||
"PUBLIC_ORIGIN",
|
||||
)
|
||||
|
||||
DOCKER_STUB = r'''
|
||||
import json, os, pathlib, sys
|
||||
a = sys.argv[1:]
|
||||
if a[0] == "compose":
|
||||
if "config" in a: op = "config"
|
||||
elif "pull" in a: op = "pull"
|
||||
elif "run" in a: op = "migrate" if "migrate" in a else "preflight"
|
||||
elif "images" in a: op = "images"
|
||||
elif "stop" in a: op = "stop"
|
||||
elif "up" in a: op = "up"
|
||||
elif "ps" in a: op = "ps"
|
||||
else: raise SystemExit("Unexpected compose invocation")
|
||||
else:
|
||||
op = {"create": "lock", "rm": "unlock", "image": "inspect"}.get(a[0], a[0])
|
||||
entry = {"op": op, "args": a, "auth_dir": os.environ.get("DOCKER_CONFIG"),
|
||||
"env": {k: os.environ.get(k) for k in
|
||||
("ADMIN_PASSWORD", "WQ_PASSWORD", "DATABASE_URL", "DEPLOY_TAG")}}
|
||||
if op == "login":
|
||||
entry["password"] = sys.stdin.read()
|
||||
auth = pathlib.Path(os.environ["DOCKER_CONFIG"])
|
||||
(auth / "config.json").write_text("test credentials")
|
||||
with open(os.environ["TEST_DOCKER_LOG"], "a") as out:
|
||||
out.write(json.dumps(entry) + "\n")
|
||||
if os.environ.get("TEST_FAIL") == op:
|
||||
raise SystemExit(23)
|
||||
if op == "lock": print("owned-lock-id")
|
||||
if op == "images": print("sha256:previous-backend\nsha256:previous-web")
|
||||
'''
|
||||
|
||||
SSH_STUB = r'''
|
||||
import json, os, pathlib, stat, subprocess, sys
|
||||
a = sys.argv[1:]
|
||||
key = pathlib.Path(a[a.index("-i") + 1])
|
||||
hosts = pathlib.Path(next(x.split("=", 1)[1] for x in a if x.startswith("UserKnownHostsFile=")))
|
||||
pathlib.Path(os.environ["TEST_SSH_LOG"]).write_text(json.dumps({
|
||||
"args": a, "key": key.read_text(), "hosts": hosts.read_text(),
|
||||
"key_mode": stat.S_IMODE(key.stat().st_mode), "key_path": str(key),
|
||||
}))
|
||||
if os.environ.get("TEST_FAIL") == "ssh":
|
||||
raise SystemExit(255)
|
||||
# Simulate a different host: none of the deployment variables are inherited.
|
||||
env = {k: v for k, v in os.environ.items() if k.startswith("TEST_") or k in ("PATH", "HOME", "TMPDIR")}
|
||||
result = subprocess.run(["bash", "--noprofile", "--norc", "-se"],
|
||||
input=sys.stdin.buffer.read(), env=env)
|
||||
raise SystemExit(result.returncode)
|
||||
'''
|
||||
|
||||
|
||||
class DeploymentTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.directory = Path(self.temp.name)
|
||||
self.bin = self.directory / "bin"
|
||||
self.bin.mkdir()
|
||||
for name, code in (("docker", DOCKER_STUB), ("ssh", SSH_STUB)):
|
||||
path = self.bin / name
|
||||
path.write_text(f"#!{sys.executable}\n" + code)
|
||||
path.chmod(0o700)
|
||||
self.log = self.directory / "docker.jsonl"
|
||||
self.ssh_log = self.directory / "ssh.json"
|
||||
self.env = {
|
||||
"PATH": str(self.bin) + os.pathsep + os.environ["PATH"],
|
||||
"HOME": str(self.directory), "TMPDIR": str(self.directory),
|
||||
"TEST_DOCKER_LOG": str(self.log), "TEST_SSH_LOG": str(self.ssh_log),
|
||||
"IMAGE_PREFIX": "registry.example.test/team/wq", "DEPLOY_TAG": "abc123",
|
||||
"REGISTRY_USERNAME": "publisher", "REGISTRY_PASSWORD": "fake-registry-token",
|
||||
"WQ_EMAIL": "test@example.test", "WQ_PASSWORD": "fake-worldquant-password",
|
||||
"DATABASE_URL": "postgresql+asyncpg://wq:fake@10.0.0.30:5432/wq",
|
||||
"ADMIN_PASSWORD": "fake-admin-password", "ENCRYPTION_KEY": "fake-key",
|
||||
"PUBLIC_ORIGIN": "https://alpha.example.test",
|
||||
"PROD_HOST": "server-b.example.test", "PROD_USER": "deploy", "PROD_PORT": "2222",
|
||||
"PROD_SSH_KEY": "fake-private-key\nsecond-line",
|
||||
"PROD_KNOWN_HOSTS": "[server-b.example.test]:2222 ssh-ed25519 fake-public-key",
|
||||
"DEPLOY_PATH": str(self.directory / "server B's project"),
|
||||
}
|
||||
|
||||
def run_script(self, script="deploy-production.sh", **overrides):
|
||||
self.log.unlink(missing_ok=True)
|
||||
self.ssh_log.unlink(missing_ok=True)
|
||||
return subprocess.run(
|
||||
["bash", str(ROOT / "scripts" / script)],
|
||||
env=self.env | overrides, capture_output=True, text=True, timeout=20,
|
||||
)
|
||||
|
||||
def calls(self):
|
||||
return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else []
|
||||
|
||||
def assert_auth_cleaned(self):
|
||||
for call in self.calls():
|
||||
if call["auth_dir"]:
|
||||
self.assertFalse(Path(call["auth_dir"]).exists())
|
||||
|
||||
def test_release_pulls_before_stopping_and_uses_only_local_images_afterward(self):
|
||||
result = self.run_script()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
calls = self.calls()
|
||||
ops = [c["op"] for c in calls]
|
||||
important = [op for op in ops if op in {"pull", "lock", "preflight", "stop", "migrate", "up", "unlock"}]
|
||||
self.assertEqual(important, ["pull", "lock", "preflight", "stop", "migrate", "up", "unlock"])
|
||||
for call in calls:
|
||||
args = call["args"]
|
||||
if call["op"] in {"preflight", "migrate", "up"}:
|
||||
self.assertEqual(args[args.index("--pull") + 1], "never")
|
||||
self.assertEqual(call["env"]["DATABASE_URL"], self.env["DATABASE_URL"])
|
||||
self.assertNotIn("build", args)
|
||||
if call["op"] == "unlock":
|
||||
self.assertEqual(args, ["rm", "owned-lock-id"])
|
||||
self.assertIn("Production is healthy", result.stdout)
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_failures_before_switch_do_not_stop_running_services(self):
|
||||
for stage in ("config", "login", "pull", "lock", "preflight"):
|
||||
with self.subTest(stage=stage):
|
||||
result = self.run_script(TEST_FAIL=stage)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
ops = [c["op"] for c in self.calls()]
|
||||
self.assertNotIn("stop", ops)
|
||||
self.assertNotIn("migrate", ops)
|
||||
self.assertNotIn("up", ops)
|
||||
self.assertNotIn("Production is healthy", result.stdout)
|
||||
self.assertEqual("unlock" in ops, stage == "preflight")
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_migration_failure_does_not_start_application(self):
|
||||
result = self.run_script(TEST_FAIL="migrate")
|
||||
self.assertEqual(result.returncode, 23)
|
||||
ops = [c["op"] for c in self.calls()]
|
||||
self.assertIn("stop", ops)
|
||||
self.assertNotIn("up", ops)
|
||||
self.assertIn("unlock", ops)
|
||||
self.assertNotIn("Production is healthy", result.stdout)
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_health_failure_is_not_reported_as_success(self):
|
||||
result = self.run_script(TEST_FAIL="up")
|
||||
self.assertEqual(result.returncode, 23)
|
||||
self.assertNotIn("Production is healthy", result.stdout)
|
||||
self.assertIn("unlock", [c["op"] for c in self.calls()])
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_missing_runtime_configuration_fails_before_docker(self):
|
||||
for key in RUNTIME_KEYS:
|
||||
with self.subTest(key=key):
|
||||
result = self.run_script(**{key: ""})
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(key, result.stderr)
|
||||
self.assertEqual(self.calls(), [])
|
||||
|
||||
def test_ssh_transmits_literal_secrets_and_only_deployment_files(self):
|
||||
marker = self.directory / "MUST_NOT_EXIST"
|
||||
secret = f"literal $value 'double\" slash\\; $(touch {marker})\nsecond line"
|
||||
result = self.run_script("deploy-remote.sh", ADMIN_PASSWORD=secret, WQ_PASSWORD=secret, REGISTRY_PASSWORD=secret)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(marker.exists())
|
||||
self.assertNotIn(secret, result.stdout + result.stderr)
|
||||
login = next(c for c in self.calls() if c["op"] == "login")
|
||||
self.assertEqual(login["password"], secret)
|
||||
preflight = next(c for c in self.calls() if c["op"] == "preflight")
|
||||
self.assertEqual(preflight["env"]["ADMIN_PASSWORD"], secret)
|
||||
self.assertEqual(preflight["env"]["WQ_PASSWORD"], secret)
|
||||
for call in self.calls():
|
||||
if call["op"] in {"preflight", "migrate", "up"}:
|
||||
self.assertEqual(call["env"]["DATABASE_URL"], self.env["DATABASE_URL"])
|
||||
ssh = json.loads(self.ssh_log.read_text())
|
||||
self.assertIn("StrictHostKeyChecking=yes", ssh["args"])
|
||||
self.assertIn("BatchMode=yes", ssh["args"])
|
||||
self.assertEqual(ssh["args"][ssh["args"].index("-p") + 1], "2222")
|
||||
self.assertEqual(ssh["key"], self.env["PROD_SSH_KEY"] + "\n")
|
||||
self.assertEqual(ssh["key_mode"], 0o600)
|
||||
self.assertNotIn(secret, " ".join(ssh["args"]))
|
||||
self.assertFalse(Path(ssh["key_path"]).exists())
|
||||
releases = list((Path(self.env["DEPLOY_PATH"]) / "releases").iterdir())
|
||||
self.assertEqual(len(releases), 1)
|
||||
files = {str(p.relative_to(releases[0])) for p in releases[0].rglob("*") if p.is_file()}
|
||||
self.assertEqual(files, {"compose.production.yaml", "scripts/deploy-production.sh"})
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_ssh_and_remote_failures_propagate_and_clean_key(self):
|
||||
for stage in ("ssh", "pull", "migrate", "up"):
|
||||
with self.subTest(stage=stage):
|
||||
result = self.run_script("deploy-remote.sh", TEST_FAIL=stage)
|
||||
self.assertEqual(result.returncode, 255 if stage == "ssh" else 23, result.stderr)
|
||||
self.assertNotIn("Production is healthy", result.stdout)
|
||||
ssh = json.loads(self.ssh_log.read_text())
|
||||
self.assertFalse(Path(ssh["key_path"]).exists())
|
||||
self.assert_auth_cleaned()
|
||||
|
||||
def test_missing_ssh_configuration_fails_before_connecting(self):
|
||||
for key in ("PROD_HOST", "PROD_USER", "PROD_SSH_KEY", "PROD_KNOWN_HOSTS", "DEPLOY_PATH"):
|
||||
with self.subTest(key=key):
|
||||
result = self.run_script("deploy-remote.sh", **{key: ""})
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(self.ssh_log.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user