From 83deb886b54a1f04fba4c6c4cdf0f13eb0c50223 Mon Sep 17 00:00:00 2001 From: laily Date: Mon, 7 Sep 2026 15:43:38 +0800 Subject: [PATCH] feat(deployer): fall back to scp when rsync is missing --- skills/deployer/README.md | 9 +- skills/deployer/SKILL.md | 2 +- skills/deployer/references/compose.md | 2 +- skills/deployer/scripts/deploy/check.py | 2 +- skills/deployer/scripts/deploy/sync.py | 93 ++++++++++----- tests/test_deployer_sync.py | 149 ++++++++++++++++++++++++ 6 files changed, 223 insertions(+), 34 deletions(-) create mode 100644 tests/test_deployer_sync.py diff --git a/skills/deployer/README.md b/skills/deployer/README.md index 8dd11cb..2b6e966 100644 --- a/skills/deployer/README.md +++ b/skills/deployer/README.md @@ -33,7 +33,7 @@ pouch add deployer -g # 全局 ## 使用前准备 -- Compose：本机 Python 3、`rsync`、`ssh`；目标机器 Docker Compose v2；`~/.ssh/config` 免密（或 agent） +- Compose：本机 Python 3、`ssh`；有 `rsync` 更好，没有则 sync 用 `scp` 兜底；目标机器 Docker Compose v2；`~/.ssh/config` 免密（或 agent） - 知道每个 Compose 服务的运行时数据放在哪（这些目录不能被同步覆盖） - Argo CD：能 clone/push GitOps 仓库（SSH 或已登录的 HTTPS），并能开 MR（GitLab `glab` / GitHub `gh` / Gitea `tea`）。本机不必长期放一份 GitOps checkout。 @@ -73,7 +73,7 @@ GitOps 我已经 clone 在 ../infra-gitops，用那个目录开 MR 1. 读服务/环境目录（及共享的父目录）的 `_config.yaml`，确定目标机器和远程路径； 项目内布局从 `.pouch/deployer/` 自动发现，无需额外配置 2. 目标是本机且镜像已在本机 docker：不 sync、不 SSH，在该环境目录直接 `docker compose` -3. 否则用 skill 自带脚本把本地目录同步到远程（rsync，自动排除 `data/`、`_data/`），再远程 `docker compose`（启动 / 重建 / 升级 / 重启） +3. 否则用 skill 自带脚本把本地目录同步到远程（优先 rsync，没有则 scp；自动排除 `data/`、`_data/`），再远程 `docker compose`（启动 / 重建 / 升级 / 重启） 4. deb 包安装走独立脚本：scp 上传到暂存目录后远程 apt 安装，失败自动修依赖 5. 同步后查看容器状态和日志确认生效 6. 只针对你指定的那一个服务操作，不会批量动整台机器 @@ -83,8 +83,9 @@ GitOps 我已经 clone 在 ../infra-gitops，用那个目录开 MR 项目内布局下，远程目录名自动带上项目前缀（如 `my-project-prod`）， 避免同一台机器上多个项目的同名环境互相覆盖；需要固定名字时在 `_config.yaml` 写 `name:`。 -重要边界：同步使用 `--delete`，远程多余的文件会被删除；数据库、证书等运行时数据 -必须放在排除目录或远程绝对路径挂载。涉及删除数据卷、清理远程文件的操作会先向你确认。 +重要边界：rsync 同步使用 `--delete`，远程多余的文件会被删除；scp 兜底不会删。 +数据库、证书等运行时数据必须放在排除目录或远程绝对路径挂载。涉及删除数据卷、 +清理远程文件的操作会先向你确认。 ## 如何判断完成 diff --git a/skills/deployer/SKILL.md b/skills/deployer/SKILL.md index 366441b..bbe3354 100644 --- a/skills/deployer/SKILL.md +++ b/skills/deployer/SKILL.md @@ -56,7 +56,7 @@ python3 -I -S /scripts/deploy/check.py --project 已具备: … 待配置: 路径 + 字段 + 可粘贴示例 + 缺了会挡住哪步 -工具链: ssh / rsync（缺则怎么装，不擅自安装） +工具链: ssh / rsync（无 rsync 时 sync 用 scp 兜底，不擅自安装） 下一步: 一句话 ``` diff --git a/skills/deployer/references/compose.md b/skills/deployer/references/compose.md index cdeae69..e735f79 100644 --- a/skills/deployer/references/compose.md +++ b/skills/deployer/references/compose.md @@ -92,7 +92,7 @@ python3 /scripts/deploy/sync.py 脚本一律 `python3 -I -S /scripts/deploy/.py`。有 `make deploy TGT=` 才用 Makefile；没有就直接调脚本，不要读 Makefile 去发明封装。 -`sync.py` 使用 `rsync -avz --delete`；本机无 rsync 时 tar over SSH 兜底。运行时数据必须放在默认排除的 `data/`、`_data/`，或 compose 挂载的远程绝对路径，否则会被清掉。 +`sync.py` 使用 `rsync -avz --delete`；本机无 rsync 时 scp 兜底（不 `--delete`）。运行时数据必须放在默认排除的 `data/`、`_data/`，或 compose 挂载的远程绝对路径，否则会被清掉。 ## 新增服务 / 环境 diff --git a/skills/deployer/scripts/deploy/check.py b/skills/deployer/scripts/deploy/check.py index 81086c7..8756b07 100644 --- a/skills/deployer/scripts/deploy/check.py +++ b/skills/deployer/scripts/deploy/check.py @@ -117,7 +117,7 @@ def check_toolchain(report: Report) -> None: rsync_ok = shutil.which("rsync") is not None lines = [ f"ssh: {'found' if ssh_ok else 'MISSING (blocks compose deploy)'}", - f"rsync: {'found' if rsync_ok else 'MISSING (tar-over-SSH fallback)'}", + f"rsync: {'found' if rsync_ok else 'MISSING (scp fallback)'}", ] if not ssh_ok: lines.append("install openssh-client") diff --git a/skills/deployer/scripts/deploy/sync.py b/skills/deployer/scripts/deploy/sync.py index 9fa5fc2..9a307f5 100755 --- a/skills/deployer/scripts/deploy/sync.py +++ b/skills/deployer/scripts/deploy/sync.py @@ -22,14 +22,14 @@ _SCRIPT_DIR = Path(__file__).resolve().parent if str(_SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPT_DIR)) -from lib import DEFAULT_BASE_PATH, project_root, rsync_ssh_args, service_info, ssh_cmd # noqa: E402 +from lib import DEFAULT_BASE_PATH, rsync_ssh_args, scp_base_args, service_info, ssh_cmd # noqa: E402 def sync_directory(info: dict) -> int: if shutil.which("rsync"): return _sync_rsync(info) - print("本机没有 rsync，改用 tar over SSH（不会删除远程多余文件）") - return _sync_tar(info) + print("本机没有 rsync，改用 scp（不会删除远程多余文件）") + return _sync_scp(info) def _sync_rsync(info: dict) -> int: @@ -63,42 +63,81 @@ def _sync_rsync(info: dict) -> int: print(f"错误: rsync 执行失败，退出码: {e.returncode}") return 1 except FileNotFoundError: - print("错误: rsync 命令未找到，请确保已安装 rsync") - return 1 + print("本机没有 rsync，改用 scp（不会删除远程多余文件）") + return _sync_scp(info) -def _sync_tar(info: dict) -> int: +def collect_sync_files(source_dir: str, excludes: list[str]) -> list[Path]: + """列出待同步文件；排除规则与 rsync `--exclude=name/` 一致（任意层级的同名目录）。""" + source = Path(source_dir) + exclude_set = set(excludes) + files: list[Path] = [] + for root, dirs, names in os.walk(source): + dirs[:] = [d for d in dirs if d not in exclude_set] + for name in names: + path = Path(root) / name + if path.is_file(): + files.append(path) + return files + + +def _sync_scp(info: dict) -> int: source_dir = info["service_dir"] remote_dir = info["remote_dir"] - tar_cmd = ["tar", "czf", "-", "-C", source_dir] - for item in info["sync_exclude"]: - tar_cmd.append(f"--exclude={item}") - tar_cmd.append(".") + node = info["node"] + source = Path(source_dir) + files = collect_sync_files(source_dir, info["sync_exclude"]) - remote_shell = ( - f"mkdir -p {shlex.quote(remote_dir)} && " - f"tar xzf - -C {shlex.quote(remote_dir)}" - ) - ssh = ssh_cmd(info, remote_shell) - print(f"正在同步 {source_dir} 到 {info['node']}:{remote_dir}") + print(f"正在同步 {source_dir} 到 {node}:{remote_dir}") if info["sync_exclude"]: print(f"排除目录: {', '.join(info['sync_exclude'])}") - print(f"执行命令: tar | {' '.join(ssh)}") print("-" * 60) - tar = subprocess.Popen(tar_cmd, stdout=subprocess.PIPE) + remote_dirs = {remote_dir} + grouped: dict[str, list[Path]] = {} + for path in files: + parent = path.relative_to(source).parent.as_posix() + grouped.setdefault(parent, []).append(path) + if parent != ".": + remote_dirs.add(f"{remote_dir}/{parent}") + + mkdir_remote = "mkdir -p " + " ".join(shlex.quote(d) for d in sorted(remote_dirs)) + mkdir_cmd = ssh_cmd(info, mkdir_remote) + print(f"执行命令: {' '.join(mkdir_cmd)}") try: - completed = subprocess.run(ssh, stdin=tar.stdout, check=False) - finally: - if tar.stdout: - tar.stdout.close() - tar.wait() - if tar.returncode: - print(f"错误: tar 打包失败，退出码: {tar.returncode}") - return tar.returncode + completed = subprocess.run(mkdir_cmd, check=False) + except FileNotFoundError: + print("错误: ssh 命令未找到") + return 1 if completed.returncode: - print(f"错误: 远程 tar 解包失败，退出码: {completed.returncode}") + print(f"错误: 远程 mkdir 失败，退出码: {completed.returncode}") return completed.returncode + + if not files: + print("没有需要同步的文件") + print("-" * 60) + print("同步完成!") + return 0 + + try: + for parent, group in sorted(grouped.items()): + dest = remote_dir if parent == "." else f"{remote_dir}/{parent}" + scp_cmd = [ + *scp_base_args(info), + "-p", + "--", + *[str(p) for p in group], + f"{node}:{dest}/", + ] + print(f"执行命令: {' '.join(scp_cmd)}") + subprocess.run(scp_cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"错误: scp 执行失败，退出码: {e.returncode}") + return 1 + except FileNotFoundError: + print("错误: scp 命令未找到，请确保已安装 openssh-client") + return 1 + print("-" * 60) print("同步完成!") return 0 diff --git a/tests/test_deployer_sync.py b/tests/test_deployer_sync.py new file mode 100644 index 0000000..3fe5581 --- /dev/null +++ b/tests/test_deployer_sync.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEPLOY_SCRIPTS = REPO_ROOT / "skills" / "deployer" / "scripts" / "deploy" +SYNC_PY = DEPLOY_SCRIPTS / "sync.py" + +if str(DEPLOY_SCRIPTS) not in sys.path: + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + +sync_spec = importlib.util.spec_from_file_location("deployer_sync", SYNC_PY) +assert sync_spec is not None and sync_spec.loader is not None +deployer_sync = importlib.util.module_from_spec(sync_spec) +sync_spec.loader.exec_module(deployer_sync) + + +def _info(service_dir: str, **overrides) -> dict: + info = { + "service_dir": service_dir, + "node": "edge", + "remote_dir": "/opt/app/svc", + "sync_exclude": ["data", "_data"], + "port": None, + "identity_file": None, + } + info.update(overrides) + return info + + +class DeployerSyncTests(unittest.TestCase): + def test_collect_sync_files_skips_excluded_directories(self) -> None: + with tempfile.TemporaryDirectory() as temp: + src = Path(temp) + (src / "compose.yaml").write_text("ok\n", encoding="utf-8") + (src / "data").mkdir() + (src / "data" / "db").write_text("nope\n", encoding="utf-8") + nested = src / "cfg" + nested.mkdir() + (nested / "app.env").write_text("x\n", encoding="utf-8") + (nested / "data").mkdir() + (nested / "data" / "cache").write_text("nope\n", encoding="utf-8") + + files = { + p.relative_to(src).as_posix() + for p in deployer_sync.collect_sync_files(str(src), ["data", "_data"]) + } + self.assertEqual(files, {"compose.yaml", "cfg/app.env"}) + + def test_prefers_rsync_when_available(self) -> None: + info = _info("/tmp/svc") + completed = subprocess.CompletedProcess(["rsync"], 0) + with mock.patch.object(deployer_sync.shutil, "which", return_value="/usr/bin/rsync"): + with mock.patch.object( + deployer_sync.subprocess, "run", return_value=completed + ) as run: + rc = deployer_sync.sync_directory(info) + self.assertEqual(rc, 0) + cmd = run.call_args[0][0] + self.assertEqual(cmd[0], "rsync") + self.assertIn("--delete", cmd) + self.assertIn("--exclude=data/", cmd) + + def test_falls_back_to_scp_when_rsync_missing(self) -> None: + with tempfile.TemporaryDirectory() as temp: + src = Path(temp) + (src / "compose.yaml").write_text("ok\n", encoding="utf-8") + (src / "data").mkdir() + (src / "data" / "db").write_text("nope\n", encoding="utf-8") + nested = src / "cfg" + nested.mkdir() + (nested / "app.env").write_text("x\n", encoding="utf-8") + + commands: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + commands.append(list(cmd)) + return subprocess.CompletedProcess(cmd, 0) + + info = _info(str(src), port=2222, identity_file="/tmp/key") + with mock.patch.object(deployer_sync.shutil, "which", return_value=None): + with mock.patch.object( + deployer_sync.subprocess, "run", side_effect=fake_run + ): + rc = deployer_sync.sync_directory(info) + + self.assertEqual(rc, 0) + self.assertTrue(commands) + mkdir_cmd = commands[0] + self.assertEqual(mkdir_cmd[0], "ssh") + self.assertIn("-p", mkdir_cmd) + self.assertIn("2222", mkdir_cmd) + self.assertTrue(any("/opt/app/svc/cfg" in part for part in mkdir_cmd)) + + scp_cmds = [c for c in commands if c and c[0] == "scp"] + self.assertEqual(len(scp_cmds), 2) + joined = "\n".join(" ".join(c) for c in scp_cmds) + self.assertIn("compose.yaml", joined) + self.assertIn("app.env", joined) + self.assertNotIn("db", joined) + self.assertNotIn("--delete", joined) + self.assertIn("-P", scp_cmds[0]) + self.assertIn("2222", scp_cmds[0]) + self.assertIn("-i", scp_cmds[0]) + self.assertIn("/tmp/key", scp_cmds[0]) + + def test_rsync_filenotfound_falls_back_to_scp(self) -> None: + with tempfile.TemporaryDirectory() as temp: + src = Path(temp) + (src / "compose.yaml").write_text("ok\n", encoding="utf-8") + commands: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + if cmd and cmd[0] == "rsync": + raise FileNotFoundError("rsync") + commands.append(list(cmd)) + return subprocess.CompletedProcess(cmd, 0) + + with mock.patch.object( + deployer_sync.shutil, "which", return_value="/usr/bin/rsync" + ): + with mock.patch.object( + deployer_sync.subprocess, "run", side_effect=fake_run + ): + rc = deployer_sync.sync_directory(_info(str(src))) + + self.assertEqual(rc, 0) + self.assertTrue(any(c and c[0] == "scp" for c in commands)) + + def test_docs_say_scp_fallback_not_tar(self) -> None: + compose = ( + REPO_ROOT / "skills" / "deployer" / "references" / "compose.md" + ).read_text(encoding="utf-8") + check = (DEPLOY_SCRIPTS / "check.py").read_text(encoding="utf-8") + self.assertIn("本机无 rsync 时 scp 兜底", compose) + self.assertNotIn("本机无 rsync 时 tar over SSH 兜底", compose) + self.assertIn("scp fallback", check) + self.assertNotIn("tar-over-SSH", check) + + +if __name__ == "__main__": + unittest.main()