feat(deployer): add deb package distribution and install via scp/apt
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
|
||||
- 想用一套固定流程把本地改好的 Docker Compose 配置发到某台服务器
|
||||
- 要升级某个服务的镜像版本、重启服务、看远程容器状态和日志
|
||||
- 有编译好的 .deb 包要装到某台机器上(scp 上传安装,或从 URL 直接拉)
|
||||
- 新加一个服务、把服务从一台机器挪到另一台、或下线旧服务
|
||||
- 想给当前项目加 prod/test/dev 三套远程环境并随时部署其中一套
|
||||
- 需要一张「哪台机器跑哪些服务」的清单
|
||||
@@ -34,6 +35,11 @@
|
||||
新增一个服务 uptime 到 vora3,先帮我建好目录结构
|
||||
vhom1 上那个 naiveproxy 为什么 sync 失败?
|
||||
|
||||
# deb 包安装
|
||||
把 ./gpt-load_1.2.0_amd64.deb 装到 web1 上
|
||||
把这个目录里的三个 .deb 都推到 deploy@nas 再安装
|
||||
web1 能出网,直接让它从 https://... 把包拉下来装
|
||||
|
||||
# 项目内环境
|
||||
给这个项目建好 .skiff/deployer,prod 和 test 分别放到两台机器上
|
||||
把 test 环境重新部署一下
|
||||
@@ -46,8 +52,9 @@ prod 的 compose 加个 redis,改完发上去
|
||||
项目内布局从 `.skiff/deployer/` 自动发现,无需额外配置
|
||||
2. 用 skill 自带脚本把本地目录同步到远程(rsync,自动排除 `data/`、`_data/`)
|
||||
3. 在远程执行对应的 `docker compose` 操作(启动 / 重建 / 升级 / 重启)
|
||||
4. 同步后查看容器状态和日志确认生效
|
||||
5. 只针对你指定的那一个服务操作,不会批量动整台机器
|
||||
4. deb 包安装走独立脚本:scp 上传到暂存目录后远程 apt 安装,失败自动修依赖
|
||||
5. 同步后查看容器状态和日志确认生效
|
||||
6. 只针对你指定的那一个服务操作,不会批量动整台机器
|
||||
|
||||
项目内布局下,远程目录名自动带上项目前缀(如 `my-project-prod`),
|
||||
避免同一台机器上多个项目的同名环境互相覆盖;需要固定名字时在 `_config.yaml` 写 `name:`。
|
||||
@@ -59,4 +66,5 @@ prod 的 compose 加个 redis,改完发上去
|
||||
|
||||
- 脚本输出显示同步完成、远程命令执行成功
|
||||
- `ps` 显示容器 Up、`logs` 无报错;升级后镜像 tag 与配置一致
|
||||
- deb 安装后 `ssh <node> dpkg -l` 能看到目标包,服务能正常启动
|
||||
- 域名/端口类服务能 curl 通
|
||||
|
||||
@@ -3,8 +3,9 @@ name: deployer
|
||||
description: >-
|
||||
管理多 VPS / NAS 的 Docker Compose 配置中心:仓库只存服务数据(compose.yaml、静态配置),
|
||||
部署方法与脚本由本 skill 提供。当用户要求部署、同步、升级、重启远程 Docker 服务,
|
||||
新增/迁移/下线服务,梳理节点与服务清单,或提到 make sync/deploy/upgrade/TGT、_config.yaml、
|
||||
rsync 同步、tar over SSH、Synology NAS 部署失败时使用。
|
||||
新增/迁移/下线服务,梳理节点与服务清单,向节点分发安装 deb 包(scp 上传 + dpkg/apt 安装,
|
||||
或从 URL 远程拉取安装),或提到 make sync/deploy/upgrade/TGT、_config.yaml、rsync 同步、
|
||||
tar over SSH、Synology NAS 部署失败时使用。
|
||||
---
|
||||
|
||||
# deployer:多机 Compose 配置中心
|
||||
@@ -17,6 +18,7 @@ description: >-
|
||||
## 何时使用
|
||||
|
||||
- 部署 / 同步 / 升级 / 重启某个远程 Docker Compose 服务
|
||||
- 向节点安装 deb 包:scp 上传本地 .deb 后 dpkg/apt 安装,或远程从 URL 直接拉取安装
|
||||
- 新增、迁移、下线一个服务;梳理「哪台机器跑什么」
|
||||
- sync 失败排查、证书丢失、改了配置不生效等运维问题
|
||||
- 提到 `make deploy TGT=...`、`TGT=`、`_config.yaml`、rsync/tar 同步
|
||||
@@ -150,6 +152,31 @@ python3 <skill-dir>/scripts/deploy/sync.py <service-path>
|
||||
`ssh <node> "cd <base_path>/<name> && docker compose down"`,数据卷按需保留或删除。
|
||||
项目环境布局:删除对应 `.skiff/deployer/{env}/` 目录即可脱离发现体系,远程清理同上。
|
||||
|
||||
### 7. 向节点安装 deb 包
|
||||
|
||||
`deb.py` 把 deb 包发到节点并安装。目标两种写法:仓库内目录
|
||||
(复用 `_config.yaml` 继承链解析 node/port/identity_file,如 `hosts/web1`),
|
||||
或裸 SSH 别名 / `user@host`(须在 `~/.ssh/config` 中,可加 `--port`/`--identity`)。
|
||||
|
||||
```bash
|
||||
# 本地 .deb → scp 上传 → 远程 apt 安装(失败自动 apt -f 修依赖),成功后删暂存包
|
||||
python3 <skill-dir>/scripts/deploy/deb.py <target> push ./foo_1.0_amd64.deb --yes
|
||||
|
||||
# 仅上传到远程暂存目录(默认 {base_path}/.debs;裸主机为 /tmp/deployer-debs)
|
||||
python3 <skill-dir>/scripts/deploy/deb.py <target> scp ./foo_1.0_amd64.deb
|
||||
|
||||
# 安装该节点暂存目录里已上传的全部 .deb(配合 scp 分步操作)
|
||||
python3 <skill-dir>/scripts/deploy/deb.py <target> dpkg --yes
|
||||
|
||||
# 远程直接从 URL 下载安装(机器能出网时免上传)
|
||||
python3 <skill-dir>/scripts/deploy/deb.py <target> apt https://example.com/foo_1.0_amd64.deb --yes
|
||||
```
|
||||
|
||||
- 非 root 用户走 `sudo -n`(需配好免密 sudo);`--yes` 传 `-y` 免交互,
|
||||
无终端交互能力,没配 sudo 免密/密钥时会直接失败。
|
||||
- 升级同版本号前想先看包信息:`ssh <node> "dpkg -I <暂存路径>"`;
|
||||
装完验证:`ssh <node> "dpkg -l | grep <pkg>"`。
|
||||
|
||||
---
|
||||
|
||||
## 注意事项
|
||||
@@ -188,9 +215,10 @@ python3 <skill-dir>/scripts/deploy/sync.py <service-path>
|
||||
|
||||
| 文件 | 用途 |
|
||||
|------|------|
|
||||
| `scripts/deploy/lib.py` | 解析服务目录、合并继承 `_config.yaml`、SSH/rsync 参数构造 |
|
||||
| `scripts/deploy/lib.py` | 解析服务目录、合并继承 `_config.yaml`、SSH/rsync/scp 参数构造 |
|
||||
| `scripts/deploy/sync.py` | rsync -avz --delete 同步;无 rsync 时 tar over SSH 兜底 |
|
||||
| `scripts/deploy/remote.py` | SSH 远程 docker compose:up/recreate/restart/upgrade/ps/logs |
|
||||
| `scripts/deploy/deb.py` | deb 包分发安装:push(scp+apt)/scp/dpkg/apt(URL) |
|
||||
| `scripts/deploy/list.py` | 扫描全部可部署服务 |
|
||||
|
||||
## references/
|
||||
|
||||
Executable
+201
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
远程分发并安装 deb 包。
|
||||
|
||||
目标 <target> 两种写法:
|
||||
1. 仓库内目录(含其祖先的 _config.yaml,解析 node/port/identity_file),
|
||||
例如 hosts/web1 或 hosts/web1/myapp —— 与 sync.py/remote.py 的目标语义一致。
|
||||
2. 裸 SSH 别名或 user@host(须在 ~/.ssh/config 中),可配合 --port/--identity。
|
||||
|
||||
用法:
|
||||
python deb.py <target> apt <deb-URL> # 远程 wget 下载 + apt 安装
|
||||
python deb.py <target> push <本地.deb> [...] # scp 上传 + dpkg 安装(失败自动 apt -f 修依赖)
|
||||
python deb.py <target> scp <本地.deb> [...] # 仅上传到远程暂存目录
|
||||
python deb.py <target> dpkg # 安装该节点暂存目录里已有的包
|
||||
|
||||
选项:
|
||||
--yes apt/dpkg 免交互(-y / --assume-yes)
|
||||
--staging-dir 远程暂存目录;默认目录目标用 {base_path}/.debs,裸主机用 /tmp/deployer-debs
|
||||
|
||||
示例:
|
||||
python deb.py hosts/web1/apt-proxy push ./gpt-load_1.0_amd64.deb
|
||||
python deb.py deploy@web1 apt https://example.com/pkg.deb --yes
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from lib import (
|
||||
project_root,
|
||||
run_remote,
|
||||
scp_base_args,
|
||||
service_info,
|
||||
ssh_config_hosts,
|
||||
)
|
||||
|
||||
BARE_HOST_STAGING = "/tmp/deployer-debs"
|
||||
|
||||
|
||||
def apt_prefix() -> str:
|
||||
"""root 直接跑 apt/dpkg,否则加 sudo -n(免密 sudo)。"""
|
||||
return 'APT="apt-get"; [ "$(id -u)" = 0 ] || APT="sudo -n apt-get"'
|
||||
|
||||
|
||||
class Target:
|
||||
"""安装目标:仓库内服务目录继承配置,或 ~/.ssh/config 中的裸主机别名。"""
|
||||
|
||||
def __init__(self, spec: str, port: int | None, identity: str | None, staging: str) -> None:
|
||||
self.spec = spec
|
||||
abs_dir = (project_root() / spec).resolve()
|
||||
if abs_dir.is_dir():
|
||||
self.info = service_info(spec)
|
||||
# 暂存目录只跟节点走(base_path/.debs),同一节点共享一个暂存区,
|
||||
# scp 与 dpkg 分步操作时才不会因目录不同而对不上
|
||||
default_staging = f"{self.info['base_path']}/.debs"
|
||||
else:
|
||||
host = spec.split("@", 1)[-1]
|
||||
if host not in ssh_config_hosts():
|
||||
print(
|
||||
f"错误: {spec!r} 既不是仓库内的服务/主机目录,"
|
||||
f"也不是 ~/.ssh/config 中的 Host 别名"
|
||||
)
|
||||
sys.exit(1)
|
||||
self.info = {
|
||||
"node": spec,
|
||||
"port": port,
|
||||
"identity_file": identity,
|
||||
}
|
||||
default_staging = BARE_HOST_STAGING
|
||||
self.staging_dir = staging or default_staging
|
||||
|
||||
@property
|
||||
def node(self) -> str:
|
||||
return self.info["node"]
|
||||
|
||||
def ssh(self, script: str) -> int:
|
||||
return run_remote(self.info, script)
|
||||
|
||||
def upload(self, local_paths: list[str]) -> list[str]:
|
||||
"""scp 上传文件到暂存目录,返回远程侧文件名列表。"""
|
||||
names = []
|
||||
for p in local_paths:
|
||||
name = Path(p).name
|
||||
if not name.endswith(".deb"):
|
||||
print(f"错误: 不是 .deb 文件: {p}")
|
||||
sys.exit(1)
|
||||
names.append(name)
|
||||
|
||||
remote = f"{self.node}:{shlex.quote(self.staging_dir)}/"
|
||||
cmd = [*scp_base_args(self.info), *local_paths, remote]
|
||||
print(f"正在上传到 {self.node}:{self.staging_dir}")
|
||||
print(f"$ {' '.join(cmd)}")
|
||||
print("-" * 60)
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"错误: scp 上传失败,退出码: {e.returncode}")
|
||||
sys.exit(e.returncode or 1)
|
||||
except FileNotFoundError:
|
||||
print("错误: scp 命令未找到")
|
||||
sys.exit(1)
|
||||
return names
|
||||
|
||||
def install_staged(self, names: list[str], *, assume_yes: bool) -> int:
|
||||
"""在远程对暂存目录里的指定包执行 apt 安装(失败自动 -f 修依赖)。"""
|
||||
quoted = " ".join(shlex.quote(f"{self.staging_dir}/{n}") for n in names)
|
||||
yes = " -y" if assume_yes else ""
|
||||
script = "\n".join(
|
||||
[
|
||||
"set -e",
|
||||
f"mkdir -p {shlex.quote(self.staging_dir)}",
|
||||
apt_prefix(),
|
||||
f"$APT install{yes} {quoted} || $APT install{yes} -f",
|
||||
]
|
||||
)
|
||||
return self.ssh(script)
|
||||
|
||||
|
||||
def _quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
def action_apt(target: Target, deb: str, *, assume_yes: bool) -> int:
|
||||
"""远程从 URL 安装:wget 到暂存目录后 apt-get install。"""
|
||||
if not deb.startswith(("http://", "https://", "ftp://")):
|
||||
print(f"提示: {deb!r} 不是 URL,请改用 push 上传本地包")
|
||||
return 2
|
||||
filename = deb.rstrip("/").rsplit("/", 1)[-1]
|
||||
staged = f"{target.staging_dir}/{filename}"
|
||||
yes = " -y" if assume_yes else ""
|
||||
script = "\n".join(
|
||||
[
|
||||
"set -e",
|
||||
f"mkdir -p {shlex.quote(target.staging_dir)}",
|
||||
apt_prefix(),
|
||||
f"wget -qO {_quote(staged)}.part {_quote(deb)}",
|
||||
f"mv {_quote(staged)}.part {_quote(staged)}",
|
||||
f"$APT install{yes} {_quote(staged)}",
|
||||
f"rm -f {_quote(staged)}",
|
||||
]
|
||||
)
|
||||
return target.ssh(script)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="向远程节点分发并安装 deb 包")
|
||||
parser.add_argument("target", help="仓库内目录(hosts/web1[/svc])或 SSH 别名(user@host)")
|
||||
parser.add_argument("action", choices=["apt", "push", "scp", "dpkg"])
|
||||
parser.add_argument("packages", nargs="*", help="apt 的 .deb URL;push/scp 的本地 .deb 文件")
|
||||
parser.add_argument("--yes", action="store_true", help="apt/dpkg 免交互(-y / --assume-yes)")
|
||||
parser.add_argument("--staging-dir", default="", help="远程暂存目录,默认 {base_path}/.debs")
|
||||
parser.add_argument("--port", type=int, help="SSH 端口(仅裸主机目标)")
|
||||
parser.add_argument("--identity", help="SSH 私钥路径(仅裸主机目标)")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.chdir(project_root())
|
||||
target = Target(args.target, args.port, args.identity, args.staging_dir)
|
||||
|
||||
if args.action == "apt":
|
||||
if len(args.packages) != 1:
|
||||
parser.error("apt 需要恰好一个 .deb URL")
|
||||
return action_apt(target, args.packages[0], assume_yes=args.yes)
|
||||
|
||||
if args.action in ("push", "scp"):
|
||||
if not args.packages:
|
||||
parser.error(f"{args.action} 至少需要一个本地 .deb 文件")
|
||||
missing = [p for p in args.packages if not Path(p).is_file()]
|
||||
if missing:
|
||||
print(f"错误: 本地文件不存在: {', '.join(missing)}")
|
||||
return 1
|
||||
names = target.upload(args.packages)
|
||||
if args.action == "scp":
|
||||
print(f"已上传: {', '.join(names)} -> {target.node}:{target.staging_dir}")
|
||||
print("后续安装: python deb.py {} dpkg".format(args.target))
|
||||
return 0
|
||||
rc = target.install_staged(names, assume_yes=args.yes)
|
||||
if rc == 0:
|
||||
staged = " ".join(shlex.quote(f"{target.staging_dir}/{n}") for n in names)
|
||||
target.ssh(f"rm -f {staged}")
|
||||
return rc
|
||||
|
||||
# dpkg:安装暂存目录中已上传的全部包
|
||||
yes = " -y" if args.yes else ""
|
||||
glob = shlex.quote(f"{target.staging_dir}") + "/*.deb"
|
||||
script = "\n".join(
|
||||
[
|
||||
apt_prefix(),
|
||||
f"ls -1 {glob} >/dev/null 2>&1 || {{ echo '错误: {target.node}:{target.staging_dir} 没有暂存的 .deb'; exit 1; }}",
|
||||
f"$APT install{yes} {glob} || $APT install{yes} -f",
|
||||
]
|
||||
)
|
||||
return target.ssh(script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -221,6 +221,35 @@ def rsync_ssh_args(info: dict) -> list[str]:
|
||||
return ["-e", shlex.join(base)]
|
||||
|
||||
|
||||
def scp_base_args(info: dict) -> list[str]:
|
||||
"""scp 可执行文件及 -P / -i 选项(注意 scp 的端口是大写 P)。"""
|
||||
args = ["scp"]
|
||||
port = info.get("port")
|
||||
if port is not None:
|
||||
args.extend(["-P", str(port)])
|
||||
identity_file = info.get("identity_file")
|
||||
if identity_file:
|
||||
args.extend(["-i", identity_file])
|
||||
return args
|
||||
|
||||
|
||||
def run_remote(info: dict, remote_command: str, *, remote_cwd: str | None = None) -> int:
|
||||
"""在远程节点执行 shell 命令并返回退出码;remote_cwd 提供时先切换目录。"""
|
||||
shell = f"cd {remote_cwd} && {remote_command}" if remote_cwd else remote_command
|
||||
endpoint = info["node"]
|
||||
if info.get("port") is not None:
|
||||
endpoint += f":{info['port']}"
|
||||
location = f"{endpoint} {remote_cwd}" if remote_cwd else endpoint
|
||||
print(f"远程执行: {location}")
|
||||
print(f"$ {remote_command}")
|
||||
print("-" * 60)
|
||||
try:
|
||||
return subprocess.run(ssh_cmd(info, shell), check=False).returncode
|
||||
except FileNotFoundError:
|
||||
print("错误: ssh 命令未找到")
|
||||
return 1
|
||||
|
||||
|
||||
def node_from_parent_dir(service_dir: str) -> str | None:
|
||||
"""父目录名若是 SSH Host 别名,则作为 node。"""
|
||||
parent = Path(service_dir.rstrip("/")).parent.name
|
||||
|
||||
@@ -21,7 +21,7 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from lib import project_root, service_info, ssh_cmd
|
||||
from lib import project_root, run_remote, service_info
|
||||
|
||||
REMOTE_COMMANDS = {
|
||||
"up": "docker compose up -d",
|
||||
@@ -33,23 +33,6 @@ REMOTE_COMMANDS = {
|
||||
}
|
||||
|
||||
|
||||
def run_ssh(info: dict, compose_command: str) -> int:
|
||||
remote_dir = info["remote_dir"]
|
||||
remote_shell = f"cd {remote_dir} && {compose_command}"
|
||||
cmd = ssh_cmd(info, remote_shell)
|
||||
endpoint = info["node"]
|
||||
if info.get("port") is not None:
|
||||
endpoint = f"{endpoint}:{info['port']}"
|
||||
print(f"远程执行: {endpoint}:{remote_dir}")
|
||||
print(f"$ {compose_command}")
|
||||
print("-" * 60)
|
||||
try:
|
||||
return subprocess.run(cmd, check=False).returncode
|
||||
except FileNotFoundError:
|
||||
print("错误: ssh 命令未找到")
|
||||
return 1
|
||||
|
||||
|
||||
def run_for_service(service_dir: str, action: str) -> int:
|
||||
info = service_info(service_dir)
|
||||
compose_command = REMOTE_COMMANDS[action]
|
||||
@@ -57,8 +40,7 @@ def run_for_service(service_dir: str, action: str) -> int:
|
||||
if info.get("port") is not None:
|
||||
endpoint = f"{endpoint}:{info['port']}"
|
||||
print(f"服务: {service_dir} -> {endpoint}:{info['remote_dir']}")
|
||||
return run_ssh(info, compose_command)
|
||||
|
||||
return run_remote(info, compose_command, remote_cwd=info["remote_dir"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
Reference in New Issue
Block a user