add project tooling and test suite for many-rsync
- Add pyproject.toml: hatchling build, many-rsync entrypoint, ruff/mypy/pytest config with 60% coverage floor - Add uv.lock for reproducible dev installs - Add .pre-commit-config.yaml: ruff (with --fix) + mypy hooks - Add test_main.py: unit tests for _build_rsync_cmd, _load_raw, and load_config covering happy paths and FATAL exit cases - Add explanation.md: architecture overview with flowchart - main.py: refactor into typed, testable functions (_load_raw, _build_rsync_cmd extracted); add RsyncParameters/Config TypedDicts; add rsync_parameters config support (rsync_path, exclude_from); harden validation (n clamped, log_level validated) - README.md: update install instructions and document all config fields including rsync_parameters
This commit is contained in:
@@ -8,14 +8,17 @@ parallel rsync runner. reads config from TOML (preferred) or JSON.
|
||||
local: foo bar --|many-rsync|--> remote/foo remote/bar
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, TypedDict, Literal
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
|
||||
class RsyncParameters(TypedDict, total=False):
|
||||
@@ -31,11 +34,6 @@ class Config(TypedDict):
|
||||
rsync_parameters: RsyncParameters
|
||||
|
||||
|
||||
try:
|
||||
import tomllib # 3.11+
|
||||
except ModuleNotFoundError:
|
||||
tomllib = None
|
||||
|
||||
HOME = Path.home()
|
||||
LOG_DIR = HOME / ".rsync-logs"
|
||||
RSYNC_BASE = ("rsync", "-avh", "--progress", "--delete", "--stats")
|
||||
@@ -45,14 +43,14 @@ logger = logging.getLogger("sync")
|
||||
|
||||
# ── config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _load_raw(path: Path) -> dict[str, Any]:
|
||||
text = path.read_text()
|
||||
if path.suffix == ".toml":
|
||||
if tomllib is None:
|
||||
sys.exit("FATAL: Python < 3.11 has no tomllib; install tomli or use JSON config")
|
||||
return tomllib.loads(text)
|
||||
if path.suffix == ".json":
|
||||
return json.loads(text)
|
||||
result: dict[str, Any] = json.loads(text)
|
||||
return result
|
||||
sys.exit(f"FATAL: unsupported config format: {path.suffix}")
|
||||
|
||||
|
||||
@@ -62,10 +60,12 @@ def load_config(path: Path) -> Config:
|
||||
if not folders:
|
||||
sys.exit("FATAL: local_folders must be a non-empty list")
|
||||
folders = [Path(f).expanduser() for f in folders]
|
||||
|
||||
|
||||
for f in folders:
|
||||
if not f.is_dir():
|
||||
sys.exit(f"FATAL: local_folders entries must exist and be folders, got: {f!r}")
|
||||
sys.exit(
|
||||
f"FATAL: local_folders entries must exist and be folders, got: {f!r}"
|
||||
)
|
||||
remote = raw.get("remote_folder")
|
||||
if not remote:
|
||||
sys.exit("FATAL: remote_folder is required")
|
||||
@@ -88,13 +88,17 @@ def load_config(path: Path) -> Config:
|
||||
rsync_params["exclude_from"] = str(exclude_from)
|
||||
|
||||
return {
|
||||
"local_folders": folders, "remote_folder": remote,
|
||||
"n": n, "log_level": level, "rsync_parameters": rsync_params,
|
||||
"local_folders": folders,
|
||||
"remote_folder": remote,
|
||||
"n": n,
|
||||
"log_level": level,
|
||||
"rsync_parameters": rsync_params,
|
||||
}
|
||||
|
||||
|
||||
# ── sync ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _build_rsync_cmd(params: RsyncParameters) -> tuple[str, ...]:
|
||||
"""Extend RSYNC_BASE with optional flags from config."""
|
||||
extra: list[str] = []
|
||||
@@ -105,7 +109,9 @@ def _build_rsync_cmd(params: RsyncParameters) -> tuple[str, ...]:
|
||||
return (*RSYNC_BASE, *extra)
|
||||
|
||||
|
||||
def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) -> tuple[Path, int]:
|
||||
def sync_folder(
|
||||
folder: Path, remote: str, ts: str, params: RsyncParameters
|
||||
) -> tuple[Path, int]:
|
||||
"""Run rsync for a single folder. Returns (folder, returncode)."""
|
||||
log_file = LOG_DIR / f"{folder.name}-{ts}.log"
|
||||
cmd: list[str | Path] = [*_build_rsync_cmd(params), f"{folder}", remote]
|
||||
@@ -116,7 +122,7 @@ def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) ->
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=HOME,
|
||||
stdin=subprocess.DEVNULL, # prevent parallel processes from fighting over terminal input
|
||||
stdin=subprocess.DEVNULL, # no terminal fights
|
||||
stdout=fh,
|
||||
stderr=subprocess.STDOUT, # interleave; nothing goes silent
|
||||
text=True,
|
||||
@@ -132,10 +138,11 @@ def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) ->
|
||||
|
||||
# ── main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg_path = Path(sys.argv[1]) if len(sys.argv) > 1 else HOME / "sync.toml"
|
||||
cfg = load_config(cfg_path)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
logging.basicConfig(
|
||||
@@ -143,9 +150,9 @@ def main() -> None:
|
||||
format="%(asctime)s %(levelname)-5s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler(LOG_DIR / f"rsync-{ts}.log"),
|
||||
]
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler(LOG_DIR / f"rsync-{ts}.log"),
|
||||
],
|
||||
)
|
||||
|
||||
folders, remote, n = cfg["local_folders"], cfg["remote_folder"], cfg["n"]
|
||||
|
||||
Reference in New Issue
Block a user