add project tooling and test suite for many-rsync

- Add pyproject.toml: hatchling build, many-rsync entrypoint, ruff/mypy/pytest config
with 60% coverage floor
- Add uv.lock for reproducible dev installs
- Add .pre-commit-config.yaml: ruff (with --fix) + mypy hooks
- Add test_main.py: unit tests for _build_rsync_cmd, _load_raw, and load_config
covering happy paths and FATAL exit cases
- Add explanation.md: architecture overview with flowchart
- main.py: refactor into typed, testable functions (_load_raw, _build_rsync_cmd
extracted); add RsyncParameters/Config TypedDicts; add rsync_parameters config support
(rsync_path, exclude_from); harden validation (n clamped, log_level validated)
- README.md: update install instructions and document all config fields including
rsync_parameters
This commit is contained in:
2026-03-31 22:05:08 +03:00
parent 4f6ef3d179
commit a168b4cbea
7 changed files with 796 additions and 20 deletions

View File

@@ -8,14 +8,17 @@ parallel rsync runner. reads config from TOML (preferred) or JSON.
local: foo bar --|many-rsync|--> remote/foo remote/bar
"""
from __future__ import annotations
import json
import logging
import subprocess
import sys
import tomllib
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, TypedDict, Literal
from typing import Any, Literal, TypedDict
class RsyncParameters(TypedDict, total=False):
@@ -31,11 +34,6 @@ class Config(TypedDict):
rsync_parameters: RsyncParameters
try:
import tomllib # 3.11+
except ModuleNotFoundError:
tomllib = None
HOME = Path.home()
LOG_DIR = HOME / ".rsync-logs"
RSYNC_BASE = ("rsync", "-avh", "--progress", "--delete", "--stats")
@@ -45,14 +43,14 @@ logger = logging.getLogger("sync")
# ── config ────────────────────────────────────────────────────────────────────
def _load_raw(path: Path) -> dict[str, Any]:
text = path.read_text()
if path.suffix == ".toml":
if tomllib is None:
sys.exit("FATAL: Python < 3.11 has no tomllib; install tomli or use JSON config")
return tomllib.loads(text)
if path.suffix == ".json":
return json.loads(text)
result: dict[str, Any] = json.loads(text)
return result
sys.exit(f"FATAL: unsupported config format: {path.suffix}")
@@ -62,10 +60,12 @@ def load_config(path: Path) -> Config:
if not folders:
sys.exit("FATAL: local_folders must be a non-empty list")
folders = [Path(f).expanduser() for f in folders]
for f in folders:
if not f.is_dir():
sys.exit(f"FATAL: local_folders entries must exist and be folders, got: {f!r}")
sys.exit(
f"FATAL: local_folders entries must exist and be folders, got: {f!r}"
)
remote = raw.get("remote_folder")
if not remote:
sys.exit("FATAL: remote_folder is required")
@@ -88,13 +88,17 @@ def load_config(path: Path) -> Config:
rsync_params["exclude_from"] = str(exclude_from)
return {
"local_folders": folders, "remote_folder": remote,
"n": n, "log_level": level, "rsync_parameters": rsync_params,
"local_folders": folders,
"remote_folder": remote,
"n": n,
"log_level": level,
"rsync_parameters": rsync_params,
}
# ── sync ──────────────────────────────────────────────────────────────────────
def _build_rsync_cmd(params: RsyncParameters) -> tuple[str, ...]:
"""Extend RSYNC_BASE with optional flags from config."""
extra: list[str] = []
@@ -105,7 +109,9 @@ def _build_rsync_cmd(params: RsyncParameters) -> tuple[str, ...]:
return (*RSYNC_BASE, *extra)
def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) -> tuple[Path, int]:
def sync_folder(
folder: Path, remote: str, ts: str, params: RsyncParameters
) -> tuple[Path, int]:
"""Run rsync for a single folder. Returns (folder, returncode)."""
log_file = LOG_DIR / f"{folder.name}-{ts}.log"
cmd: list[str | Path] = [*_build_rsync_cmd(params), f"{folder}", remote]
@@ -116,7 +122,7 @@ def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) ->
proc = subprocess.run(
cmd,
cwd=HOME,
stdin=subprocess.DEVNULL, # prevent parallel processes from fighting over terminal input
stdin=subprocess.DEVNULL, # no terminal fights
stdout=fh,
stderr=subprocess.STDOUT, # interleave; nothing goes silent
text=True,
@@ -132,10 +138,11 @@ def sync_folder(folder: Path, remote: str, ts: str, params: RsyncParameters) ->
# ── main ──────────────────────────────────────────────────────────────────────
def main() -> None:
cfg_path = Path(sys.argv[1]) if len(sys.argv) > 1 else HOME / "sync.toml"
cfg = load_config(cfg_path)
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
LOG_DIR.mkdir(parents=True, exist_ok=True)
logging.basicConfig(
@@ -143,9 +150,9 @@ def main() -> None:
format="%(asctime)s %(levelname)-5s %(message)s",
datefmt="%H:%M:%S",
handlers=[
logging.StreamHandler(),
logging.FileHandler(LOG_DIR / f"rsync-{ts}.log"),
]
logging.StreamHandler(),
logging.FileHandler(LOG_DIR / f"rsync-{ts}.log"),
],
)
folders, remote, n = cfg["local_folders"], cfg["remote_folder"], cfg["n"]