erplibre/script/git/git_local_server.py

#!/usr/bin/env python3
# © 2025-2026 TechnoLibre (http://www.technolibre.ca)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)

import argparse
import asyncio
import logging
import os
import subprocess
import sys
import xml.etree.ElementTree as ET

new_path = os.path.normpath(
    os.path.join(os.path.dirname(__file__), "..", "..")
)
sys.path.append(new_path)

from script.execute.execute import Execute

_logger = logging.getLogger(__name__)

DEFAULT_ERPLIBRE_PATH = os.path.normpath(
    os.path.join(os.path.dirname(__file__), "..", "..")
)
DEFAULT_GIT_PATH = os.path.join(os.path.expanduser("~"), ".git-server")
PRODUCTION_GIT_PATH = "/srv/git"
DEFAULT_MANIFEST = ".repo/local_manifests/erplibre_manifest.xml"
DEFAULT_REMOTE_NAME = "local"
DEFAULT_PORT = 9418
DEFAULT_JOBS = 8
ERPLIBRE_REPO_NAME = "erplibre/erplibre"
ERPLIBRE_REPO_URL = "https://github.com/erplibre"


async def _run_git(*args, cwd=None, timeout=None):
    """Run a git command asynchronously.

    Returns (stdout, stderr, returncode).
    Raises asyncio.TimeoutError on timeout.
    """
    process = await asyncio.create_subprocess_exec(
        *args,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        cwd=cwd,
    )
    try:
        stdout, stderr = await asyncio.wait_for(
            process.communicate(), timeout=timeout
        )
        return (
            stdout.decode(),
            stderr.decode(),
            process.returncode,
        )
    except asyncio.TimeoutError:
        process.kill()
        await process.communicate()
        raise


def get_config():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""\
Deploy a local git server using git daemon and configure
all repos from the ERPLibre manifest.

Actions (in order):
  1. init   - Create bare repos in the git server path
  2. remote - Add 'local' remote to each repo
  3. push   - Push all branches to the local remote
  4. serve  - Start git daemon

By default, all actions are executed sequentially.
The default path is ~/.git-server (user-level, no root needed).
Use --production-ready for /srv/git (requires root).
""",
    )
    parser.add_argument(
        "-p",
        "--path",
        default=None,
        help=(
            "Path for git server bare repos" f" (default: {DEFAULT_GIT_PATH})"
        ),
    )
    parser.add_argument(
        "--production-ready",
        action="store_true",
        help=(
            f"Use {PRODUCTION_GIT_PATH} as git server path"
            " (requires root privileges)"
        ),
    )
    parser.add_argument(
        "-m",
        "--manifest",
        default=DEFAULT_MANIFEST,
        help="Manifest XML file" f" (default: {DEFAULT_MANIFEST})",
    )
    parser.add_argument(
        "--remote-name",
        default=DEFAULT_REMOTE_NAME,
        help="Remote name to add" f" (default: {DEFAULT_REMOTE_NAME})",
    )
    parser.add_argument(
        "--port",
        type=int,
        default=DEFAULT_PORT,
        help=f"Git daemon port (default: {DEFAULT_PORT})",
    )
    parser.add_argument(
        "--action",
        choices=["init", "remote", "push", "serve", "all"],
        default="all",
        help="Action to perform (default: all)",
    )
    parser.add_argument(
        "--erplibre-root",
        default=None,
        help="ERPLibre root directory (default: auto-detect)",
    )
    parser.add_argument(
        "--push-all-branches",
        action="store_true",
        help="Push all local branches, not just the current one",
    )
    parser.add_argument(
        "--remote-url-type",
        choices=["file", "daemon"],
        default="file",
        help=(
            "Remote URL type: 'file' uses local path"
            " (push works without daemon), 'daemon'"
            " uses git:// protocol (default: file)"
        ),
    )
    parser.add_argument(
        "--unshallow",
        action="store_true",
        help=(
            "Fetch full history for shallow repos before"
            " push (slow for large repos like odoo)."
            " Default: push shallow for performance"
        ),
    )
    parser.add_argument(
        "-j",
        "--jobs",
        type=int,
        default=DEFAULT_JOBS,
        help=(
            "Parallel jobs for init/remote/push"
            f" (default: {DEFAULT_JOBS})"
        ),
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Enable verbose output",
    )
    args = parser.parse_args()

    # Resolve git server path
    if args.path and args.production_ready:
        parser.error("--path and --production-ready are mutually exclusive")
    if args.production_ready:
        args.path = PRODUCTION_GIT_PATH
    elif args.path is None:
        args.path = DEFAULT_GIT_PATH

    return args


def check_path_permissions(path):
    """Check if we have write permissions on the target
    path. Try the path itself first, then its closest
    existing parent directory."""
    check = path
    while check and not os.path.exists(check):
        check = os.path.dirname(check)
    if not check:
        check = "/"
    if os.access(check, os.W_OK):
        return True
    return False


def require_permissions_or_exit(path):
    """Exit with instructions if we cannot write to path."""
    print(f"Error: No write permission on {path}")
    print(
        "Please run this script with appropriate"
        " privileges:"
        f"\n  sudo {sys.executable}"
        f" {' '.join(sys.argv)}"
    )
    sys.exit(1)


def parse_manifest(manifest_path):
    """Parse the manifest XML and return list of projects."""
    tree = ET.parse(manifest_path)
    root = tree.getroot()

    remotes = {}
    for remote in root.findall("remote"):
        remotes[remote.get("name")] = remote.get("fetch")

    projects = []
    for project in root.findall("project"):
        name = project.get("name")
        path = project.get("path")
        remote = project.get("remote")
        revision = project.get("revision")
        projects.append(
            {
                "name": name,
                "path": path,
                "remote": remote,
                "revision": revision,
                "fetch_url": remotes.get(remote, ""),
            }
        )

    return projects


def get_erplibre_root_project(erplibre_root):
    """Build a project entry for the ERPLibre root repo.

    The root repo is not in the manifest (it is managed
    by git directly, not Google Repo) but needs to be
    included in the local git server.
    """
    result = subprocess.run(
        [
            "git",
            "-C",
            erplibre_root,
            "rev-parse",
            "--abbrev-ref",
            "HEAD",
        ],
        capture_output=True,
        text=True,
    )
    revision = (
        result.stdout.strip()
        if result.returncode == 0
        else "master"
    )

    return {
        "name": ERPLIBRE_REPO_NAME,
        "path": ".",
        "remote": "origin",
        "revision": revision,
        "fetch_url": ERPLIBRE_REPO_URL,
    }


# --- Async workers for init ---


async def _init_single_bare_repo(git_path, project, semaphore):
    """Create a single bare repo (async worker)."""
    async with semaphore:
        repo_name = project["name"]
        if not repo_name.endswith(".git"):
            repo_name += ".git"
        bare_path = os.path.join(git_path, repo_name)

        if os.path.exists(bare_path):
            _logger.info(f"  Exists: {bare_path}")
            return "skipped"

        _logger.info(f"  Creating: {bare_path}")
        _, err, rc = await _run_git(
            "git", "init", "--bare", bare_path
        )
        if rc != 0:
            _logger.warning(
                f"  Init failed: {bare_path}: {err.strip()}"
            )
            return "error"

        # Enable git daemon export
        export_file = os.path.join(
            bare_path, "git-daemon-export-ok"
        )
        open(export_file, "w").close()

        # Allow pushing from shallow clones
        await _run_git(
            "git",
            "-C",
            bare_path,
            "config",
            "receive.shallowUpdate",
            "true",
        )

        return "created"


async def init_bare_repos(git_path, projects, jobs):
    """Create bare repos for all projects (async)."""
    os.makedirs(git_path, exist_ok=True)
    semaphore = asyncio.Semaphore(jobs)

    results = await asyncio.gather(
        *[
            _init_single_bare_repo(git_path, p, semaphore)
            for p in projects
        ]
    )

    created = results.count("created")
    skipped = results.count("skipped")
    errors = results.count("error")
    print(
        f"Bare repos: {created} created,"
        f" {skipped} skipped (already exist),"
        f" {errors} errors"
    )


# --- Async workers for remote ---


def build_remote_url(git_path, repo_name, port, remote_url_type):
    """Build the remote URL based on the type."""
    if remote_url_type == "daemon":
        if port != DEFAULT_PORT:
            return f"git://localhost:{port}/{repo_name}"
        return f"git://localhost/{repo_name}"
    # Default: file path
    return os.path.join(git_path, repo_name)


async def _add_single_remote(
    erplibre_root,
    git_path,
    project,
    remote_name,
    port,
    remote_url_type,
    semaphore,
):
    """Add or update remote for a single repo (async)."""
    async with semaphore:
        repo_path = os.path.join(
            erplibre_root, project["path"]
        )
        if not os.path.isdir(repo_path):
            _logger.warning(f"  Not found: {repo_path}")
            return "error"

        repo_name = project["name"]
        if not repo_name.endswith(".git"):
            repo_name += ".git"

        remote_url = build_remote_url(
            git_path, repo_name, port, remote_url_type
        )

        # Check if remote already exists
        stdout, _, _ = await _run_git(
            "git", "-C", repo_path, "remote"
        )
        existing_remotes = stdout.strip().split("\n")

        if remote_name in existing_remotes:
            _, err, rc = await _run_git(
                "git",
                "-C",
                repo_path,
                "remote",
                "set-url",
                remote_name,
                remote_url,
            )
            if rc != 0:
                _logger.warning(
                    f"  set-url failed for"
                    f" {project['path']}: {err.strip()}"
                )
                return "error"
            _logger.info(f"  Updated: {project['path']}")
            return "updated"
        else:
            _, err, rc = await _run_git(
                "git",
                "-C",
                repo_path,
                "remote",
                "add",
                remote_name,
                remote_url,
            )
            if rc != 0:
                _logger.warning(
                    f"  add failed for"
                    f" {project['path']}: {err.strip()}"
                )
                return "error"
            _logger.info(f"  Added: {project['path']}")
            return "added"


async def add_remotes(
    erplibre_root,
    git_path,
    projects,
    remote_name,
    port,
    remote_url_type,
    jobs,
):
    """Add local remote to each repo (async).

    remote_url_type='file': uses local path, push works
      without daemon running.
    remote_url_type='daemon': uses git://localhost URL,
      requires daemon for push.
    """
    semaphore = asyncio.Semaphore(jobs)

    results = await asyncio.gather(
        *[
            _add_single_remote(
                erplibre_root,
                git_path,
                p,
                remote_name,
                port,
                remote_url_type,
                semaphore,
            )
            for p in projects
        ]
    )

    added = results.count("added")
    updated = results.count("updated")
    errors = results.count("error")
    print(
        f"Remotes: {added} added, {updated} updated,"
        f" {errors} errors"
    )


# --- Async workers for push ---


async def _is_detached_head(repo_path):
    """Check if a repo is in detached HEAD state."""
    _, _, rc = await _run_git(
        "git", "-C", repo_path, "symbolic-ref", "HEAD"
    )
    return rc != 0


async def _checkout_manifest_branch(repo_path, revision):
    """Checkout the branch from the manifest revision.

    When Google Repo syncs, repos end up in detached HEAD
    on the exact commit. We create/checkout a local branch
    matching the manifest revision so git push works.
    """
    branch = (
        revision.split("/")[-1] if "/" in revision else revision
    )

    # Check if local branch already exists
    _, _, rc = await _run_git(
        "git",
        "-C",
        repo_path,
        "show-ref",
        "--verify",
        f"refs/heads/{branch}",
    )
    if rc == 0:
        await _run_git(
            "git", "-C", repo_path, "checkout", branch
        )
    else:
        await _run_git(
            "git",
            "-C",
            repo_path,
            "checkout",
            "-b",
            branch,
        )
    return branch


async def _update_bare_head(git_path, project):
    """Update the HEAD of the bare repo to point to the
    manifest branch, so git clone checks out the right
    branch by default."""
    repo_name = project["name"]
    if not repo_name.endswith(".git"):
        repo_name += ".git"
    bare_path = os.path.join(git_path, repo_name)
    if not os.path.isdir(bare_path):
        return

    revision = project.get("revision", "")
    if not revision:
        return
    branch = (
        revision.split("/")[-1] if "/" in revision else revision
    )
    await _run_git(
        "git",
        "-C",
        bare_path,
        "symbolic-ref",
        "HEAD",
        f"refs/heads/{branch}",
    )


async def _try_unshallow(repo_path, project, remote_name):
    """Try to unshallow a repo by fetching full history."""
    shallow_file = os.path.join(
        repo_path, ".git", "shallow"
    )
    _logger.info(
        f"  Unshallowing {project['path']}..."
    )
    stdout, _, _ = await _run_git(
        "git", "-C", repo_path, "remote"
    )
    remotes = [
        r
        for r in stdout.strip().split("\n")
        if r and r != remote_name
    ]
    manifest_remote = project.get("remote", "")
    if manifest_remote in remotes:
        remotes.remove(manifest_remote)
        remotes.insert(0, manifest_remote)

    for try_remote in remotes:
        try:
            await _run_git(
                "git",
                "-C",
                repo_path,
                "fetch",
                "--unshallow",
                try_remote,
                timeout=300,
            )
        except asyncio.TimeoutError:
            continue
        if not os.path.exists(shallow_file):
            _logger.info(
                f"  Unshallowed via {try_remote}"
            )
            return
    if os.path.exists(shallow_file):
        _logger.warning(
            f"  Unshallow failed for"
            f" {project['path']},"
            " falling back to shallow push"
        )


async def _push_single_repo(
    erplibre_root,
    git_path,
    project,
    remote_name,
    push_all_branches,
    unshallow,
    semaphore,
):
    """Push a single repo to local remote (async)."""
    async with semaphore:
        repo_path = os.path.join(
            erplibre_root, project["path"]
        )
        if not os.path.isdir(repo_path):
            _logger.warning(f"  Not found: {repo_path}")
            return "error", False

        # Handle shallow repos
        shallow_file = os.path.join(
            repo_path, ".git", "shallow"
        )
        if os.path.exists(shallow_file):
            if unshallow:
                await _try_unshallow(
                    repo_path, project, remote_name
                )
            else:
                # Enable shallow push on bare repo
                repo_name = project["name"]
                if not repo_name.endswith(".git"):
                    repo_name += ".git"
                bare_path = os.path.join(
                    git_path, repo_name
                )
                if os.path.isdir(bare_path):
                    await _run_git(
                        "git",
                        "-C",
                        bare_path,
                        "config",
                        "receive.shallowUpdate",
                        "true",
                    )
                _logger.info(
                    f"  Shallow push for"
                    f" {project['path']}"
                )

        # Handle detached HEAD: checkout manifest branch
        did_checkout = False
        if await _is_detached_head(repo_path):
            revision = project.get("revision", "")
            if revision:
                branch = await _checkout_manifest_branch(
                    repo_path, revision
                )
                _logger.info(
                    f"  Checkout {branch} for"
                    f" {project['path']}"
                    " (was detached HEAD)"
                )
                did_checkout = True
            else:
                _logger.warning(
                    f"  Detached HEAD with no revision"
                    f" for {project['path']}, skipping"
                )
                return "error", False

        # Push
        try:
            if push_all_branches:
                cmd = [
                    "git",
                    "-C",
                    repo_path,
                    "push",
                    remote_name,
                    "--all",
                ]
            else:
                cmd = [
                    "git",
                    "-C",
                    repo_path,
                    "push",
                    remote_name,
                ]
            _, err, rc = await _run_git(
                *cmd, timeout=120
            )
            if rc != 0:
                _logger.warning(
                    f"  Push failed for"
                    f" {project['path']}:"
                    f" {err.strip()}"
                )
                return "error", did_checkout
            else:
                await _update_bare_head(git_path, project)
                _logger.info(
                    f"  Pushed: {project['path']}"
                )
                return "pushed", did_checkout
        except asyncio.TimeoutError:
            _logger.warning(
                f"  Timeout pushing {project['path']}"
            )
            return "error", did_checkout


async def push_to_local(
    erplibre_root,
    git_path,
    projects,
    remote_name,
    push_all_branches,
    unshallow,
    jobs,
):
    """Push to local remote for each repo (async)."""
    semaphore = asyncio.Semaphore(jobs)

    results = await asyncio.gather(
        *[
            _push_single_repo(
                erplibre_root,
                git_path,
                p,
                remote_name,
                push_all_branches,
                unshallow,
                semaphore,
            )
            for p in projects
        ]
    )

    pushed = sum(1 for s, _ in results if s == "pushed")
    errors = sum(1 for s, _ in results if s == "error")
    checkouts = sum(1 for _, c in results if c)
    print(
        f"Push: {pushed} pushed, {checkouts} branch"
        f" checkouts, {errors} errors"
    )


# --- Serve (stays synchronous — long-running daemon) ---


def print_clone_commands(git_path, projects, port):
    """Print git clone commands for all available repos."""
    print("=== Available repos to clone ===")
    base_url = (
        f"git://localhost:{port}"
        if port != DEFAULT_PORT
        else "git://localhost"
    )
    lines = []
    for project in projects:
        repo_name = project["name"]
        if not repo_name.endswith(".git"):
            repo_name += ".git"
        bare_path = os.path.join(git_path, repo_name)
        if os.path.isdir(bare_path):
            clone_path = project["path"]
            if clone_path == ".":
                clone_path = "erplibre"
            lines.append(
                f"  git clone {base_url}/{repo_name}"
                f" {clone_path}"
            )
    lines.sort()
    for line in lines:
        print(line)
    print(f"\nTotal: {len(lines)} repos available")
    print()


def serve_git_daemon(git_path, projects, port):
    """Start git daemon to serve repos."""
    print_clone_commands(git_path, projects, port)

    cmd = (
        f"git daemon --reuseaddr"
        f" --base-path={git_path}"
        f" --port={port}"
        f" --export-all"
        f" --enable=receive-pack"
        f" {git_path}"
    )
    print(f"Starting git daemon on port {port}...")
    print(f"  Base path: {git_path}")
    print(f"  URL: git://localhost:{port}/")
    print("  Press Ctrl+C to stop")

    execute = Execute()
    execute.exec_command_live(cmd, source_erplibre=False)


# --- Main ---


async def run_actions(config, erplibre_root, projects):
    """Run init/remote/push actions asynchronously."""
    action = config.action
    jobs = config.jobs

    if action in ("init", "all"):
        print("=== Creating bare repos ===")
        await init_bare_repos(config.path, projects, jobs)
        print()

    if action in ("remote", "all"):
        print("=== Adding remotes ===")
        await add_remotes(
            erplibre_root,
            config.path,
            projects,
            config.remote_name,
            config.port,
            config.remote_url_type,
            jobs,
        )
        print()

    if action in ("push", "all"):
        print("=== Pushing to local ===")
        await push_to_local(
            erplibre_root,
            config.path,
            projects,
            config.remote_name,
            config.push_all_branches,
            config.unshallow,
            jobs,
        )
        print()


def main():
    config = get_config()

    log_level = logging.DEBUG if config.verbose else logging.INFO
    logging.basicConfig(
        level=log_level,
        format="%(levelname)s: %(message)s",
    )

    # Check write permissions on target path
    if not check_path_permissions(config.path):
        require_permissions_or_exit(config.path)

    # Detect ERPLibre root
    if config.erplibre_root:
        erplibre_root = os.path.abspath(config.erplibre_root)
    else:
        erplibre_root = DEFAULT_ERPLIBRE_PATH

    manifest_path = os.path.join(erplibre_root, config.manifest)
    if not os.path.exists(manifest_path):
        print(f"Error: Manifest not found: {manifest_path}")
        sys.exit(1)

    print(f"ERPLibre root: {erplibre_root}")
    print(f"Manifest: {manifest_path}")
    print(f"Git server path: {config.path}")
    print(f"Remote name: {config.remote_name}")
    if config.production_ready:
        print("Mode: production (/srv/git)")
    else:
        print("Mode: development (~/.git-server)")
    print(f"Remote URL type: {config.remote_url_type}")
    print(f"Parallel jobs: {config.jobs}")
    print()

    projects = parse_manifest(manifest_path)
    erplibre_project = get_erplibre_root_project(erplibre_root)
    projects.append(erplibre_project)
    print(
        f"Found {len(projects)} projects"
        f" ({len(projects) - 1} from manifest"
        f" + erplibre root)"
    )
    print()

    # Run async actions (init, remote, push)
    asyncio.run(run_actions(config, erplibre_root, projects))

    # Serve is synchronous (long-running daemon)
    if config.action in ("serve", "all"):
        print("=== Starting git daemon ===")
        serve_git_daemon(config.path, projects, config.port)


if __name__ == "__main__":
    main()