patx/gitman
improve upload ux on setting tpl
Commit 2886e76 · patx · 2026-05-08T01:51:50Z
Comments
Diff
diff --git a/app.py b/app.py
index 685155a..fa42a2e 100644
--- a/app.py
+++ b/app.py
@@ -1,6 +1,7 @@
import base64
from contextlib import contextmanager
import datetime as dt
+import fcntl
import hashlib
import hmac
import html
@@ -71,6 +72,11 @@ MAX_IMPORT_BYTES = env_int("GITMAN_MAX_IMPORT_BYTES", 2 * 1024 * 1024 * 1024)
IMPORT_UPLOAD_CHUNK_BYTES = 1024 * 1024
IMPORT_UPLOAD_STALE_SECONDS = env_int("GITMAN_IMPORT_UPLOAD_STALE_SECONDS", 6 * 60 * 60, minimum=60)
GIT_IMPORT_TIMEOUT_SECONDS = env_int("GITMAN_IMPORT_TIMEOUT_SECONDS", 3600, minimum=1)
+IMPORT_FINALIZING_STALE_SECONDS = env_int(
+ "GITMAN_IMPORT_FINALIZING_STALE_SECONDS",
+ GIT_IMPORT_TIMEOUT_SECONDS + 600,
+ minimum=60,
+)
# Gunicorn reads these when app.py is used as its config file.
timeout = env_int("GITMAN_GUNICORN_TIMEOUT_SECONDS", GIT_IMPORT_TIMEOUT_SECONDS + 300, minimum=1)
graceful_timeout = timeout
@@ -281,6 +287,8 @@ class StreamingUpload:
UPLOAD_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,80}$")
+IMPORT_STATUS_FINALIZING = "finalizing"
+IMPORT_FINALIZING_MESSAGE = "Finalizing import... You can check back in a few minutes."
def validate_startup_config():
@@ -459,6 +467,14 @@ def init_db():
updated_at TEXT NOT NULL
);
+ CREATE TABLE IF NOT EXISTS repo_imports (
+ repo_id INTEGER PRIMARY KEY REFERENCES repositories(id) ON DELETE CASCADE,
+ status TEXT NOT NULL,
+ upload_id TEXT NOT NULL DEFAULT '',
+ filename TEXT NOT NULL DEFAULT '',
+ updated_at TEXT NOT NULL
+ );
+
CREATE TABLE IF NOT EXISTS auth_rate_limits (
rate_key TEXT PRIMARY KEY,
failure_count INTEGER NOT NULL,
@@ -486,6 +502,7 @@ def init_db():
ensure_repository_pages_columns(conn)
ensure_pull_request_ref_columns(conn)
ensure_repo_metadata_table(conn)
+ ensure_repo_imports_table(conn)
ensure_auth_rate_limits_table(conn)
@@ -556,6 +573,20 @@ def ensure_repo_metadata_table(conn):
)
+def ensure_repo_imports_table(conn):
+ conn.execute(
+ """
+ CREATE TABLE IF NOT EXISTS repo_imports (
+ repo_id INTEGER PRIMARY KEY REFERENCES repositories(id) ON DELETE CASCADE,
+ status TEXT NOT NULL,
+ upload_id TEXT NOT NULL DEFAULT '',
+ filename TEXT NOT NULL DEFAULT '',
+ updated_at TEXT NOT NULL
+ )
+ """
+ )
+
+
def ensure_auth_rate_limits_table(conn):
conn.execute(
"""
@@ -1732,6 +1763,97 @@ def import_upload_chunks_dir():
return path
+def import_locks_dir():
+ path = Path(tempfile.gettempdir()) / "gitman-import-locks"
+ path.mkdir(mode=0o700, parents=True, exist_ok=True)
+ return path
+
+
+@contextmanager
+def import_file_lock(lock_path):
+ lock_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600)
+ with os.fdopen(fd, "r+") as lock_file:
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+
+
+def repo_import_lock(repo):
+ return import_file_lock(import_locks_dir() / f"repo-{repo['id']}.lock")
+
+
+def upload_chunk_lock(user, repo, upload_id):
+ return import_file_lock(import_locks_dir() / f"upload-{user['id']}-{repo['id']}-{upload_id}.lock")
+
+
+def repo_import_state(repo_id):
+ with db_connect() as conn:
+ return conn.execute(
+ "SELECT * FROM repo_imports WHERE repo_id = ?",
+ (repo_id,),
+ ).fetchone()
+
+
+def mark_repo_import_finalizing(repo_id, upload_id="", filename=""):
+ now = utcnow()
+ with db_connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO repo_imports (repo_id, status, upload_id, filename, updated_at)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(repo_id) DO UPDATE SET
+ status = excluded.status,
+ upload_id = excluded.upload_id,
+ filename = excluded.filename,
+ updated_at = excluded.updated_at
+ """,
+ (
+ repo_id,
+ IMPORT_STATUS_FINALIZING,
+ (upload_id or "")[:80],
+ (filename or "")[:255],
+ now,
+ ),
+ )
+
+
+def clear_repo_import_status(repo_id, upload_id=None):
+ with db_connect() as conn:
+ if upload_id is None:
+ conn.execute("DELETE FROM repo_imports WHERE repo_id = ?", (repo_id,))
+ else:
+ conn.execute(
+ "DELETE FROM repo_imports WHERE repo_id = ? AND upload_id = ?",
+ (repo_id, upload_id),
+ )
+
+
+def repo_import_state_is_stale(state):
+ updated_at = parse_activity_time(state["updated_at"])
+ age = dt.datetime.now(dt.UTC) - updated_at
+ return age.total_seconds() > IMPORT_FINALIZING_STALE_SECONDS
+
+
+def repo_import_settings_state(repo, path):
+ state = repo_import_state(repo["id"])
+ if not state or state["status"] != IMPORT_STATUS_FINALIZING:
+ return {"finalizing": False, "message": ""}
+
+ try:
+ empty = repo_is_empty(path)
+ except (GitCommandError, OSError):
+ empty = True
+
+ if not empty or repo_import_state_is_stale(state):
+ clear_repo_import_status(repo["id"])
+ return {"finalizing": False, "message": ""}
+
+ return {"finalizing": True, "message": IMPORT_FINALIZING_MESSAGE}
+
+
def cleanup_stale_upload_chunks(path):
if not IMPORT_UPLOAD_STALE_SECONDS:
return
@@ -1784,11 +1906,89 @@ def discard_upload_chunk(source, expected_size):
remaining -= len(chunk)
-def import_complete_upload_chunk(repo, path, filename, chunk_path):
- with chunk_path.open("rb") as bundle_file:
- import_git_bundle(repo, path, StreamingUpload(filename, bundle_file))
- updated_repo = get_repo(repo["owner_username"], repo["name"])
- return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
+def import_complete_upload_chunk(repo, path, filename, chunk_path, upload_id):
+ mark_repo_import_finalizing(repo["id"], upload_id, filename)
+ try:
+ with chunk_path.open("rb") as bundle_file:
+ import_git_bundle(
+ repo,
+ path,
+ StreamingUpload(filename, bundle_file),
+ import_upload_id=upload_id,
+ import_status_already_marked=True,
+ )
+ updated_repo = get_repo(repo["owner_username"], repo["name"])
+ return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
+ finally:
+ clear_repo_import_status(repo["id"], upload_id)
+
+
+def process_import_bundle_chunk(repo, path, filename, chunk_path, owner, repo_name, total, offset, chunk_size, upload_id):
+ if offset == 0 and chunk_path.exists():
+ chunk_path.unlink()
+
+ current_size = chunk_path.stat().st_size if chunk_path.exists() else 0
+ if current_size != offset:
+ if current_size < offset:
+ discard_upload_chunk(request.environ["wsgi.input"], chunk_size)
+ if offset + chunk_size >= total and not repo_is_empty(path):
+ clear_repo_import_status(repo["id"], upload_id)
+ updated_repo = get_repo(owner, repo_name)
+ return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
+ clear_repo_import_status(repo["id"], upload_id)
+ return HTTPResponse(
+ "Upload chunk offset mismatch.\n",
+ status=409,
+ content_type="text/plain; charset=utf-8",
+ )
+ if current_size >= offset + chunk_size:
+ discard_upload_chunk(request.environ["wsgi.input"], chunk_size)
+ if current_size >= total:
+ if not repo_is_empty(path):
+ clear_repo_import_status(repo["id"], upload_id)
+ updated_repo = get_repo(owner, repo_name)
+ return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
+ try:
+ return import_complete_upload_chunk(repo, path, filename, chunk_path, upload_id)
+ except UploadTooLarge as exc:
+ clear_repo_import_status(repo["id"], upload_id)
+ abort(413, str(exc))
+ except (ValueError, GitCommandError) as exc:
+ clear_repo_import_status(repo["id"], upload_id)
+ return HTTPResponse(f"{exc}\n", status=400, content_type="text/plain; charset=utf-8")
+ return HTTPResponse("OK\n", content_type="text/plain; charset=utf-8")
+ try:
+ with chunk_path.open("r+b") as target:
+ target.truncate(offset)
+ except OSError:
+ clear_repo_import_status(repo["id"], upload_id)
+ return HTTPResponse(
+ "Upload chunk offset mismatch.\n",
+ status=409,
+ content_type="text/plain; charset=utf-8",
+ )
+
+ try:
+ save_upload_chunk(request.environ["wsgi.input"], chunk_path, chunk_size, offset)
+ current_size = chunk_path.stat().st_size
+ if current_size < total:
+ return HTTPResponse("OK\n", content_type="text/plain; charset=utf-8")
+ if current_size != total:
+ raise ValueError("Upload size mismatch.")
+
+ return import_complete_upload_chunk(repo, path, filename, chunk_path, upload_id)
+ except UploadTooLarge as exc:
+ clear_repo_import_status(repo["id"], upload_id)
+ abort(413, str(exc))
+ except (ValueError, GitCommandError) as exc:
+ clear_repo_import_status(repo["id"], upload_id)
+ return HTTPResponse(f"{exc}\n", status=400, content_type="text/plain; charset=utf-8")
+ finally:
+ try:
+ if chunk_path.exists() and chunk_path.stat().st_size >= total:
+ chunk_path.unlink()
+ except OSError:
+ pass
def bundle_ref_names(bundle_path):
@@ -1822,7 +2022,7 @@ def fetch_bundle_refspecs(bundle_path, staging_path, refspecs):
run_git_import(["fetch", str(bundle_path), *refspecs[index : index + 100]], cwd=staging_path)
-def import_git_bundle(repo, path, upload):
+def import_git_bundle(repo, path, upload, import_upload_id="", import_status_already_marked=False):
if not path.exists():
raise ValueError("Repository directory does not exist.")
if not repo_is_empty(path):
@@ -1836,49 +2036,58 @@ def import_git_bundle(repo, path, upload):
moved_target_to_backup = False
installed_staging = False
success = False
+ status_upload_id = import_upload_id or f"direct-{secrets.token_hex(8)}"
+ import_status_marked = import_status_already_marked
try:
with tempfile.NamedTemporaryFile(prefix="gitman-import-", suffix=".bundle", delete=False) as bundle_file:
bundle_path = Path(bundle_file.name)
save_bundle_upload(upload, bundle_path)
+ if not import_status_marked:
+ mark_repo_import_finalizing(repo["id"], status_upload_id, getattr(upload, "filename", ""))
+ import_status_marked = True
verification = run_git_import(["bundle", "verify", str(bundle_path)], cwd=path, check=False)
if verification.returncode != 0:
raise ValueError("Uploaded file is not a valid Git bundle.")
- staging_path = Path(tempfile.mkdtemp(prefix=f".{name}-import-", dir=path.parent))
- shutil.rmtree(staging_path)
- run_git_import(["init", "--bare", str(staging_path)])
- run_git_import(
- [
- "fetch",
- str(bundle_path),
- "+refs/heads/*:refs/heads/*",
- "+refs/tags/*:refs/tags/*",
- ],
- cwd=staging_path,
- )
- fetch_bundle_refspecs(bundle_path, staging_path, origin_branch_refspecs_for_bundle(bundle_path))
- if not list_repo_branches(staging_path):
- raise ValueError("Uploaded Git bundle does not contain any branches.")
- default_code_ref(staging_path)
- write_git_metadata(staging_path, owner, name, repo["description"])
-
- if not repo_is_empty(path):
- raise ValueError("Repository is no longer empty.")
-
- backup_path = path.with_name(f".{name}-import-backup-{secrets.token_hex(8)}")
- path.rename(backup_path)
- moved_target_to_backup = True
- staging_path.rename(path)
- installed_staging = True
- staging_path = None
+ with repo_import_lock(repo):
+ if not repo_is_empty(path):
+ raise ValueError("Git bundles can only be imported into empty repositories.")
- with db_connect() as conn:
- conn.execute("UPDATE repositories SET updated_at = ? WHERE id = ?", (utcnow(), repo["id"]))
- mark_repo_indexing(repo["id"], path)
- schedule_repo_metadata_refresh(repo["id"])
- success = True
+ staging_path = Path(tempfile.mkdtemp(prefix=f".{name}-import-", dir=path.parent))
+ shutil.rmtree(staging_path)
+ run_git_import(["init", "--bare", str(staging_path)])
+ run_git_import(
+ [
+ "fetch",
+ str(bundle_path),
+ "+refs/heads/*:refs/heads/*",
+ "+refs/tags/*:refs/tags/*",
+ ],
+ cwd=staging_path,
+ )
+ fetch_bundle_refspecs(bundle_path, staging_path, origin_branch_refspecs_for_bundle(bundle_path))
+ if not list_repo_branches(staging_path):
+ raise ValueError("Uploaded Git bundle does not contain any branches.")
+ default_code_ref(staging_path)
+ write_git_metadata(staging_path, owner, name, repo["description"])
+
+ if not repo_is_empty(path):
+ raise ValueError("Repository is no longer empty.")
+
+ backup_path = path.with_name(f".{name}-import-backup-{secrets.token_hex(8)}")
+ path.rename(backup_path)
+ moved_target_to_backup = True
+ staging_path.rename(path)
+ installed_staging = True
+ staging_path = None
+
+ with db_connect() as conn:
+ conn.execute("UPDATE repositories SET updated_at = ? WHERE id = ?", (utcnow(), repo["id"]))
+ mark_repo_indexing(repo["id"], path)
+ schedule_repo_metadata_refresh(repo["id"])
+ success = True
except Exception:
if installed_staging and path.exists():
shutil.rmtree(path)
@@ -1892,6 +2101,8 @@ def import_git_bundle(repo, path, upload):
shutil.rmtree(staging_path)
if success and backup_path and backup_path.exists():
shutil.rmtree(backup_path)
+ if import_status_marked:
+ clear_repo_import_status(repo["id"], status_upload_id)
def delete_repository(repo, path):
@@ -4316,12 +4527,15 @@ def fork_repo(owner, repo_name):
def render_repo_settings_page(repo, path, contributor_username="", error=None, notice=None):
+ import_bundle_state = repo_import_settings_state(repo, path)
return render(
"repo_settings.tpl",
repo=repo,
contributors=list_repo_contributors(repo["id"]),
contributor_username=contributor_username,
pages_settings=pages_settings_context(repo),
+ import_bundle_finalizing=import_bundle_state["finalizing"],
+ import_bundle_status_message=import_bundle_state["message"],
error=error,
notice=notice,
**repo_page_context(repo, path),
@@ -4382,63 +4596,19 @@ def repo_settings_import_bundle_chunk(owner, repo_name):
chunks_dir = import_upload_chunks_dir()
chunk_path = chunks_dir / f"{user['id']}-{repo['id']}-{upload_id}.bundle"
- if offset == 0 and chunk_path.exists():
- chunk_path.unlink()
-
- current_size = chunk_path.stat().st_size if chunk_path.exists() else 0
- if current_size != offset:
- if current_size < offset:
- discard_upload_chunk(request.environ["wsgi.input"], chunk_size)
- if offset + chunk_size >= total and not repo_is_empty(path):
- updated_repo = get_repo(owner, repo_name)
- return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
- return HTTPResponse(
- "Upload chunk offset mismatch.\n",
- status=409,
- content_type="text/plain; charset=utf-8",
- )
- if current_size >= offset + chunk_size:
- discard_upload_chunk(request.environ["wsgi.input"], chunk_size)
- if current_size >= total:
- if not repo_is_empty(path):
- updated_repo = get_repo(owner, repo_name)
- return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
- try:
- return import_complete_upload_chunk(repo, path, filename, chunk_path)
- except UploadTooLarge as exc:
- abort(413, str(exc))
- except (ValueError, GitCommandError) as exc:
- return HTTPResponse(f"{exc}\n", status=400, content_type="text/plain; charset=utf-8")
- return HTTPResponse("OK\n", content_type="text/plain; charset=utf-8")
- try:
- with chunk_path.open("r+b") as target:
- target.truncate(offset)
- except OSError:
- return HTTPResponse(
- "Upload chunk offset mismatch.\n",
- status=409,
- content_type="text/plain; charset=utf-8",
- )
-
- try:
- save_upload_chunk(request.environ["wsgi.input"], chunk_path, chunk_size, offset)
- current_size = chunk_path.stat().st_size
- if current_size < total:
- return HTTPResponse("OK\n", content_type="text/plain; charset=utf-8")
- if current_size != total:
- raise ValueError("Upload size mismatch.")
-
- return import_complete_upload_chunk(repo, path, filename, chunk_path)
- except UploadTooLarge as exc:
- abort(413, str(exc))
- except (ValueError, GitCommandError) as exc:
- return HTTPResponse(f"{exc}\n", status=400, content_type="text/plain; charset=utf-8")
- finally:
- try:
- if chunk_path.exists() and chunk_path.stat().st_size >= total:
- chunk_path.unlink()
- except OSError:
- pass
+ with upload_chunk_lock(user, repo, upload_id):
+ return process_import_bundle_chunk(
+ repo,
+ path,
+ filename,
+ chunk_path,
+ owner,
+ repo_name,
+ total,
+ offset,
+ chunk_size,
+ upload_id,
+ )
@app.route("/<owner>/<repo_name>/settings", method=["GET", "POST"])
diff --git a/scripts/git_http_runtime_matrix.py b/scripts/git_http_runtime_matrix.py
new file mode 100755
index 0000000..dbabbd3
--- /dev/null
+++ b/scripts/git_http_runtime_matrix.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from urllib.error import URLError
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import urlopen
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Compare large Git HTTP pushes across local GitMan server runtimes."
+ )
+ parser.add_argument("--source-repo", required=True, help="Local Git repository to push from.")
+ parser.add_argument("--remote-path", required=True, help="GitMan remote path, e.g. /git/alice/demo.")
+ parser.add_argument(
+ "--remote-template",
+ default="http://127.0.0.1:{port}{remote_path}",
+ help=(
+ "Remote URL template. Supports {port}, {mode}, and {remote_path}. "
+ "Embed credentials here or rely on a credential helper."
+ ),
+ )
+ parser.add_argument(
+ "--modes",
+ default="wsgiref,gunicorn-sync,gunicorn-gthread,gunicorn-config",
+ help="Comma-separated modes: wsgiref, gunicorn-sync, gunicorn-gthread, gunicorn-config.",
+ )
+ parser.add_argument("--start-port", type=int, default=18080)
+ parser.add_argument("--server-timeout", type=int, default=7500)
+ parser.add_argument("--push-timeout", type=int, default=7200)
+ parser.add_argument("--threads", type=int, default=16)
+ parser.add_argument(
+ "--refspec",
+ default="HEAD:refs/heads/gitman-runtime-matrix-{mode}",
+ help="Refspec template for the push. Supports {mode}.",
+ )
+ parser.add_argument("--http-version", default="HTTP/1.1", choices=["HTTP/1.1", "HTTP/2"])
+ parser.add_argument("--workdir", default=str(Path(__file__).resolve().parents[1]))
+ return parser.parse_args()
+
+
+def redact_url(url):
+ parts = urlsplit(url)
+ if "@" not in parts.netloc:
+ return url
+ host = parts.netloc.rsplit("@", 1)[1]
+ return urlunsplit((parts.scheme, f"<credentials>@{host}", parts.path, parts.query, parts.fragment))
+
+
+def server_command(mode, port, args):
+ if mode == "wsgiref":
+ return [sys.executable, "app.py"], {"PORT": str(port)}
+ if mode == "gunicorn-sync":
+ return [
+ "gunicorn",
+ "--config",
+ "/dev/null",
+ "--bind",
+ f"127.0.0.1:{port}",
+ "--workers",
+ "1",
+ "--worker-class",
+ "sync",
+ "--timeout",
+ str(args.server_timeout),
+ "app:app",
+ ], {}
+ if mode == "gunicorn-gthread":
+ return [
+ "gunicorn",
+ "--config",
+ "/dev/null",
+ "--bind",
+ f"127.0.0.1:{port}",
+ "--workers",
+ "1",
+ "--worker-class",
+ "gthread",
+ "--threads",
+ str(args.threads),
+ "--timeout",
+ str(args.server_timeout),
+ "app:app",
+ ], {}
+ if mode == "gunicorn-config":
+ return ["gunicorn", "--bind", f"127.0.0.1:{port}", "app:app"], {}
+ raise ValueError(f"Unsupported mode: {mode}")
+
+
+def wait_for_server(port, process, timeout=20):
+ deadline = time.monotonic() + timeout
+ url = f"http://127.0.0.1:{port}/"
+ while time.monotonic() < deadline:
+ if process.poll() is not None:
+ return False
+ try:
+ with urlopen(url, timeout=1) as response:
+ return 200 <= response.status < 500
+ except URLError:
+ time.sleep(0.2)
+ return False
+
+
+def stop_process(process):
+ if process.poll() is not None:
+ return
+ process.send_signal(signal.SIGTERM)
+ try:
+ process.wait(timeout=10)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait(timeout=10)
+
+
+def tail(path, limit=80):
+ try:
+ lines = path.read_text(errors="replace").splitlines()
+ except OSError:
+ return ""
+ return "\n".join(lines[-limit:])
+
+
+def run_mode(mode, port, args, log_dir):
+ command, extra_env = server_command(mode, port, args)
+ env = os.environ.copy()
+ env.setdefault("SECRET_KEY", "gitman-runtime-matrix-secret")
+ env.setdefault("GITMAN_GUNICORN_TIMEOUT_SECONDS", str(args.server_timeout))
+ env.update(extra_env)
+ log_path = log_dir / f"{mode}.server.log"
+ with log_path.open("wb") as log_file:
+ process = subprocess.Popen(
+ command,
+ cwd=args.workdir,
+ env=env,
+ stdout=log_file,
+ stderr=subprocess.STDOUT,
+ )
+ if not wait_for_server(port, process):
+ stop_process(process)
+ return {
+ "mode": mode,
+ "ok": False,
+ "detail": "server did not become ready",
+ "server_log": tail(log_path),
+ }
+
+ remote = args.remote_template.format(port=port, mode=mode, remote_path=args.remote_path)
+ refspec = args.refspec.format(mode=mode)
+ push_command = [
+ "git",
+ "-C",
+ args.source_repo,
+ "-c",
+ f"http.version={args.http_version}",
+ "push",
+ remote,
+ refspec,
+ ]
+ started = time.monotonic()
+ completed = subprocess.run(
+ push_command,
+ cwd=args.workdir,
+ env={**env, "GIT_TERMINAL_PROMPT": "0"},
+ capture_output=True,
+ text=True,
+ errors="replace",
+ timeout=args.push_timeout,
+ )
+ elapsed = time.monotonic() - started
+ stop_process(process)
+ return {
+ "mode": mode,
+ "ok": completed.returncode == 0,
+ "detail": f"exit={completed.returncode} elapsed={elapsed:.1f}s remote={redact_url(remote)}",
+ "stdout": completed.stdout.strip(),
+ "stderr": completed.stderr.strip(),
+ "server_log": tail(log_path),
+ }
+
+
+def main():
+ args = parse_args()
+ source_repo = Path(args.source_repo)
+ if not source_repo.exists():
+ print(f"source repo does not exist: {source_repo}", file=sys.stderr)
+ return 2
+ modes = [mode.strip() for mode in args.modes.split(",") if mode.strip()]
+ with tempfile.TemporaryDirectory(prefix="gitman-runtime-matrix-") as temp_dir:
+ log_dir = Path(temp_dir)
+ results = []
+ for index, mode in enumerate(modes):
+ port = args.start_port + index
+ print(f"== {mode} on port {port} ==", flush=True)
+ try:
+ result = run_mode(mode, port, args, log_dir)
+ except subprocess.TimeoutExpired as exc:
+ result = {"mode": mode, "ok": False, "detail": f"push timed out: {exc}"}
+ results.append(result)
+ status = "PASS" if result["ok"] else "FAIL"
+ print(f"{status}: {result['detail']}", flush=True)
+ if result.get("stderr"):
+ print(result["stderr"], flush=True)
+
+ print("\nSummary")
+ for result in results:
+ status = "PASS" if result["ok"] else "FAIL"
+ print(f"{status} {result['mode']}: {result['detail']}")
+ if any(not result["ok"] for result in results):
+ print("\nServer log tails for failures")
+ for result in results:
+ if not result["ok"] and result.get("server_log"):
+ print(f"\n--- {result['mode']} ---")
+ print(result["server_log"])
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/templates/base.tpl b/templates/base.tpl
index d51af42..2533d51 100644
--- a/templates/base.tpl
+++ b/templates/base.tpl
@@ -419,11 +419,34 @@
}
};
+ const uploadChunk = (url, body, csrfToken, onProgress, onUploadComplete) => {
+ return new Promise((resolve, reject) => {
+ const xhr = new XMLHttpRequest();
+ xhr.open("POST", url);
+ xhr.setRequestHeader("Content-Type", "application/octet-stream");
+ xhr.setRequestHeader("X-CSRF-Token", csrfToken);
+ xhr.upload.addEventListener("progress", (progressEvent) => {
+ if (progressEvent.lengthComputable) onProgress(progressEvent.loaded);
+ });
+ xhr.upload.addEventListener("load", onUploadComplete);
+ xhr.addEventListener("load", () => {
+ resolve({
+ ok: xhr.status >= 200 && xhr.status < 300,
+ status: xhr.status,
+ text: async () => xhr.responseText || "",
+ });
+ });
+ xhr.addEventListener("error", () => reject(new Error("Upload failed. Check your connection and try again.")));
+ xhr.addEventListener("abort", () => reject(new Error("Upload canceled.")));
+ xhr.send(body);
+ });
+ };
+
forms.forEach((form) => {
form.addEventListener("submit", async (event) => {
const input = form.querySelector("[data-import-bundle-file]");
const file = input && input.files ? input.files[0] : null;
- if (!file || !window.fetch) return;
+ if (!file || !window.XMLHttpRequest) return;
event.preventDefault();
@@ -434,27 +457,35 @@
url.searchParams.set("filename", file.name || "repo.bundle");
if (file.size <= 0) return;
- if (status) {
- status.className = "muted";
- status.textContent = "Uploading Git bundle...";
+ const setStatus = (message, className = "muted") => {
+ if (!status) return;
+ status.className = className;
+ status.textContent = message;
status.hidden = false;
- }
+ };
+ const setUploadStatus = (loadedBytes) => {
+ const percentage = Math.min(100, Math.floor((loadedBytes / file.size) * 100));
+ setStatus(`Uploading Git bundle... ${percentage}% - Do not leave this page.`);
+ };
+
+ setUploadStatus(0);
if (button) {
button.dataset.originalLabel = button.textContent;
button.disabled = true;
- button.textContent = "Importing...";
+ button.hidden = true;
}
try {
const chunkSize = 4 * 1024 * 1024;
const uploadId = (
- crypto.randomUUID ? crypto.randomUUID() : `${Date.now()}-${Math.random()}`
+ window.crypto && window.crypto.randomUUID ? window.crypto.randomUUID() : `${Date.now()}-${Math.random()}`
).replace(/[^A-Za-z0-9._-]/g, "");
let offset = 0;
let finalResponse = null;
while (offset < file.size) {
const end = Math.min(offset + chunkSize, file.size);
+ const isFinalChunk = end >= file.size;
url.searchParams.set("upload_id", uploadId);
url.searchParams.set("offset", String(offset));
url.searchParams.set("total", String(file.size));
@@ -464,14 +495,19 @@
for (let attempt = 1; attempt <= 8; attempt += 1) {
try {
url.searchParams.set("retry", String(attempt));
- response = await fetch(url.toString(), {
- method: "POST",
- headers: {
- "Content-Type": "application/octet-stream",
- "X-CSRF-Token": csrf ? csrf.value : "",
- },
- body: file.slice(offset, end),
- });
+ response = await uploadChunk(
+ url.toString(),
+ file.slice(offset, end),
+ csrf ? csrf.value : "",
+ (loadedBytes) => setUploadStatus(offset + loadedBytes),
+ () => {
+ if (isFinalChunk) {
+ setStatus("Finalizing import... You can check back in a few minutes.");
+ } else {
+ setUploadStatus(end);
+ }
+ }
+ );
if (response.ok) break;
lastError = new Error(
await responseMessage(response, "Upload failed.")
@@ -481,7 +517,7 @@
lastError = error;
}
if (attempt < 8) {
- if (status) status.textContent = `Retrying upload chunk... ${attempt}/7`;
+ setStatus(`Uploading Git bundle... ${Math.floor((offset / file.size) * 100)}% - Do not leave this page. Retrying chunk ${attempt}/7...`);
await new Promise((resolve) => setTimeout(resolve, Math.min(attempt * 2000, 15000)));
}
}
@@ -489,27 +525,23 @@
offset = end;
if (offset < file.size) {
- if (status) {
- status.textContent = `Uploading Git bundle... ${Math.floor((offset / file.size) * 100)}%`;
- }
+ setUploadStatus(offset);
} else {
finalResponse = response;
}
}
- if (status) status.textContent = "Importing Git bundle...";
replaceDocument(await finalResponse.text());
} catch (error) {
- if (status) {
- status.className = "alert";
- status.textContent =
- error && error.message
- ? error.message
- : "Upload failed. Check your connection and try again.";
- status.hidden = false;
- }
+ setStatus(
+ error && error.message
+ ? error.message
+ : "Upload failed. Check your connection and try again.",
+ "alert"
+ );
if (button) {
button.disabled = false;
+ button.hidden = false;
button.textContent = button.dataset.originalLabel || "Import bundle";
}
}
diff --git a/templates/repo_settings.tpl b/templates/repo_settings.tpl
index 4517b3a..90e23ed 100644
--- a/templates/repo_settings.tpl
+++ b/templates/repo_settings.tpl
@@ -23,21 +23,24 @@
</form>
</section>
-% if commit_count == 0:
+% if commit_count == 0 or import_bundle_finalizing:
<section class="panel">
<h2>Import Git bundle</h2>
- <p class="muted">Create a bundle from the source repository, then upload it here.</p>
- <pre>git bundle create repo.bundle --all</pre>
+ % if import_bundle_finalizing:
+ <p class="muted" data-import-bundle-status>{{import_bundle_status_message}}</p>
+ % else:
+ <p class="muted">Create a bundle from the source repository using
+ `git bundle create repo.bundle --all`, then upload it here.</p>
<form method="post" enctype="multipart/form-data" data-import-bundle-form data-upload-url="/{{repo['owner_username']}}/{{repo['name']}}/settings/import-bundle/chunk">
{{!csrf_field()}}
<input type="hidden" name="action" value="import_bundle">
<label>
- Git bundle
<input name="bundle" type="file" accept=".bundle,application/octet-stream" data-import-bundle-file required>
</label>
<p class="muted" data-import-bundle-status hidden></p>
<button class="button" type="submit">Import bundle</button>
</form>
+ % end
</section>
% end
diff --git a/tests/test_app.py b/tests/test_app.py
index e46efa6..6e56fe0 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -9,6 +9,8 @@ import shutil
import subprocess
import sys
import tempfile
+import threading
+import time
from pathlib import Path
from urllib.parse import urlencode, urlsplit, unquote
@@ -740,6 +742,7 @@ def test_init_db_creates_expected_tables_and_is_idempotent(isolated_app):
"pull_requests",
"repo_stars",
"custom_domains",
+ "repo_imports",
"auth_rate_limits",
}.issubset(tables)
assert {"display_name", "bio", "website"}.issubset(user_columns)
@@ -1031,6 +1034,7 @@ def test_import_git_bundle_chunked_upload_preserves_refs(isolated_app, tmp_path)
def test_import_git_bundle_chunked_upload_returns_specific_import_error(isolated_app):
owner = create_user("alice")
isolated_app.create_repository(owner, "empty", "")
+ repo = isolated_app.get_repo("alice", "empty")
client = WsgiClient(isolated_app.app)
login_client(client, "alice")
@@ -1040,6 +1044,93 @@ def test_import_git_bundle_chunked_upload_returns_specific_import_error(isolated
assert response.status_code == 400
assert response.text == "Uploaded file is not a valid Git bundle.\n"
+ assert isolated_app.repo_import_state(repo["id"]) is None
+
+
+def test_import_git_bundle_finalizing_state_hides_import_button(isolated_app):
+ owner = create_user("alice")
+ isolated_app.create_repository(owner, "empty", "")
+ repo = isolated_app.get_repo("alice", "empty")
+ isolated_app.mark_repo_import_finalizing(repo["id"], "test-upload", "repo.bundle")
+
+ client = WsgiClient(isolated_app.app)
+ login_client(client, "alice")
+ response = client.get("/alice/empty/settings")
+
+ assert response.status_code == 200
+ assert "Finalizing import... You can check back in a few minutes." in response.text
+ assert 'name="bundle"' not in response.text
+ assert ">Import bundle</button>" not in response.text
+ assert isolated_app.repo_import_state(repo["id"]) is not None
+
+
+def test_import_git_bundle_final_chunk_marks_and_clears_finalizing_state(isolated_app, monkeypatch):
+ owner = create_user("alice")
+ isolated_app.create_repository(owner, "empty", "")
+ repo = isolated_app.get_repo("alice", "empty")
+ seen_state = {}
+
+ def fake_import_git_bundle(repo_arg, path_arg, upload, import_upload_id="", import_status_already_marked=False):
+ state = isolated_app.repo_import_state(repo_arg["id"])
+ seen_state["status"] = state["status"] if state else ""
+ seen_state["upload_id"] = state["upload_id"] if state else ""
+ seen_state["already_marked"] = import_status_already_marked
+ seen_state["import_upload_id"] = import_upload_id
+ raise ValueError("import failed")
+
+ monkeypatch.setattr(isolated_app, "import_git_bundle", fake_import_git_bundle)
+
+ client = WsgiClient(isolated_app.app)
+ login_client(client, "alice")
+ client.get("/alice/empty/settings")
+
+ response = post_bundle_import_chunk(
+ client,
+ "alice",
+ "empty",
+ b"abc",
+ offset=0,
+ total=3,
+ upload_id="test-upload",
+ )
+
+ assert response.status_code == 400
+ assert response.text == "import failed\n"
+ assert seen_state == {
+ "status": isolated_app.IMPORT_STATUS_FINALIZING,
+ "upload_id": "test-upload",
+ "already_marked": True,
+ "import_upload_id": "test-upload",
+ }
+ assert isolated_app.repo_import_state(repo["id"]) is None
+
+
+def test_import_git_bundle_incomplete_chunk_clears_finalizing_state(isolated_app):
+ owner = create_user("alice")
+ isolated_app.create_repository(owner, "empty", "")
+ repo = isolated_app.get_repo("alice", "empty")
+ isolated_app.mark_repo_import_finalizing(repo["id"], "test-upload", "repo.bundle")
+
+ client = WsgiClient(isolated_app.app)
+ login_client(client, "alice")
+ client.get("/alice/empty/settings")
+
+ response = post_bundle_import_chunk(
+ client,
+ "alice",
+ "empty",
+ b"abc",
+ offset=5,
+ total=10,
+ upload_id="test-upload",
+ )
+ settings_response = client.get("/alice/empty/settings")
+
+ assert response.status_code == 409
+ assert response.text == "Upload chunk offset mismatch.\n"
+ assert isolated_app.repo_import_state(repo["id"]) is None
+ assert 'name="bundle"' in settings_response.text
+ assert ">Import bundle</button>" in settings_response.text
def test_import_git_bundle_final_chunk_retry_after_completed_import_returns_success(isolated_app):
@@ -1065,6 +1156,95 @@ def test_import_git_bundle_final_chunk_retry_after_completed_import_returns_succ
assert "Upload chunk offset mismatch." not in response.text
+def test_import_git_bundle_rechecks_empty_repo_before_fetch(isolated_app, monkeypatch):
+ owner = create_user("alice")
+ isolated_app.create_repository(owner, "empty", "")
+ repo = isolated_app.get_repo("alice", "empty")
+ path = isolated_app.repo_path("alice", "empty")
+ checked_empty = iter([True, False])
+ calls = []
+
+ def fake_repo_is_empty(repo_path):
+ assert repo_path == path
+ return next(checked_empty)
+
+ def fake_run_git_import(args, cwd=None, check=True):
+ calls.append(args)
+ if args[:2] == ["bundle", "verify"]:
+ return subprocess.CompletedProcess(args, 0, stdout="", stderr="")
+ raise AssertionError(f"unexpected import git call: {args}")
+
+ monkeypatch.setattr(isolated_app, "repo_is_empty", fake_repo_is_empty)
+ monkeypatch.setattr(isolated_app, "run_git_import", fake_run_git_import)
+
+ with pytest.raises(ValueError, match="empty repositories"):
+ isolated_app.import_git_bundle(repo, path, gitman.StreamingUpload("repo.bundle", BytesIO(b"bundle")))
+
+ assert calls
+ assert calls[0][:2] == ["bundle", "verify"]
+ assert all(call[0] != "fetch" for call in calls)
+
+
+def test_import_bundle_chunk_requests_for_same_upload_do_not_overlap(isolated_app, tmp_path, monkeypatch):
+ owner = create_user("alice")
+ isolated_app.create_repository(owner, "chunked", "")
+ repo = isolated_app.get_repo("alice", "chunked")
+ temp_root = tmp_path / "tmp"
+ active_saves = 0
+ overlap = threading.Event()
+ save_lock = threading.Lock()
+ original_save_upload_chunk = isolated_app.save_upload_chunk
+
+ def slow_save_upload_chunk(*args, **kwargs):
+ nonlocal active_saves
+ with save_lock:
+ active_saves += 1
+ if active_saves > 1:
+ overlap.set()
+ try:
+ time.sleep(0.05)
+ return original_save_upload_chunk(*args, **kwargs)
+ finally:
+ with save_lock:
+ active_saves -= 1
+
+ monkeypatch.setattr(gitman.tempfile, "gettempdir", lambda: str(temp_root))
+ monkeypatch.setattr(isolated_app, "save_upload_chunk", slow_save_upload_chunk)
+
+ clients = [WsgiClient(isolated_app.app), WsgiClient(isolated_app.app)]
+ for client in clients:
+ login_client(client, "alice")
+ client.get("/alice/chunked/settings")
+
+ responses = []
+ threads = [
+ threading.Thread(
+ target=lambda current_client=client: responses.append(
+ post_bundle_import_chunk(
+ current_client,
+ "alice",
+ "chunked",
+ b"abc",
+ offset=0,
+ total=6,
+ upload_id="same-upload",
+ )
+ )
+ )
+ for client in clients
+ ]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=2)
+
+ assert len(responses) == 2
+ assert {response.status_code for response in responses} == {200}
+ assert not overlap.is_set()
+ chunk_path = temp_root / "gitman-import-chunks" / f"{owner['id']}-{repo['id']}-same-upload.bundle"
+ assert chunk_path.read_bytes() == b"abc"
+
+
def test_import_git_bundle_rejects_invalid_upload_without_replacing_repo(isolated_app):
owner = create_user("alice")
isolated_app.create_repository(owner, "empty", "")
We added some backend helpers here to show a better status message while uploading.