patx/gitman
upload bundle tested up to 2gb
Commit 0269d24 · patx · 2026-05-06T22:28:53-04:00
Comments
No comments yet.
Diff
diff --git a/README.md b/README.md
index 86d6516..aac786b 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,7 @@ Pages-style static hosting is driven by the Git repository contents. A user site
- `GITMAN_MAX_FORM_BYTES`: maximum browser form body size, default `65536`
- `GITMAN_MAX_IMPORT_BYTES`: maximum Git bundle import upload size, default `5368709120`
- `GITMAN_IMPORT_TIMEOUT_SECONDS`: maximum Git bundle verify/fetch time, default `3600`
+- `GITMAN_GUNICORN_TIMEOUT_SECONDS`: Gunicorn worker timeout, default `GITMAN_IMPORT_TIMEOUT_SECONDS + 300`
- `GITMAN_MAX_RENDER_BYTES`: maximum file preview size, default `262144`
- `GITMAN_MAX_GIT_RESPONSE_BYTES`: maximum Git HTTP backend response size, default `268435456`
- `GITMAN_RATE_LIMIT_ENABLED`: set to `0` to disable login, signup, and Git push auth rate limiting
@@ -49,7 +50,7 @@ When `GITMAN_DEBUG` is off, `SECRET_KEY` must be set to a non-default value befo
Repositories and their Git data live on local disk, so keep the database and repo root on persistent storage. The app uses SQLite WAL mode and shells out to Git, so the process user needs read/write access to both paths and access to the configured Git executable.
-For large bundle imports behind nginx and gunicorn, set nginx `client_max_body_size` above `GITMAN_MAX_IMPORT_BYTES`, raise nginx proxy read/send timeouts for long imports, and set gunicorn's worker timeout above `GITMAN_IMPORT_TIMEOUT_SECONDS`. The server also needs enough temporary disk space for the uploaded bundle plus the staged bare repository during import.
+For large bundle imports behind nginx and gunicorn, set nginx `client_max_body_size` above `GITMAN_MAX_IMPORT_BYTES`, raise nginx proxy read/send timeouts for long imports, and set Gunicorn's worker timeout above the expected upload plus import time. The repo includes `gunicorn.conf.py`, which Gunicorn loads when run from the project root and sets the worker timeout from `GITMAN_GUNICORN_TIMEOUT_SECONDS`. The server also needs enough temporary disk space for the uploaded bundle plus the staged bare repository during import.
## License
diff --git a/app.py b/app.py
index 4657e80..55fc248 100644
--- a/app.py
+++ b/app.py
@@ -253,6 +253,12 @@ class UploadTooLarge(ValueError):
pass
+class StreamingUpload:
+ def __init__(self, filename, file):
+ self.filename = filename
+ self.file = file
+
+
def validate_startup_config():
if not DEBUG and SECRET_KEY == DEFAULT_SECRET_KEY:
raise RuntimeError("SECRET_KEY must be set to a non-default value when GITMAN_DEBUG is disabled.")
@@ -581,9 +587,15 @@ def csrf_field():
)
+def request_content_type():
+ return (request.environ.get("CONTENT_TYPE") or request.get_header("Content-Type") or "").lower()
+
+
def validate_csrf_token():
expected = request.get_cookie(CSRF_COOKIE_NAME, secret=SECRET_KEY)
- submitted = request.forms.get(CSRF_FORM_FIELD, "")
+ submitted = request.get_header("X-CSRF-Token", "") or request.query.get(CSRF_FORM_FIELD, "")
+ if not submitted and not request_content_type().startswith("application/octet-stream"):
+ submitted = request.forms.get(CSRF_FORM_FIELD, "")
if not expected or not submitted or not hmac.compare_digest(expected, submitted):
abort(403, "Invalid CSRF token.")
@@ -597,12 +609,16 @@ def request_content_length():
def is_repo_settings_multipart_request():
path = request.environ.get("PATH_INFO", request.path) or ""
- content_type = (request.environ.get("CONTENT_TYPE") or request.get_header("Content-Type") or "").lower()
- return path.endswith("/settings") and content_type.startswith("multipart/form-data")
+ return path.endswith("/settings") and request_content_type().startswith("multipart/form-data")
+
+
+def is_repo_settings_import_stream_request():
+ path = request.environ.get("PATH_INFO", request.path) or ""
+ return path.endswith("/settings/import-bundle") and request_content_type().startswith("application/octet-stream")
def browser_post_size_limit():
- if is_repo_settings_multipart_request():
+ if is_repo_settings_multipart_request() or is_repo_settings_import_stream_request():
return MAX_IMPORT_BYTES
return MAX_FORM_BYTES
@@ -3636,6 +3652,31 @@ def render_repo_settings_page(repo, path, contributor_username="", error=None, n
)
[email protected]("/<owner>/<repo_name>/settings/import-bundle")
+def repo_settings_import_bundle(owner, repo_name):
+ user = require_login()
+ repo = get_repo(owner, repo_name)
+ if not repo:
+ abort(404, "Repository not found.")
+ if not user_owns_repo(user, repo):
+ abort(403, "Only the owner can update repository settings.")
+ if not request_content_type().startswith("application/octet-stream"):
+ abort(400, "Git bundle uploads must use application/octet-stream.")
+
+ path = repo_path(owner, repo_name)
+ filename = os.path.basename(request.query.get("filename", "repo.bundle")) or "repo.bundle"
+ upload = StreamingUpload(filename, request.environ["wsgi.input"])
+ try:
+ import_git_bundle(repo, path, upload)
+ updated_repo = get_repo(owner, repo_name)
+ return render_repo_settings_page(updated_repo, path, notice="Git bundle imported.")
+ except UploadTooLarge as exc:
+ abort(413, str(exc))
+ except (ValueError, GitCommandError) as exc:
+ repo = get_repo(owner, repo_name) or repo
+ return render_repo_settings_page(repo, path, error=str(exc))
+
+
@app.route("/<owner>/<repo_name>/settings", method=["GET", "POST"])
def repo_settings(owner, repo_name):
user = require_login()
diff --git a/gunicorn.conf.py b/gunicorn.conf.py
new file mode 100644
index 0000000..a9be4df
--- /dev/null
+++ b/gunicorn.conf.py
@@ -0,0 +1,14 @@
+import os
+
+
+def env_int(name, default, minimum=1):
+ try:
+ value = int(os.environ.get(name, str(default)))
+ except ValueError:
+ return default
+ return max(minimum, value)
+
+
+import_timeout = env_int("GITMAN_IMPORT_TIMEOUT_SECONDS", 3600, minimum=1)
+timeout = env_int("GITMAN_GUNICORN_TIMEOUT_SECONDS", import_timeout + 300, minimum=1)
+graceful_timeout = timeout
diff --git a/templates/base.tpl b/templates/base.tpl
index 8586f99..e3d91cb 100644
--- a/templates/base.tpl
+++ b/templates/base.tpl
@@ -399,5 +399,66 @@
});
})();
</script>
+ <script>
+ (() => {
+ const forms = document.querySelectorAll("[data-import-bundle-form]");
+ if (!forms.length) return;
+
+ const replaceDocument = (html) => {
+ document.open();
+ document.write(html);
+ document.close();
+ };
+
+ forms.forEach((form) => {
+ form.addEventListener("submit", async (event) => {
+ const input = form.querySelector("[data-import-bundle-file]");
+ const file = input && input.files ? input.files[0] : null;
+ if (!file || !window.fetch) return;
+
+ event.preventDefault();
+
+ const status = form.querySelector("[data-import-bundle-status]");
+ const button = form.querySelector("button[type='submit']");
+ const csrf = form.querySelector('input[name="_csrf_token"]');
+ const url = new URL(form.dataset.uploadUrl, window.location.origin);
+ url.searchParams.set("filename", file.name || "repo.bundle");
+
+ if (status) {
+ status.className = "muted";
+ status.textContent = "Uploading Git bundle...";
+ status.hidden = false;
+ }
+ if (button) {
+ button.dataset.originalLabel = button.textContent;
+ button.disabled = true;
+ button.textContent = "Importing...";
+ }
+
+ try {
+ const response = await fetch(url.toString(), {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/octet-stream",
+ "X-CSRF-Token": csrf ? csrf.value : "",
+ },
+ body: file,
+ });
+ replaceDocument(await response.text());
+ } catch (error) {
+ if (status) {
+ status.className = "alert";
+ status.textContent = "Upload failed. Check your connection and try again.";
+ status.hidden = false;
+ }
+ if (button) {
+ button.disabled = false;
+ button.textContent = button.dataset.originalLabel || "Import bundle";
+ }
+ }
+ });
+ });
+ })();
+ </script>
</body>
</html>
diff --git a/templates/repo_settings.tpl b/templates/repo_settings.tpl
index f74ec04..dade147 100644
--- a/templates/repo_settings.tpl
+++ b/templates/repo_settings.tpl
@@ -28,13 +28,14 @@
<h2>Import Git bundle</h2>
<p class="muted">Create a bundle from the source repository, then upload it here.</p>
<pre>git bundle create repo.bundle --all</pre>
- <form method="post" enctype="multipart/form-data">
+ <form method="post" enctype="multipart/form-data" data-import-bundle-form data-upload-url="/{{repo['owner_username']}}/{{repo['name']}}/settings/import-bundle">
{{!csrf_field()}}
<input type="hidden" name="action" value="import_bundle">
<label>
Git bundle
- <input name="bundle" type="file" accept=".bundle,application/octet-stream" required>
+ <input name="bundle" type="file" accept=".bundle,application/octet-stream" data-import-bundle-file required>
</label>
+ <p class="muted" data-import-bundle-status hidden></p>
<button class="button" type="submit">Import bundle</button>
</form>
</section>
diff --git a/tests/test_app.py b/tests/test_app.py
index 2417733..4ab6ef9 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -277,6 +277,21 @@ def create_origin_bundle_with_refs(bundle_path):
return {"main": main_node, "feature": feature_node}
+def post_bundle_import(client, owner, repo_name, content, filename="repo.bundle"):
+ if hasattr(content, "read_bytes"):
+ content = content.read_bytes()
+ query = urlencode({"filename": filename})
+ return client.request(
+ "POST",
+ f"/{owner}/{repo_name}/settings/import-bundle?{query}",
+ headers={
+ "Content-Type": "application/octet-stream",
+ "X-CSRF-Token": client.csrf_token or "",
+ },
+ raw_body=content,
+ )
+
+
def basic_auth(username, password):
token = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
return {"Authorization": f"Basic {token}"}
@@ -769,12 +784,9 @@ def test_import_git_bundle_into_empty_repository_preserves_refs_and_metadata(iso
assert settings_response.status_code == 200
assert "Import Git bundle" in settings_response.text
assert 'name="bundle"' in settings_response.text
+ assert 'data-import-bundle-form' in settings_response.text
- response = client.post_multipart(
- "/alice/imported/settings",
- fields={"action": "import_bundle"},
- files={"bundle": ("repo.bundle", bundle_path.read_bytes(), "application/octet-stream")},
- )
+ response = post_bundle_import(client, "alice", "imported", bundle_path)
assert response.status_code == 200
assert "Git bundle imported." in response.text
@@ -803,11 +815,7 @@ def test_import_git_bundle_maps_origin_remote_branches(isolated_app, tmp_path):
login_client(client, "alice")
client.get("/alice/github-import/settings")
- response = client.post_multipart(
- "/alice/github-import/settings",
- fields={"action": "import_bundle"},
- files={"bundle": ("origin.bundle", bundle_path.read_bytes(), "application/octet-stream")},
- )
+ response = post_bundle_import(client, "alice", "github-import", bundle_path, filename="origin.bundle")
assert response.status_code == 200
imported_path = isolated_app.repo_path("alice", "github-import")
@@ -828,11 +836,7 @@ def test_import_git_bundle_rejects_invalid_upload_without_replacing_repo(isolate
login_client(client, "alice")
client.get("/alice/empty/settings")
- response = client.post_multipart(
- "/alice/empty/settings",
- fields={"action": "import_bundle"},
- files={"bundle": ("repo.bundle", b"not a git bundle", "application/octet-stream")},
- )
+ response = post_bundle_import(client, "alice", "empty", b"not a git bundle")
assert response.status_code == 200
assert "Uploaded file is not a valid Git bundle." in response.text
@@ -878,11 +882,7 @@ def test_import_git_bundle_size_limit_returns_413(isolated_app, monkeypatch):
login_client(client, "alice")
client.get("/alice/empty/settings")
- response = client.post_multipart(
- "/alice/empty/settings",
- fields={"action": "import_bundle"},
- files={"bundle": ("repo.bundle", b"x" * 200, "application/octet-stream")},
- )
+ response = post_bundle_import(client, "alice", "empty", b"x" * 200)
assert response.status_code == 413
assert "Request body too large." in response.text