From 62266dabde03645a2389972f26338a324463d1f3 Mon Sep 17 00:00:00 2001 From: Sasha Mitchell Date: Thu, 1 Oct 2026 17:15:39 +0700 Subject: [PATCH 001/153] objects: decode a byte mode before reading its digits mode_str_to_int(b"100644") returned 1830740. A byte is an int, so the digit 4 was read as 52. A text mode was already 0o100644. --- git/objects/util.py | 4 +++- test/test_util.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/git/objects/util.py b/git/objects/util.py index 64cdf71fd..1fa3b9df0 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -100,9 +100,11 @@ def mode_str_to_int(modestr: Union[bytes, str]) -> int: module regarding the rwx permissions for user, group and other, special flags and file system flags, such as whether it is a symlink. """ + if isinstance(modestr, bytes): + # A byte is an int. int(b"4"[0]) is 52, so b"100644" became 0o6767524. + modestr = modestr.decode("ascii") mode = 0 for iteration, char in enumerate(reversed(modestr[-6:])): - char = cast(Union[str, int], char) mode += int(char) << iteration * 3 # END for each char return mode diff --git a/test/test_util.py b/test/test_util.py index c6e68cf51..46417cc39 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -22,6 +22,7 @@ from git.objects.util import ( altz_to_utctz_str, from_timestamp, + mode_str_to_int, parse_actor_and_date, parse_date, tzoffset, @@ -760,3 +761,10 @@ def test_remove_password_from_command_line(self): redacted_cmd_6 = remove_password_if_present(cmd_6) assert authorization not in " ".join(redacted_cmd_6) assert "http.extraHeader=Authorization: *****" in redacted_cmd_6 + + +def test_mode_str_to_int_accepts_bytes(): + assert mode_str_to_int("100644") == 0o100644 + assert mode_str_to_int(b"100644") == 0o100644 + assert mode_str_to_int("644") == 0o644 + assert mode_str_to_int(b"120000") == 0o120000 From f24ed41dea663785f60cc4f7d5ece240a7d88174 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:37:36 +0000 Subject: [PATCH 002/153] build(deps): bump https://github.com/astral-sh/ruff-pre-commit Bumps the pre-commit group with 1 update: [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.16.5 to 0.16.9 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.16.5...v0.16.9) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.16.9 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45115f60c..9471680e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: ^(?:gitdb|smmap)/ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.5 + rev: v0.16.9 hooks: - id: ruff-check args: ["--fix"] From 97eadb888d2f99567e3113f4e455be4ca128f697 Mon Sep 17 00:00:00 2001 From: Byron Date: Thu, 1 Oct 2026 19:17:13 +0200 Subject: [PATCH 003/153] fix(submodule): validate destinations before mutation Pretty much a rubber-stamp. It won't be out there long as the replacement with CLI + Gix is already on the way. Submodule checkout destinations could pass the containment check and be rejected by the index only after cloning had changed the filesystem. This addresses `GHSA-83vg-56qc-22m7` at the shared destination boundaries, including initialization and moves as well as creation. Reuse `_validate_repo_path` before checkout mutations to enforce portable NTFS/HFS metadata-alias checks and invalid-path rejection. Validate Windows filenames and submodule-name NULs before creating directories. Compare path components with `Repo.git_dir` and `Repo.common_dir` by filesystem identity, so separately named metadata directories and their aliases are protected too. Reject metadata destinations nested inside another submodule's Git directory before cloning, reuse, or renaming. Repeat the check after cloning and disable a clone that became nested. Preflight implicit metadata renames during moves, while preserving supported metadata symlinks and relocation of a submodule's own metadata directory. Git reference: `d38352cd43ab9745686d697872408bc3249a153f`, particularly `read-cache.c::verify_path_internal`, NTFS/HFS recognition, `compat/mingw.c::is_valid_win32_path`, and `submodule.c::validate_submodule_git_dir`. Related Git tests are in `t/t7450-bad-git-dotfiles.sh` and `t/t7406-submodule-update.sh`. Regression tests first demonstrated writes before rejection and acceptance of nested and separately named metadata destinations. Tests use harmless file content and compare portable aliases with native Git index validation. Coverage includes all 16 HFS ignored characters, Windows filename rules, relative and absolute paths, metadata reuse, and nesting during cloning. Assisted-by: GPT 6.0 Astra Co-authored-by: GPT 6.0 Astra --- doc/source/changes.rst | 13 ++ git/objects/submodule/base.py | 91 ++++++++++++-- test/test_submodule.py | 225 ++++++++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+), 11 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 2f537c99d..8447228db 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,19 @@ Changelog ========= +3.2.1 +===== + +Security fixes for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-83vg-56qc-22m7 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.2.1 + 3.2.0 ===== diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 15fe877e6..74fbd39fe 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -8,6 +8,7 @@ import ntpath import os import os.path as osp +import re import shlex import stat import sys @@ -47,6 +48,7 @@ IterableList, RemoteProgress, _to_relative_path, + _validate_repo_path, join_path_native, rmtree, to_native_path_linux, @@ -305,20 +307,55 @@ def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: def _validated_name(cls, name: str) -> str: if ( not name + or "\0" in name or name.startswith(("/", "\\")) or ntpath.splitdrive(name)[0] or ".." in name.replace("\\", "/").split("/") ): raise ValueError("Invalid submodule name %r" % name) + cls._validate_windows_path(name) return name + @staticmethod + def _validate_windows_path(path: PathLike) -> None: + """Apply Git for Windows' filename checks before creating directories.""" + if sys.platform == "win32": + for component in ntpath.splitdrive(os.fspath(path))[1].replace("\\", "/").split("/"): + if component in (".", ".."): + continue + stem = component.split(".", 1)[0].rstrip(" ").upper() + if ( + component.endswith((" ", ".")) + or any(ord(char) < 32 or char in '<>:"|?*' for char in component) + or re.fullmatch(r"CON(?:IN\$|OUT\$)?|PRN|AUX|NUL|COM[1-9]|LPT[1-9]", stem) + ): + raise ValueError("Invalid submodule path on Windows: %r" % path) + @classmethod - def _module_abspath(cls, parent_repo: "Repo", path: PathLike, name: str) -> PathLike: + def _module_abspath( + cls, parent_repo: "Repo", path: PathLike, name: str, *, moving_from: Union[PathLike, None] = None + ) -> PathLike: + """Reject nested Git directories, allowing the source of a pending rename.""" + from git.repo.fun import is_git_dir + name = cls._validated_name(name) if cls._need_gitfile_submodules(parent_repo.git): + directory = osp.join(parent_repo.git_dir, "modules") + for component in to_native_path_linux(name).split("/")[:-1]: + directory = osp.join(directory, component) + if is_git_dir(directory) and ( + moving_from is None or Path(directory).resolve() != Path(moving_from).resolve() + ): + raise ValueError( + "Submodule metadata for %r is inside another Git directory: %r" % (name, directory) + ) return osp.join(parent_repo.git_dir, "modules", name) if parent_repo.working_tree_dir: - return cls._checked_abspath(parent_repo.working_tree_dir, cls._to_relative_path(parent_repo, path)) + return cls._checked_abspath( + parent_repo.working_tree_dir, + cls._to_relative_path(parent_repo, path), + git_dirs=(parent_repo.git_dir, parent_repo.common_dir), + ) raise NotADirectoryError() @classmethod @@ -360,7 +397,9 @@ def _clone_repo( path = cls._to_relative_path(repo, path) if repo.working_tree_dir is None: raise NotADirectoryError("Submodules require a working tree") - module_checkout_path = cls._checked_abspath(repo.working_tree_dir, path) + module_checkout_path = cls._checked_abspath( + repo.working_tree_dir, path, git_dirs=(repo.git_dir, repo.common_dir) + ) module_abspath = cls._module_abspath(repo, path, name) if cls._need_gitfile_submodules(repo.git): if not allow_unsafe_options: @@ -402,6 +441,16 @@ def _clone_repo( **kwargs, ) if cls._need_gitfile_submodules(repo.git): + # A concurrent clone may have turned a leading directory into a repository. + try: + cls._module_abspath(repo, path, name) + except ValueError: + clone.close() + try: + os.remove(osp.join(clone.git_dir, "HEAD")) + except FileNotFoundError: + pass + raise cls._write_git_file_and_module_config(module_checkout_path, module_abspath) return clone @@ -411,8 +460,9 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: """:return: A path guaranteed to be relative to the given parent repository :raise ValueError: - If path is not contained in the parent repository's working tree. + If path is outside the working tree or is unsafe as a submodule checkout. """ + cls._validate_windows_path(path) if parent_repo.working_tree_dir: path = _to_relative_path(parent_repo.working_tree_dir, path) else: @@ -422,6 +472,7 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: if not path or path == ".": raise ValueError("Submodule checkout path must not be the repository root") + _validate_repo_path(path) return path @property @@ -433,22 +484,35 @@ def abspath(self) -> PathLike: def _checkout_abspath(self, relative_path: PathLike, allow_final_symlink: bool = False) -> PathLike: """Check a checkout path already normalized by :meth:`_to_relative_path`.""" - return self._checked_abspath(self.repo.working_tree_dir, relative_path, allow_final_symlink) + return self._checked_abspath( + self.repo.working_tree_dir, + relative_path, + allow_final_symlink, + git_dirs=(self.repo.git_dir, self.repo.common_dir), + ) @classmethod def _checked_abspath( - cls, root: Union[PathLike, None], relative_path: PathLike, allow_final_symlink: bool = False + cls, + root: Union[PathLike, None], + relative_path: PathLike, + allow_final_symlink: bool = False, + *, + git_dirs: Sequence[PathLike] = (), ) -> str: - """Reject symlinks below a trusted root before accessing submodule paths.""" + """Reject symlinks and checkout aliases of Git directories below a trusted root.""" if root is None: raise NotADirectoryError("Submodules require a working tree") path = os.fspath(root) + metadata_dirs = set(git_dirs) components = to_native_path_linux(relative_path).split("/") for index, component in enumerate(components): path = os.fspath(join_path_native(path, component)) - if allow_final_symlink and index == len(components) - 1: - break + if metadata_dirs and osp.exists(path) and any(osp.samefile(path, directory) for directory in metadata_dirs): + raise ValueError("Submodule checkout path aliases Git metadata: %r" % relative_path) if osp.islink(path): + if allow_final_symlink and index == len(components) - 1: + break raise ValueError("Submodule path %r contains a symbolic link" % relative_path) return path @@ -1122,6 +1186,11 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = self._checked_abspath(self.repo.working_tree_dir, self.k_modules_file) # Validate the source before removing the destination. cur_path = self.abspath + module_abspath = self._module_abspath(self.repo, self.path, self.name) + if self.path == self.name: + self._module_abspath( + self.repo, module_checkout_path, os.fspath(module_checkout_path), moving_from=module_abspath + ) module_checkout_abspath = self._checkout_abspath(module_checkout_path, allow_final_symlink=True) if osp.isfile(module_checkout_abspath): raise ValueError("Cannot move repository onto a file: %s" % module_checkout_abspath) @@ -1160,7 +1229,6 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = renamed_module = True if osp.isfile(osp.join(module_checkout_abspath, ".git")): - module_abspath = self._module_abspath(self.repo, self.path, self.name) self._write_git_file_and_module_config(module_checkout_abspath, module_abspath) # END handle git file rewrite # END move physical module @@ -1522,8 +1590,8 @@ def rename(self, new_name: str) -> "Submodule": self._validated_name(self.name) self._validated_name(new_name) - destination_module_abspath = self._module_abspath(self.repo, self.path, new_name) mod = self.module() + destination_module_abspath = self._module_abspath(self.repo, self.path, new_name, moving_from=mod.git_dir) self._checked_abspath(self.repo.working_tree_dir, self.k_modules_file) # .git/config @@ -1573,6 +1641,7 @@ def module(self) -> "Repo": """ self._validated_name(self.name) module_checkout_abspath = self.abspath + self._module_abspath(self.repo, self.path, self.name) try: repo = git.Repo(module_checkout_abspath) if repo != self.repo: diff --git a/test/test_submodule.py b/test/test_submodule.py index b1f4f5156..45b285c55 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -86,6 +86,7 @@ def test_submodule_update_preserves_literal_name(tmp_path, monkeypatch, caplog, def movable_submodule(tmp_path): """Create a committed local submodule whose logical name stays fixed when moved.""" with git.Repo.init(tmp_path / "source") as source, git.Repo.init(tmp_path / "parent") as parent: + source.git.symbolic_ref("HEAD", "refs/heads/master") (tmp_path / "source" / "file").write_text("content", encoding="utf-8") source.index.add(["file"]) source.index.commit("Create source") @@ -111,6 +112,207 @@ def _move_snapshot(submodule): ) +@pytest.mark.parametrize( + "path", + [ + ".git/child", + ".GiT/child", + "nested/.git/child", + "git~1/child", + "GIT~1 . /child", + ".git. /child", + ".git:stream/child", + ".git::$INDEX_ALLOCATION/child", + "nested\\.git\\child", + "C:relative", + "nul\0name", + ] + + [ + f".g{chr(codepoint)}it/child" + for codepoint in (*range(0x200C, 0x2010), *range(0x202A, 0x202F), *range(0x206A, 0x2070), 0xFEFF) + ], +) +@pytest.mark.parametrize("operation", ["add", "clone", "update", "move", "move-module", "move-config"]) +def test_submodule_rejects_unsafe_checkout_before_mutation(movable_submodule, tmp_path, path, operation): + sm = movable_submodule + root = Path(sm.repo.working_tree_dir) + before = _move_snapshot(sm) + paths = set(root.rglob("*")) + # Check the portable metadata aliases against Git's index validation as well. + if operation == "clone" and path not in ("C:relative", "nul\0name"): + with sm.repo.git.custom_environment(GIT_INDEX_FILE=str(tmp_path / "validation-index")): + with _patch_git_config("core.protectHFS", "true"), _patch_git_config("core.protectNTFS", "true"): + with pytest.raises(GitCommandError): + sm.repo.git.update_index("--add", "--cacheinfo", f"160000,{sm.hexsha},{path}") + with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("clone attempted")): + with pytest.raises(ValueError): + if operation == "add": + sm.repo.create_submodule("new", path, sm.url) + elif operation == "clone": + Submodule._clone_repo(sm.repo, sm.url, path, "new") + elif operation == "update": + Submodule(sm.repo, sm.binsha, name="new", path=path, url=sm.url).update(init=True) + else: + sm.move(path, configuration=operation != "move-module", module=operation != "move-config") + assert _move_snapshot(sm) == before + assert set(root.rglob("*")) == paths + + +def test_add_rejects_metadata_checkout_without_writing_files(movable_submodule): + sm = movable_submodule + before = _move_snapshot(sm) + with pytest.raises(ValueError, match="Git metadata"): + sm.repo.create_submodule("new", ".git/new", sm.url) + assert not Path(sm.repo.git_dir, "new").exists() + assert not Path(sm.repo.git_dir, "modules/new").exists() + assert _move_snapshot(sm) == before + + +@pytest.mark.parametrize("absolute_path", [False, True]) +@pytest.mark.parametrize("metadata_name", ["metadata", "MeTaDaTa"]) +@pytest.mark.parametrize("operation", ["add", "clone", "update", "move"]) +def test_submodule_rejects_checkout_in_separate_metadata( + movable_submodule, tmp_path, absolute_path, metadata_name, operation +): + root = tmp_path / "separate" + with git.Repo.init(root, separate_git_dir=str(root / "metadata"), allow_unsafe_options=True) as parent: + if not (root / metadata_name).is_dir(): + pytest.skip("Requires a case-insensitive filesystem") + sm = parent.create_submodule("module", "module", movable_submodule.url) + parent.index.commit("Add submodule") + before = _move_snapshot(sm) + paths = set(root.rglob("*")) + path = root / metadata_name / "new" if absolute_path else f"{metadata_name}/new" + with pytest.raises(ValueError, match="Git metadata"): + if operation == "add": + parent.create_submodule("new", path, sm.url) + elif operation == "clone": + Submodule._clone_repo(parent, sm.url, path, "new") + elif operation == "update": + Submodule(parent, sm.binsha, name="new", path=path, url=sm.url).update(init=True) + else: + sm.move(path) + assert _move_snapshot(sm) == before + assert set(root.rglob("*")) == paths + + +@pytest.mark.parametrize("operation", ["add", "clone", "update", "rename", "move"]) +def test_submodule_rejects_nested_metadata_before_mutation(movable_submodule, operation): + sm = movable_submodule + other = sm.repo.create_submodule("other", "other", sm.url) + other.module().close() + root = Path(sm.repo.working_tree_dir) + before = _move_snapshot(sm), _move_snapshot(other) + paths = set(root.rglob("*")) + name = f"{sm.name}/child" + with pytest.raises(ValueError, match="inside.*Git directory"): + if operation == "add": + sm.repo.create_submodule(name, "new", sm.url) + elif operation == "clone": + Submodule._clone_repo(sm.repo, sm.url, "new", name) + elif operation == "update": + Submodule(sm.repo, sm.binsha, name=name, path="new", url=sm.url).update(init=True) + elif operation == "rename": + other.rename(name) + else: + other.move(name) + assert (_move_snapshot(sm), _move_snapshot(other)) == before + assert set(root.rglob("*")) == paths + + +@pytest.mark.parametrize("state", ["retained", "checked-out"]) +def test_update_rejects_existing_nested_metadata(movable_submodule, state): + sm = movable_submodule + name = f"{sm.name}/child" + checkout = Path(sm.repo.working_tree_dir, "new") + with git.Repo.clone_from( + sm.url, checkout, separate_git_dir=str(Path(sm.repo.git_dir, "modules", name)), allow_unsafe_options=True + ): + pass + if state == "retained": + shutil.rmtree(checkout) + before = _move_snapshot(sm) + paths = set(Path(sm.repo.working_tree_dir).rglob("*")) + with pytest.raises(ValueError, match="inside.*Git directory"): + Submodule(sm.repo, sm.binsha, name=name, path="new", url=sm.url).update(init=True, no_fetch=True) + assert _move_snapshot(sm) == before + assert set(Path(sm.repo.working_tree_dir).rglob("*")) == paths + + +@pytest.mark.parametrize( + "path", + [ + "CON", + "con.txt", + "CONIN$", + "conout$.txt", + "AUX .txt", + "PRN", + "NUL", + "COM9", + "LPT1", + "name:stream", + "space ", + "period.", + "line\nbreak", + "star*", + 'quote"', + "question?", + "angle<", + "angle>", + "pipe|", + ], +) +def test_submodule_rejects_windows_destination_names_before_mutation(movable_submodule, path): + sm = movable_submodule + before = _move_snapshot(sm) + paths = set(Path(sm.repo.working_tree_dir).rglob("*")) + with mock.patch("git.objects.submodule.base.sys", SimpleNamespace(platform="win32")): + with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("clone attempted")): + with pytest.raises(ValueError): + sm.repo.create_submodule("new", f"nested/{path}", sm.url) + with pytest.raises(ValueError): + sm.repo.create_submodule(f"nested/{path}", "new", sm.url) + assert _move_snapshot(sm) == before + assert set(Path(sm.repo.working_tree_dir).rglob("*")) == paths + + +@pytest.mark.parametrize("path", ["nested/space ", "nested/period."]) +def test_windows_destination_validation_precedes_normalization(tmp_path, path): + parent = SimpleNamespace(working_tree_dir=str(tmp_path)) + # Windows' GetFullPathName removes trailing spaces and periods. + with mock.patch("git.objects.submodule.base._to_relative_path", return_value=path.rstrip(" .")): + with mock.patch("git.objects.submodule.base.sys", SimpleNamespace(platform="win32")): + with pytest.raises(ValueError, match="Invalid submodule path on Windows"): + Submodule._to_relative_path(parent, path) + + +def test_submodule_can_relocate_its_own_metadata(movable_submodule): + sm = movable_submodule + sm.rename(f"{sm.name}/child") + assert Path(sm.abspath, "file").read_text() == "content" + with sm.module() as module: + assert Path(module.git_dir) == Path(sm.repo.git_dir, "modules", sm.name) + + +def test_clone_disables_metadata_that_becomes_nested(movable_submodule, monkeypatch): + sm = movable_submodule + clone_from = git.Repo.clone_from + ancestor = Path(sm.repo.git_dir, "modules/new") + + def clone_and_create_ancestor(*args, **kwargs): + clone = clone_from(*args, **kwargs) + with git.Repo.init(ancestor, bare=True): + pass + return clone + + monkeypatch.setattr(git.Repo, "clone_from", clone_and_create_ancestor) + with pytest.raises(ValueError, match="inside.*Git directory"): + Submodule._clone_repo(sm.repo, sm.url, "new", "new/child") + assert not (ancestor / "child/HEAD").exists() + assert (ancestor / "HEAD").is_file() + + @pytest.mark.parametrize("target_kind", ["relative", "absolute", "internal", "dangling"]) @pytest.mark.parametrize("configuration,module", [(True, True), (False, True), (True, False)]) @pytest.mark.parametrize("absolute_path", [False, True]) @@ -167,6 +369,28 @@ def test_move_normal_destination(movable_submodule, absolute_path): assert _move_snapshot(submodule) == before +@pytest.mark.parametrize("metadata_dir", ["git_dir", "common_dir"]) +def test_move_rejects_leaf_symlink_to_metadata(movable_submodule, tmp_path, metadata_dir): + root = tmp_path / "worktree" + movable_submodule.repo.git.worktree("add", "--detach", str(root)) + with git.Repo(root) as parent: + assert not osp.samefile(parent.git_dir, parent.common_dir) + submodule = parent.submodules[0] + submodule.update(init=True) + target = Path(getattr(parent, metadata_dir)) + destination = root / "destination" + destination.symlink_to(target, target_is_directory=True) + before = _move_snapshot(submodule) + + with pytest.raises(ValueError, match="Git metadata"): + submodule.move("destination", module=False) + + assert _move_snapshot(submodule) == before + assert destination.is_symlink() + assert destination.samefile(target) + assert Path(submodule.abspath, "file").read_text(encoding="utf-8") == "content" + + @pytest.mark.parametrize("kind", ["empty", "nonempty", "file", "dangling"]) def test_move_leaf_symlink_compatibility(movable_submodule, tmp_path, kind): """Preserve leaf-symlink replacement without modifying the external target. @@ -1410,6 +1634,7 @@ def test_update_rejects_parent_component_in_name(self, rwdir): invalid_names = ( "", + "nul\0name", "..", "../module", R"..\module", From b0ae041305ec342625b1398a673d5b8ada946fa5 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Fri, 2 Oct 2026 09:22:41 +0530 Subject: [PATCH 004/153] fix(util): create lock files in one exclusive step `LockFile._obtain_lock_or_raise()` tested for the lock with `osp.isfile()` and then created it with `open(lock_file, "w")`. Nothing keeps another holder out between the two calls, so several can pass the test and all of them set `_owns_lock`. Racing eight holders on one lock file leaves seven believing they own it, which removes the mutual exclusion that `GitConfigParser` in write mode and `RefLog.append_entry()` rely on. `osp.isfile()` also resolves symbolic links. A dangling symlink planted at `.lock` therefore reports no lock, and the following `open()` resolves it and creates the target, outside the repository. Create the lock with `os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)` instead. That is the single-step exclusive create `gitdb`'s `LockedFD.open()` and Git's own `lock_file()` already use, and `O_EXCL` fails with `EEXIST` on a symbolic link rather than resolving it, so both problems close together. The creation mode matches `LockedFD`; nothing reads a lock file's contents, and breaking a stale lock needs write permission on the containing directory rather than on the file. `FileExistsError` is translated back into the existing "did already exist" `OSError`, so `BlockingLockFile`'s retry loop and the existing `test_lock_file` and `test_blocking_lock_file` cases are unaffected. A directory at the lock path is still reported through the generic `OSError` branch. This covers acquiring the lock only; writing the locked file stays the caller's concern, as before. Adds `test_lock_file_does_not_follow_a_symlink` and `test_lock_file_is_obtained_by_a_single_holder`. Both fail on the previous code (`1 != 7`, and the symlink target gets created) and pass here. Validation: full `pytest` suite green on Python 3.11 on macOS, plus `ruff check`, `ruff format`, `mypy` and `basedpyright --warnings` clean. --- git/util.py | 15 +++++++++------ test/test_util.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/git/util.py b/git/util.py index 6f1f38401..ca942340d 100644 --- a/git/util.py +++ b/git/util.py @@ -1166,17 +1166,20 @@ def _obtain_lock_or_raise(self) -> None: if self._has_lock(): return lock_file = self._lock_file_path() - if osp.isfile(lock_file): + # Create the lock in one step, the way Git and gitdb's LockedFD do. Testing + # for the file first leaves a window in which another holder creates it and + # both proceed, and O_CREAT|O_EXCL additionally refuses to follow a symbolic + # link planted at the lock path instead of writing through it. + try: + fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as e: raise OSError( "Lock for file %r did already exist, delete %r in case the lock is illegal" % (self._file_path, lock_file) - ) - - try: - with open(lock_file, mode="w"): - pass + ) from e except OSError as e: raise OSError(str(e)) from e + os.close(fd) self._owns_lock = True diff --git a/test/test_util.py b/test/test_util.py index 46417cc39..203e3a54f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import threading import time from unittest import SkipTest, mock @@ -446,6 +447,44 @@ def test_lock_file(self): lock_file._obtain_lock_or_raise() lock_file._release_lock() + @requires_symlinks + def test_lock_file_does_not_follow_a_symlink(self): + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + outside = os.path.join(tdir, "outside-the-lock") + os.symlink(outside, my_file + ".lock") + + lock_file = LockFile(my_file) + self.assertRaises(IOError, lock_file._obtain_lock_or_raise) + assert not lock_file._has_lock() + assert not os.path.exists(outside) + + def test_lock_file_is_obtained_by_a_single_holder(self): + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + racers = 8 + at_the_line = threading.Barrier(racers) + holders = [] + guard = threading.Lock() + + def obtain(): + lock_file = LockFile(my_file) + at_the_line.wait() + try: + lock_file._obtain_lock_or_raise() + except OSError: + return + with guard: + holders.append(lock_file) + + threads = [threading.Thread(target=obtain) for _ in range(racers)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + self.assertEqual(1, len(holders)) + def test_blocking_lock_file(self): with tempfile.TemporaryDirectory() as tdir: my_file = os.path.join(tdir, "my-lock-file") From 97a546899df8b41496c71c128b9b4154bacc5fe5 Mon Sep 17 00:00:00 2001 From: Byron Date: Fri, 2 Oct 2026 04:52:11 +0000 Subject: [PATCH 005/153] fix(util): acquire Windows locks without following symlinks Took brief look only. Hope this code soon won't be present anymore. Windows follows dangling symlinks even when `os.open` uses `O_CREAT | O_EXCL`, so acquiring a lock could create its symlink target and incorrectly report ownership. The `_winapi.CreateFile` workaround also uses the ANSI API on Python 3.8 through 3.10, causing failures in Unicode directories or creating locks under mangled filenames. Use `CreateFileW` with `CREATE_NEW` and `FILE_FLAG_OPEN_REPARSE_POINT` to create the lock atomically while rejecting existing links. Declare the `ctypes` argument and return types explicitly so Unicode paths and native handle sizes are preserved. Close the handle before recording ownership, propagate Windows errors, and reject embedded NULs before the native API can truncate a path. Preserve exclusive `os.open` creation on POSIX. Expand the lock tests to cover Unicode filenames and directories, including non-BMP characters, and verify that the requested lock path is actually created and removed. Check NUL rejection, preserve both existing and missing symlink targets, and explicitly release the concurrent test's acquired locks. Reproduced the CI failure in `test_clone_from_with_path_contains_unicode` on Windows/Python 3.8.10 before the fix. The affected utility, clone, and configuration modules pass on Python 3.8.10 and 3.13.14: 169 passed, 41 skipped, and 2 expected failures on each version. The new Unicode and NUL regressions also failed before their respective fixes. `ruff check`, `ruff format --check`, `mypy --python-version=3.13`, and `basedpyright --warnings` pass. Assisted-by: GPT 6.0 Astra Co-authored-by: GPT 6 --- git/util.py | 50 +++++++++++++++++++++++++++++++++++++++++------ test/test_util.py | 37 ++++++++++++++++++++++++++++++----- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/git/util.py b/git/util.py index ca942340d..017a94ad3 100644 --- a/git/util.py +++ b/git/util.py @@ -1166,12 +1166,51 @@ def _obtain_lock_or_raise(self) -> None: if self._has_lock(): return lock_file = self._lock_file_path() - # Create the lock in one step, the way Git and gitdb's LockedFD do. Testing - # for the file first leaves a window in which another holder creates it and - # both proceed, and O_CREAT|O_EXCL additionally refuses to follow a symbolic - # link planted at the lock path instead of writing through it. + # Create the lock in one step. Checking for it first would allow another + # holder to create it between the check and the open. try: - fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + if sys.platform == "win32": + if "\0" in lock_file: + raise ValueError("embedded null character") + + import ctypes + from ctypes import wintypes + + # Unlike POSIX, Windows follows dangling symlinks even with O_EXCL. + # Open the reparse point itself so an existing link is rejected. + # Call the Unicode API directly: older _winapi.CreateFile wrappers + # use the ANSI API and can create a lock under the wrong filename. + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + create_file = kernel32.CreateFileW + create_file.argtypes = ( + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ) + create_file.restype = wintypes.HANDLE + close_handle = kernel32.CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + handle = create_file( + lock_file, + 0x40000000, # GENERIC_WRITE + 0, + None, + 1, # CREATE_NEW + 0x00200000, # FILE_FLAG_OPEN_REPARSE_POINT + None, + ) + if handle == wintypes.HANDLE(-1).value: + raise ctypes.WinError(ctypes.get_last_error()) + if not close_handle(handle): + raise ctypes.WinError(ctypes.get_last_error()) + else: + fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(fd) except FileExistsError as e: raise OSError( "Lock for file %r did already exist, delete %r in case the lock is illegal" @@ -1179,7 +1218,6 @@ def _obtain_lock_or_raise(self) -> None: ) from e except OSError as e: raise OSError(str(e)) from e - os.close(fd) self._owns_lock = True diff --git a/test/test_util.py b/test/test_util.py index 203e3a54f..eb520aac6 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -419,9 +419,11 @@ def test_it_should_dashify(self): self.assertEqual("this-is-my-argument", dashify("this_is_my_argument")) self.assertEqual("foo", dashify("foo")) - def test_lock_file(self): + @ddt.data("my-lock-file", "my-lock-file-\u0394", "\u0394/my-lock-file", "\U0001f680/my-lock-file") + def test_lock_file(self, filename): with tempfile.TemporaryDirectory() as tdir: - my_file = os.path.join(tdir, "my-lock-file") + my_file = os.path.join(tdir, filename) + os.makedirs(os.path.dirname(my_file), exist_ok=True) lock_file = LockFile(my_file) assert not lock_file._has_lock() # Release lock we don't have - fine. @@ -430,6 +432,7 @@ def test_lock_file(self): # Get lock. lock_file._obtain_lock_or_raise() assert lock_file._has_lock() + assert os.path.isfile(my_file + ".lock") # Concurrent access. other_lock_file = LockFile(my_file) @@ -438,6 +441,7 @@ def test_lock_file(self): lock_file._release_lock() assert not lock_file._has_lock() + assert not os.path.exists(my_file + ".lock") other_lock_file._obtain_lock_or_raise() self.assertRaises(IOError, lock_file._obtain_lock_or_raise) @@ -447,17 +451,36 @@ def test_lock_file(self): lock_file._obtain_lock_or_raise() lock_file._release_lock() + def test_lock_file_rejects_embedded_nul(self): + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + lock_file = LockFile(my_file + "\0suffix") + self.assertRaises(ValueError, lock_file._obtain_lock_or_raise) + assert not lock_file._has_lock() + assert not os.path.exists(my_file) + + @ddt.data(False, True) @requires_symlinks - def test_lock_file_does_not_follow_a_symlink(self): + def test_lock_file_does_not_follow_a_symlink(self, target_exists): with tempfile.TemporaryDirectory() as tdir: my_file = os.path.join(tdir, "my-lock-file") outside = os.path.join(tdir, "outside-the-lock") + content = b"Do not modify the symlink target." + if target_exists: + with open(outside, "wb") as stream: + stream.write(content) os.symlink(outside, my_file + ".lock") lock_file = LockFile(my_file) self.assertRaises(IOError, lock_file._obtain_lock_or_raise) assert not lock_file._has_lock() - assert not os.path.exists(outside) + lock_file._release_lock() + assert os.path.islink(my_file + ".lock") + if target_exists: + with open(outside, "rb") as stream: + self.assertEqual(stream.read(), content) + else: + assert not os.path.exists(outside) def test_lock_file_is_obtained_by_a_single_holder(self): with tempfile.TemporaryDirectory() as tdir: @@ -483,7 +506,11 @@ def obtain(): for thread in threads: thread.join() - self.assertEqual(1, len(holders)) + try: + self.assertEqual(1, len(holders)) + finally: + for lock_file in holders: + lock_file._release_lock() def test_blocking_lock_file(self): with tempfile.TemporaryDirectory() as tdir: From 226654af2d560b2de87f83f9c86eb9a3312c9496 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Sat, 3 Oct 2026 00:27:34 +0530 Subject: [PATCH 006/153] fix(commit): accumulate gpgsig continuation lines in linear time `Commit._deserialize` collected a `gpgsig` header by appending each continuation line to a `bytes` object with `+=`. `bytes` are immutable, so every line copied the whole signature read so far, and a signature with *n* continuation lines cost O(n^2) to parse. Commit headers are fully controlled by whoever wrote the commit, and `_deserialize` runs on the first access to any commit attribute (`author`, `message`, `parents`, ...). Reading the commits of an untrusted repository is therefore enough to hit it. Measured on Python 3.11 with a `gpgsig` header of `" x\n"` lines: | object size | before | after | | ----------- | ------- | ------ | | 2.4 MB | 9.5 s | 0.08 s | | 4.8 MB | 116.8 s | | | 9.6 MB | 547.9 s | 0.32 s | Collect the lines in a list and join them once. This is linear and produces byte-identical `gpgsig` values, so the existing `test_gpgsig` round trip is unchanged. `test_gpgsig_deserialization_is_linear` deserializes a 1.8 MB signature under a 1 s CPU-time bound. It fails on the previous code and passes here. The full suite has no new failures; the 8 tests that fail locally (non-UTF-8 trailer encoding and NUL submodule names) fail identically without this change. `ruff`, `mypy` and `basedpyright --warnings` are clean. --- git/objects/commit.py | 6 +++--- test/test_commit.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 771f87976..307b8c59b 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -885,7 +885,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": if buf[0:10] == b"encoding ": self.encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") elif buf[0:7] == b"gpgsig ": - sig = buf[buf.find(b" ") + 1 :] + b"\n" + sig_lines = [buf[buf.find(b" ") + 1 :] + b"\n"] is_next_header = False while True: sigbuf = readline() @@ -895,9 +895,9 @@ def _deserialize(self, stream: BytesIO) -> "Commit": buf = sigbuf.strip() is_next_header = True break - sig += sigbuf[1:] + sig_lines.append(sigbuf[1:]) # END read all signature - self.gpgsig = sig.rstrip(b"\n").decode(self.encoding, "ignore") + self.gpgsig = b"".join(sig_lines).rstrip(b"\n").decode(self.encoding, "ignore") if is_next_header: continue buf = readline().strip() diff --git a/test/test_commit.py b/test/test_commit.py index b5baf0e33..0a5bcdb7d 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -607,6 +607,28 @@ def test_commit_co_authors_bounds_malformed_trailer(self): # The malformed line yields nothing; the well-formed trailer still parses. assert result == [Actor("Real Name", "real@example.com")] + def test_gpgsig_deserialization_is_linear(self): + """A long gpgsig header must not make deserialization run in quadratic time.""" + num_lines = 600_000 + # Commit headers are fully attacker-controlled. Accumulating the signature with + # bytes concatenation copied it once per continuation line (O(n^2)). + data = ( + b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + b"author A 1700000000 +0000\n" + b"committer A 1700000000 +0000\n" + b"gpgsig -----BEGIN PGP SIGNATURE-----\n" + b" x\n" * num_lines + b" -----END PGP SIGNATURE-----\n" + b"\n" + b"message\n" + ) + cmt = copy.copy(self.rorepo.commit()) + start = time.process_time() + cmt._deserialize(BytesIO(data)) + elapsed = time.process_time() - start + # Leave ample CPU time for slow runners, but catch quadratic accumulation. + self.assertLess(elapsed, 1.0) + self.assertEqual(cmt.gpgsig.count("\n"), num_lines + 1) + self.assertEqual(cmt.message, "message\n") + @with_rw_directory def test_create_from_tree_with_trailers_dict(self, rw_dir): """Test that create_from_tree supports adding trailers via a dict.""" From 5fc5f3456a6816d4caa03fdb4a3b000d3170a290 Mon Sep 17 00:00:00 2001 From: Byron Date: Sat, 3 Oct 2026 04:31:59 +0200 Subject: [PATCH 007/153] review - remove test as it can totally fail on slow runners, and extending the time budget makes it hard to see if it's actually working. Let's trust what we see and look forward to a time when the in-python parsing is gone as well. --- test/test_commit.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test/test_commit.py b/test/test_commit.py index 0a5bcdb7d..b5baf0e33 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -607,28 +607,6 @@ def test_commit_co_authors_bounds_malformed_trailer(self): # The malformed line yields nothing; the well-formed trailer still parses. assert result == [Actor("Real Name", "real@example.com")] - def test_gpgsig_deserialization_is_linear(self): - """A long gpgsig header must not make deserialization run in quadratic time.""" - num_lines = 600_000 - # Commit headers are fully attacker-controlled. Accumulating the signature with - # bytes concatenation copied it once per continuation line (O(n^2)). - data = ( - b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" - b"author A 1700000000 +0000\n" - b"committer A 1700000000 +0000\n" - b"gpgsig -----BEGIN PGP SIGNATURE-----\n" + b" x\n" * num_lines + b" -----END PGP SIGNATURE-----\n" - b"\n" - b"message\n" - ) - cmt = copy.copy(self.rorepo.commit()) - start = time.process_time() - cmt._deserialize(BytesIO(data)) - elapsed = time.process_time() - start - # Leave ample CPU time for slow runners, but catch quadratic accumulation. - self.assertLess(elapsed, 1.0) - self.assertEqual(cmt.gpgsig.count("\n"), num_lines + 1) - self.assertEqual(cmt.message, "message\n") - @with_rw_directory def test_create_from_tree_with_trailers_dict(self, rw_dir): """Test that create_from_tree supports adding trailers via a dict.""" From 6795806a4d02781d7150fc71e27b7e778bb65b55 Mon Sep 17 00:00:00 2001 From: kokotatan Date: Sat, 3 Oct 2026 23:45:30 +0900 Subject: [PATCH 008/153] fix(index): accept a single PathLike in checkout `IndexFile.checkout()` documents accepting a single path, but only wraps strings before iterating. Passing one `pathlib.Path` or custom `os.PathLike` therefore raises `TypeError`, while a list containing the same object already works. Wrap single `os.PathLike` objects along with strings and widen the public annotation. Keep existing path normalization and iterable behavior. Add 36 real-repository cases covering path types, absolute and relative paths, files and directories, and single/list/iterator inputs. Eight single-PathLike cases failed before this fix. The focused tests pass, as do all pre-commit hooks, `mypy`, `basedpyright`, and strict Sphinx HTML. The full suite has 1,410 passing tests and 121 Windows symlink-privilege failures, all reproduced with identical node IDs on the pristine base. Assisted-by: OpenAI GPT-6 --- git/index/base.py | 4 ++-- test/test_index.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index c9d99840d..4ecd0c5b1 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1303,7 +1303,7 @@ def _flush_stdin_and_wait(self, proc: "Popen[bytes]", ignore_stdout: bool = Fals @default_index def checkout( self, - paths: Union[None, Iterable[PathLike]] = None, + paths: Union[None, PathLike, Iterable[PathLike]] = None, force: bool = False, fprogress: Callable = lambda *args: None, allow_unsafe_options: bool = False, @@ -1442,7 +1442,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik handle_stderr(proc, rval_iter) return rval_iter else: - if isinstance(paths, str): + if isinstance(paths, (str, os.PathLike)): paths = [paths] # Make sure we have our entries loaded before we start checkout_index, which diff --git a/test/test_index.py b/test/test_index.py index 8c33fe29d..30cb54565 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1731,6 +1731,32 @@ def test_index_file_v3_with_git_command(self, tmp_dir): assert "A file2.txt" in status_lines +class TestIndexCheckout: + @pytest.mark.parametrize("path_type", [str, Path, PathLikeMock]) + @pytest.mark.parametrize("absolute", [False, True]) + @pytest.mark.parametrize("directory", [False, True]) + @pytest.mark.parametrize("container", ["single", "list", "iterator"]) + def test_checkout_pathlike(self, tmp_path, path_type, absolute, directory, container): + with Repo.init(tmp_path) as repo: + nested = tmp_path / "nested" + nested.mkdir() + files = {"nested/first": b"first", "nested/second": b"second", "outside": b"outside"} + for name, data in files.items(): + (tmp_path / name).write_bytes(data) + repo.index.add(list(files)) + + path_name = "nested" if directory else "nested/first" + path = path_type(str(tmp_path / path_name) if absolute else path_name) + paths = path if container == "single" else [path] if container == "list" else iter([path]) + expected = {"nested/first", "nested/second"} if directory else {"nested/first"} + for name in expected: + (tmp_path / name).unlink() + + assert set(repo.index.checkout(paths)) == expected + for name, data in files.items(): + assert (tmp_path / name).read_bytes() == data + + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) def test_temporary_file_swap(self, tmp_path, file_path_type): From 6fdfa315406170a0fbc4898123e0bfda67cb9070 Mon Sep 17 00:00:00 2001 From: Yunare Maia Date: Mon, 5 Oct 2026 02:14:59 +0000 Subject: [PATCH 009/153] fix(util): redact URL credentials without corrupting the host remove_password_if_present() redacted credentials by substituting them into url.netloc with str.replace(), which rewrites every occurrence anywhere in the netloc -- including the host. https://git@github.com/user/repo.git -> https://*****@*****hub.com/user/repo.git "git" is the most common Git username and a substring of "github.com", so this hits the common case. An empty username or password makes it worse: str.replace("", "*****") matches at every position, so https://:token@fakerepo.example.com/testrepo comes back shredded into a URL roughly ten times longer, which is what lands in GitCommandError messages when a fetch fails. Rebuild the netloc from its userinfo instead, so only the credentials are replaced. The existing test already covered the empty-password case but only asserted the password is absent, which any mangling satisfies. The function is the single redaction path used for logged command lines and for exception messages (git/cmd.py, git/exc.py, git/repo/base.py), so this fixes all of them at once. --- git/util.py | 12 ++++++++---- test/test_util.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/git/util.py b/git/util.py index 017a94ad3..deebdc8d0 100644 --- a/git/util.py +++ b/git/util.py @@ -658,10 +658,14 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: if url.password is None and url.username is None: continue - if url.password is not None: - url = url._replace(netloc=url.netloc.replace(url.password, "*****")) - if url.username is not None: - url = url._replace(netloc=url.netloc.replace(url.username, "*****")) + # Redact the userinfo as a whole rather than substituting the + # username/password into the netloc: a substring replace also hits + # the host ("git" in "github.com") and an empty username or password + # matches at every position. + _, at, hostinfo = url.netloc.rpartition("@") + if at: + redacted = "*****:*****" if url.password is not None else "*****" + url = url._replace(netloc=f"{redacted}@{hostinfo}") new_cmdline[index] = urlunsplit(url) except ValueError: # This is not a valid URL. diff --git a/test/test_util.py b/test/test_util.py index eb520aac6..5b72cb432 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -828,6 +828,19 @@ def test_remove_password_from_command_line(self): assert authorization not in " ".join(redacted_cmd_6) assert "http.extraHeader=Authorization: *****" in redacted_cmd_6 + def test_remove_password_keeps_host_intact(self): + """Redaction must not touch the host, even when it contains the username.""" + redacted = remove_password_if_present(["git", "clone", "https://git@github.com/user/repo.git"]) + assert redacted == ["git", "clone", "https://*****@github.com/user/repo.git"] + + redacted = remove_password_if_present(["git", "clone", "ssh://git@github.com/u/r.git"]) + assert redacted == ["git", "clone", "ssh://*****@github.com/u/r.git"] + + def test_remove_empty_password_keeps_host_intact(self): + """An empty password must not expand into every position of the netloc.""" + redacted = remove_password_if_present(["git", "clone", "https://:@fakerepo.example.com/testrepo"]) + assert redacted == ["git", "clone", "https://*****:*****@fakerepo.example.com/testrepo"] + def test_mode_str_to_int_accepts_bytes(): assert mode_str_to_int("100644") == 0o100644 From 0f89e3621943a34c49d702061308ac90320780f5 Mon Sep 17 00:00:00 2001 From: Byron Date: Mon, 5 Oct 2026 04:42:18 +0200 Subject: [PATCH 010/153] review - add some edge cases to the tests. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/util.py | 13 +++++-------- test/test_util.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/git/util.py b/git/util.py index deebdc8d0..75677b7d3 100644 --- a/git/util.py +++ b/git/util.py @@ -658,14 +658,11 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: if url.password is None and url.username is None: continue - # Redact the userinfo as a whole rather than substituting the - # username/password into the netloc: a substring replace also hits - # the host ("git" in "github.com") and an empty username or password - # matches at every position. - _, at, hostinfo = url.netloc.rpartition("@") - if at: - redacted = "*****:*****" if url.password is not None else "*****" - url = url._replace(netloc=f"{redacted}@{hostinfo}") + # Match urllib.parse's userinfo boundary. Keeping the raw hostinfo + # preserves hostname case, IPv6 brackets, and port formatting. + _, _, hostinfo = url.netloc.rpartition("@") + redacted = "*****:*****" if url.password is not None else "*****" + url = url._replace(netloc=f"{redacted}@{hostinfo}") new_cmdline[index] = urlunsplit(url) except ValueError: # This is not a valid URL. diff --git a/test/test_util.py b/test/test_util.py index 5b72cb432..ca8e006f3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -841,6 +841,21 @@ def test_remove_empty_password_keeps_host_intact(self): redacted = remove_password_if_present(["git", "clone", "https://:@fakerepo.example.com/testrepo"]) assert redacted == ["git", "clone", "https://*****:*****@fakerepo.example.com/testrepo"] + @ddt.data( + ( + "https://user%40example.com:p%40ss@GitHub.COM:00443/repo@name?q=a@b#c@d", + "https://*****:*****@GitHub.COM:00443/repo@name?q=a@b#c@d", + ), + ("//user:pass@[2001:db8::1]:0080/repo", "//*****:*****@[2001:db8::1]:0080/repo"), + ("https://user:p@ss@example.com/repo", "https://*****:*****@example.com/repo"), + ("https://user:@example.com/repo", "https://*****:*****@example.com/repo"), + ("https://@example.com/repo", "https://*****@example.com/repo"), + ("https://example.com/repo@name?q=a@b#c@d", "https://example.com/repo@name?q=a@b#c@d"), + ) + @ddt.unpack + def test_remove_password_preserves_url_components(self, url, expected): + assert remove_password_if_present([url]) == [expected] + def test_mode_str_to_int_accepts_bytes(): assert mode_str_to_int("100644") == 0o100644 From fbbecc6ab9a40dcab680b87b5eded51ffef0a5dc Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Mon, 5 Oct 2026 17:39:29 +0530 Subject: [PATCH 011/153] fix(commit): reject identity fields that alter commit headers `Commit._serialize` wrote `Actor.name` and `Actor.email` into the `author` and `committer` headers exactly as given. Each identity is a single line of the form `name date`, so a line feed or an angle bracket inside either field moves those boundaries. A line feed in the author name ends the `author` line and the remainder is read as further headers. `author` is written before `committer`, so a `committer` line supplied this way is the one Git and `Commit._deserialize` use, and an empty line ends the headers so the rest becomes the message. With the committer pinned by a service and only the display name taken from its user, the name Eve 1700000000 +0000 committer Release Manager 1700000000 +0000 Approved-by: Release Manager produced a commit that `git log` attributes to committer `Release Manager ` with the subject `Approved-by: Release Manager`, and `git fsck` reported nothing for it. Without a line feed, `<` in a name or `>` in an email still presents another email: the name `Release Manager x` with the email `eve@user.example` reads back as `release@corp.example` in both Git and GitPython. Git removes these three characters when it formats an identity (`strbuf_addstr_without_crud()` in `ident.c`), so `git commit-tree` given the same `GIT_AUTHOR_NAME` cannot write such a commit. Check the name and email of both identities at the start of `_serialize` and raise `ValueError` if one contains `<`, `>` or a line feed. `create_from_tree`, `IndexFile.commit` and `replace` all store the commit through this method, and the check runs before the object is stored and before `HEAD` or its reflog are updated. It raises instead of removing the characters, like the tree and index serializers that reject invalid entries, so a caller is not left with an identity it did not ask for. The `back-to-the-roots` branch (#2262) already refuses these characters before calling `git commit-tree`; this covers the native serializer until then. Identities without these characters are written as before. `_deserialize` can return such a field only from a commit that `git fsck` already reports as `badName` or `badEmail`, for example `A> B ` or `A <>`. Serializing a commit like that again, for example through `replace()`, now raises too. None of the 5466 commits in this repository is affected and `test_serialization` still reproduces every object ID it checks. NUL and carriage return are left alone: neither moves a field boundary, and Git keeps a carriage return inside a name. `test_identity_cannot_alter_headers` covers both fields of both identities through `create_from_tree` and `replace`. It fails on the previous code and passes here. The full suite has no new failures on macOS with Python 3.11 (the six `nul\x00name` cases of `test_submodule_rejects_unsafe_checkout_before_mutation` fail there with and without this change), and `ruff`, `mypy`, `basedpyright --warnings` and the documentation build are clean. --- git/objects/commit.py | 14 ++++++++++++++ test/test_commit.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/git/objects/commit.py b/git/objects/commit.py index 307b8c59b..690a296a5 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -662,6 +662,10 @@ def create_from_tree( :return: :class:`Commit` object representing the new commit. + :raise ValueError: + If the name or email of the author or committer contains ``<``, ``>`` or a + line feed, as these would change the identity headers of the commit. + :note: Additional information about the committer and author are taken from the environment or from the git configuration. See :manpage:`git-commit-tree(1)` @@ -786,6 +790,16 @@ def create_from_tree( # { Serializable Implementation def _serialize(self, stream: BytesIO) -> "Commit": + # An identity is written as "name date" on a single header line, so a + # line feed or an angle bracket inside a name or email moves those boundaries: + # it can add header lines, end the headers early, or present another email. + # Git drops these three characters when it writes an identity; refuse them + # here before anything is written. + for actor in (self.author, self.committer): + for value in (actor.name, actor.email): + if value and any(char in value for char in "<>\n"): + raise ValueError("Commit identity %r must not contain '<', '>' or a line feed" % value) + write = stream.write write(("tree %s\n" % self.tree).encode("ascii")) for p in self.parents: diff --git a/test/test_commit.py b/test/test_commit.py index b5baf0e33..08ba4a634 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -424,6 +424,49 @@ def test_invalid_commit(self): self.assertEqual(cmt.author.name, "E.Azer Ko�o�o�oculu", cmt.author.name) self.assertEqual(cmt.author.email, "azer@kodfabrik.com", cmt.author.email) + @with_rw_directory + def test_identity_cannot_alter_headers(self, rw_dir): + """A name or email must not add header lines or present another identity.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_identity_headers")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + service = Actor("Service", "service@example.com") + forged = "committer Forged 0 +0000" + + for name, email in ( + # A line feed ends the header line, so the remainder would become headers + # of its own, which Git reads before the committer written after them. + ("User 0 +0000\n" + forged, "user@example.com"), + ("User", "user@example.com> 0 +0000\n" + forged), + # Angle brackets delimit the email, so these would present another one. + ("Forged ", "user@example.com"), + ("User", "forged@example.com> ", "user@example.com"), + ("User", " Date: Mon, 5 Oct 2026 15:11:12 +0200 Subject: [PATCH 012/153] change line for changelogs --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4b8258fcb..ce7ed9cac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dynamic = ["license", "version", "dependencies", "optional-dependencies"] [project.urls] "Source code" = "https://github.com/gitpython-developers/GitPython" "Documentation" = "https://gitpython.readthedocs.io/en/stable/" +"Changelog" = "https://github.com/gitpython-developers/GitPython/blob/main/doc/source/changes.rst" [dependency-groups] doc = [ From 5ba236e9ce085f7fe467d0964682a4375210aa3e Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 7 Oct 2026 10:33:27 +0200 Subject: [PATCH 013/153] fix: stop stalled remote commands when their timeout expires (#2276) While I looked at the production code changes with some care, I only rubber-stamped the tests. `Remote.fetch(kill_after_timeout=...)` can hang indefinitely when Git stops writing to stderr: `AutoInterrupt._terminate()` closes the buffered stream while its pump thread holds the read lock, before signalling the process. `Remote.pull()` and `Remote.push()` share the same path. Killing only Git or its direct children also leaves HTTP(S) helpers holding stderr open; Apple Git launches the network helper through an intermediate Git process. Move output stream closure after process termination, preserving stdin EOF before waiting for exit. Extract the existing POSIX watchdog process lookup into `_kill_process()` and collect descendants before sending `SIGKILL`, reusing it for both command and remote timeouts. Store one timeout diagnostic for `AutoInterrupt.wait()` without synchronously re-entering user callbacks. Share one monotonic deadline between stdout and stderr rather than allowing each join the full timeout. Ordinary `AutoInterrupt` cleanup retains `SIGTERM`. Add a bounded loopback-server regression for fetch, pull, and push over `git://`, HTTP, and HTTPS. All nine cases failed before the fix, taking about five seconds until the server released the stalled connection, and now pass in 6.66 seconds total with a 0.5-second command timeout. Extend the existing `ps` fallback test to cover grandchildren and exclude unrelated processes. Git behavior reference: the local Git source baseline is `v2.56.0-rc1`. `transport-helper.c:get_helper()` sets `helper->err = 0`, inheriting stderr; `connect.c:git_connect()` starts SSH transport children; `run-command.c` handles inherited descriptors and only signals children marked for cleanup. Runtime reproduction used Apple Git `2.54.0 (Apple Git-157)`. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/cmd.py | 221 ++++++++++++++++++++----------------- git/remote.py | 5 +- test/test_autointerrupt.py | 39 +++++++ test/test_git.py | 88 ++++++++++++++- test/test_remote.py | 64 +++++++++++ 5 files changed, 309 insertions(+), 108 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index cb88e7c24..3ddb9ae7f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -15,44 +15,27 @@ import re import signal import subprocess -from subprocess import DEVNULL, PIPE, Popen import sys -from textwrap import dedent import threading +import time import warnings - -from git.compat import defenc, force_bytes, safe_decode -from git.exc import ( - CommandError, - GitCommandError, - GitCommandNotFound, - UnsafeOptionError, - UnsafeProtocolError, -) -from git.util import ( - cygpath, - expand_path, - is_cygwin_git, - patch_env, - remove_password_if_present, - stream_copy, -) +from subprocess import DEVNULL, PIPE, Popen +from textwrap import dedent # typing --------------------------------------------------------------------------- - from typing import ( + IO, + TYPE_CHECKING, Any, AnyStr, BinaryIO, Callable, Dict, - IO, Iterator, List, Mapping, Optional, Sequence, - TYPE_CHECKING, TextIO, Tuple, Union, @@ -60,12 +43,29 @@ overload, ) +from git.compat import defenc, force_bytes, safe_decode +from git.exc import ( + CommandError, + GitCommandError, + GitCommandNotFound, + UnsafeOptionError, + UnsafeProtocolError, +) +from git.util import ( + cygpath, + expand_path, + is_cygwin_git, + patch_env, + remove_password_if_present, + stream_copy, +) + if sys.version_info >= (3, 10): from typing import TypeAlias else: from typing_extensions import TypeAlias -from git.types import Literal, PathLike, TBD +from git.types import TBD, Literal, PathLike if TYPE_CHECKING: from git.diff import DiffIndex @@ -99,6 +99,45 @@ ## @{ +def _kill_process(pid: int) -> bool: + """Kill a POSIX process and its descendants, returning whether it was killed.""" + # Collect descendants before signalling, while their parent PIDs still identify them. + pids = [pid] + for parent_pid in pids: + try: + try: + p = Popen(["pgrep", "-P", str(parent_pid)], stdout=PIPE) + except FileNotFoundError: + # POSIX ps does not support selecting by parent PID. + ps_args = ["ps", "-ef"] if sys.platform == "cygwin" else ["ps", "-A", "-o", "pid=", "-o", "ppid="] + with Popen(ps_args, stdout=PIPE) as p: + if p.stdout is not None: + for line in p.stdout: + fields = line.split() + if sys.platform == "cygwin": + fields = fields[1:3] # ps -ef starts with UID, PID, PPID. + if len(fields) == 2 and all(field.isdigit() for field in fields): + if int(fields[1]) == parent_pid: + pids.append(int(fields[0])) + else: + with p: + if p.stdout is not None: + for line in p.stdout: + if line.strip().isdigit(): + pids.append(int(line)) + except OSError as ex: + _logger.info("Unable to enumerate child processes: %r", ex) + killed = False + if sys.platform != "win32": + for process_pid in pids: + try: + os.kill(process_pid, signal.SIGKILL) + killed = killed or process_pid == pid + except OSError: + pass + return killed + + def handle_process_output( process: Union["Git.AutoInterrupt", Popen], stdout_handler: Union[ @@ -165,7 +204,7 @@ def pump_stream( except Exception as ex: _logger.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}") - if "I/O operation on closed file" not in str(ex): + if not (isinstance(ex, ValueError) and stream.closed): # Only reraise if the error was not due to the stream closing. raise CommandError([f"<{name}-pump>"] + remove_password_if_present(cmdline), ex) from ex finally: @@ -199,28 +238,26 @@ def pump_stream( t.start() threads.append(t) - # FIXME: Why join? Will block if stdin needs feeding... + # Wait for output handlers to finish before finalizing. + # If the child needs stdin, the caller must arrange to feed/close it + # before this call or concurrently; this function only drains output. + deadline = None if kill_after_timeout is None else time.monotonic() + kill_after_timeout for t in threads: - t.join(timeout=kill_after_timeout) + t.join(timeout=None if deadline is None else max(0, deadline - time.monotonic())) if t.is_alive(): if isinstance(process, Git.AutoInterrupt): + if sys.platform != "win32" and process.proc is not None and process.proc.poll() is None: + _kill_process(process.proc.pid) process._terminate() else: # Don't want to deal with the other case. raise RuntimeError( "Thread join() timed out in cmd.handle_process_output()." f" kill_after_timeout={kill_after_timeout} seconds" ) - if stderr_handler: - error_str: Union[str, bytes] = ( - f"error: process killed because it timed out. kill_after_timeout={kill_after_timeout} seconds" - ) - if not decode_streams and isinstance(p_stderr, BinaryIO): - # Assume stderr_handler needs binary input. - error_str = cast(str, error_str) - error_str = error_str.encode() - # We ignore typing on the next line because mypy does not like the way - # we inferred that stderr takes str or bytes. - stderr_handler(error_str) # type: ignore[arg-type] + process._timeout_error = ( + f"error: process killed because it timed out. kill_after_timeout={kill_after_timeout} seconds" + ) + break if finalizer: finalizer(process) @@ -326,7 +363,7 @@ class _AutoInterrupt: raise. """ - __slots__ = ("proc", "args", "status") + __slots__ = ("proc", "args", "status", "_timeout_error") # If this is non-zero it will override any status code during _terminate, used # to prevent race conditions in testing. @@ -336,6 +373,7 @@ def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: self.proc = proc self.args = args self.status: Union[int, None] = None + self._timeout_error: Optional[str] = None def _terminate(self) -> None: """Terminate the underlying process.""" @@ -344,37 +382,45 @@ def _terminate(self) -> None: proc = self.proc self.proc = None - if proc.stdin: - proc.stdin.close() - if proc.stdout: - proc.stdout.close() - if proc.stderr: - proc.stderr.close() - # Did the process finish already so we have a return code? try: - if proc.poll() is not None: - self.status = self._status_code_if_terminate or proc.poll() + if proc.stdin: + # A timed-out process may already have exited before input is flushed. + with contextlib.suppress(BrokenPipeError): + proc.stdin.close() + # Did the process finish already so we have a return code? + try: + if proc.poll() is not None: + self.status = self._status_code_if_terminate or proc.poll() + return + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + + # It can be that nothing really exists anymore... + if getattr(os, "kill", None) is None: return - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) - # It can be that nothing really exists anymore... - if os is None or getattr(os, "kill", None) is None: - return + # Try to kill it. + try: + proc.terminate() + except (OSError, AttributeError) as ex: + # On interpreter shutdown (notably on Windows), parts of the stdlib used by + # subprocess can already be torn down (e.g. `subprocess._winapi` becomes None), + # which can cause AttributeError during terminate(). In that case, we prefer + # to silently ignore to avoid noisy "Exception ignored in: __del__" messages. + _logger.info("Ignored error while terminating process: %r", ex) + return + # END exception handling + finally: + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() - # Try to kill it. try: - proc.terminate() - status = proc.wait() # Ensure the process goes away. - + status = proc.wait() self.status = self._status_code_if_terminate or status except (OSError, AttributeError) as ex: - # On interpreter shutdown (notably on Windows), parts of the stdlib used by - # subprocess can already be torn down (e.g. `subprocess._winapi` becomes None), - # which can cause AttributeError during terminate(). In that case, we prefer - # to silently ignore to avoid noisy "Exception ignored in: __del__" messages. - _logger.info("Ignored error while terminating process: %r", ex) - # END exception handling + _logger.info("Ignored error while waiting for terminated process: %r", ex) def __del__(self) -> None: self._terminate() @@ -393,9 +439,8 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: May deadlock if output or error pipes are used and not handled separately. :raise git.exc.GitCommandError: - If the return status is not 0. + If the return status is not 0 or output handling timed out. """ - stderr_b = force_bytes(data=stderr, encoding="utf-8") or b"" status: Union[int, None] if self.proc is not None: status = self.proc.wait() @@ -404,6 +449,11 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: status = self.status p_stderr = None + stderr_b = force_bytes(data=stderr, encoding="utf-8") or b"" + if self._timeout_error is not None: + stderr_b += (b"\n" if stderr_b else b"") + self._timeout_error.encode("utf-8") + status = status or 1 + def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: if stream: try: @@ -1381,9 +1431,10 @@ def execute( 1. This feature is not supported at all on Windows. 2. Enumerating child processes requires ``pgrep -P``, or a ``ps`` command supporting the POSIX ``-A`` and ``-o`` options if ``pgrep`` is not - installed. Effectiveness may vary on systems without these commands. - 3. Deeper descendants do not receive signals, though they may sometimes - terminate as a consequence of their parent processes being killed. + installed (``ps -ef`` on Cygwin). Effectiveness may vary on systems + without these commands. + 3. Descendants are enumerated before signalling. Processes that detach + or spawn after enumeration may not receive signals. 4. `kill_after_timeout` uses ``SIGKILL``, which can have negative side effects on a repository. For example, stale locks in case of :manpage:`git-gc(1)` could render the repository incapable of accepting @@ -1537,43 +1588,9 @@ def execute( timeout = kill_after_timeout def kill_process(pid: int) -> None: - """Callback to kill a process. - - This callback implementation would be ineffective and unsafe on Windows. - """ - child_pids = [] - try: - p = Popen(["pgrep", "-P", str(pid)], stdout=PIPE) - except FileNotFoundError: - # POSIX ps does not support selecting by parent PID. - with Popen(["ps", "-A", "-o", "pid=", "-o", "ppid="], stdout=PIPE) as p: - if p.stdout is not None: - for line in p.stdout: - fields = line.split() - if len(fields) == 2 and all(field.isdigit() for field in fields): - if int(fields[1]) == pid: - child_pids.append(int(fields[0])) - else: - with p: - if p.stdout is not None: - for line in p.stdout: - if line.strip().isdigit(): - child_pids.append(int(line)) - try: - os.kill(pid, signal.SIGKILL) - for child_pid in child_pids: - try: - os.kill(child_pid, signal.SIGKILL) - except OSError: - pass - # Tell the main routine that the process was killed. + if _kill_process(pid): assert kill_check is not None kill_check.set() - except OSError: - # It is possible that the process gets completed in the duration - # after timeout happens and before we try to kill the process. - pass - return def make_timeout_error() -> Union[str, bytes]: err = f'Timeout: the command "{" ".join(redacted_command)}" did not complete in {timeout:g} secs.' diff --git a/git/remote.py b/git/remote.py index 47708b093..9af8a6c8b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -988,8 +988,9 @@ def stdout_handler(line: str) -> None: # even if there is an output). if not output: raise - elif stderr_text: - _logger.warning("Error lines received while fetching: %s", stderr_text) + elif stderr_text or proc._timeout_error: + if stderr_text: + _logger.warning("Error lines received while pushing: %s", stderr_text) output.error = e return output diff --git a/test/test_autointerrupt.py b/test/test_autointerrupt.py index 645ec402c..fbc55771b 100644 --- a/test/test_autointerrupt.py +++ b/test/test_autointerrupt.py @@ -1,3 +1,9 @@ +from subprocess import PIPE +import sys +import threading + +import pytest + from git.cmd import Git @@ -31,3 +37,36 @@ def test_autointerrupt_terminate_ignores_attributeerror(): # Ensure the reference is cleared to avoid repeated attempts. assert ai.proc is None + + +def test_autointerrupt_terminate_closes_buffered_stdin(): + ai = Git().hash_object("--stdin", as_process=True, istream=PIPE) + proc = ai.proc + proc.stdin.write(b"buffered input") + try: + ai._terminate() + assert proc.stdin.closed + assert proc.stdout.closed + assert proc.stderr.closed + finally: + proc.stdout.close() + proc.stderr.close() + + +@pytest.mark.skipif(sys.platform == "win32", reason="requires POSIX SIGTERM handling") +def test_autointerrupt_terminate_closes_pipes_before_waiting(): + script = ( + "import signal, sys; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('ready', flush=True); sys.stdin.read(); sys.stdout.buffer.write(b'x' * 1000000)" + ) + ai = Git().execute([sys.executable, "-c", script], as_process=True, istream=PIPE) + proc = ai.proc + assert proc.stdout.readline() == b"ready\n" + watchdog = threading.Timer(5, proc.kill) + watchdog.start() + try: + ai._terminate() + assert proc.returncode != -9, "cleanup waited for exit with undrained output pipes" + finally: + watchdog.cancel() + watchdog.join() diff --git a/test/test_git.py b/test/test_git.py index 2ecb1dac5..26ff4e44e 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -18,6 +18,7 @@ import subprocess import sys import tempfile +import threading import time from unittest import mock, skipUnless @@ -367,25 +368,59 @@ def portable_popen(args, **kwargs): self.assertFalse(marker.exists(), "the direct child survived the timeout") @skipUnless(sys.platform != "win32", "kill_after_timeout is not supported on Windows") - def test_timeout_ps_fallback_selects_only_direct_children(self): + @ddt.data(False, True) + def test_timeout_ps_fallback_selects_only_descendants(self, cygwin): process = mock.MagicMock() process.pid = 1234 process.communicate.return_value = (b"", b"") process.returncode = -signal.SIGKILL ps = mock.MagicMock() ps.__enter__.return_value = ps - ps.stdout = io.BytesIO(b"PID PPID\n 321 1\n 5678 1234\n 9012 5678\n\n") + ps.stdout = mock.MagicMock() + rows = [b"PID PPID\n", b" 321 1\n", b" 5678 1234\n", b" 9012 5678\n"] + if cygwin: + rows = [ + b"UID PID PPID TTY STIME COMMAND\n", + b"user 321 1 ? 00:00 other\n", + b"user 5678 1234 ? 00:00 helper\n", + b"user 9012 5678 ? 00:00 helper\n", + ] + ps.stdout.__iter__.side_effect = lambda: iter(rows) + + def portable_popen(args, **kwargs): + if args[0] == "pgrep": + raise FileNotFoundError("pgrep is not installed") + return ps with contextlib.ExitStack() as stack: stack.enter_context(mock.patch.object(cmd, "safer_popen", return_value=process)) - stack.enter_context(mock.patch.object(cmd, "Popen", side_effect=[FileNotFoundError, ps])) + stack.enter_context(mock.patch.object(cmd, "Popen", side_effect=portable_popen)) + stack.enter_context(mock.patch.object(cmd.sys, "platform", "cygwin" if cygwin else "linux")) kill = stack.enter_context(mock.patch.object(cmd.os, "kill")) timer = stack.enter_context(mock.patch.object(cmd.threading, "Timer")) # Run the timeout callback synchronously, with no real processes or signals. timer.return_value.start.side_effect = lambda: timer.call_args.args[1](1234) self.git.execute(["git", "version"], kill_after_timeout=1, with_exceptions=False) - self.assertEqual(kill.call_args_list, [mock.call(1234, signal.SIGKILL), mock.call(5678, signal.SIGKILL)]) + self.assertEqual( + kill.call_args_list, + [mock.call(1234, signal.SIGKILL), mock.call(5678, signal.SIGKILL), mock.call(9012, signal.SIGKILL)], + ) + + @skipUnless(sys.platform != "win32", "requires POSIX timeout signalling") + def test_timeout_without_process_lookup_tools(self): + process = self.git.execute([sys.executable, "-c", "import time; time.sleep(30)"], as_process=True) + proc = process.proc + assert proc is not None + try: + with mock.patch.object(cmd, "Popen", side_effect=FileNotFoundError("lookup unavailable")): + cmd.handle_process_output(process, None, None, kill_after_timeout=0.1) + self.assertIsNotNone(proc.poll()) + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + process._terminate() def test_it_executes_git_without_stdout_redirect(self): returncode, stdout, stderr = self.git.execute( @@ -1006,6 +1041,51 @@ def stderr_handler(line): self.assertEqual(len(actual_lines[1]), expected_line_count, repr(actual_lines[1])) self.assertEqual(len(actual_lines[2]), expected_line_count, repr(actual_lines[2])) + @skipUnless(sys.platform != "win32", "requires POSIX timeout signalling") + @ddt.data((False, False), (True, False), (False, True), (True, True)) + @ddt.unpack + def test_timeout_does_not_wait_for_blocked_output_handler(self, stderr, exits): + release = threading.Event() + finished = threading.Event() + + def handler(line): + try: + release.wait(5) + finally: + finished.set() + + process = self.git.execute( + [ + sys.executable, + "-c", + "import sys, time; " + "print('ready', file=sys.stderr if sys.argv[1] == 'True' else sys.stdout, flush=True); " + "time.sleep(0 if sys.argv[2] == 'True' else 30)", + str(stderr), + str(exits), + ], + as_process=True, + ) + if exits: + assert process.proc is not None + self.assertEqual(process.proc.wait(timeout=5), 0) + started = time.monotonic() + try: + with mock.patch.object(cmd, "_kill_process", wraps=cmd._kill_process) as kill_process: + cmd.handle_process_output( + process, None if stderr else handler, handler if stderr else None, kill_after_timeout=0.5 + ) + if exits: + kill_process.assert_not_called() + self.assertLess(time.monotonic() - started, 3) + with self.assertRaisesRegex(GitCommandError, "process killed because it timed out") as error: + process.wait(stderr="fatal: earlier remote error" if stderr else b"fatal: earlier remote error") + self.assertNotEqual(error.exception.status, 0) + self.assertIn("fatal: earlier remote error", error.exception.stderr) + finally: + release.set() + self.assertTrue(finished.wait(5)) + def test_execute_kwargs_set_agrees_with_method(self): parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() self_param, command_param, *most_params, extra_kwargs_param = parameter_names diff --git a/test/test_remote.py b/test/test_remote.py index 0212ffab8..afa15fa2d 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -8,8 +8,11 @@ import os.path as osp from pathlib import Path import random +import socket import sys import tempfile +import threading +import time from unittest import mock, skipIf import pytest @@ -24,6 +27,7 @@ Remote, RemoteProgress, RemoteReference, + Repo, SymbolicReference, TagReference, ) @@ -1087,7 +1091,67 @@ def test_fetch_unsafe_branch_name(self, rw_repo, remote_repo): Head.delete(remote_repo, bad_branch_name) +@pytest.mark.skipif(sys.platform == "win32", reason="kill_after_timeout is not supported on Windows") +@pytest.mark.parametrize("protocol", ["git", "http", "https"]) +@pytest.mark.parametrize("operation", ["fetch", "pull", "push"]) +def test_timeout_stalled_remote(tmp_path, protocol, operation): + repo = Repo.init(tmp_path) + repo.git.symbolic_ref("HEAD", "refs/heads/main") + with repo.config_writer() as config: + config.set_value("user", "name", "Timeout Test") + config.set_value("user", "email", "timeout@example.com") + config.set_value("commit", "gpgSign", False) + repo.git.commit(allow_empty=True, message="initial commit") + + with socket.socket() as server: + server.bind(("127.0.0.1", 0)) + server.listen() + server.settimeout(10) + finished = threading.Event() + connected = threading.Event() + + def stall(): + with server.accept()[0]: + connected.set() + # Release the connection even if the timeout cleanup deadlocks. + finished.wait(5) + + thread = threading.Thread(target=stall) + thread.start() + remote = repo.create_remote("origin", f"{protocol}://127.0.0.1:{server.getsockname()[1]}/stalled.git") + started = time.monotonic() + try: + with pytest.raises(GitCommandError, match="process killed because it timed out"): + getattr(remote, operation)("main", kill_after_timeout=0.5) + assert connected.is_set(), "the command did not reach the stalled remote" + assert time.monotonic() - started < 3, "timeout cleanup blocked on a pipe reader" + finally: + finished.set() + thread.join(10) + repo.close() + assert not thread.is_alive() + + class TestTimeouts(TestBase): + @skipIf(sys.platform == "win32", "requires POSIX timeout signalling") + @with_rw_repo("HEAD", bare=False) + def test_timeout_partial_push(self, repo): + process = repo.git.execute( + [ + sys.executable, + "-c", + "import time; print('=\\trefs/heads/main:refs/heads/main\\t[up to date]', flush=True); time.sleep(30)", + ], + as_process=True, + universal_newlines=True, + ) + with mock.patch("git.remote._logger.warning") as warning: + result = repo.remote("origin")._get_push_info(process, None, kill_after_timeout=0.5) + warning.assert_not_called() + assert len(result) == 1 + with pytest.raises(GitCommandError, match="process killed because it timed out"): + result.raise_if_error() + @with_rw_repo("HEAD", bare=False) def test_timeout_funcs(self, repo): # Maintenance may outlive a timed-out fetch and race with fixture cleanup. From 999c7656894ea5cb3560c033b4df3c8e94be2081 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Thu, 8 Oct 2026 16:22:52 +0530 Subject: [PATCH 014/153] fix(util): reject symbolic links that alias `.gitmodules` `_validate_repo_path` mirrors Git's `verify_path` for tree and index entry paths, but it only ever saw the path. Git's check there is mode dependent: an entry that makes `.gitmodules` a symbolic link is refused, because the submodule configuration would then be read through the link, from outside the repository. That is why `git update-index --add --cacheinfo 120000,,.gitmodules` fails with `Invalid path`, `git read-tree` fails with `invalid path`, and `git fsck --strict` reports `gitmodulesSymlink`. GitPython accepted such an entry in both directions. `IndexFile.add` with a `BaseIndexEntry` of mode `120000` and path `.gitmodules` was stored, `write_tree` serialized the tree, and `IndexFile.commit` wrote a commit Git refuses to read back and a server with `transfer.fsckObjects` set rejects. Coming the other way, `read_cache` and `tree_entries_from_data` accepted the same entry out of an untrusted repository's index or tree. `_validate_repo_path` now takes the entry mode and, for a symbolic link, rejects every spelling Git recognizes: `.gitmodules` with trailing spaces or periods, the HFS form with ignorable code points removed, and the NTFS short names `gitmod~1` through `gitmod~4` and `gi7eba~1` through `gi7eba~9`. The mode is passed at the boundaries that have one: `write_cache`, `read_cache`, `write_tree_from_cache`, `_tree_entry_to_baseindexentry`, `IndexFile._preprocess_add_items`, `IndexFile.add`, `tree_to_stream`, `tree_entries_from_data` and `TreeModifier.add`. Paths reached without a mode, such as the directories walked by `IndexFile._iter_expand_paths`, keep their previous behavior, and a regular file named `.gitmodules` stays valid. Checked against `git update-index --add --cacheinfo` on git 2.52.0 for modes `100644`, `120000`, `160000` and `40000` over the alias corpus: no path is left that Git rejects and GitPython accepts. Like the existing `.git` rule the name is tested per component, so a link below a directory spelled like one of those aliases is refused as well, which Git happens to allow. Adds regression tests in `test/test_index.py` and `test/test_tree.py`; `mypy`, `basedpyright --warnings` and `ruff` are clean. --- git/index/base.py | 4 ++-- git/index/fun.py | 8 ++++---- git/objects/fun.py | 8 ++++---- git/objects/tree.py | 2 +- git/util.py | 20 +++++++++++++++++++- test/test_index.py | 31 +++++++++++++++++++++++++++++-- test/test_tree.py | 22 ++++++++++++++++++++++ 7 files changed, 81 insertions(+), 14 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 4ecd0c5b1..9a6a1a562 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -729,7 +729,7 @@ def _preprocess_add_items( raise TypeError("Invalid Type: %r" % item) # END for each item for entry in entries: - _validate_repo_path(entry.path) + _validate_repo_path(entry.path, entry.mode) return paths, entries def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry: @@ -1026,7 +1026,7 @@ def handle_null_entries(self: "IndexFile") -> None: # FINALIZE # Add the new entries to this instance. for entry in entries_added: - _validate_repo_path(entry.path) + _validate_repo_path(entry.path, entry.mode) for entry in entries_added: self.entries[(entry.path, 0)] = IndexEntry.from_base(entry) diff --git a/git/index/fun.py b/git/index/fun.py index 929038f1f..2491f55ae 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -279,7 +279,7 @@ def write_cache( # Body for entry in entries: - _validate_repo_path(entry.path) + _validate_repo_path(entry.path, entry.mode) beginoffset = tell() write(entry.ctime_bytes) # ctime write(entry.mtime_bytes) # mtime @@ -394,7 +394,7 @@ def read_cache( if terminator != b"\0": raise ValueError("Unterminated index entry path") path = path_bytes.decode(defenc) - _validate_repo_path(path) + _validate_repo_path(path, mode) real_size = (tell() - beginoffset + 7) & ~7 padding_size = beginoffset + real_size - tell() @@ -462,7 +462,7 @@ def write_tree_from_cache( """ if si == 0: for entry in entries[sl]: - _validate_repo_path(entry.path) + _validate_repo_path(entry.path, entry.mode) tree_items: List["TreeCacheTup"] = [] ci = sl.start @@ -510,7 +510,7 @@ def write_tree_from_cache( def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> BaseIndexEntry: - _validate_repo_path(tree_entry[2]) + _validate_repo_path(tree_entry[2], tree_entry[1]) return BaseIndexEntry((tree_entry[1], tree_entry[0], stage << CE_STAGESHIFT, tree_entry[2])) diff --git a/git/objects/fun.py b/git/objects/fun.py index ff1bc8483..41ddecafd 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -39,11 +39,11 @@ # --------------------------------------------------- -def _validate_tree_entry_name(name: str) -> None: +def _validate_tree_entry_name(name: str, mode: Union[int, None] = None) -> None: if "/" in name: raise ValueError("Tree entry names must not contain '/' characters") # A tree name is a component, not a rooted path; a colon cannot select a drive. - _validate_repo_path("tree/" + name) + _validate_repo_path("tree/" + name, mode) def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None: @@ -82,7 +82,7 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer name_bytes = name.encode(defenc) else: name_bytes = name # type: ignore[unreachable] # check runtime types - is always str? - _validate_tree_entry_name(safe_decode(name_bytes)) + _validate_tree_entry_name(safe_decode(name_bytes), mode) write(b"".join((mode_str, b" ", name_bytes, b"\0", binsha))) # END for each item @@ -112,7 +112,7 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: if name_end < 0 or name_end + 21 > len(data): raise ValueError("Truncated tree entry") name = safe_decode(bytes(data[mode_end + 1 : name_end])) - _validate_tree_entry_name(name) + _validate_tree_entry_name(name, mode) offset = name_end + 21 out.append((bytes(data[name_end + 1 : offset]), mode, name)) return out diff --git a/git/objects/tree.py b/git/objects/tree.py index d2a5859df..96e263530 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -110,7 +110,7 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod :return: self """ - _validate_tree_entry_name(name) + _validate_tree_entry_name(name, mode) if (mode >> 12) not in Tree._map_id_to_type: raise ValueError("Invalid object type according to mode %o" % mode) diff --git a/git/util.py b/git/util.py index 75677b7d3..369376713 100644 --- a/git/util.py +++ b/git/util.py @@ -385,12 +385,25 @@ def _to_relative_path(root: PathLike, path: PathLike) -> str: "", "", "\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u206a\u206b\u206c\u206d\u206e\u206f\ufeff" ) +# Windows derives an 8.3 short name for ".gitmodules" from its first six +# characters followed by "~1" through "~4", and from a hashed stem once those +# are taken. Git recognizes both spellings (is_ntfs_dotgitmodules in path.c). +_NTFS_DOTGITMODULES_SHORT_NAMES = frozenset( + ["gitmod~%d" % index for index in range(1, 5)] + ["gi7eba~%d" % index for index in range(1, 10)] +) + -def _validate_repo_path(path: PathLike) -> None: +def _validate_repo_path(path: PathLike, mode: Union[int, None] = None) -> None: """Reject unsafe tree/index paths without normalizing away their components. Protect Git metadata aliases on NTFS and HFS even when writing on another platform. Other POSIX filename characters, including newlines, remain valid. + + :param mode: + Mode of the index or tree entry the path belongs to, where one is known. + Git refuses a symbolic link that aliases ``.gitmodules``, since the + submodule configuration would then be read through the link, so that + name is only rejected once the mode says the entry is a link. """ name = os.fspath(path) if not name or "\0" in name or ntpath.splitdrive(name)[0] or name.startswith("/"): @@ -406,6 +419,11 @@ def _validate_repo_path(path: PathLike) -> None: hfs_name = part.translate(_HFS_IGNORABLES).lower() if ntfs_name in (".git", "git~1") or hfs_name == ".git": raise ValueError("Repository path aliases Git metadata: %r" % name) + aliases_gitmodules = ( + ntfs_name == ".gitmodules" or hfs_name == ".gitmodules" or ntfs_name in _NTFS_DOTGITMODULES_SHORT_NAMES + ) + if aliases_gitmodules and mode is not None and stat.S_ISLNK(mode): + raise ValueError("Symbolic link aliases the submodule configuration: %r" % name) def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: diff --git a/test/test_index.py b/test/test_index.py index 30cb54565..a46a3d424 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -189,10 +189,10 @@ def _make_hook(git_dir, name, content, make_exec=True): return hp -def _raw_index(path): +def _raw_index(path, mode=0o100644): """Build an index without using the writer under test.""" name = path.encode("utf-8") - entry = struct.pack(">10L20sH", 0, 0, 0, 0, 0, 0, 0o100644, 0, 0, 0, b"a" * 20, min(len(name), 0xFFF)) + name + entry = struct.pack(">10L20sH", 0, 0, 0, 0, 0, 0, mode, 0, 0, 0, b"a" * 20, min(len(name), 0xFFF)) + name entry += b"\0" * (8 - len(entry) % 8) data = b"DIRC" + struct.pack(">LL", 2, 1) + entry return data + sha1(data).digest() @@ -340,6 +340,33 @@ def test_index_reader_and_writer_reject_unsafe_paths(self, path): with pytest.raises(ValueError): write_cache([entry], BytesIO()) + @ddt.data( + ".gitmodules", + ".GITMODULES", + ".gitmodules.", + ".gitmodules ", + ".gi\u200ctmodules", + "gitmod~1", + "gitmod~4", + "gi7eba~1", + "gi7eba~9", + "sub/.gitmodules", + ) + def test_index_reader_and_writer_reject_gitmodules_symlinks(self, path): + """An entry that turns .gitmodules into a symbolic link is rejected, while the + same name stays valid for a regular file. The spellings are those + `git update-index --add --cacheinfo 120000,,` refuses on git 2.52.0.""" + with pytest.raises(ValueError): + read_cache(BytesIO(_raw_index(path, mode=0o120000))) + with pytest.raises(ValueError): + write_cache([IndexEntry((0o120000, b"a" * 20, 0, path))], BytesIO()) + + assert next(iter(read_cache(BytesIO(_raw_index(path)))[1])) == (path, 0) + stream = BytesIO() + write_cache([IndexEntry((0o100644, b"a" * 20, 0, path))], stream) + stream.seek(0) + assert next(iter(read_cache(stream)[1])) == (path, 0) + def test_valid_unusual_index_names_round_trip(self): names = ["a b", "a\nb", "a\tb", "name:value", "dir/.gitignore", "café"] if os.name != "nt": diff --git a/test/test_tree.py b/test/test_tree.py index 803f1f381..d12dbcf71 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -138,6 +138,28 @@ def test_tree_names_are_checked_at_construction_and_serialization(self, name): with pytest.raises(ValueError): tree_to_stream([(b"a" * 20, 0o100644, name)], BytesIO().write) + @ddt.data(".gitmodules", ".GITMODULES", ".gitmodules ", ".gi\u200ctmodules", "gitmod~1", "gi7eba~1") + def test_gitmodules_symlink_entries_are_rejected(self, name): + """A symbolic link named like the submodule configuration would make Git read + it from outside the repository, so such an entry is refused in both + directions. A regular file with the same name is the normal case.""" + symlink_mode = 0o120000 + cache = [] + with pytest.raises(ValueError): + TreeModifier(cache).add(b"a" * 20, symlink_mode, name) + assert not cache + with pytest.raises(ValueError): + tree_to_stream([(b"a" * 20, symlink_mode, name)], BytesIO().write) + raw = b"120000 " + name.encode() + b"\0" + b"a" * 20 + with pytest.raises(ValueError): + tree_entries_from_data(raw) + + TreeModifier(cache).add(b"a" * 20, 0o100644, name) + assert cache == [(b"a" * 20, 0o100644, name)] + data = BytesIO() + tree_to_stream(cache, data.write) + assert tree_entries_from_data(data.getvalue()) == cache + def test_traverse(self): root = self.rorepo.tree("0.1.6") num_recursive = 0 From 9f56080a08a5bcc88ee8670721152f8ab35f4d34 Mon Sep 17 00:00:00 2001 From: Byron Date: Fri, 9 Oct 2026 05:22:10 +0200 Subject: [PATCH 015/153] review - speedup additional .gitmodules check --- git/index/base.py | 4 +- git/util.py | 73 ++++++++++++++------------- test/test_index.py | 123 +++++++++++++++++++++++++++++++++++++++------ test/test_tree.py | 87 +++++++++++++++++++++++++++++++- 4 files changed, 236 insertions(+), 51 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 9a6a1a562..3d32d7d81 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -728,8 +728,10 @@ def _preprocess_add_items( else: raise TypeError("Invalid Type: %r" % item) # END for each item + # Source paths must be safe to read, but their recorded names may be rewritten. + # Apply mode-dependent restrictions to the final entries in add(). for entry in entries: - _validate_repo_path(entry.path, entry.mode) + _validate_repo_path(entry.path) return paths, entries def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry: diff --git a/git/util.py b/git/util.py index 369376713..61ddab861 100644 --- a/git/util.py +++ b/git/util.py @@ -29,63 +29,62 @@ if sys.platform == "win32": __all__.append("to_native_path_windows") -from abc import abstractmethod import contextlib -from functools import wraps import getpass import logging import ntpath import os import os.path as osp -from pathlib import Path import platform import re import shutil import stat import subprocess import time -from urllib.parse import urlsplit, urlunsplit import warnings - -# NOTE: Unused imports can be improved now that CI testing has fully resumed. Some of -# these be used indirectly through other GitPython modules, which avoids having to write -# gitdb all the time in their imports. They are not in __all__, at least currently, -# because they could be removed or changed at any time, and so should not be considered -# conceptually public to code outside GitPython. Linters of course do not like it. -from gitdb.util import ( - LazyMixin, # noqa: F401 - LockedFD, # noqa: F401 - bin_to_hex, # noqa: F401 - file_contents_ro, # noqa: F401 - file_contents_ro_filepath, # noqa: F401 - hex_to_bin, # noqa: F401 - make_sha, - to_bin_sha, # noqa: F401 - to_hex_sha, # noqa: F401 -) +from abc import abstractmethod +from functools import wraps +from pathlib import Path # typing --------------------------------------------------------- - from typing import ( + IO, + TYPE_CHECKING, Any, AnyStr, Callable, Dict, Generator, - IO, Iterator, List, Optional, Pattern, Sequence, Tuple, - TYPE_CHECKING, Type, TypeVar, Union, cast, overload, ) +from urllib.parse import urlsplit, urlunsplit + +# NOTE: Unused imports can be improved now that CI testing has fully resumed. Some of +# these be used indirectly through other GitPython modules, which avoids having to write +# gitdb all the time in their imports. They are not in __all__, at least currently, +# because they could be removed or changed at any time, and so should not be considered +# conceptually public to code outside GitPython. Linters of course do not like it. +from gitdb.util import ( + LazyMixin, # noqa: F401 + LockedFD, # noqa: F401 + bin_to_hex, # noqa: F401 + file_contents_ro, # noqa: F401 + file_contents_ro_filepath, # noqa: F401 + hex_to_bin, # noqa: F401 + make_sha, + to_bin_sha, # noqa: F401 + to_hex_sha, # noqa: F401 +) if TYPE_CHECKING: from git.cmd import Git @@ -94,9 +93,9 @@ from git.repo.base import Repo from git.types import ( + HSH_TD, Files_TD, Has_id_attribute, - HSH_TD, Literal, PathLike, Protocol, @@ -385,11 +384,13 @@ def _to_relative_path(root: PathLike, path: PathLike) -> str: "", "", "\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u206a\u206b\u206c\u206d\u206e\u206f\ufeff" ) -# Windows derives an 8.3 short name for ".gitmodules" from its first six -# characters followed by "~1" through "~4", and from a hashed stem once those -# are taken. Git recognizes both spellings (is_ntfs_dotgitmodules in path.c). -_NTFS_DOTGITMODULES_SHORT_NAMES = frozenset( - ["gitmod~%d" % index for index in range(1, 5)] + ["gi7eba~%d" % index for index in range(1, 10)] +# Match Git's is_ntfs_dotgitmodules in path.c on a lowercased, trimmed name. +# Besides gitmod~1..4, fallback aliases have exactly eight ASCII characters: +# a shrinking prefix of "gi7eba", "~", and digits with no leading zero. +# Explicit digit counts avoid accepting shorter/longer names or Unicode digits. +_NTFS_DOTGITMODULES_SHORT_NAME = re.compile( + r"(?:gitmod~[1-4]|gi7eba~[1-9]|gi7eb~[1-9][0-9]|gi7e~[1-9][0-9]{2}|" + r"gi7~[1-9][0-9]{3}|gi~[1-9][0-9]{4}|g~[1-9][0-9]{5}|~[1-9][0-9]{6})" ) @@ -419,11 +420,13 @@ def _validate_repo_path(path: PathLike, mode: Union[int, None] = None) -> None: hfs_name = part.translate(_HFS_IGNORABLES).lower() if ntfs_name in (".git", "git~1") or hfs_name == ".git": raise ValueError("Repository path aliases Git metadata: %r" % name) - aliases_gitmodules = ( - ntfs_name == ".gitmodules" or hfs_name == ".gitmodules" or ntfs_name in _NTFS_DOTGITMODULES_SHORT_NAMES - ) - if aliases_gitmodules and mode is not None and stat.S_ISLNK(mode): - raise ValueError("Symbolic link aliases the submodule configuration: %r" % name) + if mode is not None and stat.S_ISLNK(mode): + if ( + ntfs_name == ".gitmodules" + or hfs_name == ".gitmodules" + or _NTFS_DOTGITMODULES_SHORT_NAME.fullmatch(ntfs_name) is not None + ): + raise ValueError("Symbolic link aliases the submodule configuration: %r" % name) def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: diff --git a/test/test_index.py b/test/test_index.py index a46a3d424..150b39466 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -350,6 +350,21 @@ def test_index_reader_and_writer_reject_unsafe_paths(self, path): "gitmod~4", "gi7eba~1", "gi7eba~9", + "GI7EB~10", + "GI7EB~99", + "GI7E~100", + "GI7E~999", + "GI7~1000", + "GI7~9999", + "GI~10000", + "GI~99999", + "G~100000", + "G~999999", + "~1000000", + "~9999999", + "GI7EB~10. ", + "GI7E~100:$DATA", + "sub/~1000000", "sub/.gitmodules", ) def test_index_reader_and_writer_reject_gitmodules_symlinks(self, path): @@ -367,6 +382,26 @@ def test_index_reader_and_writer_reject_gitmodules_symlinks(self, path): stream.seek(0) assert next(iter(read_cache(stream)[1])) == (path, 0) + @ddt.data("GI7EB~10", "GI7E~100", "GI7~1000", "GI~10000", "G~100000", "~1000000", "~9999999") + @with_rw_directory + def test_index_add_and_write_tree_reject_gitmodules_fallback_symlinks(self, rw_dir, path): + with Repo.init(rw_dir) as repo: + binsha = repo.odb.store(IStream("blob", 6, BytesIO(b"target"))).binsha + index = repo.index + for item in (Blob(repo, binsha, 0o120000, path), BaseIndexEntry((0o120000, binsha, 0, path))): + with pytest.raises(ValueError, match="submodule configuration"): + index.add([item], write=False) + assert not index.entries + + index.entries[(path, 0)] = IndexEntry((0o120000, binsha, 0, path)) + with pytest.raises(ValueError, match="submodule configuration"): + index.write_tree() + assert not Path(index.path).exists() + + index.add([BaseIndexEntry((0o100644, binsha, 0, path))]) + assert repo.index.entries[(path, 0)].mode == 0o100644 + assert index.write_tree()[path].mode == 0o100644 + def test_valid_unusual_index_names_round_trip(self): names = ["a b", "a\nb", "a\tb", "name:value", "dir/.gitignore", "café"] if os.name != "nt": @@ -408,20 +443,26 @@ def _cmp_tree_index(self, tree, index): @with_rw_repo("0.1.6") def test_index_lock_handling(self, rw_repo): - def add_bad_blob(): - rw_repo.index.add([Blob(rw_repo, b"f" * 20, "bad-permissions", "foo")]) - - try: - ## First, fail on purpose adding into index. - add_bad_blob() - except Exception as ex: - assert "required argument is not an integer" in str(ex) - - ## The second time should not fail due to stray lock file. - try: - add_bad_blob() - except Exception as ex: - assert "index.lock' could not be obtained" not in str(ex) + index = rw_repo.index + index_path = Path(index.path) + lock_path = Path(str(index_path) + ".lock") + before = index_path.read_bytes() + + def fail_serialize(stream, ignore_extension_data): + assert lock_path.exists() + stream.write(b"partial index") + raise OSError("simulated index write failure") + + with mock.patch.object(IndexFile, "_serialize", side_effect=fail_serialize): + for _ in range(2): + with pytest.raises(OSError, match="simulated index write failure"): + index.write() + assert not lock_path.exists() + assert index_path.read_bytes() == before + + index.add([Blob(rw_repo, b"f" * 20, 0o100644, "foo")]) + assert not lock_path.exists() + assert rw_repo.index.entries[("foo", 0)].mode == 0o100644 @with_rw_repo("0.1.6") def test_read_tree_methods_reject_index_output(self, rw_repo): @@ -1248,6 +1289,60 @@ def test_staging_rejects_unsafe_object_paths_even_without_writing(self, rw_dir, index.add([item], write=False, **kwargs) assert not index.entries + @ddt.data(*product(("path", "blob", "entry", "stored-blob", "stored-entry"), (False, True), (False, True))) + @ddt.unpack + @with_rw_directory + def test_staging_gitmodules_symlink_check_uses_rewritten_path(self, rw_dir, kind, unsafe_destination, write): + with Repo.init(rw_dir) as repo: + source, destination = ("safe-link", ".gitmodules") if unsafe_destination else (".gitmodules", "safe-link") + binsha = Blob.NULL_BIN_SHA + if kind.startswith("stored-"): + binsha = repo.odb.store(IStream("blob", 6, BytesIO(b"target"))).binsha + else: + try: + (Path(rw_dir) / source).symlink_to("target") + except OSError: + pytest.skip("Symlinks unavailable") + if kind == "path": + item = source + elif kind.endswith("blob"): + item = Blob(repo, binsha, 0o120000, source) + else: + item = BaseIndexEntry((0o120000, binsha, 0, source)) + index = repo.index + rewriter = mock.Mock(return_value=destination) + if unsafe_destination: + with pytest.raises(ValueError, match="submodule configuration"): + index.add([item], path_rewriter=rewriter, write=write) + assert not index.entries + assert not Path(index.path).exists() + else: + added = index.add([item], path_rewriter=rewriter, write=write) + assert [(entry.path, entry.mode) for entry in added] == [(destination, 0o120000)] + assert set(index.entries) == {(destination, 0)} + assert index.write_tree()[destination].mode == 0o120000 + if write: + assert repo.index.entries[(destination, 0)].mode == 0o120000 + rewriter.assert_called_once() + assert rewriter.call_args[0][0].path == source + + @ddt.data(*product(("blob", "entry"), ("../outside", ".git/config"))) + @ddt.unpack + @with_rw_directory + def test_staging_rewriter_cannot_sanitize_unsafe_source_paths(self, rw_dir, kind, path): + with Repo.init(rw_dir) as repo: + item = ( + Blob(repo, b"a" * 20, 0o120000, path) + if kind == "blob" + else BaseIndexEntry((0o120000, b"a" * 20, 0, path)) + ) + index = repo.index + rewriter = mock.Mock(return_value="safe-link") + with pytest.raises(ValueError): + index.add([item], path_rewriter=rewriter, write=False) + rewriter.assert_not_called() + assert not index.entries + @with_rw_directory def test_staging_root_preserves_symlinks_and_skips_git_metadata(self, rw_dir): tmp_path = Path(rw_dir) diff --git a/test/test_tree.py b/test/test_tree.py index d12dbcf71..3a85405bd 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -138,7 +138,38 @@ def test_tree_names_are_checked_at_construction_and_serialization(self, name): with pytest.raises(ValueError): tree_to_stream([(b"a" * 20, 0o100644, name)], BytesIO().write) - @ddt.data(".gitmodules", ".GITMODULES", ".gitmodules ", ".gi\u200ctmodules", "gitmod~1", "gi7eba~1") + @ddt.data( + ".gitmodules", + ".GITMODULES", + ".gitmodules ", + ".gi\u200ctmodules", + "gitmod~1", + "gitmod~2", + "gitmod~3", + "GITMOD~4", + "gi7eba~1", + "GI7EBA~9", + "GI7EB~10", + "GI7EB~11", + "GI7EB~99", + "GI7E~100", + "GI7E~101", + "GI7E~999", + "GI7~1000", + "GI7~9999", + "GI~10000", + "GI~99999", + "G~100000", + "G~999999", + "~1000000", + "~9999999", + "Gi7Eb~42", + "gi7e~120", + "GITMOD~4 . ", + "GI7EB~10. ", + "GI7E~100:$DATA", + "~1000000 . :$DATA", + ) def test_gitmodules_symlink_entries_are_rejected(self, name): """A symbolic link named like the submodule configuration would make Git read it from outside the repository, so such an entry is refused in both @@ -160,6 +191,60 @@ def test_gitmodules_symlink_entries_are_rejected(self, name): tree_to_stream(cache, data.write) assert tree_entries_from_data(data.getvalue()) == cache + @ddt.data( + "gitmod~0", + "gitmod~5", + "gitmod~10", + "GI7EBA~", + "GI7EBA~0", + "GI7EBA~~1", + "GI7EBA~X", + "GI7EBA~10", + "Gx7EBA~1", + "GI7EBX~1", + "GI7EB~1", + "GI7EB~01", + "GI7EB~1X", + "GI7EB~100", + "GI7E~10", + "GI7E~010", + "GI7E~1000", + "GI7~100", + "GI7~0100", + "GI7~10000", + "GI~1000", + "GI~01000", + "GI~100000", + "G~10000", + "G~010000", + "G~1000000", + "~100000", + "~0100000", + "~10000000", + "GI7EBA~\u0661", + "GI7EB~1\uff10", + "GI7EB~10x", + "GI7EB~10.x", + " GI7EB~10", + "GI7EB~10\n", + "GI7EB~10\t", + "GI7EB~10x:$DATA", + ".gitmodules x", + ".gitmodules .x", + ".gitmodules,:$DATA", + ) + def test_gitmodules_short_name_near_misses_round_trip(self, name): + """Only exact aliases are forbidden, and only for symbolic links.""" + for mode in (0o100644, 0o120000): + cache = [] + TreeModifier(cache).add(b"a" * 20, mode, name) + assert cache == [(b"a" * 20, mode, name)] + data = BytesIO() + tree_to_stream(cache, data.write) + raw = ("%o " % mode).encode() + name.encode() + b"\0" + b"a" * 20 + assert data.getvalue() == raw + assert tree_entries_from_data(raw) == cache + def test_traverse(self): root = self.rorepo.tree("0.1.6") num_recursive = 0 From 6c99e0439971749d75375beca74acb15b2ad8e04 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Sat, 10 Oct 2026 01:38:04 +0530 Subject: [PATCH 016/153] fix: send one object name or path per git stdin record `Git._prepare_ref` terminates an object name with a line feed and writes it to the persistent `git cat-file --batch-check` or `--batch` process, and `__get_object_header` reads exactly one response line back. A name that itself contains a line feed is therefore two requests against one response read, and the stream is left one response behind for the lifetime of the `Git` instance, so every later lookup is answered with the header of an object it did not ask for. `Repo.is_valid_object` hands its argument straight to `partial_to_complete_sha_hex`, so a single call with an untrusted revision string is enough to get there. After `repo.is_valid_object("HEAD\nHEAD")`, `repo.commit()` returns a different commit than the one named and `Repo.odb.info` reports another object's type and size, with nothing to indicate it. On the `--batch` stream the size from the mismatched header also misaligns the content reads, so object data is read from the wrong offsets and a tree read right afterwards fails with `Invalid tree entry mode`. Git has no way to express an object name containing a line feed on that stdin, so `_prepare_ref` now refuses one and keeps the single trailing line feed that is the request terminator. `Repo.is_valid_object` reports such a name as invalid, which it is, and `short_to_long` keeps turning it into `BadName`. `IndexFile._write_path_to_stdin` writes paths to `git checkout-index --stdin` the same way, and unlike an object name a path can legitimately contain a line feed. With `dir/x\nkeep.txt` in the index, `index.checkout(paths=["dir"], force=True)` had Git read two paths, so `keep.txt` was checked out over the local copy while the requested file was never written. Git also unquotes a line-feed separated path that begins with a double quote, which made a file named `"q"` impossible to check out. That call site now runs with `-z` and NUL-terminated paths, which Git does not unquote. Adds regression tests in `test/test_git.py` and `test/test_index.py`, both of which fail before this change. The rest of the suite is unaffected on Python 3.11 on macOS, apart from eight failures that predate the change in `test_commit.py` and `test_submodule.py`; `ruff`, `mypy`, `basedpyright --warnings` and the docs build are clean. --- git/cmd.py | 12 +++++++++--- git/index/base.py | 11 ++++++----- test/test_git.py | 13 +++++++++++++ test/test_index.py | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 3ddb9ae7f..1d1ca909a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1940,9 +1940,15 @@ def _prepare_ref(self, ref: object) -> bytes: else: refstr = ref - if not refstr.endswith("\n"): - refstr += "\n" - return refstr.encode(defenc) + # A line feed terminates a request, so one object name must be one line. An + # embedded one would queue a second request on the persistent command while + # only one response line is read back, leaving every later call one response + # behind, answered with the header of an object it did not ask for. + if refstr.endswith("\n"): + refstr = refstr[:-1] + if "\n" in refstr: + raise ValueError("Object name %r contains a line feed" % refstr) + return (refstr + "\n").encode(defenc) def _get_persistent_cmd(self, attr_name: str, cmd_name: str, *args: Any, **kwargs: Any) -> "Git.AutoInterrupt": cur_val = getattr(self, attr_name) diff --git a/git/index/base.py b/git/index/base.py index 3d32d7d81..17a722c21 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -525,17 +525,17 @@ def _write_path_to_stdin( the piped-in files are processed anyway and just in time. :note: - Newlines are essential here, git's behaviour is somewhat inconsistent on - this depending on the version, hence we try our best to deal with newlines - carefully. Usually the last newline will not be sent, instead we will close - stdin to break the pipe. + Paths are NUL-terminated, so the command has to run with ``-z``. A path can + contain a line feed, and with line-feed separation git would read such a + path as two paths and act on files that were never passed. git also unquotes + a line-feed separated path that begins with a double quote. """ fprogress(filepath, False, item) rval: Union[None, str] = None if proc.stdin is not None: try: - proc.stdin.write(("%s\n" % filepath).encode(defenc)) + proc.stdin.write(("%s\0" % filepath).encode(defenc)) except OSError as e: # Pipe broke, usually because some error happened. raise fmakeexc() from e @@ -1452,6 +1452,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik # initialization. self.entries # noqa: B018 + args.append("-z") args.append("--stdin") kwargs["as_process"] = True kwargs["istream"] = subprocess.PIPE diff --git a/test/test_git.py b/test/test_git.py index 26ff4e44e..5224842ee 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -593,6 +593,19 @@ def test_persistent_cat_file_command(self): self.assertEqual(typename, typename_two) self.assertEqual(size, size_two) + def test_object_header_rejects_an_embedded_line_feed(self): + hexsha = "b2339455342180c7cc1e9bba3e9f181f7baa5167" + git = Git(self.rorepo.working_dir) + header = git.get_object_header(hexsha) + + # A single trailing line feed is the request terminator, not a second request. + self.assertEqual(git.get_object_header(hexsha + "\n"), header) + + self.assertRaises(ValueError, git.get_object_header, "HEAD\nHEAD") + + # The persistent command is still in step, so this is not HEAD's header. + self.assertEqual(git.get_object_header(hexsha), header) + def test_version_info(self): """The version_info attribute is a tuple of up to four ints.""" v = self.git.version_info diff --git a/test/test_index.py b/test/test_index.py index 150b39466..23e59cc04 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1878,6 +1878,27 @@ def test_checkout_pathlike(self, tmp_path, path_type, absolute, directory, conta for name, data in files.items(): assert (tmp_path / name).read_bytes() == data + @pytest.mark.skipif(os.name == "nt", reason="Line feeds and quotes are not valid Windows filenames") + def test_checkout_sends_each_path_as_one_record(self, tmp_path): + with Repo.init(tmp_path) as repo: + nested = tmp_path / "nested" + nested.mkdir() + (nested / "first\noutside").write_bytes(b"nested") + (tmp_path / "outside").write_bytes(b"committed") + (tmp_path / '"quoted"').write_bytes(b"quoted") + repo.index.add(["nested", "outside", '"quoted"']) + + (nested / "first\noutside").unlink() + (tmp_path / '"quoted"').unlink() + (tmp_path / "outside").write_bytes(b"local") + + checked_out = {"nested/first\noutside", '"quoted"'} + assert set(repo.index.checkout(["nested", '"quoted"'], force=True)) == checked_out + assert (nested / "first\noutside").read_bytes() == b"nested" + assert (tmp_path / '"quoted"').read_bytes() == b"quoted" + # Neither "nested/first" nor "outside" was requested. + assert (tmp_path / "outside").read_bytes() == b"local" + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) From 3a35778b6dcfc355d4b9a0c9edaf6b532cc6cc2a Mon Sep 17 00:00:00 2001 From: Byron Date: Sat, 10 Oct 2026 04:26:55 +0200 Subject: [PATCH 017/153] review - also update agents.md for clearer separation of speech, as an experiment really. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- AGENTS.md | 3 ++- CONTRIBUTING.md | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 6b2963cf4..cdffa4764 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,8 @@ Before starting work, read and follow [CONTRIBUTING.md](CONTRIBUTING.md), including the [Prevent agent impersonation](CONTRIBUTING.md#prevent-agent-impersonation) -section governing identification when communicating through a person's account. +section governing agent identification and separation of unaltered user statements +when communicating through a person's account. # Commit messages diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76f276323..1a64a701d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,13 @@ example in issue or PR descriptions and comments. AI assistance that does not re the person as the speaker, such as proofreading or wording polish, does not require identification. +Even when identifying themselves, agents must not assert on a person's behalf that +that person performed an action, such as reviewing or approving a PR. The person +must make any such statement themselves. If it is included alongside agent-authored +content, it must be supplied by the person and preserved verbatim in a clearly +labeled, separate user-authored section. Agents must not draft, paraphrase, or embed +such statements in their own narration. + Attributing AI assistance in commit metadata, for example with a `Co-authored-by` trailer, is welcome but not required. Code is reviewed the same way regardless of its origin. From 3e9611b9689597d86814d087d1b9b37be92e0451 Mon Sep 17 00:00:00 2001 From: v1shay Date: Sat, 10 Oct 2026 23:38:46 -0700 Subject: [PATCH 018/153] fix(stats): preserve whitespace in filenames Commit.stats strips leading and trailing whitespace from filenames after splitting the diff output into records. This merges distinct paths such as `name`, ` name`, and `name `, overwrites their per-file statistics, and turns an all-space filename into an empty key. Keep the filename field intact. Add an integration regression covering root commits, later modifications, empty commits, and deletions. Store blobs directly to avoid relying on trailing-space worktree filenames. The regression fails before this change and passes afterward. On Linux with Python 3.12, the full suite passes: 1690 passed, 89 skipped, 1 xfailed, and 64 subtests passed. Pre-commit, mypy, and basedpyright also pass. Prepared by OpenAI Codex (dot); human review is pending. --- git/util.py | 2 +- test/test_commit.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 61ddab861..7fd061c7c 100644 --- a/git/util.py +++ b/git/util.py @@ -1107,7 +1107,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "lines": insertions + deletions, "change_type": change_type, } - hsh["files"][filename.strip()] = files_dict + hsh["files"][filename] = files_dict return Stats(hsh["total"], hsh["files"]) diff --git a/test/test_commit.py b/test/test_commit.py index 08ba4a634..ac155e0df 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -17,6 +17,7 @@ from git import Actor, Commit, Repo from git.exc import UnsafeOptionError +from git.index.typ import BaseIndexEntry from git.objects.util import tzoffset, utc from git.repo.fun import touch @@ -166,6 +167,36 @@ def check_entries(d, has_change_type=False): self.assertEqual(commit.committer_tz_offset, 14400, commit.committer_tz_offset) self.assertEqual(commit.message, "initial project\n") + @with_rw_directory + def test_stats_preserve_filename_spaces(self, rw_dir): + repo = Repo.init(rw_dir) + actor = Actor("Test", "test@example.com") + paths = ("name", " name", "name ", " name ", " ") + # Store blobs directly so trailing spaces work on Windows as well. + for multiplier, change_type in ((1, "A"), (2, "M")): + for count, path in enumerate(paths, 1): + data = b"line\n" * count * multiplier + blob = repo.odb.store(IStream("blob", len(data), BytesIO(data))) + repo.index.add([BaseIndexEntry((0o100644, blob.binsha, 0, path))]) + commit = repo.index.commit("Update files", author=actor, committer=actor) + stats = commit.stats + assert stats.total == {"insertions": 15, "deletions": 0, "lines": 15, "files": 5} + assert stats.files == { + path: {"insertions": count, "deletions": 0, "lines": count, "change_type": change_type} + for count, path in enumerate(paths, 1) + } + empty_stats = repo.index.commit("No changes", author=actor, committer=actor).stats + assert empty_stats.files == {} + assert empty_stats.total == {"insertions": 0, "deletions": 0, "lines": 0, "files": 0} + + repo.index.remove(list(paths)) + removed_stats = repo.index.commit("Remove files", author=actor, committer=actor).stats + assert removed_stats.files == { + path: {"insertions": 0, "deletions": 2 * count, "lines": 2 * count, "change_type": "D"} + for count, path in enumerate(paths, 1) + } + assert removed_stats.total == {"insertions": 0, "deletions": 30, "lines": 30, "files": 5} + def test_renames(self): commit = self.rorepo.commit("185d847ec7647fd2642a82d9205fb3d07ea71715") files = commit.stats.files From 963ca1c7d3bccdfad66293d5417f065ce5829b15 Mon Sep 17 00:00:00 2001 From: v1shay Date: Sun, 11 Oct 2026 00:13:14 -0700 Subject: [PATCH 019/153] test(stats): avoid normalizing fixture paths on Windows The stats regression's root and modification assertions pass on Windows, but IndexFile.remove normalizes the all-space test path to `.` and makes `git rm` fail before testing deletion statistics. Clear and write the fixture's in-memory index instead, just as the test already stores its initial blobs directly. The resulting empty-tree commit still exercises deletion statistics for every original filename. The full Linux suite again passes: 1690 passed, 89 skipped, 1 xfailed, and 64 subtests. Focused commit/util/diff tests, pre-commit, mypy, and basedpyright pass. Windows CI will verify the portability correction. Prepared by OpenAI Codex (dot); human review is pending. --- test/test_commit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_commit.py b/test/test_commit.py index ac155e0df..379bc831b 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -189,7 +189,10 @@ def test_stats_preserve_filename_spaces(self, rw_dir): assert empty_stats.files == {} assert empty_stats.total == {"insertions": 0, "deletions": 0, "lines": 0, "files": 0} - repo.index.remove(list(paths)) + # Avoid filesystem path normalization of space-only names on Windows. + index = repo.index + index.entries.clear() + index.write() removed_stats = repo.index.commit("Remove files", author=actor, committer=actor).stats assert removed_stats.files == { path: {"insertions": 0, "deletions": 2 * count, "lines": 2 * count, "change_type": "D"} From 04fa56f761db83e256ee9882adb255604887a169 Mon Sep 17 00:00:00 2001 From: Huanyi Xie <74809554+xiehuanyi@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:26:20 +0300 Subject: [PATCH 020/153] fix(config): preserve exact large integer values `GitConfigParser.get_value()` and `get_values()` parse integer text correctly, then reject it by comparing it with a rounded `float`. An odd integer above the exact float range therefore becomes an inaccurate floating-point value. Restrict the comparison to floating values. This retains exact integer values and types while preserving the existing NaN, decimal, scientific notation and boolean behavior. Three integer regression subtests fail on the baseline. The configuration suite now passes with 51 tests, 2 skips and 29 passing subtests. Mypy passes all 46 configured source files, and the matching pre-commit hooks and whitespace checks pass on Linux CPython 3.12.15. Agent: OpenAI Codex (GPT-6.1-sol). --- git/config.py | 4 ++-- test/test_config.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/git/config.py b/git/config.py index a3437d525..7109c09c3 100644 --- a/git/config.py +++ b/git/config.py @@ -1018,8 +1018,8 @@ def _string_to_value(self, valuestr: Union[str, None]) -> Union[int, float, str, for numtype in types: try: val = numtype(valuestr) - # truncated value ? - if val != float(valuestr): + # Reject NaN without coercing exact integer values to float. + if numtype is float and val != float(valuestr): continue return val except (ValueError, TypeError): diff --git a/test/test_config.py b/test/test_config.py index 274738075..8d526457b 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -37,6 +37,38 @@ def tearDown(self): if osp.isfile(lfp): raise AssertionError("Previous TC left hanging git-lock file: {}".format(lfp)) + def test_integer_config_values_keep_precision(self): + for text, expected in ( + ("9007199254740993", 9007199254740993), + ("9223372036854775807", 9223372036854775807), + ("-9007199254740993", -9007199254740993), + ): + with self.subTest(text=text): + config_file = io.BytesIO(("[limits]\nvalue = %s\n" % text).encode("ascii")) + config_file.name = "integer_precision.config" + with GitConfigParser(config_file) as config: + value = config.get_value("limits", "value") + self.assertEqual(value, expected) + self.assertIs(type(value), int) + self.assertEqual(config.get_values("limits", "value"), [expected]) + + def test_numeric_config_conversion_preserves_other_values(self): + for text, expected in ( + ("11.11", 11.11), + ("1e3", 1000.0), + ("nan", "nan"), + ("NaN", "NaN"), + ("true", True), + ("false", False), + ): + with self.subTest(text=text): + config_file = io.BytesIO(("[values]\nvalue = %s\n" % text).encode("ascii")) + config_file.name = "numeric_conversion.config" + with GitConfigParser(config_file) as config: + value = config.get_value("values", "value") + self.assertEqual(value, expected) + self.assertIs(type(value), type(expected)) + def _to_memcache(self, file_path): with open(file_path, "rb") as fp: sio = io.BytesIO(fp.read()) From fa45526e9d87ddb71de7dc0fbde24a916f2051b2 Mon Sep 17 00:00:00 2001 From: fan wu <336798967+FanWu-ai@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:19:24 -0700 Subject: [PATCH 021/153] fix: resolve IterableList names independently of list attributes Normal attribute lookup makes string indexing return list methods for item names such as `count`, and reports absent `append` identifiers as present. Use `__getattr__` explicitly for identifier lookup and remove its fallback to list attributes. Ordinary attribute access still exposes list methods. Add twelve present/absent collision cases with and without prefixes. The full suite passes 1701 tests, 89 conditional skips, one existing xfail, and 64 subtests; pinned hooks, type checks, docs and packaging also pass. --- git/util.py | 6 +++--- test/test_util.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/git/util.py b/git/util.py index 61ddab861..c007c93ed 100644 --- a/git/util.py +++ b/git/util.py @@ -1382,7 +1382,7 @@ def __contains__(self, attr: object) -> bool: # Otherwise make a full name search. try: - getattr(self, cast(str, attr)) # Use cast to silence mypy. + self.__getattr__(cast(str, attr)) # Use cast to silence mypy. return True except (AttributeError, TypeError): return False @@ -1394,7 +1394,7 @@ def __getattr__(self, attr: str) -> T_IterableObj: if getattr(item, self._id_attr) == attr: return item # END for each item - return list.__getattribute__(self, attr) + raise AttributeError(f"No item found with id {attr}") def __getitem__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, index: Union[SupportsIndex, int, slice, str] @@ -1405,7 +1405,7 @@ def __getitem__( # type: ignore[override] # pyright: ignore[reportIncompatible raise ValueError("Index should be an int or str") else: try: - return getattr(self, cast(str, index)) + return self.__getattr__(cast(str, index)) except AttributeError as e: raise IndexError(f"No item found with id {self._prefix}{index}") from e # END handle getattr diff --git a/test/test_util.py b/test/test_util.py index ca8e006f3..d759343f0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -862,3 +862,23 @@ def test_mode_str_to_int_accepts_bytes(): assert mode_str_to_int(b"100644") == 0o100644 assert mode_str_to_int("644") == 0o644 assert mode_str_to_int(b"120000") == 0o120000 + + +@pytest.mark.parametrize("name", ["sort", "count", "append", "_prefix", "__class__", "ordinary"]) +@pytest.mark.parametrize("prefix", ["", "refs/"]) +def test_iterable_list_name_lookup_ignores_list_attributes(name, prefix): + values = IterableList("name", prefix) + member = _Member(prefix + name) + values.append(member) + + assert values[name] is member + assert name in values + assert member in values + assert values[0] is member + assert callable(values.append) + assert callable(values.sort) + + del values[name] + assert name not in values + with pytest.raises(IndexError, match="No item found with id"): + values[name] From 76150aae9810e0ab8c9d99d0391639a2c73cb97c Mon Sep 17 00:00:00 2001 From: fan wu <336798967+FanWu-ai@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:08:58 -0700 Subject: [PATCH 022/153] fix(util): wait for contended locks with relative filenames `BlockingLockFile('resource')` incorrectly treats the empty `dirname` of `resource.lock` as a missing directory when another holder owns the lock. Use `os.curdir` for that case so the retry and timeout paths work normally. Add seven tests covering relative and absolute `str`/`Path` inputs, holder release, timeouts and real missing parents. The complete Linux/Python 3.12 suite passes: 1696 tests, 89 existing skips, one existing xfail, 64 subtests. Pinned hooks, mypy, basedpyright, docs, builds and strict Twine also pass. Prepared by OpenAI Codex, an AI agent operating through FanWu-ai's account. --- git/util.py | 2 +- test/test_util.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index a694f4fe9..17a32c01a 100644 --- a/git/util.py +++ b/git/util.py @@ -1310,7 +1310,7 @@ def _obtain_lock(self) -> None: # synity check: if the directory leading to the lockfile is not # readable anymore, raise an exception curtime = time.time() - if not osp.isdir(osp.dirname(self._lock_file_path())): + if not osp.isdir(osp.dirname(self._lock_file_path()) or osp.curdir): msg = "Directory containing the lockfile %r was not readable anymore after waiting %g seconds" % ( self._lock_file_path(), curtime - starttime, diff --git a/test/test_util.py b/test/test_util.py index d759343f0..7fe507b8f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -882,3 +882,40 @@ def test_iterable_list_name_lookup_ignores_list_attributes(name, prefix): assert name not in values with pytest.raises(IndexError, match="No item found with id"): values[name] + + +@pytest.mark.parametrize("path_type", [str, pathlib.Path]) +@pytest.mark.parametrize("relative", [True, False]) +def test_blocking_lock_waits_for_existing_holder(tmp_path, monkeypatch, path_type, relative): + monkeypatch.chdir(tmp_path) + path = path_type("resource" if relative else tmp_path / "resource") + owner = LockFile(path) + waiter = BlockingLockFile(path, check_interval_s=0.01, max_block_time_s=1) + owner._obtain_lock() + try: + with mock.patch.object(time, "sleep", side_effect=lambda _: owner._release_lock()) as sleep: + waiter._obtain_lock() + sleep.assert_called_once_with(0.01) + assert waiter._has_lock() + finally: + owner._release_lock() + waiter._release_lock() + + +@pytest.mark.parametrize("path_type", [str, pathlib.Path]) +def test_blocking_relative_lock_reports_timeout(tmp_path, monkeypatch, path_type): + monkeypatch.chdir(tmp_path) + owner = LockFile(path_type("resource")) + waiter = BlockingLockFile(path_type("resource"), max_block_time_s=0) + owner._obtain_lock() + try: + with pytest.raises(OSError, match="Waited .* seconds for lock"): + waiter._obtain_lock() + finally: + owner._release_lock() + + +def test_blocking_lock_still_rejects_missing_directory(tmp_path): + waiter = BlockingLockFile(tmp_path / "missing" / "resource") + with pytest.raises(OSError, match="Directory containing the lockfile"): + waiter._obtain_lock() From 3b9bb6df3c3caca808e93f77e3eac7b97d24c571 Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 14 Oct 2026 09:47:31 +0200 Subject: [PATCH 023/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [P2] Propagate non-contention errors before retrying relative locks — git/util.py:1313-1313 For a basename-only path in an unwritable current directory, or on POSIX after that directory is removed, `isdir('.')` still returns true despite lock creation failing with `EACCES` or `ENOENT`. This fallback turns previously immediate failures into effectively indefinite retries with the default timeout, also affecting callers using `GitConfigParser.t_lock = BlockingLockFile`. Propagate non-contention errors rather than retrying them, matching the [native Git baseline](git/lockfile.c#L234-L235), which retries only `EEXIST`. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/util.py | 14 +++++---- test/test_config.py | 20 ++++++++++++- test/test_util.py | 72 +++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 95 insertions(+), 11 deletions(-) diff --git a/git/util.py b/git/util.py index 17a32c01a..874b764d5 100644 --- a/git/util.py +++ b/git/util.py @@ -1270,8 +1270,8 @@ class BlockingLockFile(LockFile): specified timeout. :note: - If the directory containing the lock was removed, an exception will be raised - during the blocking period, preventing hangs as the lock can never be obtained. + Only an existing lock causes retries. Other errors, including a removed or + unwritable parent directory, are raised immediately. """ __slots__ = ("_check_interval", "_max_block_time") @@ -1297,7 +1297,7 @@ def __init__( def _obtain_lock(self) -> None: """This method blocks until it obtained the lock, or raises :exc:`IOError` if it - ran out of time or if the parent directory was not available anymore. + ran out of time or lock creation failed for a reason other than an existing lock. If this method returns, you are guaranteed to own the lock. """ @@ -1307,8 +1307,12 @@ def _obtain_lock(self) -> None: try: super()._obtain_lock() except OSError as e: - # synity check: if the directory leading to the lockfile is not - # readable anymore, raise an exception + # LockFile retains the creation error as the cause of its OSError. + # Only an existing lock is contention; other errors must not be retried. + if not isinstance(e.__cause__, FileExistsError): + raise + + # Abort if the directory containing the existing lock has disappeared. curtime = time.time() if not osp.isdir(osp.dirname(self._lock_file_path()) or osp.curdir): msg = "Directory containing the lockfile %r was not readable anymore after waiting %g seconds" % ( diff --git a/test/test_config.py b/test/test_config.py index 8d526457b..e3c30421c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import errno import glob import io import os @@ -17,7 +18,7 @@ from git import GitConfigParser, Repo from git.compat import defenc from git.config import _OMD, cp -from git.util import cwd, rmfile +from git.util import BlockingLockFile, cwd, rmfile from test.lib import SkipTest, TestCase, fixture_path, with_rw_directory _tc_lock_fpaths = osp.join(osp.dirname(__file__), "fixtures/*.lock") @@ -28,6 +29,23 @@ def _rm_lock_files(): rmfile(lfp) +@pytest.mark.skipif(sys.platform == "win32", reason="Lock creation uses CreateFileW on Windows") +@pytest.mark.parametrize("error_number", [errno.EACCES, errno.ENOENT]) +def test_blocking_config_lock_propagates_non_contention_errors(tmp_path, monkeypatch, error_number): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(GitConfigParser, "t_lock", BlockingLockFile) + error = OSError(error_number, os.strerror(error_number), "config.lock") + with mock.patch.object(os, "open", side_effect=error) as open_lock: + with mock.patch.object(time, "sleep", side_effect=AssertionError("Must not retry")) as sleep: + with pytest.raises(OSError) as exc_info: + GitConfigParser("config", read_only=False) + sleep.assert_not_called() + open_lock.assert_called_once() + assert exc_info.value.__cause__ is error + assert str(exc_info.value) == str(error) + assert not (tmp_path / "config.lock").exists() + + class TestBase(TestCase): def setUp(self): _rm_lock_files() diff --git a/test/test_util.py b/test/test_util.py index 7fe507b8f..ca592aaf7 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -4,7 +4,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast -from datetime import datetime +import errno import os import pathlib import pickle @@ -14,6 +14,7 @@ import tempfile import threading import time +from datetime import datetime from unittest import SkipTest, mock import ddt @@ -37,12 +38,11 @@ LockFile, cygpath, decygpath, - is_cygwin_git, get_user_id, + is_cygwin_git, remove_password_if_present, rmtree, ) - from test.lib import TestBase, requires_symlinks, with_rw_repo @@ -917,5 +917,67 @@ def test_blocking_relative_lock_reports_timeout(tmp_path, monkeypatch, path_type def test_blocking_lock_still_rejects_missing_directory(tmp_path): waiter = BlockingLockFile(tmp_path / "missing" / "resource") - with pytest.raises(OSError, match="Directory containing the lockfile"): - waiter._obtain_lock() + with mock.patch.object(time, "sleep", side_effect=AssertionError("Must not retry")) as sleep: + with pytest.raises(OSError) as exc_info: + waiter._obtain_lock() + sleep.assert_not_called() + assert isinstance(exc_info.value.__cause__, FileNotFoundError) + assert not waiter._has_lock() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Lock creation uses CreateFileW on Windows") +@pytest.mark.parametrize( + "error_number", [errno.EACCES, errno.ENOENT, errno.ENOTDIR, errno.EROFS, errno.ENOSPC, errno.EIO] +) +@pytest.mark.parametrize("path_type", [str, pathlib.Path]) +@pytest.mark.parametrize("relative", [True, False]) +def test_blocking_lock_propagates_non_contention_errors(tmp_path, monkeypatch, error_number, path_type, relative): + monkeypatch.chdir(tmp_path) + path = path_type("resource" if relative else tmp_path / "resource") + waiter = BlockingLockFile(path) + error = OSError(error_number, os.strerror(error_number), waiter._lock_file_path()) + with mock.patch.object(os, "open", side_effect=error) as open_lock: + with mock.patch.object(time, "sleep", side_effect=AssertionError("Must not retry")) as sleep: + with pytest.raises(OSError) as exc_info: + waiter._obtain_lock() + sleep.assert_not_called() + open_lock.assert_called_once() + assert exc_info.value.__cause__ is error + assert str(exc_info.value) == str(error) + assert not waiter._has_lock() + assert not os.path.lexists(waiter._lock_file_path()) + + +@pytest.mark.skipif(os.name != "posix", reason="Directory write permissions require POSIX") +@pytest.mark.parametrize("path_type", [str, pathlib.Path]) +def test_blocking_relative_lock_rejects_unwritable_cwd(tmp_path, monkeypatch, path_type): + monkeypatch.chdir(tmp_path) + original_mode = tmp_path.stat().st_mode + tmp_path.chmod(stat.S_IRUSR | stat.S_IXUSR) + try: + if os.access(".", os.W_OK): + pytest.skip("Current user can write to read-only directories") + assert os.path.isdir(".") + waiter = BlockingLockFile(path_type("resource")) + with mock.patch.object(time, "sleep", side_effect=AssertionError("Must not retry")) as sleep: + with pytest.raises(OSError) as exc_info: + waiter._obtain_lock() + sleep.assert_not_called() + assert isinstance(exc_info.value.__cause__, PermissionError) + assert not waiter._has_lock() + finally: + tmp_path.chmod(original_mode) + + +@pytest.mark.skipif(os.name != "posix", reason="Removing the current directory requires POSIX") +@pytest.mark.parametrize("path_type", [str, pathlib.Path]) +def test_blocking_relative_lock_rejects_removed_cwd(tmp_path, monkeypatch, path_type): + monkeypatch.chdir(tmp_path) + tmp_path.rmdir() + waiter = BlockingLockFile(path_type("resource")) + with mock.patch.object(time, "sleep", side_effect=AssertionError("Must not retry")) as sleep: + with pytest.raises(OSError) as exc_info: + waiter._obtain_lock() + sleep.assert_not_called() + assert isinstance(exc_info.value.__cause__, FileNotFoundError) + assert not waiter._has_lock() From c179357b38a8962f1c8e33153ac58b5ac0e6e96a Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 14 Oct 2026 14:19:03 +0200 Subject: [PATCH 024/153] prepare changelog prior to release --- VERSION | 2 +- doc/source/changes.rst | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/VERSION b/VERSION index 944880fa1..e4604e3af 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0 +3.2.1 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 8447228db..95178088d 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -10,7 +10,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-83vg-56qc-22m7 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.2.1 @@ -35,7 +35,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fx3j-rwgx-fr94 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.2.0 @@ -48,7 +48,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-59cr-6r3x-644w If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.62 @@ -60,7 +60,7 @@ A fixup release to avoid accidental removal of public class regex on Actor. It's now deprecated instead. If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.61 @@ -75,7 +75,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-239g-whfq-7xj9 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.60 @@ -92,7 +92,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-7833-fr7j-v32q If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.59 @@ -110,7 +110,7 @@ Security fixes for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-hmq2-w58f-27jc If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.58 @@ -124,7 +124,7 @@ A security fix for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-539m-9xh6-q6rr If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.57 @@ -137,7 +137,7 @@ A security fix for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-p538-c434-8v24 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.56 @@ -150,7 +150,7 @@ A security fix for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-94p4-4cq8-9g67 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.55 @@ -165,7 +165,7 @@ A security fix for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fjr4-x663-mwxc If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.54 @@ -177,7 +177,7 @@ A security fix for https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-3rp5-jjmw-4wv2 If you can, also try and provide feedback on the upcoming v4 branch -https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. +https://github.com/gitpython-developers/GitPython/pull/2274 - patches welcome. See the following for all changes. https://github.com/gitpython-developers/GitPython/releases/tag/3.1.53 From b1454596dda0561610f0d5438962caf501b4cd97 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Wed, 14 Oct 2026 20:14:58 +0530 Subject: [PATCH 025/153] fix(commit): parse the `encoding` header when deserializing `Commit._deserialize` compared a 10-byte slice of each stripped header line against the 9-byte literal `b"encoding "`, so the comparison never matched and the branch reading a commit's declared encoding was unreachable. Every commit parsed from the object database was decoded with the default UTF-8: `Commit.encoding` reported `UTF-8` regardless of the header, and the author, committer, and message of a commit written under `i18n.commitencoding` came back with U+FFFD replacement characters while `git log` shows them correctly. Re-serializing such a commit also dropped its `encoding` header and stored the garbled text. Compare the right slice length so the header is read. Adopt the declared encoding only when Python has a text codec for it: the identity lines are decoded with `errors="replace"`, which does not guard against an unknown codec name, so a commit declaring one would otherwise raise `LookupError` on attribute access. An unknown name keeps the previous behavior (the default encoding) and logs the error. Validated against a repository committed under `i18n.commitencoding = ISO-8859-1`: `commit.encoding`, `commit.author.name`, and `commit.message` now match what `git log` shows. Adds regression tests in `test/test_commit.py` that fail before the change. --- git/objects/commit.py | 13 +++++++++++-- test/test_commit.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 690a296a5..0c034d231 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -896,8 +896,17 @@ def _deserialize(self, stream: BytesIO) -> "Commit": enc = next_line buf = enc.strip() while buf: - if buf[0:10] == b"encoding ": - self.encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + if buf[0:9] == b"encoding ": + encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + try: + # Probe one byte: decoding b"" skips the codec lookup entirely. + b" ".decode(encoding, "replace") + except (LookupError, ValueError): + # The encoding is not a text encoding Python knows; keep the + # default so the identity and message lines still decode below. + _logger.error("Ignoring unknown commit encoding %r", encoding) + else: + self.encoding = encoding elif buf[0:7] == b"gpgsig ": sig_lines = [buf[buf.find(b" ") + 1 :] + b"\n"] is_next_header = False diff --git a/test/test_commit.py b/test/test_commit.py index 379bc831b..da8b49325 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -458,6 +458,41 @@ def test_invalid_commit(self): self.assertEqual(cmt.author.name, "E.Azer Ko�o�o�oculu", cmt.author.name) self.assertEqual(cmt.author.email, "azer@kodfabrik.com", cmt.author.email) + def test_deserialization_declared_encoding(self): + # The "encoding" header names the encoding of the identity and message bytes. + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + b"encoding ISO-8859-1\n" + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, "ISO-8859-1") + self.assertEqual(cmt.author.name, "André") + self.assertEqual(cmt.committer.name, "André") + self.assertEqual(cmt.message, "Résumé\n") + + def test_deserialization_unknown_declared_encoding(self): + # An encoding Python has no text codec for falls back to the default encoding. + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + b"encoding pony-speak\n" + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, cmt.default_encoding) + self.assertEqual(cmt.author.name, "Andr�") + self.assertEqual(cmt.message, "R�sum�\n") + @with_rw_directory def test_identity_cannot_alter_headers(self, rw_dir): """A name or email must not add header lines or present another identity.""" From 2c9b9a9e0703dc1ca878f4313df83d9418e108ed Mon Sep 17 00:00:00 2001 From: Byron Date: Thu, 15 Oct 2026 05:01:58 +0200 Subject: [PATCH 026/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I didn't try to fix all the issues with this, let's just make it a little bit better hoping the code won't be around much longer. # Known limitations Commit.encoding still conflates the declared encoding with the codec used for best-effort decoding. Unsupported or malformed declarations leave the default encoding—or an earlier accepted declaration—in place, so the resulting attribute does not faithfully represent the commit’s original metadata. The rejected header bytes appear in the diagnostic but are not retained on the object. Fallback decoding with "replace" can irreversibly alter identity and message content. Reserializing such a commit can therefore change both its encoding declaration and its content; this change does not provide lossless round-tripping. Codec validation also checks only that Python accepts the codec, not that it can decode the actual commit content. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/objects/commit.py | 10 ++++++---- test/test_commit.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 0c034d231..3239079c1 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -897,14 +897,16 @@ def _deserialize(self, stream: BytesIO) -> "Commit": buf = enc.strip() while buf: if buf[0:9] == b"encoding ": - encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + raw_encoding = buf[9:] try: + # Codec names are ASCII, independent of the content encoding. + encoding = raw_encoding.decode("ascii") # Probe one byte: decoding b"" skips the codec lookup entirely. b" ".decode(encoding, "replace") except (LookupError, ValueError): - # The encoding is not a text encoding Python knows; keep the - # default so the identity and message lines still decode below. - _logger.error("Ignoring unknown commit encoding %r", encoding) + # Reject malformed names and non-text codecs without changing + # the current encoding (the default unless a valid header was read). + _logger.error("Ignoring invalid commit encoding %r", raw_encoding) else: self.encoding = encoding elif buf[0:7] == b"gpgsig ": diff --git a/test/test_commit.py b/test/test_commit.py index da8b49325..a22daa87a 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -493,6 +493,26 @@ def test_deserialization_unknown_declared_encoding(self): self.assertEqual(cmt.author.name, "Andr�") self.assertEqual(cmt.message, "R�sum�\n") + def test_deserialization_multiple_encoding_headers(self): + for headers in ( + b"encoding utf-16-be\nencoding ISO-8859-1\n", + b"encoding ISO-8859-1\nencoding utf-\xff8\n", + ): + with self.subTest(headers=headers): + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + headers + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, "ISO-8859-1") + self.assertEqual(cmt.author.name, "André") + self.assertEqual(cmt.committer.name, "André") + self.assertEqual(cmt.message, "Résumé\n") + @with_rw_directory def test_identity_cannot_alter_headers(self, rw_dir): """A name or email must not add header lines or present another identity.""" From e264eaf7ec2bfdd474012ac45b83e7c2901f868f Mon Sep 17 00:00:00 2001 From: 6801318d8d <144167388+6801318d8d@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:59:29 +0200 Subject: [PATCH 027/153] fix(index): pass a Windows hook outside the work tree by absolute path On Windows, `run_commit_hook` runs an extensionless hook through Bash and passes the hook's path relative to the working tree (`os.path.relpath`). Bash resolves that path in its own view of the filesystem, which differs from the Windows one across a mount point. Git for Windows mounts the user's temporary directory as `/tmp` (`none /tmp usertemp` in its `/etc/fstab`). For a repository under `%TEMP%` and a `core.hooksPath` outside it, such as a global `~/.config/git/hooks`, Bash starts in `/tmp/tmpXXXX/repo`, and `../../../../../.config/git/hooks/pre-commit`, correct on the Windows side, leads to `/.config/git/hooks/pre-commit`. Every commit then fails with exit code 127 and `No such file or directory`. A relative path that stays inside the working tree cannot cross such a mount, so it is kept. A path that leaves the working tree (one starting with `..`) is now passed as the absolute path in POSIX form (`C:/...`), as is already done for a hook on another drive, when the Bash is Git for Windows' own or one found on `PATH`: both accept it. The bare `bash.exe` fallback may be the WSL launcher, which cannot open a Windows absolute path, so it keeps the relative path as before. `test_run_commit_hook_outside_worktree_on_windows` now checks the three cases (outside the working tree with a found Bash, inside it, outside it with the bare fallback), and `test_pre_commit_hook_outside_worktree` runs a real hook through a `core.hooksPath` outside the working tree. Validated on Windows 11 with Git for Windows 2.50.1 and Python 3.14: `test/test_index.py` passes with an empty global Git configuration (183 passed, 3 skipped, 1 xpassed), and the new command-construction assertion fails without the change; `ruff` 0.16.9 and `mypy` report no issues on the changed files; a commit in a repository under `%TEMP%`, with a global `core.hooksPath` holding a `pre-commit` hook, succeeds instead of failing with exit code 127. Co-Authored-By: Claude Opus 5.5 --- git/index/fun.py | 21 +++++++++++----- test/test_index.py | 62 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index 2491f55ae..70c5a7e50 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -198,6 +198,11 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: if sys.platform == "win32" and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. + # Prefer Bash associated with GitPython's selected Git installation. If + # that layout is not recognized, use an explicitly configured non-system + # PATH entry. Preserve the bare fallback for installations that previously + # relied on WSL or another CreateProcess-resolved Bash. + found_bash = _git_for_windows_bash() or _which_from_path("bash.exe") try: bash_hp = osp.relpath(hp, index.repo.working_dir) except ValueError: @@ -205,12 +210,16 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: # an absolute path in this form, although a relative path is preferable # because it also works with the Windows Subsystem for Linux wrapper. bash_hp = hp - # Prefer Bash associated with GitPython's selected Git installation. If - # that layout is not recognized, use an explicitly configured non-system - # PATH entry. Preserve the bare fallback for installations that previously - # relied on WSL or another CreateProcess-resolved Bash. - bash_executable = _git_for_windows_bash() or _which_from_path("bash.exe") or "bash.exe" - cmd = [bash_executable, Path(bash_hp).as_posix()] + else: + # Bash resolves a relative path in its own view of the filesystem, + # which differs from Windows' across a mount point: Git Bash mounts + # the user's temporary directory as /tmp, so from a working tree + # there, ".." leads elsewhere. A path leaving the working tree is + # thus passed whole to a Bash found above, which accepts it; the + # bare fallback may be WSL's, which does not, and keeps it relative. + if found_bash and bash_hp.startswith(os.pardir + os.sep): + bash_hp = hp + cmd = [found_bash or "bash.exe", Path(bash_hp).as_posix()] process = safer_popen( cmd + list(args), diff --git a/test/test_index.py b/test/test_index.py index 23e59cc04..457a64ec2 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1513,6 +1513,7 @@ def test_run_commit_hook_outside_worktree_on_windows(self, rw_dir): repo = Repo.init(root / "repo") hooks_dir = root / "hooks" _make_hook(root, "fake-hook", "exit 0") + _make_hook(repo.working_dir, "fake-hook", "exit 0") system_root = root / "Windows" system_bash = system_root / "System32" / "bash.exe" git_executable = root / "Git" / "cmd" / "git.exe" @@ -1521,24 +1522,37 @@ def test_run_commit_hook_outside_worktree_on_windows(self, rw_dir): executable.parent.mkdir(parents=True) executable.touch() executable.chmod(0o755) - with repo.config_writer() as writer: - writer.set_value("core", "hooksPath", str(hooks_dir)) + + def command(hooks_path, exec_path): + with repo.config_writer() as writer: + writer.set_value("core", "hooksPath", hooks_path) + with mock.patch("git.index.fun.sys.platform", "win32"), mock.patch.object( + Git, "GIT_PYTHON_GIT_EXECUTABLE", "git" + ), mock.patch.dict(os.environ, {"SystemRoot": str(system_root)}), mock.patch( + "git.index.fun.os.get_exec_path", return_value=exec_path + ): + with mock.patch("git.index.fun.safer_popen") as popen, mock.patch( + "git.index.fun.handle_process_output" + ): + popen.return_value.returncode = 0 + run_commit_hook("fake-hook", repo.index) + return popen.call_args[0][0] # Model a normal Windows PATH: System32 (containing the WSL launcher) comes # before Git's cmd directory, while Git's Bash is not itself on PATH. This # exercises both Git-installation discovery and shell selection without # mocking either resolver's answer. - with mock.patch("git.index.fun.sys.platform", "win32"), mock.patch.object( - Git, "GIT_PYTHON_GIT_EXECUTABLE", "git" - ), mock.patch.dict(os.environ, {"SystemRoot": str(system_root)}), mock.patch( - "git.index.fun.os.get_exec_path", return_value=["", str(system_bash.parent), str(git_executable.parent)] - ): - with mock.patch("git.index.fun.safer_popen") as popen, mock.patch("git.index.fun.handle_process_output"): - popen.return_value.returncode = 0 - run_commit_hook("fake-hook", repo.index) - - command = popen.call_args[0][0] - self.assertEqual(command, [str(git_bash), "../hooks/fake-hook"]) + with_git = ["", str(system_bash.parent), str(git_executable.parent)] + # A hook outside the working tree is passed by its absolute path: a relative + # one is resolved in Bash's own view of the filesystem, where a mount point + # (Git Bash's /tmp) can make ".." lead elsewhere. + self.assertEqual(command(str(hooks_dir), with_git), [str(git_bash), (hooks_dir / "fake-hook").as_posix()]) + # A hook inside the working tree keeps its relative path. + self.assertEqual(command("hooks", with_git), [str(git_bash), "hooks/fake-hook"]) + # So does one outside it for the bare fallback, which may be WSL's bash.exe, + # unable to open a Windows absolute path. + without_git = ["", str(system_bash.parent)] + self.assertEqual(command(str(hooks_dir), without_git), ["bash.exe", "../hooks/fake-hook"]) @with_rw_directory def test_windows_bash_lookup_respects_explicit_current_directory_in_path(self, rw_dir): @@ -1671,6 +1685,28 @@ def test_pre_commit_hook_respects_core_hooks_path(self, rw_repo): output = Path(rw_repo.working_dir, "custom-hook-output.txt").read_text(encoding="utf-8") self.assertEqual(output, "ran custom hook\n") + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + raises=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.WslNoDistro, + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", + raises=HookExecutionError, + ) + @with_rw_directory + def test_pre_commit_hook_outside_worktree(self, rw_dir): + repo = Repo.init(osp.join(rw_dir, "repo")) + hp = _make_hook(rw_dir, "pre-commit", "echo 'ran outside hook' >outside-hook-output.txt") + + with repo.config_writer() as writer: + writer.set_value("core", "hooksPath", osp.dirname(hp)) + + repo.index.commit("This should run the hook outside the working tree") + output = Path(repo.working_dir, "outside-hook-output.txt").read_text(encoding="utf-8") + self.assertEqual(output, "ran outside hook\n") + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", From 833c8d8d3e5347e95f782cf1cc0b9fab60569110 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Fri, 16 Oct 2026 14:03:21 +0530 Subject: [PATCH 028/153] fix(submodule): resolve relative submodule URLs the way git does `Submodule.update` resolved a `.gitmodules` URL starting with `./` or `../` with `urllib.parse.urljoin` against the superproject's `origin` URL, and `Submodule._clone_repo` appended a `../` URL to the tracking remote's URL with `os.path.join`. Neither follows git's `relative_url`, and `urljoin` only resolves against the schemes it knows: for an `ssh://` or `git://` origin it returns the relative URL unchanged, and for an scp-like `host:path` origin more than one `../` collapses to a bare path. `git clone` then received `./planted`, a path inside the superproject's checkout, where git resolves the same entry to `ssh://host/org/super.git/planted`. A submodule was therefore cloned from a directory committed by whoever authored the superproject, through git's local clone path: `git upload-pack` ran inside that directory and its objects were copied without the `transfer.fsckObjects` checks a transport fetch applies (an object that `git clone file://...` rejects as `badDate` is accepted this way). Relative submodule URLs are common in multi-repository setups, and the superproject is untrusted input to any service that updates submodules of repositories it is given. Add `resolve_relative_url`, a port of git's `relative_url`: each leading `../` strips one component of the remote URL, a path component first, then the scp-like host before the colon, and running out of components is an error; `./` prefixes are dropped and the rest is appended. It agrees with what `git submodule init` records over 272 remote and URL pairs, including the error cases. Use it in `Submodule.update`, `Submodule._clone_repo` (which now also resolves `./` URLs, previously passed to `git clone` verbatim), and the URL change path of `RootModule.update`, which created a remote from the raw relative URL and fetched it as a path relative to the module's working tree. The remote used for resolution is unchanged: `origin` in `update`, the tracking branch's remote in `_clone_repo`. Adds regression tests in `test/test_submodule.py`; the two behavioral ones fail before the change with `./planted` reaching the clone unchanged. --- git/objects/submodule/base.py | 20 +++++--- git/objects/submodule/root.py | 12 +++-- git/objects/submodule/util.py | 94 +++++++++++++++++++++++++++++++++++ test/test_submodule.py | 63 +++++++++++++++++++++++ 4 files changed, 177 insertions(+), 12 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 74fbd39fe..d17404ea3 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -12,7 +12,6 @@ import shlex import stat import sys -import urllib.parse import uuid from io import BytesIO from pathlib import Path @@ -58,7 +57,9 @@ from .util import ( SubmoduleConfigParser, find_first_remote_branch, + is_relative_url, mkhead, + resolve_relative_url, sm_name, sm_section, ) @@ -428,10 +429,10 @@ def _clone_repo( if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - if url.startswith("../"): + if is_relative_url(url): remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name repo_remote_url = repo.remote(remote_name).url - url = os.path.join(repo_remote_url, url) + url = resolve_relative_url(repo_remote_url, url) clone = git.Repo.clone_from( url, @@ -982,13 +983,9 @@ def fetch_remotes(module_repo: "Repo") -> None: if not dry_run: if no_fetch: raise ValueError("Missing module at %r but fetching is disabled" % self.path) from None - if self.url.startswith("."): - url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) - else: - url = self.url mrepo = self._clone_repo( self.repo, - url, + self._resolved_url(), self.path, self.name, n=True, @@ -1738,6 +1735,13 @@ def url(self) -> str: """:return: The url to the repository our submodule's repository refers to""" return self._url + def _resolved_url(self) -> str: + """:return: :attr:`url`, with a ``./`` or ``../`` prefix resolved against the + url of the superproject's ``origin`` remote the way git does""" + if not is_relative_url(self.url): + return self.url + return resolve_relative_url(self.repo.remotes.origin.url, self.url) + @property def parent_commit(self) -> "Commit": """ diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index a41c50009..047f15727 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -264,10 +264,14 @@ def update( # type: ignore[override] nn = "__new_origin__" smm = sm.module() rmts = smm.remotes + # Relative urls are resolved against the superproject's remote, + # which is what the module's remotes hold. + sm_url = sm._resolved_url() + psm_url = psm._resolved_url() # Don't do anything if we already have the url we search in # place. - if len([r for r in rmts if r.url == sm.url]) == 0: + if len([r for r in rmts if r.url == sm_url]) == 0: progress.update( BEGIN | URLCHANGE, i, @@ -278,7 +282,7 @@ def update( # type: ignore[override] if not dry_run: previous_remote = None for remote in rmts: - if remote.url == psm.url: + if remote.url == psm_url: previous_remote = remote break # END if urls match @@ -305,10 +309,10 @@ def update( # type: ignore[override] # A new remote would have no cached refs. Preserve # the existing refs and tracking configuration for # offline updates instead of replacing the remote. - previous_remote.set_url(git.Git.polish_url(sm.url, expand_vars=False)) + previous_remote.set_url(git.Git.polish_url(sm_url, expand_vars=False)) else: assert nn not in [r.name for r in rmts] - smr = smm.create_remote(nn, sm.url) + smr = smm.create_remote(nn, sm_url) smr.fetch(progress=progress) # If we have a tracking branch, it should be available diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index c021510d8..9856e39ce 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -6,10 +6,13 @@ "sm_name", "mkhead", "find_first_remote_branch", + "is_relative_url", + "resolve_relative_url", "SubmoduleConfigParser", ] from io import BytesIO +import sys import weakref import git @@ -63,6 +66,97 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " raise InvalidGitRepositoryError("Didn't find remote branch '%r' in any of the given remotes" % branch_name) +def _is_dir_sep(char: str) -> bool: + return char == "/" or (sys.platform == "win32" and char == "\\") + + +def _starts_with_dot_slash(url: str) -> bool: + return len(url) >= 2 and url[0] == "." and _is_dir_sep(url[1]) + + +def _starts_with_dot_dot_slash(url: str) -> bool: + return len(url) >= 3 and url[0:2] == ".." and _is_dir_sep(url[2]) + + +def _has_dos_drive_prefix(url: str) -> bool: + return sys.platform == "win32" and len(url) >= 2 and url[0].isalpha() and url[1] == ":" + + +def _is_absolute_path(url: str) -> bool: + return (len(url) >= 1 and _is_dir_sep(url[0])) or _has_dos_drive_prefix(url) + + +def _url_is_local_not_ssh(url: str) -> bool: + colon = url.find(":") + slash = url.find("/") + return colon == -1 or (slash != -1 and slash < colon) or _has_dos_drive_prefix(url) + + +def _find_last_dir_sep(url: str) -> int: + for index in range(len(url) - 1, -1, -1): + if _is_dir_sep(url[index]): + return index + return -1 + + +def is_relative_url(url: str) -> bool: + """:return: ``True`` if `url` starts with ``./`` or ``../``, which git resolves + against the superproject's remote rather than treating as a path""" + return _starts_with_dot_slash(url) or _starts_with_dot_dot_slash(url) + + +def resolve_relative_url(remote_url: str, url: str) -> str: + """Resolve a submodule URL that starts with ``./`` or ``../`` against the + superproject's remote URL, the way git's ``relative_url`` does. + + Git treats a relative submodule URL like a relative directory: each leading + ``../`` removes the last component of `remote_url`, whether that is a path + component, the host of a URL, or the host of an scp-like ``host:path`` address. + The remainder is appended with ``/``, or with ``:`` once an scp-like host was + reached. A URL that is not relative is returned unchanged. + + :raise ValueError: + If more components are stripped than `remote_url` has, which git rejects with + "cannot strip one component off url". + """ + if not is_relative_url(url): + return url + + remote = remote_url + if remote and _is_dir_sep(remote[-1]): + remote = remote[:-1] + + is_relative = _url_is_local_not_ssh(remote) and not _is_absolute_path(remote) + if is_relative and not _starts_with_dot_slash(remote) and not _starts_with_dot_dot_slash(remote): + remote = "./" + remote + + colonsep = False + while url: + if _starts_with_dot_dot_slash(url): + url = url[3:] + separator = _find_last_dir_sep(remote) + if separator != -1: + remote = remote[:separator] + elif ":" in remote: + remote = remote[: remote.rfind(":")] + colonsep = True + elif is_relative or remote == ".": + raise ValueError("cannot strip one component off url %r" % remote) + else: + remote = "." + elif _starts_with_dot_slash(url): + url = url[2:] + else: + break + + resolved = remote + (":" if colonsep else "/") + url + if url.endswith("/"): + resolved = resolved[:-1] + if _starts_with_dot_slash(resolved): + resolved = resolved[2:] + return resolved + + # } END utilities # { Classes diff --git a/test/test_submodule.py b/test/test_submodule.py index 45b285c55..97d51d6a0 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -26,6 +26,7 @@ ) from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress +from git.objects.submodule.util import resolve_relative_url from git.repo.fun import find_submodule_git_dir, touch from git.util import HIDE_WINDOWS_KNOWN_ERRORS, cwd, join_path_native, to_native_path_linux @@ -2416,3 +2417,65 @@ def test_submodule_update_relative_url(self, rwdir): has_module = any(sm.name == "module" for sm in cloned_repo.submodules) assert has_module, "Relative submodule was not updated properly" + + def test_resolve_relative_url_like_git(self): + # Expected values are what `git submodule init` records for the same pair. + cases = [ + ("ssh://host/org/super.git", "./sub", "ssh://host/org/super.git/sub"), + ("ssh://host/org/super.git", "../sub", "ssh://host/org/sub"), + ("ssh://host:22/org/super.git", "../sub", "ssh://host:22/org/sub"), + ("git@host:org/super.git", "./sub", "git@host:org/super.git/sub"), + ("git@host:org/super.git", "../sub", "git@host:org/sub"), + ("git@host:org/super.git", "../../sub", "git@host:sub"), + ("https://host/org/super.git", "./sub/", "https://host/org/super.git/sub"), + ("https://host/org/super.git/", "././sub", "https://host/org/super.git/sub"), + ("https://host/org/super.git", ".././sub", "https://host/org/sub"), + ("https://host/org/super.git", "../../../sub", "https://sub"), + ("file:///srv/git/super.git", "../sub", "file:///srv/git/sub"), + ("/srv/git/super", "../sub", "/srv/git/sub"), + ("/srv/git/super", "../../../../sub", "sub"), + ("//share/super", "../../sub", "//sub"), + ("foo/bar", "../sub", "foo/sub"), + ("./foo", "../sub", "sub"), + ("../foo/bar", "../sub", "../foo/sub"), + ("ssh://host/org/super.git", ".sub", ".sub"), + ("ssh://host/org/super.git", "https://other/sub", "https://other/sub"), + ] + for remote_url, url, expected in cases: + assert resolve_relative_url(remote_url, url) == expected, (remote_url, url) + + for remote_url, url in [("foo", "../../sub"), ("git@host:super.git", "../../../sub"), ("../foo", "../../sub")]: + with self.assertRaisesRegex(ValueError, "cannot strip one component"): + resolve_relative_url(remote_url, url) + + @with_rw_directory + def test_update_resolves_relative_url_against_remote(self, rwdir): + parent = git.Repo.init(osp.join(rwdir, "parent")) + parent.create_remote("origin", "ssh://git@host/org/super.git") + os.mkdir(osp.join(parent.working_tree_dir, "planted")) + + for url, expected in [ + ("./planted", "ssh://git@host/org/super.git/planted"), + ("../planted", "ssh://git@host/org/planted"), + ]: + submodule = Submodule(parent, Submodule.NULL_BIN_SHA, name="module", path="module", url=url) + with mock.patch.object(Submodule, "_clone_repo", side_effect=AssertionError("stop")) as clone_repo: + with pytest.raises(AssertionError, match="stop"): + submodule.update(init=True) + assert clone_repo.call_args.args[1] == expected + + @with_rw_directory + def test_clone_repo_resolves_relative_url_against_tracking_remote(self, rwdir): + parent = git.Repo.init(osp.join(rwdir, "parent")) + parent.index.commit("initial") + parent.create_remote("origin", "git@host:org/super.git") + parent.head.reference.set_tracking_branch(git.RemoteReference(parent, "refs/remotes/origin/main")) + + for url, expected in [ + ("./planted", "git@host:org/super.git/planted"), + ("../planted", "git@host:org/planted"), + ]: + with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("stop")) as clone_from: + with pytest.raises(AssertionError, match="stop"): + Submodule._clone_repo(parent, url, "module", "module") + assert clone_from.call_args.args[0] == expected From add9920ad62af88db369c6451a1dc9400eebb814 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Fri, 16 Oct 2026 21:24:11 +0530 Subject: [PATCH 029/153] fix(tag): split tag object headers at line feeds only `TagObject._set_cache_` split the decoded tag object with `str.splitlines()` and then read the headers by position, taking `lines[3]` as the tagger. `str.splitlines()` also breaks at `\r`, `\v`, `\f`, `\x1c`-`\x1e`, `\x85`, U+2028 and U+2029, but git ends a header only at a line feed. A U+2028 inside the `tag` header of an untrusted tag object therefore shifts every later header by one line in GitPython only. With `tag v1tagger Linus Torvalds 1700000000 +0000` followed by a real `tagger Mallory ...` line, `git cat-file tag`, `git show` and `git for-each-ref --format=%(taggername)` report Mallory, while GitPython reported `tag == "v1"` and Linus Torvalds as the tagger. The object passes `git fsck --strict` and a clone with `transfer.fsckObjects=true`, because git only warns `badTagName`. The fix splits at `"\n"` and drops the trailing empty element, so tags that only use line feeds parse exactly as before. A `\r` or U+2028 inside the message is now kept, as git shows it. The new test in `test/test_refs.py` fails before the change and passes after it. `ruff`, `ruff format --check` and `mypy` are clean. On my machine the full suite has 3 failures (`test_empty_repo`, `test_rebasing`, `test_diff_with_staged_file`), and all 3 also fail without this change. --- git/objects/tag.py | 11 ++++++++++- test/test_refs.py | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/git/objects/tag.py b/git/objects/tag.py index e08d6ef10..2dbf79f1b 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -106,7 +106,16 @@ def _set_cache_(self, attr: str) -> None: """Cache all our attributes at once.""" if attr in TagObject.__slots__: ostream = self.repo.odb.stream(self.binsha) - lines: List[str] = ostream.read().decode(defenc, "replace").splitlines() + # Git ends a header only at a line feed, but str.splitlines() also + # breaks at "\r", "\v", "\f", U+2028 and similar. Splitting a tag + # object that way lets an untrusted "tag" header containing such a + # character shift every later header by a line. Split at "\n" only, + # and drop the trailing empty element so tags that use line feeds + # parse exactly as before. + data = ostream.read().decode(defenc, "replace") + lines: List[str] = data.split("\n") + if lines and lines[-1] == "": + lines.pop() _obj, hexsha = lines[0].split(" ") _type_token, type_name = lines[1].split(" ") diff --git a/test/test_refs.py b/test/test_refs.py index 2edb7b766..42bf5200e 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -7,10 +7,12 @@ import os.path as osp import tempfile import unittest +from io import BytesIO from itertools import chain from pathlib import Path from unittest.mock import patch +from gitdb import IStream from gitdb.exc import BadName import git.refs as refs @@ -139,6 +141,31 @@ def test_tags_author(self): tagger_name = tagobj.tagger.name assert tagger_name == "Michael Trier" + def test_tag_headers_split_at_line_feeds_only(self): + # A "tag" header value carrying a U+2028 (or any separator that + # str.splitlines() honors but git does not) must not shift the later + # headers. Git ends a header only at "\n", so GitPython must read the + # whole value as the tag name and the following line as the tagger, + # exactly as "git cat-file tag" does. See the TagObject parsing fix. + with tempfile.TemporaryDirectory() as tmp_dir: + with self._repo_with_initial_commit(Path(tmp_dir)) as repo: + commit_hexsha = repo.head.commit.hexsha + payload = ( + f"object {commit_hexsha}\n" + "type commit\n" + "tag v1
tagger Linus Torvalds 1700000000 +0000\n" + "tagger Mallory 1700000001 +0000\n" + "\n" + "release\n" + ).encode("utf-8") + istream = repo.odb.store(IStream("tag", len(payload), BytesIO(payload))) + tagobj = TagObject(repo, istream.binsha) + + assert tagobj.tag == "v1
tagger Linus Torvalds 1700000000 +0000" + assert tagobj.tagger.name == "Mallory" + assert tagobj.tagger.email == "m@evil.example" + assert tagobj.message == "release" + def test_tags(self): # Tag refs can point to tag objects or to commits. s = set() From 0a5938dc55c569dbf5f5c60244106ef2011cefdb Mon Sep 17 00:00:00 2001 From: Byron Date: Fri, 16 Oct 2026 11:18:29 +0200 Subject: [PATCH 030/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's basically a rubber-stamp of the review comments, hoping this code will be nothing more than interesting tests for the `gix-backend` branch, so the implementation here won't matter. - [P2] Fall back when a relative URL has no tracking remote — git/objects/submodule/base.py:432-435 For a locally initialized superproject without a tracking branch, `create_submodule(..., url='./source')` now enters this block and dereferences `None`, raising `AttributeError` even when `./source` is a valid repository. This addition succeeded before the patch; detached checkouts also fail at `active_branch`. Handle missing tracking branches by falling back to `origin` or the superproject directory, as Git does. - [P2] Resolve relative submodule URLs only once — git/objects/submodule/base.py:986-988 When `origin` is a local relative URL such as `../repos/super.git` and the branch tracks that remote, `_resolved_url()` turns `../child.git` into `../repos/child.git`. Because that result still starts with `../`, `_clone_repo()` resolves it again, producing `../repos/repos/child.git`. This can fail cloning or select the wrong repository. Ensure URLs already resolved by `update()` bypass the second resolution. - [P2] Defer old-URL resolution until a remote change is needed — git/objects/submodule/root.py:269-270 For an initialized superproject without `origin`, changing `.gitmodules` from `../source` to the absolute URL already used by the submodule's remote now raises `AttributeError` in `psm._resolved_url()`. Previously, the existing-remote check skipped URL handling and the update completed. Resolve the previous URL only after checking whether the new URL already matches a remote, so updates requiring no remote adjustment do not fail. --- git/objects/submodule/base.py | 46 ++++++---- git/objects/submodule/root.py | 34 +++---- test/test_submodule.py | 165 +++++++++++++++++++++++++++++++--- 3 files changed, 196 insertions(+), 49 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d17404ea3..00bb7e226 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -56,6 +56,8 @@ from .util import ( SubmoduleConfigParser, + _is_absolute_path, + _url_is_local_not_ssh, find_first_remote_branch, is_relative_url, mkhead, @@ -69,7 +71,7 @@ if TYPE_CHECKING: from git.index import IndexFile from git.objects.commit import Commit - from git.refs import Head, RemoteReference + from git.refs import Head from git.repo import Repo # ----------------------------------------------------------------------------- @@ -429,13 +431,8 @@ def _clone_repo( if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - if is_relative_url(url): - remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name - repo_remote_url = repo.remote(remote_name).url - url = resolve_relative_url(repo_remote_url, url) - clone = git.Repo.clone_from( - url, + cls._resolve_url(repo, url), module_checkout_path, allow_unsafe_options=allow_unsafe_options, allow_unsafe_protocols=allow_unsafe_protocols, @@ -747,8 +744,9 @@ def add( ) # END verify url - ## See #525 for ensuring git URLs in config-files are valid under Windows. - url = Git.polish_url(url, expand_vars=False) + # Relative submodule URLs must keep their prefix to resolve against the parent remote. + if not is_relative_url(url): + url = Git.polish_url(url, expand_vars=False) # It's important to add the URL to the parent config, to let `git submodule` know. # Otherwise there is a '-' character in front of the submodule listing: @@ -985,7 +983,7 @@ def fetch_remotes(module_repo: "Repo") -> None: raise ValueError("Missing module at %r but fetching is disabled" % self.path) from None mrepo = self._clone_repo( self.repo, - self._resolved_url(), + self.url, self.path, self.name, n=True, @@ -1735,12 +1733,30 @@ def url(self) -> str: """:return: The url to the repository our submodule's repository refers to""" return self._url + @staticmethod + def _resolve_url(repo: "Repo", url: str) -> str: + """Resolve relative URLs against the tracking remote, then ``origin``, + falling back to the superproject's directory the way git does.""" + if not is_relative_url(url): + return url + remote_name = "origin" + if not repo.head.is_detached: + with repo.active_branch.config_reader() as reader: + remote_name = reader.get("remote", fallback="origin") + try: + remote_url = repo.remote(remote_name).url + except ValueError: + remote_url = os.fspath(repo.working_dir) + if remote_url == ".": + remote_url = os.fspath(repo.working_dir) + resolved = resolve_relative_url(remote_url, url) + if resolved and _url_is_local_not_ssh(resolved) and not _is_absolute_path(resolved): + resolved = os.path.join(repo.working_dir, resolved) + return resolved + def _resolved_url(self) -> str: - """:return: :attr:`url`, with a ``./`` or ``../`` prefix resolved against the - url of the superproject's ``origin`` remote the way git does""" - if not is_relative_url(self.url): - return self.url - return resolve_relative_url(self.repo.remotes.origin.url, self.url) + """:return: :attr:`url` resolved against the superproject's default remote""" + return self._resolve_url(self.repo, self.url) @property def parent_commit(self) -> "Commit": diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 047f15727..c1fb4d537 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -267,7 +267,6 @@ def update( # type: ignore[override] # Relative urls are resolved against the superproject's remote, # which is what the module's remotes hold. sm_url = sm._resolved_url() - psm_url = psm._resolved_url() # Don't do anything if we already have the url we search in # place. @@ -281,28 +280,21 @@ def update( # type: ignore[override] if not dry_run: previous_remote = None - for remote in rmts: - if remote.url == psm_url: - previous_remote = remote - break - # END if urls match - # END for each remote + if len(rmts) == 1: + previous_remote = rmts[0] + else: + psm_url = psm._resolved_url() + for remote in rmts: + if remote.url == psm_url: + previous_remote = remote + break + # END if urls match + # END for each remote - # If we didn't find a matching remote, but have exactly - # one, we can safely use this one. if previous_remote is None: - if len(rmts) == 1: - previous_remote = rmts[0] - else: - # If we have not found any remote with the - # original URL we may not have a name. This is a - # special case, and its okay to fail here. - # Alternatively we could just generate a unique - # name and leave all existing ones in place. - raise InvalidGitRepositoryError( - "Couldn't find original remote-repo at url %r" % psm.url - ) - # END handle one single remote + raise InvalidGitRepositoryError( + "Couldn't find original remote-repo at url %r" % psm.url + ) # END handle check we found a remote if no_fetch: diff --git a/test/test_submodule.py b/test/test_submodule.py index 97d51d6a0..c09f0a8e1 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -53,6 +53,138 @@ def _patch_git_config(name, value): yield +@pytest.mark.parametrize("state", ["unborn", "untracked", "detached"]) +@pytest.mark.parametrize("remote", [None, ".", "origin"], ids=["working-tree", "local-origin", "origin"]) +def test_add_relative_submodule_without_tracking_remote(tmp_path, state, remote): + with git.Repo.init(tmp_path / "parent") as parent: + if state != "unborn": + parent.index.commit("Initial commit") + if state == "detached": + parent.head.reference = parent.head.commit + base = tmp_path / "remote" if remote == "origin" else tmp_path / "parent" + if remote is not None: + parent.create_remote("origin", str(base) if remote == "origin" else ".") + with git.Repo.init(base / "source") as source: + source.git.symbolic_ref("HEAD", "refs/heads/master") + source.index.commit("Initial commit") + submodule = parent.create_submodule("module", "module", "./source") + assert submodule.url == "./source" + with submodule.module() as module: + assert module.head.commit == source.head.commit + assert Path(module.remotes.origin.url).samefile(source.working_tree_dir) + parent.index.commit("Add submodule") + submodule.remove(configuration=False) + submodule.update(init=True) + with submodule.module() as module: + assert module.head.commit == source.head.commit + assert Path(module.remotes.origin.url).samefile(source.working_tree_dir) + + +@pytest.mark.parametrize("url", ["./source", "../source", "./../source"]) +def test_add_preserves_relative_url_on_cygwin(tmp_path, url): + with git.Repo.init(tmp_path / "parent") as parent, git.Repo.init(tmp_path / "source") as source: + source.index.commit("Initial commit") + with mock.patch.object(Submodule, "_clone_repo", return_value=source): + with mock.patch.object(Git, "is_cygwin", return_value=True): + submodule = parent.create_submodule("module", "module", url) + assert submodule.url == url + with submodule.config_reader() as reader: + assert reader.get_value("url") == url + with parent.config_reader() as reader: + assert reader.get_value('submodule "module"', "url") == url + + +def test_update_resolves_local_relative_remote_once(tmp_path): + with git.Repo.init(tmp_path / "parent") as parent, git.Repo.init(tmp_path / "repos" / "child.git") as source: + source.git.symbolic_ref("HEAD", "refs/heads/master") + source.index.commit("Initial commit") + parent.create_remote("origin", "../repos/super.git") + parent.head.reference.set_tracking_branch(git.RemoteReference(parent, "refs/remotes/origin/master")) + submodule = Submodule( + parent, + source.head.commit.binsha, + name="module", + path="module", + url="../child.git", + branch_path="refs/heads/master", + ) + submodule.update(init=True) + with submodule.module() as module: + assert module.head.commit == source.head.commit + assert Path(module.remotes.origin.url).samefile(source.working_tree_dir) + module.remotes.origin.fetch() + + +@pytest.mark.parametrize("no_fetch", [False, True]) +def test_root_update_anchors_relative_remote_url(tmp_path, no_fetch): + with git.Repo.init(tmp_path / "repos" / "source.git") as source, git.Repo.init(tmp_path / "parent") as parent: + source.git.symbolic_ref("HEAD", "refs/heads/master") + source.index.commit("Initial commit") + with git.Repo.clone_from(source.working_tree_dir, tmp_path / "repos" / "replacement.git") as replacement: + parent.create_remote("origin", "../repos/super.git") + submodule = parent.create_submodule("module", "module", source.working_tree_dir) + with submodule.config_writer() as writer: + writer.set_value("url", "../source.git") + previous = parent.index.commit("Record relative URL") + with submodule.config_writer() as writer: + writer.set_value("url", "../replacement.git") + parent.index.commit("Change relative URL") + + RootModule(parent).update(previous_commit=previous, no_fetch=no_fetch) + + with submodule.module() as module: + assert Path(module.remotes.origin.url).samefile(replacement.working_tree_dir) + assert module.head.commit == replacement.head.commit + module.remotes.origin.fetch() + + +@pytest.mark.parametrize("no_fetch", [False, True]) +def test_root_update_uses_single_remote_without_resolving_old_url(tmp_path, no_fetch): + with git.Repo.init(tmp_path / "source") as source, git.Repo.init(tmp_path / "parent") as parent: + source.git.symbolic_ref("HEAD", "refs/heads/master") + source.index.commit("Initial commit") + with git.Repo.clone_from(source.working_tree_dir, tmp_path / "replacement") as replacement: + parent.create_remote("origin", "../super.git") + submodule = parent.create_submodule("module", "module", source.working_tree_dir) + with submodule.config_writer() as writer: + writer.set_value("url", "../../lib.git") + previous = parent.index.commit("Record historical URL") + with submodule.config_writer() as writer: + writer.set_value("url", replacement.working_tree_dir) + parent.index.commit("Replace historical URL") + + RootModule(parent).update(previous_commit=previous, no_fetch=no_fetch) + + with submodule.module() as module: + assert Path(module.remotes.origin.url).samefile(replacement.working_tree_dir) + assert module.head.commit == replacement.head.commit + + +@pytest.mark.parametrize("dry_run", [False, True]) +def test_root_update_skips_old_url_resolution_when_remote_matches(tmp_path, dry_run): + with git.Repo.init(tmp_path / "source") as source, git.Repo.init(tmp_path / "parent") as parent: + source.index.commit("Initial commit") + submodule = parent.create_submodule("module", "module", source.working_tree_dir) + with submodule.config_writer() as writer: + writer.set_value("url", "../source") + previous = parent.index.commit("Record relative submodule URL") + with submodule.config_writer() as writer: + writer.set_value("url", Git.polish_url(source.working_tree_dir)) + parent.index.commit("Use existing remote URL") + + resolve_url = Submodule._resolved_url + + def resolve_current_url(sm): + assert sm.url != "../source", "Previous URL does not need resolving" + return resolve_url(sm) + + with mock.patch.object(Submodule, "_resolved_url", autospec=True, side_effect=resolve_current_url): + RootModule(parent).update(previous_commit=previous, no_fetch=True, dry_run=dry_run) + with submodule.module() as module: + assert module.remotes.origin.url == Git.polish_url(source.working_tree_dir) + assert module.head.commit == source.head.commit + + @pytest.mark.parametrize( "name", ["module", "$GITPYTHON_TEST_SECRET", "prefix-${GITPYTHON_TEST_SECRET}-suffix", "%GITPYTHON_TEST_SECRET%"] ) @@ -2437,6 +2569,9 @@ def test_resolve_relative_url_like_git(self): ("//share/super", "../../sub", "//sub"), ("foo/bar", "../sub", "foo/sub"), ("./foo", "../sub", "sub"), + ("./super.git", "../", ""), + ("ssh://host/org/super.git", "./", "ssh://host/org/super.git/"), + ("ssh://host/org/super.git", "../", "ssh://host/org/"), ("../foo/bar", "../sub", "../foo/sub"), ("ssh://host/org/super.git", ".sub", ".sub"), ("ssh://host/org/super.git", "https://other/sub", "https://other/sub"), @@ -2459,23 +2594,27 @@ def test_update_resolves_relative_url_against_remote(self, rwdir): ("../planted", "ssh://git@host/org/planted"), ]: submodule = Submodule(parent, Submodule.NULL_BIN_SHA, name="module", path="module", url=url) - with mock.patch.object(Submodule, "_clone_repo", side_effect=AssertionError("stop")) as clone_repo: + with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("stop")) as clone_from: with pytest.raises(AssertionError, match="stop"): submodule.update(init=True) - assert clone_repo.call_args.args[1] == expected + assert clone_from.call_args.args[0] == expected @with_rw_directory def test_clone_repo_resolves_relative_url_against_tracking_remote(self, rwdir): parent = git.Repo.init(osp.join(rwdir, "parent")) parent.index.commit("initial") - parent.create_remote("origin", "git@host:org/super.git") - parent.head.reference.set_tracking_branch(git.RemoteReference(parent, "refs/remotes/origin/main")) - - for url, expected in [ - ("./planted", "git@host:org/super.git/planted"), - ("../planted", "git@host:org/planted"), - ]: - with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("stop")) as clone_from: - with pytest.raises(AssertionError, match="stop"): - Submodule._clone_repo(parent, url, "module", "module") - assert clone_from.call_args.args[0] == expected + parent.create_remote("origin", "git@host:other/super.git") + for remote_name in ("upstream", "team/upstream"): + parent.create_remote(remote_name, "git@host:org/super.git") + with parent.head.reference.config_writer() as writer: + writer.set_value("remote", remote_name) + writer.set_value("merge", "refs/heads/main") + + for url, expected in [ + ("./planted", "git@host:org/super.git/planted"), + ("../planted", "git@host:org/planted"), + ]: + with mock.patch.object(git.Repo, "clone_from", side_effect=AssertionError("stop")) as clone_from: + with pytest.raises(AssertionError, match="stop"): + Submodule._clone_repo(parent, url, "module", "module") + assert clone_from.call_args.args[0] == expected From 7582674e201a82ad519cf37c09c16a08e64ed49e Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Sat, 17 Oct 2026 15:37:39 -0300 Subject: [PATCH 031/153] Make de api reference reproducible The API reference uses `:special-members:`, so autodoc also documents the `__dict__` of the `TypedDict` classes in `git.types`. Its repr is a `mappingproxy` containing `frozenset` objects, whose iteration order depends on string hash randomization. As a result `reference.html` differs from one build to another. Exclude `__dict__` and `__weakref__` by default. They only hold the class namespace and carry no useful documentation. The `__required_keys__` and similar attributes are still documented, since Sphinx sorts sets when it renders them on their own. The original issue was reported https://bugs.debian.org/1100149 --- doc/source/conf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/conf.py b/doc/source/conf.py index 809762483..edf488e72 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,6 +111,11 @@ # This file should be a Windows icon file (.ico) being 16x16 or 32x32 pixels large. # html_favicon = None +# Do not documento the attributes holidng a class's namespace. Their reprs embed ++# frozensets whose order depends on string hashing, which makes the generated ++# HTML vary between builds. ++autodoc_default_options = {"exclude-members": "__dict__, __weakref__"} + # Add any paths that contain custom static files (such as style sheets) here, relative # to this directory. They are copied after the builtin static files, so a file named # "default.css" will overwrite the builtin "default.css". From a3241fa7d48ea1a422d969d54101e813689014f1 Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Sat, 17 Oct 2026 15:48:30 -0300 Subject: [PATCH 032/153] Fix autodoc_default_options in conf.py Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- doc/source/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index edf488e72..a8a5427ba 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -112,9 +112,9 @@ # html_favicon = None # Do not documento the attributes holidng a class's namespace. Their reprs embed -+# frozensets whose order depends on string hashing, which makes the generated -+# HTML vary between builds. -+autodoc_default_options = {"exclude-members": "__dict__, __weakref__"} +# frozensets whose order depends on string hashing, which makes the generated +# HTML vary between builds. +autodoc_default_options = {"exclude-members": "__dict__, __weakref__"} # Add any paths that contain custom static files (such as style sheets) here, relative # to this directory. They are copied after the builtin static files, so a file named From d8c0e7bdcd6bd2932c16c8c073833a9f88d78f3c Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Sat, 17 Oct 2026 16:35:36 +0530 Subject: [PATCH 033/153] fix(diff): read raw diff records field by field `Diff._handle_diff_line` parses `git diff --raw -z` output by splitting it at every NUL followed by a colon, and `Diff._index_from_raw_format` called it once per line of stdout. With `-z`, git writes paths verbatim, so a path may start with `:` or contain a line feed, and either one breaks the record framing: - A path starting with `:` is taken as the start of a new record. Its remainder is parsed as metadata, the resulting `ValueError` is raised inside the stdout pump thread, and the caller gets back a `DiffIndex` with a pathless entry followed by nothing: every later entry is dropped, and only a log message records the failure. For a commit adding `:planted`, `docs/b` and `setup.py`, `commit.diff(NULL_TREE)` returned a single entry with `a_path == None`. - A path containing a line feed is truncated at the line feed. If the rest of the path contains a colon, it is parsed as a separate record: `docs/a\n:100644 100644 111... 222... M` produced an entry `docs/a` plus a fabricated `M` entry whose blob ids came from the file name. Paths are chosen by whoever authored the commit. A tool that uses `Commit.diff` to learn which files a commit touches can therefore be made to miss changes, or to see changes that were never made. `_index_from_raw_format` now collects all of stdout before parsing, as `_index_from_patch_format` already does, and `_handle_diff_line` reads the NUL-separated fields in order: a metadata field, then one path, or two for copies and renames. A single trailing line feed is still stripped so the existing non-`-z` fixtures parse as before. The new test in `test/test_diff.py` fails before the change and passes after it. `ruff check`, `ruff format --check` and `mypy` are clean. In a full clone the suite has 6 failures (`test_diff_with_staged_file`, `test_index_mutation`, `test_empty_repo`, `test_rebasing` and the two `test_actor_get_uid_laziness_*` tests), and all 6 also fail without this change. --- git/diff.py | 33 +++++++++++++++++---------------- test/test_diff.py | 19 ++++++++++++++++++- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/git/diff.py b/git/diff.py index b28a29529..123c22840 100644 --- a/git/diff.py +++ b/git/diff.py @@ -733,12 +733,17 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"] # Discard everything before the first colon, and the colon itself. _, _, lines = lines.partition(":") - for line in lines.split("\x00:"): - if not line: + # Each record is a metadata field starting with a colon, followed by one path, + # or two for copies and renames, each terminated by NUL. A path may itself + # start with a colon or contain a line feed, so read the fields in order + # instead of splitting at colons. + fields = iter(lines.rstrip("\n").split("\x00")) + for meta in fields: + if not meta: # The line data is empty, skip. continue - meta, _, path = line.partition("\x00") - path = path.rstrip("\x00") + if meta.startswith(":"): + meta = meta[1:] a_blob_id: Optional[str] b_blob_id: Optional[str] old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) @@ -748,7 +753,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"] change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None - path = path.strip("\n") + path = next(fields, "") a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False @@ -767,13 +772,9 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"] new_file = True elif change_type == "C": copied_file = True - a_path_str, b_path_str = path.split("\x00", 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) + b_path = next(fields, "").encode(defenc) elif change_type == "R": - a_path_str, b_path_str = path.split("\x00", 1) - a_path = a_path_str.encode(defenc) - b_path = b_path_str.encode(defenc) + b_path = next(fields, "").encode(defenc) rename_from, rename_to = a_path, b_path elif change_type == "T": # Nothing to do. @@ -817,17 +818,17 @@ def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff] # :100644 100644 687099101... 37c5e30c8... M .gitignore index: "DiffIndex" = DiffIndex() + text_list: List[bytes] = [] stderr_list: List[bytes] = [] def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None: finalize_process(proc, stderr=b"".join(stderr_list)) + # Output is read line by line, but paths may contain line feeds, so parse + # the records only once all of it has been read. handle_process_output( - proc, - lambda byt: cls._handle_diff_line(byt, repo, index), - stderr_list.append, - finalize_process_with_stderr, - decode_streams=False, + proc, text_list.append, stderr_list.append, finalize_process_with_stderr, decode_streams=False ) + cls._handle_diff_line(b"".join(text_list), repo, index) return index diff --git a/test/test_diff.py b/test/test_diff.py index bbaec72cc..06399b823 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -4,6 +4,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc +from io import BytesIO import os.path as osp import shutil import sys @@ -12,10 +13,13 @@ import ddt import pytest -from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule +from gitdb.base import IStream + +from git import NULL_TREE, Blob, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule from git.cmd import Git from git.diff import decode_path from git.exc import UnsafeOptionError +from git.index.typ import BaseIndexEntry from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory @@ -253,6 +257,19 @@ def test_diff_file_with_colon(self): Diff._handle_diff_line(output, None, res) self.assertEqual(len(res), 3) + @with_rw_directory + def test_diff_raw_paths_with_colon_or_line_feed(self, rw_dir): + repo = Repo.init(rw_dir) + binsha = repo.odb.store(IStream(Blob.type, 1, BytesIO(b"x"))).binsha + fake_meta = ":100644 100644 %s %s M" % ("1" * 40, "2" * 40) + paths = [":planted", "docs/a\n" + fake_meta, "docs/b", "setup.py"] + repo.index.add([BaseIndexEntry((0o100644, binsha, 0, path)) for path in paths]) + commit = repo.index.commit("paths") + + diffs = commit.diff(NULL_TREE) + self.assertEqual([d.b_path for d in diffs], sorted(paths)) + self.assertEqual({d.change_type for d in diffs}, {"A"}) + def test_empty_diff(self): res = [] Diff._handle_diff_line(b"", None, res) From 2a176d9dadac1ba7ed56ac87985100c095251e30 Mon Sep 17 00:00:00 2001 From: Emmanuel Arias Date: Sat, 17 Oct 2026 19:20:12 -0300 Subject: [PATCH 034/153] Fix typo in documentation comment Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a8a5427ba..032e55cfe 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,7 +111,7 @@ # This file should be a Windows icon file (.ico) being 16x16 or 32x32 pixels large. # html_favicon = None -# Do not documento the attributes holidng a class's namespace. Their reprs embed +# Do not document the attributes holding a class's namespace. Their reprs embed # frozensets whose order depends on string hashing, which makes the generated # HTML vary between builds. autodoc_default_options = {"exclude-members": "__dict__, __weakref__"} From 53df5960072f75a3dff4d9ac0ed91ba053da2050 Mon Sep 17 00:00:00 2001 From: Byron Date: Sun, 18 Oct 2026 14:14:28 +0200 Subject: [PATCH 035/153] review No changes, just notes: All this is probably beating a dead horse, but it can't be bad to have an additional test for this. Notably, the encoding/decoding portions of this are likely to fail as well. From ffefd4ffb380327687da706a389c1835c6907f06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:33:02 +0000 Subject: [PATCH 036/153] build(deps): bump astral-sh/setup-uv from 10.2.0 to 10.3.0 Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/c18668ad3cf93ea998bef934396af7bb5c839dc7...1c37ad07a6a961277cf70c0d37d6f313000f5884) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 10.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index b4046dd9b..4eff39b3c 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -48,7 +48,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false version: "0.12.13" # Released on 2026-09-10 From feda2411bb4d827093a35fdcc19ef70c52d91d92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:33:08 +0000 Subject: [PATCH 037/153] build(deps): bump Vampire/setup-wsl from 7.0.0 to 7.2.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 7.0.0 to 7.2.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v7.0.0...v7.2.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-version: 7.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3fbbe2ec6..94630dbfc 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -54,7 +54,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v7.0.0 + uses: Vampire/setup-wsl@v7.2.0 with: wsl-version: 1 From c7ce9843247c5efa788830060921afb1435a3d00 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Mon, 19 Oct 2026 01:07:28 +0530 Subject: [PATCH 038/153] fix(diff): read patch header paths the way git apply does `Diff._index_from_patch_format` takes a file's paths from the `diff --git a/ b/` line whenever a patch header has no `---` and `+++` lines, which is the case for binary files, empty files and mode-only changes. It also did so for both names of a copy, because the `copy from` line was not captured and the `copy to` line only set `copied_file`. `Diff.re_header` splits that line at the first space followed by `a/` or `b/`, but git does not quote a path for containing a space, so a path containing ` a/` or ` b/` is split in the wrong place: - A binary file `assets b/logo.png` is reported with `a_path` `assets` and `b_path` `logo.png b/assets b/logo.png`; an empty file `docs a/empty` comes back as `docs` and `empty b/docs a/empty`. - A copy of `assets b/logo.png` to `copy b/logo.png` comes back as `assets` and `logo.png b/copy b/logo.png`. - For a quoted name such as `"a/quoted\t a/tab"`, the first name is cut at the space inside the quotes, `decode_path` fails its prefix assertion, and the whole `diff(create_patch=True)` call raises `AssertionError`. Paths are chosen by whoever authored the commit. A tool that uses `Commit.diff(create_patch=True)` to learn which files a commit touches is shown an `a_path` prefix of the author's choosing, does not see the real path as `b_path`, or gets no diff at all. `re_header` now captures the `copy from` name, and the copy lines are used for the paths the way the rename lines already are. When no `---`, `+++`, rename or copy line names the file, both names on the `diff --git` line are the same path, and `_same_path_in_header` splits the line the way `git_header_name` in git's `apply.c` does: at the end of a quoted first name, or at the space where the two halves agree. A line that does not split that way keeps the previous result, so the existing fixtures parse as before. The new test in `test/test_diff.py` fails before the change and passes after it. `ruff check`, `ruff format --check` and `mypy` are clean. In a full clone the suite has 3 failures (`test_diff_with_staged_file`, `test_empty_repo` and `test_rebasing`), and all 3 also fail without this change. --- git/diff.py | 51 +++++++++++++++++++++++++++++++++++++++++++---- test/test_diff.py | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/git/diff.py b/git/diff.py index 123c22840..85a868c79 100644 --- a/git/diff.py +++ b/git/diff.py @@ -410,7 +410,8 @@ class Diff: re_header = re.compile( rb""" ^diff[ ]--git - [ ](?P"?[ab]/.+?"?)[ ](?P"?[ab]/.+?"?)\n + [ ](?P(?P"?[ab]/.+?"?) + [ ](?P"?[ab]/.+?"?))\n (?:^old[ ]mode[ ](?P\d+)\n ^new[ ]mode[ ](?P\d+)(?:\n|$))? (?:^similarity[ ]index[ ]\d+%\n @@ -419,7 +420,7 @@ class Diff: (?:^new[ ]file[ ]mode[ ](?P.+)(?:\n|$))? (?:^deleted[ ]file[ ]mode[ ](?P.+)(?:\n|$))? (?:^similarity[ ]index[ ]\d+%\n - ^copy[ ]from[ ].*\n + ^copy[ ]from[ ](?P.*)\n ^copy[ ]to[ ](?P.*)(?:\n|$))? (?:^index[ ](?P[0-9A-Fa-f]+) \.\.(?P[0-9A-Fa-f]+)[ ]?(?P.+)?(?:\n|$))? @@ -626,6 +627,39 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m return None + @staticmethod + def _same_path_in_header(names: bytes) -> Optional[bytes]: + """Return the path that a ``diff --git a/ b/`` line names twice. + + git separates the two names with a space but does not quote a path for + containing one, so once the path contains ``" a/"`` or ``" b/"`` the line alone + does not show where the first name ends. When no ``---``, ``+++``, rename or + copy line names the file, both names are the same path, and the split is + accepted only where the two halves agree, as ``git_header_name`` in git's + ``apply.c`` does. A quoted first name ends at its closing quote. + + :return: + The path without its ``a/`` prefix, or ``None`` if no split agrees. + """ + if names.startswith(b'"'): + end = 1 + while end < len(names) and names[end : end + 1] != b'"': + end += 2 if names[end : end + 1] == b"\\" else 1 + splits = [end + 1] + else: + splits = [index for index in range(len(names)) if names[index : index + 1] == b" "] + for split in splits: + if names[split : split + 1] != b" ": + continue + a_name, b_name = names[:split], names[split + 1 :] + if a_name.startswith(b'"'): + if not (a_name.endswith(b'"') and b_name.startswith(b'"') and b_name.endswith(b'"')): + continue + a_name, b_name = _unquote_path(a_name[1:-1]), _unquote_path(b_name[1:-1]) + if a_name.startswith(b"a/") and b_name.startswith(b"b/") and a_name[2:] == b_name[2:]: + return a_name[2:] + return None + @classmethod def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex["Diff"]: """Create a new :class:`DiffIndex` from the given process output which must be @@ -662,6 +696,7 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N a_mode, b_mode = None, None # For mypy. for _header in cls.re_header.finditer(text): ( + header_names, a_path_fallback, b_path_fallback, old_mode, @@ -670,6 +705,7 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N rename_to, new_file_mode, deleted_file_mode, + copied_from_name, copied_file_name, a_blob_id, b_blob_id, @@ -684,8 +720,15 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N bool(copied_file_name), ) - a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback) - b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) + # Mode changes, binary files and empty files have no `---` and `+++` lines, + # so only the `diff --git` line names them, and it names the same path twice. + named_elsewhere = any((a_path, b_path, rename_from, rename_to, copied_from_name, copied_file_name)) + same_path = None if named_elsewhere else cls._same_path_in_header(header_names) + if same_path is None: + a_path = cls._pick_best_path(a_path, rename_from or copied_from_name, a_path_fallback) + b_path = cls._pick_best_path(b_path, rename_to or copied_file_name, b_path_fallback) + else: + a_path = b_path = same_path # Our only means to find the actual text is to see what has not been matched # by our regex, and then retro-actively assign it to our index. diff --git a/test/test_diff.py b/test/test_diff.py index 06399b823..f5f0ac7c3 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -270,6 +270,42 @@ def test_diff_raw_paths_with_colon_or_line_feed(self, rw_dir): self.assertEqual([d.b_path for d in diffs], sorted(paths)) self.assertEqual({d.change_type for d in diffs}, {"A"}) + @with_rw_directory + def test_diff_patch_paths_named_only_by_the_header_line(self, rw_dir): + # Binary files, empty files, mode changes and copies have no `---` and `+++` + # lines, so their paths come from the `diff --git a/ b/` line, where + # git does not quote a space. Each path contains " a/" or " b/" so that the line + # alone does not show where the first name ends. + repo = Repo.init(rw_dir) + binary = repo.odb.store(IStream(Blob.type, 1, BytesIO(b"\0"))).binsha + empty = repo.odb.store(IStream(Blob.type, 0, BytesIO(b""))).binsha + paths = ["assets b/logo.png", "docs a/empty", "quoted\t a/tab"] + repo.index.add( + [ + BaseIndexEntry((0o100644, binary, 0, paths[0])), + BaseIndexEntry((0o100644, empty, 0, paths[1])), + BaseIndexEntry((0o100644, binary, 0, paths[2])), + ] + ) + commit = repo.index.commit("paths") + + diffs = commit.diff(NULL_TREE, create_patch=True) + self.assertEqual({(d.a_path, d.b_path) for d in diffs}, {(path, path) for path in paths}) + + repo.index.add( + [ + BaseIndexEntry((0o100755, binary, 0, paths[0])), + BaseIndexEntry((0o100644, binary, 0, "copy b/logo.png")), + ] + ) + changed = repo.index.commit("mode change and copy") + + diffs = commit.diff(changed, create_patch=True, C=True, find_copies_harder=True) + self.assertEqual( + {(d.a_path, d.b_path, d.copied_file) for d in diffs}, + {(paths[0], paths[0], False), (paths[0], "copy b/logo.png", True)}, + ) + def test_empty_diff(self): res = [] Diff._handle_diff_line(b"", None, res) From 983e182dadcc45146a3024eae4f6dc1079089d0c Mon Sep 17 00:00:00 2001 From: JingHao-Leon <102573344+JingHao-Leon@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:36:04 +0800 Subject: [PATCH 039/153] fix(config): decode escapes in unquoted config values Git decodes the standard escape sequences in unquoted config values, but `GitConfigParser._read` only called `parse_value()` when a value continued onto a following line. A single-line unquoted value therefore kept its escapes literal, so `b = first\nsecond` read back as `first\nsecond` rather than `firstsecond`. The same value also decoded differently depending only on whether it was wrapped: `b = first\nsecond \` continued onto a second line already went through `parse_value()` and was decoded. The missing decode made writes unsafe. The parsed value held no control character, so the writer's CR/LF/NUL guard did not trigger, and saving an unrelated key emitted a value that git then read differently than before. With the escape decoded, the existing guard rejects the value instead of corrupting the file. A value is decoded only when every backslash in it introduces a known escape. Git rejects an unknown escape outright, so GitPython never had to handle one; it tolerates them instead, which is what keeps a Windows path such as `C:\Users\me` usable. Decoding unconditionally would have broken that: the `\t` of `\Temp\test_foo` would become a tab while the `\U` of `\Users` stayed literal, mangling the path. Guarding on the whole value decodes `a\tb` and `foo\\bar` as git does and leaves the path alone. An existing case in `test_backslash_line_continuation` asserted the undecoded spelling `val\\` for `k = val\\` followed by a newline. Git yields `val\` there, since the second backslash escapes the first, so that expectation is corrected to match. Checked against git 2.50.1 (Apple Git-155). Co-Authored-By: Codex --- git/config.py | 38 ++++++++++++++++++++--- test/fixtures/git_config_unquoted_escapes | 12 +++++++ test/test_config.py | 26 +++++++++++++++- 3 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/git_config_unquoted_escapes diff --git a/git/config.py b/git/config.py index 7109c09c3..97deec274 100644 --- a/git/config.py +++ b/git/config.py @@ -69,6 +69,13 @@ VALID_CONFIG_OPTION_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") """Pattern for option names that can be written without changing config syntax.""" +UNQUOTED_ESCAPES = {"b": "\b", "n": "\n", "t": "\t", '"': '"', "\\": "\\"} +"""Escape sequences git recognizes in an unquoted value. + +Any other backslash escape is an error for git, which is why a value holding one is +left undecoded rather than rejected. +""" + class MetaParserBuilder(abc.ABCMeta): # noqa: B024 """Utility class wrapping base-class methods into decorators that assure read-only @@ -530,10 +537,9 @@ def parse_value(value: str) -> str: parsed: List[str] = [] whitespace: List[str] = [] quoted = escaped = False - escapes = {"b": "\b", "n": "\n", "t": "\t", '"': '"', "\\": "\\"} for char in value: if escaped: - parsed.append(escapes.get(char, "\\" + char)) + parsed.append(UNQUOTED_ESCAPES.get(char, "\\" + char)) escaped = False continue if char.isspace() and not quoted: @@ -552,6 +558,26 @@ def parse_value(value: str) -> str: parsed.append(char) return "".join(parsed) + def escapes_are_known(value: str) -> bool: + """Whether every backslash in `value` introduces an escape git recognises. + + Git rejects an unquoted value carrying an unknown escape outright. GitPython + tolerates such values, because that is what lets a Windows path such as + ``C:\\Users\\me`` stay literal, so a value is decoded only when every one + of its backslashes is part of a known escape. Decoding part of a value would + otherwise turn the ``\\t`` of ``\\Temp\\test`` into a tab while leaving the + ``\\U`` of ``\\Users`` alone. + """ + index = value.find("\\") + while index >= 0: + # A trailing backslash has nothing to escape and git drops it, so + # only a backslash followed by an unknown escape disqualifies the + # value from being decoded. + if index + 1 < len(value) and value[index + 1] not in UNQUOTED_ESCAPES: + return False + index = value.find("\\", index + 2) + return True + while True: # We assume to read binary! raw_line = fp.readline() @@ -606,7 +632,6 @@ def parse_value(value: str) -> str: # next line is appended before the complete value is # parsed. An even number means the last backslash is # escaped and the value ends there. - continued = False while True: if not is_line_continuation(optval): break @@ -620,8 +645,11 @@ def parse_value(value: str) -> str: while joined.endswith("\n") or joined.endswith("\r"): joined = joined[:-1] optval = optval[:-1] + joined - continued = True - if continued: + # Git decodes the standard escape sequences in unquoted + # values, but only a value in which every backslash is part + # of a known escape is decoded, so that a Windows path keeps + # its backslashes instead of losing them one escape at a time. + if escapes_are_known(optval): optval = parse_value(optval) elif quote_open: # Opens quoting and does not close: appears to start multi-line quoting. diff --git a/test/fixtures/git_config_unquoted_escapes b/test/fixtures/git_config_unquoted_escapes new file mode 100644 index 000000000..e146fff97 --- /dev/null +++ b/test/fixtures/git_config_unquoted_escapes @@ -0,0 +1,12 @@ +[custom] + hasnewline = first\nsecond + hastab = first\tsecond + hasbackslash = foo\\bar + hasquote = ab\"cd + hastrailingbackslash = word\\ + ordinary = hello world + withcolon = host:port + withequals = key=value + withhash = value#notcomment + hasunknownescape = foo\qbar + haswindowspath = C:\Users\RUNNER~1\AppData\Local\Temp\test_foo diff --git a/test/test_config.py b/test/test_config.py index e3c30421c..427909e83 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -365,7 +365,7 @@ def test_backslash_line_continuation(self): (b"[a]\n\tk = one\\\n two ; ignored\n", "one two"), (b'[a]\n\tk = one\\\n "two"\n', "one two"), (b"[a]\n\tk = one\\\n two\\tthree\n", "one two\tthree"), - (b"[a]\n\tk = val\\\\\n next\n", "val\\\\"), + (b"[a]\n\tk = val\\\\\n next\n", "val\\"), (b"[a]\n\tk = end\\\n", "end"), (b"[alias]\n\tco = checkout \\\n\t\t-v\n", "checkout \t\t-v"), ] @@ -1209,6 +1209,30 @@ def test_config_with_quotes_containing_escapes(self): # Cases without quotes should still parse correctly even after those, too. self.assertEqual(cr.get("custom", "unquoted"), "good evening") + def test_config_with_unquoted_escapes(self): + """Decode git's escape sequences in unquoted values, but only when every + backslash in the value is part of a known escape.""" + cr = GitConfigParser(fixture_path("git_config_unquoted_escapes"), read_only=True) + + self.assertEqual(cr.get("custom", "hasnewline"), "first\nsecond") + self.assertEqual(cr.get("custom", "hastab"), "first\tsecond") + self.assertEqual(cr.get("custom", "hasbackslash"), R"foo\bar") + self.assertEqual(cr.get("custom", "hasquote"), 'ab"cd') + self.assertEqual(cr.get("custom", "hastrailingbackslash"), "word\\") + + # Git rejects an unknown escape rather than decoding it. GitPython keeps + # such a value literal instead of rejecting it, which is what keeps a + # Windows path usable, so no backslash in one may be decoded: doing so + # would turn the \t of \Temp\test into a tab while leaving \U alone. + self.assertEqual(cr.get("custom", "hasunknownescape"), R"foo\qbar") + self.assertEqual(cr.get("custom", "haswindowspath"), R"C:\Users\RUNNER~1\AppData\Local\Temp\test_foo") + + # Values without escapes keep parsing as before. + self.assertEqual(cr.get("custom", "ordinary"), "hello world") + self.assertEqual(cr.get("custom", "withcolon"), "host:port") + self.assertEqual(cr.get("custom", "withequals"), "key=value") + self.assertEqual(cr.get("custom", "withhash"), "value") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) From 221defbbd1b427a548fd6b209b1377777fa635d2 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Tue, 20 Oct 2026 17:54:45 +0530 Subject: [PATCH 040/153] fix(repo): unquote `git check-ignore` paths in `Repo.ignored` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Repo.ignored` runs `git check-ignore -- ` and recovered the matched paths from stdout with `.replace("\\\\", "\\").replace('"', "").split("\n")`. git C-quotes a path that contains a non-ASCII byte (with the default `core.quotePath`), a double quote, a backslash or a control character, and that replacement does not reverse the quoting. With `*.env` in `.gitignore`: - `données.env` came back as `donn\303\251es.env` - `q"uote.env` came back as `q\uote.env` - `tab.env` came back as `tab\t.env` - a name containing a line feed came back with a literal `\n` A caller that compares the result with the paths it passed in, for example to leave ignored files out of a commit, an upload or an archive, therefore treats those ignored files as not ignored. Read stdout as bytes and unquote each quoted line with `git.diff._unquote_path`, which already reverses git's C-quoting for patch headers. Splitting at line feeds stays safe because git always quotes a path that contains one. `git check-ignore -z` would avoid the quoting, but git accepts `-z` only together with `--stdin`. Paths git does not quote are returned unchanged. Adds `test_ignored_items_reported_verbatim` to `test/test_repo.py`; it fails before this change and passes after. `ruff check`, `ruff format --check` and `mypy` pass. The full pytest suite run locally fails the same 83 tests with and without this change; those failures come from fixture commits missing in the local clone, not from this change. --- git/repo/base.py | 10 ++++++++-- test/test_repo.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index c39523344..a4161d620 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -26,6 +26,7 @@ from git.compat import defenc, safe_decode from git.config import GitConfigParser from git.db import GitCmdObjectDB +from git.diff import _unquote_path from git.exc import ( GitCommandError, InvalidGitRepositoryError, @@ -1158,7 +1159,7 @@ def ignored(self, *paths: PathLike) -> List[str]: Subset of those paths which are ignored """ try: - proc: str = self.git.check_ignore("--", *paths) + proc: bytes = self.git.check_ignore("--", *paths, stdout_as_string=False) except GitCommandError as err: if err.status == 1: # If return code is 1, this means none of the items in *paths are @@ -1168,7 +1169,12 @@ def ignored(self, *paths: PathLike) -> List[str]: # Raise the exception on all other return codes. raise - return proc.replace("\\\\", "\\").replace('"', "").split("\n") + # Git C-quotes a path containing special characters, including a line feed, + # so each path is on its own line. + return [ + safe_decode(_unquote_path(path[1:-1]) if path[:1] == path[-1:] == b'"' else path) + for path in proc.split(b"\n") + ] @property def active_branch(self) -> Head: diff --git a/test/test_repo.py b/test/test_repo.py index 47bf92b95..08cd4a1bf 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1657,6 +1657,17 @@ def test_ignored_items_reported(self): ["included_file.txt", "ignored_file.txt", "included_dir/file.txt", "ignored_dir/file.txt"] ) == ["ignored_file.txt", "ignored_dir/file.txt"] + def test_ignored_items_reported_verbatim(self): + with tempfile.TemporaryDirectory() as tdir: + temp_repo = Repo.init(pathlib.Path(tdir) / "repo") + + with open(pathlib.Path(temp_repo.working_tree_dir) / ".gitignore", "w") as file: + file.write("*.env\n") + + # Git C-quotes each of these in its default output format. + paths = ["donn\u00e9es.env", 'q"uote.env', "tab\t.env", "nl\nx.env"] + assert temp_repo.ignored(["keep.txt", *paths]) == paths + @requires_symlinks def test_ignored_raises_error_w_symlink(self): with tempfile.TemporaryDirectory() as tdir: From 6c4cd6ae31d66df20e9a1043e96e6da7f68177aa Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Wed, 21 Oct 2026 13:43:18 +0530 Subject: [PATCH 041/153] fix(util): redact URL credentials in config values on the command line `remove_password_if_present` is applied to every command line that GitPython logs or stores in `GitCommandError` and the other `CommandError` subclasses. It redacted an argument only when the whole argument was a URL with userinfo, or when it was an `http.extraHeader=Authorization: ...` assignment. Two common forms were missed: - A URL that is the value of a configuration assignment, such as `-c http.proxy=http://user:pass@proxy:3128` or `-c remote.origin.url=https://user:token@host/repo`. The text before `://` contains `=`, so `urlsplit` finds no scheme and no userinfo, and the argument was kept verbatim. - The `--config==` form, which is what `transform_kwargs` produces for a `config=...` keyword argument. The leading `--config=` defeated both checks, so even `--config=http.extraHeader=Authorization: Bearer ` was printed in full. For example, after `repo.git.set_persistent_git_options(c="http.proxy=http://user:secret@proxy:3128")` every failing command raised a `GitCommandError` whose message contained `user:secret`, and the same text reached the log when command tracing is on. The URL redaction moves into a small `_redact_url_credentials` helper. `remove_password_if_present` strips a leading `--config=` before the extra header check, and when the whole argument is not a URL with userinfo it tries the assignment after `--config=` (which covers `url..insteadOf` keys that carry credentials) and then the value after the first `=`. Only the userinfo is replaced, so the key and the rest of the URL stay readable, and arguments without a URL are returned unchanged. Arguments that were already redacted produce the same output as before. Adds `test_remove_password_from_config_values` to `test/test_util.py`. Its five redaction cases fail before this change and pass after it, and the existing redaction tests still pass. `ruff check`, `ruff format --check` and `mypy` are clean. --- git/util.py | 58 +++++++++++++++++++++++++++++++---------------- test/test_util.py | 27 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/git/util.py b/git/util.py index 874b764d5..473ea37db 100644 --- a/git/util.py +++ b/git/util.py @@ -652,42 +652,60 @@ def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[ return None +def _redact_url_credentials(text: str) -> Optional[str]: + """Return `text` with the userinfo of a URL redacted, or ``None`` if it has none.""" + try: + url = urlsplit(text) + except ValueError: + # This is not a valid URL. + return None + if url.password is None and url.username is None: + return None + + # Match urllib.parse's userinfo boundary. Keeping the raw hostinfo + # preserves hostname case, IPv6 brackets, and port formatting. + _, _, hostinfo = url.netloc.rpartition("@") + redacted = "*****:*****" if url.password is not None else "*****" + return urlunsplit(url._replace(netloc=f"{redacted}@{hostinfo}")) + + def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: """Redact credentials in URLs and HTTP Authorization extra headers in a command line. + URLs are also redacted when they are the value of a configuration assignment, such + as ``http.proxy=``, and assignments are recognized in both the ``-c =`` + and the ``--config==`` form. + If nothing is found, this just returns the command line as-is. This should be used for every log line that print a command line, as well as exception messages. """ new_cmdline = [] - for index, to_parse in enumerate(cmdline): - new_cmdline.append(to_parse) - config_key, separator, header = to_parse.partition("=") - header_name, colon, _ = header.partition(":") + for to_parse in cmdline: + option = "--config=" if to_parse.startswith("--config=") else "" + assignment = to_parse[len(option) :] + config_key, separator, value = assignment.partition("=") + header_name, colon, _ = value.partition(":") if ( separator and colon and config_key.lower().endswith(".extraheader") and header_name.strip().lower() == "authorization" ): - new_cmdline[index] = "%s%s%s%s *****" % (config_key, separator, header_name, colon) - continue - try: - url = urlsplit(to_parse) - # Remove password from the URL if present. - if url.password is None and url.username is None: - continue - - # Match urllib.parse's userinfo boundary. Keeping the raw hostinfo - # preserves hostname case, IPv6 brackets, and port formatting. - _, _, hostinfo = url.netloc.rpartition("@") - redacted = "*****:*****" if url.password is not None else "*****" - url = url._replace(netloc=f"{redacted}@{hostinfo}") - new_cmdline[index] = urlunsplit(url) - except ValueError: - # This is not a valid URL. + new_cmdline.append("%s%s%s%s%s *****" % (option, config_key, separator, header_name, colon)) continue + + redacted = _redact_url_credentials(to_parse) + if redacted is None and option: + redacted = _redact_url_credentials(assignment) + if redacted is not None: + redacted = option + redacted + if redacted is None and separator: + redacted = _redact_url_credentials(value) + if redacted is not None: + redacted = option + config_key + separator + redacted + new_cmdline.append(to_parse if redacted is None else redacted) return new_cmdline diff --git a/test/test_util.py b/test/test_util.py index ca592aaf7..da2f59c85 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -856,6 +856,33 @@ def test_remove_empty_password_keeps_host_intact(self): def test_remove_password_preserves_url_components(self, url, expected): assert remove_password_if_present([url]) == [expected] + @ddt.data( + ( + ["-c", "http.proxy=http://user:pass@proxy.example.com:3128"], + ["-c", "http.proxy=http://*****:*****@proxy.example.com:3128"], + ), + ( + ["-c", "remote.origin.url=https://user:pass@example.com/repo"], + ["-c", "remote.origin.url=https://*****:*****@example.com/repo"], + ), + ( + ["--config=http.proxy=http://user:pass@proxy.example.com:3128"], + ["--config=http.proxy=http://*****:*****@proxy.example.com:3128"], + ), + ( + ["--config=url.https://user:pass@example.com/.insteadOf=https://example.com/"], + ["--config=url.https://*****:*****@example.com/.insteadOf=https://example.com/"], + ), + ( + ["--config=http.extraHeader=Authorization: Bearer pass"], + ["--config=http.extraHeader=Authorization: *****"], + ), + (["-c", "user.name=pass"], ["-c", "user.name=pass"]), + ) + @ddt.unpack + def test_remove_password_from_config_values(self, cmdline, expected): + assert remove_password_if_present(cmdline) == expected + def test_mode_str_to_int_accepts_bytes(): assert mode_str_to_int("100644") == 0o100644 From de50b145e3e54e11882cfb938c14303ad0379a90 Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 21 Oct 2026 13:46:29 +0200 Subject: [PATCH 042/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [P2] Avoid copying the header at every possible separator — /Users/byron/dev/github.com/gitpython-developers/GitPython.diff-patch-header-paths/git/diff.py:650-654 For a long, space-heavy filename, this loop copies the header once per space, making parsing quadratic in path length. A Git-generated mode-only diff for the filename `b"x " * 250000` took about 3 seconds through `Tree.diff(..., create_patch=True)`, versus 0.023 seconds on the parent. Git accepts such tree entries without checkout, so larger commit-controlled paths can stall callers. Reject impossible split lengths before allocating slices, or check only the equal-length midpoint. Assisted-by: GPT 6.0 Astra Co-authored-by: GPT 6.0 Astra --- git/diff.py | 46 +++++++++++++++++++------------------- test/test_diff.py | 56 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/git/diff.py b/git/diff.py index 85a868c79..48ed863f4 100644 --- a/git/diff.py +++ b/git/diff.py @@ -9,16 +9,9 @@ import re import warnings -from git.cmd import Git, handle_process_output -from git.compat import defenc -from git.objects.base import IndexObject -from git.objects.blob import Blob -from git.objects.util import mode_str_to_int -from git.util import finalize_process, hex_to_bin - # typing ------------------------------------------------------------------ - from typing import ( + TYPE_CHECKING, Any, Iterator, List, @@ -26,12 +19,18 @@ Optional, Sequence, Tuple, - TYPE_CHECKING, TypeVar, Union, cast, ) -from git.types import PathLike, Literal + +from git.cmd import Git, handle_process_output +from git.compat import defenc +from git.objects.base import IndexObject +from git.objects.blob import Blob +from git.objects.util import mode_str_to_int +from git.types import Literal, PathLike +from git.util import finalize_process, hex_to_bin if TYPE_CHECKING: from subprocess import Popen @@ -645,19 +644,22 @@ def _same_path_in_header(names: bytes) -> Optional[bytes]: end = 1 while end < len(names) and names[end : end + 1] != b'"': end += 2 if names[end : end + 1] == b"\\" else 1 - splits = [end + 1] + split = end + 1 else: - splits = [index for index in range(len(names)) if names[index : index + 1] == b" "] - for split in splits: - if names[split : split + 1] != b" ": - continue - a_name, b_name = names[:split], names[split + 1 :] - if a_name.startswith(b'"'): - if not (a_name.endswith(b'"') and b_name.startswith(b'"') and b_name.endswith(b'"')): - continue - a_name, b_name = _unquote_path(a_name[1:-1]), _unquote_path(b_name[1:-1]) - if a_name.startswith(b"a/") and b_name.startswith(b"b/") and a_name[2:] == b_name[2:]: - return a_name[2:] + # Equal unquoted paths have equal-length names, so only the midpoint + # can separate them. Trying every space would copy the header quadratically. + if len(names) % 2 == 0: + return None + split = len(names) // 2 + if names[split : split + 1] != b" ": + return None + a_name, b_name = names[:split], names[split + 1 :] + if a_name.startswith(b'"'): + if not (a_name.endswith(b'"') and b_name.startswith(b'"') and b_name.endswith(b'"')): + return None + a_name, b_name = _unquote_path(a_name[1:-1]), _unquote_path(b_name[1:-1]) + if a_name.startswith(b"a/") and b_name.startswith(b"b/") and a_name[2:] == b_name[2:]: + return a_name[2:] return None @classmethod diff --git a/test/test_diff.py b/test/test_diff.py index f5f0ac7c3..07e2011d7 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -4,23 +4,21 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc -from io import BytesIO import os.path as osp import shutil import sys import tempfile +from io import BytesIO import ddt import pytest - from gitdb.base import IStream -from git import NULL_TREE, Blob, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule +from git import NULL_TREE, Blob, Diff, Diffable, DiffIndex, GitCommandError, Repo, Submodule from git.cmd import Git from git.diff import decode_path from git.exc import UnsafeOptionError from git.index.typ import BaseIndexEntry - from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory @@ -306,6 +304,56 @@ def test_diff_patch_paths_named_only_by_the_header_line(self, rw_dir): {(paths[0], paths[0], False), (paths[0], "copy b/logo.png", True)}, ) + @ddt.data( + (b"a/x b/x", b"x"), + (b"a/x b/x ", b"x "), + (b"a/x b/y", None), + (b"a/x b/xx", None), + (b"a/x-b/x", None), + (b"", None), + (b'"a/x\\t" "b/x\\t"', b"x\t"), + (b'"a/x\\t" "b/x\\011"', b"x\t"), + (b'"a/x\\t" "b/y\\t"', None), + (b'"a/x\\t" "b/x\\t', None), + ) + def test_same_path_in_header(self, case): + names, expected = case + self.assertEqual(Diff._same_path_in_header(names), expected) + + def test_same_path_in_header_copies_linear_bytes(self): + class SliceCountingBytes(bytes): + sliced_bytes = 0 + + def __getitem__(self, key): + result = super().__getitem__(key) + if isinstance(key, slice): + self.sliced_bytes += len(result) + return result + + path = b"x " * 1000 + names = SliceCountingBytes(b"a/" + path + b" b/" + path) + self.assertEqual(Diff._same_path_in_header(names), path) + # Count copied bytes rather than asserting a machine-dependent time limit. + self.assertLessEqual(names.sliced_bytes, 2 * len(names)) + + @with_rw_directory + def test_diff_patch_mode_only_with_long_space_heavy_path(self, rw_dir): + repo = Repo.init(rw_dir) + path = b"x " * 250000 + blob = repo.odb.store(IStream(Blob.type, 0, BytesIO(b""))).binsha + trees = [] + # Store trees directly: Git accepts paths longer than filesystem or index limits. + for mode in (b"100644", b"100755"): + data = mode + b" " + path + b"\0" + blob + tree = repo.odb.store(IStream(b"tree", len(data), BytesIO(data))) + trees.append(repo.tree(tree.binsha.hex())) + + diffs = trees[0].diff(trees[1], create_patch=True) + self.assertEqual(len(diffs), 1) + self.assertEqual((diffs[0].a_rawpath, diffs[0].b_rawpath), (path, path)) + self.assertEqual((diffs[0].a_mode, diffs[0].b_mode), (0o100644, 0o100755)) + self.assertEqual(diffs[0].diff, b"") + def test_empty_diff(self): res = [] Diff._handle_diff_line(b"", None, res) From 22f137d9aa9099f16de600be536e7a0cd203bb58 Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 21 Oct 2026 14:05:03 +0200 Subject: [PATCH 043/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [P2] Ignore comment text when validating continued values — /Users/byron/dev/github.com/gitpython-developers/GitPython.fix-config-unquoted -escape-parsing/git/config.py:576-577 With `k = one\` followed by ` two # C:\Users`, the joined value still contains the second line's comment. This check treats its `\U` as an unknown escape and skips `parse_value()`, returning `one two # C:\Users` instead of `one two`, unlike Git and `main`. Updating an unrelated option then persists the comment as part of the value. Strip or skip unquoted comment text before validating escapes. - [P2] Preserve whitespace before an EOF continuation backslash — /Users/byron/dev/github.com/gitpython-developers/GitPython.fix-config-unquoted -escape-parsing/git/config.py:652-653 If the final config line is `k = value \`, with or without a terminating newline, Git and `main` return `"value "`. The EOF branch removes the backslash first, so this newly applied `parse_value()` discards the preceding space and returns `"value"`. Updating an unrelated option also persists that change. Preserve whitespace made significant by the continuation when decoding. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/config.py | 9 +++++++-- test/test_config.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/git/config.py b/git/config.py index 97deec274..732d3658b 100644 --- a/git/config.py +++ b/git/config.py @@ -632,13 +632,15 @@ def escapes_are_known(value: str) -> bool: # next line is appended before the complete value is # parsed. An even number means the last backslash is # escaped and the value ends there. + eof_continuation = False while True: if not is_line_continuation(optval): break continuation = fp.readline() if not continuation: - # Backslash at end of file: git drops it. - optval = optval[:-1] + # Keep the backslash until parsing so preceding + # whitespace is significant even though git drops it. + eof_continuation = True break lineno = lineno + 1 joined = continuation.decode(defenc) @@ -649,8 +651,11 @@ def escapes_are_known(value: str) -> bool: # values, but only a value in which every backslash is part # of a known escape is decoded, so that a Windows path keeps # its backslashes instead of losing them one escape at a time. + optval, _ = strip_inline_comment(optval) if escapes_are_known(optval): optval = parse_value(optval) + elif eof_continuation: + optval = optval[:-1] elif quote_open: # Opens quoting and does not close: appears to start multi-line quoting. is_multi_line = True diff --git a/test/test_config.py b/test/test_config.py index 427909e83..49b8535c7 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -378,6 +378,33 @@ def test_backslash_line_continuation(self): key = "co" if section == "alias" else "k" self.assertEqual(config.get_value(section, key), expected) + @with_rw_directory + def test_continued_values_preserve_comments_and_eof_whitespace(self, rw_dir): + cases = [ + (b"one\\\n two # C:\\Users\n", "one two"), + (b"one\\\n two ; C:\\Users\n", "one two"), + (b"one\\\n two\\tthree # C:\\Users\n", "one two\tthree"), + (b'one\\\n "two # three" # C:\\Users\n', "one two # three"), + (b"value \\", "value "), + (b"value \\\n", "value "), + (b"value \t\\\n", "value \t"), + (b"one\\\n two \\\n", "one two "), + (b"C:\\Users \\\n", "C:\\Users "), + ] + config_path = osp.join(rw_dir, "config") + for value, expected in cases: + with self.subTest(value=value): + with open(config_path, "wb") as config_file: + config_file.write(b"[a]\n\tk = " + value) + + with GitConfigParser(config_path, read_only=False) as config: + self.assertEqual(config.get_value("a", "k"), expected) + config.set_value("other", "value", "updated") + + with GitConfigParser(config_path) as config: + self.assertEqual(config.get_value("a", "k"), expected) + self.assertEqual(config.get_value("other", "value"), "updated") + def test_get_value_reads_git_boolean_spellings(self): # git accepts yes/no and on/off as well as true/false, and getboolean on # this class already did. get_value returned them as strings, so "no" and From 6d81c055e2a19a79932dd7d04f97ceade4f3565c Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 21 Oct 2026 14:00:31 +0200 Subject: [PATCH 044/153] Steer agents aware from the issue + PR combo It's just noise and I don't even read such issues (nor do I read the PRs in full). --- AGENTS.md | 4 ++++ CONTRIBUTING.md | 10 ++++++++++ README.md | 8 ++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cdffa4764..a308591ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,6 +5,10 @@ including the [Prevent agent impersonation](CONTRIBUTING.md#prevent-agent-impers section governing agent identification and separation of unaltered user statements when communicating through a person's account. +[Prefer pull requests](CONTRIBUTING.md#prefer-pull-requests): a PR can serve as an +issue with a fix attached. Do not create an issue solely to follow it immediately +with a PR that fixes it; describe the problem in the PR instead. + # Commit messages Follow Conventional Commits for every commit. Every commit must have a diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a64a701d..8a155dc0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,16 @@ The following is a short step-by-step rundown of what one typically would do to - Feel free to add yourself to AUTHORS file. - Create a pull request. +## Prefer pull requests + +Pull requests are preferred when you have a fix or improvement to contribute: a PR +can be treated as an issue with a fix attached. Describe the problem and proposed +solution in the PR itself; a separate issue is not required. + +Please avoid opening an issue solely to follow it immediately with a PR that fixes +it. Open the PR directly instead. Issues are still welcome for problems without a +fix or for changes that need discussion before implementation. + ## Quality expectations Contributions must be made with care and meet the quality bar of the surrounding code. diff --git a/README.md b/README.md index 029b4fa00..8d1b590b2 100644 --- a/README.md +++ b/README.md @@ -201,8 +201,12 @@ Please have a look at the [contributions file][contributing]. - [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) - Please post on Stack Overflow and use the `gitpython` tag - [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) - - Post reproducible bugs and feature requests as a new issue. - Please be sure to provide the following information if posting bugs: + - Post reproducible bugs and feature requests as a new issue when you do not have + a fix or need discussion before implementation. If you have a fix or improvement, + [open a pull request directly](CONTRIBUTING.md#prefer-pull-requests) and describe + the problem there; you do not need to create an issue first. + Please be sure to provide the following information when reporting bugs in an + issue or PR: - GitPython version (e.g. `import git; git.__version__`) - Python version (e.g. `python --version`) - The encountered stack-trace, if applicable From f89c3431a7e825f3ffe3b4e262b67fb9318fba60 Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 21 Oct 2026 15:16:49 +0200 Subject: [PATCH 045/153] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [P1] Redact the value when the config key also contains a URL — /Users/byron/dev/github.com/gitpython-developers/GitPython.redact-config-url-c redentials/git/util.py:700-705 When a URL-specific config key and its value both contain credentials, such as `--config=http.https://alice@host.proxy=http://bob:token@proxy`, redacting the key sets `redacted`, so the value is never checked. `Git.execute` traces and `GitCommandError.command` still expose `bob:token`. Redact the key and value independently. - [P2] Recognize clone's compact -c configuration form — /Users/byron/dev/github.com/gitpython-developers/GitPython.redact-config-url-c redentials/git/util.py:686-688 When `Repo.clone_from` receives Git clone's valid `-c=` form through `multi_options`, a credentialed URL key such as `-curl.https://user:token@host/.insteadOf=https://old/` remains unchanged. The leading `-c` prevents URL parsing, while the fallback checks only the value, so traces and exceptions expose the token. Strip this prefix before checking the key. Assisted-by: GPT 6.0 Astra Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/util.py | 28 ++++++++++++++-------------- test/test_git.py | 35 +++++++++++++++++++++++++++++++++-- test/test_util.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/git/util.py b/git/util.py index 473ea37db..137318eb8 100644 --- a/git/util.py +++ b/git/util.py @@ -672,9 +672,9 @@ def _redact_url_credentials(text: str) -> Optional[str]: def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: """Redact credentials in URLs and HTTP Authorization extra headers in a command line. - URLs are also redacted when they are the value of a configuration assignment, such - as ``http.proxy=``, and assignments are recognized in both the ``-c =`` - and the ``--config==`` form. + URLs are also redacted in the key and value of a configuration assignment, such + as ``http.proxy=``. Assignments are recognized in the ``-c =``, + ``-c=``, and ``--config==`` forms. If nothing is found, this just returns the command line as-is. @@ -683,8 +683,15 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: """ new_cmdline = [] for to_parse in cmdline: - option = "--config=" if to_parse.startswith("--config=") else "" + if to_parse.startswith("--config="): + option = "--config=" + elif to_parse.startswith("-c"): + option = "-c" + else: + option = "" assignment = to_parse[len(option) :] + # Redact the whole URL first, before splitting on a possible '=' in userinfo. + assignment = _redact_url_credentials(assignment) or assignment config_key, separator, value = assignment.partition("=") header_name, colon, _ = value.partition(":") if ( @@ -696,16 +703,9 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: new_cmdline.append("%s%s%s%s%s *****" % (option, config_key, separator, header_name, colon)) continue - redacted = _redact_url_credentials(to_parse) - if redacted is None and option: - redacted = _redact_url_credentials(assignment) - if redacted is not None: - redacted = option + redacted - if redacted is None and separator: - redacted = _redact_url_credentials(value) - if redacted is not None: - redacted = option + config_key + separator + redacted - new_cmdline.append(to_parse if redacted is None else redacted) + if separator: + value = _redact_url_credentials(value) or value + new_cmdline.append(option + config_key + separator + value) return new_cmdline diff --git a/test/test_git.py b/test/test_git.py index 5224842ee..eb9d305b8 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -10,7 +10,6 @@ import logging import os import os.path as osp -from pathlib import Path import pickle import re import shutil @@ -20,6 +19,7 @@ import tempfile import threading import time +from pathlib import Path from unittest import mock, skipUnless import ddt @@ -27,7 +27,6 @@ from git import Git, GitCommandError, GitCommandNotFound, Repo, cmd, refresh from git.exc import UnsafeOptionError from git.util import cwd, finalize_process - from test.lib import TestBase, fixture_path, with_rw_directory @@ -547,6 +546,38 @@ def test_it_redacts_authorization_extra_header_from_error(self): self.assertNotIn(token, message) self.assertIn("http.extraHeader=Authorization: *****", message) + @ddt.data( + ( + "--config=http.https://alice@host.proxy=http://bob:token@proxy", + "--config=http.https://*****@host.proxy=http://*****:*****@proxy", + ), + ( + "-curl.https://user:token@host/.insteadOf=https://old/", + "-curl.https://*****:*****@host/.insteadOf=https://old/", + ), + ) + @ddt.unpack + def test_clone_redacts_config_urls_from_trace_and_error(self, option, expected): + for trace in (True, "full"): + with self.subTest(trace=trace), tempfile.TemporaryDirectory() as directory: + with mock.patch.object(Git, "GIT_PYTHON_TRACE", trace): + with self.assertLogs("git.cmd", level="INFO") as logs: + with self.assertRaises(GitCommandError) as context: + Repo.clone_from( + osp.join(directory, "missing-source"), + osp.join(directory, "destination"), + multi_options=[option], + allow_unsafe_options=True, + ) + self.assertIn(expected, context.exception.command) + message = str(context.exception) + assert logs is not None + log_output = " ".join(logs.output) + self.assertIn(expected, log_output) + for credential in ("alice", "bob", "user", "token"): + self.assertNotIn(credential, message) + self.assertNotIn(credential, log_output) + def test_it_accepts_environment_variables(self): filename = fixture_path("ls_tree_empty") with open(filename, "r") as fh: diff --git a/test/test_util.py b/test/test_util.py index da2f59c85..df8c2d452 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -849,6 +849,7 @@ def test_remove_empty_password_keeps_host_intact(self): ("//user:pass@[2001:db8::1]:0080/repo", "//*****:*****@[2001:db8::1]:0080/repo"), ("https://user:p@ss@example.com/repo", "https://*****:*****@example.com/repo"), ("https://user:@example.com/repo", "https://*****:*****@example.com/repo"), + ("https://user:p=ass@example.com/repo", "https://*****:*****@example.com/repo"), ("https://@example.com/repo", "https://*****@example.com/repo"), ("https://example.com/repo@name?q=a@b#c@d", "https://example.com/repo@name?q=a@b#c@d"), ) @@ -877,7 +878,36 @@ def test_remove_password_preserves_url_components(self, url, expected): ["--config=http.extraHeader=Authorization: Bearer pass"], ["--config=http.extraHeader=Authorization: *****"], ), + ( + ["--config=http.https://alice@host.proxy=http://bob:token@proxy"], + ["--config=http.https://*****@host.proxy=http://*****:*****@proxy"], + ), + ( + ["-c", "http.https://alice@host.proxy=http://bob:token@proxy"], + ["-c", "http.https://*****@host.proxy=http://*****:*****@proxy"], + ), + ( + ["-chttp.https://alice@host.proxy=http://bob:token@proxy"], + ["-chttp.https://*****@host.proxy=http://*****:*****@proxy"], + ), + ( + ["-curl.https://user:token@host/.insteadOf=https://old/"], + ["-curl.https://*****:*****@host/.insteadOf=https://old/"], + ), + ( + ["-chttp.proxy=http://bob:token@proxy"], + ["-chttp.proxy=http://*****:*****@proxy"], + ), + ( + ["--config=http.https://alice:token@host/.extraHeader=Authorization: Bearer pass"], + ["--config=http.https://*****:*****@host/.extraHeader=Authorization: *****"], + ), + ( + ["-chttp.extraHeader=Authorization: Bearer pass"], + ["-chttp.extraHeader=Authorization: *****"], + ), (["-c", "user.name=pass"], ["-c", "user.name=pass"]), + (["-cuser.name=pass"], ["-cuser.name=pass"]), ) @ddt.unpack def test_remove_password_from_config_values(self, cmdline, expected): From 674763717d18278b8b6bf7b4ee4fd110c8611468 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Thu, 22 Oct 2026 16:44:22 +0530 Subject: [PATCH 046/153] fix(repo): split `git blame` porcelain output at line feeds only `Repo.blame` runs `git blame -p` and iterated over its stdout with `bytes.splitlines()`, which breaks at a lone carriage return as well as at a line feed. git ends every porcelain line with a line feed only, and the content lines in that output are the raw lines of the blamed file, so a CR inside a line of file content ended the line early in GitPython and the rest of it was parsed as porcelain headers. A file line such as ok\r<40 hex digits> 2 2 1\rauthor Forged Author\rauthor-mail \r...\r\tforged() made `Repo.blame` return an extra entry that attributes `forged()` to a commit id and author of the file author's choosing, while `git blame` attributes the whole line to the commit that really added it. Short of a forgery, any line with a lone CR was truncated at the CR and the rest of it dropped. Blamed file content is controlled by whoever wrote it, so a tool that uses `Repo.blame` to decide who wrote a line (code ownership, review routing, audit) could be shown attribution chosen by that author. Iterate over a `BytesIO` of the output instead, which splits at line feeds only and keeps the terminators, so CRLF lines and the binary-line handling are unchanged. `blame_incremental` does not print file content and is unaffected. Adds `test_blame_content_with_carriage_returns` to `test/test_repo.py`; it fails before this change and passes after. `test_blame_accepts_rev_opts` mocks `Git._call_process`, so it now returns `b""` instead of a `MagicMock` to the parser. `ruff check`, `ruff format --check` and `mypy` pass. In a full clone prepared with `init-tests-after-clone.sh`, the pytest suite passes except `test_diff_with_staged_file`, `test_empty_repo` and `test_rebasing`, which fail the same way without this change. --- git/repo/base.py | 7 +++++-- test/test_repo.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index a4161d620..6bf885280 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -8,6 +8,7 @@ __all__ = ["Repo"] import gc +from io import BytesIO import logging import os import os.path as osp @@ -1358,8 +1359,10 @@ class InfoTD(TypedDict, total=False): info: InfoTD = {} - keepends = True - for line_bytes in data.splitlines(keepends): + # Git ends each porcelain line with a line feed only. bytes.splitlines() also + # splits at a lone carriage return, which a line of file content can contain, + # and would then parse the rest of that content line as blame headers. + for line_bytes in BytesIO(data): line_str = "" try: line_str = line_bytes.rstrip().decode(defenc) diff --git a/test/test_repo.py b/test/test_repo.py index 08cd4a1bf..8106a88ca 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -839,9 +839,44 @@ def test_blame_complex_revision(self, git): def test_blame_accepts_rev_opts(self, git): expected_args = ["blame", "HEAD", "-M", "-C", "-C", "--", "README.md"] boilerplate_kwargs = {"p": True, "stdout_as_string": False} + git.return_value = b"" self.rorepo.blame("HEAD", "README.md", rev_opts=["-M", "-C", "-C"]) git.assert_called_once_with(*expected_args, **boilerplate_kwargs) + @with_rw_directory + def test_blame_content_with_carriage_returns(self, rw_dir): + repo = Repo.init(rw_dir) + with repo.config_writer() as writer: + writer.set_value("user", "name", "GitPython Tests") + writer.set_value("user", "email", "gitpython@example.com") + + # A lone CR inside a line of content, followed by text shaped like blame + # porcelain headers for a forged commit and author. + forged = "\r".join( + [ + "ok", + "0" * 40 + " 2 2 1", + "author Forged Author", + "author-mail ", + "author-time 1000000000", + "committer Forged Author", + "committer-mail ", + "committer-time 1000000000", + "\tforged()", + ] + ) + tracked_file = Path(rw_dir) / "file.txt" + tracked_file.write_bytes(b"first\n" + forged.encode("ascii") + b"\n") + repo.index.add([str(tracked_file)]) + commit = repo.index.commit("init") + + blames = repo.blame("HEAD", "file.txt") + + self.assertEqual(len(blames), 1) + self.assertEqual(blames[0][0], commit) + self.assertEqual(blames[0][0].author.name, "GitPython Tests") + self.assertEqual(blames[0][1], ["first", forged]) + @with_rw_repo("HEAD", bare=False) def test_untracked_files(self, rwrepo): for run, repo_add in enumerate((rwrepo.index.add, rwrepo.git.add)): From 212573bff1afee4e307b201cd33756ed4a64d868 Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Thu, 1 Oct 2026 14:45:37 +0200 Subject: [PATCH 047/153] feat!: delegate repository storage to Git and Gix GitPython's Python implementations of repository discovery, configuration, references, object storage and indexes assume storage details that do not hold for SHA-256 objects or reftable references. Delegate these operations to Git 2.52+ and offer GixPython 0.3.0 for supported operations, using the Git CLI as the behavioral baseline. Keep library-managed commands behind argument and stdin validation, suppress implicit hooks, external diff programs, maintenance and lazy fetches where appropriate, and preserve explicit unsafe-operation opt-ins. Support SHA-1 and SHA-256 with files or reftable reference storage. Close owned repository handles before filesystem cleanup and account for Windows and Cygwin path restrictions. The optional `gix` extra selects Gitoxide through supported GixPython APIs. `GIT_PYTHON_BACKEND=cli|gix` makes selection explicit even when both backends are installed. Retain native repository handles and adapt native object, reference, index, graph, configuration, diff and status results where they match Git. Keep CLI fallbacks for missing capabilities and incompatible behavior; record reproductions, affected adapters and upstream requirements in the compatibility ledger in `doc/gix-backend.md`. Native execution never means reimplementing Git semantics in Python. Read declared commit encodings according to Git's first-header and whole-commit conversion behavior, including conversion failure and signature headers. Keep UTF-8 metadata native; use Git's text projection for other encodings and the open Gix header-order compatibility bug. Validate these cases with read-only Bash fixtures and direct Git comparisons. Configuration failure regressions exercise Git's per-mutation locks rather than the removed constructor lock. Breaking API changes follow the storage migration. Remove the Python `GitDB` backend and binary index/tree and raw reflog mutation interfaces. Replace `IndexFile.entries` with immutable `iter_entries()`/`entry()` queries and explicit edits; publish deferred index bytes only after Git validates them. Configuration uses Git's syntax and locking, reflogs expose Git's commit-reflog view, and repository object IDs have format-dependent widths. Remove deprecated exports and implicit environment-variable path expansion. Document the affected APIs, exceptions, hook behavior and replacements in `doc/source/changes.rst`. Move generated repository and filesystem initial states into Bash scripts run through `gix.testtools`. Read-only tests share prepared fixtures; mutation tests copy them by default, with explicit execution only for fixtures that cannot be relocated. Keep generated caches and archives out of version control. Tests require GixPython with either backend: CI runs fixture suites on Python 3.11+ and retains installation/import smoke checks on Python 3.8-3.10. Update the normal, minimum-Git, Alpine and Cygwin environments and the source packaging. Add downstream release checks for Bandit, MLflow, LangChain, SWE-bench and DataHub with both backends. Add a read-only repository benchmark that compares result digests, records native/fallback decisions and actual Git launches, and enforces CLI-process ceilings. Preserve historical measurements and their original revision references as evidence, including superseded results. Make scripted tests portable to Git for Windows. Define literal submodule names inside Bash from simple scenario identifiers, because MSYS expands metacharacters in native arguments and disabling that expansion breaks GixPython's quoted script launcher. Retain Windows directory-symlink types for dangling targets and build trailing-space filenames with Git's `mktree` instead of the Windows-restricted index. Compare escaped log paths and native fixture locations correctly. Keep exact Git-recorded submodule URLs while separately checking path identity, including a portable `/./` regression. Use Git's actual text projection for platform-dependent `iconv` declarations. Alpine's musl recognizes `utf-16-be` while macOS rejects that spelling. Keep Git as the text baseline; independently assert the first codec, raw dates/timezones, and native/fallback decisions. Keep malformed UTF-16 fixtures odd-sized, including the portable `UTF-16BE` spelling, so conversion fails without erasing Git's header delimiters. Assert the raw object size to prevent fixture edits from reintroducing unstable pretty output. Record these platform limitations in `doc/gix-backend.md`. Before the Windows fixture repairs, macOS validation used CPython 3.12.14, Git 2.54.0 and GixPython 0.2.0. The full CLI suite passed 1,745 tests with 151 skips, and the full Gix suite passed 1,845 tests with 51 skips. Both passed 58 subtests and retained one expected failure. CLI ran first. All 14 benchmark result digests and the existing CLI-process ceilings matched; the two-sample smoke run establishes no timing conclusion. Ruff lint/format, mypy, basedpyright, ShellCheck and Sphinx with warnings fatal passed. Hosted CI, other platforms and older interpreters were not run locally. The first Windows fixture repair passed 471 tests and 20 subtests on each backend locally but exposed a native Windows launcher failure in CI. The follow-up removes the `MSYS=noglob` override and preserves all literal-name assertions through scripted initial states. It also fixes Alpine's encoding expectation rather than changing production Git behavior. The focused commit, cleanup, fixture-health and literal-name tests passed 70 tests and 21 subtests per backend locally, CLI first, with one Windows-only skip. Repository pre-commit hooks passed. On `13202ce`, 63 of 64 hosted checks passed, including Alpine and Windows Gix. Windows CLI on Python 3.12 exposed one remaining fixture expectation: direct Git returned `x` instead of an empty message after successful UTF-16 recoding removed every newline. The failure preceded the backend comparison. Force conversion failure for those malformed fixtures and remove the unsupported empty-output assertion while retaining exact Git comparisons. No production behavior changes for this repair. The final commit module passed 40 tests and 21 subtests per backend locally, CLI first, and all hooks for the changed files passed. Hosted verification of that final fixture correction is left to the next CI run. After rebasing onto `main` at `28ab8fb3`, the Windows Python package checks on `f47ce8f5` failed both variants of `test_root_update_skips_old_url_resolution_when_remote_matches`. Git Bash recorded `C:/Users/RUNNER~1/.../source`, while repository discovery returned the equivalent `C:/Users/runneradmin/.../source`. The test compared these spellings instead of checking that root update preserved the original URL. Capture the module's recorded URL before update, require it to stay exactly unchanged, and separately check directory identity with `Path.samefile()`. This preserves the regression's URL-mutation check across Windows short-path aliases without changing production behavior. The focused relative-URL submodule tests passed 39 tests with two skips on each backend locally, CLI first, using CPython 3.12.14, Git 2.54.0 and GixPython 0.2.0. Repository hooks passed for `test/test_submodule.py`. Windows validation of this assertion correction remains with hosted CI. Refresh the stack onto `main` at `bae0c2fb`, retaining the upstream raw diff parser that reads complete NUL-delimited records and the reproducible Sphinx attribute exclusions. Adapt the new colon/line-feed path regression from removed index entry APIs to a read-only `gix.testtools` Bash fixture. Construct its trees through `git mktree -z` so Windows need not create those paths on disk, and check the exact paths and addition statuses against `git diff-tree --name-status -z` before comparing the library result. The rebased diff suite passed all 27 tests on each backend locally, CLI first, with CPython 3.12.14, Git 2.54.0 and GixPython 0.2.0. Ruff lint and format, ShellCheck and the repository hooks passed for the affected files. The rebase retains one visible commit. Full suites, documentation builds and hosted CI were not rerun for this local update. Refresh the stack onto `main` at `e5b960a4`. Retain the existing config and ignored-path implementations and adapt the upstream regressions to scripted fixtures. Direct Git comparisons confirm that unknown config escapes are errors, including Windows-style paths, and that appending a section after a dangling EOF continuation changes the preceding value. Replace seven incompatible expectations with Git's demonstrated read and write behavior, including exact serialized-byte comparisons. Document the retained general-config CLI fallback in `doc/gix-backend.md`. The tests-only resolution also exposed a real failure in the shared parser for Git CLI patch output. Retain the upstream fix for ambiguous header-only paths and copy metadata. Build the binary, empty-file, copy, mode-change and 500,000-byte path fixtures with Git plumbing so their names need not exist on the host filesystem. Preserve the upstream parser and linear-copy tests, and compare integration expectations with Git's NUL-delimited output. The resolved config, diff and repository modules passed 223 tests and 14 subtests on each backend, CLI first, with two skips per backend, using CPython 3.12.14, Git 2.54.0 and GixPython 0.2.0 on macOS. Ruff lint/format, ShellCheck and the repository hooks passed for the resolution files. The completed rebase preserves the single visible change and restores `gix-backend`. Full suites, documentation builds, other platforms and hosted CI were not rerun, and this local update was not pushed. The Windows Gix Python 3.12 check on `19352db` exposed a real adapter failure in `test_ignored_items_reported_verbatim`. Git 2.56.0.windows.2 accepts the queried path `q"uote.env` through `check-ignore --stdin -z`, but the adapter's Python metadata lookup raises `WinError 123`. Retain the Git CLI fallback when path metadata cannot be read rather than treating that failure as a missing path or inventing its kind. This also preserves Git's behavior when metadata is inaccessible because of permissions. Add portable injected-error regressions for invalid filenames and permission errors, keeping the original Windows integration regression unchanged. Record the adapter failure and missing native query capability in `doc/gix-backend.md`, without claiming a Gix excludes mismatch or lower CLI count. Focused validation passed nine tests with Gix and three ignored path tests with CLI; Ruff lint/format and the repository hooks passed. Hosted CI must validate the actual Windows repair on the new head. Refresh the stack onto `main` at `ccc48117`, retaining the upstream URL credential redaction and line-feed-only blame parsing fixes. Resolve the fixture-import conflict, preserve both blame and untracked-file regressions, and adapt the new blame repository to a read-only scripted fixture. Set its author explicitly because the fixture harness supplies author environment variables. Use the existing writable empty fixture for clone-error checks. The affected `test/test_git.py`, `test/test_repo.py` and `test/test_util.py` modules passed 358 tests and 18 subtests on each backend, CLI first, with 42 skips per backend. Validation used CPython 3.12.14, Git 2.54.0 and GixPython 0.2.0 on macOS. Ruff lint/format, ShellCheck and whitespace checks passed for the affected files. Full suites, other platforms and hosted CI were not rerun; this local rebase was not pushed. Update both the optional `gix` dependency and the shared test-fixture pin to `GixPython` 0.3.0, backed by crates.io `gix` 0.90.0 and `gix-testtools` 0.22.0. Update current installation/CI guidance while retaining historical measurements and compatibility evidence at their original versions. The default source build now uses pure Rust decompression and requires Rust 1.89+ and a system linker rather than a C/C++ compiler for those dependencies. Focused validation with the official 0.3.0 wheel confirms native discovery, HEAD and commit lookup match Git, and the shared scripted fixture API works. The reftable probe still raises native unsupported-storage errors; GitPython's existing GIX-1 fallback resolves HEAD through Git. No upstream compatibility bug is closed solely by the dependency update. Full tests, builds, benchmarks and documentation builds are outside the requested fast QA sweep. Co-authored-by: GPT-6 Astra Co-authored-by: Codex Co-authored-by: Byron --- .basedpyright/baseline.json | 172 -- .github/workflows/alpine-test.yml | 7 +- .github/workflows/backend-benchmark.yml | 81 + .github/workflows/cygwin-test.yml | 39 +- .github/workflows/downstream.yml | 51 + .github/workflows/git-cli.yml | 60 + .github/workflows/pythonpackage.yml | 105 +- .gitignore | 2 + AGENTS.md | 40 + CONTRIBUTING.md | 23 + MANIFEST.in | 2 + README.md | 10 +- doc/gix-backend.md | 1165 ++++++++++++ doc/source/changes.rst | 145 ++ doc/source/intro.rst | 8 +- doc/source/tutorial.rst | 29 +- fuzzing/README.md | 6 + fuzzing/fuzz-targets/fuzz_blob.py | 9 +- fuzzing/fuzz-targets/fuzz_config.py | 4 + fuzzing/fuzz-targets/fuzz_diff.py | 11 +- fuzzing/fuzz-targets/fuzz_repo.py | 20 +- fuzzing/local-dev-helpers/Dockerfile | 5 +- fuzzing/oss-fuzz-scripts/build.sh | 14 +- .../container-environment-bootstrap.sh | 13 +- git/__init__.py | 93 +- git/_backend.py | 1289 +++++++++++++ git/cmd.py | 399 ++-- git/compat.py | 70 - git/config.py | 1217 +++--------- git/db.py | 131 +- git/diff.py | 68 +- git/index/base.py | 817 ++++---- git/index/fun.py | 635 +------ git/index/typ.py | 82 +- git/index/util.py | 26 +- git/objects/base.py | 29 +- git/objects/commit.py | 364 ++-- git/objects/fun.py | 266 +-- git/objects/submodule/base.py | 420 +++-- git/objects/submodule/root.py | 15 +- git/objects/submodule/util.py | 7 +- git/objects/tag.py | 40 +- git/objects/tree.py | 56 +- git/objects/util.py | 122 +- git/refs/head.py | 31 +- git/refs/log.py | 450 +---- git/refs/reference.py | 32 +- git/refs/remote.py | 18 +- git/refs/symbolic.py | 651 +++---- git/refs/tag.py | 31 +- git/remote.py | 356 ++-- git/repo/base.py | 635 ++++--- git/repo/fun.py | 716 ++----- git/types.py | 47 - git/util.py | 203 +- gix-requirements.txt | 1 + pyproject.toml | 7 +- test-requirements.txt | 1 + test/cleanup.py | 81 + test/conftest.py | 127 ++ test/deprecation/__init__.py | 19 - test/deprecation/lib.py | 27 - test/deprecation/test_basic.py | 164 -- test/deprecation/test_cmd_git.py | 391 ---- test/deprecation/test_compat.py | 84 - test/deprecation/test_toplevel.py | 233 --- test/deprecation/test_types.py | 64 - test/downstream/README.md | 110 ++ test/downstream/checks/swebench.py | 75 + test/downstream/projects.json | 65 + test/downstream/run.py | 192 ++ test/lib/fixtures.py | 113 ++ test/lib/helper.py | 263 +-- test/performance/README.md | 101 + test/performance/bench_repository.py | 199 ++ test/performance/benchmark-requirements.txt | 2 + test/performance/cli-budget.json | 19 + test/performance/compare_backends.py | 65 + test/performance/lib.py | 83 - test/performance/test_benchmark_comparison.py | 22 + test/performance/test_commit.py | 129 -- test/performance/test_odb.py | 88 - test/performance/test_streams.py | 170 -- test/run-local.py | 53 + test/test_actor.py | 18 +- test/test_backend_selection.py | 28 + test/test_base.py | 54 +- test/test_blob_filter.py | 5 +- test/test_cleanup.py | 296 +++ test/test_cli_objects_index.py | 238 +++ test/test_cli_process_counts.py | 56 + test/test_cli_safety.py | 191 ++ test/test_clone.py | 97 +- test/test_command_guards.py | 112 +- test/test_commit.py | 571 +++--- test/test_config.py | 1648 +++-------------- test/test_diff.py | 478 ++--- test/test_docs.py | 17 +- test/test_exc.py | 4 +- test/test_fixture_health.py | 51 + test/test_fun.py | 316 +--- test/test_git.py | 249 ++- test/test_gix_backend.py | 752 ++++++++ test/test_imports.py | 12 +- test/test_index.py | 1299 ++++--------- test/test_installation.py | 66 +- test/test_positional_args.py | 104 +- test/test_quick_doc.py | 6 +- test/test_reflog.py | 242 ++- test/test_refs.py | 513 +++-- test/test_remote.py | 238 +-- test/test_remote_cli.py | 72 + .../{deprecation => }/test_remote_progress.py | 0 test/test_repo.py | 1213 +++++------- test/test_rev_parse.py | 174 +- test/test_submodule.py | 1502 +++++++-------- test/test_submodule_no_fetch.py | 174 +- test/test_tree.py | 165 +- test/test_typing.py | 9 +- test/test_util.py | 300 +-- tests/fixtures/benchmark-comparison.sh | 33 + tests/fixtures/benchmark.sh | 15 + tests/fixtures/cleanup.sh | 33 + tests/fixtures/clone.sh | 20 + tests/fixtures/command-guards.sh | 15 + tests/fixtures/commit.sh | 91 + tests/fixtures/config.sh | 63 + tests/fixtures/dependencies.sh | 20 + tests/fixtures/diff.sh | 116 ++ tests/fixtures/docs.sh | 4 + tests/fixtures/downstream-swebench.sh | 20 + tests/fixtures/empty.sh | 14 + tests/fixtures/fixture-lifecycle.sh | 9 + tests/fixtures/fun.sh | 27 + tests/fixtures/git-command.sh | 29 + tests/fixtures/gix-backend.sh | 187 ++ tests/fixtures/history.sh | 36 + tests/fixtures/index.sh | 126 ++ tests/fixtures/installation.sh | 15 + tests/fixtures/local-environment.sh | 14 + tests/fixtures/objects-index.sh | 81 + tests/fixtures/positional.sh | 34 + tests/fixtures/quick-doc.sh | 11 + tests/fixtures/reference-security.sh | 53 + tests/fixtures/reflog.sh | 20 + tests/fixtures/refs-input.sh | 5 + tests/fixtures/refs.sh | 35 + tests/fixtures/remote-cli.sh | 15 + tests/fixtures/remote-timeout.sh | 5 + tests/fixtures/repo.sh | 199 ++ tests/fixtures/rev-parse.sh | 64 + tests/fixtures/rmtree.sh | 21 + tests/fixtures/safety.sh | 28 + tests/fixtures/submodule-cases.sh | 121 ++ tests/fixtures/submodule-layout.sh | 81 + tests/fixtures/submodule-no-fetch.sh | 120 ++ tests/fixtures/submodule-relative.sh | 82 + tests/fixtures/submodule-source.sh | 10 + tests/fixtures/submodule-special.sh | 37 + tests/fixtures/submodule-symlinks.sh | 74 + tests/fixtures/tree-order.sh | 9 + tests/fixtures/util.sh | 21 + tox.ini | 15 + 163 files changed, 14160 insertions(+), 13423 deletions(-) create mode 100644 .github/workflows/backend-benchmark.yml create mode 100644 .github/workflows/downstream.yml create mode 100644 .github/workflows/git-cli.yml create mode 100644 doc/gix-backend.md create mode 100644 git/_backend.py create mode 100644 gix-requirements.txt create mode 100644 test/cleanup.py create mode 100644 test/conftest.py delete mode 100644 test/deprecation/__init__.py delete mode 100644 test/deprecation/lib.py delete mode 100644 test/deprecation/test_basic.py delete mode 100644 test/deprecation/test_cmd_git.py delete mode 100644 test/deprecation/test_compat.py delete mode 100644 test/deprecation/test_toplevel.py delete mode 100644 test/deprecation/test_types.py create mode 100644 test/downstream/README.md create mode 100644 test/downstream/checks/swebench.py create mode 100644 test/downstream/projects.json create mode 100644 test/downstream/run.py create mode 100644 test/lib/fixtures.py create mode 100644 test/performance/README.md create mode 100644 test/performance/bench_repository.py create mode 100644 test/performance/benchmark-requirements.txt create mode 100644 test/performance/cli-budget.json create mode 100644 test/performance/compare_backends.py delete mode 100644 test/performance/lib.py create mode 100644 test/performance/test_benchmark_comparison.py delete mode 100644 test/performance/test_commit.py delete mode 100644 test/performance/test_odb.py delete mode 100644 test/performance/test_streams.py create mode 100644 test/run-local.py create mode 100644 test/test_backend_selection.py create mode 100644 test/test_cleanup.py create mode 100644 test/test_cli_objects_index.py create mode 100644 test/test_cli_process_counts.py create mode 100644 test/test_cli_safety.py create mode 100644 test/test_gix_backend.py create mode 100644 test/test_remote_cli.py rename test/{deprecation => }/test_remote_progress.py (100%) create mode 100644 tests/fixtures/benchmark-comparison.sh create mode 100644 tests/fixtures/benchmark.sh create mode 100755 tests/fixtures/cleanup.sh create mode 100644 tests/fixtures/clone.sh create mode 100644 tests/fixtures/command-guards.sh create mode 100644 tests/fixtures/commit.sh create mode 100644 tests/fixtures/config.sh create mode 100755 tests/fixtures/dependencies.sh create mode 100644 tests/fixtures/diff.sh create mode 100644 tests/fixtures/docs.sh create mode 100644 tests/fixtures/downstream-swebench.sh create mode 100755 tests/fixtures/empty.sh create mode 100755 tests/fixtures/fixture-lifecycle.sh create mode 100755 tests/fixtures/fun.sh create mode 100644 tests/fixtures/git-command.sh create mode 100644 tests/fixtures/gix-backend.sh create mode 100755 tests/fixtures/history.sh create mode 100755 tests/fixtures/index.sh create mode 100755 tests/fixtures/installation.sh create mode 100755 tests/fixtures/local-environment.sh create mode 100644 tests/fixtures/objects-index.sh create mode 100644 tests/fixtures/positional.sh create mode 100644 tests/fixtures/quick-doc.sh create mode 100644 tests/fixtures/reference-security.sh create mode 100644 tests/fixtures/reflog.sh create mode 100644 tests/fixtures/refs-input.sh create mode 100644 tests/fixtures/refs.sh create mode 100644 tests/fixtures/remote-cli.sh create mode 100644 tests/fixtures/remote-timeout.sh create mode 100644 tests/fixtures/repo.sh create mode 100644 tests/fixtures/rev-parse.sh create mode 100644 tests/fixtures/rmtree.sh create mode 100644 tests/fixtures/safety.sh create mode 100644 tests/fixtures/submodule-cases.sh create mode 100644 tests/fixtures/submodule-layout.sh create mode 100644 tests/fixtures/submodule-no-fetch.sh create mode 100644 tests/fixtures/submodule-relative.sh create mode 100644 tests/fixtures/submodule-source.sh create mode 100644 tests/fixtures/submodule-special.sh create mode 100644 tests/fixtures/submodule-symlinks.sh create mode 100644 tests/fixtures/tree-order.sh create mode 100644 tests/fixtures/util.sh diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index f51f7ce13..f0ba85be4 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -8,75 +8,9 @@ "endColumn": 45, "lineCount": 1 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 41, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 32, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportCallIssue", - "range": { - "startColumn": 25, - "endColumn": 46, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 39, - "lineCount": 1 - } - } - ], - "./git/db.py": [ - { - "code": "reportIncompatibleMethodOverride", - "range": { - "startColumn": 8, - "endColumn": 12, - "lineCount": 1 - } - }, - { - "code": "reportIncompatibleMethodOverride", - "range": { - "startColumn": 8, - "endColumn": 14, - "lineCount": 1 - } } ], "./git/index/base.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 36, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 28, - "endColumn": 34, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -186,14 +120,6 @@ "endColumn": 25, "lineCount": 1 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 52, - "endColumn": 84, - "lineCount": 1 - } } ], "./git/objects/tag.py": [ @@ -274,32 +200,6 @@ } } ], - "./git/refs/log.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 17, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 28, - "endColumn": 30, - "lineCount": 1 - } - } - ], "./git/refs/reference.py": [ { "code": "reportIncompatibleVariableOverride", @@ -310,16 +210,6 @@ } } ], - "./git/refs/symbolic.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 15, - "endColumn": 20, - "lineCount": 1 - } - } - ], "./git/refs/tag.py": [ { "code": "reportIncompatibleMethodOverride", @@ -338,49 +228,7 @@ } } ], - "./git/remote.py": [ - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 26, - "endColumn": 38, - "lineCount": 1 - } - } - ], "./git/repo/base.py": [ - { - "code": "reportReturnType", - "range": { - "startColumn": 15, - "endColumn": 28, - "lineCount": 1 - } - }, { "code": "reportTypedDictNotRequiredAccess", "range": { @@ -445,26 +293,6 @@ "lineCount": 1 } } - ], - "./git/repo/fun.py": [ - { - "code": "reportReturnType", - "range": { - "startColumn": 11, - "endColumn": 20, - "lineCount": 1 - } - } - ], - "./test/deprecation/test_basic.py": [ - { - "code": "reportUnusedExpression", - "range": { - "startColumn": 12, - "endColumn": 62, - "lineCount": 1 - } - } ] } } diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index d87fdd079..b3558d214 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -12,6 +12,8 @@ permissions: jobs: test: runs-on: ubuntu-latest + env: + GIT_PYTHON_BACKEND: cli container: image: alpine:latest @@ -23,8 +25,8 @@ jobs: steps: - name: Prepare Alpine Linux run: | - apk add sudo git git-daemon python3 py3-pip py3-virtualenv - echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env + apk add sudo git git-daemon bash rust cargo build-base python3 python3-dev py3-pip py3-virtualenv + echo 'Defaults env_keep += "CI GITHUB_* RUNNER_* GIT_PYTHON_BACKEND"' >/etc/sudoers.d/ci_env addgroup -g 127 docker adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. adduser runner docker @@ -64,6 +66,7 @@ jobs: run: | . .venv/bin/activate pip install ./smmap ./gitdb '.[test]' + python -c 'from gix import testtools' - name: Show POSIX file ownership run: | diff --git a/.github/workflows/backend-benchmark.yml b/.github/workflows/backend-benchmark.yml new file mode 100644 index 000000000..d02dc73f1 --- /dev/null +++ b/.github/workflows/backend-benchmark.yml @@ -0,0 +1,81 @@ +name: Backend benchmark + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + compare-backends: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Ensure Git 2.52 or newer + run: | + if ! dpkg --compare-versions "$(git version | awk '{print $3}')" ge 2.52; then + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + fi + - name: Prepare Rust for GixPython + run: | + rustup update stable + rustup default stable + echo "CARGO_TARGET_DIR=$RUNNER_TEMP/gix-target" >> "$GITHUB_ENV" + - name: Cache GixPython builds + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ runner.temp }}/gix-target + ~/.cache/pip + key: benchmark-gix-${{ runner.os }}-${{ hashFiles('gix-requirements.txt') }} + - name: Install separate backends with the same interpreter + run: | + python -m venv .bench-cli + python -m venv .bench-gix + .bench-cli/bin/python -m pip install . -r test/performance/benchmark-requirements.txt + .bench-gix/bin/python -m pip install '.[gix]' -r test/performance/benchmark-requirements.txt + - name: Prepare the shared read-only benchmark fixture + run: | + .bench-cli/bin/python -c 'from pathlib import Path; from gix import testtools; print("BENCH_REPO=" + str((testtools.scripted_fixture_read_only_with_args("benchmark.sh", [str(Path.cwd())]) / "repo").resolve()))' >> "$GITHUB_ENV" + mkdir benchmark-results + - name: Measure GixPython operations, opening and discovery first + env: + GIT_PYTHON_BACKEND: gix + run: | + .bench-gix/bin/python -m test.performance.bench_repository \ + --repo "$BENCH_REPO" --expect-backend gix -o benchmark-results/gix.json + - name: Measure CLI operations, opening and discovery + env: + GIT_PYTHON_BACKEND: cli + run: | + .bench-cli/bin/python -m test.performance.bench_repository \ + --repo "$BENCH_REPO" --expect-backend cli -o benchmark-results/cli.json + - name: Check parity and publish comparison + run: | + .bench-gix/bin/python -m test.performance.compare_backends \ + benchmark-results/cli.json benchmark-results/gix.json \ + --cli-budget test/performance/cli-budget.json > benchmark-results/comparison.md + cat benchmark-results/comparison.md >> "$GITHUB_STEP_SUMMARY" + .bench-gix/bin/python -m pyperf compare_to \ + benchmark-results/cli.json benchmark-results/gix.json --table --table-format md \ + >> "$GITHUB_STEP_SUMMARY" + - name: Retain raw timings and comparison + if: always() + uses: actions/upload-artifact@v7 + with: + name: backend-benchmark + path: benchmark-results/ diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 4f7032f31..d01676f43 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -1,4 +1,4 @@ -name: test-cygwin +name: import-cygwin on: push: @@ -11,22 +11,12 @@ permissions: jobs: test: - strategy: - matrix: - selection: [fast, perf] - include: - - selection: fast - additional-pytest-args: --ignore=test/performance - - selection: perf - additional-pytest-args: test/performance - - fail-fast: false - runs-on: windows-latest env: CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" + GIT_PYTHON_BACKEND: cli defaults: run: @@ -45,7 +35,9 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v6 with: - packages: curl git python39 python-setuptools-wheel + packages: >- + curl gcc-core git libcurl-devel libexpat-devel libiconv-devel + libssl-devel make python39 python-setuptools-wheel zlib-devel add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output @@ -53,6 +45,15 @@ jobs: # Arrange for verbose output but without shell environment setup details. echo 'set -x' >~/.bash_profile + - name: Build minimum supported Cygwin Git + run: | + # Cygwin's packaged Git is older than our minimum supported version. + git clone --depth 1 --branch v2.52.0 -- https://github.com/git/git.git /tmp/git-source + # Use Cygwin's normal prefix: GitPython checks for uname next to git. + make -C /tmp/git-source -j2 prefix=/usr NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease install + hash -r + test "$(git version)" = 'git version 2.52.0' + - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" @@ -83,9 +84,12 @@ jobs: run: | python -m pip install -U 'setuptools; python_version<"3.12"' wheel - - name: Install project and test dependencies + - name: Install project run: | - pip install ./smmap ./gitdb '.[test]' + pip install ./smmap ./gitdb . + + - name: Verify Cygwin Git detection + run: python -c 'from git import Git; assert Git.is_cygwin()' - name: Show POSIX file ownership # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid @@ -135,6 +139,5 @@ jobs: python --version python -c 'import os, sys; print(f"{sys.platform=}, {os.name=}")' - - name: Test with pytest (${{ matrix.additional-pytest-args }}) - run: | - pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} + - name: Smoke check the installed package on Python 3.9 + run: python -I -c "import git; from git import _backend; assert _backend.name == 'cli'; print(git.__version__)" diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml new file mode 100644 index 000000000..9483b6586 --- /dev/null +++ b/.github/workflows/downstream.yml @@ -0,0 +1,51 @@ +name: Downstream compatibility + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + project: [langchain, mlflow, bandit, swebench, datahub] + backend: [gix, cli] + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Ensure Git 2.52 or newer + run: | + if ! dpkg --compare-versions "$(git version | awk '{print $3}')" ge 2.52; then + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + fi + - name: Prepare Rust for GixPython fixture tooling + run: | + rustup update stable + rustup default stable + echo "CARGO_TARGET_DIR=$RUNNER_TEMP/gix-target" >> "$GITHUB_ENV" + - name: Cache GixPython builds + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ runner.temp }}/gix-target + key: downstream-gix-${{ runner.os }}-${{ hashFiles('gix-requirements.txt') }} + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: '0.12.13' + enable-cache: false + - name: Test latest release against this checkout + run: uv run test/downstream/run.py "${{ matrix.project }}" --backend "${{ matrix.backend }}" diff --git a/.github/workflows/git-cli.yml b/.github/workflows/git-cli.yml new file mode 100644 index 000000000..567bc9373 --- /dev/null +++ b/.github/workflows/git-cli.yml @@ -0,0 +1,60 @@ +name: Minimum Git CLI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + git-2-52: + runs-on: ubuntu-latest + env: + GIT_PYTHON_BACKEND: cli + steps: + - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + repository: git/git + ref: v2.52.0 + path: .git-source + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - name: Build minimum supported Git + working-directory: .git-source + run: | + make -j2 prefix="$RUNNER_TEMP/git" NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease NO_CURL=YesPlease install + echo "$RUNNER_TEMP/git/bin" >> "$GITHUB_PATH" + - name: Prepare Rust for fixture tooling + run: | + rustup update stable + rustup default stable + echo "CARGO_TARGET_DIR=$RUNNER_TEMP/gix-target" >> "$GITHUB_ENV" + - name: Cache GixPython builds + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ runner.temp }}/gix-target + ~/.cache/pip + key: minimum-git-gix-${{ runner.os }}-${{ hashFiles('gix-requirements.txt') }} + - name: Install GitPython and test dependencies + run: | + python -m pip install ./smmap ./gitdb '.[test]' + python -c 'from gix import testtools' + - name: Verify native format and safety contracts + env: + GIT_CONFIG_NOSYSTEM: '1' + GIT_CONFIG_GLOBAL: /dev/null + GIT_AUTHOR_NAME: GitPython Tests + GIT_AUTHOR_EMAIL: tests@example.invalid + GIT_COMMITTER_NAME: GitPython Tests + GIT_COMMITTER_EMAIL: tests@example.invalid + run: | + git version + python -m pytest --no-cov --tb=short test/test_cli_objects_index.py test/test_cli_safety.py test/test_reflog.py test/test_config.py test/test_remote_cli.py test/test_submodule.py::test_submodule_cli_lifecycle diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 94630dbfc..53e0e08cc 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,10 +14,12 @@ permissions: jobs: test: + name: test (${{ matrix.os-type }}, ${{ matrix.python-version }}${{ matrix.backend == 'gix' && ', gix' || '' }}) strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] + python-version: ["3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] + backend: [cli] exclude: - os-type: macos python-version: "3.14t" @@ -25,17 +27,32 @@ jobs: python-version: "3.15t" include: - build-docs: true # We ensure documentation builds, except on very old interpreters. - - python-version: "3.8" - build-docs: false - experimental: false - python-version: "3.15" experimental: true - python-version: "3.15t" experimental: true + - os-type: ubuntu + python-version: "3.12" + backend: gix + build-docs: false + experimental: false + - os-type: macos + python-version: "3.12" + backend: gix + build-docs: false + experimental: false + - os-type: windows + python-version: "3.12" + backend: gix + build-docs: false + experimental: false fail-fast: false runs-on: ${{ matrix.os-type }}-latest + env: + GIT_PYTHON_BACKEND: ${{ matrix.backend }} defaults: run: @@ -51,6 +68,32 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + cache: pip + cache-dependency-path: | + pyproject.toml + requirements.txt + test-requirements.txt + gix-requirements.txt + + - name: Prepare Rust for GixPython source builds + id: rust + run: | + # GixPython 0.3.0 publishes macOS wheels only and needs Rust >= 1.89. + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version + echo "version=$(rustc --version)" >> "$GITHUB_OUTPUT" + + - name: Cache GixPython Cargo dependencies and build output + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + ${{ runner.temp }}/gix-target + key: gix-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ steps.rust.outputs.version }}-${{ hashFiles('gix-requirements.txt') }} + restore-keys: | + gix-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ steps.rust.outputs.version }}- - name: Set up WSL (Windows) if: matrix.os-type == 'windows' @@ -58,6 +101,19 @@ jobs: with: wsl-version: 1 + - name: Install current Git (Linux) + if: matrix.os-type == 'ubuntu' + run: | + sudo add-apt-repository --yes ppa:git-core/ppa + sudo apt-get update + sudo apt-get install --yes git + + - name: Install current Git (macOS) + if: matrix.os-type == 'macos' + run: | + brew install git + echo "$(brew --prefix git)/bin" >> "$GITHUB_PATH" + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh @@ -76,7 +132,13 @@ jobs: - name: Install project and test dependencies run: | - pip install ./smmap ./gitdb '.[test]' + pip install ./smmap ./gitdb ".[${EXTRAS}]" + python -c 'import os; from gix import testtools; from git import _backend; print("Backend:", _backend.name); assert _backend.name == os.environ["EXPECTED_BACKEND"]' + env: + EXTRAS: ${{ matrix.backend == 'gix' && 'test,gix' || 'test' }} + EXPECTED_BACKEND: ${{ matrix.backend }} + # Keep Cargo output when pip removes its temporary source directory. + CARGO_TARGET_DIR: ${{ runner.temp }}/gix-target - name: Show POSIX file ownership # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a @@ -159,11 +221,44 @@ jobs: - name: Test with pytest run: | - pytest --color=yes -p no:sugar --instafail -vv + if [[ "$RUNNER_OS" == "Windows" && -z "${GIT_PYTHON_GIT_EXECUTABLE+x}" ]]; then + # Absolute paths let Windows share minimum-version probes. + git_executable="$(python -c 'import os, shutil; print(os.path.abspath(shutil.which("git")))')" + export GIT_PYTHON_GIT_EXECUTABLE="$git_executable" + fi + pytest --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=test-results/pytest.xml --backend-report=test-results/backend.json continue-on-error: false + - name: Retain test results and backend operation counts + if: always() + uses: actions/upload-artifact@v7 + with: + name: tests-${{ matrix.os-type }}-${{ matrix.python-version }}-${{ matrix.backend }} + path: test-results/ + - name: Documentation if: matrix.build-docs run: | pip install --group doc make -C doc html + + import-smoke: + name: install/import (${{ matrix.os }}, ${{ matrix.python }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.8", "3.9", "3.10"] + env: + GIT_PYTHON_BACKEND: cli + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python }} + - name: Install the supported runtime + run: python -m pip install . + - name: Import the installed package + run: python -I -c "import git; from git import _backend; assert _backend.name == 'cli'; print(git.__version__)" diff --git a/.gitignore b/.gitignore index 79e543121..a3e04ea0f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ __pycache__/ .cache/ .mypy_cache/ .pytest_cache/ +tests/fixtures/generated-do-not-edit/ +tests/fixtures/generated-archives/*.tar # Transient editor files *.swp diff --git a/AGENTS.md b/AGENTS.md index a308591ea..56d2eee28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,46 @@ when communicating through a person's account. issue with a fix attached. Do not create an issue solely to follow it immediately with a PR that fixes it; describe the problem in the PR instead. +# Gix backend implementations + +Gix backend operations must use Gix through the supported GixPython (`gix`) +APIs. Python/stdlib implementations, direct filesystem inference, and custom +native implementations outside Gix are not substitutes for a missing Gix +operation. Do not implement Git behavior independently to avoid CLI calls. + +"Native" in code, documentation, reports and benchmarks means execution by +Gitoxide through GixPython's `gix` API. It never means Python-implemented Git +behavior. Every eliminated CLI call must be backed by identified Gix calls. + +Python glue may validate inputs, adapt Gix results to GitPython's return types +and formatting, manage the retained `gix.Repository`, and select a CLI fallback. +It must not invent a successful result or repair missing Git semantics with a +separate implementation. Retaining and explicitly recreating a Gix repository +through the shared accessor is allowed. + +Adapting native locations may append the fixed metadata leaf names `modules` +and `COMMIT_EDITMSG` to `Repository.git_dir()`, whose location Gix resolves. +Input paths may be canonicalized before reopening through Gix; return metadata +from that native handle. These adaptations must match Git in regression tests +and do not authorize a Python implementation of the general `--git-path` rules. + +If Gix lacks a capability, differs from the required behavior, or its use is +unclear, keep the existing Git CLI path and document the gap in +`doc/gix-backend.md`. State the expected behavior, available evidence, and what +GixPython or Gitoxide needs to expose or fix upstream. Tests, coverage reports +and benchmark ceilings must reflect that fallback; a lower CLI count alone +does not demonstrate a Gix implementation. + +Track every observed difference from the equivalent Git operation as a Git +compatibility bug, even if the Gix behavior is intentional. Gix must provide +matching behavior, at least through a mode as strict as Git. Record Git's +expected behavior, Gix's actual behavior, versions, reproduction or regression, +and the affected adapter/fallback in the ledger. Keep these bugs open when a +Gix-based adapter workaround restores parity; close them only after verifying +the upstream fix or compatible mode. Distinguish demonstrated mismatches from +missing APIs and unverified behavior. Do not repair a compatibility bug by +implementing Git semantics in Python. + # Commit messages Follow Conventional Commits for every commit. Every commit must have a diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a155dc0d..2a66f60c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,29 @@ Attributing AI assistance in commit metadata, for example with a `Co-authored-by trailer, is welcome but not required. Code is reviewed the same way regardless of its origin. +## Temporary test directories + +Tests require Python 3.11+, Git 2.52+, Bash, and the `test` extra, which installs +GixPython even when testing the CLI backend. Select `GIT_PYTHON_BACKEND=cli` or +`GIT_PYTHON_BACKEND=gix` before running pytest. + +Put initial repository and filesystem states in `tests/fixtures/*.sh` and use +`test.lib.fixtures` to run them through `gix.testtools`. Read-only tests share +cached output; mutating tests use writable copies. Use `execute=True` only when +the fixture cannot be copied, and explain the reason at its call site. Keep +operations under test in Python. Immutable parser byte samples remain in +`test/fixtures`; generated caches and archives are ignored. + +Use `test.cleanup.cleanup_directory` for cleanup outside the shared fixture +owners. Close repository handles before removing their files. Cleanup is +best-effort: it logs filesystem errors, removes whatever it can, and leaves +locked files behind without failing or skipping a test. The shared writable +repository decorators still keep failed tests' directories for debugging. + +Deletions and renames that exercise library behavior or prepare a fixture for +reuse must remain strict. If a cached fixture cannot be cleaned up, rebuild it +at a fresh location before handing it to another test. + ## Fuzzing Test Specific Documentation For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please diff --git a/MANIFEST.in b/MANIFEST.in index eac2a1514..47a3cada3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,10 +5,12 @@ include LICENSE include README.md include VERSION include requirements.txt +include gix-requirements.txt include test-requirements.txt include git/py.typed recursive-include doc * recursive-exclude test * +prune tests global-exclude __pycache__ *.pyc diff --git a/README.md b/README.md index 8d1b590b2..72fb029f1 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,11 @@ probably the skills to scratch that itch of mine: implement `git` in a way that If you like the idea and want to learn more, please head over to [gitoxide](https://github.com/Byron/gitoxide), an implementation of 'git' in [Rust](https://www.rust-lang.org). -*(Please note that `gitoxide` is not currently available for use in Python, and that Rust is required.)* +The experimental `GitPython[gix]` extra uses GixPython 0.3.0 from PyPI for +supported operations, with Git CLI fallback. Set `GIT_PYTHON_BACKEND=cli` or +`gix` to select the backend explicitly. Tests use `gix.testtools` for scripted +fixtures with either backend and require Python 3.11+. See the +[installation, test commands, and conversion ledger](doc/gix-backend.md). ## GitPython @@ -40,10 +44,10 @@ The project is open to contributions of all kinds, as well as new maintainers. ### REQUIREMENTS GitPython needs the `git` executable to be installed on the system and available in your -`PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it +`PATH` for repository operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. -- Git (1.7.x or newer) +- Git (2.52 or newer) - Python >= 3.8 The list of dependencies are listed in [`./requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/requirements.txt) and [`./test-requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/test-requirements.txt). diff --git a/doc/gix-backend.md b/doc/gix-backend.md new file mode 100644 index 000000000..1403a8ef5 --- /dev/null +++ b/doc/gix-backend.md @@ -0,0 +1,1165 @@ +# GixPython backend + +Install the `gix` extra to make GitPython use GixPython for supported operations: +`GitPython[gix]`, or `.[gix]` from this checkout. GixPython is the distribution +name; `gix` is its import name. The extra uses the official PyPI release +`GixPython==0.3.0` on CPython 3.11 or newer. On older Python versions, the +dependency is omitted and GitPython uses the CLI backend. The ordinary +GitPython installation still supports Python 3.8 or newer. + +To install from this checkout with an existing interpreter: + +```sh +uv venv --python .venv/bin/python .tox/gix +uv pip install --python .tox/gix/bin/python --editable '.[test,gix]' +``` + +`tox -e gix` also resolves the published release from the package index. + +GixPython 0.3.0 publishes macOS wheels. On Windows and Linux, installation +builds the released source distribution and requires Rust 1.89 or newer and +a system linker. Its default build uses pure Rust hashing, compression and HTTPS; +no C/C++ compiler is required for those dependencies. The release uses Gitoxide's +`gix` 0.90.0 and `gix-testtools` 0.22.0 from crates.io. The fixture suites for both +backends need GixPython. CI prepares Rust for source builds and caches built wheels, +Cargo dependencies and compilation output on the main matrix. + +The 0.3.0 upgrade probes native discovery, HEAD/commit lookup and the scripted +fixture APIs against Git on CPython 3.12. Reftable remains unsupported: on the +`empty.sh sha1 reftable` fixture, Git reports `reftable` and resolves HEAD to +`master`, while `gix.open().head()` raises an unsupported-storage error. The +existing GIX-1 CLI fallback preserves that repository's behavior. Historical +measurements and compatibility findings below retain their original versions; +the release upgrade alone does not close them. + +Python extras add dependencies; they do not leave a runtime feature bit. +Set `GIT_PYTHON_BACKEND=cli` or `GIT_PYTHON_BACKEND=gix` before importing +GitPython. Explicit Gix selection fails if `gix` cannot be imported; an invalid +selection also fails. With the variable unset, GitPython selects Gix when it +can be imported and otherwise uses CLI. Test and benchmark runners select a +backend explicitly, because both installations contain GixPython for fixtures. + +Git 2.52 or newer remains required. Unsupported operations and call options use +the existing CLI implementation. Public `repo.git.()` calls retain +their command-line behavior. Native dispatch applies to library-managed calls. + +Here and in coverage reports, **native means Gitoxide execution through +GixPython's `gix` API**. It never means Python-implemented Git behavior. Python +glue validates inputs, adapts Gix results and manages repository handles. For +example, object reads call `Repository.find_object()` and reference edits call +`Repository.edit_references_as()`. The two fixed metadata paths use +`Repository.git_dir()` plus the requested basename; that adaptation does not +provide general Git-path resolution. A `native` counter must represent a +Gix-backed operation, and backend timings include its Python glue and any CLI +fallbacks. + +## Index and object metadata lifecycle + +`IndexFile` queries immutable entry records; it does not retain a parsed index. +Deferred/virtual state consists of opaque file bytes, materialized before Git +operates and read back only after success. Private edits use Git because native +bindings cannot load an arbitrary existing index (GIX-3) or validate a checked +batch of paths (GIX-26). Split snapshots become standalone through Git and regain +split publication through Git. Native default-index queries, fresh v2 indexes +and verified tree construction remain supported. Backend coverage must count +these editing fallbacks. + +Commit metadata uses `Repository.find_commit().decode()` plus structured +`Commit.author()`/`committer()` signatures. Tag metadata uses +`Repository.find_tag().decode()` and `Tag.tagger()`. Decoded extra headers retain +`gpgsig`. Raw Gix object bytes are used only for object streams. If repository +capabilities require CLI fallback, metadata uses `git cat-file`: general formatted +Git queries cannot faithfully expose all signature headers or arbitrary tags. +The bounded public identity helpers remain Python glue. Human date parsing uses +Git until GIX-27 is exposed; trailer extraction already uses Git. + +Benchmark remeasurement on 2026-10-09 used the same fixed SHA-1/files fixture +`6ba2c0a2f9ee7feffd7e079621c4845820180c9a`. All 14 result digests match between +CLI and Gix. Gix still uses 1 CLI process for the complete read journey, 0 for +index reads and commit history, 1 for patch diffs/opening, and 3 for discovery. +The CLI journey now uses 73 launches, including 26 for the 25-commit history; +raw metadata queries preserve exact bytes. Existing Gix budgets remain valid. +The `--fast` timing samples ran alongside validation and have substantial jitter; +use their digests/counts, not their wall times, as evidence for this migration. +These read-only budgets make no claim that index editing is native. + +## Install and test without package indexes + +Prepare `.venv` and `.tox/gix` with GixPython 0.3.0. Select the backend explicitly +when running against scripted fixtures: + +```sh +GIT_PYTHON_BACKEND=cli .venv/bin/python test/run-local.py --no-cov -q +GIT_PYTHON_BACKEND=gix .tox/gix/bin/python test/run-local.py --no-cov -q \ + --backend-report=.cache/gix-coverage.json +``` + +On Windows, use Git Bash, as CI does, and the environments' `Scripts` +directories. To retain CI's coverage and pytest options: + +```sh +GIT_PYTHON_BACKEND=cli .venv/Scripts/python.exe test/run-local.py --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=.cache/windows-cli.xml --backend-report=.cache/windows-cli.json +GIT_PYTHON_BACKEND=gix .tox/gix/Scripts/python.exe test/run-local.py --color=yes -p no:sugar --instafail -vv --durations=30 \ + --junitxml=.cache/windows-gix.xml --backend-report=.cache/windows-gix.json +``` + +The Windows runner resolves `git.exe` from `PATH` to an absolute path so +GitPython can share minimum-version checks. An explicit +`GIT_PYTHON_GIT_EXECUTABLE` override is preserved for GitPython calls. +Native fixture scripts run `git` from `PATH`, which must select the intended +Git version too. Git Bash normally selects `mingw64/bin/git.exe`; the +`cmd/git.exe` and `bin/git.exe` launchers add overhead to local measurements. +The private config retains CI's `core.autocrlf=true` on Windows and appends +the test aliases directly, matching CI without introducing config includes. + +The runner uses local version tags, creates an isolated Git configuration, +prepares the historical test fixture through cached Bash scripts, gives +pytest a separate temporary root for each run, and disables package-index +access. Cleanup of these isolated directories is best-effort, so a locked +leftover cannot change pytest's exit status. Tests use local repositories, +including the tutorial example. The suite needs loopback sockets for its Git daemon and +permission to inspect its own child processes. It does not need a remote Git +server. Missing local tags or packages are errors, not invitations to download. + +The ignored `.cache/gix-wheels` directory contains the published GixPython wheel and +the build/test dependencies available on this machine. To recreate an +environment with an already-installed CPython interpreter: + +```sh +uv --cache-dir .cache/uv venv --offline --python .venv/bin/python .tox/gix +uv --cache-dir .cache/uv pip install --offline --no-index \ + --find-links .cache/gix-wheels --python .tox/gix/bin/python \ + --editable '.[test,gix]' +``` + +For another machine, populate that directory from PyPI or its package cache +before running the offline installation tests. Select wheels compatible with +the interpreter and platform; ordinary and free-threaded CPython use different +wheels. On platforms without a published wheel, first build one from the +released source distribution and retain it in the wheelhouse. + +## Scripted test fixtures + +The `test` extra requires GixPython 0.3.0 on Python 3.11+. Full fixture suites +run on Python 3.11 and newer with either backend. Python 3.8–3.10 retain +installation/import smoke coverage, including the Cygwin Python 3.9 job. +Running tests also requires Bash and Git 2.52+; `gix` here is GixPython's module. + +Initial repository and filesystem states live in `tests/fixtures/*.sh`. +`test/lib/fixtures.py` calls `gix.testtools` with an explicit lifetime: + +- `read_only_fixture()` and the default `fixture_repo()` reuse cached output. + Tests must not mutate these paths, including object storage during serialization. +- `writable_fixture()`, `fixture_repo(write=True)`, and the existing writable + decorators copy cached output with `Creation.CopyFromReadOnly`. +- `execute=True` selects `Creation.Execute` only for states that cannot move or + copy correctly, such as symlinks, unreadable files, virtual environments, or + repository metadata containing absolute paths inside its own temporary root. + +Operations under test remain in Python. Scripts prepare their inputs through +Git CLI, whose behavior defines the expectations for both backends. Test-only +configuration, malformed metadata and safety sentinels are scripted too. +Pure in-memory parser inputs remain ordinary test data. The framework closes +repository handles before best-effort cleanup, and the writable decorators +retain failed tests' directories for inspection. + +Gix testtools caches script results under +`tests/fixtures/generated-do-not-edit/`. This directory and generated `.tar` +archives are ignored; no generated archives are checked in or required. +GixPython provides deterministic fixture identities and dates and isolates Git +configuration while the scripts run. Changes to sourced data/scripts must also +change a fixture argument or version, because only the entry script is hashed. + +### GixPython 0.2.0 testtools limitations + +These are test-infrastructure bugs, separate from the Git-operation ledger: + +- **Argument memoization:** absolute arguments reset the native path used as an + in-process cache key, discarding the script name and preceding arguments. + Distinct scenarios with the same absolute source path can return the first + scenario's output. `test_fixture_cache_keeps_arguments_before_absolute_paths_distinct` + reproduces this. The shared helper uses the supported versioned postprocessing + API, with a version derived from the full script/argument tuple and an empty + callback. Upstream must include each argument and the script in the key. +- **Copy semantics:** `Creation.CopyFromReadOnly` follows symlinks and changes directory + permissions (a mode `0500` directory becomes `0755`), and cannot copy + dangling links or unreadable files. Link and permission tests explicitly use + `Creation.Execute`; upstream needs a mode that preserves links and metadata + without reading inaccessible files. +- **Error diagnostics:** failed fixture scripts include the inherited environment + in `gix.Error`, potentially exposing credentials. The helper suppresses this + error text while retaining captured script output. Upstream should omit or + redact the environment in command failures. +- **Windows script arguments:** MSYS globbing alters literal arguments from the + native fixture launcher: `prefix-${GITPYTHON_TEST_SECRET}-suffix` reached the + script as `prefix-$GITPYTHON_TEST_SECRET-suffix` in Windows CI. Setting + `MSYS=noglob` instead broke every fixture before its script ran, reporting + `$1 : No such file or directory`. GixPython 0.2.0 pins Gitoxide commit + `09f85be4e3fda42f3e8fd19c53a63758fdf6fb3d`, whose + [fixture launcher](https://github.com/GitoxideLabs/gitoxide/blob/09f85be4e3fda42f3e8fd19c53a63758fdf6fb3d/tests/tools/src/lib.rs#L1805-L1816) + passes a quoted `bash -c` program through `std::process::Command`. + Literal-name fixtures use plain scenario identifiers and define their quoted + names inside Bash; the submodule regression still asserts each exact name + before and after the operation. Upstream must preserve both launcher quoting + and literal argument contents on Windows. Do not override MSYS argument + parsing in the shared helper. +- **Cleanup:** native `TempDir.__exit__()` raises on removal errors. The Python + owner preserves this project's best-effort cleanup contract for locked files, + so cleanup cannot replace a test's actual result. The binding has no + `keep()`/`into_path()` operation to release ownership without deleting files. + Failed-test directories are moved aside while the native owner closes; if a + locked directory cannot move, the helper tries to copy its artifacts with + symlinks preserved and logs the retained path. Copy failures are logged too + and cannot replace the original test failure. Upstream needs a way to detach + temporary-directory ownership. + +These observations use the official 0.2.0 wheel and its source tag `v0.2.0` +(`5bf4247facad8caf3981bd19aa260cd657611322`). The adapters remain necessary +until the corresponding upstream behavior has been verified fixed. + +## Conversion coverage + +The pytest header names the selected backend. Native runs print operation +counts and fallback reasons, and `--backend-report=PATH` saves the same records +as JSON. A record looks like: + +```json +{"method": "IndexFile.write", "outcome": "CLI: split index preservation (GIX-13)", "count": 2} +``` + +Search the report for `not converted` to find remaining command adapters, and +for `GIX-` to find upstream limitations. A high-level method can use several +native and CLI operations; counts describe operations, not entire user calls. +The `git.backend` DEBUG logger reports these decisions outside pytest. +`git._backend.statistics()` returns a snapshot of the process-local counters. +Zero observed calls means untested in that run, not unsupported. Fixture-script +Git launches happen outside `Git.execute` and its counters. Moving setup into +Bash does not represent native backend conversion. + +Successful `Git.execute` launches also appear as `CLI process` records. This +includes raw `repo.git` calls and failed Git commands that started a process, +but excludes failed process creation, direct subprocesses in test helpers, +other Python processes and Git's own child processes. A persistent `cat-file` +launch counts once; subsequent requests through it do not. These counts apply +to both installations and are separate from native/fallback decisions. + +Pytest prints session launch totals split into setup, call, teardown and +collection/session work. `--backend-report` keeps its existing record-list +format and adds `pytest.*` phase records. The `Git.execute` record is +process-lifetime cumulative (including import-time probes); `pytest.session_total` +is the current session's total. Optional ceilings fail the test session on an +increase while allowing reductions: + +```sh +GIT_PYTHON_BACKEND=gix .tox/gix/bin/python test/run-local.py --no-cov -q \ + --backend-report=.cache/gix-processes.json \ + --max-cli-processes=TOTAL --max-cli-test-processes=CALLS +``` + +Replace `TOTAL`/`CALLS` with baselines for the same selected tests and platform. +Setup is deliberately counted separately because fixture improvements and +test additions can change it independently of native backend capability. +These are pytest phases: `unittest.TestCase.setUp()`/`tearDown()` run inside +the call phase, so that column can still include their fixture work. Counts +cover the current Python process; they are not aggregated across xdist workers. +For a stable CI assertion, the pinned repository benchmark uses per-measurement +ceilings in `test/performance/cli-budget.json`: the warm Gix journey launches +one CLI process, opening launches one, and nested discovery launches three. +Reduce these ceilings as conversions land. The other operation rows have zero +ceilings except the one-process patch diff. + +Before the Gix-only audit, this instrumentation on CPython 3.12.14/macOS arm64 and official +GixPython 0.1.0 recorded a full suite without coverage passing 1,674 tests and 38 +subtests (79 skipped, one xfailed) in 408.81 seconds. It recorded **30,898 +`Git.execute` launches**: 6,523 in pytest setup and 24,375 in call phases, +with zero in teardown or collection/session phases. The process-lifetime +counter is 30,899 because the import-time Git probe precedes pytest session +accounting. The same run recorded 21,409 fallback decisions. These totals +include explicit CLI tests and fixture commands, including non-Git commands +passed directly to `Git.execute`; they are a local baseline, not a count of +only fallback calls. The expanded warm benchmark instead isolates a user journey: +43 CLI launches versus one Gix launch, opening nine versus one, and nested +discovery eleven versus three after restoring the reference-format fallback +and sharing successful version checks. +The historical full-suite counts above predate this correction. A constant +`files` answer was not a Gix query; `GIX-1` records the remaining native +repository-format validation gap. + +| GitPython entry point / managed command | Gix implementation | Remaining CLI cases | +| --- | --- | --- | +| `Git.get_object_header`, ODB `info` | Object resolution and native header lookup | Custom storage, unsupported revision grammar | +| `Git.stream_object_data`, ODB `stream` | Object bytes in an independent stream | Objects larger than 8 MiB; GIX-2 | +| Commit metadata | Structured `Commit.decode()`, `author()` and `committer()` for UTF-8/default encoding | Other declared encodings and encoding headers in unexpected order use Git's text projection; GIX-29/30 | +| Commit/tree serialization readback | Existing ODB stream; no separate `cat-file` process | The same storage and large-object fallbacks as ODB reads | +| `Repo` opening/discovery | Native storage, worktree, object format and empty-tree metadata | Repository-format validation, unsupported layouts/environment and native validation gaps; GIX-1/14/19 | +| Standalone discovery helpers | Native storage directories and gitfiles reopened at their canonical native Git directory | Rejected candidates and the same layout/environment guards as `Repo` opening; GIX-14 | +| `Repo.rev_parse`, `rev_parse` metadata queries | Revision lookup, common directory, object format, bare flag, worktree root, fixed `modules`/`COMMIT_EDITMSG` locations | Repository-format validation, other metadata paths, symlinked metadata leaves, message searches and describe forms; GIX-1/14/17/22 | +| Revision path/mode metadata | Native revision specification, including tree paths and index stages | Custom/sparse indexes, unsupported revision grammar or path normalization | +| Tree enumeration, `ls_tree` | Native tree entries with modes, names, and object IDs | Other command options | +| Reference object reads | One exact native lookup for direct, symbolic and absent references | Partial names and native decoding errors | +| Reference-name validation | Native full-name validation without opening a repository | Standalone names outside the native grammar; GIX-20 | +| Managed `symbolic_ref` | Nonrecursive target lookup | Mutation, command-level missing-reference diagnostics | +| Reference enumeration, `for_each_ref` | Sorted reference names and literal prefixes, preserving symbolic aliases | Dangling symbolic refs, root refs, glob patterns, other formats/options; GIX-25 | +| Managed `config --get KEY` | Merged repository config snapshot | Files/streams, enumeration, mutation; GIX-12 | +| Submodule enumeration and cached fields | Dedicated native `.gitmodules` parser over the existing worktree/blob source, with raw path/URL/branch values | Other/duplicate sections, ambiguous brackets, missing/implicit fields and parser errors; GIX-12 | +| Commit hooks | No bound native lookup/execution API | All hooks, including absent hooks; GIX-23 | +| Worktree inventory, `worktree list` | Main and linked worktree metadata, including bare main repositories and locks | Prunable entries; GIX-14 | +| `Repo.merge_base`, `Repo.is_ancestor` | Native graph queries for two revisions | Octopus/fork-point and other options | +| `Commit.count` | Reachable commit count, skip/limit/first-parent | Path filters and other revision options | +| `Commit.iter_items`, `Repo.iter_commits` | First-parent walks and a single tip | General history ordering; GIX-8 | +| Index entry reads, `ls_files` | Stage/mode/OID/path and exposed index flags | Custom and sparse indexes; GIX-3/4 | +| `IndexFile.version`, `update_index` query | Native index version | Actual `update-index` mutations | +| Index editing and `IndexFile.write` | Opaque byte snapshots; atomic locked publication | Private-file edits always use Git: arbitrary native index loading and strict pathname validation are missing; GIX-3/4/13/26 | +| ODB `store`, managed `hash_object` | Native object hashing/writing with byte-fidelity preflight | Tags, large/nonseekable input and changed serialization; GIX-2/5/28 | +| `IndexFile.write_tree` | Native tree editor with child-kind and null-ID validation | Missing non-gitlink children, invalid entries; GIX-7/24 | +| Tree serialization, `mktree` | Native tree editor with child-kind and null-ID validation | Missing children, unsupported or invalid entries; GIX-7/24 | +| Fresh index preparation, `read_tree` | Empty index or index from one tree | Existing index, merges, non-v2 selection; GIX-13 | +| `Commit.create_from_tree`, `commit_tree` | Explicit author/committer and unsigned UTF-8 commit | Identity cleanup, object-kind validation, other encodings, signature formats, signing/options; GIX-6/21/24 | +| Reference log reads, `reflog` | Existence and GitPython's reflog format | Orphan logs, arbitrary formats, writes; GIX-10 | +| Committer identity, `var` | Native process/config identity from `Repository.committer()` | Per-command identity overrides, other variables and normalization/identity errors; GIX-21 | +| Selected `set_object` / reference deletion, `update_ref` | Native ref edits with GitPython's existing validation | Strict create/CAS, branch-target validation, symbolic aliases other than HEAD, symbolic detachment, active-branch HEAD log, unchanged targets and log-message cleanup; GIX-9/10/21/24 | +| Tree/commit `Diffable.diff` | Raw change records and unambiguous exact renames | Patches, paths, index/worktree/root diff, inexact/ambiguous renames; GIX-11 | +| `Commit.stats` | Native Myers line counts, binary counts, first-parent tree comparison | Diff attributes, other algorithms, gitlinks, quoted filenames, large blobs; GIX-2/18 | +| `Repo.is_dirty` | Index/tree/worktree status, configured submodule checks and pathspecs | Custom/sparse indexes and non-root command directories | +| `Repo.untracked_files` | Native directory walk | Extra status options and non-root command directories | +| `Repo.ignored` | Native excludes with tracked-file suppression | Symlink/submodule traversal, unsupported path normalization, and unavailable path metadata | + +Repository-backed native operations fall back for reftable (GIX-1) and repositories +with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts, +custom storage/environment, global Git options, and unhandled command options +also retain their CLI contracts. A broken installed extension is reported as +an import error unless CLI was explicitly selected. With automatic selection, +only an absent top-level `gix` selects CLI mode. + +## Performance work + +Use the [measurements after the Gix-only audit](#measurements-after-the-gix-only-audit) +for the corrected implementation. Older snapshots retain their original +workloads and include shortcuts that were subsequently removed. + +### Shared minimum-version checks + +Both installations reuse successful minimum-Git-version checks across fresh +`Git` wrappers with the same resolved executable, executable metadata, working +directory, effective environment and `Git.refresh()` generation. Two extra +checks in the fixed warm probe now launch zero processes instead of two; +two equivalent cold wrappers launch one probe. Public `version_info` caching +remains per instance, and old versions still fail before repository creation. + +This is shared CLI housekeeping: it caches an actual Git answer and records +no native success. It is not a Gix implementation of a missing capability. +The pinned warm Gix opening/discovery ceilings fall from 2/4 to 1/3 here; +the reference-format query and rejected-candidate diagnostics remain real CLI +calls. Cold contexts can still require a version probe. + +Global Git options and ambiguous Windows executable search retain per-instance +probes; explicit absolute executables can share checks on Windows. The shared +cache is bounded to 128 contexts. Call `Git.refresh()` if a launcher's reported +version changes through external state without changing its executable or +environment metadata. + +### Existing-repository benchmark + +[`test/performance/README.md`](../test/performance/README.md) describes the +`pyperf` harness and fixed GitPython 3.1.45 fixture. It measures eleven public-API +operations and their complete journey on one already-open `git.Repo`, plus +separate direct opening and discovery from `git/objects`. Fresh high-level +wrappers preserve the cost of actual operations; imports, fixture preparation +and parity preflight are outside timing. Native repository refresh remains +inside operation timing, so these measurements include the cost of keeping +retained state current. + +Submodule inventory, revision path/mode lookup and raw commit/tree readback +extend the already-open journey, each with a zero-CLI ceiling. The readback +measurement reads existing bytes, including the signed fixture commit, without +rewriting objects. Historical tables below retain their original workloads. + +At `244e418da6cc43de129cbe2908d11be4c5ad457a`, using official GixPython +0.1.0 and the same existing CPython 3.12.14/macOS arm64 interpreter for both +installations, with Git 2.54.0 (Apple Git-157), six worker processes with three values each produced the +following means and standard deviations. GixPython ran first, then CLI; +all eleven result digests matched. The fixture has one branch, one untracked +file and an ignored directory, with ambient Git configuration disabled. + +| Measurement | CLI (ms) | GixPython (ms) | CLI / Gix | +| --- | ---: | ---: | ---: | +| Already-open journey | 230.90 ± 6.87 | 260.14 ± 13.37 | 0.89× | +| Reference inventory | 22.77 ± 1.15 | 1.17 ± 0.26 | 19.52× | +| 25 first-parent commits with metadata | 9.24 ± 0.46 | 73.61 ± 7.24 | 0.13× | +| Root tree and README blob | 16.02 ± 1.07 | 2.40 ± 0.46 | 6.66× | +| Index entries | 8.43 ± 0.57 | 2.00 ± 0.38 | 4.21× | +| Commit count and ancestry queries | 82.28 ± 2.27 | 36.19 ± 0.84 | 2.27× | +| Latest diff and commit statistics | 29.98 ± 1.66 | 60.98 ± 2.45 | 0.49× | +| Patch diff (includes CLI fallback) | 17.87 ± 1.28 | 10.86 ± 1.01 | 1.64× | +| Dirty/untracked/ignored paths | 68.56 ± 3.18 | 64.74 ± 2.00 | 1.06× | +| Direct opening and close | 95.08 ± 4.64 | 35.84 ± 2.02 | 2.65× | +| Nested discovery and close | 112.58 ± 5.38 | 49.20 ± 4.16 | 2.29× | + +Ratios above one favor GixPython. This workload's complete journey is about +13% slower with GixPython: metadata reads and commit statistics offset gains +elsewhere. These are warm-cache measurements on one machine, with `pyperf` +stability warnings for several samples; they do not establish a general +speedup. Full-suite times below also include setup and coverage. + +The separate `Backend benchmark` CI job runs both installations on one runner, +publishes every measurement and `pyperf` significance reporting, and retains +raw JSON and comparison artifacts. Results include revisions and per-operation +native/fallback decisions. Adding a `MEASUREMENTS` entry extends the journey, +individual timings and parity checks together. Opening/discovery remain +separate lifecycle measurements. CI fails on execution/parity errors; timing +ratios are observational on shared runners. + +### Historical retained-handle and opening measurements + +These results predate the Gix-only audit. The measured implementation inferred +reference format, repaired linked-worktree metadata and rejected some discovery +candidates in Python. Its zero-CLI counts and associated opening speedups do +not establish Gix coverage and are superseded by the corrected implementation. +Keep the raw observations below only as a historical record. + +At `c49bbec0afb806df66a33fa9c5d76e21ac8f0314`, the same pinned fixture, +official GixPython 0.1.0 and existing CPython 3.12.14/macOS arm64 interpreter +were measured sequentially, Gix first and CLI second, after tests finished. +Each backend used three worker processes, three values per worker, four loops +per value and one warmup. All eleven result digests matched and the checked-in +CLI ceilings passed. Raw local results are +`/private/tmp/gitpython-retained-final-{gix,cli}.json`; this table is a journal +snapshot, while CI continues to publish fresh measurements. + +| Measurement | CLI mean ± stdev (ms) | GixPython mean ± stdev (ms) | CLI / Gix | Native / fallback | CLI launches: CLI / Gix | +| --- | ---: | ---: | ---: | ---: | ---: | +| journey | 209.44 ± 17.65 | 263.25 ± 5.52 | 0.80× | 55 / 2 | 23 / 1 | +| inventory | 19.76 ± 0.78 | 0.89 ± 0.03 | 22.30× | 3 / 0 | 3 / 0 | +| history_25 | 7.49 ± 0.17 | 79.02 ± 9.67 | 0.09× | 26 / 0 | 1 / 0 | +| browse_tree_and_blob | 13.55 ± 0.39 | 2.06 ± 0.41 | 6.59× | 5 / 0 | 2 / 0 | +| read_index | 6.96 ± 0.08 | 2.59 ± 0.47 | 2.69× | 1 / 0 | 1 / 0 | +| revision_graph | 63.27 ± 2.57 | 32.72 ± 0.22 | 1.93× | 9 / 0 | 6 / 0 | +| diff_and_stats | 20.78 ± 0.37 | 65.33 ± 1.25 | 0.32× | 5 / 0 | 3 / 0 | +| patch_diff | 14.06 ± 0.39 | 8.61 ± 0.08 | 1.63× | 3 / 2 | 2 / 1 | +| worktree_status | 56.61 ± 1.43 | 70.68 ± 2.18 | 0.80× | 3 / 0 | 5 / 0 | +| open_repository | 69.28 ± 0.59 | 1.00 ± 0.07 | 69.20× | 1 / 0 | 11 / 0 | +| discover_repository | 81.68 ± 0.68 | 0.90 ± 0.19 | 90.52× | 1 / 0 | 13 / 0 | + +The pre-audit implementation launched no CLI processes for these opening and +discovery probes. It measured about 69× and 91× faster lifecycle operations, +while the complete already-open journey was about 26% slower with Gix. These +lifecycle ratios include the shortcuts removed by the audit. Retaining a Python +handle alone does not resolve the existing history/statistics costs. Several +rows have `pyperf` sample-size or variability warnings, so timings describe +this machine and workload rather than a general speed guarantee. + +The full Gix suite without coverage passed **1,678 tests and 38 subtests**, +with 79 skips and one expected failure, in **337.51 seconds (5m 37.51s)**. +One expected path-expansion deprecation warning was reported. It recorded +**22,319 `Git.execute` launches**: 3,833 setup, 18,486 call, zero teardown or +collection/session. The process-lifetime count is 22,320 including the import +probe. Against the preceding 408.81-second/30,898-launch baseline, this local +run took 17.4% less time and launched 27.8% fewer CLI processes; the suite has +four additional regressions, and the timing comparison is observational. + +Affected tests ran with Gix before CLI: 304/264 repository, command, backend +and safety tests; 136 command-guard tests per backend; 31 positional tests per +backend; 13 process-count/budget tests per backend. Repository suites also +passed 14 subtests, with three skips. Ruff lint/format, mypy and basedpyright +passed. Both code changes are separate Tix commits: `02e2cc44` retains native +handles and the original `c49bbec0` attempted zero-launch opening/discovery. +The rewritten opening commit restores real format queries and native metadata +fallbacks; the current coverage table and ceilings reflect those requirements. + +### Measurements after the Gix-only audit + +At `5ef5c62f2e2dafe3e7ddb0deed6998fd7092d360`, the adapters used Gix for +supported operations and retained CLI fallback for missing native capabilities. +The audit removed Python reference-format inference, metadata-path construction, +committer resolution, reflog cleanup, discovery repairs and absent-hook success. +Its findings were recorded in GIX-1/10/14/21/22/23; the capability follow-up below +refines those requirements. Retained native repository state and conversions +that actually use Gix APIs remain in place. +The full-suite and timed benchmark figures in this section describe that +snapshot, before the subsequent metadata and discovery changes. + +The full Gix suite without coverage passed **1,710 tests and 38 subtests**, +with 79 skips, one expected failure and one existing path-deprecation warning. +It took **342.36 pytest seconds / 342.80 seconds including the runner +(5m 42.80s)** on existing CPython 3.12.14/macOS arm64, official GixPython 0.1.0 +and Git 2.54.0. This includes CLI-inventory and Trace2 overhead. + +The corrected run recorded **22,694 `Git.execute` launches**. The earlier +12,394-launch result included the removed Python shortcuts and is superseded +as a coverage claim. The corrected run also contains 19 additional regression +cases; these are observed totals for their respective revisions. + +| Launch source | Before audit (`d2e788a1`) | Corrected (`5ef5c62f`) | +| --- | ---: | ---: | +| Managed operations with Gix enabled | 8,836 | 16,871 | +| Managed operations with Gix explicitly disabled | 309 | 339 | +| Minimum-version probes | 2,483 | 4,698 | +| Raw `Git.execute` calls | 766 | 786 | +| **Total pytest launches** | **12,394** | **22,694** | + +The corrected total splits into 4,464 setup and 18,230 call-phase launches, +with zero in teardown or collection/session. Call phases include `unittest` +fixture work. The Python inventory reconciles exactly with the pytest counter; +the process-lifetime backend count is 22,695 including its import probe. +Trace2 additionally identifies 3,685 Git child processes and 37 processes +outside the captured wrapper. Its total overlaps the Python inventory, so the +two totals must not be added together. + +The benchmark used the same pinned GitPython 3.1.45 repository and interpreter +for both backends, sequentially after tests, Gix first and CLI second. Each +used three worker processes, three values per worker, four loops per value +and one warmup. All **14 result digests matched**, and every corrected CLI +ceiling passed. The eleven operations form one journey on an already-open +`Repo`; opening and discovery remain separate lifecycle measurements. + +| Measurement | CLI mean ± stdev (ms) | GixPython mean ± stdev (ms) | CLI / Gix | Native / fallback | CLI launches: CLI / Gix | +| --- | ---: | ---: | ---: | ---: | ---: | +| journey | 314.35 ± 2.33 | 264.94 ± 3.30 | 1.19× | 79 / 2 | 43 / 1 | +| inventory | 19.50 ± 0.31 | 0.99 ± 0.08 | 19.68× | 3 / 0 | 3 / 0 | +| history_25 | 7.59 ± 0.12 | 73.32 ± 2.17 | 0.10× | 26 / 0 | 1 / 0 | +| browse_tree_and_blob | 13.45 ± 0.21 | 1.78 ± 0.22 | 7.55× | 5 / 0 | 2 / 0 | +| read_index | 7.00 ± 0.11 | 2.09 ± 0.27 | 3.35× | 1 / 0 | 1 / 0 | +| submodule_inventory | 72.10 ± 0.71 | 3.21 ± 0.20 | 22.44× | 13 / 0 | 11 / 0 | +| revision_paths | 55.27 ± 0.42 | 1.41 ± 0.12 | 39.12× | 6 / 0 | 8 / 0 | +| object_readback | 6.60 ± 0.16 | 1.22 ± 0.10 | 5.43× | 5 / 0 | 1 / 0 | +| revision_graph | 62.26 ± 1.15 | 33.25 ± 0.36 | 1.87× | 9 / 0 | 6 / 0 | +| diff_and_stats | 21.43 ± 1.28 | 67.83 ± 7.58 | 0.32× | 5 / 0 | 3 / 0 | +| patch_diff | 13.96 ± 0.12 | 8.97 ± 0.23 | 1.56× | 3 / 2 | 2 / 1 | +| worktree_status | 36.59 ± 1.01 | 70.55 ± 4.64 | 0.52× | 3 / 0 | 5 / 0 | +| open_repository | 58.47 ± 1.12 | 7.71 ± 0.25 | 7.58× | 1 / 1 | 9 / 1 | +| discover_repository | 70.72 ± 0.68 | 20.49 ± 0.38 | 3.45× | 1 / 5 | 11 / 3 | + +The expanded warm journey is **1.19× faster** with Gix in this sample, with +43 CLI launches reduced to the single patch-diff launch. Opening is **7.58× +faster** and retains one reference-format query; discovery is **3.45× faster** +and adds two CLI queries for rejected native candidates. Its five fallback +decisions include both discovery and command-adapter decisions, while only +three processes are launched. Cold contexts can also need a version probe. +The previous zero-launch opening/discovery results included Python substitutes +and are historical only. + +History, diff/statistics and worktree-status measurements remain slower with +Gix despite making no CLI launches. They need separate profiling. Several +rows have `pyperf` sample-size or variability warnings; these measurements +describe this machine and workload rather than a general speed guarantee. + +Every affected selection passed with Gix before CLI. The final full suite above +used Gix; CLI validation used the affected selections. Repository-wide Ruff +lint/format, mypy and basedpyright passed. The benchmark comparison independently +checked result parity, fixture/source identity and all process ceilings. +Generic configuration and standalone reference-name gaps remain documented +under GIX-12/20. `Repo._get_gix_repository(recreate=True)` still explicitly +recreates retained native state; its snapshot limitations are documented below. + +Raw benchmarks, comparison tables and logs from that snapshot are under +`.cache/gix-only-audit/benchmark-*` and `.cache/gix-only-audit/pyperf-comparison.md`. +The joined inventory, per-call CSV/JSONL, grouped commands, passing full-suite +log and runner timing are in `.cache/gix-only-audit/full-inventory-5ef5c62/`. +The older `.cache/gix-conversions/benchmark-{gix,cli}.json` and +`final-inventory-d2e788a/` remain historical evidence. CI continues to publish +fresh measurements and enforce the checked-in operation-specific ceilings. + +### Native capability follow-up + +The follow-up used the same official GixPython 0.1.0 and existing CPython +3.12.14, with runtime and test code at +`ee947c98540396d5644f3a7fea7c807633f4b5b8`. Each implementation change was +amended into its corresponding Tix commit and checked with Gix before CLI. + +- Reference storage can already be identified through native config. Direct + SHA-1 and SHA-256 regressions confirm that reftable `HEAD` access raises + `gix.Error` with an unsupported-storage cause; the binding does not expose a + separate `Unsupported` exception. Strict native opening and `HEAD` access + still accept unknown repository extensions that Git rejects, so the CLI + format query remains necessary for validation (GIX-1). +- Native `git_dir()` supplies the locations for `modules` and `COMMIT_EDITMSG`, + including linked worktrees. The adapter now adds only those fixed leaf names, + after reopening noncanonical Git paths through Gix. Each ordinary lookup + drops from one CLI launch to zero. Other metadata names and symlinked leaves + retain Git's path resolution (GIX-22). +- Gix already exposes the worktree of a bare main repository. Combining its + configured `is_bare()` flag with `workdir()` fixes opening and inventory + in the adapter; the original classification difference remains tracked as + an upstream compatibility bug. Gitfiles are reopened at their + canonical native Git directory with the same strict options; supported + symlinked-target helpers now need zero CLI launches instead of one. This + also prevents an arbitrary gitfile location from becoming the worktree. + Dangling `commondir` handling and typed failure information remain gaps; + rejected candidates keep Git diagnostics (GIX-14). +- Native snapshot overrides resolve committer name, email and date. A cheap + independent repository clone is needed to isolate per-command overrides: + Python copying is unsupported and the internal binding clone shares state. + Shared snapshots and process environment remain untouched (GIX-21). +- `LogChange` already configures each ref edit's message and log policy. A + direct native regression preserves whitespace where Git normalizes it, so + message cleanup still needs a native helper or policy (GIX-10). + +Final focused validation passed **120 Gix tests and 14 subtests** (one skip) +in 22.93 seconds, followed by **34 CLI tests and 14 subtests** (two skips, +including the native-only module) in 6.14 seconds. This covered the complete +native-backend, launch-counter and benchmark-comparison modules plus the +affected shared discovery, gitfile, worktree and repository-construction tests. +Repository-wide Ruff lint/format, mypy and basedpyright passed. + +An untimed benchmark preflight, Gix first and CLI second, matched all **14 +result digests** and passed the existing launch ceilings. The warm journey +remains CLI/Gix **43/1** launches, opening **9/1**, and nested discovery +**11/3**; the new fixed-path and gitfile-helper reductions are asserted by +their focused tests. No full-suite rerun or fresh timed benchmark is claimed +for this follow-up. Logs, backend reports and the count comparison are under +`.cache/gix-capability-followup/` (`15-final-*` and `16-benchmark-*`). + +### Windows CI process overhead + +The Python 3.12 jobs in [PR #2274's Python package run](https://github.com/gitpython-developers/GitPython/actions/runs/37628358853) +spent 57m 25s in pytest on Windows and 5m 2s on Ubuntu. Their process reports +counted 85,777 and 82,683 `Git.execute` launches respectively. The Windows +submodule test modules accounted for approximately 36 minutes, based on the +timestamps of their test results; the delay was spread across many operations. + +A local Windows profile of `test_file_handle_leaks` and +`test_update_no_fetch_is_recursive[root-no-fetch]`, with CI's coverage and +pytest options, counted 2,447 launches. About 65 of 77 seconds were spent in +Git command execution, including 44 seconds in repository construction; +forced garbage collection accounted for about three seconds. Combining +`--show-ref-format`, `--show-object-format`, `--is-bare-repository` and +`--git-common-dir` in one `rev-parse` invocation reduced the same selection to +2,009 launches: three saved for each of 146 repository opens. Git still +computes all four values. The common-directory path is last and split only +after the three scalar fields, preserving paths containing newlines. + +Three paired measurements of 20 ordinary repository opens on Windows gave +median times of 300.74 ms before and 221.18 ms after, with matching metadata +and 11 versus eight launches per open. These use CPython 3.12.13 and Git +2.55.0.windows.3; timing is observational, while the launch reduction is +checked by the regression tests. + +A 50-call `git version` probe on the same host measured median times of +17.49 ms per call on Windows and 1.27 ms in Ubuntu WSL, which uses CPython +3.14.4 and Git 2.53.0. Even trivial Git commands carry appreciable startup +cost on Windows. + +This optimizes the CLI backend's construction path. Supported Gix discovery +still uses its native metadata and the separate Git format-validation query; +the Gix launch ceilings and open compatibility bugs remain unchanged. +CI now includes `--durations=30` so subsequent slow tests are visible directly. +Local Windows reproduction must retain CI's `core.autocrlf=true` setting and +place pytest's temporary directories outside a Git checkout. Isolating all +Git configuration without restoring that setting changes the newline test; +placing `--basetemp` under this checkout lets empty-directory Git probes +discover the parent repository instead. + +The full local baseline at `105114db` recorded 86,032 launches in 3,990.28s; +the two submodule modules accounted for 2,421.59s. It had 1,652 passing tests +and the two harness failures described above, both of which passed unchanged +after correcting the setup. The patched run exited successfully with 1,662 +passed, 60 skipped, nine expected failures, two unexpected passes and 40 +passing subtests. It recorded 77,732 launches in 3,715.16s. These runs +overlapped, so their wall times are not a controlled comparison. The paired +repository-open benchmark above measures the affected operation separately. +Logs, profiles, JUnit results and backend reports are retained locally under +`.cache/ci-performance/`. + +### Submodule fixture process overhead on Windows + +Windows CI and the local runner now resolve `git.exe` from `PATH` to an +absolute `GIT_PYTHON_GIT_EXECUTABLE` when no override was supplied. This enables +the existing minimum-version cache without changing the library's executable +lookup or cache invalidation. Submodule fixtures also pass `skip_hooks=True` +when committing their test history; the library default and dedicated hook +tests continue to run hooks. + +A serial Windows comparison on CPython 3.12.13, Git 2.55.0.windows.5 and +GixPython 0.1.0 selected the Windows destination-name rejection tests, +`test_update_no_fetch_restores_deinitialized_submodule`, and +`test_update_no_fetch_is_recursive`. Both runs used the direct Git executable +on `PATH`, CI's flat alias config and `core.autocrlf=true`, with profiling +enabled and coverage disabled. All 37 cases passed in both runs. + +| Test invocation and fixtures | Pytest time | Git launches | Version probes | Hook launches | +| --- | ---: | ---: | ---: | ---: | +| `GIT_PYTHON_GIT_EXECUTABLE=git`, fixture hooks enabled | 87.10s | 1,437 | 403 | 177 | +| Absolute executable, fixture hooks skipped | 77.20s | 1,128 | 271 | 0 | + +The 309 eliminated launches comprise 132 redundant version probes and three +hook checks for each of 59 fixture commits. This sample was about 11% faster; +it is not a full-suite speedup estimate. Git Bash CI already selects +`mingw64/bin/git.exe` and appends the alias config directly, so correcting the +local runner's launcher/config differences is not an additional CI gain. +Logs, profiles, JUnit results and operation reports are retained locally under +`.cache/test-cleanup/windows-submodule-{baseline,optimized}*`. + +### Test-suite setup measurements + +The fixture optimizations below are implemented as separate commits, each +validated with GixPython before CLI. +The original local measurements use official GixPython 0.1.0 and existing +CPython 3.12.14 on macOS arm64 at `89c609cf92335e75652d7725682e215a9ea080e5`. + +The full suite with coverage took 757.85 seconds wall-clock (757.04 seconds +reported by pytest): 1,649 tests and 38 subtests passed, 79 skipped, and one +expected failure. Operation reporting counted 114,800 native operations and +39,002 CLI fallback decisions. Those decisions are not a complete subprocess +count: raw `repo.git` calls and subprocesses started by Git are not all counted. + +### Retain native repository state (implemented) + +Each `git.Repo` owns a native `gix.Repository`. Managed operations access it through +`Repo._get_gix_repository()`, which reuses the instance by default. +`Repo._get_gix_repository(recreate=True)` replaces it explicitly; this method is +the control point for a future configurable reuse policy. Managed operations reuse it; +`close()` releases it, and a later operation can reopen it. Pickling excludes +native resources and restores the command's weak owner reference. Gix provides +thread-safe handle access and automatic index/ODB snapshot refresh. A per-Repo +lock serializes handle refresh, without serializing read operations. + +Retaining native state is an intentional deviation from Git's fresh process per +command. Configuration/environment and observed CLI changes trigger best-effort +refresh, but this is not a promise that every external filesystem change is +immediately visible in every native cache. Explicit recreation or closing and +reopening the `Repo` provides fresh native state. Configuration queries currently +use a separate fresh handle because the retained execution handle overrides +hooks, fsmonitor and automatic maintenance; those overrides must not leak into +configuration results. The accessor itself removes no CLI calls: ten supported +metadata queries launch zero processes before and after this refactor. + +Configuration and storage metadata changes, environment changes and raw CLI +launches cause `reload()` before reuse. Configuration queries use a separate +fresh handle so synthetic safety settings never appear as user settings. +Configurations with includes conservatively reload on each operation because +the binding does not expose all included source paths. Storage overrides, +reftable and compatibility object formats retain their existing CLI guards. + +An earlier 1,000-call benchmark without coverage measured an object read using +one native handle at 0.046 ms per call, compared with 0.635 ms through the +fresh-open stream adapter. Opening and validating a native repository alone +took 0.418 ms per call. These small repeated-read measurements do not estimate +whole-suite speedup. + +### Reuse prepared test fixtures (implemented) + +A 19-case submodule rejection sample without coverage took 15.62 seconds: +1,258 `Git.execute()` calls accounted for 11.86 seconds, while 1,995 native +repository-open/validation attempts accounted for 0.84 seconds. Most work was +fixture setup. This sample supports prioritizing repeated setup subprocesses; +it is not a profile of the entire suite. + +The original collection found 1,729 parameterized cases from 703 distinct source bodies. +Source inspection identified the following conservative set of 422 cases whose +assertions do not require changing repository state after preparation. The +remaining cases have not been exhaustively classified; these are reuse +candidates, not proof that a shared fixture is safe in every execution order. + +| Original cases | Repeated setup | Implemented change | +| --- | --- | --- | +| 166 | `TExc` (157) and `TestActor` (9) inherit repository-building `TestBase`. | Use the existing `TestCase` base without repository setup. | +| 182 | Three submodule rejection bodies repeatedly build `movable_submodule`, then check snapshots for no mutation. | Prepare logical-name baselines once and copy the parent per case, retaining fresh wrappers and independent writable files. | +| 51 | Six submodule rejection bodies prepare nested metadata, separate metadata, intermediate/leaf symlinks, or retained metadata before checking rejection. | Cache ten prepared layouts and restore complete copies at their original paths, preserving absolute Git links and symlinks. Cleanup is best-effort; a locked active copy invalidates the layout so the next case rebuilds at a fresh path. | +| 15 | Eight revision-query bodies rebuild the same four-commit graph, refs, index, and reflogs through `rev_parse_repo`. | Prepare the graph once and copy it for every consumer, including mutating cases; recreate repository, branch and commit wrappers. | +| 8 | Tree lookup bodies clone and check out `0.3.2.1` through `with_rw_repo`. | Read the historical tree directly through the existing class repository, removing clones and checkouts. | + +The `movable_submodule` baseline also serves mutating cases: all writable +refs, index, config, objects, worktree and module metadata are filesystem copies, +while the local clone source remains shared and immutable. The original 322 +consumers no longer repeat repository initialization and submodule cloning. +The original 68 `local_submodule` cases copy both source and parent from one +prepared two-commit layout because these tests also mutate the source. The +fixture relocates all source URLs and records the private URL in parent history +for `RootModule` comparisons. No mutable Git metadata is hard-linked. + +Historical dependency sources are now lazy session fixtures. Originally all +25 `TestBase` classes reconstructed both sources (50 builds), although only four +classes called the URL helpers. The suite now prepares each needed source once, +with consuming tests retaining independent writable clones. + +Four additional checks exercise isolation: edits and refs in movable copies, +restoration after deliberate mutation with an absolute symlink, private source +commits, and revision-graph changes. Existing security no-side-effect snapshots +remain in place. Native repository lifetime and refresh limitations are +documented above. + +Per-change affected tests on existing CPython 3.12.14/macOS arm64 with official +GixPython 0.1.0, without coverage (wall-clock seconds including runner setup): + +| Change | Passed cases | GixPython first | CLI second | +| --- | --- | --- | --- | +| Repository-free actor/exception tests | 166 | 0.47 s | 0.49 s | +| Lazy historical sources, all consumers | 206, plus 14 subtests; 6 skipped, 1 xfailed | 90.89 s | 160.82 s | +| Movable baseline, top-level submodule tests | 333 | 88.63 s | 178.75 s | +| Prepared rejection variants | 51 | 12.34 s | 26.23 s | +| Layout restoration check | 1 | 1.33 s | 1.62 s | +| Prepared revision graph | 23 | 4.39 s | 7.57 s | +| Historical tree lookups, whole tree module | 22 | 1.66 s | 3.86 s | +| Prepared no-fetch source and parent | 69 | 56.27 s | 117.52 s | + +These selections overlap, and their timings are validation records rather than +isolated before/after benchmarks. The full-suite measurements below provide the +broader comparison. Only the existing interpreter was used. + +At `c8ee26c7da373e28cc7ede17ef2eaadd763fcdfc`, the full GixPython suite +with coverage passed in **404.61 seconds wall-clock (6m45s)**, with pytest +reporting 404.12 seconds: 1,653 passed, 79 skipped, one expected failure, +and 38 subtests passed. Coverage remains 90%. Compared with the original +757.85-second run, this saved 353.24 seconds (46.6%, about 1.87 times faster). +This is one local before/after run per revision, not a statistical benchmark. + +The same operation counters now record 102,581 native operations and 21,402 +CLI fallback decisions, down from 114,800 and 39,002 respectively. Raw Git +calls and child processes remain outside those counters. + +The subsequent full CLI run with coverage passed in **755.78 seconds +wall-clock (12m36s)**, with pytest reporting 755.30 seconds: 1,617 passed, +80 skipped, one expected failure, and 38 subtests passed. CLI coverage is 82%; +backend-specific tests account for different collection and coverage. This CLI +run validates the optimized suite; there is no matching pre-change CLI full-run +measurement here. Repository-wide Ruff lint/format, mypy and pyright passed. + +## GixPython / Gitoxide follow-up ledger + +These are local findings, not filed issues. **Every observed divergence from +the equivalent Git operation is a compatibility bug**, including intentional +differences. Gix must offer matching behavior, at least through an API or mode +as strict as Git. This covers acceptance/rejection, validation, returned data, +ordering, paths and mutation side effects. `strict_config(True)` and +`bail_if_untrusted(True)` alone do not provide that guarantee (GIX-1/14). + +The status distinguishes reproduced bugs from missing APIs and unverified +coverage. Low-level primitives can remain available, but a compatible API or +mode is still required. An adapter workaround, including reopening through +Gix, does not close an upstream compatibility bug. Retain its expected/actual +behavior, version and reproduction until the upstream fix or compatible mode +has been verified. Python must not implement the missing Git behavior. + +Unless stated otherwise, the evidence uses official GixPython 0.1.0, its pinned +Gitoxide revision `f819565c2c4c56619c4888acef6cf3b8144cbccb`, and Git 2.54.0 on +macOS arm64 with existing CPython 3.12.14. The repository and commands below +identify the historical fixture where needed. + +| ID | Status | Finding and evidence | Required Gix behavior/API | +| --- | --- | --- | --- | +| GIX-1 | Bug; missing API | Native config exposes `extensions.refStorage`; reading `HEAD` in SHA-1/SHA-256 reftable repositories raises `gix.Error` with an unsupported-storage cause. However, strict native opening plus a successful `HEAD` read accepts unknown version-1 extensions that Git rejects. It also accepts `extensions.refStorage=files` with repository version 0, which Git rejects. | Git-compatible repository-format/extension validation, typed native errors, and eventual reftable support. A dedicated format getter is not needed just to identify storage. Keep the CLI format query because it also validates the repository, with regressions for both the unsupported-HEAD signal and unknown extensions. | +| GIX-2 | Missing API | Object lookup buffers full contents. `write_blob_stream` also reads its input into memory. | Bounded-memory object readers/writers usable by Python. The adapter currently caps native blob transfers at 8 MiB. | +| GIX-3 | Missing API | Index bindings open the repository's index but cannot load an arbitrary existing index file. `set_path()` only chooses its write destination. | An index loader accepting a path, including relative-path semantics. | +| GIX-4 | Missing API | Sparse index expansion is not exposed. | Expansion preserving skip-worktree and other metadata before editing. | +| GIX-5 | Unverified | Structured-object writers may decode/re-encode supplied bytes. This is a compatibility guard, not a demonstrated corruption bug. | A raw-object writer, or a guaranteed byte-preserving contract. The current adapter checks the round trip in an in-memory ODB first. | +| GIX-6 | Missing API | Native commit construction accepts a string message. GitPython also supports arbitrary message bytes and other declared encodings. | Byte-message commit construction with explicit encoding/signature semantics. | +| GIX-7 | Missing API | Native tree editing requires existing non-gitlink child objects. Git's `mktree --missing` / `write-tree --missing-ok` permit missing children. | Missing-object tree construction with equivalent validation. | +| GIX-8 | Bug | General revision iteration differs from Git's default ordering. The same fixture enumerated 5,439 commits in both engines, with the first ordering difference at position 178. Its 1,923-commit first-parent chain matched. | A Git-compatible walk order; counts and first-parent walks already work. See the reproduction notes below. | +| GIX-9 | Bug | `PreviousValue.MustNotExist` accepts an existing ref already at the requested target. Native tests explicitly permit this; Git's strict create/CAS rejects it. | A strict native expectation, enforced under the same ref lock. A Python existence precheck would race. | +| GIX-10 | Bug; missing API | `LogChange` preserves ` keep\t spaces ` where Git stores `keep spaces`. Updating the checked-out branch directly or through an alias omits the HEAD log entry Git appends. Updating that branch to its unchanged target also omits Git's HEAD log entry. Detaching symbolic HEAD records a null old OID instead of Git's resolved previous commit. `RefLog.Only` cannot express an independent arbitrary old OID, and reference-bound readers cannot access orphan logs. | Fix message normalization, secondary HEAD logging and old-OID selection in Gix, at least in a Git-strict transaction mode; expose raw reflog append and orphan-log access. `LogChange` already configures each edit's message, mode and force-create flag; `core.logAllRefUpdates` controls creation, not cleanup. Keep the existing `_update_ref` and `_reflog` fallbacks before mutation; do not normalize messages in Python. | +| GIX-11 | Unverified | Inexact and ambiguous rename pairing/scores have not been established as Git-compatible. This is a conservative guard, not a claimed native bug. | Verified pairing, scoring, tie-breaking and option parity. Exact unambiguous renames are native. | +| GIX-12 | Missing API | Generic config bindings lack standalone parsing, ordered section/key enumeration, multivars, unset/remove-section and source-scoped queries. Dedicated `.gitmodules` parsing is available and now used for supported submodule reads. | Those operations for `GitConfigParser` and remote/branch configuration. Native merged getters cannot replace repository-only config readers. | +| GIX-13 | Bug; missing API | Native index writing normalizes v4 to v2, offers no version setter, and expands split indexes. | Version and split-index preservation. Tests now assert that a split index remains split after an update. | +| GIX-14 | Bug; missing API | For a linked worktree of a bare main repository, `is_bare()` returns true while `git rev-parse --is-bare-repository` returns false; `workdir()` does expose the worktree. Initial gitfile opening can retain a noncanonical target and use an arbitrary gitfile's location as the worktree, unlike Git. Discovery accepts undecodable HEADs until explicit `head()` decoding and ignores dangling `commondir` symlinks that Git rejects. Bindings flatten failure kinds into `gix.Error`. Windows also exposes adapter path-formatting differences and a worker panic on an undecodable `commondir`; see the Windows reproduction below. | Provide Git-compatible classification, canonical gitfile locations and layout validation, plus typed errors instead of panics. Format native locations compatibly with Git's command output. The adapter combines Gix metadata, reopens through Gix and forces HEAD decoding, retaining CLI for remaining validation/diagnostic gaps. Those workarounds do not close these compatibility bugs. A Git-strict mode must cover these cases; strict config and trust settings alone do not. | +| GIX-15 | Bug | Native blame disagrees with Git even with Myers and rewrite tracking selected. In the fixture's `README.md`, lines 150 and 158 are attributed to the opposite commits. Incremental order also differs. | Attribution and incremental-output parity before replacing `Repo.blame` / `blame_incremental`. | +| GIX-16 | Bug; missing API | Native archive streaming takes a tree rather than a commit. Its TAR omits Git's global PAX commit comment, leaves `export-subst` placeholders literal, and writes ordinary modes as `0644` where Git uses `0664`. | Commit-aware export substitution, metadata and permission parity. Both engines respected `export-ignore` in the probe. | +| GIX-17 | Bug | Revision parsing accepts abbreviated IDs with `-dirty`, prefers the OID suffix over an exact describe-shaped tag, and treats escaped braces in message searches differently. | Git-compatible revision grammar and regex semantics. Existing `test_rev_parse.py` cases reproduce all three. | +| GIX-18 | Bug; missing API | The tree-diff resource cache reads attributes from the index; Git also consults worktree attributes. Native binary classification with textconv configured can also differ even in `to_git` mode. | Independent attribute-source control and `--no-textconv` parity. Statistics currently fall back for any effective `diff` attribute. | +| GIX-19 | Unverified | GixPython opens repositories with `extensions.compatObjectFormat`, but translation and object-write compatibility have not been validated. The installed Git reports `compatibility hash algorithm support requires Rust` when asked to translate an ID. This is an unvalidated capability guard, not evidence of missing native object mappings. | Verify compatibility-format reads, writes and OID translation against a Git build that supports them. All operations use CLI in the meantime. | +| GIX-20 | Bug; missing API | `Target.Symbolic(name)` validates full names but rejects standalone names accepted by `git check-ref-format --allow-onelevel`. Four of 430 audited calls differed: `refs` twice, `hellothere`, and `valid_one_level_refname`. Further probes include digits, punctuation and Unicode (`1`, `A1`, `HEAD_1`, `A-B`, `A.B`, `Ä`). | Expose Git-compatible partial/one-level name validation, or correct the native restriction where appropriate. This is a binding/behavior shortcoming to revisit upstream; standalone names rejected natively retain CLI validation. | +| GIX-21 | Bug; missing API | `Repository.committer()` preserves a process identity of ` Name. ` / ` email@example.invalid ` while `git var GIT_COMMITTER_IDENT` returns `Name.` / `email@example.invalid`. Native snapshot overrides for name, email and date work, but modifying the retained snapshot would leak per-command identity across threads. GixPython 0.1.0 has no general independent repository clone: Python copying fails and internal `RepoHandle::clone()` shares its `Arc` state. | Expose Git-compatible identity cleanup and a cheap independent native clone with isolated config, including per-command removal semantics. Reopening isolates state but repeats setup; `with_object_memory()` changes object-write behavior and is not a general clone. Keep `_checked_signature` validation and CLI fallback for cleanup/overrides. Never clean identity fields in Python or mutate shared snapshots/process environment to handle a command. | +| GIX-22 | Missing API | The native Git directory is sufficient for the two fixed metadata leaves used here: `modules` and `COMMIT_EDITMSG`. Git's `path.c` applies no special relocation to either, including in linked worktrees. The adapter appends only these names to the canonical native Git directory. | No new binding is needed for these ordinary paths. Other `--git-path` names and symlinked metadata leaves retain CLI resolution; a general resolver must handle config/environment overrides, common/private storage and canonical targets. `Repository.modules_path()` means `.gitmodules`, not the `modules` storage directory. | +| GIX-23 | Missing API | No hook lookup or execution API is bound in GixPython 0.1.0. The removed absence fast path read config through Gix but used Python `os.stat()` to declare success. | Bind native hook lookup with configured/default path, linked-worktree, missing/nonexecutable-hook and diagnostic semantics, plus execution where needed. Until then all managed hook calls use Git, including `--ignore-missing` no-ops. A Python filesystem check is not a Gix implementation. | +| GIX-24 | Bug | In both object formats, `new_commit_as()` accepts a blob as the tree or a parent where `git commit-tree` rejects it. Tree-editor `upsert()`/`write()` accepts object IDs whose actual kinds disagree with blob/tree/gitlink modes; `git mktree --missing` rejects all three tested mismatches. `edit_references_as()` accepts a blob target under `refs/heads/`, rejected by `git update-ref`. | Provide checked commit/tree construction and reference edits, at least in Git-strict mode. Match Git's kind and direct-branch target validation before writing and under the required ref locks, including its distinct handling of symbolic aliases. Existing `_commit_tree`, `_write_tree` and `_update_ref` guards query Gix headers and select CLI on invalid inputs; they do not replace Gix writes with Python. | +| GIX-25 | Bug; missing API | After `git symbolic-ref refs/heads/dangling refs/heads/missing`, `Repository.references().all()` enumerates the dangling name, while `git for-each-ref --format=%(refname)` omits it. Valid symbolic aliases must remain present. Both SHA-1 and SHA-256 probes reproduce this difference. | Expose Git-compatible enumeration with the same dangling-reference and diagnostic behavior, retaining the raw iterator for callers that need it. `_for_each_ref` forces Gix to resolve symbolic targets and falls back to Git on failure; keep that fallback until a compatible Gix API/mode is verified. | +| GIX-26 | Bug; missing API | `dangerously_push_entry(IndexStat(), oid, 0, 0o120000, b".gitmodules")`, followed by `sort_entries()`, `verify_entries()` and `write()`, succeeds and retains the unsafe symlink. Protected Git `update-index -z --index-info` ignores it with exit 0 and `Ignoring path .gitmodules`; protected `read-tree` rejects it. The native fresh `index_from_tree` also rejects it. | A Git-strict checked index batch editor that validates names/modes, preserves metadata and reports rejected records. `verify_entries()` verifies ordering, not pathname safety. All private-index edits use Git with `core.protectHFS=true` and `core.protectNTFS=true`, followed by requested-entry readback; no Python dictionary reconstruction or native dangerous insertion remains. | +| GIX-27 | Missing API | `gix.Time` exposes seconds/offset but no human date parser in GixPython 0.1.0. | Bind Git-compatible textual date parsing with timezone semantics. `parse_date()` uses `git var GIT_AUTHOR_IDENT` with fixed identity and per-command date, then adapts the normalized timestamp/offset. | +| GIX-28 | Bug; missing API | GixPython 0.1.0 on Gitoxide `f819565c2c4c56619c4888acef6cf3b8144cbccb` stores `object \ntype blob\ntag invalid\n\nmessage\n` through `write_object("tag", data)`. Git 2.54.0 rejects the same payload through `hash-object -t tag -w --stdin` with `missingTaggerEntry`. Byte-preserving native readback does not establish Git's validation. | Expose a Git-strict tag writer or validation API. Managed tag hashing/storage uses Git before any native write; `test_tag_storage_rejects_missing_tagger` verifies failure and absence from storage. Keep the bug open until upstream validation is verified. | +| GIX-29 | Missing API | GixPython 0.2.0 exposes raw identity/message/signature bytes without a Git-compatible text reencoder. Git 2.54.0 recodes the complete commit or retains all bytes when conversion fails; per-field Python conversion differs. The Latin-1, unknown/non-text codecs, malformed UTF-16 and signed-header cases in `test_deserialization_declared_encoding` reproduce the required behavior. | Expose whole-commit text conversion, including failed/unknown-encoding behavior. `object_metadata()` retains CLI for non-UTF-8 declarations. The fallback obtains identities/message from Git's fixed UTF-8 text projection and signatures from its recoded raw header block, retaining the original declaration separately. | +| GIX-30 | Bug (open) | GixPython 0.2.0 `Commit.decode()` reports `encoding=None` when `encoding ISO-8859-1` follows `x-extra value`; Git 2.54.0 selects `ISO-8859-1` and renders Latin-1 identity/message bytes correctly. Gix retains `(b"encoding", b"ISO-8859-1")` in `extra_headers`, but its encoding field disagrees with Git. The `late-latin1` read-only `commit.sh` fixture in `test_deserialization_declared_encoding` reproduces this and asserts the fallback. | Recognize encoding headers in Git-compatible order. `object_metadata()` conservatively retains CLI when an encoding appears in `extra_headers`; it does not select the header in Python. Keep this bug open until an upstream fix or compatible mode is verified. | + +The October 22 Windows Gix CI run on Git 2.56.0.windows.2, CPython 3.12.10, +and GixPython 0.2.0 exposed an adapter failure in +`test_ignored_items_reported_verbatim`: Python's path metadata lookup raises +`WinError 123` for `q"uote.env`, while Git's `check-ignore --stdin -z` accepts +the queried path without creating it. The adapter now retains CLI when path +metadata is unavailable, including permission errors. This is an adapter +compatibility bug, not a demonstrated mismatch in Gix's excludes matching. +GixPython would need to expose Git-compatible ignore queries with native +path-kind and traversal handling to remove this metadata-dependent fallback. +`test_ignore_metadata_errors_retain_cli` reproduces it on every host; the +original verbatim-path regression continues to cover Windows directly. + +Git selects the first encoding header, recodes the complete commit, and retains +all original bytes if conversion fails. Codec aliases vary by platform: Git 2.54.0 +on macOS rejects `utf-16-be`, while Alpine's musl 1.2.6 iconv recognizes it. The +malformed UTF-16 fixtures use odd byte counts so conversion fails even with the +portable `UTF-16BE` spelling. A later `encoding ISO-8859-1` header cannot rescue +that failure. Tests assert the odd byte counts and compare text directly with Git. + +Earlier even-sized fixtures converted successfully but erased every ASCII newline. +Their Git text projection is unstable: Git 2.56.0.windows.2 returned `x` as the +message in one CI job instead of the empty message observed elsewhere. Git's +pretty formatter assumes a header delimiter after scanning the recoded buffer; +these malformed inputs provide none. They do not provide a reliable text oracle. +Successful conversion remains covered by ASCII-compatible encodings. The native +UTF-8/default path remains in use; other declarations use a fixed UTF-8 Git text +projection. + +`Commit.encoding` remains the first declared Python text codec when valid, or +`Commit.default_encoding` for unknown, non-text or malformed codec names. That +choice for subsequent serialization does not control Git's text conversion: +Git's iconv can recognize codecs Python cannot, such as `ISO-2022-CN`. The fixture +compares against the installed Git, whose iconv support can vary by platform. + + + +Windows reproduction for GIX-14 uses CPython 3.12.13, Git +2.55.0.windows.3 and the released GixPython 0.1.0 source distribution with +the same pinned Gitoxide revision above. Both `105114db` and the CLI metadata +batching change (`f6779dbb`) reproduce the following existing failures: + +- `test_native_command_queries_match_cli` and + `test_native_worktree_inventory_includes_bare_main` return backslashes in + the adapter's native path text where Git returns forward slashes, in both + object formats. The locations agree; the command-output formatting does not. +- The `commondir=b"\xff"` subtest of + `test_repo_discovery_rejects_invalid_metadata` raises + `RuntimeError: native worker panicked` instead of the CLI backend's + `InvalidGitRepositoryError`. The binding must reject this input through its + normal native error contract so discovery can take the existing fallback. + +The adapters now format locations returned by `Repository.common_dir()`, +`workdir()` and `git_dir()` with the existing Windows-only path separator +conversion, including `Repo.common_dir` and gitfile worktree paths. Gix still +resolves every location. Ordinary, bare and linked repository metadata and +worktree-list queries compare exactly against Git without additional CLI +calls. POSIX filenames containing literal backslashes are unaffected. + +Discovery catches only the binding's exact +`RuntimeError("native worker panicked")` and selects Git's existing validation +path. The malformed metadata regression then raises `InvalidGitRepositoryError`. +Unrelated runtime +errors still propagate. Both normal native write errors and worker panics +after a mutation begins are tested to ensure they never trigger a CLI retry. +GIX-14 remains open: these adapter changes do not fix the upstream panic or +the other discovery compatibility bugs. + +The tests now pass gitfile operands in Git's Windows-compatible spelling and +compare symlink resolution against the current platform's Git output. Status +and ignore parity runs with a portable filename everywhere; only the extra +newline-filename variant is skipped on Windows. Blob-reference updates also +compare against Git: a direct branch rejects the blob, while a symbolic alias +outside `refs/heads/` accepts it on Git 2.55.0.windows.3, in both object +formats. That alias behavior is not evidence of a Gix mismatch and does not +invalidate GIX-24's separate direct-branch comparison. + +For GIX-8, at fixture commit `44e0a8ec55c42559dfcdf5117710b26261a7c937`, +compare `git rev-list HEAD` with +`repo.rev_walk([repo.head_id()]).sorting("newest_first").all()`. +The first different IDs were Git's `5a53ae6d68e318a85be78fb5fcee4d3aa9dfbb48` +and gix's `ee854dcb62220adeae4feb59bd10185e7ac02957`. +Native walk builders return new values: retain the result of +`first_parent_only()` and other builder methods. + +For GIX-15, compare `git blame --line-porcelain HEAD -- README.md` with +`repo.blame_file("README.md", repo.head_id(), +gix.BlameOptions(diff_algorithm="myers", rewrites=gix.Rewrites()))` at that same +fixture commit. The differing commits are +`2ddd5e5ef89da7f1e3b3a7d081fbc7f5c46ac11c` and +`3aacb3717ad78ec40e5b168a7ce8109aee6f156e`. `VERSION` and +`git/objects/commit.py` matched in the comparison. + +For GIX-18, set `file diff=forced` in `.gitattributes`, +`diff.forced.binary=true`, and `diff.forced.textconv=false`. Change a NUL-containing +line in `file`: the native `to_git` cache reports one insertion and one removal, +while Git with `--no-textconv --numstat` reports `-\t-\tfile` (binary). + +For GIX-24, use a repository containing an empty tree, a commit and a blob. +Pass the blob ID as the tree or a parent to `new_commit_as()` and compare +`git commit-tree BLOB` or `git commit-tree TREE -p BLOB`. Insert a commit ID +under the `blob` or `tree` kind with `Tree.edit().upsert()`, or a blob ID under +`commit`, then `write()`; Git's corresponding `mktree --missing` entry rejects +each mismatched kind. Finally, submit a `RefEdit.update_with_log()` for +`refs/heads/bad` with `Target.Object(blob_id)` and `PreviousValue.Any`; +`git update-ref refs/heads/bad BLOB` rejects that branch target. + +The GIX-10 transaction probes start with HEAD pointing to `refs/heads/main` +at a first commit. Apply `RefEdit.update_with_log()` with `PreviousValue.Any`, +`LogChange.force_create_reflog=True` and a fixed message, comparing +`git update-ref --create-reflog -m MESSAGE`. Test advancing the branch, writing +its current target again, updating through a symbolic alias, and detaching HEAD +with `with_deref(False)` / `--no-deref`. Compare both the branch and HEAD logs, +including the previous OID, as recorded in the ledger. + +The latest direct comparisons covered 12 cases in each of SHA-1 and SHA-256: +six object/branch type mismatches, identity cleanup, dangling enumeration and +four reflog transactions. Each case called Gix first and Git second. The local +probe and captured results are in +`.cache/gix-compatibility-ledger/{probe-contracts.py,contracts.json}`. +Existing [backend regressions](../test/test_gix_backend.py) protect the adapter +fallbacks, including `test_tree_writes_validate_child_object_kinds`, +`test_branch_blob_updates_match_git`, +`test_commit_identity_cleanup_matches_git` and +`test_reference_enumeration_preserves_aliases_and_skips_dangling_refs`. +This ledger update changes no runtime code or CLI budgets; the earlier +runtime validation and timings remain attributed to their measured snapshots. + +### Remaining configuration reads (GIX-12) + +The October 7 inventory identified 951 remote-enumeration reads, 674 actor +lookups, 262 remote-property reads, 202 tracking-branch lookups and 111 fetch +refspec checks. These are historical call-site counts, not additive savings. +The dedicated `.gitmodules` parser now covers supported submodule reads; +the other consumers still need generic config binding additions. + +Local probes with official GixPython 0.1.0 and the existing CPython 3.12.14 +confirmed these contracts against both GitPython backends: + +| Consumer | GitPython behavior to preserve | Why the existing native API is insufficient | +| --- | --- | --- | +| Remote enumeration | Repository-local declaration order: `z-last`, `a-first` | `remote_names()` returned the merged, sorted `a-first`, `global-only`, `z-last`. | +| Remote URL/properties | The cached raw `url` property returned the last value, `../second`; `urls` yielded both `../first` and `../second`. | Native `Remote.url(Fetch)` selected only `../first`. Generic multivalue enumeration and raw snapshot semantics are still needed. | +| Actor lookup | A supplied file/stream reader returned `Reader Actor`, independent of the repository's `Repository Actor`. | Repository identity lookup cannot honor arbitrary source lists, reader snapshots, or GitPython's identity fallback rules. | +| Tracking branch | `branch.main.remote=z-last` and `merge=refs/heads/topic` produced `refs/remotes/z-last/topic`. | Native tracking lookup applied the configured fetch mapping and returned `refs/foreign/topic`. Replacing this API requires the original scalar config values. | +| Fetch refspec presence | The cached local reader tests whether a value exists before invoking fetch. | Native remote creation/refspec access applies URL/refspec parsing and merged configuration; it does not expose this raw, source-scoped presence check. | +| General config readers | Explicit files/streams, their order, included files, duplicate values and cached reader state | `ConfigFile` exposes only `boolean`, `integer`, `string`, `set_raw_value` and `to_bstring`. `OpenOptions.isolated()` also disables includes: a local included value read as `yes` through GitPython was absent natively. | + +The needed additions are standalone generic parsing, ordered section/key and +multivalue access, source selection, and mutation/removal primitives. The +submodule adapter's conservative section guards can be removed once those +bindings exist. No generic configuration parser is implemented in Python. +The fixed general-config probe remains **2 CLI launches before and after** +this investigation; this item does not claim a process reduction. + +The October 22 rebase regressions compare unquoted escapes and continued +values directly with Git 2.54.0, using CPython 3.12.14 and GixPython 0.2.0. +Git rejects unknown escapes such as `\q` and the unescaped backslashes in +`C:\Users`, including in continued values. It preserves whitespace before a +dangling EOF continuation when reading. However, `git config set` appends a +new section header into that continuation: writing `other.value=updated` +after `k = value \` changes `a.k` to `value [other]` and writes `a.value`. +The tests assert Git's actual reads and serialized bytes rather than the +former Python parser's repair of that input. General config files and +mutations still use the CLI on both backends under GIX-12; these observations +do not demonstrate a native Gix mismatch or eliminate any CLI calls. + +## Other remaining operations + +| Operation family | Why it still uses Git | +| --- | --- | +| `Repo.init`, clone | The different bundled initialization template set remains a Git compatibility bug; a compatible mode must reproduce Git's effective templates. Git's template/reinitialization options are not exposed. Clone also needs GitPython's environment, local-copy, progress and cleanup contracts. | +| Fetch, push, pull, remote pruning | Fetch outcomes do not expose the per-ref old IDs/status/notes needed for `FetchInfo`. Push/pull porcelain is absent. Network mutations cannot be retried after a partial native attempt. | +| Checkout, reset, index checkout/merge, move/remove | No bound equivalent of checkout-index/unpack-trees preserving these worktree/index semantics. Native tree/commit merge is a different operation. | +| Branch/tag porcelain and symbolic-ref mutation | Strict creation, reflogs, config movement, signing, message cleanup and safety contracts need more than a raw ref edit. | +| Object enumeration and alternate-directory listing | No binding for complete ODB object iteration or alternate-store enumeration; `cat-file --batch-all-objects` / `count-objects` remain. | +| Name-rev, trailer parsing, cherry | No equivalent bound operation. Native describe is not name-rev. | +| Config-backed remote/submodule maintenance | Missing config enumeration/multivar/removal operations (GIX-12), plus filesystem and worktree lifecycle requirements. | +| Hook execution | Every explicit hook request, including an absent hook, retains Git-managed lookup, execution and diagnostics until GIX-23 is addressed. Native operations retain GitPython's default hook/fsmonitor/maintenance restrictions. | + +## Maintaining the adapters + +`git/_backend.py` contains the capability checks. Managed command adapters return +the same bytes/text as the existing parser expects; direct adapters return the +existing GitPython object types. Keep operand validation before dispatch. + +Gix APIs must provide Git behavior. Python glue may validate inputs, format +native results and select fallback; it must not implement missing Git semantics +or fabricate successful answers to lower CLI counts. Document missing or +unclear capabilities here and retain the CLI path until Gix provides them. +Name the actual Gix calls behind each conversion. Record every observed Git/Gix +mismatch as a compatibility bug with expected/actual behavior, version and a +reproduction or regression. A Gix-based workaround does not resolve that bug; +require an upstream fix or a verified mode as strict as Git before closing it. + +Decide whether to fall back before mutating anything. Read/preparation failures +can use Git to preserve its public diagnostics. Once `_write()` begins, +`gix.Error` failures become `GitCommandError`; other exceptions propagate. +Neither may trigger a second CLI mutation. +Access native repository state through `Repo._get_gix_repository()` for bound +commands. Reuse is the default, with explicit recreation available; keep the +snapshot deviation and refresh behavior above documented. Close native iterators +when partially consumed. + +Add each conversion with a no-subprocess check and a Git parity check where +practical, in its own commit. Update this ledger when a limitation changes; +the runtime report's reason should point to the corresponding entry. + +## CI coverage + +The [Python package workflow](../.github/workflows/pythonpackage.yml) adds one +Python 3.12 Gix job each on Ubuntu, macOS and Windows, preserving all 28 CLI +combinations and their existing check names. It installs `.[test,gix]`, verifies +the selected backend and runs the full suite with the same coverage and pytest +options. Every job retains JUnit results +and `--backend-report` operation counts under its `tests-OS-PYTHON-BACKEND` +artifact, and prints the 30 slowest test durations. The extra Gix jobs do not +duplicate the documentation build. Their check names include `gix`, such as +`test (ubuntu, 3.12, gix)`. + +On platforms without a matching wheel, pip builds the official source release. +`CARGO_TARGET_DIR` keeps compilation output outside pip's temporary source +directory. The Cargo cache distinguishes OS, architecture, Python and Rust +versions and `gix-requirements.txt`; a release change can reuse dependencies +from the same platform and toolchain. + +## Windows validation + +Local Windows checks use CPython 3.12.13, Git 2.55.0.windows.3 and GixPython +0.1.0 built from the released source distribution. The installed extension +matches the cached Windows ABI3 wheel byte-for-byte; that wheel's SHA-256 is +`1a0addd5f569e2ff7fb2b45c38be1093f6cabbead176c779533b983b86307dc2`. + +The full baseline at `f6779dbb` reproduced all 17 failures described above: +16 ordinary failures and the subtest with malformed `commondir`. The full +patched suite passed with **1,760 tests and 40 subtests**, 58 skipped, nine +expected failures and three unexpected passes in 2,242.66 seconds. The three +unexpected passes and the path-deprecation warning also appeared in the +baseline. Ruff, mypy, basedpyright and the pinned pre-commit checks passed. + +The baseline recorded 21,598 CLI launches in 2,279.10 seconds; the patched +run recorded 21,826 CLI launches and 86,936 native operations. Totals include the additional +regressions and tests that previously stopped at failing assertions. No CLI +ceilings were lowered. Both runs used isolated snapshots and CI's coverage, +Git configuration and pytest options. They overlapped, so the elapsed times +are not a controlled speed comparison. Logs, JUnit results and operation +reports are retained locally in `.cache/gix-windows/`. + +The CLI repository/discovery regressions passed with 138 tests, 14 subtests, +nine skips and one expected failure. The changed native path, discovery, +reference and mutation-error cases also passed under Ubuntu WSL: 43 tests +and 14 subtests, using CPython 3.14.4, Git 2.53.0 and a separate released-source +GixPython build. This includes the POSIX newline-filename cases. + +## Published release validation + +The official Apple Silicon ABI3 wheel for GixPython 0.1.0 has SHA-256 +`6b68758cb54a90c8eb893ac7815e74ccd9b48b197aaecec951366a6326168f52`. +It was downloaded from PyPI and verified against the published digest. +At the backend-selection commit, its SHA-1/SHA-256 smoke checks and fresh +extra-installation check passed (3 tests) on the existing CPython 3.12.14/macOS +environment. The dependency commit and all 31 descendants passed the fast QA +profile: Ruff lint/format, pre-commit, mypy, basedpyright, universal dependency +resolution, and fatal lint checks for the bundled dependencies. The full +published-wheel suite subsequently passed with coverage: 1,649 tests and +38 subtests passed, 79 skipped, and one expected failure, in 757.85 seconds +wall-clock on the same interpreter/platform. The performance notes above +record the tested revision and future optimization candidates. + +## Historical local artifact validation + +The original integration was validated with GixPython 0.1.0 from local checkout +`c8a9fabc03c326ad7b0afdf03d0bda4e62fdff52`, with Gitoxide revision +`f819565c2c4c56619c4888acef6cf3b8144cbccb` and its existing vendored fixes. +The ordinary Apple Silicon ABI3 wheel in `pygix/dist` has SHA-256 +`62a03fe63e0e043c89f8271c10540cb00d31a15cf81fc1f45d3e2feb82a44e9f`. +The installed extension was compared byte-for-byte with that wheel. + +Validation uses CPython 3.12.14 on macOS. The focused backend tests cover both +SHA-1 and SHA-256. Platform-specific skipped tests are not claims of validation +on other operating systems or free-threaded Python. The full suite and operation +report can be reproduced with the commands above. + +| Check | Result | +| --- | --- | +| Full CLI installation | 1,360 passed, 80 skipped, 1 expected failure; 38 subtests passed. | +| Full GixPython installation | 1,396 passed, 79 skipped, 1 expected failure; 38 subtests passed. | +| Expanded native parity/fallback checks | 36 passed, including object-kind validation, identity cleanup, symbolic aliases/dangling refs, and compatibility-format guards. | +| Ruff lint/format, mypy, basedpyright | Passed. | +| Offline package build | Built an sdist and rebuilt its wheel; the archive includes `gix-requirements.txt` and the wheel declares the `gix` extra. | + +The wheel directory also supported a fresh offline `.[test,gix]` installation. +The full installation test checks that its new environment selects the same +backend as the environment running the tests. Tox itself was not installed on +this machine; the underlying interpreter-based command was used directly. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 95178088d..e5361b4cf 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,151 @@ Changelog ========= +Unreleased: Git CLI migration +============================= + +GitPython now delegates repository discovery, references, configuration, object +storage, tree construction, index operations, and revision parsing to Git. The +default backend supports SHA-1 and SHA-256 repositories with either files or +reftable reference storage. Repository object IDs must no longer be assumed to +contain 20 binary bytes or 40 hexadecimal characters. +Legacy class-level ``NULL_BIN_SHA`` and ``NULL_HEX_SHA`` constants retain their +SHA-1 values; do not use their width to interpret repository object IDs. + +Git executable and safety +------------------------- + +* Git **2.52 or newer** is required for library-managed operations. Older versions + raise ``UnsupportedOperation`` before repository mutation; there is no Python + implementation fallback. ``GIT_PYTHON_GIT_EXECUTABLE`` still selects Git. +* Every managed command uses argument sequences without a shell. Existing unsafe + option/protocol checks remain, and operands and stdin records receive additional + validation. NUL-delimited ``cat-file`` requests prevent newline injection. + Previously accepted option-like revision/ref arguments can now be rejected. +* New plumbing calls suppress implicit hooks, filesystem monitors, automatic + maintenance, and lazy network fetches. Index staging continues to store raw + content rather than introduce clean filters. Diffs disable external diff and + text conversion programs; blame disables text conversion unless unsafe options + are explicitly enabled. Explicit commit hooks use ``git hook run`` and honor + ``skip_hooks``; Git combines their output, so ``HookExecutionError`` may carry + former stdout text in stderr. Trailer additions reject executable trailer + configuration instead of invoking it. +* Existing working-tree conversions retain Git's behavior: status and working-tree + diffs can run configured clean filters; checkout, clone, and archive can run + smudge filters. Archive defaults to Git's built-in formats and internal gzip; + custom format commands require ``allow_unsafe_options=True``. Tag creation + suppresses configured signing by default; signing, verification, and editor + options require the same opt-in. +* The raw ``repo.git`` interface remains available for direct Git commands. + Existing explicit unsafe-option and unsafe-protocol opt-ins remain independent. + They do not permit argument or stdin-record injection, or reordering command + options past the library's safety flags. +* Known command outcomes retain established GitPython/configparser exceptions + where distinguishable. Other failures raise ``GitCommandError`` with Git's + exit status and diagnostic output; exact error text may differ. + +API changes +----------- + +* ``GitCmdObjectDB`` no longer inherits ``LooseObjectDB``. Its object reads, + writes, existence checks, and enumeration use Git, including packed objects. + Precompressed input streams and custom object-output writers are unsupported. + ``GitDB`` exports and backend selection are removed. ``odbt`` accepts only + ``GitCmdObjectDB`` and its subclasses, rejecting unsupported types before + initialization or cloning. Remove ``odbt=GitDB`` or select ``GitCmdObjectDB``. + ``gitdb`` remains a dependency for shared types and utilities. +* ``Repo.object_format`` and ``Repo.ref_format`` report Git's storage formats. + ``Repo.alternates`` is read-only and reports effective absolute alternate + directories, including environment and transitive alternates. Direct editing + of the alternates file through this property is removed. +* Index entries retain their mode, object ID, path, and stage. Raw stat fields, + arbitrary cache flags/extensions/checksums, ``IndexFileSHA1Writer``, and the + standalone binary index read/write/merge helpers are removed. Use + ``IndexFile.from_tree()``, ``IndexFile.new()``, ``write()`` and ``write_tree()``. + Git preserves untouched metadata when an existing index is edited. Temporary + indexes isolate tree/merge operations from the real index and working tree. + Git's platform-specific index filename restrictions apply. Unsupported entries + raise ``ValueError`` before the original index changes, including names with + colons or control characters on Windows and backslashes with Cygwin Git. + Tree objects can still contain names that the working tree or index cannot + represent. + ``version`` is read-only. ``from_tree()`` accepts ``trivial``, ``aggressive``, + and ``verbose`` options; arbitrary ``read-tree`` keyword forwarding is removed. +* Standalone binary tree parsers, serializers, and multi-tree traversal helpers + are removed. Use ``Tree`` traversal/cache operations and the index interface. + Tree/commit serialization adapters that remain write their objects to the + repository to obtain Git-produced bytes. + Commit creation preserves message bytes through plain stdin. Injecting or + reusing arbitrary ``gpgsig`` headers is unsupported; modified signed commits + become unsigned. +* ``RefLog`` is associated with a reference, not a filesystem path. Keep using + ``ref.log()``, ``ref.log_entry()`` and ``ref.log_append()``. ``RefLogEntry`` now + contains ``newhexsha``, ``actor``, ``time`` and ``message``; ``oldhexsha`` and + the old tuple layout are removed. Reads follow Git's commit-reflog view, which + omits entries targeting non-commit or unavailable objects. Old IDs are never + inferred from adjacent entries. Raw reflog file/stream read, write, and rewrite + APIs are removed. Reference updates follow Git's reflog creation/update rules, + including updates when ``logmsg`` is omitted. +* ``GitConfigParser`` uses Git's configuration syntax and canonical key spelling, + and no longer subclasses ``configparser.RawConfigParser``. + Common getters, typed values, duplicate values, file/stream inputs and mutation + methods remain. Invalid Git syntax previously accepted by Python is rejected. + Empty-section creation and valueless writes are removed; valueless reads remain. + Relative includes from streams are rejected by Git. Locks cover each native + mutation rather than the writer object's lifetime. +* ``Submodule.rename()`` is removed. Moving a submodule preserves its logical name, + matching Git. Normal removal retains Git's recoverable submodule metadata. + Re-adding with ``no_checkout=True`` can reuse that metadata without changing refs; + incompatible URLs, branches, and clone-only options are rejected before mutation. + Fetch/pull results are derived from command output rather than ``FETCH_HEAD`` + file parsing. Revision strings follow Git's native revision grammar. + +* ``IndexFile.entries`` is removed. Use ``iter_entries()`` for immutable query + records, ``entry(path, stage=0)`` for exact lookup (missing entries raise + ``KeyError``), ``add()``, and ``remove()`` for edits. Python retains only opaque + index-file bytes for deferred/virtual indexes. Git edits private files and + validates readback before atomic publication. ``add(write=False)`` and + ``resolve_blobs()`` retain deferred edits; ``remove(..., write=False)`` now does + too, unless ``working_tree=True``. ``write(file_path)`` publishes to an alternate + path. Checkout and diff use pending bytes. Git owns versions, flags, sparse and + split indexes; deferred split snapshots are made standalone through Git. + Ordinary failed index edits preserve pending and published state. Worktree, + object storage and hook side effects are not rolled back. Commit hooks see the + materialized index; their index edits survive failures without advancing HEAD. + Binary-parser tests and duplicate filename-rejection matrices are replaced by + happy paths, Python glue failures and representative compatibility regressions. + Historical timing-only tests are removed; the maintained backend benchmark + checks result parity and CLI budgets. +* Commit/tag metadata uses structured Gix decoders when the Gix backend supports + the repository. The CLI backend retains raw decoding because Git has no faithful + formatted query for all commit headers or arbitrary tag objects. ``gpgsig`` + remains readable. The small ``Actor.from_string()`` and + ``parse_actor_and_date()`` identity helpers remain supported. +* ``parse_date()`` delegates text syntax and timezone interpretation to Git; + aware datetimes retain direct conversion and invalid inputs raise ``ValueError``. + ISO/RFC dates now apply their timezone to the UTC timestamp, correcting the + previous behavior. Dates without a zone use Git's local timezone, and accepted + date syntax follows the installed Git version. Use an explicit timezone for + reproducible results. ``co_authors`` now uses Git's final trailer block rather + than scanning arbitrary message lines; use a valid trailer block after a blank + line. +* Removed deprecated APIs: ``Git.USE_SHELL`` (explicit ``Git.execute(shell=...)`` + remains), ``Diff.renamed`` (use ``renamed_file``), ``Commit.trailers`` (use + ``trailers_list`` or ``trailers_dict``), ``Actor.name_email_regex`` (use + ``Actor.from_string``), ``git.util.Iterable`` (use ``IterableObj``), + ``git.compat.is_win/is_posix/is_darwin`` (use ``os.name``/``sys.platform``), and + ``git.types.Lit_commit_ish`` (use ``Literal["commit", "tag"]`` or + ``GitObjectTypeString``). Top-level typing exports and private module aliases + are removed; import from ``typing`` or the owning module. ``git.util`` now + exposes the actual utility module. Abstract ``Traversable.traverse`` and + ``list_traverse`` raise ``NotImplementedError``; use concrete implementations. +* Paths and URLs no longer expand ``$VAR``/``%VAR%`` automatically. Removed + ``expand_vars`` switches from ``Repo``, ``Repo.init``, ``expand_path``, + ``cygpath`` and ``Git.polish_url``. Expand explicitly with + ``os.path.expandvars`` if wanted; initial ``~`` expansion remains. + ``HIDE_WINDOWS_KNOWN_ERRORS`` and ``HIDE_WINDOWS_FREEZE_ERRORS`` are removed. + ``rmtree`` propagates filesystem errors rather than raising ``SkipTest``. + 3.2.1 ===== diff --git a/doc/source/intro.rst b/doc/source/intro.rst index dec2ec0fa..9bdc8b0fc 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -6,7 +6,7 @@ Overview / Install GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. -It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, or the faster, but more resource intensive git command implementation. +It provides Python objects for repository data and delegates Git operations to the Git executable. The default backend supports both SHA-1 and SHA-256 object IDs and both files and reftable reference storage. The legacy pure-Python GitDB backend is removed. The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. @@ -14,10 +14,8 @@ Requirements ============ * `Python`_ >= 3.8 -* `Git`_ 1.7.0 or newer - It should also work with older versions, but it may be that some operations - involving remotes will not work as expected. -* `GitDB`_ - a pure python git database implementation +* `Git`_ 2.52 or newer +* `GitDB`_ - shared data types and utilities * `typing_extensions`_ >= 3.7.3.4 (if python < 3.10) .. _Python: https://www.python.org diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index a1ac1eace..b3be63351 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -122,7 +122,7 @@ You can traverse down to :class:`git objects ` through :start-after: # [12-test_init_repo_object] :end-before: # ![12-test_init_repo_object] -The :class:`index ` is also called stage in git-speak. It is used to prepare new commits, and can be used to keep results of merge operations. Our index implementation allows to stream date into the index, which is useful for bare repositories that do not have a working tree. +The :class:`index ` is also called stage in git-speak. It is used to prepare new commits, and can be used to keep results of merge operations. Our index implementation allows to stream data into the index, which is useful for bare repositories that do not have a working tree. .. literalinclude:: ../../test/test_docs.py :language: python @@ -166,7 +166,7 @@ A :class:`symbolic reference ` can point to :start-after: # [3-test_references_and_objects] :end-before: # ![3-test_references_and_objects] -Access the :class:`reflog ` easily. +Access the :class:`reflog ` through a reference. Entries expose the new object ID, reflog actor, timestamp, and message in oldest-first order. They follow Git's commit-reflog view; exact old IDs and raw reflog-file access are unavailable. .. literalinclude:: ../../test/test_docs.py :language: python @@ -202,7 +202,7 @@ Change the :class:`symbolic reference ` to Understanding Objects ********************* -An Object is anything storable in git's object database. Objects contain information about their type, their uncompressed size as well as the actual data. Each object is uniquely identified by a binary SHA1 hash, being 20 bytes in size, or 40 bytes in hexadecimal notation. +An Object is anything storable in git's object database. Objects contain information about their type, their uncompressed size as well as the actual data. Each object is identified by an object ID in the repository's hash format, reported by ``repo.object_format``. SHA-1 IDs contain 20 binary bytes (40 hexadecimal characters); SHA-256 IDs contain 32 binary bytes (64 hexadecimal characters). Treat IDs returned by Git as opaque values instead of assuming a fixed width. Git only knows 4 distinct object types being :class:`Blobs `, :class:`Trees `, :class:`Commits ` and :class:`Tags `. @@ -341,7 +341,7 @@ As trees allow direct access to their intermediate child entries only, use the t The Index Object **************** -The git index is the stage containing changes to be written with the next commit or where merges finally have to take place. You may freely access and manipulate this information using the :class:`IndexFile ` object. +The git index is the stage containing changes to be written with the next commit or where merges finally have to take place. Use :meth:`IndexFile.iter_entries ` and :meth:`IndexFile.entry ` to query immutable entry records (mode, object ID, path, and stage); use ``add`` and ``remove`` to edit them. Deferred edits retain opaque index-file bytes. Git reads and writes the underlying index; raw stat fields and binary index extensions are not exposed. Modify the index with ease .. literalinclude:: ../../test/test_docs.py @@ -377,13 +377,13 @@ You can easily access configuration information for a remote by accessing option :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] -You can also specify per-call custom environments using a new context manager on the Git command, e.g. for using a specific SSH key. The following example works with `git` starting at *v2.3*:: +You can also specify per-call custom environments using a context manager on the Git command, e.g. for using a specific SSH key:: ssh_cmd = 'ssh -i id_deployment_key' with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): repo.remotes.origin.fetch() -This one sets a custom script to be executed in place of `ssh`, and can be used in `git` prior to *v2.3*:: +Alternatively, set a custom script to be executed in place of `ssh`:: ssh_executable = os.path.join(rw_dir, 'my_ssh_executable.sh') with repo.git.custom_environment(GIT_SSH=ssh_executable): @@ -522,20 +522,9 @@ resolves abbreviated object IDs through persistent ``git cat-file`` processes:: # Equivalent explicit selection: repo = Repo("path/to/repo", odbt=GitCmdObjectDB) -GitDB -===== -.. warning:: - The pure-Python ``GitDB`` backend is deprecated due to security and performance - issues. Its object parsers can exhaust resources or return incorrect object - data when processing untrusted repositories. Do not use it for untrusted data. - -Selecting ``odbt=GitDB`` (including a subclass) emits a ``DeprecationWarning``. -To migrate, remove ``odbt=GitDB`` or replace it with ``odbt=GitCmdObjectDB`` when -opening, initializing, or cloning a repository. The deprecated backend remains -available for compatibility; deprecation does not fix its parsing issues. - -The ``gitdb`` package remains a dependency because GitPython still uses its shared -types and utilities. +Only ``GitCmdObjectDB`` and subclasses can be selected through ``odbt``. The +legacy ``GitDB`` backend is removed. The ``gitdb`` package remains a dependency +for shared types and utilities. Git Command Debugging and Customization *************************************** diff --git a/fuzzing/README.md b/fuzzing/README.md index 286f529eb..0d557d658 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -46,6 +46,10 @@ capabilities, jump into the "Getting Started" section below. Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it. +The fuzz targets require **Git 2.52 or newer**, including the Git executable bundled into OSS-Fuzz artifacts. +The local development image builds pinned Git 2.52.0. The OSS-Fuzz bootstrap and build scripts reject older selected +Git executables before preparing artifacts; update the external OSS-Fuzz image when its installed Git is too old. + ### Understanding Existing Fuzz Targets Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See @@ -67,6 +71,8 @@ Contains Python files for each fuzz test. **Things to Know**: - Each fuzz test targets a specific part of GitPython's functionality. +- Repository and object targets exercise Git-backed workflows. Removed Python index/tree binary parsers are no longer + fuzz targets; malformed configuration and object IDs can now be rejected by Git or by Python input validation. - Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the functionality targeted by the test. - Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py index ce888e85f..afbdf7780 100644 --- a/fuzzing/fuzz-targets/fuzz_blob.py +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -16,17 +16,16 @@ def TestOneInput(data): with tempfile.TemporaryDirectory() as temp_dir: repo = git.Repo.init(path=temp_dir) - binsha = fdp.ConsumeBytes(20) + binsha = fdp.ConsumeBytes(repo._oid_size) mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())) path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) try: blob = git.Blob(repo, binsha, mode, path) - except AssertionError as e: - if "Require 20 byte binary sha, got" in str(e): + except ValueError: + if len(binsha) != repo._oid_size: return -1 - else: - raise e + raise _ = blob.mime_type diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 4eddc32ff..87ef6e846 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -39,6 +39,10 @@ def TestOneInput(data): git_config.read() except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): return -1 # Reject inputs raising expected exceptions + except git.GitCommandError as e: + if e.status == 128: + return -1 # Git rejects invalid includes and other configuration input. + raise # Preserve crashes and unexpected command failures. except ValueError as e: if "embedded null byte" in str(e): # The `os.path.expanduser` function, which does not accept strings diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py index d4bd68b57..3e15a3b7b 100644 --- a/fuzzing/fuzz-targets/fuzz_diff.py +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -40,8 +40,8 @@ def TestOneInput(data): repo, a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), - a_blob_id=fdp.ConsumeBytes(20), - b_blob_id=fdp.ConsumeBytes(20), + a_blob_id=fdp.ConsumeBytes(repo._oid_size * 2), + b_blob_id=fdp.ConsumeBytes(repo._oid_size * 2), a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), new_file=fdp.ConsumeBool(), @@ -55,11 +55,10 @@ def TestOneInput(data): ) except BinasciiError: return -1 - except AssertionError as e: - if "Require 20 byte binary sha, got" in str(e): + except ValueError as e: + if "Object ID does not match the repository object format" in str(e): return -1 - else: - raise e + raise _ = diff.__str__() _ = diff.a_path diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py index 7bd82c120..9045b7fee 100644 --- a/fuzzing/fuzz-targets/fuzz_repo.py +++ b/fuzzing/fuzz-targets/fuzz_repo.py @@ -1,5 +1,4 @@ import atheris -import io import sys import os import tempfile @@ -15,9 +14,7 @@ def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) - with tempfile.TemporaryDirectory() as temp_dir: - repo = git.Repo.init(path=temp_dir) - + with tempfile.TemporaryDirectory() as temp_dir, git.Repo.init(path=temp_dir) as repo: # Generate a minimal set of files based on fuzz data to minimize I/O operations. file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))] for file_path in file_paths: @@ -27,15 +24,14 @@ def TestOneInput(data): # fuzzer coverage plateaus. f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) - repo.index.add(file_paths) - repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80))) - - fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "") - - try: - fuzz_tree._deserialize(io.BytesIO(data)) - except IndexError: + message = fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80)) + if "\0" in message: return -1 + repo.index.add(file_paths) + actor = git.Actor("Fuzzing", "fuzzing@example.invalid") + commit = repo.index.commit(message, author=actor, committer=actor, skip_hooks=True) + for blob in commit.tree.blobs: + blob.data_stream.read() def main(): diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile index 426de05dd..a947ecfeb 100644 --- a/fuzzing/local-dev-helpers/Dockerfile +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -11,7 +11,10 @@ COPY . . # Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. RUN apt-get update && \ - apt-get install -y git clang && \ + apt-get install -y git clang build-essential libssl-dev zlib1g-dev libexpat1-dev libcurl4-openssl-dev && \ + git clone --depth 1 --branch v2.52.0 -- https://github.com/git/git.git /tmp/git-source && \ + make -C /tmp/git-source -j2 prefix=/usr/local NO_GETTEXT=YesPlease NO_TCLTK=YesPlease NO_PERL=YesPlease install && \ + rm -rf /tmp/git-source && \ python -m pip install --upgrade pip && \ python -m pip install atheris && \ python -m pip install -e . && \ diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index c156e872d..4b3356319 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -5,6 +5,18 @@ set -euo pipefail +git_binary="$(command -v git)" || { + printf 'GitPython fuzzing requires Git 2.52 or newer; git was not found on PATH.\n' >&2 + exit 1 +} +git_version="$("$git_binary" --version)" +if [[ ! "$git_version" =~ ^git\ version\ ([0-9]+)\.([0-9]+)(\.|[[:space:]]|$) ]] || + ((10#${BASH_REMATCH[1]} < 2 || (10#${BASH_REMATCH[1]} == 2 && 10#${BASH_REMATCH[2]} < 52))); then + printf 'GitPython fuzzing requires Git 2.52 or newer; %s reports %s. Update the container Git installation.\n' \ + "$git_binary" "$git_version" >&2 + exit 1 +fi + python3 -m pip install . find "$SRC" -maxdepth 1 \ @@ -15,5 +27,5 @@ find "$SRC" -maxdepth 1 \ # Build fuzzers in $OUT. find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do - compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:." + compile_python_fuzzer "$fuzz_harness" --add-binary="$git_binary:." --add-data="$SRC/explicit-exceptions-list.txt:." done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 924a3cbf3..9d878d7c1 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -16,6 +16,15 @@ for cmd in python3 git wget zip; do } done +git_binary="$(command -v git)" +git_version="$("$git_binary" --version)" +if [[ ! "$git_version" =~ ^git\ version\ ([0-9]+)\.([0-9]+)(\.|[[:space:]]|$) ]] || + ((10#${BASH_REMATCH[1]} < 2 || (10#${BASH_REMATCH[1]} == 2 && 10#${BASH_REMATCH[2]} < 52))); then + printf 'GitPython fuzzing requires Git 2.52 or newer; %s reports %s. Update the container Git installation.\n' \ + "$git_binary" "$git_version" >&2 + exit 1 +fi + ############# # Functions # ############# @@ -85,7 +94,7 @@ prepare_dictionaries_for_fuzz_targets() { ######################## # Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. # We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. -git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" +"$git_binary" clone --depth 1 -- https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" @@ -97,7 +106,7 @@ pushd "$SRC/gitpython/" # This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise # anticipated exceptions to reduce false positive test failures. -git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" +"$git_binary" grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" popd diff --git a/git/__init__.py b/git/__init__.py index ecc6cd94e..2af9cc08d 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -29,7 +29,6 @@ "GitCommandError", "GitCommandNotFound", "GitConfigParser", - "GitDB", "GitError", "HEAD", "Head", @@ -40,13 +39,11 @@ "IndexObject", "InvalidDBRoot", "InvalidGitRepositoryError", - "List", # Deprecated - import this from `typing` instead. "LockFile", "NULL_TREE", "NoSuchPathError", "ODBError", "Object", - "Optional", # Deprecated - import this from `typing` instead. "ParseError", "PathLike", "PushInfo", @@ -60,19 +57,15 @@ "RepositoryDirtyError", "RootModule", "RootUpdateProgress", - "Sequence", # Deprecated - import from `typing`, or `collections.abc` in 3.9+. "StageType", "Stats", "Submodule", "SymbolicReference", - "TYPE_CHECKING", # Deprecated - import this from `typing` instead. "Tag", "TagObject", "TagReference", "Tree", "TreeModifier", - "Tuple", # Deprecated - import this from `typing` instead. - "Union", # Deprecated - import this from `typing` instead. "UnmergedEntriesError", "UnsafeOptionError", "UnsafeProtocolError", @@ -88,14 +81,9 @@ __version__ = "git" -from typing import Any, List, Optional, Sequence, TYPE_CHECKING, Tuple, Union +from typing import Optional as _Optional -if TYPE_CHECKING: - from types import ModuleType - -import warnings - -from gitdb.util import to_hex_sha +from git.util import to_hex_sha from git.exc import ( AmbiguousObjectName, @@ -158,7 +146,7 @@ DiffIndex, Diffable, ) - from git.db import GitCmdObjectDB, GitDB # @NoMove + from git.db import GitCmdObjectDB # @NoMove from git.cmd import Git # @NoMove from git.repo import Repo # @NoMove from git.remote import FetchInfo, PushInfo, Remote, RemoteProgress # @NoMove @@ -169,9 +157,6 @@ IndexEntry, IndexFile, StageType, - # NOTE: This tells type checkers what util resolves to. We delete it, and it is - # really resolved by __getattr__, which warns. See below on what to use instead. - util, ) from git.util import ( # @NoMove Actor, @@ -185,82 +170,12 @@ raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc -def _warned_import(message: str, fullname: str) -> "ModuleType": - import importlib - - warnings.warn(message, DeprecationWarning, stacklevel=3) - return importlib.import_module(fullname) - - -def _getattr(name: str) -> Any: - # TODO: If __version__ is made dynamic and lazily fetched, put that case right here. - - if name == "util": - return _warned_import( - "The expression `git.util` and the import `from git import util` actually " - "reference git.index.util, and not the git.util module accessed in " - '`from git.util import XYZ` or `sys.modules["git.util"]`. This potentially ' - "confusing behavior is currently preserved for compatibility, but may be " - "changed in the future and should not be relied on.", - fullname="git.index.util", - ) - - for names, prefix in ( - ({"head", "log", "reference", "symbolic", "tag"}, "git.refs"), - ({"base", "fun", "typ"}, "git.index"), - ): - if name not in names: - continue - - fullname = f"{prefix}.{name}" - - return _warned_import( - f"{__name__}.{name} is a private alias of {fullname} and subject to " - f"immediate removal. Use {fullname} instead.", - fullname=fullname, - ) - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -if not TYPE_CHECKING: - # NOTE: The expression `git.util` gives git.index.util and `from git import util` - # imports git.index.util, NOT git.util. It may not be feasible to change this until - # the next major version, to avoid breaking code inadvertently relying on it. - # - # - If git.index.util *is* what you want, use (or import from) that, to avoid - # confusion. - # - # - To use the "real" git.util module, write `from git.util import ...`, or if - # necessary access it as `sys.modules["git.util"]`. - # - # Note also that `import git.util` technically imports the "real" git.util... but - # the *expression* `git.util` after doing so is still git.index.util! - # - # (This situation differs from that of other indirect-submodule imports that are - # unambiguously non-public and subject to immediate removal. Here, the public - # git.util module, though different, makes less discoverable that the expression - # `git.util` refers to a non-public attribute of the git module.) - # - # This had originally come about by a wildcard import. Now that all intended imports - # are explicit, the intuitive but potentially incompatible binding occurs due to the - # usual rules for Python submodule bindings. So for now we replace that binding with - # git.index.util, delete that, and let __getattr__ handle it and issue a warning. - # - # For the same runtime behavior, it would be enough to forgo importing util, and - # delete util as created naturally; __getattr__ would behave the same. But type - # checkers would not know what util refers to when accessed as an attribute of git. - del util - - # This is "hidden" to preserve static checking for undefined/misspelled attributes. - __getattr__ = _getattr - # { Initialize git executable path GIT_OK = None -def refresh(path: Optional[PathLike] = None) -> None: +def refresh(path: _Optional[PathLike] = None) -> None: """Convenience method for setting the git executable path. :param path: diff --git a/git/_backend.py b/git/_backend.py new file mode 100644 index 000000000..3264af6e5 --- /dev/null +++ b/git/_backend.py @@ -0,0 +1,1289 @@ +"""Optional native implementation of library-managed Git operations. + +Installing ``GitPython[gix]`` makes the ``gix`` module available. Unsupported +operations return ``NotImplemented`` before mutation and use the existing CLI +implementation. Native mutation failures are never retried through the CLI. +""" + +from collections import Counter +from importlib import import_module +import io +from itertools import islice +import logging +import os +import re +import stat +from threading import Lock +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, cast + +from git.compat import defenc, safe_decode +from git.exc import GitCommandError + +_selection = os.environ.get("GIT_PYTHON_BACKEND") +if _selection not in (None, "cli", "gix"): + raise ValueError("GIT_PYTHON_BACKEND must be 'cli' or 'gix'") +gix: Any = None +if _selection != "cli": + try: + gix = import_module("gix") + except ModuleNotFoundError as exc: + if exc.name != "gix" or _selection == "gix": + raise + +name = "gix" if gix is not None else "cli" +_counts: Counter = Counter() +_lock = Lock() +_logger = logging.getLogger("git.backend") + + +def record(method: str, outcome: str) -> None: + with _lock: + _counts[method, outcome] += 1 + _logger.debug("%s: %s", method, outcome) + + +def statistics() -> Dict[Tuple[str, str], int]: + """Return counts by operation and native/fallback reason for this process.""" + with _lock: + return dict(_counts) + + +class _Unsupported(Exception): + """A capability decision made before a native mutation.""" + + +def _fallback(method: str, reason: str) -> Any: + record(method, "CLI: " + reason) + return NotImplemented + + +def discover_repository(path: str, environment: Dict[str, Any]) -> Any: + """Open one discovery candidate; the caller controls parent traversal.""" + if gix is None: + return NotImplemented + effective = {**os.environ, **environment} + if any( + effective.get(key) + for key in ("GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE") + ) or any(value != os.environ.get(key) for key, value in environment.items() if key != "GIT_WORK_TREE"): + return _fallback("Repo.open", "storage or command environment") + options = gix.OpenOptions().open_path_as_is(True).bail_if_untrusted(True).strict_config(True) + options = options.config_overrides( + ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false", "core.hooksPath=" + os.devnull] + ) + try: + repo = gix.open_opts(path, options) + if os.path.isfile(path): + # Reopening also makes Gix derive the worktree from its Git directory, + # rather than retaining an arbitrary gitfile's location as a worktree. + repo = gix.open_opts(os.path.realpath(repo.git_dir()), options) + snapshot = repo.config_snapshot() + if snapshot.string("extensions.refStorage") == b"reftable": + return _fallback("Repo.open", "reftable (GIX-1)") + if snapshot.string("extensions.compatObjectFormat") is not None: + return _fallback("Repo.open", "compatibility object format (GIX-19)") + # Discovery accepts undecodable HEADs; force the native reference decoder. + repo.head() + if not repo.is_bare() and repo.workdir() is None and not effective.get("GIT_WORK_TREE"): + return _fallback("Repo.open", "missing native worktree metadata (GIX-14)") + commondir = os.path.join(repo.git_dir(), "commondir") + if os.path.lexists(commondir): + # Gix ignores invalid commondir files on common repositories. A common + # directory must have been resolved and contain actual shared storage. + if os.path.realpath(repo.common_dir()) == os.path.realpath(repo.git_dir()) or not all( + os.path.isdir(os.path.join(repo.common_dir(), entry)) for entry in ("objects", "refs") + ): + return _fallback("Repo.open", "common-directory validation (GIX-14)") + record("Repo.open", "native") + return repo + except gix.Error as exc: + # Retain Git's rejection/diagnostics for layouts Gix cannot open yet. + _logger.debug("native discovery: %s", exc) + return _fallback("Repo.open", "native discovery diagnostics") + except RuntimeError as exc: + if str(exc) != "native worker panicked": + raise + # Undecodable Windows commondir paths panic in GixPython 0.1.0. + # Discovery is read-only; keep Git's validation and error contract. + _logger.debug("native discovery: %s", exc) + return _fallback("Repo.open", "native discovery worker panic (GIX-14)") + + +def _repository(command: Any, env: Dict[str, Any], *, query_config: bool = False) -> Any: + owner = command._repo() if command._repo is not None else None + if owner is not None: + return owner._get_gix_repository(command=command, env=env, query_config=query_config) + return _open_repository(command, env, query_config=query_config) + + +def _open_repository(command: Any, env: Dict[str, Any], *, query_config: bool = False) -> Any: + if command._git_options or command._persistent_git_options: + raise _Unsupported("global command options") + overrides = {**command.environment(), **env} + effective = {**os.environ, **overrides} + path = overrides.get("GIT_DIR") + if not path: + raise _Unsupported("repository not bound") + if not os.path.isabs(path): + path = os.path.join(command.working_dir or os.getcwd(), path) + for key in ("GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE"): + if effective.get(key): + raise _Unsupported("storage environment") + supported = {"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"} + supported.update( + "GIT_%s_%s" % (role, field) for role in ("AUTHOR", "COMMITTER") for field in ("NAME", "EMAIL", "DATE") + ) + if any(key not in supported and value != os.environ.get(key) for key, value in overrides.items()): + raise _Unsupported("command environment") + options = gix.OpenOptions().open_path_as_is(True).bail_if_untrusted(True).strict_config(True) + if not query_config: + options = options.config_overrides( + ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false", "core.hooksPath=" + os.devnull] + ) + owner = command._repo() if command._repo is not None else None + state = None + repo = None + if owner is not None and not query_config: + # Gix refreshes index/ODB snapshots itself; configuration is loaded at open. + paths = [ + path, + os.fspath(owner.common_dir), + os.path.join(owner.common_dir, "config"), + os.path.join(path, "config.worktree"), + ] + paths += [ + effective.get("GIT_CONFIG_SYSTEM", "/etc/gitconfig"), + effective.get("GIT_CONFIG_GLOBAL", os.path.expanduser("~/.gitconfig")), + os.path.join(effective.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "git", "config"), + os.path.join(owner.common_dir, "objects", "info", "alternates"), + os.path.join(path, "commondir"), + os.path.join(path, "gitdir"), + ] + stamps: List[Any] = [] + for filename in paths: + try: + info = os.stat(filename) + stamps.append((info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)) + except FileNotFoundError: + stamps.append(None) + state = (path, tuple(sorted(effective.items())), tuple(stamps)) + repo = owner._gix_repository + if repo is not None and os.path.realpath(repo.git_dir()) != os.path.realpath(path): + repo = None + if repo is not None and state != owner._gix_state: + repo.reload() + if repo is None: + repo = gix.open_opts(path, options) + snapshot = repo.config_snapshot() + if snapshot.string("extensions.refStorage") == b"reftable": + raise _Unsupported("reftable (GIX-1)") + if snapshot.string("extensions.compatObjectFormat") is not None: + raise _Unsupported("compatibility object format (GIX-19)") + if effective.get("GIT_WORK_TREE") and (owner is None or query_config or state != owner._gix_state): + workdir = effective["GIT_WORK_TREE"] + if not os.path.isabs(workdir): + workdir = os.path.join(command.working_dir or os.getcwd(), workdir) + repo.set_workdir(workdir) + if owner is not None and not query_config: + owner._gix_repository = repo + # ponytail: includes can load arbitrary files; reopen until Gix exposes their source paths. + if state != owner._gix_state: + included = re.search(rb"\[include(?:if)?[\s\]]", repo.config_snapshot().plumbing().to_bstring(), re.I) + owner._gix_state = None if included else state + return repo + + +def _canonical_repository(repo: Any) -> Any: + """Have Gix resolve repository metadata again from a canonical input path.""" + path = os.fspath(repo.git_dir()) + canonical = os.path.realpath(path) + return gix.open_opts(canonical, repo.open_options()) if path != canonical else repo + + +def _check_revision_grammar(ref: str) -> None: + if ref.startswith(":/") or "^{/" in ref or "-dirty" in ref or re.search(r"-\d+-g[0-9a-fA-F]+", ref): + raise _Unsupported("revision grammar differences (GIX-17)") + + +def _oid(repo: Any, ref: Any, env: Optional[Dict[str, Any]] = None) -> Any: + ref = safe_decode(ref) if isinstance(ref, bytes) else str(ref) + if re.fullmatch(r"[a-fA-F0-9]{40}|[a-fA-F0-9]{64}", ref): + return gix.ObjectId(ref) + _check_revision_grammar(ref) + if ref.startswith(":"): + _index(repo, {"env": env or {}}) + if not hasattr(repo, "rev_parse_single"): + raise _Unsupported("revision feature disabled") + return repo.rev_parse_single(ref) + + +def object_data(command: Any, ref: bytes, *, stream: bool = False) -> Any: + """Use native object reads without changing the public cat-file interface.""" + if gix is None: + return NotImplemented + method = "stream_object_data" if stream else "get_object_header" + try: + repo = _repository(command, {}) + oid = _oid(repo, ref, command.environment()) + header = repo.try_find_header(oid) + if header is None: + raise ValueError("SHA %s could not be resolved" % oid) + result: Tuple[Any, ...] = (str(oid), header.kind(), header.size()) + if stream: + # ponytail: native lookup buffers the object; use CLI above 8 MiB until GIX-2 provides streaming. + if header.size() > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + result += (io.BytesIO(repo.find_object(oid).data),) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("%s native read: %s", method, exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def object_metadata(command: Any, oid: str, kind: str) -> Any: + """Decode commit/tag fields through Gix's structured object APIs.""" + if gix is None: + return NotImplemented + try: + repo = _repository(command, {}) + obj = getattr(repo, "find_" + kind)(gix.ObjectId(oid)) + data = obj.decode() + if kind == "commit": + if data.encoding is not None and data.encoding.lower() not in (b"utf-8", b"utf8"): + raise _Unsupported("commit text encoding (GIX-29)") + if any(key == b"encoding" for key, _ in data.extra_headers): + raise _Unsupported("commit encoding header order (GIX-30)") + result = { + "tree": data.tree.decode("ascii"), + "parents": tuple(parent.decode("ascii") for parent in data.parents), + "encoding": data.encoding, + "message": data.message, + "extra_headers": data.extra_headers, + "author": obj.author(), + "committer": obj.committer(), + } + else: + result = { + "target": data.target.decode("ascii"), + "target_kind": data.target_kind, + "name": data.name, + "message": data.message, + "signature": data.signature, + "tagger": obj.tagger(), + } + except _Unsupported as exc: + return _fallback(kind + ".decode", str(exc)) + except gix.Error as exc: + _logger.debug("%s native metadata: %s", kind, exc) + return _fallback(kind + ".decode", "native decoder diagnostics") + record(kind + ".decode", "native") + return result + + +def revision_info(command: Any, ref: str) -> Any: + """Resolve an object and its tree/index path metadata in one native parse.""" + if gix is None: + return NotImplemented + try: + _check_revision_grammar(ref) + repo = _repository(command, {}) + if ref.startswith(":"): + _index(repo, {"env": command.environment()}) + spec = repo.rev_parse(ref) + oid = spec.single() + if oid is None: + raise _Unsupported("multiple revisions") + metadata = spec.path_and_mode() + except _Unsupported as exc: + return _fallback("Repo.rev_parse", str(exc)) + except gix.Error as exc: + _logger.debug("native revision metadata: %s", exc) + return _fallback("Repo.rev_parse", "native read diagnostics") + record("Repo.rev_parse", "native") + return str(oid), metadata + + +def _rev_parse(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + from git.util import to_native_path_linux + + if args[:2] == ["--path-format=absolute", "--git-path"]: + if len(args) != 3 or args[2] not in ("modules", "COMMIT_EDITMSG"): + raise _Unsupported("Git metadata path resolution (GIX-22)") + # Git keeps these fixed leaves in the private Git directory, including + # linked worktrees. Other names have their own config/environment rules. + path = os.path.join(_canonical_repository(repo).git_dir(), args[2]) + if os.path.islink(path): + raise _Unsupported("symlinked metadata path (GIX-22)") + return os.fsencode(to_native_path_linux(path)) + b"\n" + if args == ["--path-format=absolute", "--git-common-dir"]: + return os.fsencode(to_native_path_linux(os.path.abspath(repo.common_dir()))) + b"\n" + if args == ["--is-bare-repository"]: + return b"true\n" if repo.is_bare() and repo.workdir() is None else b"false\n" + if args == ["--show-object-format"]: + return str(repo.object_hash()).encode("ascii") + b"\n" + if args == ["--show-ref-format"]: + # Config identifies reftable and HEAD reports it as unsupported, but + # neither validates unknown repository extensions as Git's query does. + raise _Unsupported("reference storage format query (GIX-1)") + if args == ["--show-toplevel"] and repo.workdir() is not None: + return os.fsencode(to_native_path_linux(os.path.abspath(repo.workdir()))) + b"\n" + if args[:1] != ["--verify"] or args[-2:-1] != ["--end-of-options"]: + raise _Unsupported("discovery or revision options") + if args[:-2] not in (["--verify"], ["--verify", "--quiet"]): + raise _Unsupported("revision options") + return str(_oid(repo, args[-1], kwargs.get("env"))).encode("ascii") + b"\n" + + +def _ls_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 3 or args[:2] != ["-z", "--full-tree"]: + raise _Unsupported("tree options") + with repo.find_tree(_oid(repo, args[2])).iter() as entries: + return b"".join( + b"%06o %s %s\t%s\0" + % ( + entry.mode(), + b"tree" if entry.kind() == "tree" else b"commit" if entry.kind() == "commit" else b"blob", + str(entry.id()).encode("ascii"), + entry.filename(), + ) + for entry in entries + ) + + +def check_ref_name(name: str) -> Any: + """Validate a full reference name without opening a repository.""" + if gix is None: + return NotImplemented + try: + gix.Target.Symbolic(name) + except gix.Error as exc: + if "/" not in name: + return _fallback("Reference.validate", "standalone reference names (GIX-20)") + record("Reference.validate", "native") + raise ValueError("Invalid reference %r" % name) from exc + record("Reference.validate", "native") + return None + + +def reference_info(command: Any, path: str) -> Any: + """Read an exact reference target, including an absent reference, in one lookup.""" + if gix is None: + return NotImplemented + try: + if path != "HEAD" and not path.startswith("refs/"): + raise _Unsupported("partial reference names") + repo = _repository(command, {}) + reference = repo.try_find_reference(path) + result = None + if reference is not None: + if reference.name() != os.fsencode(path): + raise _Unsupported("partial reference lookup") + target = reference.target() + name = target.try_name() + result = (None, safe_decode(name)) if name is not None else (str(target.try_id()), None) + except _Unsupported as exc: + return _fallback("Reference.read", str(exc)) + except gix.Error as exc: + _logger.debug("native reference read: %s", exc) + return _fallback("Reference.read", "native read diagnostics") + record("Reference.read", "native") + return result + + +def _symbolic_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 4 or args[:3] != ["--quiet", "--no-recurse", "--"]: + raise _Unsupported("reference mutation or options") + reference = repo.try_find_reference(args[-1]) + if reference is None: + raise _Unsupported("missing reference diagnostics") + target = reference.target().try_name() + if target is None: + raise GitCommandError(["git", "symbolic-ref"], 1) + return target + b"\n" + + +def _for_each_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if not args or args[0] != "--format=%(refname)": + raise _Unsupported("reference format/options") + split = args.index("--") if "--" in args else len(args) + if args[1:split] or len(args[split + 1 :]) > 1: + raise _Unsupported("root refs or reference options") + prefix = os.fsencode(args[-1]) if len(args) > split + 1 else b"" + if any(char in prefix for char in b"*?["): + raise _Unsupported("reference glob patterns") + with repo.references().all() as refs: + names = [] + for ref in refs: + names.append(ref.name()) + if ref.target().try_name() is not None: + # Git omits dangling symbolic refs; defer their diagnostics to it. + ref.follow_to_object() + return b"".join( + name + b"\n" + for name in sorted(names) + if not prefix or name == prefix or name.startswith(prefix.rstrip(b"/") + b"/") + ) + + +def submodule_config(source: Any) -> Any: + """Read the fields used by submodule objects from their selected config source.""" + if gix is None: + return NotImplemented + try: + if isinstance(source, io.BytesIO): + data = source.getvalue() + else: + with open(source, "rb") as stream: + data = stream.read() + # ponytail: no bound section enumeration; conservatively reject other '[' + # occurrences and duplicate sections until GIX-12 can preserve their order. + if b"\0" in data or re.search(rb"\[(?!submodule[ \t])", data, re.I): + raise _Unsupported("non-submodule configuration sections (GIX-12)") + modules = gix.ModulesFile.from_bytes(data) + with modules.names() as cursor: + names = list(cursor) + if data.count(b"[") != len(names): + raise _Unsupported("duplicate or ambiguous configuration sections (GIX-12)") + config = modules.config() + result = {} + for raw_name in names: + name = raw_name.decode(defenc) + values = {} + for option in ("path", "url", "branch"): + key = "submodule." + name + "." + option + value = config.string(key) + if value is None: + if option != "branch" or config.boolean(key) is not None: + raise _Unsupported("missing or implicit submodule fields") + else: + values[option] = value.decode(defenc) + result[name] = values + except _Unsupported as exc: + return _fallback("Submodule.config", str(exc)) + except (gix.Error, OSError, UnicodeError) as exc: + _logger.debug("native submodule configuration: %s", exc) + return _fallback("Submodule.config", "native parse diagnostics") + record("Submodule.config", "native") + return result + + +def _config(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) == 2 and args[0] == "--get": + snapshot = repo.config_snapshot() + value = snapshot.string(args[1]) + if value is None: + if snapshot.boolean(args[1]) is None: + raise GitCommandError(["git", "config"], 1) + value = b"" # An implicit boolean is an empty value in --get output. + return value + b"\n" + raise _Unsupported("config file parsing, enumeration, or mutation (GIX-12)") + + +def _worktree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + from git.util import to_native_path_linux + + if args != ["list", "--porcelain", "-z"] or not hasattr(repo, "worktrees"): + raise _Unsupported("worktree operation or feature disabled") + proxies = repo.worktrees() + if any(proxy.is_prunable() for proxy in proxies): + raise _Unsupported("prunable worktree diagnostics") + main = _canonical_repository(repo.main_repo()) + + def describe(local: Any) -> List[bytes]: + workdir = local.workdir() + fields = [b"worktree " + os.fsencode(to_native_path_linux(os.fspath(workdir or local.git_dir())))] + # Gix's is_bare() reflects configuration, even for a linked worktree. + if local.is_bare() and workdir is None: + fields.append(b"bare") + else: + head = local.head() + oid = head.id() + fields.append( + b"HEAD " + (str(oid).encode("ascii") if oid is not None else b"0" * local.object_hash().len_in_hex()) + ) + fields.append(b"detached" if head.is_detached() else b"branch " + head.referent_name()) + return fields + + records = [b"\0".join(describe(main)) + b"\0\0"] + for proxy in sorted(proxies, key=lambda proxy: os.fsencode(proxy.base())): + fields = describe(proxy.into_repo()) + if proxy.is_locked(): + reason = proxy.lock_reason() + fields.append(b"locked" + (b" " + reason if reason else b"")) + records.append(b"\0".join(fields) + b"\0\0") + return b"".join(records) + + +def _worktree_root(command: Any, repo: Any) -> str: + root = repo.workdir() + if ( + root is None + or repo.prefix() is not None + or os.path.realpath(command.working_dir or os.getcwd()) != os.path.realpath(root) + ): + raise _Unsupported("worktree or command working directory") + return os.fspath(root) + + +def is_dirty(command: Any, index: bool, working_tree: bool, untracked: bool, submodules: bool, path: Any) -> Any: + if gix is None: + return NotImplemented + method = "Repo.is_dirty" + try: + repo = _repository(command, {}) + _worktree_root(command, repo) + if not hasattr(repo, "status"): + raise _Unsupported("status feature disabled") + patterns = [os.fspath(path)] if path else [] + if any("\0" in pattern for pattern in patterns): + raise _Unsupported("invalid pathspec") + status = ( + repo.status() + .index(_index(repo, {"env": command.environment()})) + .untracked_files("files" if untracked else "none") + .tree_index_track_renames(None) + .index_worktree_rewrites(None) + .index_worktree_submodules("configured" if submodules else "all", check_dirty=submodules) + ) + dirty = False + with status.into_iter(patterns) as entries: + for item in entries: + if item.summary() is None: + continue + if item.kind == "TreeIndex": + if index and (submodules or item.details["entry_mode"] != 0o160000): + dirty = True + break + elif item.details["kind"] == "DirectoryContents": + if untracked and item.details["entry"]["status"] == "Untracked": + dirty = True + break + elif working_tree: + dirty = True + break + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("is_dirty native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return dirty + + +def untracked_files(command: Any, args: Tuple[Any, ...], options: Dict[str, Any]) -> Any: + if gix is None: + return NotImplemented + method = "Repo.untracked_files" + try: + if options.keys() - {"ignore_submodules"}: + raise _Unsupported("status options") + repo = _repository(command, {}) + _worktree_root(command, repo) + if not hasattr(repo, "dirwalk_iter"): + raise _Unsupported("dirwalk feature disabled") + index = _index(repo, {"env": command.environment()}) + patterns = command._unpack_args([arg for arg in args if arg is not None]) + if any("\0" in path for path in patterns): + raise _Unsupported("invalid pathspec") + walk_options = repo.dirwalk_options().emit_untracked("matching").emit_tracked(False) + result = [] + with repo.dirwalk_iter(index, patterns, walk_options) as entries: + for item in entries: + entry = item.entry + if entry.status == "Untracked": + path = entry.rela_path + if entry.disk_kind in ("Directory", "Repository"): + path += b"/" + result.append(path) + result.sort() + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("untracked_files native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return [safe_decode(path) for path in result] + + +def ignored(command: Any, paths: Sequence[Any]) -> Any: + if gix is None: + return NotImplemented + method = "Repo.ignored" + try: + repo = _repository(command, {}) + root = _worktree_root(command, repo) + if not hasattr(repo, "excludes"): + raise _Unsupported("excludes feature disabled") + index = _index(repo, {"env": command.environment()}) + excludes = repo.excludes(index) + with index.entries() as entries: + tracked = {entry.path() for entry in entries} + result = [] + for path in paths: + path = os.fspath(path) + relative = os.path.relpath(path, root) if os.path.isabs(path) else path + relative = relative.replace(os.sep, "/") + if any(part in (".", "..", ".git") for part in relative.split("/")) or relative.startswith(":"): + raise _Unsupported("ignore path normalization") + # check-ignore rejects traversal through symlinks and tracked gitlinks. + parent = relative.rstrip("/") + while "/" in parent: + parent = parent.rsplit("/", 1)[0] + if os.path.islink(os.path.join(root, parent)) or os.fsencode(parent) in tracked: + raise _Unsupported("ignore path traverses symlink or submodule") + if os.fsencode(relative.rstrip("/")) in tracked: + continue + mode: Optional[int] + try: + mode = os.lstat(os.path.join(root, relative)).st_mode + except FileNotFoundError: + mode = stat.S_IFDIR if relative.endswith("/") else None + except OSError as exc: + raise _Unsupported("ignore path metadata unavailable") from exc + else: + mode = 0o40000 if stat.S_ISDIR(mode) else 0o120000 if stat.S_ISLNK(mode) else 0o100644 + if excludes.at_entry(os.fsencode(relative), mode).is_excluded(): + result.append(path) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("ignored native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def _merge_base(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if not hasattr(repo, "merge_base"): + raise _Unsupported("revision feature disabled") + ancestor = args[:1] == ["--is-ancestor"] + if ancestor: + args = args[1:] + if len(args) != 3 or args[0] != "--": + raise _Unsupported("merge-base options") + one, two = (_oid(repo, value) for value in args[1:]) + if kwargs.get("all"): + ids = repo.merge_bases_many(one, [two]) + else: + base = repo.merge_base(one, two) + ids = [base] if base is not None else [] + if not ids or (ancestor and ids[0] != one): + raise GitCommandError(["git", "merge-base"], 1) + return b"" if ancestor else b"".join(str(oid).encode("ascii") + b"\n" for oid in ids) + + +def _walk(repo: Any, rev: str, options: Dict[str, Any]) -> Iterator[str]: + if not hasattr(repo, "rev_walk"): + raise _Unsupported("revision feature disabled") + if options.keys() - {"max_count", "skip", "first_parent"}: + raise _Unsupported("history options or ordering (GIX-8)") + if options.get("first_parent") not in (None, True, False): + raise _Unsupported("history options") + start, count = options.get("skip", 0), options.get("max_count") + if type(start) is not int or start < 0 or (count is not None and (type(count) is not int or count < 0)): + raise _Unsupported("history limits") + tip = repo.find_object(_oid(repo, rev)).peel_to_commit().id + platform = repo.rev_walk([tip]) + if options.get("first_parent"): + platform = platform.first_parent_only() + cursor = platform.all() + + def iterate() -> Iterator[str]: + try: + with cursor: + for item in islice(cursor, start, None if count is None else start + count): + yield str(item.id) + except gix.Error as exc: + raise GitCommandError(["gix", "rev-list", rev], 128, str(exc)) from exc + + return iterate() + + +def history(command: Any, rev: str, paths: Any, options: Dict[str, Any], *, count: bool = False) -> Any: + """Count reachable commits, or lazily walk the first-parent chain.""" + if gix is None: + return NotImplemented + method = "Commit.count" if count else "Commit.iter_items" + try: + if paths: + raise _Unsupported("history path filtering") + if ( + not count + and not options.get("first_parent") + and not (options.get("max_count") == 1 and not options.get("skip")) + ): + raise _Unsupported("Git history ordering (GIX-8)") + iterator = _walk(_repository(command, {}), rev, options) + result = sum(1 for _ in iterator) if count else iterator + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("%s native preparation: %s", method, exc) + return _fallback(method, "native preparation diagnostics") + record(method, "native") + return result + + +def _date(signature: Any) -> bytes: + offset = signature.time.offset + hours, minutes = divmod(abs(offset) // 60, 60) + return b"%d %s%02d%02d" % (signature.time.seconds, b"-" if offset < 0 else b"+", hours, minutes) + + +def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) == 3 and args[:2] == ["exists", "--"]: + reference = repo.try_find_reference(args[-1]) + if reference is None: + raise _Unsupported("orphan reflog lookup") + if not reference.log_exists(): + raise GitCommandError(["git", "reflog", "exists"], 1) + return b"" + if ( + len(args) != 10 + or args[:8] + != [ + "show", + "--format=%H%x00%gn%x00%ge%x00%gD%x00%gs", + "--date=raw", + "-z", + "--no-abbrev", + "--no-decorate", + "--no-notes", + "--no-color", + ] + or args[-1] != "--" + ): + raise _Unsupported("reflog writing or format") + reference = repo.try_find_reference(args[-2]) + if reference is None: + raise _Unsupported("orphan reflog lookup") + cursor = reference.log_iter().rev() + if cursor is None: + return b"" + output = [] + with cursor: + for line in cursor: + if not repo.has_object(line.new_oid) or repo.find_header(line.new_oid).kind() != "commit": + continue + signature = line.signature + output.append( + b"\0".join( + [ + str(line.new_oid).encode("ascii"), + signature.name, + signature.email, + os.fsencode(args[-2]) + b"@{" + _date(signature) + b"}", + line.message, + ] + ) + + b"\0" + ) + return b"".join(output) + + +def _index(repo: Any, kwargs: Dict[str, Any]) -> Any: + if not hasattr(repo, "index_or_empty"): + raise _Unsupported("index feature disabled") + path = kwargs.get("env", {}).get("GIT_INDEX_FILE", os.environ.get("GIT_INDEX_FILE")) + if path and not os.path.isabs(path): + raise _Unsupported("relative index path") + if path and os.path.abspath(path) != os.path.abspath(repo.index_path()): + raise _Unsupported("custom index path (GIX-3)") + index = repo.index_or_empty() + if index.is_sparse(): + raise _Unsupported("sparse index (GIX-4)") + return index + + +def _ls_files(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["--stage", "-v", "-z", "--full-name"]: + raise _Unsupported("index listing options") + with _index(repo, kwargs).entries() as entries: + output = [] + for entry in entries: + flag = b"S" if entry.flags & (1 << 30) else b"M" if entry.stage() else b"H" + if entry.flags & (1 << 15): + flag = flag.lower() + output.append( + b"%s %06o %s %d\t%s\0" % (flag, entry.mode, str(entry.id).encode("ascii"), entry.stage(), entry.path()) + ) + return b"".join(output) + + +def _update_index(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["--show-index-version"]: + raise _Unsupported("index mutation options") + return str(_index(repo, kwargs).version()).encode("ascii") + b"\n" + + +def _index_from_tree(repo: Any, tree: Any) -> Any: + if not hasattr(repo, "index_from_tree"): + raise _Unsupported("index feature disabled") + if ( + repo.config_snapshot().integer("index.version") not in (None, 2) + or os.environ.get("GIT_INDEX_VERSION", "2") != "2" + ): + raise _Unsupported("new index version selection (GIX-13)") + return repo.index_from_tree(tree) + + +def _write(method: str, function: Callable[[], Any]) -> Any: + """Once a write starts, an error must not cause a second attempt through Git.""" + try: + return function() + except gix.Error as exc: + raise GitCommandError(["gix", method], 128, str(exc)) from exc + + +def _input(kwargs: Dict[str, Any]) -> bytes: + stream = kwargs.get("istream") + if stream is None or not hasattr(stream, "seekable") or not stream.seekable(): + raise _Unsupported("nonseekable input") + start = stream.tell() + try: + stream.seek(0, 2) + if stream.tell() - start > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + stream.seek(start) + data = stream.read() + finally: + stream.seek(start) + if not isinstance(data, bytes): + raise _Unsupported("nonbinary input") + return data + + +def _hash_object(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) not in (3, 4) or args[0] != "-t" or args[-1] != "--stdin": + raise _Unsupported("object hashing options") + write = args[2:-1] == ["-w"] + if args[2:-1] not in ([], ["-w"]): + raise _Unsupported("object hashing options") + kind = args[1] + if kind not in ("blob", "tree", "commit", "tag"): + raise _Unsupported("object kind") + if kind == "tag": + raise _Unsupported("strict tag object validation (GIX-28)") + data = _input(kwargs) + # Validate and check the encoded bytes in memory before touching object storage. + memory = repo.with_object_memory() + oid = memory.write_object(kind, data) + if memory.find_object(oid).data != data: + raise _Unsupported("object serialization changes bytes (GIX-5)") + if write: + oid = _write("hash_object", lambda: repo.write_object(kind, data)) + kwargs["istream"].seek(len(data), 1) + return str(oid).encode("ascii") + b"\n" + + +_ENTRY_KINDS = {0o100644: "blob", 0o100755: "exe", 0o120000: "link", 0o160000: "commit", 0o40000: "tree"} + + +def _write_tree(repo: Any, entries: Sequence[Tuple[bytes, int, str]]) -> str: + names = {name for name, _mode, _oid in entries} + if len(names) != len(entries): + raise _Unsupported("duplicate or overlapping tree paths") + for name in names: + while b"/" in name: + name = name.rsplit(b"/", 1)[0] + if name in names: + raise _Unsupported("duplicate or overlapping tree paths") + for _path, mode, oid in entries: + object_id = gix.ObjectId(oid) + if object_id.is_null(): + raise _Unsupported("null tree object IDs") + header = repo.try_find_header(object_id) + if header is None: + if mode != 0o160000: + raise _Unsupported("missing tree children (GIX-7)") + elif header.kind() != ("tree" if mode == 0o40000 else "commit" if mode == 0o160000 else "blob"): + raise _Unsupported("tree child object kind") + with repo.empty_tree().edit() as editor: + for path, mode, oid in entries: + editor.upsert(path, _ENTRY_KINDS[mode], gix.ObjectId(oid)) + return str(_write("write_tree", editor.write)) + + +def write_tree(command: Any, entries: Sequence[Tuple[bytes, int, str]]) -> Any: + if gix is None: + return NotImplemented + try: + result = _write_tree(_repository(command, {}), entries) + except _Unsupported as exc: + return _fallback("IndexFile.write_tree", str(exc)) + except gix.Error as exc: + _logger.debug("write_tree native preparation: %s", exc) + return _fallback("IndexFile.write_tree", "native preparation diagnostics") + record("IndexFile.write_tree", "native") + return result + + +def tree_diff(repository: Any, left: Any, right: Any, paths: Any, patch: bool, options: Dict[str, Any]) -> Any: + """Build GitPython's raw tree diff directly from native change records.""" + if gix is None: + return NotImplemented + from git.diff import Diff, DiffIndex, Lit_change_type + + method = "Diffable.diff" + try: + if not hasattr(left, "hexsha") or not (hasattr(right, "hexsha") or isinstance(right, str)): + raise _Unsupported("index/worktree/root diff") + if patch or paths: + raise _Unsupported("patch formatting or path filtering") + if options.keys() - {"R", "no_renames"} or any(type(value) is not bool for value in options.values()): + raise _Unsupported("diff options") + if "no_renames" in options and not options["no_renames"]: + raise _Unsupported("configured rename detection") + repo = _repository(repository.git, {}) + if not hasattr(repo, "diff_tree_to_tree"): + raise _Unsupported("tree-diff feature disabled") + before = repo.find_object(_oid(repo, left.hexsha)).peel_to_tree() + after = repo.find_object(_oid(repo, getattr(right, "hexsha", right))).peel_to_tree() + if options.get("R"): + before, after = after, before + diff_options = gix.DiffOptions().track_path().track_rewrites(None) + changes = repo.diff_tree_to_tree(before, after, diff_options) + if not options.get("no_renames"): + added = [str(c.id()) for c in changes if c.kind == "Addition" and c.entry_mode() != 0o40000] + deleted = [str(c.id()) for c in changes if c.kind == "Deletion" and c.entry_mode() != 0o40000] + if added and deleted: + if (set(added) - set(deleted) and set(deleted) - set(added)) or ( + len(set(added)) != len(added) or len(set(deleted)) != len(deleted) + ): + raise _Unsupported("inexact or ambiguous rename detection (GIX-11)") + # Exact rewrites need no blob filters, textconv, or external diff drivers. + diff_options = diff_options.track_rewrites(gix.Rewrites(percentage=None)) + changes = repo.diff_tree_to_tree(before, after, diff_options) + result: DiffIndex[Diff] = DiffIndex() + for change in changes: + details = change.details + kind, path, mode, oid = change.kind, change.location(), change.entry_mode(), str(change.id()) + previous_mode = details.get("previous_entry_mode", details.get("source_entry_mode", mode)) + previous_oid = str(details.get("previous_id", details.get("source_id", change.id()))) + if mode == 0o40000 or previous_mode == 0o40000: + if mode != previous_mode: + raise _Unsupported("directory type change") + continue + new_file, deleted_file, renamed = kind == "Addition", kind == "Deletion", kind == "Rewrite" + change_type = ( + "A" + if new_file + else "D" + if deleted_file + else "R" + if renamed + else ("T" if mode & 0o170000 != previous_mode & 0o170000 else "M") + ) + source = change.source_location() if renamed else path + result.append( + Diff( + repository, + source, + path, + None if new_file else previous_oid, + None if deleted_file else oid, + "%06o" % (0 if new_file else previous_mode), + "%06o" % (0 if deleted_file else mode), + new_file, + deleted_file, + False, + source if renamed else None, + path if renamed else None, + "", + cast(Lit_change_type, change_type), + 100 if renamed else None, + ) + ) + result.sort(key=lambda diff: diff.b_rawpath or b"") + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("tree_diff native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def commit_stats(commit: Any) -> Any: + """Count changed lines natively, retaining Git's statistics representation.""" + if gix is None: + return NotImplemented + from git.util import Stats + + method = "Commit.stats" + try: + repo = _repository(commit.repo.git, {}) + if not hasattr(repo, "diff_tree_to_tree") or not hasattr(repo, "attributes_only"): + raise _Unsupported("diff or attributes feature disabled") + if repo.config_snapshot().string("diff.algorithm") not in (None, b"myers", b"default"): + raise _Unsupported("statistics diff algorithm") + native = repo.find_commit(gix.ObjectId(commit.hexsha)) + with native.parent_ids() as parents: + parent = next(parents, None) + before = repo.empty_tree() if parent is None else repo.find_commit(parent).tree() + changes = repo.diff_tree_to_tree(before, native.tree(), gix.DiffOptions().track_path().track_rewrites(None)) + index = _index(repo, {"env": commit.repo.git.environment()}) + # The blob cache reads index attributes, whereas Git also reads worktree attributes. + attributes = [repo.attributes_only(index, source) for source in ("id_mapping", "worktree_then_id_mapping")] + cache = repo.diff_resource_cache_for_tree_diff() + lines = [] + for change in sorted(changes, key=lambda change: change.location()): + mode = change.entry_mode() + previous_mode = change.details.get("previous_entry_mode", mode) + if mode == 0o40000 or previous_mode == 0o40000: + if mode != previous_mode: + raise _Unsupported("statistics directory type change") + continue + if mode == 0o160000 or previous_mode == 0o160000: + raise _Unsupported("statistics gitlinks") + path = change.location() + if any(byte < 32 or byte >= 127 or byte in b'"\\' for byte in path): + raise _Unsupported("statistics filename quoting") + for stack in attributes: + outcome = stack.selected_attribute_matches(["diff"]) + stack.at_entry(path, mode).matching_attributes(outcome) + with outcome.iter_selected() as matches: + if any(match.state != "unspecified" for match in matches): + raise _Unsupported("statistics diff attributes (GIX-18)") + for oid in (change.id(), change.details.get("previous_id")): + if oid is not None and repo.find_header(oid).size() > 8 * 1024 * 1024: + raise _Unsupported("large object streaming (GIX-2)") + counts = change.diff(cache).line_counts() + cache.clear_resource_cache() + kind = {"Addition": "A", "Deletion": "D", "Modification": "M"}[change.kind] + if mode & 0o170000 != previous_mode & 0o170000: + kind = "T" + lines.append( + "%s\t%d\t%d\t%s\n" + % (kind, counts.insertions if counts else 0, counts.removals if counts else 0, safe_decode(path)) + ) + result = Stats._list_from_string(commit.repo, "".join(lines)) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except gix.Error as exc: + _logger.debug("commit_stats native read: %s", exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + return result + + +def _mktree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["-z", "--missing"]: + raise _Unsupported("tree writing options") + data = _input(kwargs) + entries = [] + for record in data.split(b"\0"): + if not record: + continue + metadata, path = record.split(b"\t", 1) + mode_bytes, kind, oid_bytes = metadata.split() + mode = int(mode_bytes, 8) + if mode not in _ENTRY_KINDS or b"/" in path or path in (b"", b".", b"..", b".git"): + raise _Unsupported("tree entry") + expected = b"tree" if mode == 0o40000 else b"commit" if mode == 0o160000 else b"blob" + if kind != expected: + raise _Unsupported("tree entry kind") + entries.append((path, mode, oid_bytes.decode("ascii"))) + oid = _write_tree(repo, entries) + kwargs["istream"].seek(len(data), 1) + return oid.encode("ascii") + b"\n" + + +def _read_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) != 1 or (args[0] != "--empty" and args[0].startswith("-")): + raise _Unsupported("index merge options") + path = kwargs.get("env", {}).get("GIT_INDEX_FILE") + if not path or os.path.exists(path): + raise _Unsupported("existing index metadata") + tree = repo.empty_tree() if args == ["--empty"] else repo.find_object(_oid(repo, args[0])).peel_to_tree() + index = _index_from_tree(repo, tree) + index.set_path(path) + _write("read_tree", index.write) + return b"" + + +def _checked_signature(signature: Any) -> Any: + # Git strips "crud" at the edges and angle brackets/newlines within identities. + crud = bytes(range(33)) + b".,:;<>\"'\\" + for value in (signature.name, signature.email): + if not value or value != value.strip(crud) or any(char in value for char in b"<>\r\n\0"): + raise _Unsupported("identity normalization") + return signature + + +def _signature(env: Dict[str, Any], role: str) -> Any: + prefix = "GIT_" + role + "_" + name, email, date = (env.get(prefix + field) for field in ("NAME", "EMAIL", "DATE")) + # Adapt the normalized date emitted by GitPython commit construction, not Git object bytes. + match = re.fullmatch(r"(-?\d+) ([+-])(\d\d)(\d\d)", date or "") + if not name or not email or match is None: + raise _Unsupported("identity resolution") + seconds, sign, hours, minutes = match.groups() + offset = (int(hours) * 60 + int(minutes)) * 60 * (-1 if sign == "-" else 1) + return _checked_signature(gix.Signature(name, email, int(seconds), offset)) + + +def _committer(repo: Any, env: Dict[str, Any]) -> Any: + for field in ("NAME", "EMAIL", "DATE"): + key = "GIT_COMMITTER_" + field + if key in env and env[key] != os.environ.get(key): + raise _Unsupported("per-command committer identity (GIX-21)") + if os.environ.get(key) == "": + raise _Unsupported("empty identity") + signature = repo.committer() + if signature is None: + raise _Unsupported("identity resolution") + return _checked_signature(signature) + + +def _var(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if args != ["GIT_COMMITTER_IDENT"]: + raise _Unsupported("Git variable") + signature = _committer(repo, kwargs.get("env", {})) + return signature.name + b" <" + signature.email + b"> " + _date(signature) + b"\n" + + +def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if "--" not in args: + raise _Unsupported("reference transactions (GIX-9)") + split = args.index("--") + options, operands = args[:split], args[split + 1 :] + deref, force_log, delete, message = True, False, False, "" + cursor = iter(options) + for option in cursor: + if option == "--no-deref": + deref = False + elif option == "--create-reflog": + force_log = True + elif option == "-d": + delete = True + elif option == "-m": + message = next(cursor, "") + else: + raise _Unsupported("reference options") + # LogChange sets each edit's message, but its writer does not normalize it. + if re.search(r"[\t\r\n\v\f]|^ | $| {2}", message): + raise _Unsupported("reflog message cleanup (GIX-10)") + if delete: + if len(operands) != 1 or deref: + raise _Unsupported("reference deletion options") + edit = gix.RefEdit.delete(operands[0], gix.PreviousValue.Any).with_deref(False) + _write("update_ref", lambda: repo.edit_references_as([edit])) + return b"" + if len(operands) != 2: + raise _Unsupported("strict reference creation / compare-and-swap (GIX-9)") + name, target = operands + oid = _oid(repo, target) + kind = repo.find_header(oid).kind() + if (name == "HEAD" or name.startswith("refs/heads/")) and kind != "commit": + raise _Unsupported("branch target validation") + reference = repo.try_find_reference(name) + if reference is not None: + if deref and name != "HEAD" and reference.target().try_name() is not None: + raise _Unsupported("symbolic reference update (GIX-10)") + if not deref and reference.target().try_name() is not None: + raise _Unsupported("detaching symbolic ref reflog (GIX-10)") + if reference.follow_to_object() == oid: + raise _Unsupported("unchanged reference reflog (GIX-10)") + head = repo.try_find_reference("HEAD") + for _ in range(5): + if head is None or head.target().try_name() is None: + break + if head.target().try_name() == os.fsencode(name): + raise _Unsupported("active branch HEAD reflog (GIX-10)") + head = head.follow() + log = gix.LogChange() + log.force_create_reflog = force_log + log.message = message + edit = gix.RefEdit.update_with_log(name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref(deref) + signature = _committer(repo, kwargs.get("env", {})) + _write("update_ref", lambda: repo.edit_references_as([edit], signature)) + return b"" + + +def _commit_tree(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: + if len(args) < 2 or args[0] != "--no-gpg-sign" or args[2::2] != ["-p"] * len(args[2::2]): + raise _Unsupported("commit options") + if len(args) % 2 or any(setting.lower() != "i18n.commitencoding=utf-8" for setting in kwargs.get("_config", ())): + raise _Unsupported("commit encoding (GIX-6)") + data = _input(kwargs) + try: + message = data.decode("utf-8") + except UnicodeDecodeError: + raise _Unsupported("commit message bytes (GIX-6)") from None + parents = args[3::2] + if len(set(parents)) != len(parents): + raise _Unsupported("duplicate parents") + author = _signature(kwargs.get("env", {}), "AUTHOR") + committer = _signature(kwargs.get("env", {}), "COMMITTER") + tree = _oid(repo, args[1]) + # Native commit creation accepts literal IDs, whereas commit-tree verifies kinds. + if repo.find_header(tree).kind() != "tree" or any( + repo.find_header(_oid(repo, p)).kind() != "commit" for p in parents + ): + raise _Unsupported("commit object kinds") + commit = _write("commit_tree", lambda: repo.new_commit_as(committer, author, message, tree, parents)) + kwargs["istream"].seek(len(data), 1) + return str(commit.id).encode("ascii") + b"\n" + + +_HANDLERS: Dict[str, Callable[[Any, List[str], Dict[str, Any]], bytes]] = { + "rev_parse": _rev_parse, + "ls_tree": _ls_tree, + "symbolic_ref": _symbolic_ref, + "for_each_ref": _for_each_ref, + "merge_base": _merge_base, + "ls_files": _ls_files, + "update_index": _update_index, + "hash_object": _hash_object, + "mktree": _mktree, + "read_tree": _read_tree, + "commit_tree": _commit_tree, + "reflog": _reflog, + "var": _var, + "update_ref": _update_ref, + "config": _config, + "worktree": _worktree, +} + + +def dispatch( + command: Any, method: str, args: Tuple[Any, ...], kwargs: Dict[str, Any], config: Sequence[str] = () +) -> Any: + if gix is None: + return NotImplemented + handler = _HANDLERS.get(method) + if handler is None: + return _fallback(method, "not converted") + allowed = {"env", "stdout_as_string", "strip_newline_in_stdout", "with_extended_output"} + if method == "merge_base": + allowed.add("all") + if method in ("hash_object", "mktree", "commit_tree"): + allowed.add("istream") + if kwargs.keys() - allowed: + return _fallback(method, "command/process options") + if config and method != "commit_tree": + return _fallback(method, "configuration overrides") + if config: + kwargs = dict(kwargs, _config=config) + kwargs = dict(kwargs, env={**command.environment(), **(kwargs.get("env") or {})}) + try: + args_list = command._unpack_args([arg for arg in args if arg is not None]) + repo = _repository(command, kwargs.get("env", {}), query_config=method == "config") + output = handler(repo, args_list, kwargs) + except _Unsupported as exc: + return _fallback(method, str(exc)) + except GitCommandError: + record(method, "native") + raise + except gix.Error as exc: + _logger.debug("%s native read: %s", method, exc) + return _fallback(method, "native read diagnostics") + record(method, "native") + if kwargs.get("strip_newline_in_stdout", True) and output.endswith(b"\n"): + output = output[:-1] + result = safe_decode(output) if kwargs.get("stdout_as_string", True) else output + return (0, result, "") if kwargs.get("with_extended_output") else result diff --git a/git/cmd.py b/git/cmd.py index 1d1ca909a..1f1e65a26 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = ["GitMeta", "Git"] +__all__ = ["Git"] import contextlib import io @@ -13,12 +13,12 @@ import logging import os import re +import shutil import signal import subprocess import sys import threading import time -import warnings from subprocess import DEVNULL, PIPE, Popen from textwrap import dedent @@ -44,12 +44,14 @@ ) from git.compat import defenc, force_bytes, safe_decode +from git import _backend from git.exc import ( CommandError, GitCommandError, GitCommandNotFound, UnsafeOptionError, UnsafeProtocolError, + UnsupportedOperation, ) from git.util import ( cygpath, @@ -589,79 +591,7 @@ def __del__(self) -> None: _CatFileContentStream.__qualname__ = "Git.CatFileContentStream" -_USE_SHELL_DEFAULT_MESSAGE = ( - "Git.USE_SHELL is deprecated, because only its default value of False is safe. " - "It will be removed in a future release." -) - -_USE_SHELL_DANGER_MESSAGE = ( - "Setting Git.USE_SHELL to True is unsafe and insecure, as the effect of special " - "shell syntax cannot usually be accounted for. This can result in a command " - "injection vulnerability and arbitrary code execution. Git.USE_SHELL is deprecated " - "and will be removed in a future release." -) - - -def _warn_use_shell(*, extra_danger: bool) -> None: - warnings.warn( - _USE_SHELL_DANGER_MESSAGE if extra_danger else _USE_SHELL_DEFAULT_MESSAGE, - DeprecationWarning, - stacklevel=3, - ) - - -class _GitMeta(type): - """Metaclass for :class:`Git`. - - This helps issue :class:`DeprecationWarning` if :attr:`Git.USE_SHELL` is used. - """ - - def __getattribute(cls, name: str) -> Any: - if name == "USE_SHELL": - _warn_use_shell(extra_danger=False) - return super().__getattribute__(name) - - def __setattr(cls, name: str, value: Any) -> Any: - if name == "USE_SHELL": - _warn_use_shell(extra_danger=value) - super().__setattr__(name, value) - - if not TYPE_CHECKING: - # To preserve static checking for undefined/misspelled attributes while letting - # the methods' bodies be type-checked, these are defined as non-special methods, - # then bound to special names out of view of static type checkers. (The original - # names invoke name mangling (leading "__") to avoid confusion in other scopes.) - __getattribute__ = __getattribute - __setattr__ = __setattr - - -GitMeta = _GitMeta -"""Alias of :class:`Git`'s metaclass, whether it is :class:`type` or a custom metaclass. - -Whether the :class:`Git` class has the default :class:`type` as its metaclass or uses a -custom metaclass is not documented and may change at any time. This statically checkable -metaclass alias is equivalent at runtime to ``type(Git)``. This should almost never be -used. Code that benefits from it is likely to be remain brittle even if it is used. - -In view of the :class:`Git` class's intended use and :class:`Git` objects' dynamic -callable attributes representing git subcommands, it rarely makes sense to inherit from -:class:`Git` at all. Using :class:`Git` in multiple inheritance can be especially tricky -to do correctly. Attempting uses of :class:`Git` where its metaclass is relevant, such -as when a sibling class has an unrelated metaclass and a shared lower bound metaclass -might have to be introduced to solve a metaclass conflict, is not recommended. - -:note: - The correct static type of the :class:`Git` class itself, and any subclasses, is - ``Type[Git]``. (This can be written as ``type[Git]`` in Python 3.9 later.) - - :class:`GitMeta` should never be used in any annotation where ``Type[Git]`` is - intended or otherwise possible to use. This alias is truly only for very rare and - inherently precarious situations where it is necessary to deal with the metaclass - explicitly. -""" - - -class Git(metaclass=_GitMeta): +class Git: """The Git class manages communication with the Git binary. It provides a convenient interface to calling the Git binary, such as in:: @@ -686,6 +616,7 @@ class Git(metaclass=_GitMeta): "_git_options", "_persistent_git_options", "_environment", + "_repo", ) _excluded_ = ( @@ -693,8 +624,12 @@ class Git(metaclass=_GitMeta): "cat_file_header", "_version_info", "_version_info_token", + "_repo", ) + _version_info: Optional[Tuple[int, ...]] + _version_info_token: object + # Match Git's leading transport selector, including an empty helper name. # Git also selects the command-executing ext helper for an ext:// URL. re_unsafe_protocol = re.compile(r"([A-Za-z0-9][A-Za-z0-9+.-]*|)::|ext://") @@ -725,55 +660,6 @@ def __setstate__(self, d: Dict[str, Any]) -> None: GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) """Enables debugging of GitPython's git commands.""" - USE_SHELL: bool = False - """Deprecated. If set to ``True``, a shell will be used when executing git commands. - - Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython - functions should be updated to use the default value of ``False`` instead. ``True`` - is unsafe unless the effect of syntax treated specially by the shell is fully - considered and accounted for, which is not possible under most circumstances. As - detailed below, it is also no longer needed, even where it had been in the past. - - It is in many if not most cases a command injection vulnerability for an application - to set :attr:`USE_SHELL` to ``True``. Any attacker who can cause a specially crafted - fragment of text to make its way into any part of any argument to any git command - (including paths, branch names, etc.) can cause the shell to read and write - arbitrary files and execute arbitrary commands. Innocent input may also accidentally - contain special shell syntax, leading to inadvertent malfunctions. - - In addition, how a value of ``True`` interacts with some aspects of GitPython's - operation is not precisely specified and may change without warning, even before - GitPython 4.0.0 when :attr:`USE_SHELL` may be removed. This includes: - - * Whether or how GitPython automatically customizes the shell environment. - - * Whether, outside of Windows (where :class:`subprocess.Popen` supports lists of - separate arguments even when ``shell=True``), this can be used with any GitPython - functionality other than direct calls to the :meth:`execute` method. - - * Whether any GitPython feature that runs git commands ever attempts to partially - sanitize data a shell may treat specially. Currently this is not done. - - Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows - in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as - GitPython solves that problem more robustly and safely by using the - ``CREATE_NO_WINDOW`` process creation flag on Windows. - - Because Windows path search differs subtly based on whether a shell is used, in rare - cases changing this from ``True`` to ``False`` may keep an unusual git "executable", - such as a batch file, from being found. To fix this, set the command name or full - path in the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable or pass the - full path to :func:`git.refresh` (or invoke the script using a ``.exe`` shim). - - Further reading: - - * :meth:`Git.execute` (on the ``shell`` parameter). - * https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a - * https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags - * https://github.com/python/cpython/issues/91558#issuecomment-1100942950 - * https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw - """ - _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" @@ -787,6 +673,7 @@ def __setstate__(self, d: Dict[str, Any]) -> None: """ _refresh_token = object() # Since None would match an initial _version_info_token. + _version_check_cache: Dict[Tuple[Any, ...], Tuple[int, ...]] = {} @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: @@ -960,34 +847,29 @@ def is_cygwin(cls) -> bool: @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Literal[False] = ..., expand_vars: bool = ...) -> str: ... + def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: ... @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> str: ... + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: ... @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> PathLike: + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: """Remove any backslashes from URLs to be written in config files. Windows might create config files containing paths with backslashes, but git stops liking them as it will escape the backslashes. Hence we undo the escaping just to be sure. - :param expand_vars: - Expand environment variables and an initial ``~``. Disable this for values - obtained from an untrusted source, such as remote URLs. """ if is_cygwin is None: is_cygwin = cls.is_cygwin() if is_cygwin: - url = cygpath(url, expand_vars=expand_vars) + url = cygpath(url) else: - if expand_vars: - url = os.path.expandvars(url) - if url.startswith("~"): - url = os.path.expanduser(url) + if url.startswith("~"): + url = os.path.expanduser(url) url = url.replace("\\\\", "\\").replace("\\", "/") return url @@ -1129,6 +1011,139 @@ def _option_candidates(cls, args: Sequence[Any] = (), kwargs: Optional[Mapping[s ) return options + @staticmethod + def _check_operand(value: Any, label: str = "operand") -> str: + """Validate a single name/revision, before Git can interpret it as an option. + + Paths and free-form payloads need their own validation and framing instead. + """ + value = safe_decode(value) if isinstance(value, bytes) else str(value) + if value.startswith("-") or any(char in value for char in "\0\r\n"): + raise UnsafeOptionError(f"Invalid {label}: {value!r}") + return value + + def _require_version(self) -> None: + key = None + if self._version_info_token is not self._refresh_token: + key = self._version_check_key() + if key is not None: + version = self._version_check_cache.get(key) + if version is not None: + self._version_info = version + self._version_info_token = key[0] + version = self.version_info + if version < (2, 52): + raise UnsupportedOperation("GitPython requires Git 2.52 or newer for repository operations") + if key is not None and key not in self._version_check_cache: + # ponytail: clear at 128 contexts; use LRU if diverse environments churn. + if len(self._version_check_cache) >= 128: + self._version_check_cache.clear() + self._version_check_cache[key] = version + + def _version_check_key(self) -> Optional[Tuple[Any, ...]]: + """Identify the executable and context of a minimum-version probe.""" + executable = self.GIT_PYTHON_GIT_EXECUTABLE + if not executable or self._git_options or self._persistent_git_options: + return None + environment = {**os.environ, "LANGUAGE": "C", "LC_ALL": "C", **self._environment} + try: + cwd = os.path.realpath(self._working_dir or os.getcwd()) + if not os.access(cwd, os.X_OK): + return None + resolved: Optional[str] + if os.path.isabs(executable): + resolved = executable + elif sys.platform == "win32": + # CreateProcess searches in the parent context, unlike execvpe. + return None + elif os.path.dirname(executable): + resolved = os.path.join(cwd, executable) + else: + path = environment.get("PATH") + path = os.defpath if path is None else path + resolved = shutil.which( + executable, path=os.pathsep.join(os.path.join(cwd, p) for p in path.split(os.pathsep)) + ) + if resolved is None: + return None + info = os.stat(resolved) + except OSError: + return None + return ( + self._refresh_token, + type(self), + executable, + resolved, + cwd, + tuple(sorted(environment.items())), + (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns), + ) + + def _call_process_safe( + self, + method: str, + *args: Any, + _allow_hooks: bool = False, + _allow_network: bool = False, + _config: Sequence[str] = (), + **kwargs: Any, + ) -> Any: + """Run library-owned plumbing without introducing executable configuration. + + Callers validate their operands and any forwarded options using the existing + command-specific guards. This does not restrict the public raw Git interface. + """ + self._check_operand(method, "command") + if kwargs.get("shell"): + raise UnsafeOptionError("Library operations cannot run through a shell") + if _allow_hooks and method != "hook": + raise UnsafeOptionError("Only explicit hook operations may enable hooks") + insertion = kwargs.get("insert_kwargs_after") + if insertion is not None and not ( + method == "remote" and args and args[0] == insertion and insertion in ("add", "set-url", "update") + ): + raise UnsafeOptionError("Library command options cannot be reordered past safety flags") + for setting in _config: + key, separator, value = setting.partition("=") + if not separator or ( + key.lower() not in ("i18n.commitencoding", "diff.mnemonicprefix", "fetch.output", "core.abbrev") + and not (key in ("core.protectHFS", "core.protectNTFS") and value == "true") + and not (key == "protocol.file.allow" and value in ("always", "never", "user")) + and not (key in ("tar.tgz.command", "tar.tar.gz.command") and value == "git archive gzip") + ): + raise UnsafeOptionError(f"Unsupported internal Git configuration: {key!r}") + for arg in self._unpack_args([arg for arg in args if arg is not None]) + self.transform_kwargs( + **{key: value for key, value in kwargs.items() if key not in execute_kwargs} + ): + if "\0" in arg: + raise UnsafeOptionError("Git arguments cannot contain NUL bytes") + options = ["--no-pager", "--no-optional-locks"] + # These settings would become visible as user configuration in `config`. + if method != "config": + config = ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false"] + if not _allow_hooks: + config.append(f"core.hooksPath={os.devnull}") + for setting in [*config, *_config]: + options.extend(("-c", setting)) + elif _config: + raise ValueError("Configuration queries must not include synthetic settings") + native = _backend.dispatch(self, method, args, kwargs, _config) + if native is not NotImplemented: + return native + self._require_version() + env = dict(kwargs.pop("env", {}) or {}) + env.update(LC_ALL="C", LANGUAGE="C") + if not _allow_network: + env.update(GIT_NO_LAZY_FETCH="1", GIT_TERMINAL_PROMPT="0") + return self._call_process( + method, + *args, + _safe_git_options=options, + shell=False, + env=env, + **{key: value for key, value in kwargs.items() if key != "shell"}, + ) + AutoInterrupt: TypeAlias = _AutoInterrupt CatFileContentStream: TypeAlias = _CatFileContentStream @@ -1148,21 +1163,17 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: self._persistent_git_options: List[str] = [] # Extra environment variables to pass to git commands - self._environment: Dict[str, str] = {} + self._environment: Dict[str, Optional[str]] = {} + self._repo: Any = None # Weak reference; the Repo owns native resources. # Cached version slots - self._version_info: Union[Tuple[int, ...], None] = None - self._version_info_token: object = None + self._version_info = None + self._version_info_token = None # Cached command slots self.cat_file_header: Union[None, TBD] = None self.cat_file_all: Union[None, TBD] = None - def __getattribute__(self, name: str) -> Any: - if name == "USE_SHELL": - _warn_use_shell(extra_danger=False) - return super().__getattribute__(name) - def __getattr__(self, name: str) -> Any: """A convenience method as it allows to call the command as if it was an object. @@ -1232,7 +1243,7 @@ def version_info(self) -> Tuple[int, ...]: return self._version_info # Run "git version" and parse it. - process_version = self._call_process("version") + process_version = cast(str, self._call_process("version", shell=False)) version_string = process_version.split(" ")[2] version_fields = version_string.split(".")[:4] leading_numeric_fields = itertools.takewhile(str.isdigit, version_fields) @@ -1378,11 +1389,11 @@ def execute( Windows, the program parses the arguments itself, so multi-word strings can work but are not portable. - Avoid ``shell=True`` (and :attr:`Git.USE_SHELL`): this runs the command in + Avoid ``shell=True``: this runs the command in a shell, which is generally unsafe. The shell interprets metacharacters such as ``;``, ``|``, ``&``, ``$(...)``, ``$VAR``, ``%VAR%``, and ``^`` (depending on the platform) as syntax. Any untrusted text in the command - can then execute arbitrary OS commands. See :attr:`Git.USE_SHELL`. + can then execute arbitrary OS commands. Producing a sequence automatically by :func:`shlex.split` and passing it as the command is far safer than ``shell=True``. But :func:`shlex.split` @@ -1450,7 +1461,7 @@ def execute( :param shell: Whether to invoke commands through a shell (see :class:`Popen(..., shell=True) `). - If this is not ``None``, it overrides :attr:`USE_SHELL`. + Use a shell explicitly; the default is ``False``. Passing ``shell=True`` to this or any other GitPython function should be avoided, as it is unsafe under most circumstances. This is because it is @@ -1517,16 +1528,18 @@ def execute( # Start the process. inline_env = env - env = os.environ.copy() + environment: Dict[str, Optional[str]] = dict(os.environ) # Attempt to force all output to plain ASCII English, which is what some parsing # code may expect. # According to https://askubuntu.com/a/311796, we are setting LANGUAGE as well # just to be sure. - env["LANGUAGE"] = "C" - env["LC_ALL"] = "C" - env.update(self._environment) + environment["LANGUAGE"] = "C" + environment["LC_ALL"] = "C" + environment.update(self._environment) if inline_env is not None: - env.update(inline_env) + environment.update(inline_env) + # Internal or per-call None overrides remove inherited variables. + env = {key: value for key, value in environment.items() if value is not None} if sys.platform == "win32": if kill_after_timeout is not None: @@ -1541,12 +1554,7 @@ def execute( stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: - # Get the value of USE_SHELL with no deprecation warning. Do this without - # warnings.catch_warnings, to avoid a race condition with application code - # configuring warnings. The value could be looked up in type(self).__dict__ - # or Git.__dict__, but those can break under some circumstances. This works - # the same as self.USE_SHELL in more situations; see Git.__getattribute__. - shell = super().__getattribute__("USE_SHELL") + shell = False _logger.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, @@ -1572,6 +1580,10 @@ def execute( except cmd_not_found_exception as err: raise GitCommandNotFound(redacted_command, err) from err else: + _backend.record("Git.execute", "CLI process") + owner = self._repo() if self._repo is not None else None + if owner is not None: + owner._gix_state = None # Replace with a typeguard for Popen[bytes]? proc.stdout = cast(BinaryIO, proc.stdout) proc.stderr = cast(BinaryIO, proc.stderr) @@ -1692,7 +1704,7 @@ def as_text(stdout_value: Union[bytes, str, None]) -> str: else: return stdout_value - def environment(self) -> Dict[str, str]: + def environment(self) -> Dict[str, Optional[str]]: return self._environment def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: @@ -1776,7 +1788,7 @@ def _unpack_args(cls, arg_list: Sequence[Any]) -> List[str]: for arg in arg_list: outlist.extend(cls._unpack_args(arg)) else: - outlist.append(str(arg_list)) + outlist.append(os.fsdecode(arg_list) if isinstance(arg_list, os.PathLike) else str(arg_list)) return outlist @@ -1861,6 +1873,7 @@ def _call_process( """ # Handle optional arguments prior to calling transform_kwargs. # Otherwise these'll end up in args, which is bad. + safe_git_options = kwargs.pop("_safe_git_options", ()) exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} @@ -1894,12 +1907,14 @@ def _call_process( call.extend(self._git_options) self._git_options = () + call.extend(safe_git_options) + call.append(dashify(method)) call.extend(args_list) return self.execute(call, **exec_kwargs) - def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: + def _parse_object_header(self, header_line: Union[str, bytes]) -> Tuple[str, str, int]: """ :param header_line: A line of the form:: @@ -1912,6 +1927,8 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :raise ValueError: If the header contains indication for an error due to incorrect input sha. """ + if isinstance(header_line, bytes): + header_line = header_line.decode("ascii", "replace") tokens = header_line.split() if len(tokens) != 3: if not tokens: @@ -1926,48 +1943,56 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: # END handle actual return value # END error handling - if len(tokens[0]) != 40: + if ( + not re.fullmatch(r"[0-9a-fA-F]+", tokens[0]) + or len(tokens[0]) % 2 + or tokens[1] not in ("blob", "tree", "commit", "tag") + or not tokens[2].isdigit() + ): raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) def _prepare_ref(self, ref: object) -> bytes: - # Required for command to separate refs on stdin, as bytes. + # `cat-file -Z` separates both requests and responses with NUL, so paths + # containing newlines cannot inject requests or desynchronize the process. if isinstance(ref, bytes): - # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text. - refstr: str = ref.decode("ascii") + refstr: str = ref.decode(defenc, "surrogateescape") elif not isinstance(ref, str): refstr = str(ref) # Could be ref-object. else: refstr = ref - # A line feed terminates a request, so one object name must be one line. An - # embedded one would queue a second request on the persistent command while - # only one response line is read back, leaving every later call one response - # behind, answered with the header of an object it did not ask for. - if refstr.endswith("\n"): - refstr = refstr[:-1] - if "\n" in refstr: - raise ValueError("Object name %r contains a line feed" % refstr) - return (refstr + "\n").encode(defenc) + if "\0" in refstr or refstr.startswith("-"): + raise UnsafeOptionError("Object queries cannot contain NUL or start with '-'") + return refstr.encode(defenc, "surrogateescape") + b"\0" def _get_persistent_cmd(self, attr_name: str, cmd_name: str, *args: Any, **kwargs: Any) -> "Git.AutoInterrupt": cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val - options = {"istream": PIPE, "as_process": True} + options: Dict[str, Any] = {"istream": PIPE, "as_process": True} options.update(kwargs) - cmd = self._call_process(cmd_name, *args, **options) + cmd = self._call_process_safe(cmd_name, *args, **options) setattr(self, attr_name, cmd) cmd = cast("Git.AutoInterrupt", cmd) return cmd - def __get_object_header(self, cmd: "Git.AutoInterrupt", ref: Union[str, bytes]) -> Tuple[str, str, int]: + def __get_object_header(self, cmd: "Git.AutoInterrupt", request: bytes) -> Tuple[str, str, int]: if cmd.stdin and cmd.stdout: - cmd.stdin.write(self._prepare_ref(ref)) + cmd.stdin.write(request) cmd.stdin.flush() - return self._parse_object_header(cmd.stdout.readline()) + header = bytearray() + while True: + char = cmd.stdout.read(1) + if not char: + cmd.wait() + raise ValueError("Git closed the object stream before its response") + if char == b"\0": + break + header.extend(char) + return self._parse_object_header(bytes(header)) else: raise ValueError("cmd stdin was empty") @@ -1982,8 +2007,12 @@ def get_object_header(self, ref: Union[str, bytes]) -> Tuple[str, str, int]: :return: (hexsha, type_string, size_as_int) """ - cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) - return self.__get_object_header(cmd, ref) + request = self._prepare_ref(ref) + native = _backend.object_data(self, request[:-1]) + if native is not NotImplemented: + return native + cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True, Z=True) + return self.__get_object_header(cmd, request) def get_object_data(self, ref: Union[str, bytes]) -> Tuple[str, str, int, bytes]: """Similar to :meth:`get_object_header`, but returns object data as well. @@ -2009,8 +2038,12 @@ def stream_object_data(self, ref: Union[str, bytes]) -> Tuple[str, str, int, "Gi This method is not threadsafe. You need one independent :class:`Git` instance per thread to be safe! """ - cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) - hexsha, typename, size = self.__get_object_header(cmd, ref) + request = self._prepare_ref(ref) + native = _backend.object_data(self, request[:-1], stream=True) + if native is not NotImplemented: + return native + cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True, Z=True) + hexsha, typename, size = self.__get_object_header(cmd, request) cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() return (hexsha, typename, size, self.CatFileContentStream(size, cmd_stdout)) diff --git a/git/compat.py b/git/compat.py index d7d9a55a9..2fb1c8474 100644 --- a/git/compat.py +++ b/git/compat.py @@ -11,9 +11,7 @@ """ import locale -import os import sys -import warnings from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 @@ -24,9 +22,7 @@ AnyStr, Dict, # noqa: F401 IO, # noqa: F401 - List, Optional, - TYPE_CHECKING, Tuple, # noqa: F401 Type, # noqa: F401 Union, @@ -36,72 +32,6 @@ # --------------------------------------------------------------------------- -_deprecated_platform_aliases = { - "is_win": os.name == "nt", - "is_posix": os.name == "posix", - "is_darwin": sys.platform == "darwin", -} - - -def _getattr(name: str) -> Any: - try: - value = _deprecated_platform_aliases[name] - except KeyError: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None - - warnings.warn( - f"{__name__}.{name} and other is_ aliases are deprecated. " - "Write the desired os.name or sys.platform check explicitly instead.", - DeprecationWarning, - stacklevel=2, - ) - return value - - -if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. - __getattr__ = _getattr - - -def __dir__() -> List[str]: - return [*globals(), *_deprecated_platform_aliases] - - -is_win: bool -"""Deprecated alias for ``os.name == "nt"`` to check for native Windows. - -This is deprecated because it is clearer to write out :attr:`os.name` or -:attr:`sys.platform` checks explicitly, especially in cases where it matters which is -used. - -:note: - ``is_win`` is ``False`` on Cygwin, but is often wrongly assumed ``True``. To detect - Cygwin, use ``sys.platform == "cygwin"``. -""" - -is_posix: bool -"""Deprecated alias for ``os.name == "posix"`` to check for Unix-like ("POSIX") systems. - -This is deprecated because it clearer to write out :attr:`os.name` or -:attr:`sys.platform` checks explicitly, especially in cases where it matters which is -used. - -:note: - For POSIX systems, more detailed information is available in :attr:`sys.platform`, - while :attr:`os.name` is always ``"posix"`` on such systems, including macOS - (Darwin). -""" - -is_darwin: bool -"""Deprecated alias for ``sys.platform == "darwin"`` to check for macOS (Darwin). - -This is deprecated because it clearer to write out :attr:`os.name` or -:attr:`sys.platform` checks explicitly. - -:note: - For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while - ``sys.platform == "darwin"``. -""" - defenc = sys.getfilesystemencoding() """The encoding used to convert between Unicode and bytes filenames.""" diff --git a/git/config.py b/git/config.py index 732d3658b..faaddf3f6 100644 --- a/git/config.py +++ b/git/config.py @@ -3,140 +3,31 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Parser for reading and writing configuration files.""" +"""Git configuration access through ``git config``.""" __all__ = ["GitConfigParser", "SectionConstraint"] -import abc import configparser as cp -import fnmatch -import inspect -import logging import os import os.path as osp import re import sys -from functools import wraps -from io import BufferedReader, IOBase - -# typing------------------------------------------------------- -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, - cast, -) +import tempfile +from contextlib import contextmanager +from typing import Any, Dict, Generic, Iterator, List, OrderedDict, Sequence, Tuple, TypeVar, Union, TYPE_CHECKING from git.compat import defenc, force_text +from git.exc import GitCommandError from git.types import _T, ConfigLevels_Tup, Lit_config_levels, PathLike, assert_never -from git.util import LockFile if TYPE_CHECKING: from io import BytesIO - from git.repo.base import Repo T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser") T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool, None) - OrderedDict_OMD = OrderedDict[str, List[T_OMD_value]] - -# ------------------------------------------------------------- - -_logger = logging.getLogger(__name__) - CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") -"""The configuration level of a configuration file.""" - -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeif )\"(gitdir|gitdir/i|onbranch|hasconfig:remote\.\*\.url):(.+)\"") -"""Section pattern to detect conditional includes. - -See: https://git-scm.com/docs/git-config#_conditional_includes -""" - -UNSAFE_CONFIG_CHARS_RE = re.compile(r"[\r\n\x00]") -"""Characters that cannot be safely written in config names or values.""" - -VALID_CONFIG_OPTION_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") -"""Pattern for option names that can be written without changing config syntax.""" - -UNQUOTED_ESCAPES = {"b": "\b", "n": "\n", "t": "\t", '"': '"', "\\": "\\"} -"""Escape sequences git recognizes in an unquoted value. - -Any other backslash escape is an error for git, which is why a value holding one is -left undecoded rather than rejected. -""" - - -class MetaParserBuilder(abc.ABCMeta): # noqa: B024 - """Utility class wrapping base-class methods into decorators that assure read-only - properties.""" - - def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": - """Equip all base-class methods with a needs_values decorator, and all non-const - methods with a :func:`set_dirty_and_flush_changes` decorator in addition to - that. - """ - kmm = "_mutating_methods_" - if kmm in clsdict: - mutating_methods = clsdict[kmm] - for base in bases: - methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for method_name, method in methods: - if method_name in clsdict: - continue - method_with_values = needs_values(method) - if method_name in mutating_methods: - method_with_values = set_dirty_and_flush_changes(method_with_values) - # END mutating methods handling - - clsdict[method_name] = method_with_values - # END for each name/method pair - # END for each base - # END if mutating methods configuration is set - - new_type = super().__new__(cls, name, bases, clsdict) - return new_type - - -def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method for ensuring we read values (on demand) before we try to access - them.""" - - @wraps(func) - def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: - self.read() - return func(self, *args, **kwargs) - - # END wrapper method - return assure_data_present - - -def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method that checks whether given non constant function may be called. - - If so, the instance will be set dirty. Additionally, we flush the changes right to - disk. - """ - - def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: - rval = non_const_func(self, *args, **kwargs) - self._dirty = True - self.write() - return rval - - # END wrapper method - flush_changes.__name__ = non_const_func.__name__ - return flush_changes class SectionConstraint(Generic[T_ConfigParser]): @@ -153,6 +44,10 @@ class SectionConstraint(Generic[T_ConfigParser]): _valid_attrs_ = ( "get_value", + "get_values", + "add_value", + "items", + "items_all", "set_value", "get", "set", @@ -298,60 +193,19 @@ def get_config_path(config_level: Lit_config_levels) -> str: ) -class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): - """Implements specifics required to read git style configuration files. - - This variation behaves much like the :manpage:`git-config(1)` command, such that the - configuration will be read on demand based on the filepath given during - initialization. - - The changes will automatically be written once the instance goes out of scope, but - can be triggered manually as well. - - The configuration file will be locked if you intend to change values preventing - other instances to write concurrently. - - :note: - Section and option names are case-insensitive; quoted subsection names are - case-sensitive. Names retain their first spelling when enumerated or written. - Case variants are merged, preserving all values in the order they are read. - - :note: - If used as a context manager, this will release the locked file. - - :note: - Options without a value are stored as ``None`` and written without ``=``. - :meth:`get_value` and :meth:`get_values` return an empty string for them, - while :meth:`getboolean` returns ``True``. An explicit empty value is - stored as an empty string and reads as ``False`` with :meth:`getboolean`. - """ +class GitConfigParser: + """Read and modify Git configuration using Git's parser and file locking. - # { Configuration - t_lock = LockFile - """The lock type determines the type of lock to use in new configuration readers. + File paths, byte streams, and lists of sources are accepted for reading. Writers + operate on one source, updating it immediately. Git locks each mutation; a writer + does not reserve a lifetime lock. Git canonicalizes enumerated section/option + names, while preserving subsection case and duplicate values. - They must be compatible to the :class:`~git.util.LockFile` interface. - A suitable alternative would be the :class:`~git.util.BlockingLockFile`. + Empty sections, raw configuration parsing/serialization, and writing valueless + options are not supported. Existing valueless options remain readable. """ - re_comment = re.compile(r"^\s*[#;]") - # } END configuration - - optvalueonly_source = r"\s*(?P