ignore files ignored by .gitignore, allow source tree merge (not just 1 level) (#15)

Author: gaborbernat

.gitignore

@@ -20,7 +20,7 @@ lib64/
 parts/
 sdist/
 var/
-*.egg-info/
+*.egg-info
 .installed.cfg
 *.egg
 
@@ -36,7 +36,7 @@ pip-delete-this-directory.txt
 
 # Unit test / coverage reports
 htmlcov/
-.tox/
+.tox
 .coverage
 .coverage.*
 .cache
@@ -64,4 +64,4 @@ target/
 typed-src/
 
 pip-wheel-metadata
-.mypy_cache
+.mypy_cache

retype.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 """Re-apply type annotations from .pyi stubs to your codebase."""
 
+import os
 import re
 import sys
 import threading
@@ -15,6 +16,7 @@
 from pathlib import Path
 
 import click
+from pathspec import PathSpec
 from typed_ast import ast3
 
 __version__ = "19.5.0"
@@ -99,11 +101,18 @@ def retype_path(
 ):
     """Recursively retype files or directories given. Generate errors."""
     if src.is_dir():
-        for child in src.iterdir():
-            if child == pyi_dir or child == targets:
-                continue
+        extra_ignore = []
+        for folder in [pyi_dir, targets]:
+            try:
+                extra_ignore.append("/{}".format(folder.relative_to(src)))
+            except ValueError:
+                pass
+        for file in walk_not_git_ignored(
+            src, lambda p: p.suffix == ".py", extra_ignore
+        ):
+            nested = file.relative_to(src).parent
             yield from retype_path(
-                child, pyi_dir / src.name, targets / src.name, quiet=quiet, hg=hg
+                file, pyi_dir / nested, targets / nested, quiet=quiet, hg=hg
             )
     elif src.suffix == ".py" or src_explicitly_given:
         try:
@@ -1409,6 +1418,58 @@ def new(n, prefix=None):
     return n
 
 
+def _load_ignore(at_path, parent_spec, ignores):
+    ignore_file = at_path / ".gitignore"
+    if not ignore_file.exists():
+        return parent_spec
+    lines = ignore_file.read_text().split(os.linesep)
+    spec = PathSpec.from_lines("gitwildmatch", lines)
+    spec = PathSpec(parent_spec.patterns + spec.patterns)
+    ignores[at_path] = spec
+    return spec
+
+
+def walk_not_git_ignored(path, keep, extra_ignore):
+    path = path.absolute()
+    spec = PathSpec.from_lines("gitwildmatch", [".git"] + extra_ignore)
+    ignores = {}
+    # detect git folder, collect ignores up to root
+    at = path
+    while True:
+        git_exist = (at / ".git").exists()
+        if git_exist:
+            ignores[at.parent] = spec
+            # go down back and load all ignores
+            for part in (".",) + path.relative_to(at).parts:
+                at = at / part
+                spec = _load_ignore(at, spec, ignores)
+            break
+        if at == at.parent:
+            ignores[path] = spec
+            break
+        at = at.parent
+
+    # now walk from root, collect new ignores and evaluate
+    for root, dirs, files in os.walk(str(path)):
+        root_path = Path(root).relative_to(path)  # current path
+        current_path = path / root
+        parent_spec = ignores.get(current_path) or next(
+            ignores[p] for p in current_path.parents if p in ignores
+        )
+        spec = _load_ignore(Path(root), parent_spec, ignores)
+        for file_name in files:
+            result = root_path / file_name
+            if (
+                file_name != ".gitignore"
+                and keep(result)
+                and not spec.match_file(str(result))
+            ):
+                yield path / result
+        for cur_dir in list(dirs):
+            if spec.match_file(str(root_path / cur_dir)):
+                dirs.remove(cur_dir)
+
+
 _as = Leaf(token.NAME, "as", prefix=" ")
 _colon = Leaf(token.COLON, ":")
 _comma = Leaf(token.COMMA, ",")

setup.py

@@ -63,7 +63,7 @@
         )
     ],
     zip_safe=False,
-    install_requires=["click", "typed-ast"],
+    install_requires=["click", "typed-ast", "pathspec >= 0.5.9, <1"],
     test_suite="tests.test_retype",
     classifiers=[
         "Development Status :: 3 - Alpha",

tests/test_discovery.py

@@ -0,0 +1,56 @@
+from collections import namedtuple
+
+import pytest
+
+from retype import walk_not_git_ignored
+
+
+@pytest.fixture()
+def build(tmp_path):
+    def _build(files):
+        for file, content in files.items():
+            dest = tmp_path / file
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(content, encoding="utf-8")
+        return tmp_path
+
+    return _build
+
+
+Case = namedtuple("Case", ["files", "found", "cwd"])
+WALK_TESTS = {
+    "no_ignore_root": Case({"a.py": ""}, ["a.py"], "."),
+    "no_ignore_root_within": Case({"b/a.py": ""}, ["b/a.py"], "."),
+    "no_ignore_keep_py_only": Case(
+        {"a.py": "", "a.pyi": "", "b/a.pyi": "", "b/a.py": ""}, ["a.py", "b/a.py"], "."
+    ),
+    "ignore_py_at_root": Case(
+        {".gitignore": "*.py", "a.py": "", "b/a.py": ""}, [], "."
+    ),
+    "ignore_py_nested": Case(
+        {"a.py": "", "b/a.py": "", "b/.gitignore": "*.py"}, ["a.py"], "."
+    ),
+    "git_nested_no_res": Case(
+        {".git/demo": "", ".gitignore": "*.py", "b/c/d/a.py": ""}, [], "b/c/d"
+    ),
+    "git_nested_has_res": Case(
+        {".git/demo": "", "b/c/a.py": "", "b/c/d/.gitignore": "*.py", "b/c/d/a.py": ""},
+        ["a.py"],
+        "b/c",
+    ),
+}
+
+
+@pytest.mark.parametrize("case", WALK_TESTS.values(), ids=list(WALK_TESTS.keys()))
+def test_walk(case: Case, build, monkeypatch):
+    path = build(case.files)
+    dest = path / case.cwd
+    monkeypatch.chdir(dest)
+    result = [
+        str(f.relative_to(dest))
+        for f in walk_not_git_ignored(
+            dest, lambda p: p.suffix == ".py", extra_ignore=[]
+        )
+    ]
+    expected = [str(f) for f in case.found]
+    assert result == expected

tox.ini

@@ -50,7 +50,7 @@ force_grid_wrap = 0
 use_parentheses = True
 line_length = 88
 known_first_party = retype
-known_third_party = click,setuptools,typed_ast
+known_third_party = click,pathspec,pytest,setuptools,typed_ast
 
 [testenv:lint]
 description = run the flake8 checker
@@ -72,11 +72,8 @@ description = try to merge our types against our types
 deps = pip >= 19.1.1
        mypy == 0.701
 changedir = {envtmpdir}
-commands = python -c 'import glob; import shutil; from pathlib import Path; \
-           [print(f, (Path()/"out"/shutil.copy(f, ".")).with_suffix(".py").absolute()) \
-           for f in glob.glob(r"{toxinidir}/*.py")]'
-           python '{envsitepackagesdir}/retype.py' -p "{toxinidir}/types" -t out .
-           mypy out --strict --ignore-missing-imports
+commands = python '{envsitepackagesdir}/retype.py' -p '{toxinidir}/types' -t {envtmpdir} {toxinidir}
+           mypy {envtmpdir} --strict --ignore-missing-imports {posargs}
 
 [testenv:package_readme]
 description = check that the long description is valid (need for PyPi)
@@ -89,7 +86,6 @@ commands = python -m pep517.build --out-dir {envtmpdir}/build --binary .
 
 [testenv:dev]
 description = generate a DEV environment
-deps = pip >= 19.1.1
 usedevelop = True
 commands = python -m pip list --format=columns
            python -c 'import sys; print(sys.executable)'

types/retype.pyi

@@ -2,6 +2,8 @@ from lib2to3.pytree import Leaf, Node
 from pathlib import Path
 from typing import (
     Callable,
+    Dict,
+    Generator,
     Iterable,
     Iterator,
     List,
@@ -12,6 +14,7 @@ from typing import (
     Union,
 )
 
+from pathspec import PathSpec
 from typed_ast import ast3
 
 _LN = Union[Node, Leaf]
@@ -149,3 +152,9 @@ def _dn_call(call: ast3.Call) -> List[str]: ...
 def _dn_attribute(attr: ast3.Attribute) -> List[str]: ...
 def _nuin_node(node: Node, name: Leaf) -> bool: ...
 def _nuin_leaf(leaf: Leaf, name: Leaf) -> bool: ...
+def walk_not_git_ignored(
+    path: Path, keep: Callable[[Path], bool], extra_ignore: List[str]
+) -> Generator[Path, None, None]: ...
+def _load_ignore(
+    at_path: Path, parent_spec: PathSpec, ignores: Dict[Path, PathSpec]
+) -> PathSpec: ...

types/tests/test_discovery.pyi

@@ -0,0 +1,15 @@
+from pathlib import Path
+from typing import Callable, Dict, NamedTuple
+
+from _pytest.monkeypatch import MonkeyPatch
+
+Case: NamedTuple = ...
+
+WALK_TESTS: Dict[str, Case]
+
+def build(tmp_path: Path) -> Callable[[Dict[str, str]], Path]:
+    def _build(files: Dict[str, str]) -> Path: ...
+
+def test_walk(
+    case: Case, build: Callable[[Dict[str, str]], Path], monkeypatch: MonkeyPatch
+) -> None: ...

types/tests/test_retype.pyi

@@ -5,16 +5,29 @@ _E = TypeVar("_E", bound=Exception)
 
 class RetypeTestCase(TestCase):
     def assertReapply(
-        self, pyi_txt: str, src_txt: str, expected_txt: str, *, incremental: bool
+        self,
+        pyi_txt: str,
+        src_txt: str,
+        expected_txt: str,
+        *,
+        incremental: bool = ...,
+        replace_any: bool = ...,
     ) -> None: ...
     def assertReapplyVisible(
-        self, pyi_txt: str, src_txt: str, expected_txt: str, *, incremental: bool
+        self,
+        pyi_txt: str,
+        src_txt: str,
+        expected_txt: str,
+        *,
+        incremental: bool = ...,
+        replace_any: bool = ...,
     ) -> None: ...
     def assertReapplyRaises(
         self,
         pyi_txt: str,
         src_txt: str,
         expected_exception: Type[_E],
         *,
-        incremental: bool,
+        incremental: bool = ...,
+        replace_any: bool = ...,
     ) -> _E: ...