Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make --exclude only apply to recursively found files #1591

Merged
merged 3 commits into from
Aug 13, 2020
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 47 additions & 23 deletions src/black/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,7 @@ def get_sources(
exclude_regexes = [exclude_regex]
if force_exclude_regex is not None:
exclude_regexes.append(force_exclude_regex)
gitignore = get_gitignore(root)

for s in src:
p = Path(s)
Expand All @@ -597,17 +598,27 @@ def get_sources(
include_regex,
exclude_regexes,
report,
get_gitignore(root),
gitignore,
)
)
elif s == "-":
sources.add(p)
elif p.is_file():
sources.update(
gen_python_files(
[p], root, None, exclude_regexes, report, get_gitignore(root)
)
)
# Hard-exclude any files that matches the `--force-exclude` regex.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment makes more sense on line 613 below

normalized_path = normalize_path_maybe_ignore(p, root, report)
if normalized_path is None:
continue

normalized_path = "/" + normalized_path
if force_exclude_regex:
force_exclude_match = force_exclude_regex.search(normalized_path)
else:
force_exclude_match = None
if force_exclude_match and force_exclude_match.group(0):
report.path_ignored(p, "matches the --force-exclude regular expression")
continue

sources.add(p)
else:
err(f"invalid path: {s}")
return sources
Expand Down Expand Up @@ -5759,6 +5770,29 @@ def get_gitignore(root: Path) -> PathSpec:
return PathSpec.from_lines("gitwildmatch", lines)


def normalize_path_maybe_ignore(
path: Path, root: Path, report: "Report"
) -> Optional[str]:
"""Normalize `path`. May return `None` if `path` was ignored.

`report` is where "path ignored" output goes.
"""
try:
normalized_path = path.resolve().relative_to(root).as_posix()
except OSError as e:
report.path_ignored(path, f"cannot be read because {e}")
return None

except ValueError:
if path.is_symlink():
report.path_ignored(path, f"is a symbolic link that points outside {root}")
return None

raise

return normalized_path


def gen_python_files(
paths: Iterable[Path],
root: Path,
Expand All @@ -5767,44 +5801,34 @@ def gen_python_files(
report: "Report",
gitignore: PathSpec,
) -> Iterator[Path]:
"""Generate all files under `path` whose paths are not excluded by the
`exclude` regex, but are included by the `include` regex.
"""Generate all files under `path` whose paths are not excluded by
`exclude_regexes`, but are included by the `include` regex.

Symbolic links pointing outside of the `root` directory are ignored.

`report` is where output about exclusions goes.
"""
assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
for child in paths:
# Then ignore with `exclude` option.
try:
normalized_path = child.resolve().relative_to(root).as_posix()
except OSError as e:
report.path_ignored(child, f"cannot be read because {e}")
normalized_path = normalize_path_maybe_ignore(child, root, report)
if normalized_path is None:
continue
except ValueError:
if child.is_symlink():
report.path_ignored(
child, f"is a symbolic link that points outside {root}"
)
continue

raise

# First ignore files matching .gitignore
if gitignore.match_file(normalized_path):
report.path_ignored(child, "matches the .gitignore file content")
continue

# Then ignore with `exclude` and `--force-exclude` options.
normalized_path = "/" + normalized_path
if child.is_dir():
normalized_path += "/"

is_excluded = False
for exclude in exclude_regexes:
for opt_name, exclude in zip(("--exclude", "--force-exclude"), exclude_regexes):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes me a bit uneasy because it assumes that exclude_regexes exactly matches these two flags, when the type doesn't demand that. It would be cleaner to make exclude_regex and force_exclude_regex separate arguments to this function, or to make exclude_regexes an iterable of (option name, pattern) pairs.

Copy link
Collaborator Author

@ichard26 ichard26 Aug 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is not my cleanest code ever as it's also using the behaviour of zip ignoring trailing unmatched values. I have already thought about moving the force exclude regex into its own argument but I decided not doing so as it would have made more sense when --extend-exclude is implemented. No problems doing this now than later!

exclude_match = exclude.search(normalized_path) if exclude else None
if exclude_match and exclude_match.group(0):
report.path_ignored(child, "matches the --exclude regular expression")
report.path_ignored(child, f"matches the {opt_name} regular expression")
is_excluded = True
break
if is_excluded:
Expand Down
49 changes: 39 additions & 10 deletions tests/test_black.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,20 @@ def skip_if_exception(e: str) -> Iterator[None]:
raise


class FakeContext(click.Context):
"""A fake click Context for when calling functions that need it."""

def __init__(self) -> None:
self.default_map: Dict[str, Any] = {}


class FakeParameter(click.Parameter):
"""A fake click Parameter for when calling functions that need it."""

def __init__(self) -> None:
pass


class BlackRunner(CliRunner):
"""Modify CliRunner so that stderr is not merged with stdout.

Expand Down Expand Up @@ -1556,6 +1570,31 @@ def test_include_exclude(self) -> None:
)
self.assertEqual(sorted(expected), sorted(sources))

@patch("black.find_project_root", lambda *args: THIS_DIR.resolve())
def test_exclude_for_issue_1572(self) -> None:
# Exclude shouldn't touch files that were explicitly given to Black through the
# CLI. Exclude is supposed to only apply to the recursive discovery of files.
# https://github.com/psf/black/issues/1572
path = THIS_DIR / "data" / "include_exclude_tests"
include = ""
exclude = r"/exclude/|a\.py"
src = str(path / "b/exclude/a.py")
report = black.Report()
expected = [Path(path / "b/exclude/a.py")]
sources = list(
black.get_sources(
ctx=FakeContext(),
src=(src,),
quiet=True,
verbose=False,
include=include,
exclude=exclude,
force_exclude=None,
report=report,
)
)
self.assertEqual(sorted(expected), sorted(sources))

def test_gitignore_exclude(self) -> None:
path = THIS_DIR / "data" / "include_exclude_tests"
include = re.compile(r"\.pyi?$")
Expand Down Expand Up @@ -1777,16 +1816,6 @@ def test_parse_pyproject_toml(self) -> None:

def test_read_pyproject_toml(self) -> None:
test_toml_file = THIS_DIR / "test.toml"

# Fake a click context and parameter so mypy stays happy
class FakeContext(click.Context):
def __init__(self) -> None:
self.default_map: Dict[str, Any] = {}

class FakeParameter(click.Parameter):
def __init__(self) -> None:
pass

fake_ctx = FakeContext()
black.read_pyproject_toml(
fake_ctx, FakeParameter(), str(test_toml_file),
Expand Down