Spaces:
Sleeping
Sleeping
from functools import partial | |
from glob import glob | |
from distutils.util import convert_path | |
import distutils.command.build_py as orig | |
import os | |
import fnmatch | |
import textwrap | |
import io | |
import distutils.errors | |
import itertools | |
import stat | |
import warnings | |
from pathlib import Path | |
from typing import Dict, Iterable, Iterator, List, Optional, Tuple | |
from setuptools._deprecation_warning import SetuptoolsDeprecationWarning | |
from setuptools.extern.more_itertools import unique_everseen | |
def make_writable(target): | |
os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE) | |
class build_py(orig.build_py): | |
"""Enhanced 'build_py' command that includes data files with packages | |
The data files are specified via a 'package_data' argument to 'setup()'. | |
See 'setuptools.dist.Distribution' for more details. | |
Also, this version of the 'build_py' command allows you to specify both | |
'py_modules' and 'packages' in the same setup operation. | |
""" | |
editable_mode: bool = False | |
existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. | |
def finalize_options(self): | |
orig.build_py.finalize_options(self) | |
self.package_data = self.distribution.package_data | |
self.exclude_package_data = self.distribution.exclude_package_data or {} | |
if 'data_files' in self.__dict__: | |
del self.__dict__['data_files'] | |
self.__updated_files = [] | |
def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, | |
link=None, level=1): | |
# Overwrite base class to allow using links | |
if link: | |
infile = str(Path(infile).resolve()) | |
outfile = str(Path(outfile).resolve()) | |
return super().copy_file(infile, outfile, preserve_mode, preserve_times, | |
link, level) | |
def run(self): | |
"""Build modules, packages, and copy data files to build directory""" | |
if not (self.py_modules or self.packages) or self.editable_mode: | |
return | |
if self.py_modules: | |
self.build_modules() | |
if self.packages: | |
self.build_packages() | |
self.build_package_data() | |
# Only compile actual .py files, using our base class' idea of what our | |
# output files are. | |
self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0)) | |
def __getattr__(self, attr): | |
"lazily compute data files" | |
if attr == 'data_files': | |
self.data_files = self._get_data_files() | |
return self.data_files | |
return orig.build_py.__getattr__(self, attr) | |
def build_module(self, module, module_file, package): | |
outfile, copied = orig.build_py.build_module(self, module, module_file, package) | |
if copied: | |
self.__updated_files.append(outfile) | |
return outfile, copied | |
def _get_data_files(self): | |
"""Generate list of '(package,src_dir,build_dir,filenames)' tuples""" | |
self.analyze_manifest() | |
return list(map(self._get_pkg_data_files, self.packages or ())) | |
def get_data_files_without_manifest(self): | |
""" | |
Generate list of ``(package,src_dir,build_dir,filenames)`` tuples, | |
but without triggering any attempt to analyze or build the manifest. | |
""" | |
# Prevent eventual errors from unset `manifest_files` | |
# (that would otherwise be set by `analyze_manifest`) | |
self.__dict__.setdefault('manifest_files', {}) | |
return list(map(self._get_pkg_data_files, self.packages or ())) | |
def _get_pkg_data_files(self, package): | |
# Locate package source directory | |
src_dir = self.get_package_dir(package) | |
# Compute package build directory | |
build_dir = os.path.join(*([self.build_lib] + package.split('.'))) | |
# Strip directory from globbed filenames | |
filenames = [ | |
os.path.relpath(file, src_dir) | |
for file in self.find_data_files(package, src_dir) | |
] | |
return package, src_dir, build_dir, filenames | |
def find_data_files(self, package, src_dir): | |
"""Return filenames for package's data files in 'src_dir'""" | |
patterns = self._get_platform_patterns( | |
self.package_data, | |
package, | |
src_dir, | |
) | |
globs_expanded = map(partial(glob, recursive=True), patterns) | |
# flatten the expanded globs into an iterable of matches | |
globs_matches = itertools.chain.from_iterable(globs_expanded) | |
glob_files = filter(os.path.isfile, globs_matches) | |
files = itertools.chain( | |
self.manifest_files.get(package, []), | |
glob_files, | |
) | |
return self.exclude_data_files(package, src_dir, files) | |
def get_outputs(self, include_bytecode=1) -> List[str]: | |
"""See :class:`setuptools.commands.build.SubCommand`""" | |
if self.editable_mode: | |
return list(self.get_output_mapping().keys()) | |
return super().get_outputs(include_bytecode) | |
def get_output_mapping(self) -> Dict[str, str]: | |
"""See :class:`setuptools.commands.build.SubCommand`""" | |
mapping = itertools.chain( | |
self._get_package_data_output_mapping(), | |
self._get_module_mapping(), | |
) | |
return dict(sorted(mapping, key=lambda x: x[0])) | |
def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: | |
"""Iterate over all modules producing (dest, src) pairs.""" | |
for (package, module, module_file) in self.find_all_modules(): | |
package = package.split('.') | |
filename = self.get_module_outfile(self.build_lib, package, module) | |
yield (filename, module_file) | |
def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: | |
"""Iterate over package data producing (dest, src) pairs.""" | |
for package, src_dir, build_dir, filenames in self.data_files: | |
for filename in filenames: | |
target = os.path.join(build_dir, filename) | |
srcfile = os.path.join(src_dir, filename) | |
yield (target, srcfile) | |
def build_package_data(self): | |
"""Copy data files into build directory""" | |
for target, srcfile in self._get_package_data_output_mapping(): | |
self.mkpath(os.path.dirname(target)) | |
_outf, _copied = self.copy_file(srcfile, target) | |
make_writable(target) | |
def analyze_manifest(self): | |
self.manifest_files = mf = {} | |
if not self.distribution.include_package_data: | |
return | |
src_dirs = {} | |
for package in self.packages or (): | |
# Locate package source directory | |
src_dirs[assert_relative(self.get_package_dir(package))] = package | |
if ( | |
getattr(self, 'existing_egg_info_dir', None) | |
and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() | |
): | |
egg_info_dir = self.existing_egg_info_dir | |
manifest = Path(egg_info_dir, "SOURCES.txt") | |
files = manifest.read_text(encoding="utf-8").splitlines() | |
else: | |
self.run_command('egg_info') | |
ei_cmd = self.get_finalized_command('egg_info') | |
egg_info_dir = ei_cmd.egg_info | |
files = ei_cmd.filelist.files | |
check = _IncludePackageDataAbuse() | |
for path in self._filter_build_files(files, egg_info_dir): | |
d, f = os.path.split(assert_relative(path)) | |
prev = None | |
oldf = f | |
while d and d != prev and d not in src_dirs: | |
prev = d | |
d, df = os.path.split(d) | |
f = os.path.join(df, f) | |
if d in src_dirs: | |
if f == oldf: | |
if check.is_module(f): | |
continue # it's a module, not data | |
else: | |
importable = check.importable_subpackage(src_dirs[d], f) | |
if importable: | |
check.warn(importable) | |
mf.setdefault(src_dirs[d], []).append(path) | |
def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]: | |
""" | |
``build_meta`` may try to create egg_info outside of the project directory, | |
and this can be problematic for certain plugins (reported in issue #3500). | |
Extensions might also include between their sources files created on the | |
``build_lib`` and ``build_temp`` directories. | |
This function should filter this case of invalid files out. | |
""" | |
build = self.get_finalized_command("build") | |
build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base) | |
norm_dirs = [os.path.normpath(p) for p in build_dirs if p] | |
for file in files: | |
norm_path = os.path.normpath(file) | |
if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs): | |
yield file | |
def get_data_files(self): | |
pass # Lazily compute data files in _get_data_files() function. | |
def check_package(self, package, package_dir): | |
"""Check namespace packages' __init__ for declare_namespace""" | |
try: | |
return self.packages_checked[package] | |
except KeyError: | |
pass | |
init_py = orig.build_py.check_package(self, package, package_dir) | |
self.packages_checked[package] = init_py | |
if not init_py or not self.distribution.namespace_packages: | |
return init_py | |
for pkg in self.distribution.namespace_packages: | |
if pkg == package or pkg.startswith(package + '.'): | |
break | |
else: | |
return init_py | |
with io.open(init_py, 'rb') as f: | |
contents = f.read() | |
if b'declare_namespace' not in contents: | |
raise distutils.errors.DistutilsError( | |
"Namespace package problem: %s is a namespace package, but " | |
"its\n__init__.py does not call declare_namespace()! Please " | |
'fix it.\n(See the setuptools manual under ' | |
'"Namespace Packages" for details.)\n"' % (package,) | |
) | |
return init_py | |
def initialize_options(self): | |
self.packages_checked = {} | |
orig.build_py.initialize_options(self) | |
self.editable_mode = False | |
self.existing_egg_info_dir = None | |
def get_package_dir(self, package): | |
res = orig.build_py.get_package_dir(self, package) | |
if self.distribution.src_root is not None: | |
return os.path.join(self.distribution.src_root, res) | |
return res | |
def exclude_data_files(self, package, src_dir, files): | |
"""Filter filenames for package's data files in 'src_dir'""" | |
files = list(files) | |
patterns = self._get_platform_patterns( | |
self.exclude_package_data, | |
package, | |
src_dir, | |
) | |
match_groups = (fnmatch.filter(files, pattern) for pattern in patterns) | |
# flatten the groups of matches into an iterable of matches | |
matches = itertools.chain.from_iterable(match_groups) | |
bad = set(matches) | |
keepers = (fn for fn in files if fn not in bad) | |
# ditch dupes | |
return list(unique_everseen(keepers)) | |
def _get_platform_patterns(spec, package, src_dir): | |
""" | |
yield platform-specific path patterns (suitable for glob | |
or fn_match) from a glob-based spec (such as | |
self.package_data or self.exclude_package_data) | |
matching package in src_dir. | |
""" | |
raw_patterns = itertools.chain( | |
spec.get('', []), | |
spec.get(package, []), | |
) | |
return ( | |
# Each pattern has to be converted to a platform-specific path | |
os.path.join(src_dir, convert_path(pattern)) | |
for pattern in raw_patterns | |
) | |
def assert_relative(path): | |
if not os.path.isabs(path): | |
return path | |
from distutils.errors import DistutilsSetupError | |
msg = ( | |
textwrap.dedent( | |
""" | |
Error: setup script specifies an absolute path: | |
%s | |
setup() arguments must *always* be /-separated paths relative to the | |
setup.py directory, *never* absolute paths. | |
""" | |
).lstrip() | |
% path | |
) | |
raise DistutilsSetupError(msg) | |
class _IncludePackageDataAbuse: | |
"""Inform users that package or module is included as 'data file'""" | |
MESSAGE = """\ | |
Installing {importable!r} as data is deprecated, please list it in `packages`. | |
!!\n\n | |
############################ | |
# Package would be ignored # | |
############################ | |
Python recognizes {importable!r} as an importable package, | |
but it is not listed in the `packages` configuration of setuptools. | |
{importable!r} has been automatically added to the distribution only | |
because it may contain data files, but this behavior is likely to change | |
in future versions of setuptools (and therefore is considered deprecated). | |
Please make sure that {importable!r} is included as a package by using | |
the `packages` configuration field or the proper discovery methods | |
(for example by using `find_namespace_packages(...)`/`find_namespace:` | |
instead of `find_packages(...)`/`find:`). | |
You can read more about "package discovery" and "data files" on setuptools | |
documentation page. | |
\n\n!! | |
""" | |
def __init__(self): | |
self._already_warned = set() | |
def is_module(self, file): | |
return file.endswith(".py") and file[:-len(".py")].isidentifier() | |
def importable_subpackage(self, parent, file): | |
pkg = Path(file).parent | |
parts = list(itertools.takewhile(str.isidentifier, pkg.parts)) | |
if parts: | |
return ".".join([parent, *parts]) | |
return None | |
def warn(self, importable): | |
if importable not in self._already_warned: | |
msg = textwrap.dedent(self.MESSAGE).format(importable=importable) | |
warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) | |
self._already_warned.add(importable) | |