diff options
Diffstat (limited to 'pyproject_buildrequires.py')
| -rw-r--r-- | pyproject_buildrequires.py | 588 |
1 files changed, 0 insertions, 588 deletions
diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py deleted file mode 100644 index 515da95..0000000 --- a/pyproject_buildrequires.py +++ /dev/null @@ -1,588 +0,0 @@ -import glob -import io -import os -import sys -import importlib.metadata -import argparse -import traceback -import json -import subprocess -import re -import tempfile -import email.parser -import pathlib -import zipfile - -from pyproject_requirements_txt import convert_requirements_txt -from pyproject_wheel import parse_config_settings_args - - -# Some valid Python version specifiers are not supported. -# Allow only the forms we know we can handle. -VERSION_RE = re.compile(r'[a-zA-Z0-9.-]+(\.\*)?') - - -class EndPass(Exception): - """End current pass of generating requirements""" - - -# nb: we don't use functools.partial to be able to use pytest's capsys -# see https://github.com/pytest-dev/pytest/issues/8900 -def print_err(*args, **kwargs): - kwargs.setdefault('file', sys.stderr) - print(*args, **kwargs) - - -try: - from packaging.requirements import Requirement, InvalidRequirement - from packaging.utils import canonicalize_name -except ImportError as e: - print_err('Import error:', e) - # already echoed by the %pyproject_buildrequires macro - sys.exit(0) - -# uses packaging, needs to be imported after packaging is verified to be present -from pyproject_convert import convert - - -def guess_reason_for_invalid_requirement(requirement_str): - if ':' in requirement_str: - message = ( - 'It might be an URL. ' - '%pyproject_buildrequires cannot handle all URL-based requirements. ' - 'Add PackageName@ (see PEP 508) to the URL to at least require any version of PackageName.' - ) - if '@' in requirement_str: - message += ' (but note that URLs might not work well with other features)' - return message - if '/' in requirement_str: - return ( - 'It might be a local path. ' - '%pyproject_buildrequires cannot handle local paths as requirements. ' - 'Use an URL with PackageName@ (see PEP 508) to at least require any version of PackageName.' - ) - # No more ideas - return None - - -class Requirements: - """Requirement gatherer. The macro will eventually print out output_lines.""" - def __init__(self, get_installed_version, extras=None, - generate_extras=False, python3_pkgversion='3', config_settings=None): - self.get_installed_version = get_installed_version - self.output_lines = [] - self.extras = set() - - if extras: - for extra in extras: - self.add_extras(*extra.split(',')) - - self.missing_requirements = False - self.ignored_alien_requirements = [] - - self.generate_extras = generate_extras - self.python3_pkgversion = python3_pkgversion - self.config_settings = config_settings - - def add_extras(self, *extras): - self.extras |= set(e.strip() for e in extras) - - @property - def marker_envs(self): - if self.extras: - return [{'extra': e} for e in sorted(self.extras)] - return [{'extra': ''}] - - def evaluate_all_environments(self, requirement): - for marker_env in self.marker_envs: - if requirement.marker.evaluate(environment=marker_env): - return True - return False - - def add(self, requirement_str, *, package_name=None, source=None): - """Output a Python-style requirement string as RPM dep""" - print_err(f'Handling {requirement_str} from {source}') - - try: - requirement = Requirement(requirement_str) - except InvalidRequirement: - hint = guess_reason_for_invalid_requirement(requirement_str) - message = f'Requirement {requirement_str!r} from {source} is invalid.' - if hint: - message += f' Hint: {hint}' - raise ValueError(message) - - if requirement.url: - print_err( - f'WARNING: Simplifying {requirement_str!r} to {requirement.name!r}.' - ) - - name = canonicalize_name(requirement.name) - if (requirement.marker is not None and - not self.evaluate_all_environments(requirement)): - print_err(f'Ignoring alien requirement:', requirement_str) - self.ignored_alien_requirements.append(requirement_str) - return - - # Handle self-referencing requirements - if package_name and canonicalize_name(package_name) == name: - # Self-referential extras need to be handled specially - if requirement.extras: - if not (requirement.extras <= self.extras): # only handle it if needed - # let all further requirements know we want those extras - self.add_extras(*requirement.extras) - # re-add all of the alien requirements ignored in the past - # they might no longer be alien now - self.readd_ignored_alien_requirements(package_name=package_name) - else: - print_err(f'Ignoring self-referential requirement without extras:', requirement_str) - return - - # We need to always accept pre-releases as satisfying the requirement - # Otherwise e.g. installed cffi version 1.15.0rc2 won't even satisfy the requirement for "cffi" - # https://bugzilla.redhat.com/show_bug.cgi?id=2014639#c3 - requirement.specifier.prereleases = True - - try: - # TODO: check if requirements with extras are satisfied - installed = self.get_installed_version(requirement.name) - except importlib.metadata.PackageNotFoundError: - print_err(f'Requirement not satisfied: {requirement_str}') - installed = None - if installed and installed in requirement.specifier: - print_err(f'Requirement satisfied: {requirement_str}') - print_err(f' (installed: {requirement.name} {installed})') - if requirement.extras: - print_err(f' (extras are currently not checked)') - else: - self.missing_requirements = True - - if self.generate_extras: - extra_names = [f'{name}[{extra.lower()}]' for extra in sorted(requirement.extras)] - else: - extra_names = [] - - for name in [name] + extra_names: - together = [] - for specifier in sorted( - requirement.specifier, - key=lambda s: (s.operator, s.version), - ): - if not VERSION_RE.fullmatch(str(specifier.version)): - raise ValueError( - f'Unknown character in version: {specifier.version}. ' - + '(This might be a bug in pyproject-rpm-macros.)', - ) - together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion), - specifier.operator, specifier.version)) - if len(together) == 0: - dep = python3dist(name, python3_pkgversion=self.python3_pkgversion) - self.output_lines.append(dep) - elif len(together) == 1: - self.output_lines.append(together[0]) - else: - self.output_lines.append(f"({' with '.join(together)})") - - def check(self, *, source=None): - """End current pass if any unsatisfied dependencies were output""" - if self.missing_requirements: - print_err(f'Exiting dependency generation pass: {source}') - raise EndPass(source) - - def extend(self, requirement_strs, **kwargs): - """add() several requirements""" - for req_str in requirement_strs: - self.add(req_str, **kwargs) - - def readd_ignored_alien_requirements(self, **kwargs): - """add() previously ignored alien requirements again.""" - requirements, self.ignored_alien_requirements = self.ignored_alien_requirements, [] - kwargs.setdefault('source', 'Previously ignored alien requirements') - self.extend(requirements, **kwargs) - - -def toml_load(opened_binary_file): - try: - # tomllib is in the standard library since 3.11.0b1 - import tomllib - except ImportError: - try: - import tomli as tomllib - except ImportError as e: - print_err('Import error:', e) - # already echoed by the %pyproject_buildrequires macro - sys.exit(0) - return tomllib.load(opened_binary_file) - - -def get_backend(requirements): - try: - f = open('pyproject.toml', 'rb') - except FileNotFoundError: - pyproject_data = {} - else: - with f: - pyproject_data = toml_load(f) - - buildsystem_data = pyproject_data.get('build-system', {}) - requirements.extend( - buildsystem_data.get('requires', ()), - source='build-system.requires', - ) - - backend_name = buildsystem_data.get('build-backend') - if not backend_name: - # https://www.python.org/dev/peps/pep-0517/: - # If the pyproject.toml file is absent, or the build-backend key is - # missing, the source tree is not using this specification, and tools - # should revert to the legacy behaviour of running setup.py - # (either directly, or by implicitly invoking the [following] backend). - # If setup.py is also not present program will mimick pip's behavior - # and end with an error. - if not os.path.exists('setup.py'): - raise FileNotFoundError('File "setup.py" not found for legacy project.') - backend_name = 'setuptools.build_meta:__legacy__' - - # Note: For projects without pyproject.toml, this was already echoed - # by the %pyproject_buildrequires macro, but this also handles cases - # with pyproject.toml without a specified build backend. - # If the default requirements change, also change them in the macro! - requirements.add('setuptools >= 40.8', source='default build backend') - # PEP 517 doesn't mandate depending on wheel when the default backend is used. - # Historically, it used to be assumed as necessary, but later it turned out to be wrong. - # See the removal in pip and build: - # https://github.com/pypa/pip/pull/12449 - # https://github.com/pypa/build/pull/716 - # However, the requirement *will* be generated by setuptools anyway - # as part of get_requires_for_build_wheel(). - # So we might as well keep it to skip one redundant step. - requirements.add('wheel', source='default build backend') - - requirements.check(source='build backend') - - backend_path = buildsystem_data.get('backend-path') - if backend_path: - # PEP 517 example shows the path as a list, but some projects don't follow that - if isinstance(backend_path, str): - backend_path = [backend_path] - sys.path = backend_path + sys.path - - module_name, _, object_name = backend_name.partition(":") - backend_module = importlib.import_module(module_name) - - if object_name: - return getattr(backend_module, object_name) - - return backend_module - - -def generate_build_requirements(backend, requirements): - get_requires = getattr(backend, 'get_requires_for_build_wheel', None) - if get_requires: - new_reqs = get_requires(config_settings=requirements.config_settings) - requirements.extend(new_reqs, source='get_requires_for_build_wheel') - requirements.check(source='get_requires_for_build_wheel') - - -def parse_metadata_file(metadata_file): - return email.parser.Parser().parse(metadata_file, headersonly=True) - - -def requires_from_parsed_metadata_file(message): - return {k: message.get_all(k, ()) for k in ('Requires', 'Requires-Dist')} - - -def package_name_from_parsed_metadata_file(message): - return message.get('name') - - -def package_name_and_requires_from_metadata_file(metadata_file): - message = parse_metadata_file(metadata_file) - package_name = package_name_from_parsed_metadata_file(message) - requires = requires_from_parsed_metadata_file(message) - return package_name, requires - - -def generate_run_requirements_hook(backend, requirements): - hook_name = 'prepare_metadata_for_build_wheel' - prepare_metadata = getattr(backend, hook_name, None) - if not prepare_metadata: - raise ValueError( - 'The build backend cannot provide build metadata ' - '(incl. runtime requirements) before build. ' - 'Use the provisional -w flag to build the wheel and parse the metadata from it, ' - 'or use the -R flag not to generate runtime dependencies.' - ) - dir_basename = prepare_metadata('.', config_settings=requirements.config_settings) - with open(dir_basename + '/METADATA') as metadata_file: - name, requires = package_name_and_requires_from_metadata_file(metadata_file) - for key, req in requires.items(): - requirements.extend(req, - package_name=name, - source=f'hook generated metadata: {key} ({name})') - - -def find_built_wheel(wheeldir): - wheels = glob.glob(os.path.join(wheeldir, '*.whl')) - if not wheels: - return None - if len(wheels) > 1: - raise RuntimeError('Found multiple wheels in %{_pyproject_wheeldir}, ' - 'this is not supported with %pyproject_buildrequires -w.') - return wheels[0] - - -def generate_run_requirements_wheel(backend, requirements, wheeldir): - # Reuse the wheel from the previous round of %pyproject_buildrequires (if it exists) - wheel = find_built_wheel(wheeldir) - if not wheel: - # pip is already echoed from the macro - # but we need to explicitly restart if has not yet been installed - # see https://bugzilla.redhat.com/2169855 - requirements.add('pip >= 19', source='%pyproject_buildrequires -w') - requirements.check(source='%pyproject_buildrequires -w') - import pyproject_wheel - returncode = pyproject_wheel.build_wheel( - wheeldir=wheeldir, - stdout=sys.stderr, - config_settings=requirements.config_settings, - ) - if returncode != 0: - raise RuntimeError('Failed to build the wheel for %pyproject_buildrequires -w.') - wheel = find_built_wheel(wheeldir) - if not wheel: - raise RuntimeError('Cannot locate the built wheel for %pyproject_buildrequires -w.') - - print_err(f'Reading metadata from {wheel}') - with zipfile.ZipFile(wheel) as wheelfile: - for name in wheelfile.namelist(): - if name.count('/') == 1 and name.endswith('.dist-info/METADATA'): - with io.TextIOWrapper(wheelfile.open(name), encoding='utf-8') as metadata_file: - name, requires = package_name_and_requires_from_metadata_file(metadata_file) - for key, req in requires.items(): - requirements.extend(req, - package_name=name, - source=f'built wheel metadata: {key} ({name})') - break - else: - raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.') - - -def generate_run_requirements(backend, requirements, *, build_wheel, wheeldir): - if build_wheel: - generate_run_requirements_wheel(backend, requirements, wheeldir) - else: - generate_run_requirements_hook(backend, requirements) - - -def generate_tox_requirements(toxenv, requirements): - toxenv = ','.join(toxenv) - requirements.add('tox-current-env >= 0.0.6', source='tox itself') - requirements.check(source='tox itself') - with tempfile.NamedTemporaryFile('r') as deps, \ - tempfile.NamedTemporaryFile('r') as extras, \ - tempfile.NamedTemporaryFile('r') as provision: - r = subprocess.run( - [sys.executable, '-m', 'tox', - '--print-deps-to', deps.name, - '--print-extras-to', extras.name, - '--no-provision', provision.name, - '-q', '-r', '-e', toxenv], - check=False, - encoding='utf-8', - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - if r.stdout: - print_err(r.stdout, end='') - - provision_content = provision.read() - if provision_content and r.returncode != 0: - provision_requires = json.loads(provision_content) - if provision_requires.get('minversion') is not None: - requirements.add(f'tox >= {provision_requires["minversion"]}', - source='tox provision (minversion)') - if 'requires' in provision_requires: - requirements.extend(provision_requires["requires"], - source='tox provision (requires)') - requirements.check(source='tox provision') # this terminates the script - raise RuntimeError( - 'Dependencies requested by tox provisioning appear installed, ' - 'but tox disagreed.') - else: - r.check_returncode() - - deplines = deps.read().splitlines() - packages = convert_requirements_txt(deplines) - requirements.add_extras(*extras.read().splitlines()) - requirements.extend(packages, - source=f'tox --print-deps-only: {toxenv}') - - -def python3dist(name, op=None, version=None, python3_pkgversion="3"): - prefix = f"python{python3_pkgversion}dist" - - if op is None: - if version is not None: - raise AssertionError('op and version go together') - return f'{prefix}({name})' - else: - return f'{prefix}({name}) {op} {version}' - - -def generate_requires( - *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, - get_installed_version=importlib.metadata.version, # for dep injection - generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True, - output, config_settings=None, -): - """Generate the BuildRequires for the project in the current directory - - The generated BuildRequires are written to the provided output. - - This is the main Python entry point. - """ - requirements = Requirements( - get_installed_version, extras=extras or [], - generate_extras=generate_extras, - python3_pkgversion=python3_pkgversion, - config_settings=config_settings, - ) - - try: - if (include_runtime or toxenv) and not use_build_system: - raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options') - if requirement_files: - for req_file in requirement_files: - requirements.extend( - convert_requirements_txt(req_file, pathlib.Path(req_file.name)), - source=f'requirements file {req_file.name}' - ) - requirements.check(source='all requirements files') - if use_build_system: - backend = get_backend(requirements) - generate_build_requirements(backend, requirements) - if toxenv: - include_runtime = True - generate_tox_requirements(toxenv, requirements) - if include_runtime: - generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir) - except EndPass: - return - finally: - output.write_text(os.linesep.join(requirements.output_lines) + os.linesep) - - -def main(argv): - parser = argparse.ArgumentParser( - description='Generate BuildRequires for a Python project.', - prog='%pyproject_buildrequires', - add_help=False, - ) - parser.add_argument( - '--help', action='help', - default=argparse.SUPPRESS, - help=argparse.SUPPRESS, - ) - parser.add_argument( - '-r', '--runtime', action='store_true', default=True, - help=argparse.SUPPRESS, # Generate run-time requirements (backwards-compatibility only) - ) - parser.add_argument( - '--generate-extras', action='store_true', - help=argparse.SUPPRESS, - ) - parser.add_argument( - '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', - default="3", help=argparse.SUPPRESS, - ) - parser.add_argument( - '--output', type=pathlib.Path, required=True, help=argparse.SUPPRESS, - ) - parser.add_argument( - '--wheeldir', metavar='PATH', default=None, - help=argparse.SUPPRESS, - ) - parser.add_argument( - '-x', '--extras', metavar='EXTRAS', action='append', - help='comma separated list of "extras" for runtime requirements ' - '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)', - ) - parser.add_argument( - '-t', '--tox', action='store_true', - help=('generate test tequirements from tox environment ' - '(implies --runtime)'), - ) - parser.add_argument( - '-e', '--toxenv', metavar='TOXENVS', action='append', - help=('specify tox environments (comma separated and/or repeated)' - '(implies --tox)'), - ) - parser.add_argument( - '-w', '--wheel', action='store_true', default=False, - help=('Generate run-time requirements by building the wheel ' - '(useful for build backends without the prepare_metadata_for_build_wheel hook)'), - ) - parser.add_argument( - '-R', '--no-runtime', action='store_false', dest='runtime', - help="Don't generate run-time requirements (implied by -N)", - ) - parser.add_argument( - '-N', '--no-use-build-system', dest='use_build_system', - action='store_false', help='Use -N to indicate that project does not use any build system', - ) - parser.add_argument( - 'requirement_files', nargs='*', type=argparse.FileType('r'), - metavar='REQUIREMENTS.TXT', - help=('Add buildrequires from file'), - ) - parser.add_argument( - '-C', - dest='config_settings', - action='append', - help='Configuration settings to pass to the PEP 517 backend', - ) - - args = parser.parse_args(argv) - - if not args.use_build_system: - args.runtime = False - - if args.wheel: - if not args.wheeldir: - raise ValueError('--wheeldir must be set when -w.') - - if args.toxenv: - args.tox = True - - if args.tox: - args.runtime = True - if not args.toxenv: - _default = f'py{sys.version_info.major}{sys.version_info.minor}' - args.toxenv = [os.getenv('RPM_TOXENV', _default)] - - if args.extras: - args.runtime = True - - try: - generate_requires( - include_runtime=args.runtime, - build_wheel=args.wheel, - wheeldir=args.wheeldir, - toxenv=args.toxenv, - extras=args.extras, - generate_extras=args.generate_extras, - python3_pkgversion=args.python3_pkgversion, - requirement_files=args.requirement_files, - use_build_system=args.use_build_system, - output=args.output, - config_settings=parse_config_settings_args(args.config_settings), - ) - except Exception: - # Log the traceback explicitly (it's useful debug info) - traceback.print_exc() - exit(1) - - -if __name__ == '__main__': - main(sys.argv[1:]) |