diff options
16 files changed, 1522 insertions, 0 deletions
diff --git a/0007-More-robust-spec-file-presence-checking.patch b/0007-More-robust-spec-file-presence-checking.patch new file mode 100644 index 0000000..a1dde64 --- /dev/null +++ b/0007-More-robust-spec-file-presence-checking.patch @@ -0,0 +1,292 @@ +From 1108810bdefd0d880517b274acd6a3bd0d4156e0 Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Tue, 21 Mar 2023 02:44:04 +0100 +Subject: [PATCH 07/12] More robust spec file presence checking + +Some commands (verrel, sources, prep, import, ...) need to check +whether the dist-git repository is in the correct state. It means +at least the presence of the specfile. +In the beginning, rpkg detects layouts. Layouts determine the file +structure of the repository. For example, most commands can't +be executed for the RetiredLayout (there is no specfile). +When the repository directory exists, some layout can be always +detected. Therefore '--path' argument is now checked for +a valid directory. +The timeout change in the request fixes the new bandit's finding. + +Fixes: #663 +JIRA: RHELCMP-11387 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 9 ++++--- + pyrpkg/cli.py | 8 +++--- + pyrpkg/layout/__init__.py | 4 +-- + pyrpkg/utils.py | 14 ++++++++++ + tests/commands/test_push.py | 54 +++++++++++++++++++------------------ + tests/test_cli.py | 12 ++++++--- + 6 files changed, 63 insertions(+), 38 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 776cb21..028d195 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -923,9 +923,8 @@ class Commands(object): + def load_spec(self): + """This sets the spec attribute""" + +- if self.layout is None: ++ if self.layout is None or isinstance(self.layout, layout.IncompleteLayout): + raise rpkgError('Spec file is not available') +- + if self.is_retired(): + raise rpkgError('This package or module is retired. The action has stopped.') + +@@ -1166,8 +1165,10 @@ class Commands(object): + + @property + def sources_filename(self): +- if self.layout is None: +- return os.path.join(self.path, 'sources') ++ if self.layout is None or isinstance(self.layout, layout.IncompleteLayout): ++ raise rpkgError('Spec file is not available') ++ if self.is_retired(): ++ raise rpkgError('This package or module is retired. The action has stopped.') + return os.path.join( + self.path, self.layout.sources_file_template.replace("{0.repo_name}", self.repo_name)) + +diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py +index c3672b3..1bcf6e4 100644 +--- a/pyrpkg/cli.py ++++ b/pyrpkg/cli.py +@@ -386,7 +386,7 @@ class cliClient(object): + help='Run Koji commands as a different user') + # Let the user define a path to work in rather than cwd + self.parser.add_argument('--path', default=None, +- type=utils.u, ++ type=utils.validate_path, + help='Define the directory to work in ' + '(defaults to cwd)') + # Verbosity +@@ -911,8 +911,9 @@ class cliClient(object): + if 'path' in args: + # Without "path", we can't really test... + url = '%(protocol)s://%(host)s/%(path)s/info/refs?service=git-receive-pack' % args +- resp = requests.head(url, auth=HTTPBasicAuth(args['username'], +- args['password'])) ++ resp = requests.head(url, ++ auth=HTTPBasicAuth(args['username'], args['password']), ++ timeout=15) + if resp.status_code == 401: + return self.oidc_client.report_token_issue() + +@@ -2363,6 +2364,7 @@ class cliClient(object): + + def import_srpm(self): + uploadfiles = self.cmd.import_srpm(self.args.srpm) ++ self.load_cmd() # to reload layouts - because a specfile could appear during import + if uploadfiles: + try: + self.cmd.upload(uploadfiles, replace=True, offline=self.args.offline) +diff --git a/pyrpkg/layout/__init__.py b/pyrpkg/layout/__init__.py +index 762af0d..850ddc2 100644 +--- a/pyrpkg/layout/__init__.py ++++ b/pyrpkg/layout/__init__.py +@@ -12,8 +12,8 @@ + from pyrpkg.errors import LayoutError + + from .base import MetaLayout +-from .layouts import (DistGitLayout, IncompleteLayout, # noqa: F401 +- RetiredLayout, SRPMLayout) ++from .layouts import (DistGitLayout, DistGitResultsDirLayout, # noqa: F401 ++ IncompleteLayout, RetiredLayout, SRPMLayout) + + + def build(path, hint=None): +diff --git a/pyrpkg/utils.py b/pyrpkg/utils.py +index ceb4906..3337bdb 100644 +--- a/pyrpkg/utils.py ++++ b/pyrpkg/utils.py +@@ -26,11 +26,25 @@ if six.PY3: + def u(s): + return s + ++ def validate_path(s): ++ abspath = os.path.abspath(s) ++ if os.path.exists(abspath): ++ return s ++ else: ++ raise argparse.ArgumentTypeError('given path \'{0}\' doesn\'t exist'.format(abspath)) ++ + getcwd = os.getcwd + else: + def u(s): + return s.decode('utf-8') + ++ def validate_path(s): ++ abspath = os.path.abspath(s.decode('utf-8')) ++ if os.path.exists(abspath): ++ return s.decode('utf-8') ++ else: ++ raise argparse.ArgumentTypeError('given path \'{0}\' doesn\'t exist'.format(abspath)) ++ + getcwd = os.getcwdu + + +diff --git a/tests/commands/test_push.py b/tests/commands/test_push.py +index ef8057a..79c3a8b 100644 +--- a/tests/commands/test_push.py ++++ b/tests/commands/test_push.py +@@ -1,9 +1,13 @@ + # -*- coding: utf-8 -*- + + import os ++import subprocess + + import git + ++import pyrpkg ++from pyrpkg.sources import SourcesFile ++ + from . import CommandTestCase + + SPECFILE_TEMPLATE = """Name: test +@@ -22,11 +26,6 @@ Test + %%install + rm -f $RPM_BUILD_ROOT%%{_sysconfdir}/""" + +-CLONE_CONFIG = ''' +- bz.default-component %(module)s +- sendemail.to %(module)s-owner@fedoraproject.org +-''' +- + + class CommandPushTestCase(CommandTestCase): + +@@ -45,28 +44,30 @@ class CommandPushTestCase(CommandTestCase): + + self.make_new_git(self.module) + +- import pyrpkg +- cmd = pyrpkg.Commands(self.path, self.lookaside, +- self.lookasidehash, +- self.lookaside_cgi, self.gitbaseurl, +- self.anongiturl, self.branchre, self.kojiprofile, +- self.build_client, self.user, self.dist, +- self.target, self.quiet) +- cmd.clone_config_rpms = CLONE_CONFIG +- cmd.clone(self.module, anon=True) +- cmd.path = os.path.join(self.path, self.module) +- os.chdir(os.path.join(self.path, self.module)) ++ moduledir = os.path.join(self.gitroot, self.module) ++ subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], ++ cwd=self.path, stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE) ++ ++ self.cloned_dir = os.path.join(self.path, self.module) ++ self.cmd = pyrpkg.Commands(self.cloned_dir, self.lookaside, ++ self.lookasidehash, ++ self.lookaside_cgi, self.gitbaseurl, ++ self.anongiturl, self.branchre, self.kojiprofile, ++ self.build_client, self.user, self.dist, ++ self.target, self.quiet) ++ os.chdir(self.cloned_dir) + + spec_file = 'module.spec' + with open(spec_file, 'w') as f: + f.write(SPECFILE_TEMPLATE % '') + +- cmd.repo.index.add([spec_file]) +- cmd.repo.index.commit("add SPEC") ++ self.cmd.repo.index.add([spec_file]) ++ self.cmd.repo.index.commit("add SPEC") + + # Now, change directory to parent and test the push + os.chdir(self.path) +- cmd.push(no_verify=True) ++ self.cmd.push(no_verify=True) + + + class TestPushWithPatches(CommandTestCase): +@@ -76,18 +77,20 @@ class TestPushWithPatches(CommandTestCase): + + self.make_new_git(self.module) + +- import pyrpkg +- self.cmd = pyrpkg.Commands(self.path, self.lookaside, ++ moduledir = os.path.join(self.gitroot, self.module) ++ subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], ++ cwd=self.path, stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE) ++ ++ self.cloned_dir = os.path.join(self.path, self.module) ++ self.cmd = pyrpkg.Commands(self.cloned_dir, self.lookaside, + self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, + self.kojiprofile, + self.build_client, self.user, self.dist, + self.target, self.quiet) +- self.cmd.clone_config_rpms = CLONE_CONFIG +- self.cmd.clone(self.module, anon=True) +- self.cmd.path = os.path.join(self.path, self.module) +- os.chdir(os.path.join(self.path, self.module)) ++ os.chdir(self.cloned_dir) + + # Track SPEC and a.patch in git + spec_file = 'module.spec' +@@ -103,7 +106,6 @@ Patch3: d.path + f.write(patch_file) + + # Track c.patch in sources +- from pyrpkg.sources import SourcesFile + sources_file = SourcesFile(self.cmd.sources_filename, + self.cmd.source_entry_type) + file_hash = self.cmd.lookasidecache.hash_file('c.patch') +diff --git a/tests/test_cli.py b/tests/test_cli.py +index df053aa..868ad1f 100644 +--- a/tests/test_cli.py ++++ b/tests/test_cli.py +@@ -1841,9 +1841,11 @@ class TestMockbuild(CliTestCase): + @patch('pyrpkg.Commands._config_dir_basic') + @patch('pyrpkg.Commands._config_dir_other') + @patch('os.path.exists', return_value=False) ++ @patch('pyrpkg.utils.validate_path') + def test_use_mock_config_got_from_koji( +- self, exists, config_dir_other, config_dir_basic): ++ self, validate_path, exists, config_dir_other, config_dir_basic): + mock_layout = layout.DistGitLayout(root_dir=self.cloned_repo_path) ++ validate_path.return_value = self.cloned_repo_path + with patch('pyrpkg.layout.build', return_value=mock_layout): + config_dir_basic.return_value = '/path/to/config-dir' + +@@ -1859,9 +1861,11 @@ class TestMockbuild(CliTestCase): + + @patch('pyrpkg.Commands._config_dir_basic') + @patch('os.path.exists', return_value=False) ++ @patch('pyrpkg.utils.validate_path') + def test_fail_to_store_mock_config_in_created_config_dir( +- self, exists, config_dir_basic): ++ self, validate_path, exists, config_dir_basic): + config_dir_basic.side_effect = rpkgError ++ validate_path.return_value = self.cloned_repo_path + + cli_cmd = ['rpkg', '--path', self.cloned_repo_path, + '--release', 'rhel-7', 'mockbuild'] +@@ -1870,10 +1874,12 @@ class TestMockbuild(CliTestCase): + @patch('pyrpkg.Commands._config_dir_basic') + @patch('pyrpkg.Commands._config_dir_other') + @patch('os.path.exists', return_value=False) ++ @patch('pyrpkg.utils.validate_path') + def test_fail_to_populate_mock_config( +- self, exists, config_dir_other, config_dir_basic): ++ self, validate_path, exists, config_dir_other, config_dir_basic): + config_dir_basic.return_value = '/path/to/config-dir' + config_dir_other.side_effect = rpkgError ++ validate_path.return_value = self.cloned_repo_path + + cli_cmd = ['rpkg', '--path', self.cloned_repo_path, + '--release', 'rhel-7', 'mockbuild'] +-- +2.39.2 + diff --git a/0008-Update-to-spec-file-presence-checking.patch b/0008-Update-to-spec-file-presence-checking.patch new file mode 100644 index 0000000..723415f --- /dev/null +++ b/0008-Update-to-spec-file-presence-checking.patch @@ -0,0 +1,43 @@ +From 791fd03b4de1324508583ab53c89cc67459db355 Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Tue, 21 Mar 2023 13:44:38 +0100 +Subject: [PATCH 08/12] Update to spec file presence checking + +Using a different approach to checking the layout. Older way prevented +`retire` function working correctly. Layouts are detected at the +beginning of the run and the result stays the same, unlike the direct +checking files like dead.package in function `is_retired`. + +Fixes: #663 +JIRA: RHELCMP-11387 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 028d195..e8f4886 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -925,7 +925,7 @@ class Commands(object): + + if self.layout is None or isinstance(self.layout, layout.IncompleteLayout): + raise rpkgError('Spec file is not available') +- if self.is_retired(): ++ if isinstance(self.layout, layout.RetiredLayout): + raise rpkgError('This package or module is retired. The action has stopped.') + + # Get a list of ".spec" files in the path we're looking at +@@ -1167,7 +1167,7 @@ class Commands(object): + def sources_filename(self): + if self.layout is None or isinstance(self.layout, layout.IncompleteLayout): + raise rpkgError('Spec file is not available') +- if self.is_retired(): ++ if isinstance(self.layout, layout.RetiredLayout): + raise rpkgError('This package or module is retired. The action has stopped.') + return os.path.join( + self.path, self.layout.sources_file_template.replace("{0.repo_name}", self.repo_name)) +-- +2.39.2 + diff --git a/0009-Add-more-information-about-pre-push-hook.patch b/0009-Add-more-information-about-pre-push-hook.patch new file mode 100644 index 0000000..60bad27 --- /dev/null +++ b/0009-Add-more-information-about-pre-push-hook.patch @@ -0,0 +1,44 @@ +From 0393dc39bf450cf20df9db63bac135c078f64a14 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com> +Date: Tue, 28 Mar 2023 08:53:30 +0200 +Subject: [PATCH 09/12] Add more information about pre-push hook +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +It's not obvious to many users where the check is coming from, and they +have the power to edit the script or delete it completely. Let's try to +improve that a bit. + +Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com> +--- + pyrpkg/__init__.py | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index e8f4886..7a3c9c6 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -1806,6 +1806,10 @@ class Commands(object): + hook_content = textwrap.dedent(""" + #!/bin/bash + ++ # This file was generated by {0} when cloning the repository. ++ # You can edit it to your liking or delete completely. It will not ++ # be recreated. ++ + _remote="$1" + _url="$2" + +@@ -4429,7 +4433,7 @@ class Commands(object): + return self._repo_name, version, release + + def pre_push_check(self, ref): +- show_hint = ('Hint: this check (pre-push hook script) can be bypassed by adding ' ++ show_hint = ('Hint: this check (.git/hooks/pre-push script) can be bypassed by adding ' + 'the argument \'--no-verify\' argument to the push command.') + try: + commit = self.repo.commit(ref) +-- +2.39.2 + diff --git a/0010-pre-push-check-have-to-use-spectool-with-define.patch b/0010-pre-push-check-have-to-use-spectool-with-define.patch new file mode 100644 index 0000000..bfe7094 --- /dev/null +++ b/0010-pre-push-check-have-to-use-spectool-with-define.patch @@ -0,0 +1,146 @@ +From d5be51eec99108c3809551b615064d0c5cbe628a Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Tue, 28 Mar 2023 19:58:06 +0200 +Subject: [PATCH 10/12] `pre-push-check` have to use spectool with --define + +To get all defined source files and patches from the specfile, +the 'spectool' utility needs '--define' argument(s) to set specific +paths for the repository. + +JIRA: RHELCMP-11466 +Fixes: #672 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 57 +++++++++++++++------------ + tests/commands/test_pre_push_check.py | 3 +- + 2 files changed, 33 insertions(+), 27 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 7a3c9c6..584c141 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -4442,30 +4442,41 @@ class Commands(object): + sys.exit(1) + + try: ++ clone_dir = tempfile.mkdtemp(prefix="pre_push_hook_") ++ for cmd in [ ++ ('git', 'clone', self.path, clone_dir), ++ ('git', 'checkout', ref), ++ ]: ++ ret, _, _ = self._run_command(cmd, cwd=clone_dir, ++ # suppress unwanted printing of command line messages ++ return_stdout=True, return_stderr=True) ++ if ret != 0: ++ self.log.error('Command \'{0}\' failed. Push operation ' ++ 'was cancelled.'.format(' '.join(cmd))) ++ self.log.warning(show_hint) ++ sys.exit(2) ++ ++ # get all source files from the specfile (including patches) + # Assume, that specfile names are same in the active branch + # and in the pushed branch (git checkout f37 && git push origin rawhide) + # in this case 'f37' is active branch and 'rawhide' is pushed branch. + specfile_path_absolute = os.path.join(self.layout.specdir, self.spec) + # convert to relative path + specfile_path = os.path.relpath(specfile_path_absolute, start=self.path) +- spec_content = self.repo.git.cat_file("-p", "{0}:{1}".format(ref, specfile_path)) +- except Exception: +- # It might be the case of an empty commit +- self.log.warning('Specfile doesn\'t exist. Push operation continues.') +- return +- +- # load specfile content from pushed branch and save it into a temporary file +- with tempfile.NamedTemporaryFile(mode="w+") as temporary_spec: +- temporary_spec.write(spec_content) +- temporary_spec.flush() +- # get all source files from the specfile (including patches) +- cmd = ('spectool', '-l', temporary_spec.name) +- ret, stdout, _ = self._run_command(cmd, return_text=True, return_stdout=True) ++ cmd = ['spectool', '-l', os.path.join(clone_dir, specfile_path)] ++ # extract just '--define' arguments from rpmdefines ++ for opt, val in zip(self.rpmdefines[0::2], self.rpmdefines[1::2]): ++ if opt == '--define': ++ cmd.extend((opt, val)) ++ ret, stdout, _ = self._run_command(cmd, cwd=clone_dir, ++ return_text=True, return_stdout=True) + if ret != 0: + self.log.error('Command \'{0}\' failed. Push operation ' + 'was cancelled.'.format(' '.join(cmd))) + self.log.warning(show_hint) +- sys.exit(2) ++ sys.exit(3) ++ finally: ++ self._cleanup_tmp_dir(clone_dir) + + source_files = [] + # extract source files from the spectool's output +@@ -4490,22 +4501,16 @@ class Commands(object): + sources_file_path_absolute = self.sources_filename + # convert to relative path + sources_file_path = os.path.relpath(sources_file_path_absolute, start=self.path) +- sources_file_content = self.repo.git.cat_file( +- '-p', '{0}:{1}'.format(ref, sources_file_path)) ++ ++ # parse 'sources' files content ++ sourcesf = SourcesFile(sources_file_path, self.source_entry_type) ++ sourcesf_entries = set(item.file for item in sourcesf.entries) + except Exception: + self.log.warning('\'sources\' file doesn\'t exist. Push operation continues.') + # NOTE: check doesn't fail when 'sources' file doesn't exist. Just skips the rest. + # it might be the case of the push without 'sources' = retiring the repository + return + +- # load 'sources' file content from pushed branch and save it into a temporary file +- with tempfile.NamedTemporaryFile(mode="w+") as temporary_sources_file: +- temporary_sources_file.write(sources_file_content) +- temporary_sources_file.flush() +- # parse 'sources' files content +- sourcesf = SourcesFile(temporary_sources_file.name, self.source_entry_type) +- sourcesf_entries = set(item.file for item in sourcesf.entries) +- + # list of all files (their relative paths) in the commit + repo_entries = set(item.path for item in commit.tree.traverse() if item.type != "tree") + +@@ -4518,7 +4523,7 @@ class Commands(object): + 'nor tracked in git. ' + 'Push operation was cancelled'.format(source_file)) + self.log.warning(show_hint) +- sys.exit(3) ++ sys.exit(4) + + # verify all file entries in 'sources' were uploaded to the lookaside cache + for entry in sourcesf.entries: +@@ -4532,6 +4537,6 @@ class Commands(object): + self.log.error('Source file (or tarball) \'{}\' wasn\'t uploaded to the lookaside ' + 'cache. Push operation was cancelled.'.format(filename)) + self.log.warning(show_hint) +- sys.exit(4) ++ sys.exit(5) + + return 0 # The push operation continues +diff --git a/tests/commands/test_pre_push_check.py b/tests/commands/test_pre_push_check.py +index 5e314b9..ee151c1 100644 +--- a/tests/commands/test_pre_push_check.py ++++ b/tests/commands/test_pre_push_check.py +@@ -37,6 +37,7 @@ class TestPrePushCheck(CommandTestCase): + def setUp(self): + super(TestPrePushCheck, self).setUp() + ++ self.dist = "rhel-8" + self.make_new_git(self.module) + + moduledir = os.path.join(self.gitroot, self.module) +@@ -87,7 +88,7 @@ Patch3: d.patch + with self.assertRaises(SystemExit) as exc: + self.cmd.pre_push_check("HEAD") + +- self.assertEqual(exc.exception.code, 3) ++ self.assertEqual(exc.exception.code, 4) + log_error.assert_called_once_with("Source file 'b.patch' was neither listed in the " + "'sources' file nor tracked in git. Push operation " + "was cancelled") +-- +2.39.2 + diff --git a/0011-A-HEAD-query-into-a-lookaside-cache.patch b/0011-A-HEAD-query-into-a-lookaside-cache.patch new file mode 100644 index 0000000..a3281b2 --- /dev/null +++ b/0011-A-HEAD-query-into-a-lookaside-cache.patch @@ -0,0 +1,97 @@ +From 77cd608e596af94811c22a16ff58a265d9c7381e Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Fri, 31 Mar 2023 14:09:09 +0200 +Subject: [PATCH 11/12] A HEAD query into a lookaside cache + +A query about whether some file is present in the lookaside cache was +under authentication and it prevented using command `pre-push-check` +for those without the 'packager' permission. +Added another method (based on HTTP HEAD), that allows the same check +without authentication. + +JIRA: RHELCMP-11485 +Fixes: https://pagure.io/fedpkg/issue/513 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 2 +- + pyrpkg/lookaside.py | 36 ++++++++++++++++++++++++++++++++++-- + 2 files changed, 35 insertions(+), 3 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 584c141..15203b7 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -4529,7 +4529,7 @@ class Commands(object): + for entry in sourcesf.entries: + filename = entry.file + hash = entry.hash +- file_exists_in_lookaside = self.lookasidecache.remote_file_exists( ++ file_exists_in_lookaside = self.lookasidecache.remote_file_exists_head( + self.ns_repo_name if self.lookaside_namespaced else self.repo_name, + filename, + hash) +diff --git a/pyrpkg/lookaside.py b/pyrpkg/lookaside.py +index 90f0f1e..ecbf12b 100644 +--- a/pyrpkg/lookaside.py ++++ b/pyrpkg/lookaside.py +@@ -22,7 +22,7 @@ import sys + + import pycurl + import six +-from six.moves import http_client ++from six.moves import http_client, urllib + + from .errors import (AlreadyUploadedError, DownloadError, InvalidHashType, + UploadError) +@@ -157,7 +157,7 @@ class CGILookasideCache(object): + return + + self.log.info("Downloading %s", filename) +- urled_file = filename.replace(' ', '%20') ++ urled_file = urllib.parse.quote(filename) + url = self.get_download_url(name, urled_file, hash, hashtype, **kwargs) + if isinstance(url, six.text_type): + url = url.encode('utf-8') +@@ -200,6 +200,38 @@ class CGILookasideCache(object): + if not self.file_is_valid(outfile, hash, hashtype=hashtype): + raise DownloadError('%s failed checksum' % filename) + ++ def remote_file_exists_head(self, name, filename, hash): ++ """Verify whether a file exists on the lookaside cache. ++ Uses a HTTP HEAD request and doesn't require authentication. ++ ++ :param str name: The name of the module. (usually the name of the ++ SRPM). This can include the namespace as well (depending on what ++ the server side expects). ++ :param str filename: The name of the file to check for. ++ :param str hash: The known good hash of the file. ++ """ ++ ++ urled_file = urllib.parse.quote(filename) ++ url = self.get_download_url(name, urled_file, hash, self.hashtype) ++ ++ c = pycurl.Curl() ++ c.setopt(pycurl.URL, url) ++ c.setopt(pycurl.NOBODY, True) ++ c.setopt(pycurl.FOLLOWLOCATION, 1) ++ ++ try: ++ c.perform() ++ status = c.getinfo(pycurl.RESPONSE_CODE) ++ except Exception as e: ++ raise DownloadError(e) ++ finally: ++ c.close() ++ ++ if status != 200: ++ self.log.debug('Unavailable file \'%s\' at %s' % (filename, url)) ++ return False ++ return True ++ + def remote_file_exists(self, name, filename, hash): + """Verify whether a file exists on the lookaside cache + +-- +2.39.2 + diff --git a/0012-pre-push-hook-script-contains-a-user-s-config.patch b/0012-pre-push-hook-script-contains-a-user-s-config.patch new file mode 100644 index 0000000..ff2676d --- /dev/null +++ b/0012-pre-push-hook-script-contains-a-user-s-config.patch @@ -0,0 +1,197 @@ +From 1f03eb9102f765c36cc201a499d815732e67dd39 Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Mon, 27 Mar 2023 23:34:12 +0200 +Subject: [PATCH 12/12] pre-push hook script contains a user's config + +When the `clone` command is called with an argument +-C|--config <config_file> +this argument is placed to the generated pre-push script. + +Fixes: #667 +JIRA: RHELCMP-11394 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 23 ++++++++++++------- + pyrpkg/cli.py | 6 +++-- + tests/commands/test_clone.py | 44 ++++++++++++++++++++++++++++++++++++ + 3 files changed, 63 insertions(+), 10 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 15203b7..9996402 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -1566,7 +1566,8 @@ class Commands(object): + return + + def clone(self, repo, path=None, branch=None, bare_dir=None, +- anon=False, target=None, depth=None, extra_args=None): ++ anon=False, target=None, depth=None, extra_args=None, ++ config_path=None): + """Clone a repo, optionally check out a specific branch. + + :param str repo: the name of the repository to clone. +@@ -1583,6 +1584,7 @@ class Commands(object): + to the specified number of commits. + :param list extra_args: additional arguments that are passed to + the clone command. ++ :param str config_path: path to the global config file + """ + + if not path: +@@ -1638,7 +1640,7 @@ class Commands(object): + + if not bare_dir: + self._add_git_excludes(os.path.join(path, git_dir)) +- self._add_git_pre_push_hook(os.path.join(path, git_dir)) ++ self._add_git_pre_push_hook(os.path.join(path, git_dir), config_path) + + return + +@@ -1654,7 +1656,7 @@ class Commands(object): + return repo + + def clone_with_dirs(self, repo, anon=False, target=None, depth=None, +- extra_args=None): ++ extra_args=None, config_path=None): + """Clone a repo old style with subdirs for each branch. + + :param str repo: name of the repository to clone. +@@ -1666,6 +1668,7 @@ class Commands(object): + to the specified number of commits. + :param list extra_args: additional arguments that are passed to + the clone command. ++ :param str config_path: path to the global config file + """ + + self._push_url = None +@@ -1724,7 +1727,7 @@ class Commands(object): + + # Add excludes + self._add_git_excludes(branch_path) +- self._add_git_pre_push_hook(branch_path) ++ self._add_git_pre_push_hook(branch_path, config_path) + except (git.GitCommandError, OSError) as e: + raise rpkgError('Could not locally clone %s from %s: %s' + % (branch, repo_path, e)) +@@ -1787,7 +1790,7 @@ class Commands(object): + git_excludes.write() + self.log.debug('Git-excludes patterns were added into %s' % git_excludes_path) + +- def _add_git_pre_push_hook(self, conf_dir): ++ def _add_git_pre_push_hook(self, repo_dir, config_path=None): + """ + Create pre-push hook script and write it in the location: + <repository_directory>/.git/hooks/pre-push +@@ -1803,6 +1806,10 @@ class Commands(object): + self.log.debug('Pre-push hook script was NOT added - missing ' + 'the packaging tool like fedpkg, rhpkg, ...') + return ++ ++ # in case the clone command run with 'x-pkg -C <config_path> clone <repo_name>' ++ config_arg = ' -C "{0}"'.format(os.path.realpath(config_path)) if config_path else "" ++ + hook_content = textwrap.dedent(""" + #!/bin/bash + +@@ -1818,7 +1825,7 @@ class Commands(object): + do + command -v {0} >/dev/null 2>&1 || {{ echo >&2 "Warning: '{0}' is missing, \\ + pre-push check is omitted. See .git/hooks/pre-push"; exit 0; }} +- {0} pre-push-check "$local_sha" ++ {0}{1} pre-push-check "$local_sha" + ret_code=$? + if [ $ret_code -ne 0 ] && [ $exit_code -eq 0 ]; then + exit_code=$ret_code +@@ -1826,8 +1833,8 @@ class Commands(object): + done + + exit $exit_code +- """).strip().format(tool_name) +- git_pre_push_hook_path = os.path.join(conf_dir, '.git/hooks/pre-push') ++ """).strip().format(tool_name, config_arg) ++ git_pre_push_hook_path = os.path.join(repo_dir, '.git/hooks/pre-push') + if not os.path.exists(os.path.dirname(git_pre_push_hook_path)): + # prepare ".git/hooks" directory if it is missing + os.makedirs(os.path.dirname(git_pre_push_hook_path)) +diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py +index 1bcf6e4..3d8ce33 100644 +--- a/pyrpkg/cli.py ++++ b/pyrpkg/cli.py +@@ -2182,14 +2182,16 @@ class cliClient(object): + anon=self.args.anonymous, + target=self.args.clone_target, + depth=self.args.depth, +- extra_args=self.extra_args) ++ extra_args=self.extra_args, ++ config_path=self.args.config) + else: + self.cmd.clone(self.args.repo[0], + branch=self.args.branch, + anon=self.args.anonymous, + target=self.args.clone_target, + depth=self.args.depth, +- extra_args=self.extra_args) ++ extra_args=self.extra_args, ++ config_path=self.args.config) + + def commit(self): + if self.args.with_changelog and not self.args.message: +diff --git a/tests/commands/test_clone.py b/tests/commands/test_clone.py +index f741864..6ef1300 100644 +--- a/tests/commands/test_clone.py ++++ b/tests/commands/test_clone.py +@@ -95,6 +95,50 @@ class CommandCloneTestCase(CommandTestCase): + + shutil.rmtree(altpath) + ++ def test_clone_anonymous_pre_push_hook(self): ++ self.make_new_git(self.module) ++ ++ altpath = tempfile.mkdtemp(prefix='rpkg-tests.') ++ ++ cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, ++ self.lookaside_cgi, self.gitbaseurl, ++ self.anongiturl, self.branchre, self.kojiprofile, ++ self.build_client, self.user, self.dist, ++ self.target, self.quiet) ++ cmd.clone(self.module, anon=True, config_path=None) ++ ++ moduledir = os.path.join(self.path, self.module) ++ self.assertTrue(os.path.isfile(os.path.join(moduledir, '.git/hooks/pre-push'))) ++ ++ with open(os.path.join(moduledir, '.git/hooks/pre-push')) as git_hook_script: ++ content = git_hook_script.read() ++ pattern = '__main__.py pre-push-check "$local_sha"' ++ self.assertIn(pattern, content) ++ ++ shutil.rmtree(altpath) ++ ++ def test_clone_anonymous_pre_push_hook_config(self): ++ self.make_new_git(self.module) ++ ++ altpath = tempfile.mkdtemp(prefix='rpkg-tests.') ++ ++ cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, ++ self.lookaside_cgi, self.gitbaseurl, ++ self.anongiturl, self.branchre, self.kojiprofile, ++ self.build_client, self.user, self.dist, ++ self.target, self.quiet) ++ cmd.clone(self.module, anon=True, config_path="/home/conf/rhpkg.conf") ++ ++ moduledir = os.path.join(self.path, self.module) ++ self.assertTrue(os.path.isfile(os.path.join(moduledir, '.git/hooks/pre-push'))) ++ ++ with open(os.path.join(moduledir, '.git/hooks/pre-push')) as git_hook_script: ++ content = git_hook_script.read() ++ pattern = '__main__.py -C "/home/conf/rhpkg.conf" pre-push-check "$local_sha"' ++ self.assertIn(pattern, content) ++ ++ shutil.rmtree(altpath) ++ + def test_clone_anonymous_with_branch(self): + self.make_new_git(self.module, + branches=['rpkg-tests-1', 'rpkg-tests-2']) +-- +2.39.2 + diff --git a/0013-Fix-unittests-for-clone-and-pre-push-hook-script.patch b/0013-Fix-unittests-for-clone-and-pre-push-hook-script.patch new file mode 100644 index 0000000..991fb22 --- /dev/null +++ b/0013-Fix-unittests-for-clone-and-pre-push-hook-script.patch @@ -0,0 +1,49 @@ +From 1d82b7eaf98e695689a7dc10bd308030e3c13eea Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Sat, 1 Apr 2023 01:34:34 +0200 +Subject: [PATCH] Fix unittests for `clone` and pre-push hook script + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + tests/commands/test_clone.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/tests/commands/test_clone.py b/tests/commands/test_clone.py +index 6ef1300..85fdfd1 100644 +--- a/tests/commands/test_clone.py ++++ b/tests/commands/test_clone.py +@@ -1,5 +1,6 @@ + import os + import shutil ++import sys + import tempfile + + import git +@@ -110,9 +111,10 @@ class CommandCloneTestCase(CommandTestCase): + moduledir = os.path.join(self.path, self.module) + self.assertTrue(os.path.isfile(os.path.join(moduledir, '.git/hooks/pre-push'))) + ++ clonned_by = os.path.basename(sys.argv[0]) + with open(os.path.join(moduledir, '.git/hooks/pre-push')) as git_hook_script: + content = git_hook_script.read() +- pattern = '__main__.py pre-push-check "$local_sha"' ++ pattern = '{0} pre-push-check "$local_sha"'.format(clonned_by) + self.assertIn(pattern, content) + + shutil.rmtree(altpath) +@@ -132,9 +134,11 @@ class CommandCloneTestCase(CommandTestCase): + moduledir = os.path.join(self.path, self.module) + self.assertTrue(os.path.isfile(os.path.join(moduledir, '.git/hooks/pre-push'))) + ++ clonned_by = os.path.basename(sys.argv[0]) + with open(os.path.join(moduledir, '.git/hooks/pre-push')) as git_hook_script: + content = git_hook_script.read() +- pattern = '__main__.py -C "/home/conf/rhpkg.conf" pre-push-check "$local_sha"' ++ pattern = '{0} -C "/home/conf/rhpkg.conf" pre-push-check ' \ ++ '"$local_sha"'.format(clonned_by) + self.assertIn(pattern, content) + + shutil.rmtree(altpath) +-- +2.39.2 + diff --git a/0014-import_srpm-allow-pre-generated-srpms.patch b/0014-import_srpm-allow-pre-generated-srpms.patch new file mode 100644 index 0000000..f596265 --- /dev/null +++ b/0014-import_srpm-allow-pre-generated-srpms.patch @@ -0,0 +1,105 @@ +From d87cb37fa2fea2ed535b9085a1f4c607083e1c2e Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Tue, 4 Apr 2023 01:40:23 +0200 +Subject: [PATCH 1/6] import_srpm: allow pre-generated srpms + +When active, do not care specfile in the srpm is processed by +rpmautospec. Can be activated only directly via pyrpkg 'Commands' +object. + +Relates: https://github.com/fedora-copr/copr/issues/2317 +Fixes: #655 +RHELCMP-11085 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 6 +++++- + tests/test_cli.py | 41 +++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 46 insertions(+), 1 deletion(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 9996402..ecb99c9 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -110,7 +110,7 @@ class Commands(object): + build_client, user=None, + dist=None, target=None, quiet=False, + distgit_namespaced=False, realms=None, lookaside_namespaced=False, +- git_excludes=None, results_dir='root'): ++ git_excludes=None, results_dir='root', allow_pre_generated_srpm=False): + """Init the object and some configuration details.""" + + # Path to operate on, most often pwd +@@ -239,6 +239,9 @@ class Commands(object): + # Layout setup + self.layout = layout.build(self.path, + 'resultsdir' if self.results_dir == 'subdir' else None) ++ # A Configuration value used in 'import_srpm' command (comes from the Copr team) ++ # If pre-generated srpms are allowed, don't care specfile is processed by rpmautospec ++ self.allow_pre_generated_srpm = allow_pre_generated_srpm + + # Define properties here + # Properties allow us to "lazy load" various attributes, which also means +@@ -1471,6 +1474,7 @@ class Commands(object): + # the dist-git repo without any specfiles - right after initialization) we are + # not able determine which the main specfile is. + if file.endswith('.spec') and not file.startswith('.') \ ++ and not self.allow_pre_generated_srpm \ + and spec_file_processed_by_rpmautospec(file, target_dir): + raise rpkgError('SRPM was processed by rpmautospec ' + '(specfile "{}" was analyzed)'.format(file)) +diff --git a/tests/test_cli.py b/tests/test_cli.py +index 868ad1f..02620ef 100644 +--- a/tests/test_cli.py ++++ b/tests/test_cli.py +@@ -1784,6 +1784,47 @@ class TestImportSrpm(LookasideCacheMock, CliTestCase): + self.assertFilesExist(['package.rpmlintrc'], search_dir=self.chaos_repo) + self.assertFilesNotExist(['the_file_is_not_in_reserved.yaml'], search_dir=self.chaos_repo) + ++ @patch('pyrpkg.spec_file_processed_by_rpmautospec') ++ def test_import_srpm_not_processed_by_rpmautospec(self, rpmautospec_processed): ++ cli_cmd = ['rpkg', '--path', self.chaos_repo, '--name', 'docpkg', ++ 'import', '--skip-diffs', self.srpm_file] ++ ++ rpmautospec_processed.return_value = False ++ with patch('sys.argv', new=cli_cmd): ++ cli = self.new_cli() ++ with patch('pyrpkg.lookaside.CGILookasideCache.upload', self.lookasidecache_upload): ++ cli.import_srpm() # no exception should be raised ++ rpmautospec_processed.assert_called_once() ++ ++ @patch('pyrpkg.spec_file_processed_by_rpmautospec') ++ def test_import_srpm_processed_by_rpmautospec(self, rpmautospec_processed): ++ cli_cmd = ['rpkg', '--path', self.chaos_repo, '--name', 'docpkg', ++ 'import', '--skip-diffs', self.srpm_file] ++ ++ rpmautospec_processed.return_value = True ++ with patch('sys.argv', new=cli_cmd): ++ cli = self.new_cli() ++ with patch('pyrpkg.lookaside.CGILookasideCache.upload', self.lookasidecache_upload): ++ six.assertRaisesRegex( ++ self, ++ rpkgError, ++ 'SRPM was processed by rpmautospec', ++ cli.import_srpm) ++ rpmautospec_processed.assert_called_once() ++ ++ @patch('pyrpkg.spec_file_processed_by_rpmautospec') ++ def test_import_srpm_processed_by_rpmautospec_allowed(self, rpmautospec_processed): ++ cli_cmd = ['rpkg', '--path', self.chaos_repo, '--name', 'docpkg', ++ 'import', '--skip-diffs', self.srpm_file] ++ ++ rpmautospec_processed.return_value = True ++ with patch('sys.argv', new=cli_cmd): ++ cli = self.new_cli() ++ cli.cmd.allow_pre_generated_srpm = True ++ with patch('pyrpkg.lookaside.CGILookasideCache.upload', self.lookasidecache_upload): ++ cli.import_srpm() # no exception should be raised ++ rpmautospec_processed.assert_not_called() ++ + + class TestMockbuild(CliTestCase): + """Test mockbuild command""" +-- +2.39.2 + diff --git a/0015-Ignore-missing-spec-file-in-pre-push-hook.patch b/0015-Ignore-missing-spec-file-in-pre-push-hook.patch new file mode 100644 index 0000000..b32833c --- /dev/null +++ b/0015-Ignore-missing-spec-file-in-pre-push-hook.patch @@ -0,0 +1,37 @@ +From 3ebfeae20c74de0ca4b26b22135e1996265ea4ce Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com> +Date: Wed, 5 Apr 2023 11:41:23 +0200 +Subject: [PATCH 2/6] Ignore missing spec file in pre-push hook +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +For modules or containers there will be no spec file, and there is +nothing to block the push on. + +Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com> +--- + pyrpkg/__init__.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index ecb99c9..d3a7a1c 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -4471,7 +4471,12 @@ class Commands(object): + # Assume, that specfile names are same in the active branch + # and in the pushed branch (git checkout f37 && git push origin rawhide) + # in this case 'f37' is active branch and 'rawhide' is pushed branch. +- specfile_path_absolute = os.path.join(self.layout.specdir, self.spec) ++ try: ++ specfile_path_absolute = os.path.join(self.layout.specdir, self.spec) ++ except rpkgError: ++ # No specfile found, nothing to check ++ return ++ + # convert to relative path + specfile_path = os.path.relpath(specfile_path_absolute, start=self.path) + cmd = ['spectool', '-l', os.path.join(clone_dir, specfile_path)] +-- +2.39.2 + diff --git a/0016-Check-remote-file-with-correct-hash.patch b/0016-Check-remote-file-with-correct-hash.patch new file mode 100644 index 0000000..9bf9eac --- /dev/null +++ b/0016-Check-remote-file-with-correct-hash.patch @@ -0,0 +1,61 @@ +From 4bd4ab1823a7d4bc218b8057b7f00808fabf7648 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com> +Date: Wed, 5 Apr 2023 09:34:33 +0200 +Subject: [PATCH 3/6] Check remote file with correct hash +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The configured hashtype doesn't have to actually be used. There can be +old repos that still use md5. + +JIRA: RHELCMP-11508 +Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com> +--- + pyrpkg/__init__.py | 3 ++- + pyrpkg/lookaside.py | 5 +++-- + 2 files changed, 5 insertions(+), 3 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index d3a7a1c..0b9a869 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -4548,7 +4548,8 @@ class Commands(object): + file_exists_in_lookaside = self.lookasidecache.remote_file_exists_head( + self.ns_repo_name if self.lookaside_namespaced else self.repo_name, + filename, +- hash) ++ hash, ++ hashtype=entry.hashtype) + if not file_exists_in_lookaside: + self.log.error('Source file (or tarball) \'{}\' wasn\'t uploaded to the lookaside ' + 'cache. Push operation was cancelled.'.format(filename)) +diff --git a/pyrpkg/lookaside.py b/pyrpkg/lookaside.py +index ecbf12b..3efcd88 100644 +--- a/pyrpkg/lookaside.py ++++ b/pyrpkg/lookaside.py +@@ -200,7 +200,7 @@ class CGILookasideCache(object): + if not self.file_is_valid(outfile, hash, hashtype=hashtype): + raise DownloadError('%s failed checksum' % filename) + +- def remote_file_exists_head(self, name, filename, hash): ++ def remote_file_exists_head(self, name, filename, hash, hashtype): + """Verify whether a file exists on the lookaside cache. + Uses a HTTP HEAD request and doesn't require authentication. + +@@ -209,10 +209,11 @@ class CGILookasideCache(object): + the server side expects). + :param str filename: The name of the file to check for. + :param str hash: The known good hash of the file. ++ :param str hashtype: The type of hash + """ + + urled_file = urllib.parse.quote(filename) +- url = self.get_download_url(name, urled_file, hash, self.hashtype) ++ url = self.get_download_url(name, urled_file, hash, hashtype or self.hashtype) + + c = pycurl.Curl() + c.setopt(pycurl.URL, url) +-- +2.39.2 + diff --git a/0017-Allow-empty-commits-when-uses_rpmautospec.patch b/0017-Allow-empty-commits-when-uses_rpmautospec.patch new file mode 100644 index 0000000..448cbc9 --- /dev/null +++ b/0017-Allow-empty-commits-when-uses_rpmautospec.patch @@ -0,0 +1,35 @@ +From d2c63c32306732695b7fe5f9dda3deecb7899f4f Mon Sep 17 00:00:00 2001 +From: Jiri Kyjovsky <j1.kyjovsky@gmail.com> +Date: Sat, 15 Apr 2023 13:46:21 +0200 +Subject: [PATCH 4/6] Allow empty commits when `uses_rpmautospec` + +To avoid situations where the command `commit` by default does +commit when no change in repo is present. + +Fixes: #677 +JIRA: RHELCMP-11489 +Merges: https://pagure.io/rpkg/pull-request/688 + +Signed-off-by: Jiri Kyjovsky <j1.kyjovsky@gmail.com> +--- + pyrpkg/__init__.py | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 0b9a869..187796e 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -1867,7 +1867,9 @@ class Commands(object): + + # construct the git command + # We do this via subprocess because the git module is terrible. +- cmd = ['git', 'commit', '--allow-empty'] ++ cmd = ['git', 'commit'] ++ if not self.is_retired() and self.uses_rpmautospec: ++ cmd.append('--allow-empty') + if signoff: + cmd.append('-s') + if self.quiet: +-- +2.39.2 + diff --git a/0018-Config-file-option-to-skip-the-hook-script-creation.patch b/0018-Config-file-option-to-skip-the-hook-script-creation.patch new file mode 100644 index 0000000..fa9aa01 --- /dev/null +++ b/0018-Config-file-option-to-skip-the-hook-script-creation.patch @@ -0,0 +1,126 @@ +From b48eb502d330ec7a543805d7f185ea270df75b90 Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Wed, 12 Apr 2023 00:42:04 +0200 +Subject: [PATCH 5/6] Config file option to skip the hook script creation + +A new option named "skip_hooks" can be added to the config file +(into the main section). It accepts boolean values and when +the option is present and set, it prevents creating the pre-push +hook script during cloning a dist-git repository. + +Fixes: https://pagure.io/fedpkg/issue/515 +JIRA: RHELCMP-11491 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 14 ++++++++++---- + pyrpkg/cli.py | 13 +++++++++++-- + 2 files changed, 21 insertions(+), 6 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 187796e..7fddff7 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -1571,7 +1571,7 @@ class Commands(object): + + def clone(self, repo, path=None, branch=None, bare_dir=None, + anon=False, target=None, depth=None, extra_args=None, +- config_path=None): ++ config_path=None, skip_hooks=None): + """Clone a repo, optionally check out a specific branch. + + :param str repo: the name of the repository to clone. +@@ -1589,6 +1589,7 @@ class Commands(object): + :param list extra_args: additional arguments that are passed to + the clone command. + :param str config_path: path to the global config file ++ :param bool skip_hooks: skip creation pre-push hook script + """ + + if not path: +@@ -1644,7 +1645,8 @@ class Commands(object): + + if not bare_dir: + self._add_git_excludes(os.path.join(path, git_dir)) +- self._add_git_pre_push_hook(os.path.join(path, git_dir), config_path) ++ if not skip_hooks: ++ self._add_git_pre_push_hook(os.path.join(path, git_dir), config_path) + + return + +@@ -1660,7 +1662,7 @@ class Commands(object): + return repo + + def clone_with_dirs(self, repo, anon=False, target=None, depth=None, +- extra_args=None, config_path=None): ++ extra_args=None, config_path=None, skip_hooks=None): + """Clone a repo old style with subdirs for each branch. + + :param str repo: name of the repository to clone. +@@ -1673,6 +1675,7 @@ class Commands(object): + :param list extra_args: additional arguments that are passed to + the clone command. + :param str config_path: path to the global config file ++ :param bool skip_hooks: skip creation pre-push hook script + """ + + self._push_url = None +@@ -1731,7 +1734,8 @@ class Commands(object): + + # Add excludes + self._add_git_excludes(branch_path) +- self._add_git_pre_push_hook(branch_path, config_path) ++ if not skip_hooks: ++ self._add_git_pre_push_hook(branch_path, config_path) + except (git.GitCommandError, OSError) as e: + raise rpkgError('Could not locally clone %s from %s: %s' + % (branch, repo_path, e)) +@@ -1820,6 +1824,8 @@ class Commands(object): + # This file was generated by {0} when cloning the repository. + # You can edit it to your liking or delete completely. It will not + # be recreated. ++ # Creating this file can be also prevented by adding an option ++ # "skip_hooks = True" into the {0}'s config file; [{0}] section. + + _remote="$1" + _url="$2" +diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py +index 3d8ce33..a1f3f44 100644 +--- a/pyrpkg/cli.py ++++ b/pyrpkg/cli.py +@@ -2177,13 +2177,21 @@ class cliClient(object): + self.log.warning("Repo name should't contain '.git' suffix. " + "Correcting the repo name: '%s'" % repo) + ++ skip_hooks = None ++ if self.config.has_option(self.name, "skip_hooks"): ++ try: ++ skip_hooks = self.config.getboolean(self.name, "skip_hooks") ++ except ValueError: ++ self.log.error("Error: config file option 'skip_hooks'") ++ raise + if self.args.branches: + self.cmd.clone_with_dirs(self.args.repo[0], + anon=self.args.anonymous, + target=self.args.clone_target, + depth=self.args.depth, + extra_args=self.extra_args, +- config_path=self.args.config) ++ config_path=self.args.config, ++ skip_hooks=skip_hooks) + else: + self.cmd.clone(self.args.repo[0], + branch=self.args.branch, +@@ -2191,7 +2199,8 @@ class cliClient(object): + target=self.args.clone_target, + depth=self.args.depth, + extra_args=self.extra_args, +- config_path=self.args.config) ++ config_path=self.args.config, ++ skip_hooks=skip_hooks) + + def commit(self): + if self.args.with_changelog and not self.args.message: +-- +2.39.2 + diff --git a/0019-Pre-push-hook-won-t-check-private-branches.patch b/0019-Pre-push-hook-won-t-check-private-branches.patch new file mode 100644 index 0000000..37a7c52 --- /dev/null +++ b/0019-Pre-push-hook-won-t-check-private-branches.patch @@ -0,0 +1,45 @@ +From 4553da364d7d8a974ab0c08834ee0a54320da2cb Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Tue, 18 Apr 2023 16:06:43 +0200 +Subject: [PATCH 6/6] Pre-push hook won't check private branches + +The pre-push hook script was failing the when user tried to push +a private branch. It required using the --release argument with +the pre-push-check command and passing additional arguments into +the hook script. That was found unreliable and private branches +won't be checked. + +Fixes: #683 +JIRA: RHELCMP-11528 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 7fddff7..3f934d3 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -4489,9 +4489,15 @@ class Commands(object): + specfile_path = os.path.relpath(specfile_path_absolute, start=self.path) + cmd = ['spectool', '-l', os.path.join(clone_dir, specfile_path)] + # extract just '--define' arguments from rpmdefines +- for opt, val in zip(self.rpmdefines[0::2], self.rpmdefines[1::2]): +- if opt == '--define': +- cmd.extend((opt, val)) ++ try: ++ for opt, val in zip(self.rpmdefines[0::2], self.rpmdefines[1::2]): ++ if opt == '--define': ++ cmd.extend((opt, val)) ++ except rpkgError: ++ # this exception was caused probably by using a private branch ++ self.log.warning('The pre-push script can\'t check private branches. ' ++ 'Push operation continues.') ++ return + ret, stdout, _ = self._run_command(cmd, cwd=clone_dir, + return_text=True, return_stdout=True) + if ret != 0: +-- +2.39.2 + diff --git a/0020-Use-release-s-rpmdefines-in-unused-sources-check.patch b/0020-Use-release-s-rpmdefines-in-unused-sources-check.patch new file mode 100644 index 0000000..6be7cee --- /dev/null +++ b/0020-Use-release-s-rpmdefines-in-unused-sources-check.patch @@ -0,0 +1,170 @@ +From 8667d5379161183b306bdd4a6733c666cd2ef310 Mon Sep 17 00:00:00 2001 +From: Otto Liljalaakso <otto.liljalaakso@iki.fi> +Date: Sun, 2 Apr 2023 17:21:00 +0300 +Subject: [PATCH 1/2] Use release's rpmdefines in unused sources check + +Conditional Source: tags are problematic and, in fact, forbidden in at +least Fedora. However, there are packages that conditionalize packages +based on macros such as %{rhel} or %{fedora}. 'x-pkg sources' did not +handle such packages correctly, because when the specfile was parsed +to check for unused sources, values for those macros were not set. This +was different from other commands which set such macros based on the +value of --release parameter or Git branch name. + +Improve support for conditional Source: tags by using the standard set +of rpmdefines when the specfile is parsed in 'fedpkg sources'. + +Fixes: #671 +JIRA: RHELCMP-11465 +Merges: https://pagure.io/rpkg/pull-request/678 + +Signed-off-by: Otto Liljalaakso <otto.liljalaakso@iki.fi> +--- + pyrpkg/__init__.py | 21 +++++++++++++++------ + pyrpkg/spec.py | 12 +++++++----- + tests/test_cli.py | 21 ++++++++++++++++++++- + tests/test_spec.py | 8 ++++++-- + 4 files changed, 48 insertions(+), 14 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 3f934d3..817ef33 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -2261,13 +2261,22 @@ class Commands(object): + sourcesf = SourcesFile(self.sources_filename, self.source_entry_type) + + try: +- specf = SpecFile(os.path.join(self.layout.specdir, self.spec), +- self.layout.sourcedir) +- spec_parsed = True +- except Exception: +- self.log.warning("Parsing specfile for used sources failed. " +- "Falling back to downloading all sources.") ++ # Try resolving rpmdefines separately. This produces a clear error ++ # message in the common failure case of custom branch name. ++ self.rpmdefines ++ except Exception as err: ++ self.log.warning("Parsing specfile for used sources failed: %s" % err) ++ self.log.warning("Falling back to downloading all sources.") + spec_parsed = False ++ else: ++ try: ++ specf = SpecFile(os.path.join(self.layout.specdir, self.spec), ++ self.rpmdefines) ++ spec_parsed = True ++ except Exception: ++ self.log.warning("Parsing specfile for used sources failed. " ++ "Falling back to downloading all sources.") ++ spec_parsed = False + + args = dict() + if self.lookaside_request_params: +diff --git a/pyrpkg/spec.py b/pyrpkg/spec.py +index d72f1fb..5400de3 100644 +--- a/pyrpkg/spec.py ++++ b/pyrpkg/spec.py +@@ -18,16 +18,16 @@ class SpecFile(object): + r'^((source[0-9]*|patch[0-9]*)\s*:\s*(?P<val>.*))\s*$', + re.IGNORECASE) + +- def __init__(self, spec, sourcedir): ++ def __init__(self, spec, rpmdefines): + self.spec = spec +- self.sourcedir = sourcedir ++ self.rpmdefines = rpmdefines + self.sources = [] + + self.parse() + + def parse(self): + """Call rpmspec and find source tags from the result.""" +- stdout = run(self.spec, self.sourcedir) ++ stdout = run(self.spec, self.rpmdefines) + for line in stdout.splitlines(): + m = self.sourcefile_expression.match(line) + if not m: +@@ -38,8 +38,10 @@ class SpecFile(object): + self.sources.append(val) + + +-def run(spec, sourcedir): +- cmdline = ['rpmspec', '--define', "_sourcedir %s" % sourcedir, '-P', spec] ++def run(spec, rpmdefines): ++ cmdline = ['rpmspec'] ++ cmdline.extend(rpmdefines) ++ cmdline.extend(['-P', spec]) + try: + process = subprocess.Popen(cmdline, + stdout=subprocess.PIPE, +diff --git a/tests/test_cli.py b/tests/test_cli.py +index 02620ef..58df047 100644 +--- a/tests/test_cli.py ++++ b/tests/test_cli.py +@@ -1607,6 +1607,25 @@ class TestSources(LookasideCacheMock, CliTestCase): + def test_unused_sources_are_not_downloaded(self): + self._upload_unused() + ++ cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'sources'] ++ with patch('sys.argv', new=cli_cmd): ++ with patch('pyrpkg.Commands.rpmdefines', ++ new=['--define', '_sourcedir %s' % self.cloned_repo_path]): ++ cli = self.new_cli() ++ with patch('pyrpkg.lookaside.CGILookasideCache.download', ++ new=self.lookasidecache_download): ++ cli.sources() ++ ++ path = os.path.join(self.cloned_repo_path, 'unused.patch') ++ self.assertFalse(os.path.exists(path)) ++ ++ @patch('pyrpkg.Commands.load_rpmdefines') ++ def test_download_sources_including_unused(self, rpmdefines): ++ self._upload_unused() ++ # SpecFile parsing executes 'rpmspec', that needs '--define' arguments from rpmdefines ++ # when rpmdefines raises eception, SpecFile parsing fails --> all sources are downloaded. ++ rpmdefines.side_effect = rpkgError ++ + cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'sources'] + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() +@@ -1615,7 +1634,7 @@ class TestSources(LookasideCacheMock, CliTestCase): + cli.sources() + + path = os.path.join(self.cloned_repo_path, 'unused.patch') +- self.assertFalse(os.path.exists(path)) ++ self.assertTrue(os.path.exists(path)) + + def test_force_option_downloads_unused_sources(self): + self._upload_unused() +diff --git a/tests/test_spec.py b/tests/test_spec.py +index eefc475..0c7907a 100644 +--- a/tests/test_spec.py ++++ b/tests/test_spec.py +@@ -10,6 +10,10 @@ from pyrpkg.errors import rpkgError + class SpecFileTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') ++ self.rpmdefines = ["--define", "_sourcedir %s" % self.workdir, ++ "--define", "_specdir %s" % self.workdir, ++ "--define", "_builddir %s" % self.workdir, ++ "--eval", "%%undefine rhel"] + self.specfile = os.path.join(self.workdir, self._testMethodName) + + # Write common header +@@ -43,7 +47,7 @@ class SpecFileTestCase(unittest.TestCase): + "PAtch999: https://remote.patch-sourcce.org/another-patch.bz2\n") + spec_fd.close() + +- s = spec.SpecFile(self.specfile, self.workdir) ++ s = spec.SpecFile(self.specfile, self.rpmdefines) + actual = s.sources + expected = [ + "tarball.tar.gz", +@@ -65,4 +69,4 @@ class SpecFileTestCase(unittest.TestCase): + self.assertRaises(rpkgError, + spec.SpecFile, + self.specfile, +- self.workdir) ++ self.rpmdefines) +-- +2.40.0 + diff --git a/0021-Do-not-require-sources-file-for-all-namespaces.patch b/0021-Do-not-require-sources-file-for-all-namespaces.patch new file mode 100644 index 0000000..a4c71aa --- /dev/null +++ b/0021-Do-not-require-sources-file-for-all-namespaces.patch @@ -0,0 +1,60 @@ +From 079a64dde258f45e26fe35de86b1a0915f4973cd Mon Sep 17 00:00:00 2001 +From: Ondrej Nosek <onosek@redhat.com> +Date: Thu, 27 Apr 2023 23:05:48 +0200 +Subject: [PATCH 2/2] Do not require 'sources' file for all namespaces + +Requirement for 'sources' file for all layouts except the RetiredLayout +(and thus all namespaces) was too restrictive and unexpected. +Partially reverts the commit 1108810bdefd0d880517b274acd6a3bd0d4156e0. + +Fixes: #684 +JIRA: RHELCMP-11529 + +Signed-off-by: Ondrej Nosek <onosek@redhat.com> +--- + pyrpkg/__init__.py | 2 -- + pyrpkg/cli.py | 1 - + tests/test_cli.py | 2 +- + 3 files changed, 1 insertion(+), 4 deletions(-) + +diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py +index 817ef33..11b8dae 100644 +--- a/pyrpkg/__init__.py ++++ b/pyrpkg/__init__.py +@@ -1168,8 +1168,6 @@ class Commands(object): + + @property + def sources_filename(self): +- if self.layout is None or isinstance(self.layout, layout.IncompleteLayout): +- raise rpkgError('Spec file is not available') + if isinstance(self.layout, layout.RetiredLayout): + raise rpkgError('This package or module is retired. The action has stopped.') + return os.path.join( +diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py +index a1f3f44..dc1eb4e 100644 +--- a/pyrpkg/cli.py ++++ b/pyrpkg/cli.py +@@ -2375,7 +2375,6 @@ class cliClient(object): + + def import_srpm(self): + uploadfiles = self.cmd.import_srpm(self.args.srpm) +- self.load_cmd() # to reload layouts - because a specfile could appear during import + if uploadfiles: + try: + self.cmd.upload(uploadfiles, replace=True, offline=self.args.offline) +diff --git a/tests/test_cli.py b/tests/test_cli.py +index 58df047..6e4ec6a 100644 +--- a/tests/test_cli.py ++++ b/tests/test_cli.py +@@ -1610,7 +1610,7 @@ class TestSources(LookasideCacheMock, CliTestCase): + cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'sources'] + with patch('sys.argv', new=cli_cmd): + with patch('pyrpkg.Commands.rpmdefines', +- new=['--define', '_sourcedir %s' % self.cloned_repo_path]): ++ new=['--define', '_sourcedir %s' % self.cloned_repo_path]): + cli = self.new_cli() + with patch('pyrpkg.lookaside.CGILookasideCache.download', + new=self.lookasidecache_download): +-- +2.40.0 + @@ -41,6 +41,21 @@ Patch3: 0003-Remove-Environment-Markers-syntax.patch Patch4: 0004-Process-source-URLs-with-fragment-in-pre-push-hook.patch Patch5: 0005-container-build-update-signing-intent-help-for-OSBS-.patch Patch6: 0006-Do-not-generate-pre-push-hook-script-in-some-cases.patch +Patch7: 0007-More-robust-spec-file-presence-checking.patch +Patch8: 0008-Update-to-spec-file-presence-checking.patch +Patch9: 0009-Add-more-information-about-pre-push-hook.patch +Patch10: 0010-pre-push-check-have-to-use-spectool-with-define.patch +Patch11: 0011-A-HEAD-query-into-a-lookaside-cache.patch +Patch12: 0012-pre-push-hook-script-contains-a-user-s-config.patch +Patch13: 0013-Fix-unittests-for-clone-and-pre-push-hook-script.patch +Patch14: 0014-import_srpm-allow-pre-generated-srpms.patch +Patch15: 0015-Ignore-missing-spec-file-in-pre-push-hook.patch +Patch16: 0016-Check-remote-file-with-correct-hash.patch +Patch17: 0017-Allow-empty-commits-when-uses_rpmautospec.patch +Patch18: 0018-Config-file-option-to-skip-the-hook-script-creation.patch +Patch19: 0019-Pre-push-hook-won-t-check-private-branches.patch +Patch20: 0020-Use-release-s-rpmdefines-in-unused-sources-check.patch +Patch21: 0021-Do-not-require-sources-file-for-all-namespaces.patch %description Python library for interacting with rpm+git |