diff options
| -rw-r--r-- | copr-backend.spec | 67 | ||||
| -rw-r--r-- | euler_msgbus.patch | 83 | ||||
| -rw-r--r-- | helpers.patch | 35 | ||||
| -rw-r--r-- | print_queues.patch | 12 | ||||
| -rw-r--r-- | signatrust_bin.patch | 105 | ||||
| -rw-r--r-- | sources | 4 | ||||
| -rw-r--r-- | support_signatrust_backend.patch | 825 |
7 files changed, 1079 insertions, 52 deletions
diff --git a/copr-backend.spec b/copr-backend.spec index 86a223c..ca104ea 100644 --- a/copr-backend.spec +++ b/copr-backend.spec @@ -3,14 +3,14 @@ %endif %global prunerepo_version 1.20 -%global tests_version 5 +%global tests_version 2 %global tests_tar test-data-copr-backend -%global copr_common_version 0.20.1.dev1 +%global copr_common_version 0.19 Name: copr-backend -Version: 1.173 -Release: 2%{?dist} +Version: 1.168 +Release: 1%{?dist} Summary: Backend for Copr License: GPL-2.0-or-later @@ -21,6 +21,12 @@ URL: https://github.com/fedora-copr/copr # tito build --tgz --tag %%name-%%version-%%release Source0: %{name}-%{version}.tar.gz Source1: https://github.com/fedora-copr/%{tests_tar}/archive/v%{tests_version}/%{tests_tar}-%{tests_version}.tar.gz +Patch1: helpers.patch +Patch2: euler_msgbus.patch +Patch3: print_queues.patch +#Patch4: redis_helpers.patch # should patch to copr common +Patch5: support_signatrust_backend.patch +Patch6: signatrust_bin.patch BuildArch: noarch BuildRequires: asciidoc @@ -37,7 +43,7 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools BuildRequires: python3-copr -BuildRequires: python3-copr-common >= %copr_common_version +BuildRequires: python3-copr-common = %copr_common_version BuildRequires: python3-daemon BuildRequires: python3-dateutil BuildRequires: python3-distro @@ -56,7 +62,6 @@ BuildRequires: python3-retask BuildRequires: python3-setproctitle BuildRequires: python3-sphinx BuildRequires: python3-tabulate -BuildRequires: python3-zstandard BuildRequires: modulemd-tools >= 0.6 BuildRequires: prunerepo >= %prunerepo_version BuildRequires: dnf @@ -75,7 +80,7 @@ Requires: obs-signd Requires: openssh-clients Requires: prunerepo >= %prunerepo_version Requires: python3-copr -Requires: python3-copr-common >= %copr_common_version +Requires: python3-copr-common = %copr_common_version Recommends: python3-copr-messaging Requires: python3-daemon Requires: python3-dateutil @@ -124,7 +129,7 @@ only. %prep -%setup -q -a 1 +%autosetup -p1 %build @@ -174,6 +179,8 @@ install -d %{buildroot}%{_sysconfdir}/logstash.d install -d %{buildroot}%{_datadir}/logstash/patterns/ cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern +cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/ + install -d %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d/copr_backend.conf @@ -216,6 +223,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %config(noreplace) %{_sysconfdir}/logrotate.d/copr-backend %dir %{_pkgdocdir} %doc %{_pkgdocdir}/lighttpd +%doc %{_pkgdocdir}/playbooks %dir %{_sysconfdir}/copr %config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf %{_unitdir}/*.service @@ -236,48 +244,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %doc %{_pkgdocdir}/ %exclude %{_pkgdocdir}/lighttpd +%exclude %{_pkgdocdir}/playbooks %changelog -* Fri Feb 21 2025 mywaaagh_admin <pkwarcraft@hotmail.com> 1.173-2 -- fix for openeuler - -* Thu Nov 23 2023 Pavel Raiskup <praiskup@redhat.com> 1.173-1 -- enforce createrepo_c gzip compression (f39+ switched to zstd) -- self-identify the resalloc resource in logs -- dropping the documentary playbooks from copr-backend payload -- nicer unknown-resalloc-tickets output -- worker to not call keygen for source builds at all -- don't sign products of srpm-build -- longer timeout for fallback generating GPG keys after build -- recreate missing repodata so that prunerepo doesn't traceback -- use the rename HashWorkerLimit instead of GroupWorkerLimit -- provide per-arch & per-owner worker limit implemented -- collect and compress fedora-review logs after run -- react on staled SSH connections in some cases - -* Tue Aug 15 2023 Pavel Raiskup <praiskup@redhat.com> 1.172-1 -- dump the /update/ payload to worker.log -- don't run external command(s) to collect built packages -- don't eat the "build detail collecting" traceback -- fixes in the unknown-resalloc-tickets.py helper -- more careful format_evr() method -- fix tests for zst compression on F39+ -- log task dict in case of error returned from redis -- skip builds for ExcludeArch and "not" ExclusiveArch -- offload NEVRA (s)rpm parsing to copr-rpmbuild -- redis authentication support added - -* Tue Jun 06 2023 Pavel Raiskup <praiskup@redhat.com> 1.171-1 -- copr_prune_results.py: work-around the arg_max problem - -* Mon Jun 05 2023 Pavel Raiskup <praiskup@redhat.com> 1.170-1 -- copr_prune_results.py: don't enforce appstream-builder, ask FE -- copr_prune_results.py: just one API call to FE per one repo - -* Tue May 23 2023 Jakub Kadlcik <frostyx@email.cz> 1.169-1 -- Forking: better handle FileExistsError -- Run the copr-rpmbuild command with task URL, not build ID - * Wed Apr 05 2023 Jiri Kyjovsky <j1.kyjovsky@gmail.com> 1.168-1 - Bump version for release mess @@ -1680,5 +1649,3 @@ copr_log_hitcounter * Mon Jun 17 2013 Miroslav Suchý <msuchy@redhat.com> 1.1-1 - new package built with tito - - diff --git a/euler_msgbus.patch b/euler_msgbus.patch new file mode 100644 index 0000000..8973ce1 --- /dev/null +++ b/euler_msgbus.patch @@ -0,0 +1,83 @@ +diff --git a/backend/copr_backend/euler_msgbus.py b/backend/copr_backend/euler_msgbus.py +new file mode 100644 +index 000000000..1395249be +--- /dev/null ++++ b/backend/copr_backend/euler_msgbus.py +@@ -0,0 +1,77 @@ ++import socket ++ ++from kafka import KafkaProducer ++import os ++import datetime ++import uuid ++import json ++import ssl ++ ++ ++def message_from_worker_job(topic, job, who, ip, pid): ++ message = {} ++ content = { ++ 'user': job.submitter, ++ 'copr': job.project_name, ++ 'owner': job.project_owner, ++ 'pkg': job.package_name, ++ 'build': job.build_id, ++ 'chroot': job.chroot, ++ 'version': job.package_version, ++ 'status': job.status, ++ } ++ content.update({'ip': ip, 'who': who, 'pid': pid}) ++ message_types = { ++ 'build.start': { ++ 'what': "build start: user:{user} copr:{copr}" \ ++ " pkg:{pkg} build:{build} ip:{ip} pid:{pid}", ++ }, ++ 'chroot.start': { ++ 'what': "chroot start: chroot:{chroot} user:{user}" \ ++ " copr:{copr} pkg:{pkg} build:{build} ip:{ip} pid:{pid}", ++ }, ++ 'build.end': { ++ 'what': "build end: user:{user} copr:{copr} build:{build}" \ ++ " pkg:{pkg} version:{version} ip:{ip} pid:{pid} status:{status}", ++ }, ++ } ++ content['what'] = message_types[topic]['what'].format(**content) ++ message['body'] = content ++ now = datetime.datetime.now().isoformat() ++ headers = { ++ "openEuler_messaging_schema": "eur." + topic, ++ "sent-at": now, ++ } ++ message['headers'] = headers ++ message['id'] = str(uuid.uuid4()) ++ message['topic'] = "org.openEuler.prod.eur." + topic ++ return message ++ ++ ++class MessageSender: ++ def __init__(self, backend_opts, name, log): ++ self.log = log ++ self.name = name ++ self.pid = os.getpid() ++ self.opts = backend_opts ++ ++ def announce(self, topic, job, host): ++ msg = message_from_worker_job(topic, job, self.name, host, self.pid) ++ self.send_message(msg) ++ ++ def send_message(self, msg): ++ """ Send message to kafka """ ++ ++ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ++ context.verify_mode = ssl.CERT_REQUIRED ++ context.load_verify_locations("etc/copr/kafka.crt") ++ producer = KafkaProducer(bootstrap_servers=self.opts.message.bootstrap_servers, ++ api_version=(3, 5, 0), ++ sasl_mechanism="PLAIN", ++ ssl_context=context, ++ security_protocol='SASL_SSL', ++ sasl_plain_username=self.opts.message.user_name, ++ sasl_plain_password=self.opts.message.password, ++ value_serializer=lambda v: json.dumps(v).encode('utf-8')) ++ producer.send(str(self.opts.message.topic), msg) ++ producer.flush() diff --git a/helpers.patch b/helpers.patch new file mode 100644 index 0000000..33be500 --- /dev/null +++ b/helpers.patch @@ -0,0 +1,35 @@ +diff --git a/copr_backend/helpers.py b/copr_backend/helpers.py +index df2819e0..c85c7396 100644 +--- a/copr_backend/helpers.py ++++ b/copr_backend/helpers.py +@@ -288,6 +288,9 @@ class BackendConfigReader(object): + opts.redis_port = _get_conf( + cp, "backend", "redis_port", "6379") + ++ opts.redis_pwd = _get_conf( ++ cp, "backend", "redis_password", None) ++ + opts.redis_db = _get_conf( + cp, "backend", "redis_db", "0") + + +@@ -416,6 +416,19 @@ class BackendConfigReader(object): + opts.aws_cloudfront_distribution = _get_conf( + cp, "backend", "aws_cloudfront_distribution", None) + ++ opts.message = Munch() ++ opts.message.bootstrap_servers = _get_conf( ++ cp, "message", "bootstrap_servers", None) ++ ++ opts.message.user_name = _get_conf( ++ cp, "message", "user_name", None) ++ ++ opts.message.password = _get_conf( ++ cp, "message", "password", None) ++ ++ opts.message.topic = _get_conf( ++ cp, "message", "topic", None) ++ + # ssh options + opts.ssh = Munch() + opts.ssh.builder_config = _get_conf( diff --git a/print_queues.patch b/print_queues.patch new file mode 100644 index 0000000..83d7fc8 --- /dev/null +++ b/print_queues.patch @@ -0,0 +1,12 @@ +diff --git a/run/print_queues.py b/run/print_queues.py +index 849f8f6a..f48d1157 100755 +--- a/run/print_queues.py ++++ b/run/print_queues.py +@@ -14,6 +14,7 @@ redis_config = { + 'host': opts['redis_host'], + 'port': opts['redis_port'], + 'db': opts['redis_db'], ++ 'password': opts['redis_pwd'], + } + + for i in range(0, NUM_QUEUES): diff --git a/signatrust_bin.patch b/signatrust_bin.patch new file mode 100644 index 0000000..fcba220 --- /dev/null +++ b/signatrust_bin.patch @@ -0,0 +1,105 @@ +diff --git a/run/copr_fix_gpg.py b/run/copr_fix_gpg.py +index 24edf74e..bfa70c05 100755 +--- a/run/copr_fix_gpg.py ++++ b/run/copr_fix_gpg.py +@@ -9,7 +9,7 @@ from urllib.parse import urlparse + import pwd + + from copr_backend.helpers import BackendConfigReader, call_copr_repo, run_cmd +-from copr_backend.sign import get_pubkey, unsign_rpms_in_dir, sign_rpms_in_dir, create_user_keys, create_gpg_email ++from copr_backend.sign import new_signer, create_gpg_email + + logging.basicConfig( + filename="/var/log/copr-backend/fix_gpg.log", +@@ -69,12 +69,14 @@ def fix_copr(args, opts, copr_full_name): + log.info('Ignoring %s. Directory does not exist.', copr_path) + return + ++ signer = new_signer(opts) ++ + log.info("Generate key-pair on copr-keygen (if not generated) for email %s", + create_gpg_email(owner, coprname, opts.sign_domain)) +- create_user_keys(owner, coprname, opts) ++ signer.create_user_keys(owner, coprname, opts) + + log.info("Regenerate pubkey.gpg in copr %s", copr_path) +- get_pubkey(owner, coprname, log, opts.sign_domain, os.path.join(copr_path, 'pubkey.gpg')) ++ signer.get_pubkey(owner, coprname, log, opts.sign_domain, os.path.join(copr_path, 'pubkey.gpg')) + + # Match the "00001231-anycharacer" directory names. Compile once, use many. + builddir_matcher = re.compile(r"\d{8,}-") +@@ -111,8 +113,8 @@ def fix_copr(args, opts, copr_full_name): + + log.info("Processing rpms in builddir %s", builddir_path) + try: +- unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign +- sign_rpms_in_dir(owner, coprname, builddir_path, chroot, opts, log) ++ signer.unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign ++ signer.sign_rpms_in_dir(owner, coprname, builddir_path, chroot, opts, log) + except Exception as e: + log.exception(str(e)) + continue +diff --git a/run/copr_sign_unsigned.py b/run/copr_sign_unsigned.py +index 33eaeae3..e17e65bb 100755 +--- a/run/copr_sign_unsigned.py ++++ b/run/copr_sign_unsigned.py +@@ -12,7 +12,7 @@ import pwd + + from copr_backend.helpers import (BackendConfigReader, create_file_logger, + uses_devel_repo, call_copr_repo) +-from copr_backend.sign import get_pubkey, sign_rpms_in_dir, create_user_keys ++from copr_backend.sign import new_signer + from copr_backend.exceptions import CoprSignNoKeyError + + +@@ -26,10 +26,11 @@ log = logging.getLogger(__name__) + def check_signed_rpms_in_pkg_dir(pkg_dir, user, project, opts, chroot_dir, devel): + success = True + ++ signer = new_signer(opts) + logger = create_file_logger("run.check_signed_rpms_in_pkg_dir", + "/tmp/copr_check_signed_rpms.log") + try: +- sign_rpms_in_dir(user, project, pkg_dir, chroot_dir, opts, log=logger) ++ signer.sign_rpms_in_dir(user, project, pkg_dir, chroot_dir, opts, log=logger) + log.info("running createrepo for %s", pkg_dir) + call_copr_repo(directory=chroot_dir, devel=devel, logger=log) + except Exception as err: +@@ -80,9 +81,10 @@ def check_pubkey(pubkey_path, user, project, opts): + log.info("Pubkey for %s/%s exists: %s", user, project, pubkey_path) + return True + else: ++ signer = new_signer(opts) + log.info("Missing pubkey for %s/%s", user, project) + try: +- get_pubkey(user, project, log, opts.sign_domain, pubkey_path) ++ signer.get_pubkey(user, project, log, opts.sign_domain, pubkey_path) + return True + except Exception as err: + log.exception(err) +@@ -102,6 +104,8 @@ def main(): + opts = BackendConfigReader().read() + log.info("Starting pubkey fill, destdir: %s", opts.destdir) + ++ signer = new_signer(opts) ++ + log.debug("list dir: %s", os.listdir(opts.destdir)) + for user_name in os.listdir(opts.destdir): + if user_name in users_done_old: +@@ -116,13 +120,13 @@ def main(): + log.info("Checking project dir: %s", project_name) + + try: +- get_pubkey(user_name, project_name, log, opts.sign_domain) ++ signer.get_pubkey(user_name, project_name, log, opts.sign_domain) + log.info("Key-pair exists for %s/%s", user_name, project_name) + except CoprSignNoKeyError: +- create_user_keys(user_name, project_name, opts) ++ signer.create_user_keys(user_name, project_name, opts) + log.info("Created new key-pair for %s/%s", user_name, project_name) + except Exception as err: +- log.error("Failed to get pubkey for {}/{}, mark as failed, skipping") ++ log.error("Failed to get pubkey for {}/{}, mark as failed, skipping".format(user_name, project_name)) + log.exception(err) + failed = True + continue @@ -1,2 +1,2 @@ -c3ce07d660a16ccb0c1c2e1f3fbcfcd8 copr-backend-1.173.tar.gz -c94d55e21a4935b3e62f673fbae65af2 test-data-copr-backend-5.tar.gz +8dacd18f46475296d0199365db977649 copr-backend-1.168.tar.gz +a35697a68640fb4c0bcfef02a93eac63 test-data-copr-backend-2.tar.gz diff --git a/support_signatrust_backend.patch b/support_signatrust_backend.patch new file mode 100644 index 0000000..9e1fde5 --- /dev/null +++ b/support_signatrust_backend.patch @@ -0,0 +1,825 @@ +diff --git a/copr_backend/actions.py b/copr_backend/actions.py +index 39722b84..6da2dfb6 100644 +--- a/copr_backend/actions.py ++++ b/copr_backend/actions.py +@@ -21,13 +21,13 @@ from copr_common.worker_manager import WorkerManager + + from copr_backend.worker_manager import BackendQueueTask + +-from .sign import create_user_keys, CoprKeygenRequestError ++from .sign import CoprKeygenRequestError + from .exceptions import CreateRepoError, CoprSignError, FrontendClientException + from .helpers import (get_redis_logger, silent_remove, ensure_dir_exists, + get_chroot_arch, format_filename, + uses_devel_repo, call_copr_repo, build_chroot_log_name, + copy2_but_hardlink_rpms) +-from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey ++from .sign import new_signer + + + class Action(object): +@@ -92,6 +92,8 @@ class Action(object): + + self.log = log if log else get_redis_logger(self.opts, "backend.actions", "actions") + ++ self.signer = new_signer(opts) ++ + def __str__(self): + return "<{}(Action): {}>".format(self.__class__.__name__, self.data) + +@@ -147,7 +149,7 @@ class GPGMixin(object): + # skip key creation, most probably sign component is unused + return True + try: +- create_user_keys(ownername, projectname, self.opts) ++ self.signer.create_user_keys(ownername, projectname, self.opts) + return True + except CoprKeygenRequestError as e: + self.log.exception(e) +@@ -176,7 +178,7 @@ class Fork(Action, GPGMixin): + # Generate brand new gpg key. + self.generate_gpg_key(data["user"], data["copr"]) + # Put the new public key into forked build directory. +- get_pubkey(data["user"], data["copr"], self.log, self.opts.sign_domain, pubkey_path) ++ self.signer.get_pubkey(data["user"], data["copr"], self.log, self.opts.sign_domain, pubkey_path) + + chroot_paths = set() + for chroot, src_dst_dir in builds_map.items(): +@@ -206,9 +208,9 @@ class Fork(Action, GPGMixin): + continue + + # Drop old signatures coming from original repo and re-sign. +- unsign_rpms_in_dir(dst_path, opts=self.opts, log=self.log) ++ self.signer.unsign_rpms_in_dir(dst_path, opts=self.opts, log=self.log) + if sign: +- sign_rpms_in_dir(data["user"], data["copr"], dst_path, ++ self.signer.sign_rpms_in_dir(data["user"], data["copr"], dst_path, + chroot, opts=self.opts, log=self.log) + + self.log.info("Forked build %s as %s", src_path, dst_path) +diff --git a/copr_backend/background_worker.py b/copr_backend/background_worker.py +index 4b5f1313..bc05edbd 100644 +--- a/copr_backend/background_worker.py ++++ b/copr_backend/background_worker.py +@@ -9,7 +9,7 @@ import logging + from copr_common.background_worker import BackgroundWorker + from copr_backend.frontend import FrontendClient + from copr_backend.helpers import (BackendConfigReader, get_redis_logger) +- ++from copr_backend.sign import new_signer + + class BackendBackgroundWorker(BackgroundWorker): + """ +@@ -28,6 +28,7 @@ class BackendBackgroundWorker(BackgroundWorker): + + self.frontend_client = FrontendClient(self.opts, self.log, + try_indefinitely=True) ++ self.signer = new_signer(self.opts) + + def _switch_logger_to_redis(self): + logger_name = '{}.{}.pid-{}'.format( +diff --git a/copr_backend/background_worker_build.py b/copr_backend/background_worker_build.py +index 233549b4..afc948e8 100644 +--- a/copr_backend/background_worker_build.py ++++ b/copr_backend/background_worker_build.py +@@ -27,7 +27,7 @@ from copr_backend.helpers import ( + ) + from copr_backend.job import BuildJob +-from copr_backend.msgbus import MessageSender ++from copr_backend.euler_msgbus import MessageSender +-from copr_backend.sign import sign_rpms_in_dir, get_pubkey ++from copr_backend.sign import new_signer + from copr_backend.sshcmd import SSHConnection, SSHConnectionError + from copr_backend.vm_alloc import ResallocHostFactory + +@@ -599,7 +599,7 @@ class BuildBackgroundWorker(BackendBackgroundWorker): + self.log.info("Going to sign pkgs from source: %s in chroot: %s", + self.job.task_id, self.job.chroot_dir) + +- sign_rpms_in_dir( ++ self.signer.sign_rpms_in_dir( + self.job.project_owner, + self.job.project_name, + os.path.join(self.job.chroot_dir, self.job.target_dir_name), +@@ -712,7 +712,7 @@ class BuildBackgroundWorker(BackendBackgroundWorker): + # TODO: uncomment this when key revoke/change will be implemented + # if os.path.exists(pubkey_path): + # return +- get_pubkey(user, project, self.log, self.opts.sign_domain, pubkey_path) ++ self.signer.get_pubkey(user, project, self.log, self.opts.sign_domain, pubkey_path) + self.log.info("Added pubkey for user %s project %s into: %s", + user, project, pubkey_path) + +diff --git a/copr_backend/constants.py b/copr_backend/constants.py +index a529be28..83bcb8fb 100644 +--- a/copr_backend/constants.py ++++ b/copr_backend/constants.py +@@ -13,6 +13,7 @@ DEF_BUILD_USER = "mockbuilder" + DEF_DESTDIR = os.getcwd() + DEF_MACROS = {} + DEF_BUILDROOT_PKGS = "" ++DEF_SIGN_BACKEND = "obs-signd" + + + DEF_CONSECUTIVE_FAILURE_THRESHOLD = 10 +diff --git a/copr_backend/exceptions.py b/copr_backend/exceptions.py +index 21afb14c..0865fcc8 100644 +--- a/copr_backend/exceptions.py ++++ b/copr_backend/exceptions.py +@@ -48,8 +48,8 @@ class CoprKeygenRequestError(Exception): + + def __str__(self): + out = super(CoprKeygenRequestError, self).__str__() +- out += "\nrequest to copr-keygen: {}\n".format(self.request) +- if self.response: ++ out += "\nrequest to key_backend: {}\n".format(self.request) ++ if self.response is not None: + out += "status code: {}\n" "response content: {}\n" \ + .format(self.response.status_code, self.response.content) + return out +diff --git a/copr_backend/helpers.py b/copr_backend/helpers.py +index 05348366..db4af7f4 100644 +--- a/copr_backend/helpers.py ++++ b/copr_backend/helpers.py +@@ -31,7 +31,7 @@ from munch import Munch + from copr_common.redis_helpers import get_redis_connection + from copr.v3 import Client + from copr_backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \ +- CONSECUTIVE_FAILURE_REDIS_KEY, default_log_format ++ CONSECUTIVE_FAILURE_REDIS_KEY, default_log_format, DEF_SIGN_BACKEND + from copr_backend.exceptions import CoprBackendError, CoprBackendSrpmError + + from . import constants +@@ -309,6 +309,18 @@ class BackendConfigReader(object): + opts.sign_domain = _get_conf( + cp, "backend", "sign_domain", DOMAIN) + ++ opts.sign_backend = _get_conf( ++ cp, "backend", "sign_backend", DEF_SIGN_BACKEND) ++ ++ opts.signatrust_host = _get_conf( ++ cp, "backend", "signatrust_host", "") ++ ++ opts.signatrust_token = _get_conf( ++ cp, "backend", "signatrust_token", "") ++ ++ opts.signatrust_key_expire = _get_conf( ++ cp, "backend", "signatrust_key_expire", 1825, mode="int") ++ + opts.build_groups = [] + for group_id in range(opts.build_groups_count): + archs = _get_conf(cp, "backend", +diff --git a/copr_backend/sign.py b/copr_backend/sign.py +index 6655fb98..31af3c8a 100644 +--- a/copr_backend/sign.py ++++ b/copr_backend/sign.py +@@ -4,8 +4,11 @@ + Wrapper for /bin/sign from obs-sign package + """ + ++from datetime import datetime, timedelta, timezone + from subprocess import Popen, PIPE, SubprocessError + import os ++import re ++import requests + import time + + from packaging import version +@@ -16,8 +19,6 @@ from .exceptions import CoprSignError, CoprSignNoKeyError, \ + CoprKeygenRequestError + + +-SIGN_BINARY = "/bin/sign" +- + def create_gpg_email(username, projectname, domain): + """ + Creates canonical name_email to identify gpg key +@@ -25,78 +26,6 @@ def create_gpg_email(username, projectname, domain): + + return "{}#{}@copr.{}".format(username, projectname, domain) + +- +-def call_sign_bin(cmd, log): +- """ +- Call /bin/sign and return (rc, stdout, stderr). Re-try the call +- automatically upon certain failures (if that makes sense). +- """ +- cmd_pretty = ' '.join(cmd) +- for attempt in [1, 2, 3]: +- log.info("Calling '%s' (attempt #%s)", cmd_pretty, attempt) +- try: +- handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") +- stdout, stderr = handle.communicate() +- except (SubprocessError, OSError) as err: +- new_err = CoprSignError("Failed to invoke '{}'".format(cmd_pretty)) +- raise new_err from err +- +- if handle.returncode != 0: +- log.warning("Command '%s' failed with: %s", +- cmd_pretty, stderr.rstrip()) +- sleeptime = 20 +- log.warning("Going to sleep %ss and re-try.", sleeptime) +- time.sleep(sleeptime) +- continue +- break +- return handle.returncode, stdout, stderr +- +- +-def get_pubkey(username, projectname, log, sign_domain, outfile=None): +- """ +- Retrieves public key for user/project from signer host. +- +- :param sign_domain: the domain name of the sign key +- :param outfile: [optional] file to write obtained key +- :return: public keys +- +- :raises CoprSignError: failed to retrieve key, see error message +- :raises CoprSignNoKeyError: if there are no such user in keyring +- """ +- usermail = create_gpg_email(username, projectname, sign_domain) +- cmd = [SIGN_BINARY, "-u", usermail, "-p"] +- +- returncode, stdout, stderr = call_sign_bin(cmd, log) +- if returncode != 0: +- if "unknown key:" in stderr: +- raise CoprSignNoKeyError( +- "There are no gpg keys for user {} in keyring".format(username), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- raise CoprSignError( +- msg="Failed to get user pubkey\n" +- "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- +- if outfile: +- with open(outfile, "w") as handle: +- handle.write(stdout) +- +- return stdout +- +- +-def _sign_one(path, email, hashtype, log): +- cmd = [SIGN_BINARY, "-4", "-h", hashtype, "-u", email, "-r", path] +- returncode, stdout, stderr = call_sign_bin(cmd, log) +- if returncode != 0: +- raise CoprSignError( +- msg="Failed to sign {} by user {}".format(path, email), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- return stdout, stderr +- +- + def gpg_hashtype_for_chroot(chroot, opts): + """ + Given the chroot name (in "mock format", like "fedora-rawhide-x86_64") +@@ -135,133 +64,438 @@ def gpg_hashtype_for_chroot(chroot, opts): + # fallback to sha256 + return "sha256" + ++# a sign interface ++class Signer(object): ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """get public key""" ++ raise NotImplementedError ++ ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """batch sign rpms""" ++ raise NotImplementedError ++ ++ @classmethod ++ def create_user_keys(cls, username, projectname, opts): ++ """create user key pair""" ++ raise NotImplementedError ++ ++ @classmethod ++ def _sign_one(cls, path, email, hashtype, log): ++ """sign one rpm""" ++ raise NotImplementedError ++ ++ @classmethod ++ def _unsign_one(cls, path): ++ # Requires rpm-sign package ++ cmd = ["/usr/bin/rpm", "--delsign", path] ++ handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") ++ stdout, stderr = handle.communicate() + +-def sign_rpms_in_dir(username, projectname, path, chroot, opts, log): +- """ +- Signs rpms using obs-signd. +- +- If some some pkgs failed to sign, entire build marked as failed, +- but we continue to try sign other pkgs. ++ if handle.returncode != 0: ++ err = CoprSignError( ++ msg="Failed to unsign {}".format(path), ++ return_code=handle.returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- :param username: copr username +- :param projectname: copr projectname +- :param path: directory with rpms to be signed +- :param chroot: chroot name where we sign packages, affects the hash type +- :param Munch opts: backend config ++ raise err ++ ++ return stdout, stderr ++ ++ @classmethod ++ def call_sign_bin(cls, cmd, log): ++ """ ++ Call sign_cmd and return (rc, stdout, stderr). Re-try the call ++ automatically upon certain failures (if that makes sense). ++ """ ++ cmd_pretty = ' '.join(cmd) ++ for attempt in [1, 2, 3]: ++ log.info("Calling '%s' (attempt #%s)", cmd_pretty, attempt) ++ try: ++ handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") ++ stdout, stderr = handle.communicate() ++ except (SubprocessError, OSError) as err: ++ new_err = CoprSignError("Failed to invoke '{}'".format(cmd_pretty)) ++ raise new_err from err ++ ++ if handle.returncode != 0: ++ log.warning("Command '%s' failed with: %s", ++ cmd_pretty, stderr.rstrip()) ++ sleeptime = 20 ++ log.warning("Going to sleep %ss and re-try.", sleeptime) ++ time.sleep(sleeptime) ++ continue ++ break ++ return handle.returncode, stdout, stderr ++ ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """ ++ Signs rpms using obs-signd. ++ ++ If some some pkgs failed to sign, entire build marked as failed, ++ but we continue to try sign other pkgs. ++ ++ :param username: copr username ++ :param projectname: copr projectname ++ :param path: directory with rpms to be signed ++ :param chroot: chroot name where we sign packages, affects the hash type ++ :param Munch opts: backend config ++ ++ :type log: logging.Logger ++ ++ :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package ++ """ ++ rpm_list = [ ++ os.path.join(path, filename) ++ for filename in os.listdir(path) ++ if filename.endswith(".rpm") ++ ] + +- :type log: logging.Logger ++ if not rpm_list: ++ return + +- :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package +- """ +- rpm_list = [ +- os.path.join(path, filename) +- for filename in os.listdir(path) +- if filename.endswith(".rpm") +- ] ++ hashtype = gpg_hashtype_for_chroot(chroot, opts) + +- if not rpm_list: +- return ++ try: ++ cls.get_pubkey(username, projectname, log, opts.sign_domain) ++ except CoprSignNoKeyError: ++ cls.create_user_keys(username, projectname, opts) ++ ++ errors = [] # tuples (rpm_filepath, exception) ++ for rpm in rpm_list: ++ try: ++ cls._sign_one(rpm, create_gpg_email(username, projectname, opts.sign_domain), ++ hashtype, log) ++ log.info("signed rpm: %s", rpm) ++ ++ except CoprSignError as e: ++ log.exception("failed to sign rpm: %s", rpm) ++ errors.append((rpm, e)) ++ ++ if errors: ++ raise CoprSignError("Rpm sign failed, affected rpms: {}" ++ .format([err[0] for err in errors])) ++ ++ @classmethod ++ def unsign_rpms_in_dir(cls, path, opts, log): ++ """ ++ :param path: directory with rpms to be signed ++ :param Munch opts: backend config ++ :type log: logging.Logger ++ :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package ++ """ ++ rpm_list = [ ++ os.path.join(path, filename) ++ for filename in os.listdir(path) ++ if filename.endswith(".rpm") ++ ] ++ ++ if not rpm_list: ++ return ++ ++ errors = [] # tuples (rpm_filepath, exception) ++ for rpm in rpm_list: ++ try: ++ cls._unsign_one(rpm) ++ log.info("unsigned rpm: %s", rpm) ++ ++ except CoprSignError as e: ++ log.exception("failed to unsign rpm: %s", rpm) ++ errors.append((rpm, e)) ++ ++ if errors: ++ raise CoprSignError("Rpm unsign failed, affected rpms: {}" ++ .format([err[0] for err in errors])) ++ ++def new_signer(opts): ++ if opts.sign_backend == "obs-signd": ++ return ObsSign ++ elif opts.sign_backend == "signatrust": ++ Signatrust.signatrust_token = opts.signatrust_token ++ Signatrust.signatrust_host = opts.signatrust_host ++ return Signatrust ++ else: ++ raise CoprSignError("invalid signer type: {}".format(opts.signer_type)) ++ ++class ObsSign(Signer): ++ sign_cmd = "/bin/sign" ++ ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """ ++ Retrieves public key for user/project from signer host. ++ ++ :param sign_domain: the domain name of the sign key ++ :param outfile: [optional] file to write obtained key ++ :return: public keys ++ ++ :raises CoprSignError: failed to retrieve key, see error message ++ :raises CoprSignNoKeyError: if there are no such user in keyring ++ """ ++ usermail = create_gpg_email(username, projectname, sign_domain) ++ cmd = [cls.sign_cmd, "-u", usermail, "-p"] ++ ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ if "unknown key:" in stderr: ++ raise CoprSignNoKeyError( ++ "There are no gpg keys for user {} in keyring".format(username), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) ++ raise CoprSignError( ++ msg="Failed to get user pubkey\n" ++ "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- hashtype = gpg_hashtype_for_chroot(chroot, opts) ++ if outfile: ++ with open(outfile, "w") as handle: ++ handle.write(stdout) + +- try: +- get_pubkey(username, projectname, log, opts.sign_domain) +- except CoprSignNoKeyError: +- create_user_keys(username, projectname, opts) ++ return stdout + +- errors = [] # tuples (rpm_filepath, exception) +- for rpm in rpm_list: ++ @classmethod ++ def _sign_one(cls, path, email, hashtype, log): ++ cmd = [cls.sign_cmd, "-4", "-h", hashtype, "-u", email, "-r", path] ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ raise CoprSignError( ++ msg="Failed to sign {} by user {}".format(path, email), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) ++ return stdout, stderr ++ ++ @classmethod ++ def create_user_keys(cls, username, projectname, opts): ++ """ ++ Generate a new key-pair at sign host ++ ++ :param username: ++ :param projectname: ++ :param opts: backend config ++ ++ :return: None ++ """ ++ data = { ++ "name_real": "{}_{}".format(username, projectname), ++ "name_email": create_gpg_email(username, projectname, opts.sign_domain) ++ } ++ ++ log = get_redis_logger(opts, "sign", "actions") ++ keygen_url = "http://{}/gen_key".format(opts.keygen_host) ++ query = dict(url=keygen_url, data=data, method="post") ++ try: ++ request = SafeRequest(log=log) ++ response = request.send(**query) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to create key-pair for user: {}," ++ " project:{} with error: {}" ++ .format(username, projectname, e), request=query) ++ ++ if response.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to create key-pair for user: {}, project:{}, status_code: {}, response: {}" ++ .format(username, projectname, response.status_code, response.text), ++ request=query, response=response) ++ ++class Signatrust(Signer): ++ sign_cmd = "/usr/local/bin/client" ++ prefix = "" ++ signatrust_host = "" ++ signatrust_token = "" ++ ++ @classmethod ++ def get_prefix(cls): ++ """ ++ Get prefix of the user ++ ++ As in copr, we set key attr with visibility=private ++ These keys' name were prefixed by user's email like: ++ tommylikehu@gmail.com:mywaaagh_admin_test ++ """ ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } ++ try: ++ r = requests.get("{}/api/v1/users/info".format(cls.signatrust_host), headers=headers).json() ++ cls.prefix = r.get("email") ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get userinfo", request="/api/v1/users/info") ++ ++ @classmethod ++ def get_key_name(cls, username, projectname, prefix=True): ++ """ ++ copr key_name rule in signatrust: ++ <user prefix>:<user_name>_<project_name> ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ if prefix: ++ return "{}:{}_{}".format(cls.prefix, username, projectname) ++ return "{}_{}".format(username, projectname) ++ ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """ ++ get public key ++ ++ https://domain:port/api/v1/keys/<key_name>/public_key ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } ++ ++ key_name = cls.get_key_name(username, projectname) ++ url = "{}/api/v1/keys/{}/public_key".format(cls.signatrust_host, key_name) + try: +- _sign_one(rpm, create_gpg_email(username, projectname, opts.sign_domain), +- hashtype, log) +- log.info("signed rpm: %s", rpm) ++ r = requests.get(url, headers=headers) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get public_key", request="/api/v1/keys/{}/public_key".format(key_name), response=r) + +- except CoprSignError as e: +- log.exception("failed to sign rpm: %s", rpm) +- errors.append((rpm, e)) ++ if r.status_code == 404: ++ raise CoprSignNoKeyError( ++ "There are no gpg keys for user {} in keyring".format(username), ++ return_code=r.status_code, ++ cmd="GET {}".format(url), stdout="", stderr="") ++ elif r.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to get user pubkey\n", ++ request="/api/v1/keys/{}/public_key".format(key_name), response=r) ++ ++ if outfile: ++ with open(outfile, "wb") as handle: ++ handle.write(r.content) ++ ++ return r.content ++ ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """batch sign rpms""" ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ if not cls._key_existed(username, projectname, opts): ++ cls.create_user_keys(username, projectname, opts) ++ ++ # when we migrate copr keys into signatrust ++ # we fellow the rules: ++ # key_name = <user_email>:<user_name>_<project_name> ++ cmd = [cls.sign_cmd, "-c", "/etc/signatrust.toml", "add", "--file-type", "rpm", "--key-type", "pgp", "--key-name", cls.get_key_name(username, projectname), path] ++ ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ raise CoprSignError( ++ msg="Failed to sign rpms\n" ++ "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- if errors: +- raise CoprSignError("Rpm sign failed, affected rpms: {}" +- .format([err[0] for err in errors])) ++ if stderr: ++ # signatrust client will print error message for one rpm per line ++ failed_list = re.findall(r"failed to sign file (.*.rpm) due to error", stderr) ++ ++ if failed_list: ++ failed_rpms = " ".join(map(os.path.basename, failed_list)) ++ log.exception("failed to sign rpm: %s".format(failed_rpms)) ++ raise CoprSignError("Rpm sign failed, affected rpms: {}" ++ .format(failed_rpms)) ++ ++ @classmethod ++ def unsign_rpms_in_dir(cls, path, opts, log): ++ """ ++ signatrust will replace the signature infomation defaultly, ++ so there is no need to unsign, just return ++ """ ++ return + ++ @classmethod ++ def _key_existed(cls, username, projectname, opts): ++ """ ++ check keyname existence + +-def create_user_keys(username, projectname, opts): +- """ +- Generate a new key-pair at sign host ++ HEAD /api/v1/keys/ ++ """ ++ if not cls.prefix: ++ cls.get_prefix() + +- :param username: +- :param projectname: +- :param opts: backend config ++ key_name = cls.get_key_name(username, projectname, prefix=False) + +- :return: None +- """ +- data = { +- "name_real": "{}_{}".format(username, projectname), +- "name_email": create_gpg_email(username, projectname, opts.sign_domain) +- } +- +- log = get_redis_logger(opts, "sign", "actions") +- keygen_url = "http://{}/gen_key".format(opts.keygen_host) +- query = dict(url=keygen_url, data=data, method="post") +- try: +- request = SafeRequest(log=log) +- response = request.send(**query) +- except Exception as e: +- raise CoprKeygenRequestError( +- msg="Failed to create key-pair for user: {}," +- " project:{} with error: {}" +- .format(username, projectname, e), request=query) +- +- if response.status_code >= 400: +- raise CoprKeygenRequestError( +- msg="Failed to create key-pair for user: {}, project:{}, status_code: {}, response: {}" +- .format(username, projectname, response.status_code, response.text), +- request=query, response=response) +- +- +-def _unsign_one(path): +- # Requires rpm-sign package +- cmd = ["/usr/bin/rpm", "--delsign", path] +- handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") +- stdout, stderr = handle.communicate() +- +- if handle.returncode != 0: +- err = CoprSignError( +- msg="Failed to unsign {}".format(path), +- return_code=handle.returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- +- raise err +- +- return stdout, stderr +- +- +-def unsign_rpms_in_dir(path, opts, log): +- """ +- :param path: directory with rpms to be signed +- :param Munch opts: backend config +- :type log: logging.Logger +- :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package +- """ +- rpm_list = [ +- os.path.join(path, filename) +- for filename in os.listdir(path) +- if filename.endswith(".rpm") +- ] ++ query = { ++ "name": key_name, ++ "visibility": "private" ++ } + +- if not rpm_list: +- return ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } + +- errors = [] # tuples (rpm_filepath, exception) +- for rpm in rpm_list: + try: +- _unsign_one(rpm) +- log.info("unsigned rpm: %s", rpm) +- +- except CoprSignError as e: +- log.exception("failed to unsign rpm: %s", rpm) +- errors.append((rpm, e)) ++ res = requests.head("{}/api/v1/keys/name_identical".format(opts.signatrust_host), headers=headers, params=query) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to check key existence", request="/api/v1/keys/name_identical", response=res) ++ ++ # signatrust return 200 means key name available ++ if res.status_code == 200: ++ return False ++ # signatrust return 409 means key name redundant ++ elif res.status_code == 409: ++ return True ++ else: ++ raise CoprKeygenRequestError( ++ msg="Failed to check key existence", request="/api/v1/keys/name_identical", response=res) ++ ++ @classmethod ++ def create_user_keys(cls, username, projectname, opts): ++ """ ++ create user key pair ++ ++ POST /api/v1/keys/ ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ if cls._key_existed(username, projectname, opts): ++ return ++ time_format = "%Y-%m-%d %H:%M:%S%z" ++ expire = datetime.now(datetime.now(timezone.utc).astimezone().tzinfo) + timedelta(days=opts.signatrust_key_expire) ++ data = { ++ "name": cls.get_key_name(username, projectname, prefix=False), ++ "description": "gpg key to sign rpms in {}/{}".format(username, projectname), ++ "key_type": "pgp", ++ "visibility": "private", # we use private key type for those key will not be seen by other users ++ "attributes": { ++ "digest_algorithm": "sha2_256", ++ "key_type": "rsa", ++ "key_length": "2048", ++ "email": "{}".format(create_gpg_email(username, projectname, opts.sign_domain)), ++ }, ++ "expire_at": datetime.strftime(expire, time_format) ++ } ++ ++ headers = { ++ "content-type": "application/json", ++ "accept": "application/json", ++ "Authorization": opts.signatrust_token ++ } + +- if errors: +- raise CoprSignError("Rpm unsign failed, affected rpms: {}" +- .format([err[0] for err in errors])) ++ try: ++ res = requests.post("{}/api/v1/keys/".format(opts.signatrust_host), headers=headers, json=data) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get userinfo", request="/api/v1/keys/", response=res) ++ ++ if res.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to create user payload: {}".format(data), request="/api/v1/keys/", response=res) |