diff options
author | CoprDistGit <infra@openeuler.org> | 2025-03-21 17:37:41 +0000 |
---|---|---|
committer | CoprDistGit <infra@openeuler.org> | 2025-03-21 17:37:41 +0000 |
commit | bed3ab1ed2c9a92b20bae1d0f11f17d9051bfff9 (patch) | |
tree | 76779ef5ca00cab259eb516c4c44eef0b3048cc9 | |
parent | 2132b310609fdedcd5427676a98f1174869b3a82 (diff) |
automatic import of copr-backend
-rw-r--r-- | copr-backend.spec | 1472 | ||||
-rw-r--r-- | euler_msgbus.patch | 83 | ||||
-rw-r--r-- | fail_to_build_in_oe2403.patch | 14 | ||||
-rw-r--r-- | signatrust_bin.patch | 105 | ||||
-rw-r--r-- | sources | 2 | ||||
-rw-r--r-- | support_signatrust_backend.patch | 1150 |
6 files changed, 1371 insertions, 1455 deletions
diff --git a/copr-backend.spec b/copr-backend.spec index 86a223c..ae9f5f4 100644 --- a/copr-backend.spec +++ b/copr-backend.spec @@ -1,7 +1,3 @@ -%if 0%{?rhel} <= 7 && 0%{?rhel} > 0 -%global _pkgdocdir %{_docdir}/%{name}-%{version} -%endif - %global prunerepo_version 1.20 %global tests_version 5 %global tests_tar test-data-copr-backend @@ -10,7 +6,7 @@ Name: copr-backend Version: 1.173 -Release: 2%{?dist} +Release: 1%{?dist} Summary: Backend for Copr License: GPL-2.0-or-later @@ -20,7 +16,13 @@ URL: https://github.com/fedora-copr/copr # git clone %%url && cd copr # tito build --tgz --tag %%name-%%version-%%release Source0: %{name}-%{version}.tar.gz -Source1: https://github.com/fedora-copr/%{tests_tar}/archive/v%{tests_version}/%{tests_tar}-%{tests_version}.tar.gz +Source1: %{tests_tar}-%{tests_version}.tar.gz +Patch1: fail_to_build_in_oe2403.patch +Patch2: euler_msgbus.patch +#Patch3: print_queues.patch +#Patch4: redis_helpers.patch # should patch to copr common +Patch5: support_signatrust_backend.patch +Patch6: signatrust_bin.patch BuildArch: noarch BuildRequires: asciidoc @@ -37,7 +39,8 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools BuildRequires: python3-copr -BuildRequires: python3-copr-common >= %copr_common_version +BuildRequires: python3-kafka-python +BuildRequires: python3-copr-common = %copr_common_version BuildRequires: python3-daemon BuildRequires: python3-dateutil BuildRequires: python3-distro @@ -56,10 +59,10 @@ BuildRequires: python3-retask BuildRequires: python3-setproctitle BuildRequires: python3-sphinx BuildRequires: python3-tabulate -BuildRequires: python3-zstandard BuildRequires: modulemd-tools >= 0.6 BuildRequires: prunerepo >= %prunerepo_version BuildRequires: dnf +BuildRequires: rpmdevtools Requires: (copr-selinux if selinux-policy-targeted) Requires: ansible @@ -75,7 +78,7 @@ Requires: obs-signd Requires: openssh-clients Requires: prunerepo >= %prunerepo_version Requires: python3-copr -Requires: python3-copr-common >= %copr_common_version +Requires: python3-copr-common = %copr_common_version Recommends: python3-copr-messaging Requires: python3-daemon Requires: python3-dateutil @@ -101,6 +104,7 @@ Requires: rsync Requires: modulemd-tools >= 0.6 Recommends: util-linux-core Requires: zstd +Requires: python3-kafka-python Requires(post): systemd Requires(preun): systemd @@ -124,7 +128,8 @@ only. %prep -%setup -q -a 1 +%autosetup -a1 -p1 + %build @@ -174,6 +179,7 @@ install -d %{buildroot}%{_sysconfdir}/logstash.d install -d %{buildroot}%{_datadir}/logstash/patterns/ cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern + install -d %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d/copr_backend.conf @@ -238,1447 +244,5 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %exclude %{_pkgdocdir}/lighttpd %changelog -* Fri Feb 21 2025 mywaaagh_admin <pkwarcraft@hotmail.com> 1.173-2 -- fix for openeuler - -* Thu Nov 23 2023 Pavel Raiskup <praiskup@redhat.com> 1.173-1 -- enforce createrepo_c gzip compression (f39+ switched to zstd) -- self-identify the resalloc resource in logs -- dropping the documentary playbooks from copr-backend payload -- nicer unknown-resalloc-tickets output -- worker to not call keygen for source builds at all -- don't sign products of srpm-build -- longer timeout for fallback generating GPG keys after build -- recreate missing repodata so that prunerepo doesn't traceback -- use the rename HashWorkerLimit instead of GroupWorkerLimit -- provide per-arch & per-owner worker limit implemented -- collect and compress fedora-review logs after run -- react on staled SSH connections in some cases - -* Tue Aug 15 2023 Pavel Raiskup <praiskup@redhat.com> 1.172-1 -- dump the /update/ payload to worker.log -- don't run external command(s) to collect built packages -- don't eat the "build detail collecting" traceback -- fixes in the unknown-resalloc-tickets.py helper -- more careful format_evr() method -- fix tests for zst compression on F39+ -- log task dict in case of error returned from redis -- skip builds for ExcludeArch and "not" ExclusiveArch -- offload NEVRA (s)rpm parsing to copr-rpmbuild -- redis authentication support added - -* Tue Jun 06 2023 Pavel Raiskup <praiskup@redhat.com> 1.171-1 -- copr_prune_results.py: work-around the arg_max problem - -* Mon Jun 05 2023 Pavel Raiskup <praiskup@redhat.com> 1.170-1 -- copr_prune_results.py: don't enforce appstream-builder, ask FE -- copr_prune_results.py: just one API call to FE per one repo - -* Tue May 23 2023 Jakub Kadlcik <frostyx@email.cz> 1.169-1 -- Forking: better handle FileExistsError -- Run the copr-rpmbuild command with task URL, not build ID - -* Wed Apr 05 2023 Jiri Kyjovsky <j1.kyjovsky@gmail.com> 1.168-1 -- Bump version for release mess - -* Tue Apr 04 2023 Jiri Kyjovsky <j1.kyjovsky@gmail.com> 1.167-1 -- Run createrepo without --database -- Make copr_messaging optional - -* Wed Mar 22 2023 Jiri Kyjovsky <j1.kyjovsky@gmail.com> 1.166-1 -- Don't include package name into srpm result dir name -- Remove libmodulemd1 dependency -- Hardlink RPMs while doing rawhide_to_release -- Make sign key domain name configurable -* Wed Jan 25 2023 Jakub Kadlcik <frostyx@email.cz> 1.165-1 -- Skip the test_run_prunerepo test because of Koji - -* Tue Jan 24 2023 Jakub Kadlcik <frostyx@email.cz> 1.164-1 -- Fix chroot version parsing with new python-packaging -- Fix traceback for non-existing tasks -- Python: drop the unneeded marshmallow dep -- Log general exceptions to find issues more easily -- Proper log argument formatting instead of .format -- Use spdx license - -* Wed Nov 30 2022 Pavel Raiskup <praiskup@redhat.com> 1.163-1 -- start systemd services after the redis.service -- build worker - list the built RPMs with rpm --nosignature - -* Sat Nov 26 2022 Jakub Kadlcik <frostyx@email.cz> 1.162-1 -- use OpenPGP v4 signatures -- migrate from pipes to shlex -- require redis.service to be started -- move to GitHub home page -- add resultdir cleaner -- move dispatcher and background workers to copr-common -- de-prio IO for the analyze-results script -- don't traceback when there are no files in the S3 storage -- allow devel instance to remove access files -- send non-CDN hitcounter hits in chunks -- copr-backend-unknown-resalloc-tickets script -- work with multiple CDN hostnames per instance -- move setup_script_logger to copr-common - -* Mon Sep 26 2022 Pavel Raiskup <praiskup@redhat.com> 1.161-1 -- sign everything EPEL-5+ with sha256 hashalgo - -* Tue Sep 20 2022 Jakub Kadlcik <frostyx@email.cz> 1.160-1 -- aws-hitcounter: remove temporary files as soon as possible -- aws-hitcounter: ignore downloaded SRPM files -- aws-hitcounter: decode special characters from URLs - -* Tue Aug 16 2022 Pavel Raiskup <praiskup@redhat.com> 1.159-1 -- count only hits from an appropriate CDN hostname -- add option for infinite number of attempts to the hitcounter script -- print more reasonable output from AWS hitcounter script - -* Tue Aug 16 2022 Jiri Kyjovsky <j1.kyjovsky@gmail.com> 1.158-1 -- log every request that is sent to frontend - -* Tue Jul 26 2022 Jakub Kadlcik <frostyx@email.cz> 1.157-1 -- Don't use --keep-all-metadata -- Search for comps.xml in chroot dir - -* Tue Jun 21 2022 Jakub Kadlcik <frostyx@email.cz> 1.156-1 -- Consolidate the two hitcounter scripts -- Dump Resalloc ticket ID and hostname to backend.log -- Automatically restart services in %%post -- Don't count RPMs downloaded from Mock -- Attempt to sign multiple times -- Try multiple attempts of creating GPG keys - -* Mon Apr 04 2022 Pavel Raiskup <praiskup@redhat.com> 1.155-1 -- fix slow priority queue filling -- speedup the background-process spawner - -* Fri Mar 18 2022 Pavel Raiskup <praiskup@redhat.com> 1.154-1 -- copr_fix_gpg: automatically refresh CDN cache -- copr_fix_gpg: don't use --skip-stat for copr-repo when RPMs are re-signed -- action processor: properly return failures of the fork action -- sign EL8+ with sha256 hash algorithm -- copr_fix_gpg: add a new --chroot option -- copr_fix_gpg: skip non-chroot directories -- add hitcounter script for AWS CDN -- backend: tasks in concurrent sandboxes reprioritized to be more fair - -* Wed Feb 02 2022 Silvie Chlupova <schlupov@redhat.com> 1.153-1 -- less aggressive final_prunerepo setting -- analyze-results: dump data for projects' chroots -- basic build tagging -- better "regenerate repo" instructions -- limit RubyGems and PyPI package names length -- Disable coverage analysis during RPM build -- python code for removing unused tarballs on dist-git server - -* Wed Nov 10 2021 Silvie Chlupova <schlupov@redhat.com> 1.152-1 -- Fixup ACR handling -- Drop the unused pid file from the specfile -* Thu Sep 30 2021 Silvie Chlupova 1.151-1 -- backend: use lock(timeout=5) to work-around fair-locks - -* Wed Aug 25 2021 Pavel Raiskup <praiskup@redhat.com> 1.150-1 -- request arch_noarch resalloc tag for source RPM builds -- re-try /bin/sign call upon connection timeout -- drop 'check_consecutive_build_fails' script - -* Tue Jun 15 2021 Pavel Raiskup <praiskup@redhat.com> 1.149-1 -- new weekly cron-job for analyzing storage use (graphs, statistics) -- added some convenience globals into copr_backend.setup module -- keep the max batch size really on the MAX_IN_BATCH limit -- require up2date rpmbuild version -- new action for removing CoprDirs (triggered by cron on frontend) -- fix the CompsUpdate action -- users now can disable appstream metadata generation themselves (without admins asistance) -- handle results.json given by builder, and provide it to frontend (served as APIv3 later) -- pruner: allow pruning also the finalized chroots on demand -- invent FE-BE API version, so backend politely waits for an updated copr-frontend version - -* Thu May 13 2021 Pavel Raiskup <praiskup@redhat.com> 1.148-1 -- work with builders also over ipv6 - -* Sun May 02 2021 Pavel Raiskup <praiskup@redhat.com> 1.147-1 -- fix logging traceback for module builds -- call creatrepo_c with --update if possible -- don't do full createrpeo with --rpms-to-remove - -* Fri Apr 30 2021 Pavel Raiskup <praiskup@redhat.com> 1.146-1 -- better fixes for the appstream-builder generated files -- new helper script named /bin/copr-assure-permissions -- do not rsync-copy permissions from the builder -- log the partial "prunerepo" effects into resultdir -- better, more verbose call_copr_repo logging - -* Tue Apr 27 2021 Jakub Kadlcik <frostyx@email.cz> 1.145-1 -- backend: make the walk_limited test not dependend on its output order, pt2 - -* Tue Apr 27 2021 Jakub Kadlcik <frostyx@email.cz> 1.144-1 -- backend: make the walk_limited test not dependend on its output order - -* Tue Apr 27 2021 Jakub Kadlcik <frostyx@email.cz> 1.143-1 -- backend: fix copr_prune_results logging once more -- backend: better logging in prunerepo -- backend: prunerepo: don't re-createrepo when no rpm is removed -- backend: catch correct client exceptions in copr_prune_results -- test: backend: change prunerepo logic, use get_rpms_to_remove from prunerepo -- backend: new fixture for testing prunerepo -- backend: use safe defaults if APIv3 result doesn't contain what it should -- backend: migrate to APIv3 -- backend: better how-to-redirect logs comment -- backend, frontend, keygen, distgit: keep cca 3 months of logs -- backend: don't createrepo in srpm-builds on delete -- backend: test walk_limited function from helpers - -* Tue Mar 16 2021 Pavel Raiskup <praiskup@redhat.com> 1.142-1 -- prepare for the centos-stream-8 rename -- add script to prune srpm-build directories -- pruner: correctly deliver the final prunerepo stamp to frontend -- pruner: logging through RedisLogHandler -- pruner: better parallelization -- pruner: re-try be-fe communication upon failures -- require up2date modulemd-tools - -* Wed Jan 20 2021 Pavel Raiskup <praiskup@redhat.com> 1.141-1 -- run prunerepo in parallel -- add one-shot copr_find_wrong_chroot_artifacts.py script -- support modulemd v2 - -* Tue Dec 01 2020 Pavel Raiskup <praiskup@redhat.com> 1.140-1 -- fix frontend-client post arguments - -* Mon Nov 30 2020 Pavel Raiskup <praiskup@redhat.com> 1.139-1 -- require appropriate common version -- use common for repeatedly sends requests to frontend - -* Mon Nov 30 2020 Pavel Raiskup <praiskup@redhat.com> 1.138-1 -- get back to using standard createrepo_c command from createrepo_mod -- drop call to stomp's conn.start() (it was dropped) - -* Mon Nov 09 2020 Jakub Kadlcik <frostyx@email.cz> 1.137-1 -- backend: test action for deleting chroot -- backend: fix testsuite for the new createrepo_c -- frontend: don't set ended_on for canceled builds -- all: run pytest with -vv in package build -- common, cli, python, rpmbuild, frontend, backend: DistGit source method -- backend: use createrepo_mod tool for generating module repodata - -* Wed Aug 12 2020 Pavel Raiskup <praiskup@redhat.com> 1.136-1 -- testsuite: give more time to the slow Koji builders - -* Mon Aug 10 2020 Pavel Raiskup <praiskup@redhat.com> 1.135-1 -- prioritize all non-background jobs -- fix up libmodulemd dependency - -* Fri Jun 19 2020 Pavel Raiskup <praiskup@redhat.com> 1.134-1 -- fix copr-repo to work with absolute paths -- automatically batch the createrepo requests -- scheduler is now fair, and ordered -- indefinitely retry workers' talk to frontend -- allow canceling also "starting" builds -- more verbose delete action in logs -- cleanup the example configuration -- use FileHandler for backend.log, fixes traceback - -* Tue Jun 09 2020 Pavel Raiskup <praiskup@redhat.com> 1.133-1 -- better build task priority processing -- dump attempt to send message to backend.log -- drop the VMM concept, replaced with resalloc -- delegate more work to the builder code -- external blob tarball for unittests -- buggy error handler in pkg_name_evr() -- basic build task priority -- the reschedule-all builds idiom removed -- fix the build cancelation -- drop duplicate BuildRequire on python-requests -- require the newest version of copr-common -- minimalize the transfered amount of information about actions from FE -- process actions in regard to their priority -- move backend's code to standard PYTHONPATH -- move ActionResult to copr_common.enums -- actions/builds use the same WorkerManager logic -- more verbose rawhide to release action processing - -* Wed Feb 05 2020 Pavel Raiskup <praiskup@redhat.com> 1.132-1 -- better handle invalid options in copr-repo --add/--delete -- copr-repo: optimize-out useless createrepo_c runs -- move initial createrepo check from dispatcher to worker -- don't send messages on bus N-times -- /bin/copr-repo now respects .disable-appstream files -- drop unused build_deleting_without_createrepo option - -* Wed Jan 15 2020 Tomas Hrnciar <thrnciar@redhat.com> 1.131-1 -- put build-ID.log file to resultdir -- call call_copr_repo if initial createrepo failed -- Build Dispatcher does not wait forever till repo is created, - it creates it manually -- properly delete logs for old builds -- delete build-ID.log files again -- edit repositories only by new 'copr-repo' tool -- fix multi-build delete -- fix for not saving end time of actions -- lower traffic in build_dispatcher log -- more resilient redis logging -- attempt to publish on msgbus N-times -- log service: move RequiredBy to [Install] -- keep worker ID in proc title - -* Fri Dec 06 2019 Pavel Raiskup <praiskup@redhat.com> 1.130-1 -- backend: execute actions with sane umask=0022 - -* Wed Dec 04 2019 Pavel Raiskup <praiskup@redhat.com> 1.129-1 -- do not start a build if copr_base is not available yet -- systemd services' restart re-ordering -- de-duplicate frontend_.update() call when reattaching to existing build -- allow specifying timeout for spawn/terminate playbooks -- removing dependecy on euca2ools in spec -- send `uses_devel_repo' as a part of task info -- correctly configure logrotate -- get_redis_logger: skip log entries bellow log_level -- delete leftover action workers from redis - -* Fri Oct 11 2019 Pavel Raiskup <praiskup@redhat.com> 1.128-1 -- restart copr-backend sub-services on failure -- don't kill action processors by 'systemctl restart' -- lower the log traffic in build_dispatcher.log - -* Thu Oct 03 2019 Pavel Raiskup <praiskup@redhat.com> 1.127-1 -- fix testsuite for slow Koji builders - -* Thu Oct 03 2019 Pavel Raiskup <praiskup@redhat.com> 1.126-1 -- more reliable communication with frontend (#1021) -- only ask for auto_createrepo once per project -- parallel handling of actions (#1007) -- don't provide builder-live.log once the build ended, and - add 'copr-compress-live-logs' helper (#985) -- less exceptions in logs -- project forking fixes -- depend on copr-messaging, not fedora-messaging -- fixes for copr_print_results_to_delete.py script - -* Wed Aug 28 2019 Dominik Turecek <dturecek@redhat.com> 1.125-1 -- minimize redis traffic for looping over pending-jobs (issue#902) -- batch delete builds into a single action (issue#688) -- admin opt-out createrepo after build-deleting -- fix wrong message validation class -- refine cleanup_vm_nova.py -- depend on copr-messaging - -* Mon Jul 29 2019 Pavel Raiskup <praiskup@redhat.com> 1.124-1 -- run createrepo immediately, don't wait for build (issue#833) -- compress backend-live.log by calling gzip (issue#86) -- use copr-messaging module for validating outgoing messages -- don't run appstream-builder for PR dirs -- don't run createrepo for srpm directories -- skip VMs with failing live-check in scheduler -- sandbox builds per user/submitter/project -- drop unused compat code for droped /bin/copr-builder -- do not call appstream builder with --max-threads (issue#717) -- added copr_print_results_to_delete.py script, should help - us with removal of orphaned resources on backend storage (issue#712) -- allow disabling appstream builder per project (issue#738) -- tabular output from copr_get_vm_info.py - -* Wed Apr 24 2019 Jakub Kadlčík <frostyx@email.cz> 1.123-1 -- clean data for failed builds; fix #619 -- replace runnecessary regex with str.endswith -- move clean_copr from prunerepo to our codebase -- cleanup_vm_nova.py: use yaml.safe_load -- don't rely on createrepo from prunerepo -- simplify logging through redis -- run sign command without sudo to fix #636 -- encode 'msg' in LogRecord sooner -- fix charset warnings on redis-py v3 -- fix default arguments in redis scripts -- don't prunerepo too old directories -- LogHandler: don't drop exc_info from LogRecord -- use the correct data for rawhide_to_release createrepo -- make copr_prune_results skip already pruned outdated chroots -- require libmodulemd in at least 1.7.0 -- remove dependency on python3-configparser - -* Mon Feb 11 2019 Jakub Kadlčík <frostyx@email.cz> 1.122-1 -- Add requires python3-novaclient -- Set the architecture for which the module has been built -- Generate module artifacts in the correct format -- Compress the modules.yaml file - -* Fri Jan 11 2019 Miroslav Suchý <msuchy@redhat.com> 1.121-1 -- remove data from outdated chroots - -* Thu Jan 10 2019 Miroslav Suchý <msuchy@redhat.com> 1.120-1 -- update list of copr services -- Use oslo_concurrency for craeterepo locking -- use run_cmd() in pkg_name_evr() -- drop "downloading" state -- allow blacklisting packages from chroots - -* Fri Oct 19 2018 Miroslav Suchý <msuchy@redhat.com> 1.119-1 -- optimize copr_log_hitcounter.py for speed a bit -- move selinux rules to copr-selinux -- fix traceback for non-serializable log message -- fix tracebacks in copr-backend-log -- more robust run_tests.sh -- remove unused imports -- py3 compat for msgbus support -- use git_dir_archive instead of git_dir_pack -- migrate from deprecated python3-modulemd to libmodulemd -- doc: remove warning that _static directory does not exists -- doc: the undeline need to be at least as long as the title - -* Thu Aug 23 2018 clime <clime@redhat.com> 1.118-1 -- fix logging exception -- send proper arguments for rawhide_to_release -- packaging: Python 2/3, RHEL/Fedora fixes - -* Mon Aug 06 2018 clime <clime@redhat.com> 1.117-1 -- None task protection -- pagure integration -- use manual .pyc file generation -- remove unused imports, ad. pr#327 -- resolving pylint warnings -- for py3 use unittest.mock -- fix msgbus ContentType to application/json - -* Fri May 18 2018 clime <clime@redhat.com> 1.116-1 -- fix #291 forks are incomplete -- log more information about incoming actions -- preparation for opensuse-leap-15.0-x86_64 - -* Thu Apr 26 2018 Dominik Turecek <dturecek@redhat.com> 1.115-1 -- rpkg deployment into COPR - containers + releng continuation -- fix pagure bugs #269, #273, #221 and #268 -- cleanup in test_helpers, one test added -- change order of args in StrictRedis call -- add comment about expected usage of the copr_log_hitcounter script -- try to send hit data to frontend several times from -copr_log_hitcounter - -* Mon Feb 26 2018 clime <clime@redhat.com> 1.114-1 -- add possibility for copr_log_hitcounter to ignore multiple subnets - -* Fri Feb 23 2018 clime <clime@redhat.com> 1.113-1 -- original builder deprecation -- remove Group tag - -* Mon Feb 19 2018 clime <clime@redhat.com> 1.112-1 -- Shebangs cleanup -- escapes in changelogs - -* Sun Feb 18 2018 clime <clime@redhat.com> 1.111-1 -- use netaddr instead of IPy module -- sleep after each load_jobs iteration -- python3 conversion -- UMB: adding content type -- add source_status field for Builds -- generate module artifacts rpms -- the rsync log is actually renderred directly into result dir now -- mockchain.log renamed to backend.log -- pg#192 missing records in mockchain.log -- enable running tests in spec file -- enable and update vmmamanger tests, fix three minor bugs in the - manager -- frontend now presents the whole job queue state to - backend -- copy only module builds into the repo directory - -* Wed Dec 20 2017 clime <clime@redhat.com> 1.110-1 -- exception handling for hit counting when IP address cannot be parsed - -* Mon Dec 18 2017 Dominik Turecek <dturecek@redhat.com> 1.109-1 -- terminate also 'in_use' builders if health checks have failed -- make --detached the last arg for copr-rpmbuild -- update copr_log_hitcounter to check ip against ignored pattern -- new msg bus options -- disable DNF makecache timer/service -- fix message duplication for multi-bus scenario - -* Thu Nov 16 2017 Miroslav Suchý <msuchy@redhat.com> 1.108-1 -- optimize createrepo_c -- Revert "[backend] remove --ignore-lock from createrepo_c" - -* Thu Nov 09 2017 clime <clime@redhat.com> 1.107-1 -- kill all processes in copr-rpmbuild's process group -- add --drop-resultdir switch to copr-rpmbuild call -- release_vm immediately after VM is no longer needed -- remove --ignore-lock from createrepo_c - -* Wed Oct 18 2017 clime <clime@redhat.com> 1.106-1 -- run copr-rpmbuild with --verbose option - -* Wed Sep 27 2017 clime <clime@redhat.com> 1.105-1 -- remove uneeded yum dep - -* Tue Sep 26 2017 clime <clime@redhat.com> 1.104-1 -- update copr-rpmbuild command for the new options -- change arguments to build_id and chroot -- #128 AppStream data collection vetoes addons -- fix rpm download stats collection -- module-stuff update - -* Fri Sep 15 2017 clime <clime@redhat.com> 1.103-1 -- update fedora image version to 26 -- fixes for recent code - -* Thu Sep 07 2017 clime <clime@redhat.com> 1.102-1 -- srpms are now being built from upstream on builders - -* Wed Jun 14 2017 clime <clime@redhat.com> 1.101-1 -- remove unused helpers.run_ssh + function spacing fixup -- cancel-build action fix - -* Fri Jun 09 2017 clime <clime@redhat.com> 1.100-1 -- extend check for a builder package present on a builder machine -- arbitrary dist-git branching -- remove --add-cache-id from appstream-builder call, see Bug 1426166 -- change to using a standalone builder package - -* Wed May 03 2017 clime <clime@redhat.com> 1.99-1 -- missing on_success_build call added back to sign packages and recreate repo after each build - -* Mon Apr 24 2017 clime <clime@redhat.com> 1.98-1 -- Bug 1444804 - Logs are not present for failed builds - -* Wed Apr 19 2017 clime <clime@redhat.com> 1.97-1 -- do not condrestart optional logstash service -- standalone builder option -- build reattaching after copr-backend(-build) service restart -- live mockchain log -- use openssh instead of paramiko -- update cleanup_vm_nova script -- remove buggy logging -- removed Sphinx as a dependency... -- verbose log everything we have about failed playbook -- replace fedorahosted links -- make systemd services out of ActionDispatcher and BuildDispatcher - -* Thu Jan 26 2017 clime <clime@redhat.com> 1.96-1 -- Fixes for building COPR Backend and Dist-git on EL7 -- simplified/improved logging of exceptions mainly -- don't use sha256 checksum for rhel-5* repos, too -- drop mentions of the max_builds_per_vm optoin -- switched usage of deprecated ansible Runner for python-paramiko module -- os_nova filter plugin fixed for python-novaclient 3 -- support for STOMP msg buses -- fix Bug 1402689 regarding job cancellation -- jobgrab service is no more -- respect 'do_sign' option when forking -- fix buildroot_cmd for rhel mock profiles - -* Thu Dec 01 2016 clime <clime@redhat.com> 1.95-1 -- use buildroot_pkgs substitution type according to job.chroot -- use timeout command to respect timeout param coming from frontend -- don't ship unitfiles in %%bindir -- move createrepo to the end of the rawhide_to_release handler -- modulemd 1.0.2 compatibility -- Bug 1397119 - Error reading SSH protocol banner -- added auto-prune project's option -- Bug 1086139 - [RFE] provide UI to cancel a build -- Fix misleading debug statement -- fix exception logging in ensure_dir_exists helper -- Fix chroot_setup_cmd regex for custom chroot - -* Mon Sep 19 2016 clime <clime@redhat.com> 1.94-1 -- also provide default version and release for generated modules.json - -* Mon Sep 19 2016 clime <clime@redhat.com> 1.93-1 -- fix NameError: global name 'result' is not defined -- fix exception logging -- Modularity support -- Bug 1357564 - RFE: allow downloading of mock profiles (reproducible builds) -- "safer" exception handling for actions - -* Mon Aug 15 2016 clime <clime@redhat.com> 1.92-1 -- wrap feedback about actions to frontend into try-except -- log even the traceback from forking -- use makedirs instead of mkpath in fork action -- if anything bad happens, log exception in generate_gpg_key action -- also restart copr-backend-vmm and copr-backend-log when (re)installing -- Bug 1361344 - RFE: Allow denial of build deletion and resubmitting at project or group level -- catch errors in fork action -- set action result for comps.xml and module_md.yaml file deletion -- backend fork action now takes care of new gpg-key generation instead of frontend -- removed no longer supported --api-version=0.8 arg from appstream-builder command line -- specify module_md as module type -- fix saving comps.xml and module_md.yaml into empty copr (with no build) -- module_md.yaml is added to repodata now similarly to appstream.xml -- support for generation of module dist tags -- module_md.yaml uploading for a chroot -- simplified build and action task workflow -- use copy of the mock (chroot) config, not the original in /etc/mock/ - -* Wed Jun 22 2016 Miroslav Suchý <msuchy@redhat.com> 1.91-1 -- configure more packages to run pylint -- terminate machine which was only partially spawned -- [copr-prune-results] do not sys.exit if prunerepo returns non-zero status, - just raise an exception -- more of log file migration -- claim /var/log/copr-backend in %%files -- adjust log path in runtime files -- update conf file log path directives -- change logdir to /var/log/copr-backend/ - -* Fri May 27 2016 Miroslav Suchý <msuchy@redhat.com> 1.90-1 -- do not use --log-dir in appstream-builder - -* Tue May 24 2016 Miroslav Suchý <miroslav@suchy.cz> 1.89-1 -- use correct conditional in requires - -* Mon May 23 2016 Miroslav Suchý <msuchy@redhat.com> 1.88-1 -- backend: change logstash requires to soft requires -- 1336360 - allow custom chroots - -* Fri May 13 2016 Miroslav Suchý <msuchy@redhat.com> 1.87-1 -- workaround for BZ 1334200 -- more info in logs by default -- print seconds just as int -- unsign gpg from forked packages before signing them with new key -- sign forked packages @TODO We need to delsign them first - -* Fri May 06 2016 Miroslav Suchý <msuchy@redhat.com> 1.86-1 -- more info in logs by default -- unsign gpg from forked packages before signing them with new key - -* Thu May 05 2016 Miroslav Suchý <msuchy@redhat.com> 1.85-1 -- also be tolerant about sign/unsign failures on particular rpm -- just log errors (exception) during particular copr fixing, do not - interrupt the whole process -- added additional check on copr path existence into copr_fix_gpg.py -- allow sudo /usr/bin/rpm for `copr` user -- look into build dirs (subdirs of a chroot) for rpms to be re-signed -- on F24+ use just ansible -- Run rpm-sign with sudo when unsigning -- script to fix gpg keys & rpm signatures -- define functions for deleting gpg signatures from packages -- removed temporary mock workaround from Dockerfile (no - longer needed) - -* Thu Apr 28 2016 Miroslav Suchý <msuchy@redhat.com> 1.84-1 -- Bug 1327996 - config_opts['use_host_resolv'] is not set back to - True if it was False before - -* Fri Apr 22 2016 Miroslav Suchý <msuchy@redhat.com> 1.83-1 -- run createrepo on forked project (RhBug: 1329076) -- Bug 1327852 - /usr/bin/check_consecutive_build_fails.py errors -- we need to stick to ansible1.9 -- more escaping -- prunning down testresults :) -- a few unittests for copr_prune_results.py script -- unit test "fixes" -- fix error when forking into existing project -- (mockremote): improve chroot_setup_cmd replacement for EL-5 -- copr_prune_results.py - python path fix -- Bug 1324514 - copr createrepo error messages - fix for errors of - type one -- Bug 1324514 - copr createrepo error messages - fix for errors of - type 2 - -* Thu Mar 24 2016 Jakub Kadlčík <jkadlcik@redhat.com> 1.82-1 -- use timeout variable from config - -* Mon Mar 14 2016 Jakub Kadlčík <jkadlcik@redhat.com> 1.81-1 -- support project forking -- support building from PyPI -- support for redis_host, redis_port, redis_db config options -- dockerized-backend project moved under backend/docker -- run createrepo in rawhide_to_release -- specify rawhide name when calling rawhide_to_release - -* Fri Jan 29 2016 Miroslav Suchý <msuchy@redhat.com> 1.80-1 -- do not fail when when you receive job with architecture which does not have - queue -- fix 1260780 - Build fails after successful package generation - - just add a log error message pointing to an rsync log -- jobgrabcontrol.py/retask misuse fix -- "localhost-targeted" spawn and terminate playbooks added for testing -- [frontend]implement rawhide to release feature First create new - chroots: python manage.py create_chroot fedora-24-i386 fedora-24-x86_64 -- abstraction above [BE <-> JG <-> Builders] channels -- don't traceback backend if frontend is not yet up&running -- do not preserve user and group when rsyncing - -* Wed Dec 23 2015 Miroslav Suchý <msuchy@redhat.com> 1.79-1 -- fix packaging issues in epel-7+ - -* Mon Nov 16 2015 Miroslav Suchý <miroslav@suchy.cz> 1.78-1 -- handle_generate_gpg_key skips key creation when signing is disabled -- Added test_handle_generate_gpg_key -- fixed failing tests -- show when createrepo is waiting for lock -- do not block builds when processing too much actions - -* Fri Nov 06 2015 Miroslav Suchý <msuchy@redhat.com> 1.77-1 -- we need to have recent python-copr -- create copr-backend-service script to handle all copr services -- [backend] fix not starting job_grab - -* Tue Oct 13 2015 Miroslav Suchý <msuchy@redhat.com> 1.76-1 -- createrepo action run infinitely when applied to - deleted project - -* Mon Sep 21 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.75-1 -- [backend] run copr-backend-log service before other components - -* Mon Sep 21 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.74-1 -- [backend] add executable bit to run/copr_run_job_grab.py - -* Mon Sep 21 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.73-1 -- added context manager `local_file_logger` -- eliminated global multiprocessing.Lock -- split backend daemon: extracted RedisLogHandler, JobGrab, VMM -- replace python-bunch with python-munch -- added comps.xml support - -* Tue Aug 04 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.72-1 -- support new results naming in the build deletion action -- fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path -- using package name and versiong given in the build task; cleanup; -- handle error's caused by failure to obtain srpm from dist-git -- repairing test for newest changes -- rsync update + several fixes -- building from dist git -- fix vm spawn check: spawner count child processes per build group; - -* Wed Jul 01 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.71-1 -- add small script to print queues -- AppData supproted -- copy mockchain and rsync logs to resdir (RhBug:1221519) -- note which modules still stops us from migrating to python3 - -* Mon Jun 15 2015 Miroslav Suchý <msuchy@redhat.com> 1.70-1 -- alter vm_name= regexp -- polishing Bug 1195867 - Move or delete logs when rebuilding failed - build. -- backup only info and log files -- have just one backup directory per results directory -- clean results from previous build -- alter IP= regexp -- disabled appdata until fixed -- unable appdata in createrepo -- more safe VmMaster.check_one_vm_for_dead_builder function -- adding support for AppData -- new requirement form AppData support -- createrepo_unsafe now returns only STDOUT and raise exception on - errors - -* Mon Jun 01 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.69-1 -- removed creation of symlinks for log.gz -- catch exception during Worker.can_start_job -- config cleanup - -* Thu May 28 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.68-1 -- [backend] add config option for VM health check timeout -- [backend] moved config parameters from Threshold class into the backend - config file - -* Thu May 21 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.67-1 -- [backend] Handle unexpected exception VmMaster::check_one_vm_for_dead_builder - -* Thu May 21 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.66-1 -- [backend] fix race condition in check for dead worker - -* Wed May 20 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.65-1 -- [backend] Rescheduling unfinished builds before stop -- fix indentation -- [backend] request frontend to reschedule old unfinished builds at startup -- [backend] update sytemd unit: removed obsolete - EnviromentFile=/home/copr/cloud/ec2rc.variable directive - -* Tue May 19 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.64-1 -- [backend] check for aux process state and restart if needed - -* Fri May 15 2015 Miroslav Suchý <msuchy@redhat.com> 1.63-1 -- BR python-sphinx - -* Fri May 15 2015 Miroslav Suchý <msuchy@redhat.com> 1.62-1 -- [backend] small cleanup, need more tests - -* Fri May 15 2015 Miroslav Suchý <msuchy@redhat.com> 1.61-1 -- [backend] notify job_grab to remove job from added when start_job failed -- [backend] [vmm] terminate VM with state IN_USE only when builder process is - missed -- [backend] bugfix VMM.get_all_vm_in_group : VM could be removed before load - occures -- [backend] job_grab: postpone build is we already serving more builds - than`max_vm_per_user` option -- [backend] fix build logging -- [backend] defer sending job to worker if job owner acquired too much VMs -- [backend] replaced Thresholds.dirty_vm_terminating_timeout with config option - vm_dirty_terminating_timeout -- [backend] Thread's doesn't have a pid -- [backend] vm manage: user threading instead of multiprocessing -- [backend] VMM aware cleunup_vm_nova -- [backend] moving to nginx to serve results. lighttpd couldnt server pre- - compressed properly -- [backend] script to clean up erred and forgotten VM's using python-novaclient -- [backend] new documentation -- [backend] repaired unittests -- [backend] updated builder playbooks -- [backend] updated example spawn playbook -- [backend] don't provide logstash config directly, add only example to - documentation -- [backend] do logging from multiply processes through redis pubsub; some fixes - to VM-management -- [image_builder] initial release, due to OS bug, we cannot create snapshot - after provision through API, need to do it manually in the WebUI. -- [backend] tests cleanup -- [backend] ensure that prune script running under the copr user; simpler - `copr_find_obsolete_builds` -- [backend] safer copr_prune_results script, unittests -- [backend][frontend] Send for delete action only `src_pkg_name` instead of - original URL. -- [backend] returned script to call createrepo from cli -- [copr] don't allow acquire VMs that was last checked before server restart. -- [backend] Added limit to acquire_vm based on VMs used by the same username at - the current moment. -- New python dependencies -- [backend] run tmp redis-server for tests -- [backend] Dedicated and more complex management for builder machines. - [frontend] Now builds failed due to VM errors reschedulted faster. - -* Fri Mar 20 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.60-1 -- [backend][spec] start/stop redis server during package build tests - -* Fri Mar 20 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.59-1 -- [backend][hotfix] 1203753 : don't process delete action if src_pkg is - mallformed - -* Mon Mar 02 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.58-1 -- [rhbz:#1185959] - RFE: Present statistics about project - popularity. A few more counters for downloads from backend's result - directory. -- [backend] [rhbz:#1191037] RFE: Include package name and version in fedmsg - notification -- [rhbz:#1091640] RFE: Release specific additional repos -- [rhbz:#1119300] [RFE] allow easy add copr repos in using - repository lis -- [backend][frontend] removing code related to multiply source rpms in build. - Build.pkgs now expected to have exactly one src.rpm. -- [copr] backend: script fixes, dropped create_repo cli script -- more file descriptors on builder -- [rhbz:#1171796] copr sometimes doesn't delete build from repository -- [rhbz:#1073333] Record consecutive builds fails to redis. Added - script to produce warnings for nagios check from failures recorded to redis. -- correctly print job representation - -* Fri Jan 23 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.57-1 -- call correct Worker method on backend termination -- put gpg pubkey to the project results root dir (one level up from - the chroot dir) -- don't kill Worker from errors during job build -- [rhbz:#1169782] RFE - Show package "version-release" instead of - just "version" -- [rhbz:#1117446] add a build id tagfile into the package directory -- Updated unittests to reflect latest changes. -- builder: use only one log file for rsync per build -- dispatcher: run terminate_instance safely -- cleanup example config -- cleanup mockremote.builder -- Builder.download don't use Popen+PIPE.communicate with rsync, - output redirected to the files. -- disable networking only when required; python style exception - handling in mockremote*; removed run/copr_mockremote -- test build with disabled networking -- simplified mockremote.builder.Builder.check_for_ans_error; new - method mockremote.builder.Builder.run_ansible_with_check -- daemons.dispatched.Worker: don't fail when wrong group_id was - provided -- add vm_ip to worker process title (rhbz: 1182637) - -* Wed Jan 14 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.56-1 -- [backend] [.spec] fix %%files section - -* Wed Jan 14 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.55-1 -- [backend] [bugfix] set pythonpath in systemd unit to run /usr/bin/copr_be.py -- [backend] [RHBZ:#1182106] JobGrabber dies when action raises an exception. -- [backend] Moved scripts into /usr/bin/ Renamed copr{-,_}be.py. - -* Wed Jan 07 2015 Miroslav Suchý <msuchy@redhat.com> 1.54-1 -- 1179713 - workaround for 1179806 -- run script unbufferred otherwise log is written after full block -- express that it is n-th projects -- fix permissions on prune script - -* Mon Jan 05 2015 Valentin Gologuzov <vgologuz@redhat.com> 1.53-1 -- [backend, frontend] [RHBZ:#1176364] Wrong value for the build timeout. - -* Mon Dec 15 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.52-1 -- fixed config option `results_baseurl` usage, in mockremote - -* Fri Dec 12 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.51-1 -- updated BuildRequires; cleanup imports -- package sign: generate gpg usermail with special symbol -- bugfix: when dispatcher has vm_ip it shouldn't start new VM; -- run tests during rpm build -- minor docstring fix - -* Wed Dec 10 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.50-1 -- [backend] added option to control ansible ssh transport, changed by default - to `paramiko` [frontend] bugfix api create new -- [backend] removed spawn_vars options, to be able to spawn VMs in advance -- [backend] unittest for backend.daemons.log -- [backend] massive refactoring and unittest coverage -- [backend] backend.sign: discover `keygen_host` from backend config file - -* Tue Nov 25 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.49-1 -- [backend] small bug in dispatcher - -* Tue Nov 25 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.48-1 -- bugfixes, disabled debug prints, fixed PEP8 violations - -* Thu Nov 20 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.47-1 -- refactored mockremote, added explicit BuildJob class -- allow to spawn builder in advance -- copr-prune-repo respects auto_createrepo option -- bugfix: repeated config reads produced constantly growing lists - -* Fri Oct 24 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.46-1 -- [backend] added handling of new action type: "createrepo" -- [backend] added dependency on `python-copr` -- [backend] added to mockchroot -a /devel/repodata subfolder -- [backend] new config option to define the public frontend api endpoint -- [backend] conditional execution of createrepo_c -- [backend] unittest for Action and minor refactoring -- [backend] rotate backend.log as well - -* Thu Sep 18 2014 Miroslav Suchý <msuchy@redhat.com> 1.45-1 -- [backend][keygen] minor fixes/typos - -* Thu Sep 18 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.44-1 -- [backend] type fix - -* Thu Sep 18 2014 Valentin Gologuzov <vgologuz@redhat.com> 1.43-1 -- [backend] config parsing: convert fields to proper data type. -- [backend] added option to disable package signing. -- [keygen] new component for copr: gpg key generation for package sign -- [backend] broadcast both submitter and owner to fedmsg -- [backend] example backend config: changes url protocol to HTTPS. - -* Mon Aug 25 2014 Adam Samalik <asamalik@redhat.com> 1.42-1 -- [backend] [RHBZ:1128606 ] For rhel-5 builds pass "--checksum md5" to - `createrepo_c` command. -- [backend] fix of builder test -- [backend] test builder instance after spawning -- [backend] never give up while spawning an OpenStack VM -- [backend] worker's log filename correction -- [backend] task id in worker process' name -- [backend] async build playbooks - -* Thu Aug 14 2014 Miroslav Suchý <miroslav@suchy.cz> 1.41-1 -- [backend] fix of fix -- [backend] couple of fixes - -* Wed Aug 13 2014 Miroslav Suchý <msuchy@redhat.com> 1.40-1 -- [backend] queue cleaning -- [backend] experimental build groups for more architectures -- [backend] fix of a strange beaviour of retask -- [backend] fedmsg shows submitter instead of project owner -- [backend] new task queue for workers using retask -- epel-7 comps workaround is need no more, since CENTOS7 have been released - -* Tue Jul 22 2014 Miroslav Suchý <msuchy@redhat.com> 1.39-1 -- FrontendCallback prettified -- Starting state implemented, cancelling fixed -- [backend] faster skipping - -* Tue Jul 15 2014 Miroslav Suchý <msuchy@redhat.com> 1.38-1 -- [backend] built pkgs fix - -* Tue Jul 15 2014 Miroslav Suchý <msuchy@redhat.com> 1.37-1 -- [backend] shell command uses pipes.quote -- Return the chroot that finished when sending build.end -- better and safer deleting of builds -- [backend] separate playbooks for each architecture -- [backend] built pkgs - include subpackages -- [backend] skipped status and package details implemented -- document vm_name option - -* Thu Jun 19 2014 Miroslav Suchý <msuchy@redhat.com> 1.36-1 -- backend: migrate to nova ansible module -- backend: make sure that exit() exit whole script not just sub-shell -- backend: allow passing additional info to playbooks -- handle {spawn,terminate}_instance equally -- backend: stop if you could not change to directory -- W:310, 8: Attribute 'abort' defined outside __init__ (attribute-defined- - outside-init) -- W:139, 0: Dangerous default value [] as argument (dangerous-default-value) - W:139, 0: Dangerous default value [0] as argument (dangerous-default-value) - W:139, 0: Dangerous default value ['stdout', 'stderr'] as argument - (dangerous-default-value) -- W:543, 4: Dangerous default value DEF_MACROS ({}) as argument (dangerous- - default-value) -- W:543, 4: Dangerous default value DEF_REPOS ([]) as argument (dangerous- - default-value) -- W:677,24: Unused variable 'out' (unused-variable) W:677,20: Unused variable - 'rc' (unused-variable) -- W:297,12: Unused variable 'hn' (unused-variable) -- C:116, 0: Unnecessary parens after 'print' keyword (superfluous-parens) -- W: 72,28: Unused variable 'out' (unused-variable) W: 72,24: Unused variable - 'rc' (unused-variable) -- fix typo in exception message printing -- 1102788 - Increase number of file descriptors on the build machine - -* Fri May 30 2014 Miroslav Suchý <msuchy@redhat.com> 1.35-1 -- follow selinux packaging draft -- [backend] epel 5 repo fix (sha256 -> sha) - -* Thu Apr 24 2014 Miroslav Suchý <msuchy@redhat.com> 1.34-1 -- if directory does not exist, do not try to delete it - -* Tue Apr 15 2014 Miroslav Suchý <miroslav@suchy.cz> 1.33-1 -- do not publish copr.worker messages -- better count workers - -* Thu Apr 10 2014 Miroslav Suchý <msuchy@redhat.com> 1.32-1 -- include ec2rc in service unit file - -* Wed Apr 09 2014 Miroslav Suchý <msuchy@redhat.com> 1.31-1 -- 1077791 - set perm of cronfile to 755 -- 1077791 - add LICENSE to -doc subpackage -- 1077791 - remove make as BR - -* Tue Mar 18 2014 Miroslav Suchý <msuchy@redhat.com> 1.30-1 -- [backend] exclude files which are part of main package -- copr-backend.src:113: W: mixed-use-of-spaces-and-tabs (spaces: line 5, tab: - line 113) - -* Tue Mar 18 2014 Miroslav Suchý <msuchy@redhat.com> 1.29-1 -- move backend into separate package - -* Thu Feb 27 2014 Miroslav Suchý <msuchy@redhat.com> 1.28-1 -- [backend] - pass lock to Actions - -* Wed Feb 26 2014 Miroslav Suchý <msuchy@redhat.com> 1.27-1 -- [frontend] update to jquery 1.11.0 -- [fronted] link username to fas -- [cli] allow to build into projects of other users -- [backend] do not create repo in destdir -- [backend] ensure that only one createrepo is running at the same time -- [cli] allow to get data from sent build -- temporary workaround for BZ 1065251 -- Chroot details API now uses GET instead of POST -- when deleting/canceling task, go to same page -- add copr modification to web api -- 1063311 - admin should be able to delete task -- [frontend] Stray end tag h4. -- [frontend] another s/coprs/projects/ rename -- [frontend] provide info about last successful build -- [spec] rhel5 needs group definition even in subpackage -- [frontend] move 'you agree' text to dd -- [frontend] add margin to chroots-set -- [frontend] add margin to field label -- [frontend] put disclaimer to paragraph tags -- [frontend] use black font color -- [frontend] use default filter instead of *_not_filled -- [frontend] use markdown template filter -- [frontend] use isdigit instead of is_int -- [frontend] move Serializer to helpers -- [frontend] fix coding style and py3 compatibility -- [cli] fix coding style and py3 compatibility -- [backend] fix coding style and py3 compatibility - -* Tue Jan 28 2014 Miroslav Suchý <miroslav@suchy.cz> 1.26-1 -- lower testing date -- move localized_time into filters -- [frontend] update user data after login -- [frontend] use iso-8601 date - -* Mon Jan 27 2014 Miroslav Suchý <msuchy@redhat.com> 1.25-1 -- 1044085 - move timezone modification out of template and make it actually - work -- clean up temp data if any -- [db] timezone can be nullable -- [frontend] actually save the timezone to model -- fix colision of revision id -- 1044085 - frontend: display time in user timezone -- [frontend] rebuild stuck task -- disable test on i386 -- use experimental createrepo_c to get rid of lock on temp files -- [frontend] - do not throw ISE when build_id is malformed -- [tests] add test for BuildLogic.add -- [tests] add test for build resubmission -- [frontend] permission checking is done in BuildLogic.add -- [frontend] remove BuildLogic.new, use BL.add only -- [api] fix validation error handling -- [cli] fix initial_pkgs and repos not sent to backend -- [frontend] fix BuildsLogic.new not assigning copr to build -- [frontend] allow resubmitting builds from monitor -- [frontend] allow GET on repeat_build -- [frontend] 1050904 - monitor shows not submitted chroots -- [frontend] rename active_mock_chroots to active_chroots -- [frontend] rename MockChroot.chroot_name to .name -- [frontend] 1054474 - drop Copr.build_count nonsense -- [tests] fix https and repo generation -- [tests] return exit code from manage.py test -- 1054472 - Fix deleting multiple SRPMs -- [spec] tighten acl on copr-be.conf -- [backend] - add missing import -- 1054082 - general: encode to utf8 if err in mimetext -- [backend] lock log file before writing -- 1055594 - mockremote: always unquote pkg url -- 1054086 - change vendor tag -- mockremote: rawhide instead of $releasever in repos when in rawhide chroot -- 1055499 - do not replace version with $releasever on rawhide -- 1055119 - do not propagate https until it is properly signed -- fix spellings on chroot edit page -- 1054341 - be more verbose about allowed licenses -- 1054594 - temporary disable https in repo file - -* Thu Jan 16 2014 Miroslav Suchý <msuchy@redhat.com> 1.24-1 -- add BR python-markdown -- [fronted] don't add description to .repo files -- [spec] fix with_tests conditional -- add build deletion -- 1044158 - do not require fas username prior to login -- replace http with https in copr-cli and in generated repo file -- [cli] UX changes - explicitly state that pkgs is URL -- 1053142 - only build copr-cli on el6 -- [frontend] correctly handle mangled chroot -- [frontend] do not traceback when user malform url -- [frontend] change default description and instructions to sound more - dangerously -- 1052075 - do not set chroots on repeated build -- 1052071 - do not throw ISE when copr does not exist - -* Mon Jan 13 2014 Miroslav Suchý <msuchy@redhat.com> 1.23-1 -- [backend] rhel7-beta do not have comps -- 1052073 - correctly parse malformed chroot - -* Fri Jan 10 2014 Miroslav Suchý <msuchy@redhat.com> 1.22-1 -- [backend] if we could not spawn VM, wait a moment and try again -- [backend] use createrepo_c instead of createrepo -- 1050952 - check if copr_url exist in config -- [frontend] replace newlines in description by space in repo file - -* Wed Jan 08 2014 Miroslav Suchý <msuchy@redhat.com> 1.21-1 -- 1049460 - correct error message -- [cron] manually clean /var/tmp after createrepo - -* Wed Jan 08 2014 Miroslav Suchý <msuchy@redhat.com> 1.20-1 -- [cli] no need to set const with action=store_true -- [cli] code cleanup -- 1049460 - print nice error when projects does not exist -- 1049392 - require python-setuptools -- [backend] add --verbose to log to stderr -- [backend] handle KeyboardInterrupt without tons of tracebacks -- 1048508 - fix links at projects lists -- [backend] in case of error the output is in e.output -- [selinux] allow httpd to search -- [backend] set number of worker in name of process -- [logrotate] rotate every week unconditionally -- [backend] do not traceback if jobfile is mangled -- [backend] print error messages to stderr -- [cli] do not require additional arguments for --nowait -- [backend] replace procname with setproctitle -- [cli] use copr.fedoraproject.org as default url -- [frontend] show monitor even if last build have been canceled -- [backend] call correct function -- [cli] print errors to stderr -- 1044136 - do not print TB if config in mangled -- 1044165 - Provide login and token information in the same form as entered to - ~/.config-copr -- [frontend] code cleanup -- [frontend] move rendering of .repo file to helpers -- 1043649 - in case of Fedora use $releasever in repo file -- [frontend] condition should be in reverse - -* Mon Dec 16 2013 Miroslav Suchý <msuchy@redhat.com> 1.19-1 -- [backend] log real cause if ansible crash -- [frontend] try again if whoosh does not get lock -- [backend] if frontend does not respond, repeat -- print yum repos nicely -- Bump the copr-cli release to 0.2.0 with all the changes made -- Refer to the man page for more information about the configuration file for - copr-cli -- Rework the layout of the list command -- Fix parsing the copr_url from the configuration file -- [backend] run createrepo as copr user -- 1040615 - wrap lines with long URL - -* Wed Dec 11 2013 Miroslav Suchý <msuchy@redhat.com> 1.18-1 -- [frontend] inicialize variable - -* Wed Dec 11 2013 Miroslav Suchý <msuchy@redhat.com> 1.17-1 -- [frontend] fix latest build variable overwrite - -* Wed Dec 11 2013 Miroslav Suchý <msuchy@redhat.com> 1.16-1 -- [backend] store jobs in id-chroot.json file -- [frontend] handle unknown build/chroot status -- use newstyle ansible variables - -* Tue Dec 10 2013 Miroslav Suchý <msuchy@redhat.com> 1.15-1 -- [frontend] smarter package name parsing -- [frontend] extend range to allow 0 -- handle default timeout on backend -- initial support for SCL -- [backend] create word readable files in result directory -- [backend] print tracebacks -- [frontend] monitor: display only pkg name w/o version -- [doc] update api docs -- [doc] update copr-cli manpage -- [cli] list only name, description and instructions -- [cli] add support for build status & build monitor -- [frontend] add build status to API -- [playbook] do not overwrite mockchain -- [backend] add spece between options -- [backend] pass mock options correctly -- [frontend] support markdown in description and instructions -- [backend] Add macros to mockchain define arguments -- [backend] Pass copr username and project name to MockRemote -- [backend] Handle additional macro specification in MockRemote -- [frontend] monitor: show results per package -- [frontend] add favicon -- [backend] quote strings before passing to mockchain -- send chroots with via callback to frontend -- [cli] change cli to new api call -- enhance API documentation -- add yum_repos to coprs/user API call -- [frontend] provide link to description of allowed content -- [backend] we pass just one chroot -- [backend] - variable play is not defined -- if createrepo fail, run it again -- [cron] fix syntax error -- [man] state that --chroot for create command is required -- [spec] enable tests -- [howto] add note about upgrading db schema -- [frontend]: add copr monitor -- [tests]: replace test_allowed_one -- [tests]: fix for BuildChroots & new backend view -- [frontend] rewrite backend view to use Build <-> Chroot relation -- [frontend] add Build <-> Chroot relation -- 1030493 - [cli] check that at least one chroot is entered -- [frontend] typo -- fixup! [tests]: fix test_build_logic to handle BuildChroot -- fixup! [frontend] add ActionsLogic -- [tests]: fix test_build_logic to handle BuildChroot -- [spec] enable/disable test using variable -- add migration script - add table build_chroot -- [frontend] skip legal-flag actions when dumping waiting actions -- [frontend] rewrite backend view to use Build <-> Chroot relation -- [frontend] add ActionsLogic -- [frontend] create BuildChroot objects on new build -- [frontend] add Build <-> Chroot relation -- [frontend] add StatusEnum -- [frontend] fix name -> coprname typo -- [frontend] remove unused imports -- [frontend] add missing json import -- [backend] rework ip address extraction -- ownership of /etc/copr should be just normal -- [backend] - wrap up returning action in "action" blok -- [backend] rename backend api url -- [backend] handle "rename" action -- [backend] handle "delete" action -- base handling of actions -- move callback to frontend to separate object -- secure waiting_actions with password -- pick only individual builds -- make address, where we send legal flags, configurable -- send email to root after legal flag have been raised - -* Fri Nov 08 2013 Miroslav Suchý <msuchy@redhat.com> 1.14-1 -- 1028235 - add disclaimer about repos -- fix pagination -- fix one failing test - -* Wed Nov 06 2013 Miroslav Suchý <msuchy@redhat.com> 1.13-1 -- suggest correct name of repo file -- we could not use releasever macro -- no need to capitalize Projects -- another s/copr/project -- add link to header for sign-in -- fix failing tests -- UX - let textarea will full widht of box -- UX - make background of hovered builds darker -- generate yum repo for each chroot of copr -- align table header same way as ordinary rows -- enable resulting repo and disable gpgchecks - -* Mon Nov 04 2013 Miroslav Suchý <msuchy@redhat.com> 1.12-1 -- do not send parameters when we neither need them nor use them -- authenticate using api login, not using username -- disable editing name of project -- Add commented out WTF_CSRF_ENABLED = True to configs -- Use new session for each test -- fix test_coprs_general failures -- fix test_coprs_builds failures -- Add WTF_CSRF_ENABLED = False to unit test config -- PEP8 fixes -- Fix compatibility with wtforms 0.9 -- typo s/submited/submitted/ -- UX - show details of build only after click -- add link to FAQ to footer -- UX - add placeholders -- UX - add asterisk to required fields -- dynamicly generate url for home -- add footer - -* Sat Oct 26 2013 Miroslav Suchý <msuchy@redhat.com> 1.11-1 -- catch IOError from libravatar if there is no network - -* Fri Oct 25 2013 Miroslav Suchý <msuchy@redhat.com> 1.10-1 -- do not normalize url -- specify full prefix of http -- execute playbook using /usr/bin/ansible-playbook -- use ssh transport -- check after connection is made -- add notes about debuging mockremote -- clean up instance even when worker fails -- normalize paths before using -- do not use exception variable -- operator should be preceded and followed by space -- remove trailing whitespace -- convert comment to docstring -- use ssh transport -- do not create new ansible connection, reuse self.conn -- run copr-be.py as copr -- s/Copr/Project/ where we use copr in meaning of projects -- number will link to those coprs, to which it refers -- run log and jobgrab as copr user -- log event to log file -- convert comment into docstring -- use unbufferred output for copr-be.py -- hint how to set ec2 variables -- document sleeptime -- document copr_url for copr-cli -- document how to set api key for copr-cli -- do not create list of list -- document SECRET_KEY variable -- make note how to become admin -- instruct people to install selinux with frontend - -* Thu Oct 03 2013 Miroslav Suchý <msuchy@redhat.com> 1.9-1 -- prune old builds -- require python-decorator -- remove requirements.txt -- move TODO-backend to our wiki -- create pid file in /var/run/copr-backend -- add backend service file for systemd -- remove daemonize option in config -- use python logging -- create pid file in /var/run by default -- do not create destdir -- use daemon module instead of home brew function -- fix default location of copr-be.conf -- 2 tests fixed, one still failing -- fix failing test test_fail_on_missing_dash -- fixing test_fail_on_nonexistent_copr test -- run frontend unit tests when building package -- Adjust URLs in the unit-tests to their new structure -- Adjust the CLI to call the adjuste endpoint of the API -- Adjust API endpoint to reflects the UI endpoints in their url structure -- First pass at adding fedmsg hooks. - -* Tue Sep 24 2013 Miroslav Suchý <msuchy@redhat.com> 1.8-1 -- 1008532 - require python2-devel -- add note about ssh keys to copr-setup.txt -- set home of copr user to system default - -* Mon Sep 23 2013 Miroslav Suchý <msuchy@redhat.com> 1.7-1 -- 1008532 - backend should own _pkgdocdir -- 1008532 - backend should owns /etc/copr as well -- 1008532 - require logrotate -- 1008532 - do not distribute empty copr.if -- 1008532 - use %%{?_smp_mflags} macro with make -- move jobsdir to /var/lib/copr/jobs -- correct playbooks path -- selinux with enforce can be used for frontend - -* Wed Sep 18 2013 Miroslav Suchý <msuchy@redhat.com> 1.6-1 -- add BR python-devel -- generate selinux type for /var/lib/copr and /var/log/copr -- clean up backend setup instructions -- initial selinux subpackage - -* Mon Sep 16 2013 Miroslav Suchý <msuchy@redhat.com> 1.5-1 -- 1008532 - use __python2 instead of __python -- 1008532 - do not mark man page as doc -- 1008532 - preserve timestamp - -* Mon Sep 16 2013 Miroslav Suchý <msuchy@redhat.com> 1.4-1 -- add logrotate file - -* Mon Sep 16 2013 Miroslav Suchý <msuchy@redhat.com> 1.3-1 -- be clear how we create tgz - -* Mon Sep 16 2013 Miroslav Suchý <msuchy@redhat.com> 1.2-1 -- fix typo -- move frontend data into /var/lib/copr -- no need to own /usr/share/copr by copr-fe -- mark application as executable -- coprs_frontend does not need to be owned by copr-fe -- add executable attribute to copr-be.py -- remove shebang from dispatcher.py -- squeeze description into 80 chars -- fix typo -- frontend need argparse too -- move results into /var/lib/copr/public_html -- name of dir is just copr-%%version -- Remove un-necessary quote that breaks the tests -- Adjust unit-tests to the new urls -- Update the URL to be based upon a /user/copr/<action> structure -- comment config copr-be.conf and add defaults -- put examples of builderpb.yml and terminatepb.yml into doc dir -- more detailed description of copr-be.conf -- move files in config directory not directory itself -- include copr-be.conf -- include copr-be.py -- create copr with lighttpd group -- edit backend part of copr-setup.txt -- remove fedora16 and add 19 and 20 -- create -doc subpackage with python documentation -- add generated documentation on gitignore list -- add script to generate python documentation -- copr-setup.txt change to for mock -- rhel6 do not know _pkgdocdir macro -- make instruction clear -- require recent whoosh -- add support for libravatar -- include backend in rpm -- add notes about lighttpd config files and how to deploy them -- do not list file twice -- move log file to /var/log -- change destdir in copr-be.conf.example -- lightweight is the word and buildsystem has more meaning than 'koji'. -- restart apache after upgrade of frontend -- own directory where backend put results -- removal of hidden-file-or-dir - /usr/share/copr/coprs_frontend/coprs/logic/.coprs_logic.py.swo -- copr-backend.noarch: W: spelling-error %%description -l en_US latests -> - latest, latest's, la tests -- simplify configuration - introduce /etc/copr/copr*.conf -- Replace "with" statements with @TransactionDecorator decorator -- add python-flexmock to deps of frontend -- remove sentence which does not have meaning -- change api token expiration to 120 days and make it configurable -- create_chroot must be run as copr-fe user -- add note that you have to add chroots to db -- mark config.py as config so it is not overwritten during upgrade -- own directory data/whooshee/copr_user_whoosheer -- gcc is not needed -- sqlite db must be owned by copr-fe user -- copr does not work with selinux -- create subdirs under data/openid_store -- suggest to install frontend as package from copr repository -- on el6 add python-argparse to BR -- add python-requests to BR -- add python-setuptools to BR -- maintain apache configuration on one place only -- apache 2.4 changed access control -- require python-psycopg2 -- postgresql server is not needed -- document how to create db -- add to HOWTO how to create db -- require python-alembic -- add python-flask-script and python-flask-whooshee to requirements -- change user in coprs.conf.example to copr-fe -- fix paths in coprs.conf.example -- copr is noarch package -- add note where to configure frontend -- move frontend to /usr/share/copr/coprs_frontend -- put production placeholders in coprs_frontend/coprs/config.py -- put frontend into copr.spec -- web application should be put in /usr/share/%%{name} - -* Mon Jun 17 2013 Miroslav Suchý <msuchy@redhat.com> 1.1-1 -- new package built with tito - - +* Wed Mar 12 2025 lichaoran <pkwarcraft@hotmail.com> 1.168-1 +- Init package diff --git a/euler_msgbus.patch b/euler_msgbus.patch new file mode 100644 index 0000000..5eca90a --- /dev/null +++ b/euler_msgbus.patch @@ -0,0 +1,83 @@ +diff --git a/copr_backend/euler_msgbus.py b/copr_backend/euler_msgbus.py +new file mode 100644 +index 000000000..1395249be +--- /dev/null ++++ b/copr_backend/euler_msgbus.py +@@ -0,0 +1,77 @@ ++import socket ++ ++from kafka import KafkaProducer ++import os ++import datetime ++import uuid ++import json ++import ssl ++ ++ ++def message_from_worker_job(topic, job, who, ip, pid): ++ message = {} ++ content = { ++ 'user': job.submitter, ++ 'copr': job.project_name, ++ 'owner': job.project_owner, ++ 'pkg': job.package_name, ++ 'build': job.build_id, ++ 'chroot': job.chroot, ++ 'version': job.package_version, ++ 'status': job.status, ++ } ++ content.update({'ip': ip, 'who': who, 'pid': pid}) ++ message_types = { ++ 'build.start': { ++ 'what': "build start: user:{user} copr:{copr}" \ ++ " pkg:{pkg} build:{build} ip:{ip} pid:{pid}", ++ }, ++ 'chroot.start': { ++ 'what': "chroot start: chroot:{chroot} user:{user}" \ ++ " copr:{copr} pkg:{pkg} build:{build} ip:{ip} pid:{pid}", ++ }, ++ 'build.end': { ++ 'what': "build end: user:{user} copr:{copr} build:{build}" \ ++ " pkg:{pkg} version:{version} ip:{ip} pid:{pid} status:{status}", ++ }, ++ } ++ content['what'] = message_types[topic]['what'].format(**content) ++ message['body'] = content ++ now = datetime.datetime.now().isoformat() ++ headers = { ++ "openEuler_messaging_schema": "eur." + topic, ++ "sent-at": now, ++ } ++ message['headers'] = headers ++ message['id'] = str(uuid.uuid4()) ++ message['topic'] = "org.openEuler.prod.eur." + topic ++ return message ++ ++ ++class MessageSender: ++ def __init__(self, backend_opts, name, log): ++ self.log = log ++ self.name = name ++ self.pid = os.getpid() ++ self.opts = backend_opts ++ ++ def announce(self, topic, job, host): ++ msg = message_from_worker_job(topic, job, self.name, host, self.pid) ++ self.send_message(msg) ++ ++ def send_message(self, msg): ++ """ Send message to kafka """ ++ ++ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ++ context.verify_mode = ssl.CERT_REQUIRED ++ context.load_verify_locations("etc/copr/kafka.crt") ++ producer = KafkaProducer(bootstrap_servers=self.opts.message.bootstrap_servers, ++ api_version=(3, 5, 0), ++ sasl_mechanism="PLAIN", ++ ssl_context=context, ++ security_protocol='SASL_SSL', ++ sasl_plain_username=self.opts.message.user_name, ++ sasl_plain_password=self.opts.message.password, ++ value_serializer=lambda v: json.dumps(v).encode('utf-8')) ++ producer.send(str(self.opts.message.topic), msg) ++ producer.flush() diff --git a/fail_to_build_in_oe2403.patch b/fail_to_build_in_oe2403.patch new file mode 100644 index 0000000..be32aee --- /dev/null +++ b/fail_to_build_in_oe2403.patch @@ -0,0 +1,14 @@ +diff --git a/tests/test_modifyrepo.py b/backend/tests/test_modifyrepo.py +index 330ff68f7..2c1bd0a66 100644 +--- a/tests/test_modifyrepo.py ++++ b/tests/test_modifyrepo.py +@@ -431,7 +431,8 @@ def test_copr_repo_run_createrepo(self, popen, do_stat, chroot, database_option) + "--workers", "8", "--general-compress-type=gz", "--update"] + additional_args + + @pytest.mark.skipif( +- distro.id() == 'fedora' and int(distro.version()) >= 36, ++ (distro.id() == 'fedora' and int(distro.version()) >= 36) or ++ (distro.id() == 'openeuler' and float(distro.version()) >= 24.03), + reason="createrepo_c dropped md5 checksum support" + ) + def test_copr_repo_el5(self, f_third_build): diff --git a/signatrust_bin.patch b/signatrust_bin.patch new file mode 100644 index 0000000..fcba220 --- /dev/null +++ b/signatrust_bin.patch @@ -0,0 +1,105 @@ +diff --git a/run/copr_fix_gpg.py b/run/copr_fix_gpg.py +index 24edf74e..bfa70c05 100755 +--- a/run/copr_fix_gpg.py ++++ b/run/copr_fix_gpg.py +@@ -9,7 +9,7 @@ from urllib.parse import urlparse + import pwd + + from copr_backend.helpers import BackendConfigReader, call_copr_repo, run_cmd +-from copr_backend.sign import get_pubkey, unsign_rpms_in_dir, sign_rpms_in_dir, create_user_keys, create_gpg_email ++from copr_backend.sign import new_signer, create_gpg_email + + logging.basicConfig( + filename="/var/log/copr-backend/fix_gpg.log", +@@ -69,12 +69,14 @@ def fix_copr(args, opts, copr_full_name): + log.info('Ignoring %s. Directory does not exist.', copr_path) + return + ++ signer = new_signer(opts) ++ + log.info("Generate key-pair on copr-keygen (if not generated) for email %s", + create_gpg_email(owner, coprname, opts.sign_domain)) +- create_user_keys(owner, coprname, opts) ++ signer.create_user_keys(owner, coprname, opts) + + log.info("Regenerate pubkey.gpg in copr %s", copr_path) +- get_pubkey(owner, coprname, log, opts.sign_domain, os.path.join(copr_path, 'pubkey.gpg')) ++ signer.get_pubkey(owner, coprname, log, opts.sign_domain, os.path.join(copr_path, 'pubkey.gpg')) + + # Match the "00001231-anycharacer" directory names. Compile once, use many. + builddir_matcher = re.compile(r"\d{8,}-") +@@ -111,8 +113,8 @@ def fix_copr(args, opts, copr_full_name): + + log.info("Processing rpms in builddir %s", builddir_path) + try: +- unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign +- sign_rpms_in_dir(owner, coprname, builddir_path, chroot, opts, log) ++ signer.unsign_rpms_in_dir(builddir_path, opts, log) # first we need to unsign by using rpm-sign before we sign with obs-sign ++ signer.sign_rpms_in_dir(owner, coprname, builddir_path, chroot, opts, log) + except Exception as e: + log.exception(str(e)) + continue +diff --git a/run/copr_sign_unsigned.py b/run/copr_sign_unsigned.py +index 33eaeae3..e17e65bb 100755 +--- a/run/copr_sign_unsigned.py ++++ b/run/copr_sign_unsigned.py +@@ -12,7 +12,7 @@ import pwd + + from copr_backend.helpers import (BackendConfigReader, create_file_logger, + uses_devel_repo, call_copr_repo) +-from copr_backend.sign import get_pubkey, sign_rpms_in_dir, create_user_keys ++from copr_backend.sign import new_signer + from copr_backend.exceptions import CoprSignNoKeyError + + +@@ -26,10 +26,11 @@ log = logging.getLogger(__name__) + def check_signed_rpms_in_pkg_dir(pkg_dir, user, project, opts, chroot_dir, devel): + success = True + ++ signer = new_signer(opts) + logger = create_file_logger("run.check_signed_rpms_in_pkg_dir", + "/tmp/copr_check_signed_rpms.log") + try: +- sign_rpms_in_dir(user, project, pkg_dir, chroot_dir, opts, log=logger) ++ signer.sign_rpms_in_dir(user, project, pkg_dir, chroot_dir, opts, log=logger) + log.info("running createrepo for %s", pkg_dir) + call_copr_repo(directory=chroot_dir, devel=devel, logger=log) + except Exception as err: +@@ -80,9 +81,10 @@ def check_pubkey(pubkey_path, user, project, opts): + log.info("Pubkey for %s/%s exists: %s", user, project, pubkey_path) + return True + else: ++ signer = new_signer(opts) + log.info("Missing pubkey for %s/%s", user, project) + try: +- get_pubkey(user, project, log, opts.sign_domain, pubkey_path) ++ signer.get_pubkey(user, project, log, opts.sign_domain, pubkey_path) + return True + except Exception as err: + log.exception(err) +@@ -102,6 +104,8 @@ def main(): + opts = BackendConfigReader().read() + log.info("Starting pubkey fill, destdir: %s", opts.destdir) + ++ signer = new_signer(opts) ++ + log.debug("list dir: %s", os.listdir(opts.destdir)) + for user_name in os.listdir(opts.destdir): + if user_name in users_done_old: +@@ -116,13 +120,13 @@ def main(): + log.info("Checking project dir: %s", project_name) + + try: +- get_pubkey(user_name, project_name, log, opts.sign_domain) ++ signer.get_pubkey(user_name, project_name, log, opts.sign_domain) + log.info("Key-pair exists for %s/%s", user_name, project_name) + except CoprSignNoKeyError: +- create_user_keys(user_name, project_name, opts) ++ signer.create_user_keys(user_name, project_name, opts) + log.info("Created new key-pair for %s/%s", user_name, project_name) + except Exception as err: +- log.error("Failed to get pubkey for {}/{}, mark as failed, skipping") ++ log.error("Failed to get pubkey for {}/{}, mark as failed, skipping".format(user_name, project_name)) + log.exception(err) + failed = True + continue @@ -1,2 +1,2 @@ -22717aa0348a40c6e80c746cc6c818ca copr-backend-1.173.tar.gz +5ed3646f3887a4ff1656c1f7a05d78eb copr-backend-1.173.tar.gz c94d55e21a4935b3e62f673fbae65af2 test-data-copr-backend-5.tar.gz diff --git a/support_signatrust_backend.patch b/support_signatrust_backend.patch new file mode 100644 index 0000000..d803d26 --- /dev/null +++ b/support_signatrust_backend.patch @@ -0,0 +1,1150 @@ +diff --git a/copr_backend/actions.py b/backend/copr_backend/actions.py +index 39722b843..6da2dfb65 100644 +--- a/copr_backend/actions.py ++++ b/copr_backend/actions.py +@@ -21,13 +21,13 @@ from copr_common.worker_manager import WorkerManager + + from copr_backend.worker_manager import BackendQueueTask + +-from .sign import create_user_keys, CoprKeygenRequestError ++from .sign import CoprKeygenRequestError + from .exceptions import CreateRepoError, CoprSignError, FrontendClientException + from .helpers import (get_redis_logger, silent_remove, ensure_dir_exists, + get_chroot_arch, format_filename, + uses_devel_repo, call_copr_repo, build_chroot_log_name, + copy2_but_hardlink_rpms) +-from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey ++from .sign import new_signer + + + class Action(object): +@@ -92,6 +92,8 @@ class Action(object): + + self.log = log if log else get_redis_logger(self.opts, "backend.actions", "actions") + ++ self.signer = new_signer(opts) ++ + def __str__(self): + return "<{}(Action): {}>".format(self.__class__.__name__, self.data) + +@@ -147,7 +149,7 @@ class GPGMixin(object): + # skip key creation, most probably sign component is unused + return True + try: +- create_user_keys(ownername, projectname, self.opts) ++ self.signer.create_user_keys(ownername, projectname, self.opts) + return True + except CoprKeygenRequestError as e: + self.log.exception(e) +@@ -176,7 +178,7 @@ class Fork(Action, GPGMixin): + # Generate brand new gpg key. + self.generate_gpg_key(data["user"], data["copr"]) + # Put the new public key into forked build directory. +- get_pubkey(data["user"], data["copr"], self.log, self.opts.sign_domain, pubkey_path) ++ self.signer.get_pubkey(data["user"], data["copr"], self.log, self.opts.sign_domain, pubkey_path) + + chroot_paths = set() + for chroot, src_dst_dir in builds_map.items(): +@@ -206,9 +208,9 @@ class Fork(Action, GPGMixin): + continue + + # Drop old signatures coming from original repo and re-sign. +- unsign_rpms_in_dir(dst_path, opts=self.opts, log=self.log) ++ self.signer.unsign_rpms_in_dir(dst_path, opts=self.opts, log=self.log) + if sign: +- sign_rpms_in_dir(data["user"], data["copr"], dst_path, ++ self.signer.sign_rpms_in_dir(data["user"], data["copr"], dst_path, + chroot, opts=self.opts, log=self.log) + + self.log.info("Forked build %s as %s", src_path, dst_path) +diff --git a/copr_backend/background_worker.py b/backend/copr_backend/background_worker.py +index 4b5f13135..bc05edbd7 100644 +--- a/copr_backend/background_worker.py ++++ b/copr_backend/background_worker.py +@@ -9,7 +9,7 @@ import logging + from copr_common.background_worker import BackgroundWorker + from copr_backend.frontend import FrontendClient + from copr_backend.helpers import (BackendConfigReader, get_redis_logger) +- ++from copr_backend.sign import new_signer + + class BackendBackgroundWorker(BackgroundWorker): + """ +@@ -28,6 +28,7 @@ class BackendBackgroundWorker(BackgroundWorker): + + self.frontend_client = FrontendClient(self.opts, self.log, + try_indefinitely=True) ++ self.signer = new_signer(self.opts) + + def _switch_logger_to_redis(self): + logger_name = '{}.{}.pid-{}'.format( +diff --git a/copr_backend/background_worker_build.py b/backend/copr_backend/background_worker_build.py +index 0eee39ab8..c12d4921d 100644 +--- a/copr_backend/background_worker_build.py ++++ b/copr_backend/background_worker_build.py +@@ -27,8 +27,8 @@ from copr_backend.helpers import ( + call_copr_repo, run_cmd, register_build_result, format_evr, + ) + from copr_backend.job import BuildJob +-from copr_backend.msgbus import MessageSender +-from copr_backend.sign import sign_rpms_in_dir, get_pubkey ++from copr_backend.euler_msgbus import MessageSender ++from copr_backend.sign import new_signer + from copr_backend.sshcmd import SSHConnection, SSHConnectionError + from copr_backend.vm_alloc import ResallocHostFactory + +@@ -622,7 +622,7 @@ class BuildBackgroundWorker(BackendBackgroundWorker): + self.log.info("Going to sign pkgs from source: %s in chroot: %s", + self.job.task_id, self.job.chroot_dir) + +- sign_rpms_in_dir( ++ self.signer.sign_rpms_in_dir( + self.job.project_owner, + self.job.project_name, + os.path.join(self.job.chroot_dir, self.job.target_dir_name), +@@ -736,7 +736,7 @@ class BuildBackgroundWorker(BackendBackgroundWorker): + # TODO: uncomment this when key revoke/change will be implemented + # if os.path.exists(pubkey_path): + # return +- get_pubkey(user, project, self.log, self.opts.sign_domain, pubkey_path) ++ self.signer.get_pubkey(user, project, self.log, self.opts.sign_domain, pubkey_path) + self.log.info("Added pubkey for user %s project %s into: %s", + user, project, pubkey_path) + +diff --git a/copr_backend/constants.py b/backend/copr_backend/constants.py +index a529be28a..83bcb8fbe 100644 +--- a/copr_backend/constants.py ++++ b/copr_backend/constants.py +@@ -13,6 +13,7 @@ DEF_BUILD_USER = "mockbuilder" + DEF_DESTDIR = os.getcwd() + DEF_MACROS = {} + DEF_BUILDROOT_PKGS = "" ++DEF_SIGN_BACKEND = "obs-signd" + + + DEF_CONSECUTIVE_FAILURE_THRESHOLD = 10 +diff --git a/copr_backend/exceptions.py b/backend/copr_backend/exceptions.py +index 21afb14c6..0865fcc8c 100644 +--- a/copr_backend/exceptions.py ++++ b/copr_backend/exceptions.py +@@ -48,8 +48,8 @@ class CoprKeygenRequestError(Exception): + + def __str__(self): + out = super(CoprKeygenRequestError, self).__str__() +- out += "\nrequest to copr-keygen: {}\n".format(self.request) +- if self.response: ++ out += "\nrequest to key_backend: {}\n".format(self.request) ++ if self.response is not None: + out += "status code: {}\n" "response content: {}\n" \ + .format(self.response.status_code, self.response.content) + return out +diff --git a/copr_backend/helpers.py b/backend/copr_backend/helpers.py +index 75fa5e62d..291228912 100644 +--- a/copr_backend/helpers.py ++++ b/copr_backend/helpers.py +@@ -31,7 +31,7 @@ from munch import Munch + from copr_common.redis_helpers import get_redis_connection + from copr.v3 import Client + from copr_backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \ +- CONSECUTIVE_FAILURE_REDIS_KEY, default_log_format ++ CONSECUTIVE_FAILURE_REDIS_KEY, default_log_format, DEF_SIGN_BACKEND + from copr_backend.exceptions import CoprBackendError + + from . import constants +@@ -309,6 +309,18 @@ class BackendConfigReader(object): + opts.sign_domain = _get_conf( + cp, "backend", "sign_domain", DOMAIN) + ++ opts.sign_backend = _get_conf( ++ cp, "backend", "sign_backend", DEF_SIGN_BACKEND) ++ ++ opts.signatrust_host = _get_conf( ++ cp, "backend", "signatrust_host", "") ++ ++ opts.signatrust_token = _get_conf( ++ cp, "backend", "signatrust_token", "") ++ ++ opts.signatrust_key_expire = _get_conf( ++ cp, "backend", "signatrust_key_expire", 1825, mode="int") ++ + opts.build_groups = [] + for group_id in range(opts.build_groups_count): + archs = _get_conf(cp, "backend", +diff --git a/copr_backend/sign.py b/backend/copr_backend/sign.py +index e21653e78..95f674255 100644 +--- a/copr_backend/sign.py ++++ b/copr_backend/sign.py +@@ -25,78 +25,6 @@ def create_gpg_email(username, projectname, domain): + + return "{}#{}@copr.{}".format(username, projectname, domain) + +- +-def call_sign_bin(cmd, log): +- """ +- Call /bin/sign and return (rc, stdout, stderr). Re-try the call +- automatically upon certain failures (if that makes sense). +- """ +- cmd_pretty = ' '.join(cmd) +- for attempt in [1, 2, 3]: +- log.info("Calling '%s' (attempt #%s)", cmd_pretty, attempt) +- try: +- handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") +- stdout, stderr = handle.communicate() +- except (SubprocessError, OSError) as err: +- new_err = CoprSignError("Failed to invoke '{}'".format(cmd_pretty)) +- raise new_err from err +- +- if handle.returncode != 0: +- log.warning("Command '%s' failed with: %s", +- cmd_pretty, stderr.rstrip()) +- sleeptime = 20 +- log.warning("Going to sleep %ss and re-try.", sleeptime) +- time.sleep(sleeptime) +- continue +- break +- return handle.returncode, stdout, stderr +- +- +-def get_pubkey(username, projectname, log, sign_domain, outfile=None): +- """ +- Retrieves public key for user/project from signer host. +- +- :param sign_domain: the domain name of the sign key +- :param outfile: [optional] file to write obtained key +- :return: public keys +- +- :raises CoprSignError: failed to retrieve key, see error message +- :raises CoprSignNoKeyError: if there are no such user in keyring +- """ +- usermail = create_gpg_email(username, projectname, sign_domain) +- cmd = [SIGN_BINARY, "-u", usermail, "-p"] +- +- returncode, stdout, stderr = call_sign_bin(cmd, log) +- if returncode != 0: +- if "unknown key:" in stderr: +- raise CoprSignNoKeyError( +- "There are no gpg keys for user {} in keyring".format(username), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- raise CoprSignError( +- msg="Failed to get user pubkey\n" +- "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- +- if outfile: +- with open(outfile, "w") as handle: +- handle.write(stdout) +- +- return stdout +- +- +-def _sign_one(path, email, hashtype, log): +- cmd = [SIGN_BINARY, "-4", "-h", hashtype, "-u", email, "-r", path] +- returncode, stdout, stderr = call_sign_bin(cmd, log) +- if returncode != 0: +- raise CoprSignError( +- msg="Failed to sign {} by user {}".format(path, email), +- return_code=returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- return stdout, stderr +- +- + def gpg_hashtype_for_chroot(chroot, opts): + """ + Given the chroot name (in "mock format", like "fedora-rawhide-x86_64") +@@ -136,133 +64,436 @@ def gpg_hashtype_for_chroot(chroot, opts): + return "sha256" + + +-def sign_rpms_in_dir(username, projectname, path, chroot, opts, log): +- """ +- Signs rpms using obs-signd. ++# a sign interface ++class Signer(object): ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """get public key""" ++ raise NotImplementedError + +- If some some pkgs failed to sign, entire build marked as failed, +- but we continue to try sign other pkgs. ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """batch sign rpms""" ++ raise NotImplementedError + +- :param username: copr username +- :param projectname: copr projectname +- :param path: directory with rpms to be signed +- :param chroot: chroot name where we sign packages, affects the hash type +- :param Munch opts: backend config ++ @classmethod ++ def create_user_keys(username, projectname, opts, try_indefinitely=False): ++ """create user key pair""" ++ raise NotImplementedError + +- :type log: logging.Logger ++ @classmethod ++ def _sign_one(cls, path, email, hashtype, log): ++ """sign one rpm""" ++ raise NotImplementedError + +- :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package +- """ ++ @classmethod ++ def _unsign_one(cls, path): ++ # Requires rpm-sign package ++ cmd = ["/usr/bin/rpm", "--delsign", path] ++ handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") ++ stdout, stderr = handle.communicate() ++ ++ if handle.returncode != 0: ++ err = CoprSignError( ++ msg="Failed to unsign {}".format(path), ++ return_code=handle.returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- rpm_list = [ +- os.path.join(path, filename) +- for filename in os.listdir(path) +- if filename.endswith(".rpm") +- ] ++ raise err ++ ++ return stdout, stderr ++ ++ @classmethod ++ def call_sign_bin(cls, cmd, log): ++ """ ++ Call sign_cmd and return (rc, stdout, stderr). Re-try the call ++ automatically upon certain failures (if that makes sense). ++ """ ++ cmd_pretty = ' '.join(cmd) ++ for attempt in [1, 2, 3]: ++ log.info("Calling '%s' (attempt #%s)", cmd_pretty, attempt) ++ try: ++ handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") ++ stdout, stderr = handle.communicate() ++ except (SubprocessError, OSError) as err: ++ new_err = CoprSignError("Failed to invoke '{}'".format(cmd_pretty)) ++ raise new_err from err ++ ++ if handle.returncode != 0: ++ log.warning("Command '%s' failed with: %s", ++ cmd_pretty, stderr.rstrip()) ++ sleeptime = 20 ++ log.warning("Going to sleep %ss and re-try.", sleeptime) ++ time.sleep(sleeptime) ++ continue ++ break ++ return handle.returncode, stdout, stderr ++ ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """ ++ Signs rpms using obs-signd. ++ ++ If some some pkgs failed to sign, entire build marked as failed, ++ but we continue to try sign other pkgs. ++ ++ :param username: copr username ++ :param projectname: copr projectname ++ :param path: directory with rpms to be signed ++ :param chroot: chroot name where we sign packages, affects the hash type ++ :param Munch opts: backend config ++ ++ :type log: logging.Logger ++ ++ :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package ++ """ ++ rpm_list = [ ++ os.path.join(path, filename) ++ for filename in os.listdir(path) ++ if filename.endswith(".rpm") ++ ] + +- if not rpm_list: +- return ++ if not rpm_list: ++ return + +- hashtype = gpg_hashtype_for_chroot(chroot, opts) ++ hashtype = gpg_hashtype_for_chroot(chroot, opts) + +- try: +- get_pubkey(username, projectname, log, opts.sign_domain) +- except CoprSignNoKeyError: +- create_user_keys(username, projectname, opts, try_indefinitely=True) ++ try: ++ cls.get_pubkey(username, projectname, log, opts.sign_domain) ++ except CoprSignNoKeyError: ++ cls.create_user_keys(username, projectname, opts, try_indefinitely=True) ++ ++ errors = [] # tuples (rpm_filepath, exception) ++ for rpm in rpm_list: ++ try: ++ cls._sign_one(rpm, create_gpg_email(username, projectname, opts.sign_domain), ++ hashtype, log) ++ log.info("signed rpm: %s", rpm) ++ ++ except CoprSignError as e: ++ log.exception("failed to sign rpm: %s", rpm) ++ errors.append((rpm, e)) ++ ++ if errors: ++ raise CoprSignError("Rpm sign failed, affected rpms: {}" ++ .format([err[0] for err in errors])) ++ ++ @classmethod ++ def unsign_rpms_in_dir(cls, path, opts, log): ++ """ ++ :param path: directory with rpms to be signed ++ :param Munch opts: backend config ++ :type log: logging.Logger ++ :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package ++ """ ++ rpm_list = [ ++ os.path.join(path, filename) ++ for filename in os.listdir(path) ++ if filename.endswith(".rpm") ++ ] ++ ++ if not rpm_list: ++ return ++ ++ errors = [] # tuples (rpm_filepath, exception) ++ for rpm in rpm_list: ++ try: ++ cls._unsign_one(rpm) ++ log.info("unsigned rpm: %s", rpm) ++ ++ except CoprSignError as e: ++ log.exception("failed to unsign rpm: %s", rpm) ++ errors.append((rpm, e)) ++ ++ if errors: ++ raise CoprSignError("Rpm unsign failed, affected rpms: {}" ++ .format([err[0] for err in errors])) ++ ++def new_signer(opts): ++ if opts.sign_backend == "signatrust": ++ Signatrust.signatrust_token = opts.signatrust_token ++ Signatrust.signatrust_host = opts.signatrust_host ++ return Signatrust ++ else: # keep obs-signd as default backend ++ return ObsSign ++ ++class ObsSign(Signer): ++ sign_cmd = "/bin/sign" ++ ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """ ++ Retrieves public key for user/project from signer host. ++ ++ :param sign_domain: the domain name of the sign key ++ :param outfile: [optional] file to write obtained key ++ :return: public keys ++ ++ :raises CoprSignError: failed to retrieve key, see error message ++ :raises CoprSignNoKeyError: if there are no such user in keyring ++ """ ++ usermail = create_gpg_email(username, projectname, sign_domain) ++ cmd = [cls.sign_cmd, "-u", usermail, "-p"] ++ ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ if "unknown key:" in stderr: ++ raise CoprSignNoKeyError( ++ "There are no gpg keys for user {} in keyring".format(username), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) ++ raise CoprSignError( ++ msg="Failed to get user pubkey\n" ++ "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- errors = [] # tuples (rpm_filepath, exception) +- for rpm in rpm_list: ++ if outfile: ++ with open(outfile, "w") as handle: ++ handle.write(stdout) ++ ++ return stdout ++ ++ @classmethod ++ def _sign_one(cls, path, email, hashtype, log): ++ cmd = [cls.sign_cmd, "-4", "-h", hashtype, "-u", email, "-r", path] ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ raise CoprSignError( ++ msg="Failed to sign {} by user {}".format(path, email), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) ++ return stdout, stderr ++ ++ @classmethod ++ def create_user_keys(cls, username, projectname, opts, try_indefinitely=False): ++ """ ++ Generate a new key-pair at sign host ++ ++ :param username: ++ :param projectname: ++ :param opts: backend config ++ ++ :return: None ++ """ ++ data = { ++ "name_real": "{}_{}".format(username, projectname), ++ "name_email": create_gpg_email(username, projectname, opts.sign_domain) ++ } ++ ++ log = get_redis_logger(opts, "sign", "actions") ++ keygen_url = "http://{}/gen_key".format(opts.keygen_host) ++ query = dict(url=keygen_url, data=data, method="post") ++ try: ++ request = SafeRequest(log=log, try_indefinitely=try_indefinitely) ++ response = request.send(**query) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to create key-pair for user: {}," ++ " project:{} with error: {}" ++ .format(username, projectname, e), request=query) ++ ++ if response.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to create key-pair for user: {}, project:{}, status_code: {}, response: {}" ++ .format(username, projectname, response.status_code, response.text), ++ request=query, response=response) ++ ++class Signatrust(Signer): ++ sign_cmd = "/usr/local/bin/client" ++ prefix = "" ++ signatrust_host = "" ++ signatrust_token = "" ++ ++ @classmethod ++ def get_prefix(cls): ++ """ ++ Get prefix of the user ++ ++ As in copr, we set key attr with visibility=private ++ These keys' name were prefixed by user's email like: ++ tommylikehu@gmail.com:mywaaagh_admin_test ++ """ ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } ++ try: ++ r = requests.get("{}/api/v1/users/info".format(cls.signatrust_host), headers=headers).json() ++ cls.prefix = r.get("email") ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get userinfo", request="/api/v1/users/info") ++ ++ @classmethod ++ def get_key_name(cls, username, projectname, prefix=True): ++ """ ++ copr key_name rule in signatrust: ++ <user prefix>:<user_name>_<project_name> ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ if prefix: ++ return "{}:{}_{}".format(cls.prefix, username, projectname) ++ return "{}_{}".format(username, projectname) ++ ++ @classmethod ++ def get_pubkey(cls, username, projectname, log, sign_domain, outfile=None): ++ """ ++ get public key ++ ++ https://domain:port/api/v1/keys/<key_name>/public_key ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } ++ ++ key_name = cls.get_key_name(username, projectname) ++ url = "{}/api/v1/keys/{}/public_key".format(cls.signatrust_host, key_name) + try: +- _sign_one(rpm, create_gpg_email(username, projectname, opts.sign_domain), +- hashtype, log) +- log.info("signed rpm: %s", rpm) ++ r = requests.get(url, headers=headers) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get public_key", request="/api/v1/keys/{}/public_key".format(key_name), response=r) + +- except CoprSignError as e: +- log.exception("failed to sign rpm: %s", rpm) +- errors.append((rpm, e)) ++ if r.status_code == 404: ++ raise CoprSignNoKeyError( ++ "There are no gpg keys for user {} in keyring".format(username), ++ return_code=r.status_code, ++ cmd="GET {}".format(url), stdout="", stderr="") ++ elif r.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to get user pubkey\n", ++ request="/api/v1/keys/{}/public_key".format(key_name), response=r) ++ ++ if outfile: ++ with open(outfile, "wb") as handle: ++ handle.write(r.content) ++ ++ return r.content ++ ++ @classmethod ++ def sign_rpms_in_dir(cls, username, projectname, path, chroot, opts, log): ++ """batch sign rpms""" ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ if not cls._key_existed(username, projectname, opts): ++ cls.create_user_keys(username, projectname, opts) ++ ++ # when we migrate copr keys into signatrust ++ # we fellow the rules: ++ # key_name = <user_email>:<user_name>_<project_name> ++ cmd = [cls.sign_cmd, "-c", "/etc/signatrust.toml", "add", "--file-type", "rpm", "--key-type", "pgp", "--key-name", cls.get_key_name(username, projectname), path] ++ ++ returncode, stdout, stderr = cls.call_sign_bin(cmd, log) ++ if returncode != 0: ++ raise CoprSignError( ++ msg="Failed to sign rpms\n" ++ "sign stdout: {}\n sign stderr: {}\n".format(stdout, stderr), ++ return_code=returncode, ++ cmd=cmd, stdout=stdout, stderr=stderr) + +- if errors: +- raise CoprSignError("Rpm sign failed, affected rpms: {}" +- .format([err[0] for err in errors])) ++ if stderr: ++ # signatrust client will print error message for one rpm per line ++ failed_list = re.findall(r"failed to sign file (.*.rpm) due to error", stderr) ++ ++ if failed_list: ++ failed_rpms = " ".join(map(os.path.basename, failed_list)) ++ log.exception("failed to sign rpm: %s".format(failed_rpms)) ++ raise CoprSignError("Rpm sign failed, affected rpms: {}" ++ .format(failed_rpms)) ++ ++ @classmethod ++ def unsign_rpms_in_dir(cls, path, opts, log): ++ """ ++ signatrust will replace the signature infomation defaultly, ++ so there is no need to unsign, just return ++ """ ++ return + ++ @classmethod ++ def _key_existed(cls, username, projectname, opts): ++ """ ++ check keyname existence + +-def create_user_keys(username, projectname, opts, try_indefinitely=False): +- """ +- Generate a new key-pair at sign host ++ HEAD /api/v1/keys/ ++ """ ++ if not cls.prefix: ++ cls.get_prefix() + +- :param username: +- :param projectname: +- :param opts: backend config ++ key_name = cls.get_key_name(username, projectname, prefix=False) + +- :return: None +- """ +- data = { +- "name_real": "{}_{}".format(username, projectname), +- "name_email": create_gpg_email(username, projectname, opts.sign_domain) +- } +- +- log = get_redis_logger(opts, "sign", "actions") +- keygen_url = "http://{}/gen_key".format(opts.keygen_host) +- query = dict(url=keygen_url, data=data, method="post") +- try: +- request = SafeRequest(log=log, try_indefinitely=try_indefinitely) +- response = request.send(**query) +- except Exception as e: +- raise CoprKeygenRequestError( +- msg="Failed to create key-pair for user: {}," +- " project:{} with error: {}" +- .format(username, projectname, e), request=query) +- +- if response.status_code >= 400: +- raise CoprKeygenRequestError( +- msg="Failed to create key-pair for user: {}, project:{}, status_code: {}, response: {}" +- .format(username, projectname, response.status_code, response.text), +- request=query, response=response) +- +- +-def _unsign_one(path): +- # Requires rpm-sign package +- cmd = ["/usr/bin/rpm", "--delsign", path] +- handle = Popen(cmd, stdout=PIPE, stderr=PIPE, encoding="utf-8") +- stdout, stderr = handle.communicate() +- +- if handle.returncode != 0: +- err = CoprSignError( +- msg="Failed to unsign {}".format(path), +- return_code=handle.returncode, +- cmd=cmd, stdout=stdout, stderr=stderr) +- +- raise err +- +- return stdout, stderr +- +- +-def unsign_rpms_in_dir(path, opts, log): +- """ +- :param path: directory with rpms to be signed +- :param Munch opts: backend config +- :type log: logging.Logger +- :raises: :py:class:`backend.exceptions.CoprSignError` failed to sign at least one package +- """ +- rpm_list = [ +- os.path.join(path, filename) +- for filename in os.listdir(path) +- if filename.endswith(".rpm") +- ] ++ query = { ++ "name": key_name, ++ "visibility": "private" ++ } + +- if not rpm_list: +- return ++ headers = { ++ "accept": "application/json", ++ "Authorization": cls.signatrust_token ++ } + +- errors = [] # tuples (rpm_filepath, exception) +- for rpm in rpm_list: + try: +- _unsign_one(rpm) +- log.info("unsigned rpm: %s", rpm) +- +- except CoprSignError as e: +- log.exception("failed to unsign rpm: %s", rpm) +- errors.append((rpm, e)) ++ res = requests.head("{}/api/v1/keys/name_identical".format(opts.signatrust_host), headers=headers, params=query) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to check key existence", request="/api/v1/keys/name_identical", response=res) ++ ++ # signatrust return 200 means key name available ++ if res.status_code == 200: ++ return False ++ # signatrust return 409 means key name redundant ++ elif res.status_code == 409: ++ return True ++ else: ++ raise CoprKeygenRequestError( ++ msg="Failed to check key existence", request="/api/v1/keys/name_identical", response=res) ++ ++ @classmethod ++ def create_user_keys(cls, username, projectname, opts, try_indefinitely=False): ++ """ ++ create user key pair ++ ++ POST /api/v1/keys/ ++ """ ++ if not cls.prefix: ++ cls.get_prefix() ++ ++ if cls._key_existed(username, projectname, opts): ++ return ++ time_format = "%Y-%m-%d %H:%M:%S%z" ++ expire = datetime.now(datetime.now(timezone.utc).astimezone().tzinfo) + timedelta(days=opts.signatrust_key_expire) ++ data = { ++ "name": cls.get_key_name(username, projectname, prefix=False), ++ "description": "gpg key to sign rpms in {}/{}".format(username, projectname), ++ "key_type": "pgp", ++ "visibility": "private", # we use private key type for those key will not be seen by other users ++ "attributes": { ++ "digest_algorithm": "sha2_256", ++ "key_type": "rsa", ++ "key_length": "2048", ++ "email": "{}".format(create_gpg_email(username, projectname, opts.sign_domain)), ++ }, ++ "expire_at": datetime.strftime(expire, time_format) ++ } ++ ++ headers = { ++ "content-type": "application/json", ++ "accept": "application/json", ++ "Authorization": opts.signatrust_token ++ } + +- if errors: +- raise CoprSignError("Rpm unsign failed, affected rpms: {}" +- .format([err[0] for err in errors])) ++ try: ++ res = requests.post("{}/api/v1/keys/".format(opts.signatrust_host), headers=headers, json=data) ++ except Exception as e: ++ raise CoprKeygenRequestError( ++ msg="Failed to get userinfo", request="/api/v1/keys/", response=res) ++ ++ if res.status_code >= 400: ++ raise CoprKeygenRequestError( ++ msg="Failed to create user payload: {}".format(data), request="/api/v1/keys/", response=res) +diff --git a/tests/daemons/test_log.py b/backend/tests/daemons/test_log.py +index 73b1a777a..c68b7d918 100644 +--- a/tests/daemons/test_log.py ++++ b/tests/daemons/test_log.py +@@ -45,7 +45,8 @@ class TestLog(object): + self.log_file = os.path.join(self.log_dir, "copr.log") + self.opts = Munch( + verbose=False, +- log_dir=self.log_dir ++ log_dir=self.log_dir, ++ sign_backend = "obs-signd" + ) + print("\n log dir: {}".format(self.log_dir)) + self.queue = MagicMock() +diff --git a/tests/daemons/unused_test_job_grab.py b/backend/tests/daemons/unused_test_job_grab.py +index be3e64c56..79f6472b7 100644 +--- a/tests/daemons/unused_test_job_grab.py ++++ b/tests/daemons/unused_test_job_grab.py +@@ -87,6 +87,7 @@ class TestJobGrab(object): + redis_host="127.0.0.1", + redis_port=6379, + redis_db=0, ++ sign_backend = "obs-signd" + ) + + self.queue = MagicMock() +diff --git a/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py +index 6620f01b1..8c9ef2369 100644 +--- a/tests/run/test_copr_prune_results.py ++++ b/tests/run/test_copr_prune_results.py +@@ -59,7 +59,8 @@ class TestPruneResults(object): + self.opts = Munch( + prune_days=14, + frontend_base_url = '<frontend_url>', +- destdir=self.testresults_dir ++ destdir=self.testresults_dir, ++ sign_backend = "obs-signd" + ) + + def teardown_method(self, method): +diff --git a/tests/test_action.py b/backend/tests/test_action.py +index 7da3ab09c..4c935eb53 100644 +--- a/tests/test_action.py ++++ b/tests/test_action.py +@@ -57,6 +57,7 @@ class TestAction(object): + results_baseurl=RESULTS_ROOT_URL, + + do_sign=False, ++ sign_backend = "obs-signd", + + keygen_host="example.com" + ) +@@ -136,7 +137,7 @@ class TestAction(object): + @mock.patch("copr_backend.actions.os.makedirs") + @mock.patch("copr_backend.actions.copy_tree") + @mock.patch("copr_backend.actions.os.path.exists") +- @mock.patch("copr_backend.actions.unsign_rpms_in_dir") ++ @mock.patch("copr_backend.sign.ObsSign.unsign_rpms_in_dir") + @mock.patch("copr_backend.helpers.subprocess.Popen") + def test_action_handle_forks(self, mc_popen, mc_unsign_rpms_in_dir, + mc_exists, mc_copy_tree, _mc_os_makedirs, +diff --git a/tests/test_background_worker_build.py b/backend/tests/test_background_worker_build.py +index 1dfe19563..258564067 100644 +--- a/tests/test_background_worker_build.py ++++ b/tests/test_background_worker_build.py +@@ -77,6 +77,7 @@ def _reset_build_worker(): + # Don't waste time with mocking. We don't want to log anywhere, and we want + # to let BuildBackgroundWorker adjust the handlers. + worker.log.handlers = [] ++ worker.opts.sign_backend = "obs-signd" + return worker + + def _fake_host(): +diff --git a/tests/test_frontend.py b/backend/tests/test_frontend.py +index eeb8dc8f4..06a6e605b 100644 +--- a/tests/test_frontend.py ++++ b/tests/test_frontend.py +@@ -48,6 +48,7 @@ class TestFrontendClient(object): + self.opts = Munch( + frontend_base_url="http://example.com/", + frontend_auth="12345678", ++ sign_backend = "obs-signd" + ) + self.fc = FrontendClient(self.opts) + +diff --git a/tests/test_helpers.py b/backend/tests/test_helpers.py +index 8da70269d..e23565054 100644 +--- a/tests/test_helpers.py ++++ b/tests/test_helpers.py +@@ -31,6 +31,7 @@ class TestHelpers(object): + self.opts = Munch( + redis_db=9, + redis_port=7777, ++ sign_backend = "obs-signd" + ) + + self.rc = get_redis_connection(self.opts) +diff --git a/tests/test_sign.py b/backend/tests/test_sign.py +index bf2dd1b8c..ebb8f2be3 100644 +--- a/tests/test_sign.py ++++ b/tests/test_sign.py +@@ -10,10 +10,10 @@ import pytest + + from copr_backend.exceptions import CoprSignError, CoprSignNoKeyError, CoprKeygenRequestError + from copr_backend.sign import ( +- get_pubkey, _sign_one, sign_rpms_in_dir, create_user_keys, ++ new_signer, + gpg_hashtype_for_chroot, +- call_sign_bin, + ) ++from copr_backend.constants import DEF_SIGN_BACKEND + + STDOUT = "stdout" + STDERR = "stderr" +@@ -33,6 +33,8 @@ class TestSign(object): + self.opts = Munch(keygen_host="example.com") + self.opts.gently_gpg_sha256 = False + self.opts.sign_domain = "fedorahosted.org" ++ self.opts.sign_backend = DEF_SIGN_BACKEND ++ self.signer = new_signer(self.opts) + + def teardown_method(self, method): + if self.tmp_dir_path: +@@ -60,7 +62,7 @@ class TestSign(object): + mc_handle.returncode = 0 + mc_popen.return_value = mc_handle + +- result = get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) ++ result = self.signer.get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) + assert result == STDOUT + assert mc_popen.call_args[0][0] == ['/bin/sign', '-u', self.usermail, '-p'] + +@@ -70,7 +72,7 @@ class TestSign(object): + mc_popen.side_effect = IOError(STDERR) + + with pytest.raises(CoprSignError): +- get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) ++ self.signer.get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) + + + @mock.patch("copr_backend.sign.time.sleep") +@@ -82,7 +84,7 @@ class TestSign(object): + mc_popen.return_value = mc_handle + + with pytest.raises(CoprSignNoKeyError) as err: +- get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) ++ self.signer.get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) + + assert "There are no gpg keys for user foo in keyring" in str(err) + +@@ -95,7 +97,7 @@ class TestSign(object): + mc_popen.return_value = mc_handle + + with pytest.raises(CoprSignError) as err: +- get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) ++ self.signer.get_pubkey(self.username, self.projectname, MagicMock(), self.opts.sign_domain) + + assert "Failed to get user pubkey" in str(err) + +@@ -108,7 +110,7 @@ class TestSign(object): + + outfile_path = os.path.join(self.tmp_dir_path, "out.pub") + assert not os.path.exists(outfile_path) +- result = get_pubkey(self.username, self.projectname, MagicMock(), ++ result = self.signer.get_pubkey(self.username, self.projectname, MagicMock(), + self.opts.sign_domain, outfile_path) + assert result == STDOUT + assert os.path.exists(outfile_path) +@@ -124,7 +126,7 @@ class TestSign(object): + mc_popen.return_value = mc_handle + + fake_path = "/tmp/pkg.rpm" +- result = _sign_one(fake_path, self.usermail, "sha1", MagicMock()) ++ result = self.signer._sign_one(fake_path, self.usermail, "sha1", MagicMock()) + assert STDOUT, STDERR == result + + expected_cmd = ['/bin/sign', "-4", "-h", "sha1", "-u", self.usermail, +@@ -137,7 +139,7 @@ class TestSign(object): + + fake_path = "/tmp/pkg.rpm" + with pytest.raises(CoprSignError): +- _sign_one(fake_path, self.usermail, "sha256", MagicMock()) ++ self.signer._sign_one(fake_path, self.usermail, "sha256", MagicMock()) + + @mock.patch("copr_backend.sign.time.sleep") + @mock.patch("copr_backend.sign.Popen") +@@ -149,12 +151,11 @@ class TestSign(object): + + fake_path = "/tmp/pkg.rpm" + with pytest.raises(CoprSignError): +- _sign_one(fake_path, self.usermail, "sha256", MagicMock()) ++ self.signer._sign_one(fake_path, self.usermail, "sha256", MagicMock()) + +- @staticmethod + @mock.patch("copr_backend.sign.time.sleep") + @mock.patch("copr_backend.sign.Popen") +- def test_call_sign_bin_repeatedly(mc_popen, _sleep): ++ def test_call_sign_bin_repeatedly(self, mc_popen, _sleep): + """ + Test that we attempt to run /bin/sign multiple times if it returns + non-zero exit status +@@ -163,13 +164,13 @@ class TestSign(object): + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 1 + mc_popen.return_value = mc_handle +- call_sign_bin(cmd=[], log=MagicMock()) ++ self.signer.call_sign_bin(cmd=[], log=MagicMock()) + assert mc_popen.call_count == 3 + + @mock.patch("copr_backend.sign.SafeRequest.send") + def test_create_user_keys(self, mc_request): + mc_request.return_value.status_code = 200 +- create_user_keys(self.username, self.projectname, self.opts) ++ self.signer.create_user_keys(self.username, self.projectname, self.opts) + + assert mc_request.called + expected_call = mock.call( +@@ -183,7 +184,7 @@ class TestSign(object): + def test_create_user_keys_error_1(self, mc_request): + mc_request.side_effect = IOError() + with pytest.raises(CoprKeygenRequestError) as err: +- create_user_keys(self.username, self.projectname, self.opts) ++ self.signer.create_user_keys(self.username, self.projectname, self.opts) + + assert "Failed to create key-pair" in str(err) + +@@ -195,16 +196,16 @@ class TestSign(object): + mc_request.return_value.content = "error: {}".format(code) + + with pytest.raises(CoprKeygenRequestError) as err: +- create_user_keys(self.username, self.projectname, self.opts) ++ self.signer.create_user_keys(self.username, self.projectname, self.opts) + assert "Failed to create key-pair for user: foo, project:bar" in str(err) + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_nothing(self, mc_gp, mc_cuk, mc_so, + tmp_dir): + # empty target dir doesn't produce error +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "epel-8-x86_64", self.opts, + log=MagicMock()) + +@@ -212,13 +213,13 @@ class TestSign(object): + assert not mc_cuk.called + assert not mc_so.called + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_ok(self, mc_gp, mc_cuk, mc_so, + tmp_dir, tmp_files): + +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "fedora-rawhide-x86_64", + self.opts, log=MagicMock()) + +@@ -234,15 +235,15 @@ class TestSign(object): + assert os.path.join(self.tmp_dir_path, name) in pathes + assert len(pathes) == count + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_error_on_pubkey( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_gp.side_effect = CoprSignError("foobar") + with pytest.raises(CoprSignError): +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "epel-7-x86_64", self.opts, + log=MagicMock()) + +@@ -250,15 +251,15 @@ class TestSign(object): + assert not mc_cuk.called + assert not mc_so.called + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_no_pub_key( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_gp.side_effect = CoprSignNoKeyError("foobar") + +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "rhel-7-x86_64", self.opts, + log=MagicMock()) + +@@ -266,9 +267,9 @@ class TestSign(object): + assert mc_cuk.called + assert mc_so.called + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_sign_error_one( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + +@@ -276,7 +277,7 @@ class TestSign(object): + None, CoprSignError("foobar"), None + ] + with pytest.raises(CoprSignError): +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "fedora-36-x86_64", self.opts, + log=MagicMock()) + +@@ -285,15 +286,15 @@ class TestSign(object): + + assert mc_so.called + +- @mock.patch("copr_backend.sign._sign_one") +- @mock.patch("copr_backend.sign.create_user_keys") +- @mock.patch("copr_backend.sign.get_pubkey") ++ @mock.patch("copr_backend.sign.ObsSign._sign_one") ++ @mock.patch("copr_backend.sign.ObsSign.create_user_keys") ++ @mock.patch("copr_backend.sign.ObsSign.get_pubkey") + def test_sign_rpms_id_dir_sign_error_all( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_so.side_effect = CoprSignError("foobar") + with pytest.raises(CoprSignError): +- sign_rpms_in_dir(self.username, self.projectname, ++ self.signer.sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, "fedora-36-i386", self.opts, + log=MagicMock()) + |