1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
diff --git a/coprs_frontend/coprs/__init__.py b/coprs_frontend/coprs/__init__.py
index 5332f21e..652ce93a 100644
--- a/coprs_frontend/coprs/__init__.py
+++ b/coprs_frontend/coprs/__init__.py
@@ -94,6 +94,7 @@ cache_rcp = RedisConnectionProvider(config=app.config, db=1)
cache = Cache(app, config={
'CACHE_REDIS_HOST': cache_rcp.host,
'CACHE_REDIS_PORT': cache_rcp.port,
+ 'CACHE_REDIS_PASSWORD': cache_rcp.password,
})
app.cache = cache
diff --git a/coprs_frontend/coprs/logic/packages_logic.py b/coprs_frontend/coprs/logic/packages_logic.py
index 84f23d6e..be2b067c 100644
--- a/coprs_frontend/coprs/logic/packages_logic.py
+++ b/coprs_frontend/coprs/logic/packages_logic.py
@@ -1,7 +1,7 @@
import json
from typing import List, Optional
-from sqlalchemy import bindparam, Integer, func, or_
+from sqlalchemy import bindparam, Integer, func, or_, and_
from sqlalchemy.sql import true, text
from sqlalchemy.orm import selectinload
@@ -22,10 +22,18 @@ log = app.logger
class PackagesLogic(object):
@classmethod
- def count(cls):
+ def count(cls, success=True):
"""
Get packages count
"""
+ if success:
+ id = "succeed_package_count"
+ count = app.cache.get(id)
+ if not count:
+ count = cls.get_all_success_packages().count()
+ app.cache.set(id, count, 3600)
+ return count
+
return models.Package.query.count()
@classmethod
@@ -433,3 +441,116 @@ class PackagesLogic(object):
user.name,
package.name,
package.copr.full_name)
+
+ @classmethod
+ def get_all_success_packages(cls):
+ """
+ Get all succeed packages
+ """
+ copr_ids = [
+ copr.id
+ for copr in models.Copr.query.filter(models.Copr.deleted == False)
+ .with_entities(models.Copr.id)
+ .all()
+ ]
+
+ pkg_ids = [
+ pkg.id
+ for pkg in models.Package.query.filter(models.Package.copr_id.in_(copr_ids))
+ .with_entities(models.Package.id)
+ .all()
+ ]
+
+ pkg_ids = cls.get_packages_with_success_builds_ids(pkg_ids)
+
+ packages = models.Package.query.filter(
+ and_(models.Package.id.in_(pkg_ids), models.Package.copr_id.in_(copr_ids))
+ ).order_by(models.Package.name)
+
+ return packages
+
+ @classmethod
+ def get_packages_with_success_builds_ids(cls, pkg_ids):
+ """
+ Obtain the list of package ids with the latest build assigned.
+ Parameters:
+
+ :param packages: Don't query the list of Package objects from DB, but
+ use the given 'packages' array.
+ :return: array of Package ids, with assigned latest Build object
+ """
+ builds_ids = (
+ models.Build.query.join(models.CoprDir)
+ .filter(models.Build.package_id.in_(pkg_ids))
+ .with_entities(func.max(models.Build.id))
+ .group_by(models.Build.package_id)
+ )
+
+ builds = (
+ models.Build.query.filter(models.Build.id.in_(builds_ids))
+ .options(selectinload("build_chroots"))
+ .yield_per(1000)
+ .all()
+ )
+
+ results = []
+ for build in builds:
+ if build.status == StatusEnum("succeeded"):
+ results.append(build.package_id)
+
+ return results
+
+ @classmethod
+ def get_packages_with_succ_builds(cls, small_build=True, packages=None):
+ """
+ Obtain the list of package objects with the
+ latest build assigned.
+ Parameters:
+
+ :param small_build: Don't assign full Build objects, but only a limited
+ objects with necessary info.
+ :param packages: Don't query the list of Package objects from DB, but
+ use the given 'packages' array.
+ :return: array of Package objects, with assigned latest Build object
+ """
+ if packages is None:
+ return
+
+ pkg_ids = [package.id for package in packages]
+ builds_ids = (
+ models.Build.query.join(models.CoprDir)
+ .filter(models.Build.package_id.in_(pkg_ids))
+ .with_entities(func.max(models.Build.id))
+ .group_by(models.Build.package_id)
+ )
+
+ # map package.id => package object in packages array
+ packages_map = {package.id: package for package in packages}
+
+ builds = (
+ models.Build.query.filter(models.Build.id.in_(builds_ids))
+ .options(selectinload("build_chroots"))
+ .yield_per(1000)
+ )
+
+ for build in builds:
+
+ class SmallBuild:
+ pass
+
+ if not build.package_id:
+ continue
+
+ if small_build:
+ small_build_object = SmallBuild()
+ for param in ["state", "status", "pkg_version", "submitted_on"]:
+ # we don't want to keep all the attributes here in memory, and
+ # also we don't need any further info about assigned
+ # build_chroot(s). So we only pick the info we need, and throw
+ # the expensive objects away.
+ setattr(small_build_object, param, getattr(build, param))
+ packages_map[build.package_id].latest_build = small_build_object
+ else:
+ packages_map[build.package_id].latest_build = build
+
+ return packages
diff --git a/coprs_frontend/coprs/views/coprs_ns/coprs_packages.py b/coprs_frontend/coprs/views/coprs_ns/coprs_packages.py
index 527c73d1..363c532f 100644
--- a/coprs_frontend/coprs/views/coprs_ns/coprs_packages.py
+++ b/coprs_frontend/coprs/views/coprs_ns/coprs_packages.py
@@ -23,6 +23,7 @@ from coprs.views.misc import (
from coprs.logic.complex_logic import ComplexLogic
from coprs.logic.packages_logic import PackagesLogic
from coprs.logic.users_logic import UsersLogic
+from coprs.logic.coprs_logic import CoprsLogic
from coprs.exceptions import (ActionInProgressException, ObjectNotFound, NoPackageSourceException,
InsufficientRightsException, MalformedArgumentException)
@@ -322,3 +323,32 @@ def copr_delete_package(copr, package_id):
flask.flash("Package has been deleted successfully.")
return flask.redirect(helpers.copr_url("coprs_ns.copr_packages", copr))
+
+
+@coprs_ns.route("/packages_statistics")
+@req_with_pagination
+def packages_statistics(page=1):
+ flashes = flask.session.pop('_flashes', [])
+ query_packages = PackagesLogic.get_all_success_packages()
+
+ count = query_packages.count()
+
+ pagination = None
+ if query_packages.count() > 1000:
+ pagination = query_packages.paginate(page=page, per_page=50)
+ packages = pagination.items
+ else:
+ packages = query_packages.all()
+
+ packages = PackagesLogic.get_packages_with_succ_builds(packages=packages)
+
+ response = flask.Response(
+ stream_with_context(helpers.stream_template(
+ "coprs/show/package_statistics.html",
+ packages=packages,
+ flashes=flashes,
+ serverside_pagination=pagination,
+ count=count,
+ )))
+ flask.session.pop('_flashes', [])
+ return response
\ No newline at end of file
|
