From 9f08e6190e1c98ee2b83bfed8f6bdd09ca605d21 Mon Sep 17 00:00:00 2001 From: grumo35 Date: Mon, 31 Jan 2022 11:24:04 +0100 Subject: [PATCH 001/120] Update __init__.py Added basic proxy support from environment variables, need to add this option in parameter of patchman cli. --- util/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/__init__.py b/util/__init__.py index ddbe63eb..bc669c1e 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from os import getenv import sys import requests @@ -33,6 +34,11 @@ from progressbar import Bar, ETA, Percentage, ProgressBar from patchman.signals import error_message +http_proxy=getenv('http_proxy') +proxies = { + 'http': http_proxy, + 'https': http_proxy, +} if ProgressBar.__dict__.get('maxval'): pbar2 = False @@ -130,7 +136,7 @@ def get_url(url): """ res = None try: - res = requests.get(url, stream=True) + res = requests.get(url,proxies=proxies, stream=True) except requests.exceptions.Timeout: error_message.send(sender=None, text='Timeout - {0!s}'.format(url)) except requests.exceptions.TooManyRedirects: From cd577bf3fdfb77f23b44f1ca0283009dd01041ef Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:48:33 -0400 Subject: [PATCH 002/120] Update __init__.py --- util/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index f27e232a..457e6782 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -29,24 +29,23 @@ from time import time from tqdm import tqdm -from patchman.signals import error_message, info_message, debug_message - -http_proxy = getenv('http_proxy') -http_proxy = getenv('https_proxy') -proxies = { - 'http': http_proxy, - 'https': https_proxy, -} - from django.utils.timezone import make_aware from django.utils.dateparse import parse_datetime from django.conf import settings +from patchman.signals import error_message, info_message, debug_message pbar = None verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') +http_proxy = getenv('http_proxy') +https_proxy = getenv('https_proxy') +proxies = { + 'http': http_proxy, + 'https': https_proxy, +} + def get_verbosity(): """ Get the global verbosity level From 80a2d60e60d9b8d947d33f5b45966c59d0158a7a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:49:38 -0400 Subject: [PATCH 003/120] Update __init__.py --- util/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index 457e6782..10ee21f7 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,13 +14,13 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from os import getenv import requests import bz2 import magic import zlib import lzma +import os from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -39,8 +39,8 @@ verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') -http_proxy = getenv('http_proxy') -https_proxy = getenv('https_proxy') +http_proxy = os.getenv('http_proxy') +https_proxy = os.getenv('https_proxy') proxies = { 'http': http_proxy, 'https': https_proxy, @@ -122,7 +122,7 @@ def get_url(url, headers={}, params={}): response = None try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') - response = requests.get(url, headers=headers, params=params, stream=True, proxies = proxies, timeout=30) + response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response From f07ffcc69b806414483b740afce68d9097e5fa11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 21 Oct 2024 11:12:16 +0100 Subject: [PATCH 004/120] Added Dockerfile and docker-compose.yaml --- docker/Dockerfile | 12 +++++++++ docker/docker-compose.yaml | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/docker-compose.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..09c8ef01 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:bookworm-slim + +RUN apt update \ + && apt install -y curl mariadb-client python3-mysqldb python3-pymemcache + +RUN curl -sS https://repo.openbytes.ie/openbytes.gpg > /usr/share/keyrings/openbytes.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian bookworm main" > /etc/apt/sources.list.d/patchman.list + +RUN apt update \ + && apt install -y python3-patchman + +EXPOSE 80 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..b4af7fd0 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,52 @@ +--- +networks: + patchman: + name: patchman + driver: bridge + ipam: + driver: default + config: + - subnet: 10.0.0.0/24 + gateway: 10.0.0.1 + +services: + patchman: + container_name: patchman + image: patchman:latest + build: + context: ./patchman + restart: unless-stopped + entrypoint: ["/usr/sbin/apache2ctl", "-DFOREGROUND"] + networks: + patchman: + ipv4_address: '10.0.0.10' + volumes: + - /opt/docker/patchman/conf/local_settings.py:/etc/patchman/local_settings.py:ro + - /opt/docker/patchman/conf/patchman.nos.mgt.conf:/etc/apache2/sites-enabled/000-default.conf:ro + depends_on: + - patchman-db + - memcached + + patchman-db: + container_name: patchman-db + image: mysql:latest + restart: unless-stopped + command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"] + networks: + patchman: + ipv4_address: '10.0.0.11' + environment: + MYSQL_ROOT_PASSWORD: changeme + MYSQL_DATABASE: changeme + MYSQL_USER: changeme + MYSQL_PASSWORD: changeme + volumes: + - /opt/docker/patchman/data:/var/lib/mysql:rw + + memcached: + container_name: memcached + image: memcached:latest + restart: unless-stopped + networks: + patchman: + ipv4_address: '10.0.0.12' From 3b242b903ce37b0937158593e63202350b1cfba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:38:08 +0000 Subject: [PATCH 005/120] Added necessary packages to allow realtime processing with Celery --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 09c8ef01..ff707d3d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim RUN apt update \ - && apt install -y curl mariadb-client python3-mysqldb python3-pymemcache + && apt install -y curl mariadb-client python3-mysqldb python3-pymemcache python3-celery python3-redis python-celery-common RUN curl -sS https://repo.openbytes.ie/openbytes.gpg > /usr/share/keyrings/openbytes.gpg RUN echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian bookworm main" > /etc/apt/sources.list.d/patchman.list From fc9bdc6fd81f5777ee54bf2ff66cb6bc9654d92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:41:11 +0000 Subject: [PATCH 006/120] Added entrypoint script to start Apache httpd and Celery worker simultaneously --- docker/Dockerfile | 3 +++ docker/docker-entrypoint.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 docker/docker-entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index ff707d3d..daf8a662 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,3 +10,6 @@ RUN apt update \ && apt install -y python3-patchman EXPOSE 80 + +COPY ./docker-entrypoint.sh docker-entrypoint.sh +ENTRYPOINT ./docker-entrypoint.sh diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 00000000..b29a4193 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# Starts Apache httpd process +/usr/sbin/apache2ctl -DFOREGROUND & + +# Starts Celery for for realtime processing of reports from clients +C_FORCE_ROOT=1 celery -b redis://redis:6379/0 -A patchman worker -l INFO -E + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? From 2bf1974589ce38da5ddccb314a235e6b57281ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:42:57 +0000 Subject: [PATCH 007/120] Added patchman-email installation --- docker/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index daf8a662..0af580cf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,6 +9,10 @@ RUN echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbyt RUN apt update \ && apt install -y python3-patchman +RUN curl -sS https://raw.githubusercontent.com/RicardoJeronimo/patchman/master/email/patchman-email > /usr/bin/patchman-email \ + && curl -sS https://raw.githubusercontent.com/RicardoJeronimo/patchman/master/etc/patchman/patchman-email.conf > /etc/patchman/patchman-email.conf +RUN chmod +x /usr/bin/patchman-email + EXPOSE 80 COPY ./docker-entrypoint.sh docker-entrypoint.sh From 66e2616313f2e6e73f8ddc9202ad082c3adeeaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:46:55 +0000 Subject: [PATCH 008/120] Added patchman-email dependencies --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0af580cf..78de78f9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim RUN apt update \ - && apt install -y curl mariadb-client python3-mysqldb python3-pymemcache python3-celery python3-redis python-celery-common + && apt install -y curl mariadb-client python-celery-common python3-celery python3-mysqldb python3-pymemcache python3-redis sendmail sharutils uuid-runtime weasyprint RUN curl -sS https://repo.openbytes.ie/openbytes.gpg > /usr/share/keyrings/openbytes.gpg RUN echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian bookworm main" > /etc/apt/sources.list.d/patchman.list From 613950ea342e941dd7ca416c6d73e463577138aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:51:03 +0000 Subject: [PATCH 009/120] Added Redis container for Celery realtime processing --- docker/docker-compose.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b4af7fd0..d427159e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -20,12 +20,10 @@ services: networks: patchman: ipv4_address: '10.0.0.10' - volumes: - - /opt/docker/patchman/conf/local_settings.py:/etc/patchman/local_settings.py:ro - - /opt/docker/patchman/conf/patchman.nos.mgt.conf:/etc/apache2/sites-enabled/000-default.conf:ro depends_on: - patchman-db - memcached + - redis patchman-db: container_name: patchman-db @@ -40,8 +38,6 @@ services: MYSQL_DATABASE: changeme MYSQL_USER: changeme MYSQL_PASSWORD: changeme - volumes: - - /opt/docker/patchman/data:/var/lib/mysql:rw memcached: container_name: memcached @@ -50,3 +46,11 @@ services: networks: patchman: ipv4_address: '10.0.0.12' + + redis: + container_name: redis + image: redis:latest + restart: unless-stopped + networks: + patchman: + ipv4_address: '10.0.0.13' From 4ba672fce590859fd650d8579efde8a1ace03057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 28 Nov 2024 12:56:02 +0000 Subject: [PATCH 010/120] Adjusted build context --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d427159e..325af2a0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -14,7 +14,7 @@ services: container_name: patchman image: patchman:latest build: - context: ./patchman + context: . restart: unless-stopped entrypoint: ["/usr/sbin/apache2ctl", "-DFOREGROUND"] networks: From 565c5ee475f3192ecfbf64b68a4b6524498eb836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 11:19:40 +0000 Subject: [PATCH 011/120] Changed Dockerfile to install from source instead of repository --- docker/Dockerfile | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 78de78f9..c3b9b7c3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,19 +1,25 @@ FROM debian:bookworm-slim -RUN apt update \ - && apt install -y curl mariadb-client python-celery-common python3-celery python3-mysqldb python3-pymemcache python3-redis sendmail sharutils uuid-runtime weasyprint +RUN apt update && apt upgrade +RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-colorama python3-debian python3-defusedxml python3-django python3-django-extensions python3-django-tagging python3-djangorestframework python3-humanize python3-lxml python3-magic python3-mysqldb python3-pip python3-progressbar python3-pymemcache python3-redis python3-requests python3-rpm sendmail sharutils uuid-runtime weasyprint -RUN curl -sS https://repo.openbytes.ie/openbytes.gpg > /usr/share/keyrings/openbytes.gpg -RUN echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian bookworm main" > /etc/apt/sources.list.d/patchman.list +RUN pip3 install django-bootstrap3 -RUN apt update \ - && apt install -y python3-patchman +COPY ./patchman/* /srv/ +COPY ./sbin/* /usr/bin/ +COPY ./email/patchman-email /usr/bin/ +COPY ./patchman/etc/local_settings.py /etc/patchman/ +COPY ./email/patchman-email.conf /etc/patchman/ +COPY ./patchman/etc/apache.conf.example /etc/apache2/conf-available/patchman.conf -RUN curl -sS https://raw.githubusercontent.com/RicardoJeronimo/patchman/master/email/patchman-email > /usr/bin/patchman-email \ - && curl -sS https://raw.githubusercontent.com/RicardoJeronimo/patchman/master/etc/patchman/patchman-email.conf > /etc/patchman/patchman-email.conf -RUN chmod +x /usr/bin/patchman-email +RUN a2enmod wsgi +RUN a2enconf patchman +RUN chown -R :www-data /etc/patchman +RUN chmod -R g+r /etc/patchman +RUN chown -R :www-data /var/lib/patchman +RUN chmod -R g+w /var/lib/patchman/db EXPOSE 80 -COPY ./docker-entrypoint.sh docker-entrypoint.sh +COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh ENTRYPOINT ./docker-entrypoint.sh From 0e56a1f7c2069aa7c194f5f5b9143b01b81d4afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 11:59:46 +0000 Subject: [PATCH 012/120] Fixed django-bootstrap3 installation and some file origins --- docker/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c3b9b7c3..dd5cd666 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,19 +3,20 @@ FROM debian:bookworm-slim RUN apt update && apt upgrade RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-colorama python3-debian python3-defusedxml python3-django python3-django-extensions python3-django-tagging python3-djangorestframework python3-humanize python3-lxml python3-magic python3-mysqldb python3-pip python3-progressbar python3-pymemcache python3-redis python3-requests python3-rpm sendmail sharutils uuid-runtime weasyprint -RUN pip3 install django-bootstrap3 +RUN pip3 install django-bootstrap3 --break-system-packages COPY ./patchman/* /srv/ COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ -COPY ./patchman/etc/local_settings.py /etc/patchman/ -COPY ./email/patchman-email.conf /etc/patchman/ -COPY ./patchman/etc/apache.conf.example /etc/apache2/conf-available/patchman.conf +COPY ./etc/patchman/local_settings.py /etc/patchman/ +COPY ./etc/patchman/patchman-email.conf /etc/patchman/ +COPY ./etc/patchman/apache.conf.example /etc/apache2/conf-available/patchman.conf RUN a2enmod wsgi RUN a2enconf patchman RUN chown -R :www-data /etc/patchman RUN chmod -R g+r /etc/patchman +RUN mkdir -p /var/lib/patchman/db RUN chown -R :www-data /var/lib/patchman RUN chmod -R g+w /var/lib/patchman/db From 529b9b6b727e19f709e8bd7750a62776d6619b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 12:18:31 +0000 Subject: [PATCH 013/120] Entrypoint only starts Celery worker if an environment varible exists --- docker/docker-entrypoint.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index b29a4193..47ddf74f 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -4,7 +4,17 @@ /usr/sbin/apache2ctl -DFOREGROUND & # Starts Celery for for realtime processing of reports from clients -C_FORCE_ROOT=1 celery -b redis://redis:6379/0 -A patchman worker -l INFO -E +if [ ! -z "${CELERY_BROKER}" ]; then + broker="${CELERY_BROKER}" + + if [ ! -z "${CELERY_BROKER_PORT}" ]; then + brokerPort="${CELERY_BROKER_PORT}" + else + brokerPort=6379 + fi + + C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E +fi # Wait for any process to exit wait -n From 5efeccba8e6e270acb37d9914f0a9757bfaea68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 13:27:31 +0000 Subject: [PATCH 014/120] Changed entrypoint interpreter to Bash --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 47ddf74f..f7b5fed4 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Starts Apache httpd process /usr/sbin/apache2ctl -DFOREGROUND & From 4c0e74ed80c363978f8dc0fe1bc194356f690190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 15:02:54 +0000 Subject: [PATCH 015/120] Fixed httpd virtualhost location --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index dd5cd666..14d0f5c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,10 +10,10 @@ COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ COPY ./etc/patchman/patchman-email.conf /etc/patchman/ -COPY ./etc/patchman/apache.conf.example /etc/apache2/conf-available/patchman.conf +COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.conf RUN a2enmod wsgi -RUN a2enconf patchman +RUN a2ensite patchman RUN chown -R :www-data /etc/patchman RUN chmod -R g+r /etc/patchman RUN mkdir -p /var/lib/patchman/db From 53b237f890bbb967062a23e84dae3bfe16772287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 15:26:43 +0000 Subject: [PATCH 016/120] Added httpd service reload --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 14d0f5c4..3f725b12 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,12 +14,15 @@ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.co RUN a2enmod wsgi RUN a2ensite patchman + RUN chown -R :www-data /etc/patchman RUN chmod -R g+r /etc/patchman RUN mkdir -p /var/lib/patchman/db RUN chown -R :www-data /var/lib/patchman RUN chmod -R g+w /var/lib/patchman/db +RUN service apache2 reload + EXPOSE 80 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh From 05112559c5eacc62f97b8fb64982a05abd313b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 15:44:10 +0000 Subject: [PATCH 017/120] Reverted change made in 787f257 --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f725b12..eda57ff1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,8 +21,6 @@ RUN mkdir -p /var/lib/patchman/db RUN chown -R :www-data /var/lib/patchman RUN chmod -R g+w /var/lib/patchman/db -RUN service apache2 reload - EXPOSE 80 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh From d04fb1503e9b495ec2e23e392bd516abb695d950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 15:47:38 +0000 Subject: [PATCH 018/120] Created missing directory and optimized instructions --- docker/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eda57ff1..e6c57126 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,10 @@ RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery- RUN pip3 install django-bootstrap3 --break-system-packages -COPY ./patchman/* /srv/ +RUN mkdir -p /srv/patchman +RUN mkdir -p /var/lib/patchman/db + +COPY ./patchman/* /srv/patchman COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ @@ -15,11 +18,8 @@ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.co RUN a2enmod wsgi RUN a2ensite patchman -RUN chown -R :www-data /etc/patchman -RUN chmod -R g+r /etc/patchman -RUN mkdir -p /var/lib/patchman/db -RUN chown -R :www-data /var/lib/patchman -RUN chmod -R g+w /var/lib/patchman/db +RUN chown -R :www-data /etc/patchman && chmod -R g+r /etc/patchman +RUN chown -R :www-data /var/lib/patchman && chmod -R g+w /var/lib/patchman/db EXPOSE 80 From 119468a59c26e26bcaf478b1e0d7ee486c3a0b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 16:17:48 +0000 Subject: [PATCH 019/120] Fixed COPY instruction and made Python scripts executable --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e6c57126..81201306 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ RUN pip3 install django-bootstrap3 --break-system-packages RUN mkdir -p /srv/patchman RUN mkdir -p /var/lib/patchman/db -COPY ./patchman/* /srv/patchman +COPY ./patchman /srv/patchman COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ @@ -20,6 +20,7 @@ RUN a2ensite patchman RUN chown -R :www-data /etc/patchman && chmod -R g+r /etc/patchman RUN chown -R :www-data /var/lib/patchman && chmod -R g+w /var/lib/patchman/db +RUN chmod +x /srv/patchman/patchman/*.py EXPOSE 80 From edba5417b8aceea7f97a325b398a7ddebf827c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 16:23:49 +0000 Subject: [PATCH 020/120] Fixed COPY instruction --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 81201306..999e673f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,10 +5,10 @@ RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery- RUN pip3 install django-bootstrap3 --break-system-packages -RUN mkdir -p /srv/patchman +RUN mkdir -p /srv/patchman/patchman RUN mkdir -p /var/lib/patchman/db -COPY ./patchman /srv/patchman +COPY ./patchman /srv/patchman/patchman COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ From 40d67b8377d0a606eb9acbac93008952e7eee203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 16:57:24 +0000 Subject: [PATCH 021/120] Fixed COPY instruction for the full project tree to be copied into /srv --- docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 999e673f..8d03284e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,10 +5,9 @@ RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery- RUN pip3 install django-bootstrap3 --break-system-packages -RUN mkdir -p /srv/patchman/patchman -RUN mkdir -p /var/lib/patchman/db +RUN mkdir -p /srv/patchman && mkdir -p /var/lib/patchman/db -COPY ./patchman /srv/patchman/patchman +COPY . /srv/patchman/ COPY ./sbin/* /usr/bin/ COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ From 7dc4b597488eaf17a7c1839e93a2c45028d7776e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 17:31:46 +0000 Subject: [PATCH 022/120] Changed ENTRYPOINT to meet JSONArgsRecommended check --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d03284e..eaa73e7e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,4 +24,4 @@ RUN chmod +x /srv/patchman/patchman/*.py EXPOSE 80 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh -ENTRYPOINT ./docker-entrypoint.sh +ENTRYPOINT ["docker-entrypoint.sh"] From df805bed1c41999c338a2c23b194b6b8d71230da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 17:50:25 +0000 Subject: [PATCH 023/120] Exposed port 443 --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index eaa73e7e..e1f4ff86 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,7 @@ RUN chown -R :www-data /var/lib/patchman && chmod -R g+w /var/lib/patchman/db RUN chmod +x /srv/patchman/patchman/*.py EXPOSE 80 +EXPOSE 443 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] From 981ec28db70c6c2652dcb307391b24370c8a525a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 18:12:03 +0000 Subject: [PATCH 024/120] Reverted changes made in 3178c02 --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e1f4ff86..eaa73e7e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,6 @@ RUN chown -R :www-data /var/lib/patchman && chmod -R g+w /var/lib/patchman/db RUN chmod +x /srv/patchman/patchman/*.py EXPOSE 80 -EXPOSE 443 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] From 4d7b5375212baeb624370e20f80c8009dbe4a7e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 21 Jan 2025 18:28:27 +0000 Subject: [PATCH 025/120] Fixed entrypoint --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eaa73e7e..64e86e41 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,4 +24,4 @@ RUN chmod +x /srv/patchman/patchman/*.py EXPOSE 80 COPY ./docker/docker-entrypoint.sh docker-entrypoint.sh -ENTRYPOINT ["docker-entrypoint.sh"] +ENTRYPOINT ["./docker-entrypoint.sh"] From 64caeeecbb0cbb36450865182f2a43181a4626ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 10:44:11 +0000 Subject: [PATCH 026/120] Entrypoint script configures SECRET_KEY if not set --- docker/docker-entrypoint.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index f7b5fed4..2c2bec86 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,5 +1,14 @@ #!/bin/bash +# Configure SECRET_KEY if not set +if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then + if [ ! -z "${SECRET_KEY}" ]; then + sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" /etc/patchman/local_settings.py + else + patchman-set-secret-key + fi +fi + # Starts Apache httpd process /usr/sbin/apache2ctl -DFOREGROUND & From d9d1eb1183c1e13315eb18df2f4c3cfa0dca104b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 11:05:56 +0000 Subject: [PATCH 027/120] Added to do items --- docker/docker-entrypoint.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2c2bec86..056c2743 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,5 +1,14 @@ #!/bin/bash +# Configure ADMINS +# To do + +# Configure DATABASES +# To do + +# Configure TIME_ZONE +# To do + # Configure SECRET_KEY if not set if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then if [ ! -z "${SECRET_KEY}" ]; then @@ -9,6 +18,12 @@ if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | fi fi +# Configure CACHES +# To do + +# Configure ASYNC_PROCESSING +# To do + # Starts Apache httpd process /usr/sbin/apache2ctl -DFOREGROUND & From 5695f754a31af50c69c34372a6e6792883b161ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 11:30:47 +0000 Subject: [PATCH 028/120] Entrypoint now configures Memcached --- docker/docker-entrypoint.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 056c2743..aa223628 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -19,7 +19,19 @@ if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | fi # Configure CACHES -# To do +if [ ! -z "${MEMCACHED_ADDR}" ]; then + memcachedAddr="${MEMCACHED_ADDR}" + + if [ ! -z "${MEMCACHED_PORT}" ]; then + memcachedPort="${MEMCACHED_PORT}" + else + memcachedPort="11211" + fi + + sed -i "s/'LOCATION': '127.0.0.1:11211'/'LOCATION': '"$memcachedAddr":"$memcachedPort"'/g" /etc/patchman/local_settings.py +else + sed -i '41,49 {s/^/#/}' /etc/patchman/local_settings.py +fi # Configure ASYNC_PROCESSING # To do From 930d17dc2b27f237ba95550434d868538dbf761d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 11:57:17 +0000 Subject: [PATCH 029/120] Entrypoint now configures ADMINS and TIME_ZONE --- docker/docker-entrypoint.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index aa223628..1888040d 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,13 +1,21 @@ #!/bin/bash # Configure ADMINS -# To do +if [ ! -z "${ADMIN_NAME}" ]; then + sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' /etc/patchman/local_settings.py +fi + +if [ ! -z "${ADMIN_EMAIL}" ]; then + sed -i '6 {s/you@example.com/'"${ADMIN_EMAIL}"'/}' /etc/patchman/local_settings.py +fi # Configure DATABASES # To do # Configure TIME_ZONE -# To do +if [ ! -z "${TIMEZONE}" ]; then + sed -i '18 {s/America\/New_York/'"${TIMEZONE}"'/}' /etc/patchman/local_settings.py +fi # Configure SECRET_KEY if not set if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then From c70e52ebef09b16e42af5bf6c8e5d2c97e3d9c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 11:59:35 +0000 Subject: [PATCH 030/120] Entrypoint now starts httpd after celery broker --- docker/docker-entrypoint.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 1888040d..066b57c8 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -44,9 +44,6 @@ fi # Configure ASYNC_PROCESSING # To do -# Starts Apache httpd process -/usr/sbin/apache2ctl -DFOREGROUND & - # Starts Celery for for realtime processing of reports from clients if [ ! -z "${CELERY_BROKER}" ]; then broker="${CELERY_BROKER}" @@ -57,9 +54,12 @@ if [ ! -z "${CELERY_BROKER}" ]; then brokerPort=6379 fi - C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E + C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E & fi +# Starts Apache httpd process +/usr/sbin/apache2ctl -DFOREGROUND + # Wait for any process to exit wait -n From bbee6ae63283a4d16a3e1dafc23bba8b3b560c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 12:34:51 +0000 Subject: [PATCH 031/120] Fixed TIMEZONE sed where a slash needs to be escaped --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 066b57c8..612f4fe9 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -14,7 +14,7 @@ fi # Configure TIME_ZONE if [ ! -z "${TIMEZONE}" ]; then - sed -i '18 {s/America\/New_York/'"${TIMEZONE}"'/}' /etc/patchman/local_settings.py + sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' /etc/patchman/local_settings.py fi # Configure SECRET_KEY if not set From a11f447a6f4ebaeac65466d1de2047a313f67171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 13:58:48 +0000 Subject: [PATCH 032/120] Entrypoint now configures USE_ASYNC_PROCESSING --- docker/docker-entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 612f4fe9..db5baec4 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -41,9 +41,6 @@ else sed -i '41,49 {s/^/#/}' /etc/patchman/local_settings.py fi -# Configure ASYNC_PROCESSING -# To do - # Starts Celery for for realtime processing of reports from clients if [ ! -z "${CELERY_BROKER}" ]; then broker="${CELERY_BROKER}" @@ -54,6 +51,9 @@ if [ ! -z "${CELERY_BROKER}" ]; then brokerPort=6379 fi + echo "USE_ASYNC_PROCESSING = True" >> /etc/patchman/local_settings.py + echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0" >> /etc/patchman/local_settings.py + C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E & fi From a6e10a987dda45e82aa3274bee84c9c2208c89d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 14:33:25 +0000 Subject: [PATCH 033/120] Minor bugfix --- docker/docker-entrypoint.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index db5baec4..f7341bb8 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -51,8 +51,14 @@ if [ ! -z "${CELERY_BROKER}" ]; then brokerPort=6379 fi - echo "USE_ASYNC_PROCESSING = True" >> /etc/patchman/local_settings.py - echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0" >> /etc/patchman/local_settings.py + if [ -z $(grep "USE_ASYNC_PROCESSING" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then + echo "" >> /etc/patchman/local_settings.py + echo "USE_ASYNC_PROCESSING = True" >> /etc/patchman/local_settings.py + fi + + if [ -z $(grep "CELERY_BROKER_URL" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then + echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0'" >> /etc/patchman/local_settings.py + fi C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E & fi From 6472e166dd554804a17533f4563b582b6e51d160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 15:44:15 +0000 Subject: [PATCH 034/120] Entrypoint now configures DATABASES --- docker/docker-entrypoint.sh | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index f7341bb8..3eec0e48 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -10,14 +10,46 @@ if [ ! -z "${ADMIN_EMAIL}" ]; then fi # Configure DATABASES -# To do +if [ ! -z "${DB_ENGINE}"]; then + sed -i '9,14 {s/^/#/}' /etc/patchman/local_settings.py + + if [ "${DB_ENGINE}" == "MySQL"]; then + cat <<-EOF >> /etc/patchman/local_settings.py + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '"${DB_DATABASE}"', + 'USER': '"${DB_USER}"', + 'PASSWORD': '"${DB_PASSWORD}"', + 'HOST': '"${DB_HOST}"', + 'PORT': '"${DB_PORT}"', + 'STORAGE_ENGINE': 'INNODB', + 'CHARSET' : 'utf8' + } + } + EOF + elif [ "${DB_ENGINE}" == "PostgreSQL"]; then + cat <<-EOF >> /etc/patchman/local_settings.py + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2' + 'NAME': '"${DB_DATABASE}"', + 'USER': '"${DB_USER}"', + 'PASSWORD': '"${DB_PASSWORD}"', + 'HOST': '"${DB_HOST}"', + 'PORT': '"${DB_PORT}"', + 'CHARSET' : 'utf8' + } + EOF + fi +fi # Configure TIME_ZONE if [ ! -z "${TIMEZONE}" ]; then sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' /etc/patchman/local_settings.py fi -# Configure SECRET_KEY if not set +# Configure SECRET_KEY if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then if [ ! -z "${SECRET_KEY}" ]; then sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" /etc/patchman/local_settings.py From 8cb178ba05ba7c2e3091f632bacede6b28de9d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 15:52:12 +0000 Subject: [PATCH 035/120] Minor bugfix --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 3eec0e48..17a5dfdb 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -40,7 +40,7 @@ if [ ! -z "${DB_ENGINE}"]; then 'PORT': '"${DB_PORT}"', 'CHARSET' : 'utf8' } - EOF + EOF fi fi From 6d460f6379f373644c4bd351789367304bac6d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 16:05:56 +0000 Subject: [PATCH 036/120] Minor bugfixes --- docker/docker-entrypoint.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 17a5dfdb..3d4c83f5 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -10,34 +10,36 @@ if [ ! -z "${ADMIN_EMAIL}" ]; then fi # Configure DATABASES -if [ ! -z "${DB_ENGINE}"]; then +if [ ! -z "${DB_ENGINE}" ]; then sed -i '9,14 {s/^/#/}' /etc/patchman/local_settings.py - if [ "${DB_ENGINE}" == "MySQL"]; then + if [ "${DB_ENGINE}" == "MySQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'NAME': '"${DB_DATABASE}"', - 'USER': '"${DB_USER}"', - 'PASSWORD': '"${DB_PASSWORD}"', - 'HOST': '"${DB_HOST}"', - 'PORT': '"${DB_PORT}"', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', 'STORAGE_ENGINE': 'INNODB', 'CHARSET' : 'utf8' } } EOF - elif [ "${DB_ENGINE}" == "PostgreSQL"]; then + elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2' - 'NAME': '"${DB_DATABASE}"', - 'USER': '"${DB_USER}"', - 'PASSWORD': '"${DB_PASSWORD}"', - 'HOST': '"${DB_HOST}"', - 'PORT': '"${DB_PORT}"', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', 'CHARSET' : 'utf8' } EOF From 5e60b6b0134eec421c9e1feee0a0fd17fda66237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 16:20:33 +0000 Subject: [PATCH 037/120] Minor bugfixes --- docker/docker-entrypoint.sh | 43 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 3d4c83f5..a5d6f477 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,33 +15,34 @@ if [ ! -z "${DB_ENGINE}" ]; then if [ "${DB_ENGINE}" == "MySQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py - + DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'STORAGE_ENGINE': 'INNODB', - 'CHARSET' : 'utf8' - } + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'STORAGE_ENGINE': 'INNODB', + 'CHARSET' : 'utf8' + } } EOF elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py - + DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2' - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'CHARSET' : 'utf8' - } + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2' + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'CHARSET' : 'utf8' + } + } EOF fi fi From 4266367cce1785b11eb01a23a6b05d1163f28ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 16:31:21 +0000 Subject: [PATCH 038/120] Minor indentation fixes --- docker/docker-entrypoint.sh | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index a5d6f477..21056d5e 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -16,33 +16,33 @@ if [ ! -z "${DB_ENGINE}" ]; then if [ "${DB_ENGINE}" == "MySQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'STORAGE_ENGINE': 'INNODB', - 'CHARSET' : 'utf8' - } - } + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'STORAGE_ENGINE': 'INNODB', + 'CHARSET' : 'utf8' + } + } EOF elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2' - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'CHARSET' : 'utf8' - } - } + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2' + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'CHARSET' : 'utf8' + } + } EOF fi fi From 6ff4a4b445ad06bf5bf3b26a8fa6887dc0233de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 16:38:12 +0000 Subject: [PATCH 039/120] Minor indentation fixes --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 21056d5e..4bca848e 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -35,7 +35,7 @@ if [ ! -z "${DB_ENGINE}" ]; then DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2' - 'NAME': '${DB_DATABASE}', + 'NAME': '${DB_DATABASE}', 'USER': '${DB_USER}', 'PASSWORD': '${DB_PASSWORD}', 'HOST': '${DB_HOST}', From 929772ab5771c391aae64a385830593a88a3f47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 16:56:32 +0000 Subject: [PATCH 040/120] Sed commands that comment lines don't do it again on container restarts --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4bca848e..16b5425d 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -11,7 +11,7 @@ fi # Configure DATABASES if [ ! -z "${DB_ENGINE}" ]; then - sed -i '9,14 {s/^/#/}' /etc/patchman/local_settings.py + sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py if [ "${DB_ENGINE}" == "MySQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py @@ -73,7 +73,7 @@ if [ ! -z "${MEMCACHED_ADDR}" ]; then sed -i "s/'LOCATION': '127.0.0.1:11211'/'LOCATION': '"$memcachedAddr":"$memcachedPort"'/g" /etc/patchman/local_settings.py else - sed -i '41,49 {s/^/#/}' /etc/patchman/local_settings.py + sed -i '41,49 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py fi # Starts Celery for for realtime processing of reports from clients From c7f2d09a3417a081cebf1a7193834aaf0bdda9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 17:49:59 +0000 Subject: [PATCH 041/120] Entrypoint now checks if a MySQL or PostgreSQL database is already configured --- docker/docker-entrypoint.sh | 65 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 16b5425d..16e79435 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,37 +13,40 @@ fi if [ ! -z "${DB_ENGINE}" ]; then sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py - if [ "${DB_ENGINE}" == "MySQL" ]; then - cat <<-EOF >> /etc/patchman/local_settings.py - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'STORAGE_ENGINE': 'INNODB', - 'CHARSET' : 'utf8' - } - } - EOF - elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then - cat <<-EOF >> /etc/patchman/local_settings.py - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2' - 'NAME': '${DB_DATABASE}', - 'USER': '${DB_USER}', - 'PASSWORD': '${DB_PASSWORD}', - 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', - 'CHARSET' : 'utf8' - } - } - EOF + if [ $(grep "ENGINE" /etc/patchman/local_settings.py | wc -l) < 2 ]; then + if [ "${DB_ENGINE}" == "MySQL" ]; then + cat <<-EOF >> /etc/patchman/local_settings.py + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'STORAGE_ENGINE': 'INNODB', + 'CHARSET' : 'utf8' + } + } + EOF + + elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then + cat <<-EOF >> /etc/patchman/local_settings.py + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': '${DB_DATABASE}', + 'USER': '${DB_USER}', + 'PASSWORD': '${DB_PASSWORD}', + 'HOST': '${DB_HOST}', + 'PORT': '${DB_PORT}', + 'CHARSET' : 'utf8' + } + } + EOF + fi fi fi From ac22d4d65376f2421c3c1f4a25130511706d78d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 18:05:18 +0000 Subject: [PATCH 042/120] Minor bugfix --- docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 16e79435..4ddfc0af 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,7 +13,7 @@ fi if [ ! -z "${DB_ENGINE}" ]; then sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py - if [ $(grep "ENGINE" /etc/patchman/local_settings.py | wc -l) < 2 ]; then + if [[ $(grep "ENGINE" /etc/patchman/local_settings.py | wc -l) < 2 ]]; then if [ "${DB_ENGINE}" == "MySQL" ]; then cat <<-EOF >> /etc/patchman/local_settings.py From fbd377c36464fe484704cc7764c4a00433724e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 30 Jan 2025 18:28:45 +0000 Subject: [PATCH 043/120] Binary files are now symlinked instead of copied --- docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 64e86e41..4be32964 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,12 +8,15 @@ RUN pip3 install django-bootstrap3 --break-system-packages RUN mkdir -p /srv/patchman && mkdir -p /var/lib/patchman/db COPY . /srv/patchman/ -COPY ./sbin/* /usr/bin/ -COPY ./email/patchman-email /usr/bin/ COPY ./etc/patchman/local_settings.py /etc/patchman/ COPY ./etc/patchman/patchman-email.conf /etc/patchman/ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.conf +RUN ln -s /srv/patchman/manage.py /usr/bin/patchman-manage +RUN ln -s /srv/patchman/sbin/patchman /usr/bin/patchman +RUN ln -s /srv/patchman/sbin/patchman-set-secret-key /usr/bin/patchman-set-secret-key +RUN ln -s /srv/patchman/email/patchman-email /usr/bin/patchman-email + RUN a2enmod wsgi RUN a2ensite patchman From 64d74915fa9f614066cb4a22e62a44baff311e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 3 Apr 2025 16:56:30 +0100 Subject: [PATCH 044/120] Added build step via setup.py --- docker/Dockerfile | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4be32964..272c6406 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,28 +1,23 @@ FROM debian:bookworm-slim -RUN apt update && apt upgrade -RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-colorama python3-debian python3-defusedxml python3-django python3-django-extensions python3-django-tagging python3-djangorestframework python3-humanize python3-lxml python3-magic python3-mysqldb python3-pip python3-progressbar python3-pymemcache python3-redis python3-requests python3-rpm sendmail sharutils uuid-runtime weasyprint +RUN apt -y update && apt -y upgrade +RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-redis python3-rpm sendmail sharutils uuid-runtime weasyprint -RUN pip3 install django-bootstrap3 --break-system-packages - -RUN mkdir -p /srv/patchman && mkdir -p /var/lib/patchman/db +WORKDIR /srv/patchman COPY . /srv/patchman/ -COPY ./etc/patchman/local_settings.py /etc/patchman/ -COPY ./etc/patchman/patchman-email.conf /etc/patchman/ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.conf -RUN ln -s /srv/patchman/manage.py /usr/bin/patchman-manage -RUN ln -s /srv/patchman/sbin/patchman /usr/bin/patchman -RUN ln -s /srv/patchman/sbin/patchman-set-secret-key /usr/bin/patchman-set-secret-key -RUN ln -s /srv/patchman/email/patchman-email /usr/bin/patchman-email +RUN /srv/patchman/setup.py install + +COPY ./etc/patchman/patchman-email.conf /etc/patchman/ +COPY /srv/patchman/email/patchman-email /usr/bin/patchman-email RUN a2enmod wsgi RUN a2ensite patchman -RUN chown -R :www-data /etc/patchman && chmod -R g+r /etc/patchman -RUN chown -R :www-data /var/lib/patchman && chmod -R g+w /var/lib/patchman/db -RUN chmod +x /srv/patchman/patchman/*.py +RUN mkdir -p /var/lib/patchman/db +RUN chown :www-data /var/lib/patchman/db && chmod 2770 /var/lib/patchman/db EXPOSE 80 From d3e3120ae496d49e32819374b5661896d83ec975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 3 Apr 2025 17:01:30 +0100 Subject: [PATCH 045/120] Entrypoint syncs database on container first start --- docker/docker-entrypoint.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4ddfc0af..29856066 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,6 +79,20 @@ else sed -i '41,49 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py fi +# Sync database on container first start +if [ ! -f /var/lib/patchman/.firstrun ]; then + patchman-manage makemigrations + patchman-manage migrate --run-syncdb --fake-initial + patchman-manage collectstatic + + # If SQLite is being used, allow httpd to write + if [ -z "${DB_ENGINE}" ]; then + chmod 660 /var/lib/patchman/db/patchman.db + fi + + touch /var/lib/patchman/.firstrun +fi + # Starts Celery for for realtime processing of reports from clients if [ ! -z "${CELERY_BROKER}" ]; then broker="${CELERY_BROKER}" From 098a68365351bff991e853ebb36a61e1b640b050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 3 Apr 2025 17:25:00 +0100 Subject: [PATCH 046/120] Added PostgreSQL dependency --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 272c6406..030efa07 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim RUN apt -y update && apt -y upgrade -RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-redis python3-rpm sendmail sharutils uuid-runtime weasyprint +RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-psycopg2 python3-redis python3-rpm sendmail sharutils uuid-runtime weasyprint WORKDIR /srv/patchman From e7acfab1655df7990613c9a60a5711a0e3636032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 3 Apr 2025 19:18:08 +0100 Subject: [PATCH 047/120] Fixed COPY instruction --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 030efa07..b11e8372 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,7 @@ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.co RUN /srv/patchman/setup.py install COPY ./etc/patchman/patchman-email.conf /etc/patchman/ -COPY /srv/patchman/email/patchman-email /usr/bin/patchman-email +COPY ./email/patchman-email /usr/bin/patchman-email RUN a2enmod wsgi RUN a2ensite patchman From c5bba1a66f5c67892937bccf8987530d83a74bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 12:20:17 +0100 Subject: [PATCH 048/120] Added GitHub Action to build container --- .github/workflows/docker-build.yaml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/docker-build.yaml diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml new file mode 100644 index 00000000..5c8a3e89 --- /dev/null +++ b/.github/workflows/docker-build.yaml @@ -0,0 +1,33 @@ +name: Docker Image CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "docker" ] + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{vars.DOCKERHUB_USERNAME}}/{{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:v3.0.19 + cache-from: type=gha + cache-to: type=gha,mode=max From 744cacee81ede8342029e59ae86bb5649bcd44f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 14:41:13 +0100 Subject: [PATCH 049/120] Latest tag is now used for container versioning during builds --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 5c8a3e89..9b2e5a11 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -28,6 +28,6 @@ jobs: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: ${{vars.DOCKERHUB_USERNAME}}/{{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:v3.0.19 + tags: ${{vars.DOCKERHUB_USERNAME}}/{{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:${{github.ref_name}} cache-from: type=gha cache-to: type=gha,mode=max From a6fa2e67cc2ede113ab478f779d3cd38df0e24e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 15:06:13 +0100 Subject: [PATCH 050/120] Removed build steps for branch integrity --- docker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b11e8372..f67fb67d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,9 +10,6 @@ COPY ./etc/patchman/apache.conf.example /etc/apache2/sites-available/patchman.co RUN /srv/patchman/setup.py install -COPY ./etc/patchman/patchman-email.conf /etc/patchman/ -COPY ./email/patchman-email /usr/bin/patchman-email - RUN a2enmod wsgi RUN a2ensite patchman From 9acf35497f6bb4616fe58c363a5b91a4587ae289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 17:36:24 +0100 Subject: [PATCH 051/120] Removed build dependencies for branch integrity --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f67fb67d..1a04543c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim RUN apt -y update && apt -y upgrade -RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-psycopg2 python3-redis python3-rpm sendmail sharutils uuid-runtime weasyprint +RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-psycopg2 python3-redis python3-rpm WORKDIR /srv/patchman From d0abc9d76c38da20196a499b26242a43597efabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 17:40:44 +0100 Subject: [PATCH 052/120] Fixed minor bug and removed build on pull requests --- .github/workflows/docker-build.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 9b2e5a11..bc6dd659 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -3,8 +3,6 @@ name: Docker Image CI on: push: branches: [ "main" ] - pull_request: - branches: [ "docker" ] jobs: docker: @@ -28,6 +26,6 @@ jobs: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: ${{vars.DOCKERHUB_USERNAME}}/{{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:${{github.ref_name}} + tags: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:${{github.ref_name}} cache-from: type=gha cache-to: type=gha,mode=max From d164c82300ed56d2df30bddd05bc61c9b1b1fc6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 17:43:46 +0100 Subject: [PATCH 053/120] Minor bugfix --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index bc6dd659..c6a8fb87 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -26,6 +26,6 @@ jobs: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars_DOCKERHUB_CONTAINER}}:${{github.ref_name}} + tags: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:${{github.ref_name}} cache-from: type=gha cache-to: type=gha,mode=max From 838dcf7be0b79c2b56d771054972c7a396080a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 18:33:38 +0100 Subject: [PATCH 054/120] Improved Docker Compose example --- docker/docker-compose.yaml | 45 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 325af2a0..80e3ba6a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,25 +1,23 @@ --- -networks: - patchman: - name: patchman - driver: bridge - ipam: - driver: default - config: - - subnet: 10.0.0.0/24 - gateway: 10.0.0.1 - services: patchman: container_name: patchman - image: patchman:latest - build: - context: . + image: furlongm/patchman restart: unless-stopped - entrypoint: ["/usr/sbin/apache2ctl", "-DFOREGROUND"] - networks: - patchman: - ipv4_address: '10.0.0.10' + environment: + TIMEZONE: America/New_York + ADMIN_NAME: admin_name + ADMIN_EMAIL: admin_mail@domain.tld + DB_ENGINE: MySQL + DB_DATABASE: patchman + DB_USER: user + DB_PASSWORD: changeme + DB_HOST: patchman-db + DB_PORT: 3306 + MEMCACHED_ADDR: memcached + MEMCACHED_PORT: 11211 + CELERY_BROKER: redis + CELERY_BROKER_PORT: 6379 depends_on: - patchman-db - memcached @@ -30,27 +28,18 @@ services: image: mysql:latest restart: unless-stopped command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"] - networks: - patchman: - ipv4_address: '10.0.0.11' environment: MYSQL_ROOT_PASSWORD: changeme - MYSQL_DATABASE: changeme - MYSQL_USER: changeme + MYSQL_DATABASE: patchman + MYSQL_USER: user MYSQL_PASSWORD: changeme memcached: container_name: memcached image: memcached:latest restart: unless-stopped - networks: - patchman: - ipv4_address: '10.0.0.12' redis: container_name: redis image: redis:latest restart: unless-stopped - networks: - patchman: - ipv4_address: '10.0.0.13' From 459fa28f60770e40f0bbac60db30873702e11c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 18:39:07 +0100 Subject: [PATCH 055/120] Environment variable DB_PORT is now optional --- docker/docker-entrypoint.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 29856066..18b6deaa 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,6 +15,12 @@ if [ ! -z "${DB_ENGINE}" ]; then if [[ $(grep "ENGINE" /etc/patchman/local_settings.py | wc -l) < 2 ]]; then if [ "${DB_ENGINE}" == "MySQL" ]; then + if [ ! -z "${DB_PORT}" ]; then + dbPort="${DB_PORT}" + else + dbPort="3306" + fi + cat <<-EOF >> /etc/patchman/local_settings.py DATABASES = { @@ -24,7 +30,7 @@ if [ ! -z "${DB_ENGINE}" ]; then 'USER': '${DB_USER}', 'PASSWORD': '${DB_PASSWORD}', 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', + 'PORT': '"$dbPort"', 'STORAGE_ENGINE': 'INNODB', 'CHARSET' : 'utf8' } @@ -32,6 +38,12 @@ if [ ! -z "${DB_ENGINE}" ]; then EOF elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then + if [ ! -z "${DB_PORT}" ]; then + dbPort="${DB_PORT}" + else + dbPort="5432" + fi + cat <<-EOF >> /etc/patchman/local_settings.py DATABASES = { From 32454bb08e6c2019a1228ed95bf8fe08dc06a550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 18:52:39 +0100 Subject: [PATCH 056/120] Minor improvements --- docker/docker-entrypoint.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 18b6deaa..a64d33d5 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,21 +1,21 @@ #!/bin/bash # Configure ADMINS -if [ ! -z "${ADMIN_NAME}" ]; then +if [ -n "${ADMIN_NAME}" ]; then sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' /etc/patchman/local_settings.py fi -if [ ! -z "${ADMIN_EMAIL}" ]; then +if [ -n "${ADMIN_EMAIL}" ]; then sed -i '6 {s/you@example.com/'"${ADMIN_EMAIL}"'/}' /etc/patchman/local_settings.py fi # Configure DATABASES -if [ ! -z "${DB_ENGINE}" ]; then +if [ -n "${DB_ENGINE}" ]; then sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py - if [[ $(grep "ENGINE" /etc/patchman/local_settings.py | wc -l) < 2 ]]; then + if [[ $(grep -c "ENGINE" /etc/patchman/local_settings.py) -lt 2 ]]; then if [ "${DB_ENGINE}" == "MySQL" ]; then - if [ ! -z "${DB_PORT}" ]; then + if [ -n "${DB_PORT}" ]; then dbPort="${DB_PORT}" else dbPort="3306" @@ -38,7 +38,7 @@ if [ ! -z "${DB_ENGINE}" ]; then EOF elif [ "${DB_ENGINE}" == "PostgreSQL" ]; then - if [ ! -z "${DB_PORT}" ]; then + if [ -n "${DB_PORT}" ]; then dbPort="${DB_PORT}" else dbPort="5432" @@ -63,13 +63,13 @@ if [ ! -z "${DB_ENGINE}" ]; then fi # Configure TIME_ZONE -if [ ! -z "${TIMEZONE}" ]; then +if [ -n "${TIMEZONE}" ]; then sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' /etc/patchman/local_settings.py fi # Configure SECRET_KEY -if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then - if [ ! -z "${SECRET_KEY}" ]; then +if [ -z "$(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then + if [ -n "${SECRET_KEY}" ]; then sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" /etc/patchman/local_settings.py else patchman-set-secret-key @@ -77,10 +77,10 @@ if [ -z $(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | fi # Configure CACHES -if [ ! -z "${MEMCACHED_ADDR}" ]; then +if [ -n "${MEMCACHED_ADDR}" ]; then memcachedAddr="${MEMCACHED_ADDR}" - if [ ! -z "${MEMCACHED_PORT}" ]; then + if [ -n "${MEMCACHED_PORT}" ]; then memcachedPort="${MEMCACHED_PORT}" else memcachedPort="11211" @@ -106,21 +106,21 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ ! -z "${CELERY_BROKER}" ]; then +if [ -n "${CELERY_BROKER}" ]; then broker="${CELERY_BROKER}" - if [ ! -z "${CELERY_BROKER_PORT}" ]; then + if [ -n "${CELERY_BROKER_PORT}" ]; then brokerPort="${CELERY_BROKER_PORT}" else brokerPort=6379 fi - if [ -z $(grep "USE_ASYNC_PROCESSING" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then + if [ -z "$(grep "USE_ASYNC_PROCESSING" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then echo "" >> /etc/patchman/local_settings.py echo "USE_ASYNC_PROCESSING = True" >> /etc/patchman/local_settings.py fi - if [ -z $(grep "CELERY_BROKER_URL" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'") ]; then + if [ -z "$(grep "CELERY_BROKER_URL" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0'" >> /etc/patchman/local_settings.py fi From 06e81ec849c29681870ef51671d73638f75cf8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 18:55:04 +0100 Subject: [PATCH 057/120] Moved local_settings.py path to a variable --- docker/docker-entrypoint.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index a64d33d5..48666097 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,19 +1,21 @@ #!/bin/bash +conf="/etc/patchman/local_settings.py" + # Configure ADMINS if [ -n "${ADMIN_NAME}" ]; then - sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' /etc/patchman/local_settings.py + sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' "$conf" fi if [ -n "${ADMIN_EMAIL}" ]; then - sed -i '6 {s/you@example.com/'"${ADMIN_EMAIL}"'/}' /etc/patchman/local_settings.py + sed -i '6 {s/you@example.com/'"${ADMIN_EMAIL}"'/}' "$conf" fi # Configure DATABASES if [ -n "${DB_ENGINE}" ]; then - sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py + sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' "$conf" - if [[ $(grep -c "ENGINE" /etc/patchman/local_settings.py) -lt 2 ]]; then + if [[ $(grep -c "ENGINE" "$conf") -lt 2 ]]; then if [ "${DB_ENGINE}" == "MySQL" ]; then if [ -n "${DB_PORT}" ]; then dbPort="${DB_PORT}" @@ -21,7 +23,7 @@ if [ -n "${DB_ENGINE}" ]; then dbPort="3306" fi - cat <<-EOF >> /etc/patchman/local_settings.py + cat <<-EOF >> "$conf" DATABASES = { 'default': { @@ -44,7 +46,7 @@ if [ -n "${DB_ENGINE}" ]; then dbPort="5432" fi - cat <<-EOF >> /etc/patchman/local_settings.py + cat <<-EOF >> "$conf" DATABASES = { 'default': { @@ -64,13 +66,13 @@ fi # Configure TIME_ZONE if [ -n "${TIMEZONE}" ]; then - sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' /etc/patchman/local_settings.py + sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' "$conf" fi # Configure SECRET_KEY -if [ -z "$(grep "SECRET_KEY" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then +if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then if [ -n "${SECRET_KEY}" ]; then - sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" /etc/patchman/local_settings.py + sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" "$conf" else patchman-set-secret-key fi @@ -86,9 +88,9 @@ if [ -n "${MEMCACHED_ADDR}" ]; then memcachedPort="11211" fi - sed -i "s/'LOCATION': '127.0.0.1:11211'/'LOCATION': '"$memcachedAddr":"$memcachedPort"'/g" /etc/patchman/local_settings.py + sed -i "s/'LOCATION': '127.0.0.1:11211'/'LOCATION': '"$memcachedAddr":"$memcachedPort"'/g" "$conf" else - sed -i '41,49 {/^#/ ! s/\(.*\)/#\1/}' /etc/patchman/local_settings.py + sed -i '41,49 {/^#/ ! s/\(.*\)/#\1/}' "$conf" fi # Sync database on container first start @@ -115,13 +117,13 @@ if [ -n "${CELERY_BROKER}" ]; then brokerPort=6379 fi - if [ -z "$(grep "USE_ASYNC_PROCESSING" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then - echo "" >> /etc/patchman/local_settings.py - echo "USE_ASYNC_PROCESSING = True" >> /etc/patchman/local_settings.py + if [ -z "$(grep "USE_ASYNC_PROCESSING" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then + echo "" >> "$conf" + echo "USE_ASYNC_PROCESSING = True" >> "$conf" fi - if [ -z "$(grep "CELERY_BROKER_URL" /etc/patchman/local_settings.py | cut -d " " -f 3 | tr -d "'")" ]; then - echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0'" >> /etc/patchman/local_settings.py + if [ -z "$(grep "CELERY_BROKER_URL" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then + echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0'" >> "$conf" fi C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E & From ebcc531c6e482bac3985e461e92595168e9bbcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Fri, 4 Apr 2025 18:56:51 +0100 Subject: [PATCH 058/120] Improved Docker Compose example --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 80e3ba6a..da001a54 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -2,7 +2,7 @@ services: patchman: container_name: patchman - image: furlongm/patchman + image: furlongm/patchman:latest restart: unless-stopped environment: TIMEZONE: America/New_York From c23385fff545419fa5b59f5887c9fe7ac43845cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 11:14:20 +0100 Subject: [PATCH 059/120] Adjusted entrypoint script to reflect changes made to local_settings.py in e55bfce --- docker/docker-entrypoint.sh | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 48666097..d839f9d7 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -66,7 +66,7 @@ fi # Configure TIME_ZONE if [ -n "${TIMEZONE}" ]; then - sed -i '18 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' "$conf" + sed -i '22 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' "$conf" fi # Configure SECRET_KEY @@ -79,18 +79,22 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if [ -n "${MEMCACHED_ADDR}" ]; then - memcachedAddr="${MEMCACHED_ADDR}" +if ["${USE_CACHE}" == True ] && [ -n "${REDIS_ADDR}" ]; then + redisAddr="${REDIS_ADDR}" - if [ -n "${MEMCACHED_PORT}" ]; then - memcachedPort="${MEMCACHED_PORT}" + if [ -n "${REDIS_PORT}" ]; then + redisPort="${REDIS_PORT}" else - memcachedPort="11211" + redisPort="6379" fi - sed -i "s/'LOCATION': '127.0.0.1:11211'/'LOCATION': '"$memcachedAddr":"$memcachedPort"'/g" "$conf" -else - sed -i '41,49 {/^#/ ! s/\(.*\)/#\1/}' "$conf" + # Comment DummyCache Block + sed -i '44,48 {/^#/ ! s/\(.*\)/#\1/}' "$conf" + + # Uncomment RedisCache Block + sed -i '52,58 {s/^# //}' "$conf" + + sed -i "55 {s/127.0.0.1:6379/$redisAddr:$redisPort/}" "$conf" fi # Sync database on container first start @@ -108,13 +112,13 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ -n "${CELERY_BROKER}" ]; then - broker="${CELERY_BROKER}" +if ["${USE_CELERY}" == True ] && [ -n "${REDIS_ADDR}" ]; then + redisAddr="${REDIS_ADDR}" - if [ -n "${CELERY_BROKER_PORT}" ]; then - brokerPort="${CELERY_BROKER_PORT}" + if [ -n "${REDIS_PORT}" ]; then + redisPort="${REDIS_PORT}" else - brokerPort=6379 + redisPort=6379 fi if [ -z "$(grep "USE_ASYNC_PROCESSING" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then @@ -123,10 +127,10 @@ if [ -n "${CELERY_BROKER}" ]; then fi if [ -z "$(grep "CELERY_BROKER_URL" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then - echo "CELERY_BROKER_URL = 'redis://"$broker":"$brokerPort"/0'" >> "$conf" + echo "CELERY_BROKER_URL = 'redis://$redisAddr:$redisPort/0'" >> "$conf" fi - C_FORCE_ROOT=1 celery -b redis://"$broker":"$brokerPort"/0 -A patchman worker -l INFO -E & + C_FORCE_ROOT=1 celery -b redis://"$redisAddr":"$redisPort"/0 -A patchman worker -l INFO -E & fi # Starts Apache httpd process From 3040e6b6ae4ef13df286eea521f16e58db990763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 11:25:29 +0100 Subject: [PATCH 060/120] Minor bugfix --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index d839f9d7..86a80987 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,7 +79,7 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if ["${USE_CACHE}" == True ] && [ -n "${REDIS_ADDR}" ]; then +if [ "${USE_CACHE}" == True ] && [ -n "${REDIS_ADDR}" ]; then redisAddr="${REDIS_ADDR}" if [ -n "${REDIS_PORT}" ]; then @@ -112,7 +112,7 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if ["${USE_CELERY}" == True ] && [ -n "${REDIS_ADDR}" ]; then +if [ "${USE_CELERY}" == True ] && [ -n "${REDIS_ADDR}" ]; then redisAddr="${REDIS_ADDR}" if [ -n "${REDIS_PORT}" ]; then From b2a6013689a6d29b5a23c9162e5787a52c0f25f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 11:40:44 +0100 Subject: [PATCH 061/120] Added git dependency to Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a04543c..34a603a2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim RUN apt -y update && apt -y upgrade -RUN apt install -y apache2 libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-psycopg2 python3-redis python3-rpm +RUN apt install -y apache2 git libapache2-mod-wsgi-py3 mariadb-client python-celery-common python3-celery python3-debian python3-defusedxml python3-lxml python3-mysqldb python3-pip python3-progressbar python3-psycopg2 python3-redis python3-rpm WORKDIR /srv/patchman From caf0b440e5e35c2ab015b046767d14b15f5899bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 12:17:35 +0100 Subject: [PATCH 062/120] Minor bugfix --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 86a80987..cf7cff82 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,7 +79,7 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if [ "${USE_CACHE}" == True ] && [ -n "${REDIS_ADDR}" ]; then +if [ "${USE_CACHE}" ] && [ -n "${REDIS_ADDR}" ]; then redisAddr="${REDIS_ADDR}" if [ -n "${REDIS_PORT}" ]; then @@ -112,7 +112,7 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ "${USE_CELERY}" == True ] && [ -n "${REDIS_ADDR}" ]; then +if [ "${USE_CELERY}" ] && [ -n "${REDIS_ADDR}" ]; then redisAddr="${REDIS_ADDR}" if [ -n "${REDIS_PORT}" ]; then From 8b1d99776e587176524921029bbb6d87abc5d16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 12:29:31 +0100 Subject: [PATCH 063/120] Added CACHE_TIMEOUT environment variable to entrypoint script --- docker/docker-entrypoint.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index cf7cff82..75148471 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -95,6 +95,10 @@ if [ "${USE_CACHE}" ] && [ -n "${REDIS_ADDR}" ]; then sed -i '52,58 {s/^# //}' "$conf" sed -i "55 {s/127.0.0.1:6379/$redisAddr:$redisPort/}" "$conf" + + if [ -n "${CACHE_TIMEOUT}" ]; then + sed -i "56 {s/30/${CACHE_TIMEOUT}/}" "$conf" + fi fi # Sync database on container first start From 66cf8dc6ab7d32df96f55b84528edd4aa0adeba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 12:34:43 +0100 Subject: [PATCH 064/120] Improved Docker Compose example --- docker/docker-compose.yaml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index da001a54..77f80400 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -5,22 +5,24 @@ services: image: furlongm/patchman:latest restart: unless-stopped environment: - TIMEZONE: America/New_York ADMIN_NAME: admin_name ADMIN_EMAIL: admin_mail@domain.tld + TIMEZONE: America/New_York DB_ENGINE: MySQL + DB_HOST: patchman-db + DB_PORT: 3306 DB_DATABASE: patchman DB_USER: user DB_PASSWORD: changeme - DB_HOST: patchman-db - DB_PORT: 3306 - MEMCACHED_ADDR: memcached - MEMCACHED_PORT: 11211 - CELERY_BROKER: redis - CELERY_BROKER_PORT: 6379 + REDIS_HOST: redis + REDIS_PORT: 6379 + USE_CELERY: True + USE_CACHE: True + CACHE_TIMEOUT: 20 + ports: + - 80:80/tcp depends_on: - patchman-db - - memcached - redis patchman-db: @@ -34,11 +36,6 @@ services: MYSQL_USER: user MYSQL_PASSWORD: changeme - memcached: - container_name: memcached - image: memcached:latest - restart: unless-stopped - redis: container_name: redis image: redis:latest From f06c6ec4f1a1da472fff98ccd239ab087a0dcca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 12:35:55 +0100 Subject: [PATCH 065/120] Changed Redis environment variable name --- docker/docker-entrypoint.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 75148471..95e01d73 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,8 +79,8 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if [ "${USE_CACHE}" ] && [ -n "${REDIS_ADDR}" ]; then - redisAddr="${REDIS_ADDR}" +if [ "${USE_CACHE}" ] && [ -n "${REDIS_HOST}" ]; then + redisHost="${REDIS_HOST}" if [ -n "${REDIS_PORT}" ]; then redisPort="${REDIS_PORT}" @@ -94,7 +94,7 @@ if [ "${USE_CACHE}" ] && [ -n "${REDIS_ADDR}" ]; then # Uncomment RedisCache Block sed -i '52,58 {s/^# //}' "$conf" - sed -i "55 {s/127.0.0.1:6379/$redisAddr:$redisPort/}" "$conf" + sed -i "55 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" if [ -n "${CACHE_TIMEOUT}" ]; then sed -i "56 {s/30/${CACHE_TIMEOUT}/}" "$conf" @@ -116,8 +116,8 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ "${USE_CELERY}" ] && [ -n "${REDIS_ADDR}" ]; then - redisAddr="${REDIS_ADDR}" +if [ "${USE_CELERY}" ] && [ -n "${REDIS_HOST}" ]; then + redisHost="${REDIS_HOST}" if [ -n "${REDIS_PORT}" ]; then redisPort="${REDIS_PORT}" @@ -131,10 +131,10 @@ if [ "${USE_CELERY}" ] && [ -n "${REDIS_ADDR}" ]; then fi if [ -z "$(grep "CELERY_BROKER_URL" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then - echo "CELERY_BROKER_URL = 'redis://$redisAddr:$redisPort/0'" >> "$conf" + echo "CELERY_BROKER_URL = 'redis://$redisHost:$redisPort/0'" >> "$conf" fi - C_FORCE_ROOT=1 celery -b redis://"$redisAddr":"$redisPort"/0 -A patchman worker -l INFO -E & + C_FORCE_ROOT=1 celery -b redis://"$redisHost":"$redisPort"/0 -A patchman worker -l INFO -E & fi # Starts Apache httpd process From ea045e87f89f25b530ea2602cf8dadcc956444f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 13:01:05 +0100 Subject: [PATCH 066/120] Adjusted entrypoint script to reflect changes made to local_settings.py in c910a2c --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 95e01d73..4a570d3d 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -13,9 +13,9 @@ fi # Configure DATABASES if [ -n "${DB_ENGINE}" ]; then - sed -i '9,14 {/^#/ ! s/\(.*\)/#\1/}' "$conf" + sed -i '9,18 {/^#/ ! s/\(.*\)/#\1/}' "$conf" - if [[ $(grep -c "ENGINE" "$conf") -lt 2 ]]; then + if [[ $(grep -v "#" "$conf" | grep -c "ENGINE") -lt 2 ]]; then if [ "${DB_ENGINE}" == "MySQL" ]; then if [ -n "${DB_PORT}" ]; then dbPort="${DB_PORT}" From 8ed49c6fd908162d55206a38826a19f6d081f246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Mon, 7 Apr 2025 13:15:00 +0100 Subject: [PATCH 067/120] Minor bugfix --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4a570d3d..10dce42d 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -32,7 +32,7 @@ if [ -n "${DB_ENGINE}" ]; then 'USER': '${DB_USER}', 'PASSWORD': '${DB_PASSWORD}', 'HOST': '${DB_HOST}', - 'PORT': '"$dbPort"', + 'PORT': '$dbPort', 'STORAGE_ENGINE': 'INNODB', 'CHARSET' : 'utf8' } @@ -55,7 +55,7 @@ if [ -n "${DB_ENGINE}" ]; then 'USER': '${DB_USER}', 'PASSWORD': '${DB_PASSWORD}', 'HOST': '${DB_HOST}', - 'PORT': '${DB_PORT}', + 'PORT': '$dbPort', 'CHARSET' : 'utf8' } } From d44a2ce5d9ff30abb6caf3037307706cdbc4c5d5 Mon Sep 17 00:00:00 2001 From: "furlongm@gmail.com" Date: Wed, 23 Apr 2025 17:35:03 -0400 Subject: [PATCH 068/120] handle duplicate CVSSes better --- security/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/security/models.py b/security/models.py index a847ab02..b60acb47 100644 --- a/security/models.py +++ b/security/models.py @@ -125,19 +125,20 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] - existing = self.cvss_scores.filter(version=version, vector_string=vector_string) - if existing: - cvss = existing.first() - else: + try: cvss, created = CVSS.objects.get_or_create( version=version, vector_string=vector_string, score=score, severity=severity, ) - cvss.score = score - cvss.severity = severity - cvss.save() + except CVSS.MultipleObjectsReturned: + matching_cvsses = CVSS.objects.filter( + version=version, + vector_string=vector_string, + ) + cvss = matching_cvsses.first() + matching_cvsses.exclude(id=cvss.id).delete() self.cvss_scores.add(cvss) def fetch_cve_data(self, fetch_nist_data=False, sleep_secs=6): From 57284bd5cfacf8abaa9a553d5bab1dbb24c8d944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 24 Apr 2025 11:46:40 +0100 Subject: [PATCH 069/120] Adjusted entrypoint script to reflect changes made to local_settings.py in c877223 --- docker/docker-entrypoint.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 10dce42d..f81f9088 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -89,15 +89,15 @@ if [ "${USE_CACHE}" ] && [ -n "${REDIS_HOST}" ]; then fi # Comment DummyCache Block - sed -i '44,48 {/^#/ ! s/\(.*\)/#\1/}' "$conf" + sed -i '47,51 {/^#/ ! s/\(.*\)/#\1/}' "$conf" # Uncomment RedisCache Block - sed -i '52,58 {s/^# //}' "$conf" + sed -i '55,61 {s/^# //}' "$conf" - sed -i "55 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" + sed -i "58 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" if [ -n "${CACHE_TIMEOUT}" ]; then - sed -i "56 {s/30/${CACHE_TIMEOUT}/}" "$conf" + sed -i "59 {s/30/${CACHE_TIMEOUT}/}" "$conf" fi fi From bf626c90dd697dfd7bc41f542ae55c88af296e2a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 8 Apr 2025 15:49:31 -0400 Subject: [PATCH 070/120] reduce max charfield length for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index 5655f8b0..c22d1727 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=65535, null=True)), + ('description', models.CharField(blank=True, max_length=21844, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=65535)), + ('description', models.CharField(max_length=21844)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 382cd29ad3aeba4481fa4d14e38ce75bcff37874 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 00:05:34 -0400 Subject: [PATCH 071/120] further reduce charfield size for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index c22d1727..5f922c9a 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=21844, null=True)), + ('description', models.CharField(blank=True, max_length=255, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=21844)), + ('description', models.CharField(max_length=255)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 20a42edbe72459f2ddb70c438e283262f5a9df95 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 29 Apr 2025 16:10:52 -0400 Subject: [PATCH 072/120] reduce URLField max_length to 765 --- errata/migrations/0001_initial.py | 2 +- security/migrations/0005_reference_cve_references.py | 2 +- security/models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/errata/migrations/0001_initial.py b/errata/migrations/0001_initial.py index 85fe88b4..d02a7dc8 100644 --- a/errata/migrations/0001_initial.py +++ b/errata/migrations/0001_initial.py @@ -19,7 +19,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('er_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], ), migrations.CreateModel( diff --git a/security/migrations/0005_reference_cve_references.py b/security/migrations/0005_reference_cve_references.py index 97251add..f94cf7d5 100644 --- a/security/migrations/0005_reference_cve_references.py +++ b/security/migrations/0005_reference_cve_references.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('ref_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], options={ 'unique_together': {('ref_type', 'url')}, diff --git a/security/models.py b/security/models.py index b60acb47..9c097eed 100644 --- a/security/models.py +++ b/security/models.py @@ -29,7 +29,7 @@ class Reference(models.Model): ref_type = models.CharField(max_length=255) - url = models.URLField(max_length=2000) + url = models.URLField(max_length=765) class Meta: unique_together = ['ref_type', 'url'] From bf5478cf5d5374b693d2641d526db1a1e02f47e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 15:45:31 +0000 Subject: [PATCH 073/120] Bump django from 4.2.20 to 4.2.21 Bumps [django](https://github.com/django/django) from 4.2.20 to 4.2.21. - [Commits](https://github.com/django/django/compare/4.2.20...4.2.21) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.21 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dca4fe03..2f72fae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.20 +Django==4.2.21 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 57e5c0d4861b7c80d081e1b85a8b48f9bd1cb332 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 23:28:39 +0000 Subject: [PATCH 074/120] Bump django from 4.2.21 to 4.2.22 Bumps [django](https://github.com/django/django) from 4.2.21 to 4.2.22. - [Commits](https://github.com/django/django/compare/4.2.21...4.2.22) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.22 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..3418b5d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.21 +Django==4.2.22 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 6a45e90ed63efadd4e1e7c246d174feceb91c439 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:47:24 +0000 Subject: [PATCH 075/120] Bump requests from 2.32.3 to 2.32.4 Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.3...v2.32.4) --- updated-dependencies: - dependency-name: requests dependency-version: 2.32.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..955c7bbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 chardet==5.2.0 -requests==2.32.3 +requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 django-filter==25.1 From e6f5b9d45bfec05932c0763b32b98ba4cf88e203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 25 Jun 2025 10:03:54 +0100 Subject: [PATCH 076/120] CACHES and CELERY_BROKER_URL configuration can now be set to false when value isn't NULL --- docker/docker-entrypoint.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index f81f9088..ebf2277a 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,8 +79,12 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if [ "${USE_CACHE}" ] && [ -n "${REDIS_HOST}" ]; then - redisHost="${REDIS_HOST}" +if [ "${USE_CACHE}" ]; then + if [ -n "${REDIS_HOST}" ]; then + redisHost="${REDIS_HOST}" + else + redisHost="127.0.0.1" + fi if [ -n "${REDIS_PORT}" ]; then redisPort="${REDIS_PORT}" @@ -116,8 +120,12 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ "${USE_CELERY}" ] && [ -n "${REDIS_HOST}" ]; then - redisHost="${REDIS_HOST}" +if [ "${USE_CELERY}" ]; then + if [ -n "${REDIS_HOST}" ]; then + redisHost="${REDIS_HOST}" + else + redisHost="127.0.0.1" + fi if [ -n "${REDIS_PORT}" ]; then redisPort="${REDIS_PORT}" From 8b90eca954f688f08976238ae9582983899ba4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 25 Jun 2025 10:22:24 +0100 Subject: [PATCH 077/120] Fixed if statements when checking if variables are TRUE or FALSE --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index ebf2277a..cf0ac72f 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -79,7 +79,7 @@ if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then fi # Configure CACHES -if [ "${USE_CACHE}" ]; then +if "${USE_CACHE}"; then if [ -n "${REDIS_HOST}" ]; then redisHost="${REDIS_HOST}" else @@ -120,7 +120,7 @@ if [ ! -f /var/lib/patchman/.firstrun ]; then fi # Starts Celery for for realtime processing of reports from clients -if [ "${USE_CELERY}" ]; then +if "${USE_CELERY}"; then if [ -n "${REDIS_HOST}" ]; then redisHost="${REDIS_HOST}" else From 56748532f099b6bd1dfaf202e55e4a160dc9fa3f Mon Sep 17 00:00:00 2001 From: vtalos Date: Thu, 17 Jul 2025 19:33:36 +0300 Subject: [PATCH 078/120] Remove unused dependency 'chardet' from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 088f0870..a39eb8cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ django-bootstrap3==23.1 python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 -chardet==5.2.0 requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 From 94fcb04694d204c32d4098fe9981f255eb4a843e Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 5 Aug 2025 20:15:16 -0400 Subject: [PATCH 079/120] get_or_create_module only returns module --- modules/utils.py | 4 ++-- repos/repo_types/yum.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 817a610c..f56a0f62 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -23,7 +23,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object - Returns the module and a boolean for created + Returns the module """ created = False m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) @@ -46,7 +46,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): arch=m_arch, repo=repo, ) - return module, created + return module def get_matching_modules(name, stream, version, context, arch): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index d08c7393..7ac85816 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -91,7 +91,7 @@ def extract_module_metadata(data, url, repo): packages.add(package) from modules.utils import get_or_create_module - module, created = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) + module = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) package_ids = [] for package in packages: From 1480468f32ed5386c678e5fa0972eaf4c708c879 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 02:19:45 +0000 Subject: [PATCH 080/120] Bump django from 4.2.22 to 4.2.24 Bumps [django](https://github.com/django/django) from 4.2.22 to 4.2.24. - [Commits](https://github.com/django/django/compare/4.2.22...4.2.24) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.24 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a39eb8cc..9d2baa9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.22 +Django==4.2.24 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 72d1c2eb97039747483b3f3f9bd28a60cd757692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 16:59:33 +0100 Subject: [PATCH 081/120] Added README --- docker/README.md | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..56d6210c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,93 @@ +# Patchman + +Patchman is a Django-based patch status monitoring tool for linux systems. Patchman provides a web interface for monitoring the package updates available for Linux hosts. + +Source: https://github.com/ricardojeronimo/patchman + +Upstream: https://github.com/furlongm/patchman + + +## Getting Started + +To get started, pull the latest image from Docker Hub and run it. +``` +docker pull ricardojeronim0/patchman:latest +docker run -d -p 80:80 --name patchman ricardojeronim0/patchman +``` + +This container will run migrations on first startup, but you still need to create a superuser before being able to log in on the web interface. + +``` +docker exec -it patchman patchman-manage createsuperuser +``` + +## Configuration + +This container is configured using environment variables. The following variables are available to customize the container's behavior. + +| Variable | Default Value | Description | +| :--- | :--- | :--- | +| `ADMIN_Name` | `Your Name` | Your name | +| `ADMIN_EMAIL` | `you@example.com` | Your e-mail address | +| `TIMEZONE` | `America/New_York` | Your timezone | +| `DB_ENGINE` | `SQLite` | Database engine to be used. Choose between `MySQL` or `PostgreSQL`, leave empty to use default `SQLite` | +| `DB_HOST` | | Database hostname, IP or container name | +| `DB_PORT` |` | Database port | +| `DB_DATABASE` | | Database name | +| `DB_USER` | | Database user | +| `DB_PASSWORD` | | Database password | +| `REDIS_HOST` | `127.0.0.1` | Redis hostname, IP or container name | +| `REDIS_PORT` | `6379` | Redis port | +| `USE_CELERY` | `False` | Change to `True` for realtime processing of reports from clients | +| `USE_CACHE` | `False` | Change to `True` cache contents and reduce the load on the server | +| `CACHE_TIMEOUT` | `30` | Cache time in seconds. Be aware that the UI results may be out of date for this amount of time | + + +## Docker Compose Example + +For more complex deployments, `docker-compose` is the recommended approach. Below is an example `docker-compose.yaml` file that demonstrates how to configure the container and connect it to a separate MySQL service, and Redis for async processing and/or caching. + +```yaml +--- +services: + patchman: + container_name: patchman + image: ricardojeronim0/patchman:latest + restart: unless-stopped + environment: + ADMIN_NAME: admin_name + ADMIN_EMAIL: admin_mail@domain.tld + TIMEZONE: America/New_York + DB_ENGINE: MySQL + DB_HOST: patchman-db + DB_PORT: 3306 + DB_DATABASE: patchman + DB_USER: user + DB_PASSWORD: changeme + REDIS_HOST: redis + REDIS_PORT: 6379 + USE_CELERY: True + USE_CACHE: True + CACHE_TIMEOUT: 20 + ports: + - 80:80/tcp + depends_on: + - patchman-db + - redis + + patchman-db: + container_name: patchman-db + image: mysql:latest + restart: unless-stopped + command: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"] + environment: + MYSQL_ROOT_PASSWORD: changeme + MYSQL_DATABASE: patchman + MYSQL_USER: user + MYSQL_PASSWORD: changeme + + redis: + container_name: redis + image: redis:latest + restart: unless-stopped +``` From 102ed5a838972482a240295116b8af82e8e0264a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 17:21:42 +0100 Subject: [PATCH 082/120] Changed build action to update Docker Hub description with README.md file --- .github/workflows/docker-build.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index c6a8fb87..c9271ed8 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -29,3 +29,11 @@ jobs: tags: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:latest,${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}}:${{github.ref_name}} cache-from: type=gha cache-to: type=gha,mode=max + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} + readme-filepath: ./docker/README.md From d95e71c2eac648b9defc5011d9ce76e0362b0766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 17:51:08 +0100 Subject: [PATCH 083/120] Fixed README.md file location --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index c9271ed8..c338ca61 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -36,4 +36,4 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} - readme-filepath: ./docker/README.md + readme-filepath: ./README.md From 68ee20d80626bccaf30c33f781202c77c258aa63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 17:59:58 +0100 Subject: [PATCH 084/120] Fixed README.md file location --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index c338ca61..d893d08a 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -36,4 +36,4 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} - readme-filepath: ./README.md + readme-filepath: README.md From 6944d4d78b5e779a373d13ef5596365873a9cb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:10:23 +0100 Subject: [PATCH 085/120] Fixed README.md file location --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index d893d08a..d039fbd7 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -36,4 +36,4 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} - readme-filepath: README.md + readme-filepath: docker/README.md From 54c8cfaddd3148c1ba91d54b5487b1f920fc6d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:17:15 +0100 Subject: [PATCH 086/120] Fixed README.md file location --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index d039fbd7..b3c7873b 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -36,4 +36,4 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} - readme-filepath: docker/README.md + readme-filepath: ../../docker/README.md From f5b51c6931284faf74f7cc4aa23ebd4e3866c848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:29:47 +0100 Subject: [PATCH 087/120] Fixed README.md file location --- .github/workflows/docker-build.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index b3c7873b..64f68c4d 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -30,10 +30,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Checkout for next job + - uses: actions/checkout@v4 + - name: Docker Hub Description uses: peter-evans/dockerhub-description@v4 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: ${{vars.DOCKERHUB_USERNAME}}/${{vars.DOCKERHUB_CONTAINER}} - readme-filepath: ../../docker/README.md + readme-filepath: docker/README.md From 463b76a64b6db70dc6684aa35a59ffad9ebebf8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:32:35 +0100 Subject: [PATCH 088/120] Fixed checkout job on build action --- .github/workflows/docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 64f68c4d..9f2206d5 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -31,7 +31,7 @@ jobs: cache-to: type=gha,mode=max - name: Checkout for next job - - uses: actions/checkout@v4 + uses: actions/checkout@v4 - name: Docker Hub Description uses: peter-evans/dockerhub-description@v4 From 641697737b0c72f6f0fe204d790b52a1b67145a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:42:32 +0100 Subject: [PATCH 089/120] Changed README --- docker/README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 56d6210c..a7c86239 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,4 @@ -# Patchman - -Patchman is a Django-based patch status monitoring tool for linux systems. Patchman provides a web interface for monitoring the package updates available for Linux hosts. - Source: https://github.com/ricardojeronimo/patchman - Upstream: https://github.com/furlongm/patchman @@ -27,7 +22,7 @@ This container is configured using environment variables. The following variable | Variable | Default Value | Description | | :--- | :--- | :--- | -| `ADMIN_Name` | `Your Name` | Your name | +| `ADMIN_NAME` | `Your Name` | Your name | | `ADMIN_EMAIL` | `you@example.com` | Your e-mail address | | `TIMEZONE` | `America/New_York` | Your timezone | | `DB_ENGINE` | `SQLite` | Database engine to be used. Choose between `MySQL` or `PostgreSQL`, leave empty to use default `SQLite` | From 89568898702b467e04b9dbd4e4cc95e65b64a97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Wed, 10 Sep 2025 18:45:54 +0100 Subject: [PATCH 090/120] Changed README --- docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/README.md b/docker/README.md index a7c86239..3310fccf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,5 @@ Source: https://github.com/ricardojeronimo/patchman + Upstream: https://github.com/furlongm/patchman From b6162963e7811081eb7a4f60ef04a594d23d5866 Mon Sep 17 00:00:00 2001 From: Will Furnell Date: Fri, 12 Sep 2025 13:33:45 +0100 Subject: [PATCH 091/120] Package types are in the Package class --- packages/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models.py b/packages/models.py index f4c9c59e..74a83c0c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -195,11 +195,11 @@ def __str__(self): rel = f'-{self.release}' else: rel = '' - if self.packagetype == self.GENTOO: + if self.packagetype == Package.GENTOO: return f'{self.category}/{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype in [self.DEB, self.ARCH]: + elif self.packagetype in [Package.DEB, Package.ARCH]: return f'{self.name}_{epo}{self.version}{rel}_{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype == self.RPM: + elif self.packagetype == Package.RPM: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' else: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' From 3676e78cd0afbd0467241f8fb62f54626f2508fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:00:50 +0000 Subject: [PATCH 092/120] Bump django from 4.2.24 to 4.2.25 Bumps [django](https://github.com/django/django) from 4.2.24 to 4.2.25. - [Commits](https://github.com/django/django/compare/4.2.24...4.2.25) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.25 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d2baa9e..9d277d38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.24 +Django==4.2.25 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 1c260013c8cdb1bb5913a1151e37bde309ac438d Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:54:33 -0400 Subject: [PATCH 093/120] bump redis --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d277d38..2f264c9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ python-magic==0.4.27 gitpython==3.1.44 tenacity==8.2.3 celery==5.4.0 -redis==5.2.1 +redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 From 0f5445448421f17efb66d07d3f511b6327732440 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:59:40 -0400 Subject: [PATCH 094/120] Update license in common.py --- util/templatetags/common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 6737c438..2aea1e5e 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -1,12 +1,10 @@ -# Copyright 2010 VPAC -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see import re From ce9f4f07dc176cfdab072f425acfdd69e9b728eb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:11:25 -0400 Subject: [PATCH 095/120] fix licenses --- hosts/templatetags/report_alert.py | 7 +++---- setup.py | 2 +- util/filterspecs.py | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index 3a3e3a9a..a28c5058 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -1,11 +1,10 @@ -# Copyright 2016-2021 Marcus Furlong +# Copyright 2016-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -13,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from datetime import timedelta diff --git a/setup.py b/setup.py index d8249a67..6ec6d974 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # diff --git a/util/filterspecs.py b/util/filterspecs.py index 1c845ff3..722b45df 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -1,12 +1,11 @@ # Copyright 2010 VPAC -# Copyright 2014-2021 Marcus Furlong +# Copyright 2014-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +13,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from django.utils.safestring import mark_safe from django.db.models.query import QuerySet From c651c3f222ec7c53af47cfaedfcb5d7254b47710 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:57:40 -0400 Subject: [PATCH 096/120] use GPL-3.0-only for debian copyright --- debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index ab051037..5202ff0e 100644 --- a/debian/copyright +++ b/debian/copyright @@ -6,7 +6,7 @@ Source: https://github.com/furlongm/patchman Files: * Copyright: 2011-2012 VPAC http://www.vpac.org 2013-2021 Marcus Furlong -License: GPL-3.0 +License: GPL-3.0-only This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 3 only. From fb9b56c453f1c93b617b2b2d98e8bb4db005c8f7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 20 Oct 2025 20:47:26 -0400 Subject: [PATCH 097/120] fix tag handling Signed-off-by: Marcus Furlong --- hosts/models.py | 2 +- hosts/utils.py | 19 +++++++++++++++++-- sbin/patchman | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index a6c451b5..5b7b3979 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -85,7 +85,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {self.tags}\n' + text += f'Tags : {" ".join(self.tags.slugs())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' diff --git a/hosts/utils.py b/hosts/utils.py index b328129f..f07d5d1e 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -18,8 +18,9 @@ from socket import gethostbyaddr, gaierror, herror from django.db import transaction, IntegrityError +from taggit.models import Tag -from patchman.signals import error_message +from patchman.signals import error_message, info_message def update_rdns(host): @@ -62,7 +63,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.osvariant = osvariant host.domain = domain host.lastreport = report.created - host.tags = report.tags + host.tags.set(report.tags.split(','), clear=True) if report.reboot == 'True': host.reboot_required = True else: @@ -73,3 +74,17 @@ def get_or_create_host(report, arch, osvariant, domain): if host: host.check_rdns() return host + + +def clean_tags(): + """ Delete Tags that have no Host + """ + tags = Tag.objects.filter( + host__isnull=True, + ) + tlen = tags.count() + if tlen == 0: + info_message.send(sender=None, text='No orphaned Tags found.') + else: + info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + tags.delete() diff --git a/sbin/patchman b/sbin/patchman index 9cc6048e..c0911434 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -34,6 +34,7 @@ from errata.utils import mark_errata_security_updates, enrich_errata, \ scan_package_updates_for_affected_packages from errata.tasks import update_errata from hosts.models import Host +from hosts.utils import clean_tags from modules.utils import clean_modules from packages.utils import clean_packages, clean_packageupdates, clean_packagenames from repos.models import Repository @@ -362,6 +363,7 @@ def dbcheck(remove_duplicates=False): clean_repos() clean_modules() clean_packageupdates() + clean_tags() def collect_args(): From eee56752f2dcfdaa9e63361bb20f7aa5b99138fa Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 21 Oct 2025 22:35:58 -0400 Subject: [PATCH 098/120] fix some flake8-bugbear bugs Signed-off-by: Marcus Furlong --- patchman/urls.py | 2 +- repos/repo_types/gentoo.py | 2 +- repos/repo_types/rpm.py | 2 +- util/__init__.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/patchman/urls.py b/patchman/urls.py index 337d6b63..ee786566 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -44,7 +44,7 @@ router.register(r'package', package_views.PackageViewSet) router.register(r'package-update', package_views.PackageUpdateViewSet) router.register(r'cve', security_views.CVEViewSet) -router.register(r'reference', security_views.ReferenceViewSet), +router.register(r'reference', security_views.ReferenceViewSet) router.register(r'erratum', errata_views.ErratumViewSet) router.register(r'repo', repo_views.RepositoryViewSet) router.register(r'mirror', repo_views.MirrorViewSet) diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 94df139a..8e4198d9 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -226,7 +226,7 @@ def extract_gentoo_overlay_ebuilds(t): """ Extract ebuilds from a Gentoo overlay tarball """ extracted_ebuilds = {} - for root, dirs, files in os.walk(t): + for root, _, files in os.walk(t): for name in files: if fnmatch(name, '*.ebuild'): package_name = root.replace(t + '/', '') diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index aa3354c7..d9501cde 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -69,7 +69,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True) - for i, mirror in enumerate(enabled_mirrors): + for mirror in enabled_mirrors: res = find_mirror_url(mirror.url, formats) if not res: mirror.fail() diff --git a/util/__init__.py b/util/__init__.py index a56ed3b6..4958dfb0 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -107,10 +107,14 @@ def fetch_content(response, text='', ljust=35): wait=wait_exponential(multiplier=1, min=1, max=10), reraise=False, ) -def get_url(url, headers={}, params={}): +def get_url(url, headers=None, params=None): """ Perform a http GET on a URL. Return None on error. """ response = None + if not headers: + headers = {} + if not params: + params = {} try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, timeout=30) From 2bcd4dac0bd7dd81f4ebe4aa483532aec04a8650 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:45 -0400 Subject: [PATCH 099/120] fix package filter list for errata --- packages/views.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/views.py b/packages/views.py index cd53fa6e..c55a6c72 100644 --- a/packages/views.py +++ b/packages/views.py @@ -62,9 +62,16 @@ def package_list(request): if 'affected_by_errata' in request.GET: affected_by_errata = request.GET['affected_by_errata'] == 'true' if affected_by_errata: - packages = packages.filter(erratum__isnull=False) + packages = packages.filter(affected_by_erratum__isnull=False) else: - packages = packages.filter(erratum__isnull=True) + packages = packages.filter(affected_by_erratum__isnull=True) + + if 'provides_fix_in_erratum' in request.GET: + provides_fix_in_erratum = request.GET['provides_fix_in_erratum'] == 'true' + if provides_fix_in_erratum: + packages = packages.filter(provides_fix_in_erratum__isnull=False) + else: + packages = packages.filter(provides_fix_in_erratum__isnull=True) if 'installed_on_hosts' in request.GET: installed_on_hosts = request.GET['installed_on_hosts'] == 'true' @@ -102,6 +109,8 @@ def package_list(request): filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) + filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', + {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Installed on Hosts', 'installed_on_hosts', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Available in Repos', 'available_in_repos', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) From 856f41f9f7cdc8fbebb0c6c3685a014526efdc36 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:29 -0400 Subject: [PATCH 100/120] add support for zstd compression in deb and rpm repos fixes: #698 Signed-off-by: Marcus Furlong --- debian/control | 2 +- repos/repo_types/deb.py | 7 ++++++- repos/repo_types/rpm.py | 2 ++ requirements.txt | 1 + setup.cfg | 1 + util/__init__.py | 17 +++++++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 67026269..7bfe320e 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit + python3-redis, python3-git, python3-django-taggit, python3-zstandard Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 25d8eba7..1d3607c5 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -71,7 +71,12 @@ def refresh_deb_repo(repo): are and then fetches and extracts packages from those files. """ - formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages'] + formats = [ + 'Packages.xz', + 'Packages.bz2', + 'Packages.gz', + 'Packages', + ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index d9501cde..93d47007 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -57,10 +57,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): which type of repo it is, then refreshes the mirrors """ formats = [ + 'repodata/repomd.xml.zst', 'repodata/repomd.xml.xz', 'repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', + 'suse/repodata/repomd.xml.zst', 'suse/repodata/repomd.xml.xz', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', diff --git a/requirements.txt b/requirements.txt index 2f264c9b..788f4241 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 +zstandard==0.25.0 diff --git a/setup.cfg b/setup.cfg index 7af9ccb0..b1d5ee4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ requires = /usr/bin/python3 python3-importlib-metadata python3-cvss python3-redis + python3-zstandard redis celery python3-django-celery-beat diff --git a/util/__init__.py b/util/__init__.py index ac6f8f1b..4a3f9caa 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -21,6 +21,11 @@ import zlib import lzma import os +try: + # python 3.14+ - can also remove the dependency at that stage + from compression import zstd +except ImportError: + import zstandard as zstd from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -202,6 +207,16 @@ def unxz(contents): error_message.send(sender=None, text='lzma: ' + e) +def unzstd(contents): + """ unzstd contents in memory and return the data + """ + try: + zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() + return zstddata + except zstd.ZstdError as e: + error_message.send(sender=None, text='zstd: ' + e) + + def extract(data, fmt): """ Extract the contents based on mimetype or file ending. Return the unmodified data if neither mimetype nor file ending matches, otherwise @@ -214,6 +229,8 @@ def extract(data, fmt): m = magic.open(magic.MAGIC_MIME) m.load() mime = m.buffer(data).split(';')[0] + if mime == 'application/zstd' or fmt.endswith('zst'): + return unzstd(data) if mime == 'application/x-xz' or fmt.endswith('xz'): return unxz(data) elif mime == 'application/x-bzip2' or fmt.endswith('bz2'): From 8d9da89ebccd367de221b891ad02f99b205d9c19 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:41 -0400 Subject: [PATCH 101/120] simplify logging --- arch/utils.py | 10 ++--- errata/models.py | 12 ++--- errata/sources/distros/arch.py | 5 ++- errata/sources/distros/centos.py | 5 ++- errata/sources/distros/debian.py | 7 +-- errata/sources/distros/rocky.py | 11 ++--- errata/sources/distros/ubuntu.py | 7 +-- errata/sources/repos/yum.py | 5 ++- errata/tasks.py | 2 +- errata/utils.py | 9 ++-- hosts/models.py | 14 +++--- hosts/tasks.py | 4 +- hosts/utils.py | 8 ++-- modules/utils.py | 8 ++-- packages/utils.py | 26 +++++------ patchman/receivers.py | 21 ++++----- patchman/signals.py | 8 ++-- reports/models.py | 10 ++--- reports/tasks.py | 4 +- reports/utils.py | 5 ++- repos/models.py | 18 ++++---- repos/repo_types/arch.py | 11 ++--- repos/repo_types/deb.py | 11 ++--- repos/repo_types/gentoo.py | 15 ++++--- repos/repo_types/rpm.py | 8 ++-- repos/repo_types/yast.py | 5 ++- repos/repo_types/yum.py | 23 +++++----- repos/utils.py | 37 ++++++++-------- sbin/patchman | 76 ++++++++++++++++---------------- security/models.py | 8 ++-- util/__init__.py | 24 +++++----- util/logging.py | 42 ++++++++++++++++++ 32 files changed, 258 insertions(+), 201 deletions(-) create mode 100644 util/logging.py diff --git a/arch/utils.py b/arch/utils.py index 1498fdec..04d0b350 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from arch.models import PackageArchitecture, MachineArchitecture -from patchman.signals import info_message +from util.logging import info_message def clean_package_architectures(): @@ -24,9 +24,9 @@ def clean_package_architectures(): parches = PackageArchitecture.objects.filter(package__isnull=True) plen = parches.count() if plen == 0: - info_message.send(sender=None, text='No orphaned PackageArchitectures found.') + info_message(text='No orphaned PackageArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures') + info_message(text=f'Removing {plen} orphaned PackageArchitectures') parches.delete() @@ -39,9 +39,9 @@ def clean_machine_architectures(): ) mlen = marches.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned MachineArchitectures found.') + info_message(text='No orphaned MachineArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures') + info_message(text=f'Removing {mlen} orphaned MachineArchitectures') marches.delete() diff --git a/errata/models.py b/errata/models.py index b10daf4d..cfc9bd0d 100644 --- a/errata/models.py +++ b/errata/models.py @@ -25,7 +25,7 @@ from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from patchman.signals import error_message +from util.logging import error_message from util import get_url @@ -70,7 +70,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -84,7 +84,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -93,7 +93,7 @@ def fetch_osv_dev_data(self): osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}' res = get_url(osv_dev_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}') + error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}') return data = res.content osv_dev_json = json.loads(data) @@ -102,7 +102,7 @@ def fetch_osv_dev_data(self): def parse_osv_dev_data(self, osv_dev_json): name = osv_dev_json.get('id') if name != self.name: - error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}') + error_message(text=f'Erratum name mismatch - {self.name} != {name}') return related = osv_dev_json.get('related') if related: @@ -155,7 +155,7 @@ def add_cve(self, cve_id): """ Add a CVE to an Erratum object """ if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit(): - error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}') + error_message(text=f'Not a CVE ID: {cve_id}') return self.cves.add(get_or_create_cve(cve_id)) diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 40d0dada..87c6c47a 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,7 +20,8 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from packages.models import Package from packages.utils import find_evr, get_matching_packages, get_or_create_package from util import get_url, fetch_content @@ -99,7 +100,7 @@ def process_arch_erratum(advisory, osrelease): add_arch_erratum_references(e, advisory) add_arch_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_arch_linux_osrelease(): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index eefb2b88..d2722a6b 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -20,7 +20,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import parse_package_string, get_or_create_package -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type @@ -34,7 +35,7 @@ def update_centos_errata(): if actual_checksum != expected_checksum: e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) else: if data: parse_centos_errata(bunzip2(data)) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 93ae2bd5..1ae919e4 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -27,7 +27,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, find_evr -from patchman.signals import error_message, pbar_start, pbar_update, warning_message +from util.logging import error_message, warning_message +from patchman.signals import pbar_start, pbar_update from util import get_url, fetch_content, get_setting_of_type, extract DSCs = {} @@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames): for package in packages: process_debian_erratum_fixed_packages(e, package) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def parse_debian_erratum_package(line, accepted_codenames): @@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version): """ Fetch the package list from a DSC file for a given source package/version """ if not DSCs.get(package) or not DSCs[package].get(version): - warning_message.send(sender=None, text=f'No DSC found for {package} {version}') + warning_message(text=f'No DSC found for {package} {version}') return source_url = DSCs[package][version]['url'] res = get_url(source_url) diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 693d7b0c..16d4d12c 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -25,7 +25,8 @@ from packages.models import Package from packages.utils import parse_package_string, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, info_message, error_message +from util import get_url, fetch_content +from util.logging import info_message, error_message def update_rocky_errata(concurrent_processing=True): @@ -50,16 +51,16 @@ def check_rocky_errata_endpoint_health(rocky_errata_api_host): health = json.loads(data) if health.get('status') == 'ok': s = f'Rocky Errata API healthcheck OK: {rocky_errata_healthcheck_url}' - info_message.send(sender=None, text=s) + info_message(text=s) return True else: s = f'Rocky Errata API healthcheck FAILED: {rocky_errata_healthcheck_url}' - error_message.send(sender=None, text=s) + error_message(text=s) return False except Exception as e: s = f'Rocky Errata API healthcheck exception occured: {rocky_errata_healthcheck_url}\n' s += str(e) - error_message.send(sender=None, text=s) + error_message(text=s) return False @@ -194,7 +195,7 @@ def process_rocky_erratum(advisory): add_rocky_erratum_oses(e, advisory) add_rocky_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_rocky_erratum_references(e, advisory): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 7f50962c..d1ce7cc5 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -28,7 +28,8 @@ from packages.models import Package from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update def update_ubuntu_errata(concurrent_processing=False): @@ -45,7 +46,7 @@ def update_ubuntu_errata(concurrent_processing=False): else: e = 'Ubuntu USN DB checksum mismatch, skipping Ubuntu errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) def fetch_ubuntu_usn_db(): @@ -126,7 +127,7 @@ def process_usn(usn_id, advisory, accepted_releases): add_ubuntu_erratum_references(e, usn_id, advisory) add_ubuntu_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_ubuntu_erratum_osreleases(e, affected_releases, accepted_releases): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index dfeed879..f361d10e 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -23,7 +23,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from patchman.signals import pbar_start, pbar_update, error_message +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url @@ -38,7 +39,7 @@ def extract_updateinfo(data, url, concurrent_processing=True): elen = root.__len__() updates = root.findall('update') except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing updateinfo file from {url} : {e}') + error_message(text=f'Error parsing updateinfo file from {url} : {e}') if concurrent_processing: extract_updateinfo_concurrently(updates, elen) else: diff --git a/errata/tasks.py b/errata/tasks.py index fe53b415..f1d6eeee 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -22,7 +22,7 @@ from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from patchman.signals import error_message +from util.logging import error_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type diff --git a/errata/utils.py b/errata/utils.py index d8099db4..a8d8d424 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -21,7 +21,8 @@ from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from patchman.signals import pbar_start, pbar_update, warning_message +from util.logging import warning_message +from patchman.signals import pbar_start, pbar_update def get_or_create_erratum(name, e_type, issue_date, synopsis): @@ -36,16 +37,16 @@ def get_or_create_erratum(name, e_type, issue_date, synopsis): days_delta = abs(e.issue_date.date() - issue_date_tz.date()).days updated = False if e.e_type != e_type: - warning_message.send(sender=None, text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') + warning_message(text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') e.e_type = e_type updated = True if days_delta > 1: text = f'Updating {name} issue date `{e.issue_date.date()}` -> `{issue_date_tz.date()}`' - warning_message.send(sender=None, text=text) + warning_message(text=text) e.issue_date = issue_date_tz updated = True if e.synopsis != synopsis: - warning_message.send(sender=None, text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') + warning_message(text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') e.synopsis = synopsis updated = True if updated: diff --git a/hosts/models.py b/hosts/models.py index 5b7b3979..650544dc 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -34,7 +34,7 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from patchman.signals import info_message +from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo @@ -90,7 +90,7 @@ def show(self): text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) @@ -114,13 +114,13 @@ def check_rdns(self): if self.check_dns: update_rdns(self) if self.hostname.lower() == self.reversedns.lower(): - info_message.send(sender=None, text='Reverse DNS matches') + info_message(text='Reverse DNS matches') else: text = 'Reverse DNS mismatch found: ' text += f'{self.hostname} != {self.reversedns}' - info_message.send(sender=None, text=text) + info_message(text=text) else: - info_message.send(sender=None, text='Reverse DNS check disabled') + info_message(text='Reverse DNS check disabled') def clean_reports(self): """ Remove all but the last 3 reports for a host @@ -131,7 +131,7 @@ def clean_reports(self): for report in Report.objects.filter(host=self).order_by('-created')[3:]: report.delete() if rlen > 0: - info_message.send(sender=None, text=f'{self.hostname}: removed {rlen} old reports') + info_message(text=f'{self.hostname}: removed {rlen} old reports') def get_host_repo_packages(self): if self.host_repos_only: @@ -163,7 +163,7 @@ def process_update(self, package, highest_package): security = True update = get_or_create_package_update(oldpackage=package, newpackage=highest_package, security=security) self.updates.add(update) - info_message.send(sender=None, text=f'{update}') + info_message(text=f'{update}') return update.id def find_updates(self): diff --git a/hosts/tasks.py b/hosts/tasks.py index 2fdce96f..1643901d 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -20,7 +20,7 @@ from hosts.models import Host from util import get_datetime_now -from patchman.signals import info_message +from util.logging import info_message @shared_task @@ -78,4 +78,4 @@ def find_all_host_updates_homogenous(): phost.updated_at = ts phost.save() updated_hosts.append(phost) - info_message.send(sender=None, text=f'Added the same updates to {phost}') + info_message(text=f'Added the same updates to {phost}') diff --git a/hosts/utils.py b/hosts/utils.py index f07d5d1e..d6e663cf 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -20,7 +20,7 @@ from django.db import transaction, IntegrityError from taggit.models import Tag -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message def update_rdns(host): @@ -70,7 +70,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = False host.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) if host: host.check_rdns() return host @@ -84,7 +84,7 @@ def clean_tags(): ) tlen = tags.count() if tlen == 0: - info_message.send(sender=None, text='No orphaned Tags found.') + info_message(text='No orphaned Tags found.') else: - info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + info_message(text=f'{tlen} orphaned Tags found.') tags.delete() diff --git a/modules/utils.py b/modules/utils.py index f56a0f62..05c57c80 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from django.db import IntegrityError -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message from modules.models import Module from arch.models import PackageArchitecture @@ -37,7 +37,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): repo=repo, ) except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) module = Module.objects.get( name=name, stream=stream, @@ -73,7 +73,7 @@ def clean_modules(): ) mlen = modules.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned Modules found.') + info_message(text='No orphaned Modules found.') else: - info_message.send(sender=None, text=f'{mlen} orphaned Modules found.') + info_message(text=f'{mlen} orphaned Modules found.') modules.delete() diff --git a/packages/utils.py b/packages/utils.py index 9b098225..f00f6710 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -22,7 +22,7 @@ from arch.models import PackageArchitecture from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString -from patchman.signals import error_message, info_message, warning_message +from util.logging import error_message, info_message, warning_message def convert_package_to_packagestring(package): @@ -141,7 +141,7 @@ def parse_redhat_package_string(pkg_str): name, epoch, ver, rel, dist, arch = m.groups() else: e = f'Error parsing package string: "{pkg_str}"' - error_message.send(sender=None, text=e) + error_message(text=e) return if dist: rel = f'{rel}.{dist}' @@ -195,7 +195,7 @@ def get_or_create_package(name, epoch, version, release, arch, p_type): package = packages.first() # TODO this should handle gentoo package categories too, otherwise we may be deleting packages # that should be kept - warning_message.send(sender=None, text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') + warning_message(text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') packages.exclude(id=package.id).delete() return package @@ -218,10 +218,10 @@ def get_or_create_package_update(oldpackage, newpackage, security): except MultipleObjectsReturned: e = 'Error: MultipleObjectsReturned when attempting to add package \n' e += f'update with oldpackage={oldpackage} | newpackage={newpackage}:' - error_message.send(sender=None, text=e) + error_message(text=e) updates = PackageUpdate.objects.filter(oldpackage=oldpackage, newpackage=newpackage) for update in updates: - error_message.send(sender=None, text=str(update)) + error_message(text=str(update)) return try: if update: @@ -281,13 +281,13 @@ def clean_packageupdates(): for update in package_updates: if update.host_set.count() == 0: text = f'Removing unused PackageUpdate {update}' - info_message.send(sender=None, text=text) + info_message(text=text) update.delete() for duplicate in package_updates: if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ update.security == duplicate.security and update.id != duplicate.id: text = f'Removing duplicate PackageUpdate: {update}' - info_message.send(sender=None, text=text) + info_message(text=text) for host in duplicate.host_set.all(): host.updates.remove(duplicate) host.updates.add(update) @@ -307,12 +307,12 @@ def clean_packages(remove_duplicates=False): ) plen = packages.count() if plen == 0: - info_message.send(sender=None, text='No orphaned Packages found.') + info_message(text='No orphaned Packages found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned Packages') + info_message(text=f'Removing {plen} orphaned Packages') packages.delete() if remove_duplicates: - info_message.send(sender=None, text='Checking for duplicate Packages...') + info_message(text='Checking for duplicate Packages...') for package in Package.objects.all(): potential_duplicates = Package.objects.filter( name=package.name, @@ -326,7 +326,7 @@ def clean_packages(remove_duplicates=False): if potential_duplicates.count() > 1: for dupe in potential_duplicates: if dupe.id != package.id: - info_message.send(sender=None, text=f'Removing duplicate Package {dupe}') + info_message(text=f'Removing duplicate Package {dupe}') dupe.delete() @@ -336,7 +336,7 @@ def clean_packagenames(): names = PackageName.objects.filter(package__isnull=True) nlen = names.count() if nlen == 0: - info_message.send(sender=None, text='No orphaned PackageNames found.') + info_message(text='No orphaned PackageNames found.') else: - info_message.send(sender=None, text=f'Removing {nlen} orphaned PackageNames') + info_message(text=f'Removing {nlen} orphaned PackageNames') names.delete() diff --git a/patchman/receivers.py b/patchman/receivers.py index 5ec32cdd..8d8893ca 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -21,7 +21,8 @@ from django.dispatch import receiver from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update, \ + info_message_s, warning_message_s, error_message_s, debug_message_s from django.conf import settings @@ -47,36 +48,36 @@ def pbar_update_receiver(**kwargs): update_pbar(index) -@receiver(info_message) -def print_info_message(sender=None, **kwargs): - """ Receiver to print an info message, no color +@receiver(info_message_s) +def print_info_message(**kwargs): + """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.RESET_ALL + Fore.RESET + text) -@receiver(warning_message) +@receiver(warning_message_s) def print_warning_message(**kwargs): - """ Receiver to print a warning message in yellow text + """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.BRIGHT + Fore.YELLOW + text) -@receiver(error_message) +@receiver(error_message_s) def print_error_message(**kwargs): - """ Receiver to print an error message in red text + """ Receiver to handle an error message, red text """ text = str(kwargs.get('text')) if text: tqdm.write(Style.BRIGHT + Fore.RED + text) -@receiver(debug_message) +@receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to print a debug message in blue, if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if verbose and DEBUG are set """ text = str(kwargs.get('text')) if get_verbosity() and settings.DEBUG and text: diff --git a/patchman/signals.py b/patchman/signals.py index 917a48e4..799b9c98 100644 --- a/patchman/signals.py +++ b/patchman/signals.py @@ -19,7 +19,7 @@ pbar_start = Signal() pbar_update = Signal() -info_message = Signal() -warning_message = Signal() -error_message = Signal() -debug_message = Signal() +info_message_s = Signal() +warning_message_s = Signal() +error_message_s = Signal() +debug_message_s = Signal() diff --git a/reports/models.py b/reports/models.py index 6818ea23..d529804b 100644 --- a/reports/models.py +++ b/reports/models.py @@ -19,7 +19,7 @@ from django.urls import reverse from hosts.utils import get_or_create_host -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message class Report(models.Model): @@ -97,11 +97,11 @@ def process(self, find_updates=True, verbose=False): """ Process a report and extract os, arch, domain, packages, repos etc """ if not self.os or not self.kernel or not self.arch: - error_message.send(sender=None, text=f'Error: OS, kernel or arch not sent with report {self.id}') + error_message(text=f'Error: OS, kernel or arch not sent with report {self.id}') return if self.processed: - info_message.send(sender=None, text=f'Report {self.id} has already been processed') + info_message(text=f'Report {self.id} has already been processed') return from reports.utils import get_arch, get_os, get_domain @@ -111,7 +111,7 @@ def process(self, find_updates=True, verbose=False): host = get_or_create_host(self, arch, osvariant, domain) if verbose: - info_message.send(sender=None, text=f'Processing report {self.id} - {self.host}') + info_message(text=f'Processing report {self.id} - {self.host}') from reports.utils import process_packages, process_repos, process_updates, process_modules process_repos(report=self, host=host) @@ -124,5 +124,5 @@ def process(self, find_updates=True, verbose=False): if find_updates: if verbose: - info_message.send(sender=None, text=f'Finding updates for report {self.id} - {self.host}') + info_message(text=f'Finding updates for report {self.id} - {self.host}') host.find_updates() diff --git a/reports/tasks.py b/reports/tasks.py index db9e4103..fe294e8d 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -21,7 +21,7 @@ from hosts.models import Host from reports.models import Report -from util import info_message +from util.logging import info_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -48,5 +48,5 @@ def clean_reports_with_no_hosts(): for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): text = f'Deleting report {report.id} for Host `{report.host}` as the host no longer exists' - info_message.send(sender=None, text=text) + info_message(text=text) report.delete() diff --git a/reports/utils.py b/reports/utils.py index 641f90df..76b6e09c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -26,7 +26,8 @@ from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant from packages.models import Package, PackageCategory from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.models import Repository, Mirror, MirrorPackage from repos.utils import get_or_create_repo @@ -93,7 +94,7 @@ def process_packages(report, host): host.packages.add(package) else: if pkg_str[0].lower() != 'gpg-pubkey': - info_message.send(sender=None, text=f'No package returned for {pkg_str}') + info_message(text=f'No package returned for {pkg_str}') pbar_update.send(sender=None, index=i + 1) for package in host.packages.all(): diff --git a/repos/models.py b/repos/models.py index 181a103d..a1db2a93 100644 --- a/repos/models.py +++ b/repos/models.py @@ -26,7 +26,7 @@ from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from patchman.signals import info_message, warning_message, error_message +from util.logging import info_message, warning_message, error_message class Repository(models.Model): @@ -72,7 +72,7 @@ def show(self): text += f'arch: {self.arch}\n' text += 'Mirrors:' - info_message.send(sender=None, text=text) + info_message(text=text) for mirror in self.mirror_set.all(): mirror.show() @@ -99,10 +99,10 @@ def refresh(self, force=False): refresh_gentoo_repo(self) else: text = f'Error: unknown repo type for repo {self.id}: {self.repotype}' - error_message.send(sender=None, text=text) + error_message(text=text) else: text = 'Repo requires authentication, not updating' - warning_message.send(sender=None, text=text) + warning_message(text=text) def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors @@ -168,7 +168,7 @@ def show(self): text = f' {self.id} : {self.url}\n' text += ' last updated: ' text += f'{self.timestamp} checksum: {self.packages_checksum}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def fail(self): """ Records that the mirror has failed @@ -178,10 +178,10 @@ def fail(self): """ if self.repo.auth_required: text = f'Mirror requires authentication, not updating - {self.url}' - warning_message.send(sender=None, text=text) + warning_message(text=text) return text = f'No usable mirror found at {self.url}' - error_message.send(sender=None, text=text) + error_message(text=text) default_max_mirror_failures = 28 max_mirror_failures = get_setting_of_type( setting_name='MAX_MIRROR_FAILURES', @@ -191,11 +191,11 @@ def fail(self): self.fail_count = self.fail_count + 1 if max_mirror_failures == -1: text = f'Mirror has failed {self.fail_count} times, but MAX_MIRROR_FAILURES=-1, not disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) elif self.fail_count > max_mirror_failures: self.refresh = False text = f'Mirror has failed {self.fail_count} times (max={max_mirror_failures}), disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) self.last_access_ok = False self.save() diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 6e85b153..09719428 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,7 +18,8 @@ from io import BytesIO from packages.models import PackageString -from patchman.signals import info_message, warning_message, pbar_start, pbar_update +from util.logging import info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages from util import get_datetime_now, get_checksum, Checksum @@ -34,7 +35,7 @@ def refresh_arch_repo(repo): for i, mirror in enumerate(enabled_mirrors): if i >= max_mirrors: text = f'{max_mirrors} Mirrors already refreshed (max={max_mirrors}), skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) break res = find_mirror_url(mirror.url, [fname]) @@ -42,7 +43,7 @@ def refresh_arch_repo(repo): continue mirror_url = res.url text = f'Found Arch Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -54,7 +55,7 @@ def refresh_arch_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum @@ -111,5 +112,5 @@ def extract_arch_packages(data): packagetype='A') packages.add(package) else: - info_message.send(sender=None, text='No Packages found in Repo') + info_message(text='No Packages found in Repo') return packages diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 1d3607c5..c6c26d78 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -19,7 +19,8 @@ from debian.debian_support import Version from packages.models import PackageString -from patchman.signals import error_message, pbar_start, pbar_update, info_message, warning_message +from util.logging import error_message, info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url from util import get_datetime_now, get_checksum, Checksum, extract @@ -30,7 +31,7 @@ def extract_deb_packages(data, url): try: extracted = extract(data, url).decode('utf-8') except UnicodeDecodeError as e: - error_message.send(sender=None, text=f'Skipping {url} : {e}') + error_message(text=f'Skipping {url} : {e}') return package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) @@ -61,7 +62,7 @@ def extract_deb_packages(data, url): packagetype='D') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages @@ -86,7 +87,7 @@ def refresh_deb_repo(repo): continue mirror_url = res.url text = f'Found deb Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -98,7 +99,7 @@ def refresh_deb_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 8e4198d9..e440f0d5 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -26,7 +26,8 @@ from packages.models import PackageString from packages.utils import find_evr -from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid @@ -56,7 +57,7 @@ def refresh_gentoo_main_repo(repo): if mirror.packages_checksum == checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue res = get_url(mirror.url) @@ -70,7 +71,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue extracted = extract(data, mirror.url) - info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}') + info_message(text=f'Found Gentoo Repo - {mirror.url}') computed_checksum = get_checksum(data, Checksum.md5) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'): @@ -165,7 +166,7 @@ def get_gentoo_overlay_mirrors(repo_name): if element.text.startswith('http'): mirrors.append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_overlays_url}: {e}') + error_message(text=f'Error parsing {gentoo_overlays_url}: {e}') return mirrors @@ -199,7 +200,7 @@ def get_gentoo_mirror_urls(): if element.get('protocol') == 'http': mirrors[name]['urls'].append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_distfiles_url}: {e}') + error_message(text=f'Error parsing {gentoo_distfiles_url}: {e}') mirror_urls = [] # for now, ignore region data and choose MAX_MIRRORS mirrors at random for _, v in mirrors.items(): @@ -274,7 +275,7 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds): ) packages.add(package) plen = len(packages) - info_message.send(sender=None, text=f'Extracted {plen} Packages', plen=plen) + info_message(text=f'Extracted {plen} Packages', plen=plen) return packages @@ -282,7 +283,7 @@ def extract_gentoo_overlay_packages(mirror): """ Extract packages from gentoo overlay repo """ t = tempfile.mkdtemp() - info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}') + info_message(text=f'Extracting Gentoo packages from {mirror.url}') git.Repo.clone_from(mirror.url, t, depth=1) packages = set() extracted_ebuilds = extract_gentoo_overlay_ebuilds(t) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 93d47007..51661809 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,7 +16,7 @@ from django.db.models import Q -from patchman.signals import info_message, warning_message +from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data @@ -47,7 +47,7 @@ def max_mirrors_refreshed(repo, checksum, ts): have_checksum_and_ts = repo.mirror_set.filter(mirrors_q).count() if have_checksum_and_ts >= max_mirrors: text = f'{max_mirrors} Mirrors already have this checksum and timestamp, skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) return True return False @@ -87,11 +87,11 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): if mirror_url.endswith('content'): text = f'Found yast rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yast_repo(mirror, repo_data) else: text = f'Found yum rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yum_repo(mirror, repo_data, mirror_url, errata_only) if mirror.last_access_ok: mirror.timestamp = ts diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index 0ef54358..bf594040 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,7 +17,8 @@ import re from packages.models import PackageString -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -65,5 +66,5 @@ def extract_yast_packages(data): packagetype='R') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index 7ac85816..bc0fbc4b 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -22,7 +22,8 @@ from errata.sources.repos.yum import extract_updateinfo from packages.models import Package, PackageString from packages.utils import get_or_create_package, parse_package_string -from patchman.signals import warning_message, error_message, pbar_start, pbar_update +from util.logging import warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -50,7 +51,7 @@ def get_repomd_url(mirror_url, data, url_type='primary'): checksum = grandchild.text checksum_type = grandchild.attrib.get('type') except ElementTree.ParseError as e: - error_message.send(sender=None, text=(f'Error parsing repomd from {mirror_url}: {e}')) + error_message(text=(f'Error parsing repomd from {mirror_url}: {e}')) if not location: return None, None, None url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location @@ -65,7 +66,7 @@ def extract_module_metadata(data, url, repo): try: modules_yaml = yaml.safe_load_all(extracted) except yaml.YAMLError as e: - error_message.send(sender=None, text=f'Error parsing modules.yaml: {e}') + error_message(text=f'Error parsing modules.yaml: {e}') mlen = len(re.findall(r'---', yaml.dump(extracted.decode()))) pbar_start.send(sender=None, ptext=f'Extracting {mlen} Modules ', plen=mlen) @@ -150,10 +151,10 @@ def extract_yum_packages(data, url): i += 1 else: text = f'Error parsing Package: {name} {epoch} {version} {release} {arch}' - error_message.send(sender=None, text=text) + error_message(text=text) elem.clear() except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing yum primary.xml from {url}: {e}') + error_message(text=f'Error parsing yum primary.xml from {url}: {e}') return packages @@ -162,7 +163,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='updateinfo') if not url: - warning_message.send(sender=None, text=f'No Errata metadata found in {mirror_url}') + warning_message(text=f'No Errata metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -177,7 +178,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): if mirror.errata_checksum and mirror.errata_checksum == checksum: text = 'Mirror Errata checksum has not changed, skipping Erratum refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.errata_checksum = checksum @@ -191,7 +192,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='modules') if not url: - warning_message.send(sender=None, text=f'No Module metadata found in {mirror_url}') + warning_message(text=f'No Module metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -206,7 +207,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): if mirror.modules_checksum and mirror.modules_checksum == checksum: text = 'Mirror Modules checksum has not changed, skipping Module refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.modules_checksum = checksum @@ -220,7 +221,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='primary') if not url: - warning_message.send(sender=None, text=f'No Package metadata found in {mirror_url}') + warning_message(text=f'No Package metadata found in {mirror_url}') data = fetch_mirror_data( mirror=mirror, url=url, @@ -234,7 +235,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): if mirror.packages_checksum and mirror.packages_checksum == checksum: text = 'Mirror Packages checksum has not changed, skipping Package refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.packages_checksum = checksum diff --git a/repos/utils.py b/repos/utils.py index 49b5d07f..c11c41ad 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -26,7 +26,8 @@ from packages.models import Package from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from patchman.signals import info_message, warning_message, error_message, debug_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -77,7 +78,7 @@ def update_mirror_packages(mirror, packages): package = convert_packagestring_to_package(strpackage) mirror_package, c = MirrorPackage.objects.get_or_create(mirror=mirror, package=package) except Package.MultipleObjectsReturned: - error_message.send(sender=None, text=f'Duplicate Package found in {mirror}: {strpackage}') + error_message(text=f'Duplicate Package found in {mirror}: {strpackage}') def find_mirror_url(stored_mirror_url, formats): @@ -89,7 +90,7 @@ def find_mirror_url(stored_mirror_url, formats): if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = f"{mirror_url.rstrip('/')}/{fmt}" - debug_message.send(sender=None, text=f'Checking for Mirror at {mirror_url}') + debug_message(text=f'Checking for Mirror at {mirror_url}') try: res = get_url(mirror_url) except RetryError: @@ -133,7 +134,7 @@ def get_metalink_urls(url): if greatgreatgrandchild.attrib.get('protocol') in ['https', 'http']: metalink_urls.append(greatgreatgrandchild.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing metalink {url}: {e}') + error_message(text=f'Error parsing metalink {url}: {e}') return metalink_urls @@ -152,12 +153,12 @@ def get_mirrorlist_urls(url): return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) if mirror_urls: - debug_message.send(sender=None, text=f'Found mirrorlist: {url}') + debug_message(text=f'Found mirrorlist: {url}') return mirror_urls else: - debug_message.send(sender=None, text=f'Not a mirrorlist: {url}') + debug_message(text=f'Not a mirrorlist: {url}') except Exception as e: - error_message.send(sender=None, text=f'Error attempting to parse a mirrorlist: {e} {url}') + error_message(text=f'Error attempting to parse a mirrorlist: {e} {url}') def add_mirrors_from_urls(repo, mirror_urls): @@ -172,7 +173,7 @@ def add_mirrors_from_urls(repo, mirror_urls): existing = repo.mirror_set.filter(q).count() if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' - warning_message.send(sender=None, text=text) + warning_message(text=text) break from repos.models import Mirror # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? @@ -180,7 +181,7 @@ def add_mirrors_from_urls(repo, mirror_urls): m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) if c: text = f'Added Mirror - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) def check_for_mirrorlists(repo): @@ -193,7 +194,7 @@ def check_for_mirrorlists(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found mirrorlist - {mirror.url}') + info_message(text=f'Found mirrorlist - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -210,7 +211,7 @@ def check_for_metalinks(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found metalink - {mirror.url}') + info_message(text=f'Found metalink - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -249,9 +250,9 @@ def mirror_checksum_is_valid(computed, provided, mirror, metadata_type): """ if not computed or computed != provided: text = f'Checksum failed for mirror {mirror.id}, not refreshing {metadata_type} metadata' - error_message.send(sender=None, text=text) + error_message(text=text) text = f'Found checksum: {computed}\nExpected checksum: {provided}' - error_message.send(sender=None, text=text) + error_message(text=text) mirror.last_access_ok = False mirror.fail() return False @@ -296,9 +297,9 @@ def clean_repos(): repos = Repository.objects.filter(mirror__isnull=True) rlen = repos.count() if rlen == 0: - info_message.send(sender=None, text='No Repositories with zero Mirrors found.') + info_message(text='No Repositories with zero Mirrors found.') else: - info_message.send(sender=None, text=f'Removing {rlen} empty Repositories.') + info_message(text=f'Removing {rlen} empty Repositories.') repos.delete() @@ -309,13 +310,13 @@ def remove_mirror_trailing_slashes(): mirrors = Mirror.objects.filter(url__endswith='/') mlen = mirrors.count() if mlen == 0: - info_message.send(sender=None, text='No Mirrors with trailing slashes found.') + info_message(text='No Mirrors with trailing slashes found.') else: - info_message.send(sender=None, text=f'Removing trailing slashes from {mlen} Mirrors.') + info_message(text=f'Removing trailing slashes from {mlen} Mirrors.') for mirror in mirrors: mirror.url = mirror.url.rstrip('/') try: mirror.save() except IntegrityError: - warning_message.send(sender=None, text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') + warning_message(text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') mirror.delete() diff --git a/sbin/patchman b/sbin/patchman index c0911434..47d89b06 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from reports.models import Report from reports.tasks import clean_reports_with_no_hosts from security.utils import update_cves, update_cwes from util import set_verbosity, get_datetime_now -from patchman.signals import info_message +from util.logging import info_message def get_host(host=None, action='Performing action'): @@ -64,7 +64,7 @@ def get_host(host=None, action='Performing action'): matches = Host.objects.filter(hostname__startswith=host).count() text = f'{matches} Hosts match hostname "{host}"' - info_message.send(sender=None, text=text) + info_message(text=text) return host_obj @@ -84,7 +84,7 @@ def get_hosts(hosts=None, action='Performing action'): host_objs.append(host_obj) else: text = f'{action} for all Hosts\n' - info_message.send(sender=None, text=text) + info_message(text=text) host_objs = Host.objects.all() return host_objs @@ -107,7 +107,7 @@ def get_repos(repo=None, action='Performing action', only_enabled=False): else: repos = Repository.objects.all() - info_message.send(sender=None, text=text) + info_message(text=text) return repos @@ -118,9 +118,9 @@ def refresh_repos(repo=None, force=False): repos = get_repos(repo, 'Refreshing metadata', True) for repo in repos: text = f'Repository {repo.id} : {repo}' - info_message.send(sender=None, text=text) + info_message(text=text) repo.refresh(force) - info_message.send(sender=None, text='') + info_message(text='') def list_repos(repos=None): @@ -161,10 +161,10 @@ def host_updates_alt(host=None): hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) if host not in updated_hosts: host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') host.updated_at = ts host.save() @@ -200,10 +200,10 @@ def host_updates_alt(host=None): phost.save() updated_hosts.append(phost) text = f'Added the same updates to {phost}' - info_message.send(sender=None, text=text) + info_message(text=text) else: text = 'Updates already added in this run' - info_message.send(sender=None, text=text) + info_message(text=text) def host_updates(host=None): @@ -211,9 +211,9 @@ def host_updates(host=None): """ hosts = get_hosts(host, 'Finding updates') for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') def diff_hosts(hosts): @@ -236,47 +236,47 @@ def diff_hosts(hosts): repo_diff_AB = reposA.difference(reposB) repo_diff_BA = reposB.difference(reposA) - info_message.send(sender=None, text=f'+ {hostA.hostname}') - info_message.send(sender=None, text=f'- {hostB.hostname}') + info_message(text=f'+ {hostA.hostname}') + info_message(text=f'- {hostB.hostname}') if hostA.os != hostB.os: - info_message.send(sender=None, text='\nOperating Systems') - info_message.send(sender=None, text=f'+ {hostA.os}') - info_message.send(sender=None, text=f'- {hostB.os}') + info_message(text='\nOperating Systems') + info_message(text=f'+ {hostA.os}') + info_message(text=f'- {hostB.os}') else: - info_message.send(sender=None, text='\nNo OS differences') + info_message(text='\nNo OS differences') if hostA.arch != hostB.arch: - info_message.send(sender=None, text='\nArchitecture') - info_message.send(sender=None, text=f'+ {hostA.arch}') - info_message.send(sender=None, text=f'- {hostB.arch}') + info_message(text='\nArchitecture') + info_message(text=f'+ {hostA.arch}') + info_message(text=f'- {hostB.arch}') else: - info_message.send(sender=None, text='\nNo Architecture differences') + info_message(text='\nNo Architecture differences') if hostA.kernel != hostB.kernel: - info_message.send(sender=None, text='\nKernels') - info_message.send(sender=None, text=f'+ {hostA.kernel}') - info_message.send(sender=None, text=f'- {hostB.kernel}') + info_message(text='\nKernels') + info_message(text=f'+ {hostA.kernel}') + info_message(text=f'- {hostB.kernel}') else: - info_message.send(sender=None, text='\nNo Kernel differences') + info_message(text='\nNo Kernel differences') if len(package_diff_AB) != 0 or len(package_diff_BA) != 0: - info_message.send(sender=None, text='\nPackages') + info_message(text='\nPackages') for package in package_diff_AB: - info_message.send(sender=None, text=f'+ {package}') + info_message(text=f'+ {package}') for package in package_diff_BA: - info_message.send(sender=None, text=f'- {package}') + info_message(text=f'- {package}') else: - info_message.send(sender=None, text='\nNo Package differences') + info_message(text='\nNo Package differences') if len(repo_diff_AB) != 0 or len(repo_diff_BA) != 0: - info_message.send(sender=None, text='\nRepositories') + info_message(text='\nRepositories') for repo in repo_diff_AB: - info_message.send(sender=None, text=f'+ {repo}') + info_message(text=f'+ {repo}') for repo in repo_diff_BA: - info_message.send(sender=None, text=f'- {repo}') + info_message(text=f'- {repo}') else: - info_message.send(sender=None, text='\nNo Repo differences') + info_message(text='\nNo Repo differences') def delete_hosts(hosts=None): @@ -286,7 +286,7 @@ def delete_hosts(hosts=None): matching_hosts = get_hosts(hosts) for host in matching_hosts: text = f'Deleting host: {host.hostname}:' - info_message.send(sender=None, text=text) + info_message(text=text) host.delete() @@ -300,7 +300,7 @@ def toggle_host_hro(hosts=None, host_repos_only=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} host_repos_only') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.host_repos_only = host_repos_only host.save() @@ -315,7 +315,7 @@ def toggle_host_check_dns(hosts=None, check_dns=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} check_dns') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.check_dns = check_dns host.save() @@ -347,7 +347,7 @@ def process_reports(host=None, force=False): text = 'Processing Reports for all Hosts' reports = Report.objects.filter(processed=force).order_by('created') - info_message.send(sender=None, text=text) + info_message(text=text) for report in reports: report.process(find_updates=False) diff --git a/security/models.py b/security/models.py index 9c097eed..7f674a0b 100644 --- a/security/models.py +++ b/security/models.py @@ -152,7 +152,7 @@ def fetch_mitre_cve_data(self): mitre_cve_url = f'https://cveawg.mitre.org/api/cve/{self.cve_id}' res = get_url(mitre_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} MITRE data') cve_json = json.loads(data) @@ -162,7 +162,7 @@ def fetch_osv_dev_cve_data(self): osv_dev_cve_url = f'https://api.osv.dev/v1/vulns/{self.cve_id}' res = get_url(osv_dev_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} OSV data') cve_json = json.loads(data) @@ -186,7 +186,7 @@ def fetch_nist_cve_data(self): res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') cve_json = json.loads(data) self.parse_nist_cve_data(cve_json) @@ -197,7 +197,7 @@ def parse_nist_cve_data(self, cve_json): cve = vulnerability.get('cve') cve_id = cve.get('id') if cve_id != self.cve_id: - error_message.send(sender=None, text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') + error_message(text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') return metrics = cve.get('metrics') for metric, score_data in metrics.items(): diff --git a/util/__init__.py b/util/__init__.py index 4a3f9caa..c3dfd6bd 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -38,7 +38,7 @@ from django.utils.dateparse import parse_datetime from django.conf import settings -from patchman.signals import error_message, info_message, debug_message +from util.logging import error_message, info_message, debug_message pbar = None verbose = None @@ -109,7 +109,7 @@ def fetch_content(response, text='', ljust=35): data += chunk return data else: - info_message.send(sender=None, text=text) + info_message(text=text) return response.content @@ -128,16 +128,16 @@ def get_url(url, headers=None, params=None): if not params: params = {} try: - debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') + debug_message(text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) - debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') + debug_message(text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response response.raise_for_status() except requests.exceptions.TooManyRedirects: - error_message.send(sender=None, text=f'Too many redirects - {url}') + error_message(text=f'Too many redirects - {url}') except ConnectionError: - error_message.send(sender=None, text=f'Connection error - {url}') + error_message(text=f'Connection error - {url}') return response @@ -180,7 +180,7 @@ def gunzip(contents): wbits = zlib.MAX_WBITS | 32 return zlib.decompress(contents, wbits) except zlib.error as e: - error_message.send(sender=None, text='gunzip: ' + str(e)) + error_message(text='gunzip: ' + str(e)) def bunzip2(contents): @@ -191,10 +191,10 @@ def bunzip2(contents): return bzip2data except IOError as e: if e == 'invalid data stream': - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) except ValueError as e: if e == "couldn't find end of stream": - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) def unxz(contents): @@ -204,7 +204,7 @@ def unxz(contents): xzdata = lzma.decompress(contents) return xzdata except lzma.LZMAError as e: - error_message.send(sender=None, text='lzma: ' + e) + error_message(text='lzma: ' + e) def unzstd(contents): @@ -214,7 +214,7 @@ def unzstd(contents): zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() return zstddata except zstd.ZstdError as e: - error_message.send(sender=None, text='zstd: ' + e) + error_message(text=f'zstd: {e}') def extract(data, fmt): @@ -253,7 +253,7 @@ def get_checksum(data, checksum_type): checksum = get_md5(data) else: text = f'Unknown checksum type: {checksum_type}' - error_message.send(sender=None, text=text) + error_message(text=text) return checksum diff --git a/util/logging.py b/util/logging.py new file mode 100644 index 00000000..dd79d296 --- /dev/null +++ b/util/logging.py @@ -0,0 +1,42 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from datetime import datetime + +from patchman.signals import info_message_s +from patchman.signals import warning_message_s +from patchman.signals import error_message_s +from patchman.signals import debug_message_s + + +def info_message(text): + ts = datetime.now() + info_message_s.send(sender=None, text=text, ts=ts) + + +def warning_message(text): + ts = datetime.now() + warning_message_s.send(sender=None, text=text, ts=ts) + + +def debug_message(text): + ts = datetime.now() + debug_message_s.send(sender=None, text=text, ts=ts) + + +def error_message(text): + ts = datetime.now() + error_message_s.send(sender=None, text=text, ts=ts) From 00fbd6e42581ffedf93cdfedd47154cb6ffdeea2 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:50 -0400 Subject: [PATCH 102/120] use redis for caching and use locks for tasks --- errata/tasks.py | 68 ++++++++++++++++++++-------------- etc/patchman/local_settings.py | 16 +++----- reports/tasks.py | 15 +++++++- repos/tasks.py | 17 ++++++++- security/tasks.py | 33 ++++++++++++++--- 5 files changed, 101 insertions(+), 48 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index f1d6eeee..d9aaf3d4 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -16,13 +16,15 @@ from celery import shared_task +from django.core.cache import cache + from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message +from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type @@ -44,34 +46,44 @@ def update_yum_repo_errata(repo_id=None, force=False): def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ - errata_os_updates = [] - erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] - erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] - if erratum_type: - if erratum_type not in erratum_types: - error_message.send(sender=None, text=f'Erratum type `{erratum_type}` not in {erratum_types}') - else: - errata_os_updates = erratum_type + lock_key = 'update_errata_lock' + # lock will expire after 48 hours + lock_expire = 60 * 60 * 48 + + if cache.add(lock_key, 'true', lock_expire): + try: + errata_os_updates = [] + erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] + erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + if erratum_type: + if erratum_type not in erratum_types: + error_message(text=f'Erratum type `{erratum_type}` not in {erratum_types}') + else: + errata_os_updates = erratum_type + else: + errata_os_updates = get_setting_of_type( + setting_name='ERRATA_OS_UPDATES', + setting_type=list, + default=erratum_type_defaults, + ) + if 'yum' in errata_os_updates: + update_yum_repo_errata(repo_id=repo, force=force) + if 'arch' in errata_os_updates: + update_arch_errata() + if 'alma' in errata_os_updates: + update_alma_errata() + if 'rocky' in errata_os_updates: + update_rocky_errata() + if 'debian' in errata_os_updates: + update_debian_errata() + if 'ubuntu' in errata_os_updates: + update_ubuntu_errata() + if 'centos' in errata_os_updates: + update_centos_errata() + finally: + cache.delete(lock_key) else: - errata_os_updates = get_setting_of_type( - setting_name='ERRATA_OS_UPDATES', - setting_type=list, - default=erratum_type_defaults, - ) - if 'yum' in errata_os_updates: - update_yum_repo_errata(repo_id=repo, force=force) - if 'arch' in errata_os_updates: - update_arch_errata() - if 'alma' in errata_os_updates: - update_alma_errata() - if 'rocky' in errata_os_updates: - update_rocky_errata() - if 'debian' in errata_os_updates: - update_debian_errata() - if 'ubuntu' in errata_os_updates: - update_ubuntu_errata() - if 'centos' in errata_os_updates: - update_centos_errata() + warning_message('Already updating Errata, skipping task.') @shared_task diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..9e7ca21b 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -44,22 +44,16 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', + 'TIMEOUT': 0, } } -# Uncomment to enable redis caching for e.g. 30 seconds -# Note that the UI results may be out of date for this amount of time -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.redis.RedisCache', -# 'LOCATION': 'redis://127.0.0.1:6379', -# 'TIMEOUT': 30, -# } -# } - from datetime import timedelta # noqa from celery.schedules import crontab # noqa CELERY_BEAT_SCHEDULE = { diff --git a/reports/tasks.py b/reports/tasks.py index fe294e8d..d2a47e8f 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -17,11 +17,12 @@ from celery import shared_task +from django.core.cache import cache from django.db.utils import OperationalError from hosts.models import Host from reports.models import Report -from util.logging import info_message +from util.logging import info_message, warning_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -29,7 +30,17 @@ def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - report.process() + lock_key = f'process_report_lock_{report_id}' + # lock will expire after 1 hour + lock_expire = 60 * 60 + + if cache.add(lock_key, 'true', lock_expire): + try: + report.process() + finally: + cache.delete(lock_key) + else: + warning_message(f'Already processing report {report_id}, skipping task.') @shared_task diff --git a/repos/tasks.py b/repos/tasks.py index 39098fa8..436da82a 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from repos.models import Repository +from util.logging import warning_message @shared_task @@ -32,5 +35,15 @@ def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ repos = Repository.objects.filter(enabled=True) - for repo in repos: - refresh_repo.delay(repo.id, force) + lock_key = 'refresh_repos_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(lock_key, 'true', lock_expire): + try: + for repo in repos: + refresh_repo.delay(repo.id, force) + finally: + cache.delete(lock_key) + else: + warning_message('Already refreshing repos, skipping task.') diff --git a/security/tasks.py b/security/tasks.py index a04bb1c8..ce60df83 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from security.models import CVE, CWE +from util.logging import warning_message @shared_task @@ -31,8 +34,18 @@ def update_cve(cve_id): def update_cves(): """ Task to update all CVEs """ - for cve in CVE.objects.all(): - update_cve.delay(cve.id) + lock_key = 'update_cves_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cve in CVE.objects.all(): + update_cve.delay(cve.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CVEs, skipping task.') @shared_task @@ -45,7 +58,17 @@ def update_cwe(cwe_id): @shared_task def update_cwes(): - """ Task to update all CWEa + """ Task to update all CWEs """ - for cwe in CWE.objects.all(): - update_cwe.delay(cwe.id) + lock_key = 'update_cwes_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cwe in CWE.objects.all(): + update_cwe.delay(cwe.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CWEs, skipping task.') From fd6f9aadf43f8e988967bbdc2efe0b3dc1665f94 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 18:38:04 -0400 Subject: [PATCH 103/120] add errata source options to config file --- errata/sources/distros/debian.py | 2 +- errata/sources/distros/ubuntu.py | 2 +- etc/patchman/local_settings.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 1ae919e4..ece3754d 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -264,7 +264,7 @@ def get_accepted_debian_codenames(): """ Get acceptable Debian OS codenames Can be overridden by specifying DEBIAN_CODENAMES in settings """ - default_codenames = ['bookworm', 'bullseye'] + default_codenames = ['bookworm', 'trixie'] accepted_codenames = get_setting_of_type( setting_name='DEBIAN_CODENAMES', setting_type=list, diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index d1ce7cc5..6fafb40a 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -203,7 +203,7 @@ def get_accepted_ubuntu_codenames(): """ Get acceptable Ubuntu OS codenames Can be overridden by specifying UBUNTU_CODENAMES in settings """ - default_codenames = ['focal', 'jammy', 'noble'] + default_codenames = ['jammy', 'noble'] accepted_codenames = get_setting_of_type( setting_name='UBUNTU_CODENAMES', setting_type=list, diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..ab93c89d 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -41,6 +41,18 @@ # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14 +# list of errata sources to update, remove unwanted ones to improve performance +ERRATA_OS_UPDATES = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + +# list of Alma Linux releases to update +ALMA_RELEASES = [8, 9, 10] + +# list of Debian Linux releases to update +DEBIAN_CODENAMES = ['bookworm', 'trixie'] + +# list of Ubuntu Linux releases to update +UBUNTU_CODENAMES = ['jammy', 'noble'] + # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False From f06729e89aafa10910a421a21761d60d54345896 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 14 May 2025 22:39:29 -0400 Subject: [PATCH 104/120] remove daily cronjob in favour of patchman-celery --- debian/python3-patchman.cron.daily | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 debian/python3-patchman.cron.daily diff --git a/debian/python3-patchman.cron.daily b/debian/python3-patchman.cron.daily deleted file mode 100644 index d4752f75..00000000 --- a/debian/python3-patchman.cron.daily +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/usr/bin/patchman -a -q From a59a23b5e26b3e2c6436c140055bf55a1b47c712 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 30 Oct 2025 22:02:44 -0400 Subject: [PATCH 105/120] add isort check --- .github/workflows/lint-and-test.yml | 4 +++ arch/admin.py | 3 ++- arch/serializers.py | 2 +- arch/utils.py | 2 +- arch/views.py | 7 ++--- domains/admin.py | 1 + errata/admin.py | 1 + errata/models.py | 7 +++-- errata/sources/distros/alma.py | 2 +- errata/sources/distros/arch.py | 10 ++++--- errata/sources/distros/centos.py | 7 ++--- errata/sources/distros/debian.py | 8 +++--- errata/sources/distros/rocky.py | 12 +++++---- errata/sources/distros/ubuntu.py | 13 ++++++--- errata/sources/repos/yum.py | 4 +-- errata/tasks.py | 7 +++-- errata/utils.py | 4 +-- errata/views.py | 7 +++-- etc/patchman/local_settings.py | 4 ++- hooks/yum/patchman.py | 1 + hooks/zypper/patchman.py | 3 ++- hosts/admin.py | 1 + hosts/migrations/0001_initial.py | 3 ++- hosts/migrations/0002_initial.py | 2 +- .../0004_remove_host_tags_host_tags.py | 3 ++- hosts/migrations/0006_migrate_to_tz_aware.py | 1 + hosts/migrations/0007_alter_host_tags.py | 2 +- hosts/models.py | 3 ++- hosts/tasks.py | 1 - hosts/templatetags/report_alert.py | 2 +- hosts/utils.py | 4 +-- hosts/views.py | 23 ++++++++-------- modules/admin.py | 1 + modules/migrations/0001_initial.py | 2 +- modules/utils.py | 4 +-- modules/views.py | 7 +++-- operatingsystems/admin.py | 3 ++- operatingsystems/forms.py | 4 +-- operatingsystems/migrations/0002_initial.py | 2 +- operatingsystems/migrations/0003_os_arch.py | 2 +- operatingsystems/serializers.py | 2 +- operatingsystems/views.py | 17 +++++++----- packages/admin.py | 1 + packages/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20250207_1319.py | 2 +- packages/serializers.py | 2 +- packages/utils.py | 4 ++- packages/views.py | 13 ++++----- patchman/__init__.py | 3 +-- patchman/celery.py | 3 ++- patchman/receivers.py | 16 +++++------ patchman/urls.py | 3 +-- patchman/wsgi.py | 3 +-- reports/admin.py | 1 + .../migrations/0004_migrate_to_tz_aware.py | 1 + reports/models.py | 6 +++-- reports/tasks.py | 1 - reports/utils.py | 13 ++++++--- reports/views.py | 19 ++++++------- repos/admin.py | 3 ++- repos/forms.py | 7 +++-- repos/migrations/0001_initial.py | 2 +- repos/migrations/0003_migrate_to_tz_aware.py | 1 + repos/models.py | 9 +++---- repos/repo_types/arch.py | 9 ++++--- repos/repo_types/deb.py | 9 ++++--- repos/repo_types/gentoo.py | 16 +++++++---- repos/repo_types/rpm.py | 7 +++-- repos/repo_types/yast.py | 2 +- repos/repo_types/yum.py | 5 ++-- repos/serializers.py | 2 +- repos/tasks.py | 1 - repos/utils.py | 18 +++++++++---- repos/views.py | 27 ++++++++++--------- sbin/patchman | 21 +++++++++------ security/admin.py | 2 +- security/models.py | 4 +-- security/tasks.py | 1 - security/views.py | 11 ++++---- setup.py | 3 ++- util/__init__.py | 23 +++++++++------- util/filterspecs.py | 5 ++-- util/logging.py | 8 +++--- util/tasks.py | 4 ++- util/templatetags/common.py | 7 +++-- util/views.py | 6 ++--- 86 files changed, 292 insertions(+), 212 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 20b221aa..436698be 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,10 @@ jobs: pip install flake8 flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --max-line-length=120 --show-source --statistics + - name: Check isort + run: | + pip install isort + isort --check --profile=django . - name: Set secret key run: ./sbin/patchman-set-secret-key - name: Test with django diff --git a/arch/admin.py b/arch/admin.py index 624a3720..5224711c 100644 --- a/arch/admin.py +++ b/arch/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from arch.models import PackageArchitecture, MachineArchitecture + +from arch.models import MachineArchitecture, PackageArchitecture admin.site.register(PackageArchitecture) admin.site.register(MachineArchitecture) diff --git a/arch/serializers.py b/arch/serializers.py index 5319e796..a5765128 100644 --- a/arch/serializers.py +++ b/arch/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer): diff --git a/arch/utils.py b/arch/utils.py index 04d0b350..3db6ac70 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture from util.logging import info_message diff --git a/arch/views.py b/arch/views.py index 56a2a150..21f6b7c7 100644 --- a/arch/views.py +++ b/arch/views.py @@ -16,9 +16,10 @@ from rest_framework import viewsets -from arch.models import PackageArchitecture, MachineArchitecture -from arch.serializers import PackageArchitectureSerializer, \ - MachineArchitectureSerializer +from arch.models import MachineArchitecture, PackageArchitecture +from arch.serializers import ( + MachineArchitectureSerializer, PackageArchitectureSerializer, +) class PackageArchitectureViewSet(viewsets.ModelViewSet): diff --git a/domains/admin.py b/domains/admin.py index 2ef883e3..5cb0fee3 100644 --- a/domains/admin.py +++ b/domains/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from domains.models import Domain admin.site.register(Domain) diff --git a/errata/admin.py b/errata/admin.py index 88190ff6..ac4b8a50 100644 --- a/errata/admin.py +++ b/errata/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from errata.models import Erratum diff --git a/errata/models.py b/errata/models.py index cfc9bd0d..8c21bcfa 100644 --- a/errata/models.py +++ b/errata/models.py @@ -16,17 +16,16 @@ import json -from django.db import models +from django.db import IntegrityError, models from django.urls import reverse -from django.db import IntegrityError +from errata.managers import ErratumManager from packages.models import Package, PackageUpdate from packages.utils import find_evr, get_matching_packages -from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from util.logging import error_message from util import get_url +from util.logging import error_message class Erratum(models.Model): diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index e0f2d4ae..0091b8bf 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -22,8 +22,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string -from util import get_url, fetch_content, get_setting_of_type from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_setting_of_type, get_url def update_alma_errata(concurrent_processing=True): diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 87c6c47a..e22de403 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,11 +20,13 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from util.logging import error_message -from patchman.signals import pbar_start, pbar_update from packages.models import Package -from packages.utils import find_evr, get_matching_packages, get_or_create_package -from util import get_url, fetch_content +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, +) +from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_url +from util.logging import error_message def update_arch_errata(concurrent_processing=False): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index d2722a6b..8f4aa4a1 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -15,14 +15,15 @@ # along with Patchman. If not, see import re + from defusedxml import ElementTree from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package -from util.logging import error_message +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type +from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url +from util.logging import error_message def update_centos_errata(): diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index ece3754d..8025b1bf 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -18,18 +18,18 @@ import csv import re from datetime import datetime -from debian.deb822 import Dsc from io import StringIO +from debian.deb822 import Dsc from django.db import connections from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, find_evr -from util.logging import error_message, warning_message +from packages.utils import find_evr, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, get_setting_of_type, extract +from util import extract, fetch_content, get_setting_of_type, get_url +from util.logging import error_message, warning_message DSCs = {} diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 16d4d12c..2d805985 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -14,19 +14,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import json import concurrent.futures -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +import json from django.db import connections from django.db.utils import OperationalError +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content -from util.logging import info_message, error_message +from util import fetch_content, get_url +from util.logging import error_message, info_message def update_rocky_errata(concurrent_processing=True): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 6fafb40a..5616331f 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -16,8 +16,8 @@ import concurrent.futures import csv -import os import json +import os from io import StringIO from urllib.parse import urlparse @@ -26,10 +26,15 @@ from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages -from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from util.logging import error_message +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update +from util import ( + bunzip2, fetch_content, get_setting_of_type, get_sha256, get_url, +) +from util.logging import error_message def update_ubuntu_errata(concurrent_processing=False): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index f361d10e..8b6732c4 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -16,17 +16,17 @@ import concurrent.futures from io import BytesIO -from defusedxml import ElementTree +from defusedxml import ElementTree from django.db import connections from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from util.logging import error_message from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url +from util.logging import error_message def extract_updateinfo(data, url, concurrent_processing=True): diff --git a/errata/tasks.py b/errata/tasks.py index d9aaf3d4..8ded3a79 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -15,19 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache -from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata -from errata.sources.distros.debian import update_debian_errata +from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.centos import update_centos_errata +from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type +from util.logging import error_message, warning_message @shared_task diff --git a/errata/utils.py b/errata/utils.py index a8d8d424..e0a5e01b 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -18,11 +18,11 @@ from django.db import connections -from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from util.logging import warning_message from patchman.signals import pbar_start, pbar_update +from util import tz_aware_datetime +from util.logging import warning_message def get_or_create_erratum(name, e_type, issue_date, synopsis): diff --git a/errata/views.py b/errata/views.py index 42d12f71..8e1c0b2f 100644 --- a/errata/views.py +++ b/errata/views.py @@ -14,16 +14,15 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from operatingsystems.models import OSRelease from errata.models import Erratum from errata.serializers import ErratumSerializer +from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index f7b98cf8..40dd2efd 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -66,8 +66,10 @@ } } -from datetime import timedelta # noqa +from datetime import timedelta # noqa + from celery.schedules import crontab # noqa + CELERY_BEAT_SCHEDULE = { 'process_all_unprocessed_reports': { 'task': 'reports.tasks.process_reports', diff --git a/hooks/yum/patchman.py b/hooks/yum/patchman.py index 343144eb..52f9cc8b 100644 --- a/hooks/yum/patchman.py +++ b/hooks/yum/patchman.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see import os + from yum.plugins import TYPE_CORE requires_api_version = '2.1' diff --git a/hooks/zypper/patchman.py b/hooks/zypper/patchman.py index 14781565..d9d478f3 100755 --- a/hooks/zypper/patchman.py +++ b/hooks/zypper/patchman.py @@ -18,8 +18,9 @@ # # zypp system plugin for patchman -import os import logging +import os + from zypp_plugin import Plugin diff --git a/hosts/admin.py b/hosts/admin.py index 8a42e8cc..43bf31da 100644 --- a/hosts/admin.py +++ b/hosts/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from hosts.models import Host, HostRepo diff --git a/hosts/migrations/0001_initial.py b/hosts/migrations/0001_initial.py index 43366684..0037e094 100644 --- a/hosts/migrations/0001_initial.py +++ b/hosts/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.db import migrations, models + try: import tagging.fields has_tagging = True diff --git a/hosts/migrations/0002_initial.py b/hosts/migrations/0002_initial.py index cc59a70e..6c453c49 100644 --- a/hosts/migrations/0002_initial.py +++ b/hosts/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/hosts/migrations/0004_remove_host_tags_host_tags.py b/hosts/migrations/0004_remove_host_tags_host_tags.py index 84e7affe..bf03a84e 100644 --- a/hosts/migrations/0004_remove_host_tags_host_tags.py +++ b/hosts/migrations/0004_remove_host_tags_host_tags.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.18 on 2025-02-04 23:37 +import taggit.managers from django.apps import apps from django.db import migrations -import taggit.managers + try: import tagging # noqa except ImportError: diff --git a/hosts/migrations/0006_migrate_to_tz_aware.py b/hosts/migrations/0006_migrate_to_tz_aware.py index e36bbf1f..c14ea50b 100644 --- a/hosts/migrations/0006_migrate_to_tz_aware.py +++ b/hosts/migrations/0006_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Host = apps.get_model('hosts', 'Host') for host in Host.objects.all(): diff --git a/hosts/migrations/0007_alter_host_tags.py b/hosts/migrations/0007_alter_host_tags.py index 3858b847..3910a06f 100644 --- a/hosts/migrations/0007_alter_host_tags.py +++ b/hosts/migrations/0007_alter_host_tags.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.19 on 2025-02-28 19:53 -from django.db import migrations import taggit.managers +from django.db import migrations class Migration(migrations.Migration): diff --git a/hosts/models.py b/hosts/models.py index 650544dc..8ea5e3d5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -24,6 +24,7 @@ from version_utils.rpm import labelCompare except ImportError: from rpm import labelCompare + from taggit.managers import TaggableManager from arch.models import MachineArchitecture @@ -34,9 +35,9 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo +from util.logging import info_message class Host(models.Model): diff --git a/hosts/tasks.py b/hosts/tasks.py index 1643901d..226652f6 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.db.models import Count from hosts.models import Host diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index a28c5058..48d8f966 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,9 +17,9 @@ from datetime import timedelta from django.template import Library -from django.utils.html import format_html from django.templatetags.static import static from django.utils import timezone +from django.utils.html import format_html from util import get_setting_of_type diff --git a/hosts/utils.py b/hosts/utils.py index d6e663cf..44441f9b 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from socket import gethostbyaddr, gaierror, herror +from socket import gaierror, gethostbyaddr, herror -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from taggit.models import Tag from util.logging import error_message, info_message diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffa..8f20ab19 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages - -from taggit.models import Tag +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets +from taggit.models import Tag -from util.filterspecs import Filter, FilterBar -from hosts.models import Host, HostRepo -from domains.models import Domain from arch.models import MachineArchitecture -from operatingsystems.models import OSVariant, OSRelease -from reports.models import Report +from domains.models import Domain from hosts.forms import EditHostForm -from hosts.serializers import HostSerializer, HostRepoSerializer +from hosts.models import Host, HostRepo +from hosts.serializers import HostRepoSerializer, HostSerializer +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from util.filterspecs import Filter, FilterBar @login_required diff --git a/modules/admin.py b/modules/admin.py index 33b94d20..9cf21e6c 100644 --- a/modules/admin.py +++ b/modules/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from modules.models import Module admin.site.register(Module) diff --git a/modules/migrations/0001_initial.py b/modules/migrations/0001_initial.py index 12a8e278..9c27d425 100644 --- a/modules/migrations/0001_initial.py +++ b/modules/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/modules/utils.py b/modules/utils.py index 05c57c80..0d669478 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,10 +15,10 @@ # along with Patchman. If not, see from django.db import IntegrityError -from util.logging import error_message, info_message -from modules.models import Module from arch.models import PackageArchitecture +from modules.models import Module +from util.logging import error_message, info_message def get_or_create_module(name, stream, version, context, arch, repo): diff --git a/modules/views.py b/modules/views.py index b897a709..2d017220 100644 --- a/modules/views.py +++ b/modules/views.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - -from rest_framework import viewsets, permissions +from django.shortcuts import get_object_or_404, render +from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer diff --git a/operatingsystems/admin.py b/operatingsystems/admin.py index 15f5e200..4884b5eb 100644 --- a/operatingsystems/admin.py +++ b/operatingsystems/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from operatingsystems.models import OSVariant, OSRelease + +from operatingsystems.models import OSRelease, OSVariant class OSReleaseAdmin(admin.ModelAdmin): diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index 548a7d88..fa319182 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ModelForm, ModelMultipleChoiceField -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant from repos.models import Repository diff --git a/operatingsystems/migrations/0002_initial.py b/operatingsystems/migrations/0002_initial.py index 517a3f9a..04cbb411 100644 --- a/operatingsystems/migrations/0002_initial.py +++ b/operatingsystems/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/migrations/0003_os_arch.py b/operatingsystems/migrations/0003_os_arch.py index 2778ca3f..4d0e0f93 100644 --- a/operatingsystems/migrations/0003_os_arch.py +++ b/operatingsystems/migrations/0003_os_arch.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:02 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/serializers.py b/operatingsystems/serializers.py index 8418c720..be178909 100644 --- a/operatingsystems/serializers.py +++ b/operatingsystems/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant class OSVariantSerializer(serializers.HyperlinkedModelSerializer): diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f92..6009f119 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,19 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse - from rest_framework import viewsets from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from operatingsystems.forms import AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm -from operatingsystems.serializers import OSVariantSerializer, OSReleaseSerializer +from operatingsystems.forms import ( + AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm, +) +from operatingsystems.models import OSRelease, OSVariant +from operatingsystems.serializers import ( + OSReleaseSerializer, OSVariantSerializer, +) @login_required diff --git a/packages/admin.py b/packages/admin.py index 979ba779..bc4b1aaa 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from packages.models import Package, PackageName, PackageUpdate diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 07e7bcb0..bfea3e1a 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/migrations/0002_auto_20250207_1319.py b/packages/migrations/0002_auto_20250207_1319.py index 1563d139..4c744203 100644 --- a/packages/migrations/0002_auto_20250207_1319.py +++ b/packages/migrations/0002_auto_20250207_1319.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:19 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/serializers.py b/packages/serializers.py index 902cb3e0..b6e3fb83 100644 --- a/packages/serializers.py +++ b/packages/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from packages.models import PackageName, Package, PackageUpdate +from packages.models import Package, PackageName, PackageUpdate class PackageNameSerializer(serializers.HyperlinkedModelSerializer): diff --git a/packages/utils.py b/packages/utils.py index f00f6710..87395ff6 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -21,7 +21,9 @@ from django.db import IntegrityError, transaction from arch.models import PackageArchitecture -from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString +from packages.models import ( + Package, PackageCategory, PackageName, PackageString, PackageUpdate, +) from util.logging import error_message, info_message, warning_message diff --git a/packages/views.py b/packages/views.py index c55a6c72..413faee0 100644 --- a/packages/views.py +++ b/packages/views.py @@ -15,17 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar -from packages.models import PackageName, Package, PackageUpdate from arch.models import PackageArchitecture -from packages.serializers import PackageNameSerializer, PackageSerializer, PackageUpdateSerializer +from packages.models import Package, PackageName, PackageUpdate +from packages.serializers import ( + PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/patchman/__init__.py b/patchman/__init__.py index af122cc6..321dd7e5 100644 --- a/patchman/__init__.py +++ b/patchman/__init__.py @@ -14,10 +14,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from .receivers import * # noqa - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app +from .receivers import * # noqa __all__ = ('celery_app',) diff --git a/patchman/celery.py b/patchman/celery.py index 3c58edc5..c47f994d 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -15,10 +15,11 @@ # along with Patchman. If not, see import os + from celery import Celery os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa +from django.conf import settings # noqa app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/patchman/receivers.py b/patchman/receivers.py index 8d8893ca..19312ed5 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -15,16 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from colorama import init, Fore, Style -from tqdm import tqdm - +from colorama import Fore, Style, init +from django.conf import settings from django.dispatch import receiver +from tqdm import tqdm -from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, \ - info_message_s, warning_message_s, error_message_s, debug_message_s - -from django.conf import settings +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, + warning_message_s, +) +from util import create_pbar, get_verbosity, update_pbar init(autoreset=True) diff --git a/patchman/urls.py b/patchman/urls.py index ee786566..2ae64f56 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -15,12 +15,11 @@ # You should have received a copy of the GNU General Public License # along with If not, see -from django.conf.urls import include, handler404, handler500 # noqa from django.conf import settings +from django.conf.urls import handler404, handler500, include # noqa from django.contrib import admin from django.urls import path from django.views import static - from rest_framework import routers from arch import views as arch_views diff --git a/patchman/wsgi.py b/patchman/wsgi.py index 9a9b4b7f..16f02d5a 100644 --- a/patchman/wsgi.py +++ b/patchman/wsgi.py @@ -19,7 +19,6 @@ from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa - +from django.conf import settings # noqa application = get_wsgi_application() diff --git a/reports/admin.py b/reports/admin.py index a37ec4d0..66e0bf5b 100644 --- a/reports/admin.py +++ b/reports/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from reports.models import Report diff --git a/reports/migrations/0004_migrate_to_tz_aware.py b/reports/migrations/0004_migrate_to_tz_aware.py index 98176510..20510dcd 100644 --- a/reports/migrations/0004_migrate_to_tz_aware.py +++ b/reports/migrations/0004_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Report = apps.get_model('reports', 'Report') for report in Report.objects.all(): diff --git a/reports/models.py b/reports/models.py index d529804b..f1e0f6f3 100644 --- a/reports/models.py +++ b/reports/models.py @@ -104,7 +104,7 @@ def process(self, find_updates=True, verbose=False): info_message(text=f'Report {self.id} has already been processed') return - from reports.utils import get_arch, get_os, get_domain + from reports.utils import get_arch, get_domain, get_os arch = get_arch(self.arch) osvariant = get_os(self.os, arch) domain = get_domain(self.domain) @@ -113,7 +113,9 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import process_packages, process_repos, process_updates, process_modules + from reports.utils import ( + process_modules, process_packages, process_repos, process_updates, + ) process_repos(report=self, host=host) process_modules(report=self, host=host) process_packages(report=self, host=host) diff --git a/reports/tasks.py b/reports/tasks.py index d2a47e8f..07fb3004 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -16,7 +16,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from django.db.utils import OperationalError diff --git a/reports/utils.py b/reports/utils.py index 76b6e09c..8b12f046 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -23,13 +23,18 @@ from domains.models import Domain from hosts.models import HostRepo from modules.utils import get_or_create_module -from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant +from operatingsystems.utils import ( + get_or_create_osrelease, get_or_create_osvariant, +) from packages.models import Package, PackageCategory -from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from util.logging import info_message +from packages.utils import ( + find_evr, get_or_create_package, get_or_create_package_update, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update -from repos.models import Repository, Mirror, MirrorPackage +from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo +from util.logging import info_message def process_repos(report, host): diff --git a/reports/views.py b/reports/views.py index ccef1bb2..f247deb2 100644 --- a/reports/views.py +++ b/reports/views.py @@ -15,20 +15,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -from django.http import HttpResponse, Http404 -from django.views.decorators.csrf import csrf_exempt -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages from django.db.utils import OperationalError +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from util.filterspecs import Filter, FilterBar from reports.models import Report +from util.filterspecs import Filter, FilterBar @retry( diff --git a/repos/admin.py b/repos/admin.py index bea87567..a516ff8a 100644 --- a/repos/admin.py +++ b/repos/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from repos.models import Repository, Mirror, MirrorPackage + +from repos.models import Mirror, MirrorPackage, Repository class MirrorAdmin(admin.ModelAdmin): diff --git a/repos/forms.py b/repos/forms.py index 0800a5c3..9cb66897 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField, TextInput, Form, ModelChoiceField, ValidationError from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ( + Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, + ValidationError, +) -from repos.models import Repository, Mirror +from repos.models import Mirror, Repository class EditRepoForm(ModelForm): diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878..1ae96a98 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/repos/migrations/0003_migrate_to_tz_aware.py b/repos/migrations/0003_migrate_to_tz_aware.py index dddd78ba..38e30488 100644 --- a/repos/migrations/0003_migrate_to_tz_aware.py +++ b/repos/migrations/0003_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Mirror = apps.get_model('repos', 'Mirror') for mirror in Mirror.objects.all(): diff --git a/repos/models.py b/repos/models.py index a1db2a93..9b9082af 100644 --- a/repos/models.py +++ b/repos/models.py @@ -20,13 +20,12 @@ from arch.models import MachineArchitecture from packages.models import Package -from util import get_setting_of_type - -from repos.repo_types.deb import refresh_deb_repo -from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo +from repos.repo_types.deb import refresh_deb_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from util.logging import info_message, warning_message, error_message +from repos.repo_types.rpm import refresh_repo_errata, refresh_rpm_repo +from util import get_setting_of_type +from util.logging import error_message, info_message, warning_message class Repository(models.Model): diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 09719428..390b321d 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,10 +18,13 @@ from io import BytesIO from packages.models import PackageString -from util.logging import info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages -from util import get_datetime_now, get_checksum, Checksum +from repos.utils import ( + fetch_mirror_data, find_mirror_url, get_max_mirrors, + update_mirror_packages, +) +from util import Checksum, get_checksum, get_datetime_now +from util.logging import info_message, warning_message def refresh_arch_repo(repo): diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index c6c26d78..33d1f2c4 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -15,14 +15,17 @@ # along with Patchman. If not, see import re + from debian.deb822 import Packages from debian.debian_support import Version from packages.models import PackageString -from util.logging import error_message, info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url -from util import get_datetime_now, get_checksum, Checksum, extract +from repos.utils import ( + fetch_mirror_data, find_mirror_url, update_mirror_packages, +) +from util import Checksum, extract, get_checksum, get_datetime_now +from util.logging import error_message, info_message, warning_message def extract_deb_packages(data, url): diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index e440f0d5..a0584618 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -14,22 +14,28 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import git import os import shutil import tarfile import tempfile -from defusedxml import ElementTree from fnmatch import fnmatch from io import BytesIO from pathlib import Path +import git +from defusedxml import ElementTree + from packages.models import PackageString from packages.utils import find_evr -from util.logging import info_message, warning_message, error_message from patchman.signals import pbar_start, pbar_update -from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages -from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid +from repos.utils import ( + add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages, +) +from util import ( + Checksum, extract, fetch_content, get_checksum, get_datetime_now, get_url, + response_is_valid, +) +from util.logging import error_message, info_message, warning_message def refresh_gentoo_main_repo(repo): diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 51661809..5ffbb708 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,11 +16,14 @@ from django.db.models import Q -from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo -from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data +from repos.utils import ( + check_for_metalinks, check_for_mirrorlists, fetch_mirror_data, + find_mirror_url, get_max_mirrors, +) from util import get_datetime_now +from util.logging import info_message, warning_message def refresh_repo_errata(repo): diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index bf594040..e37b9934 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,10 +17,10 @@ import re from packages.models import PackageString -from util.logging import info_message from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract +from util.logging import info_message def refresh_yast_repo(mirror, data): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index bc0fbc4b..1e96db39 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -15,17 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from repos.models import Repository diff --git a/repos/utils.py b/repos/utils.py index c11c41ad..29e9cdb3 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -17,17 +17,24 @@ import re from io import BytesIO -from defusedxml import ElementTree -from tenacity import RetryError +from defusedxml import ElementTree from django.db import IntegrityError from django.db.models import Q +from tenacity import RetryError from packages.models import Package -from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package -from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from util.logging import info_message, warning_message, error_message, debug_message +from packages.utils import ( + convert_package_to_packagestring, convert_packagestring_to_package, +) from patchman.signals import pbar_start, pbar_update +from util import ( + Checksum, extract, fetch_content, get_checksum, get_setting_of_type, + get_url, response_is_valid, +) +from util.logging import ( + debug_message, error_message, info_message, warning_message, +) def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -176,6 +183,7 @@ def add_mirrors_from_urls(repo, mirror_urls): warning_message(text=text) break from repos.models import Mirror + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? # that is what metalink urls return now m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) diff --git a/repos/views.py b/repos/views.py index 199c834e..1f0c2bfa 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,24 +15,27 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect -from django.http import HttpResponse -from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse -from django.db.models import Q from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError - +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar +from arch.models import MachineArchitecture from hosts.models import HostRepo -from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease -from arch.models import MachineArchitecture -from repos.forms import EditRepoForm, LinkRepoForm, CreateRepoForm, EditMirrorForm -from repos.serializers import RepositorySerializer, MirrorSerializer, MirrorPackageSerializer +from repos.forms import ( + CreateRepoForm, EditMirrorForm, EditRepoForm, LinkRepoForm, +) +from repos.models import Mirror, MirrorPackage, Repository +from repos.serializers import ( + MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/sbin/patchman b/sbin/patchman index 47d89b06..c415abec 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -17,32 +17,37 @@ # along with Patchman. If not, see +import argparse import os import sys -import argparse +from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count -from django import setup as django_setup os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa + django_setup() from arch.utils import clean_architectures -from errata.utils import mark_errata_security_updates, enrich_errata, \ - scan_package_updates_for_affected_packages from errata.tasks import update_errata +from errata.utils import ( + enrich_errata, mark_errata_security_updates, + scan_package_updates_for_affected_packages, +) from hosts.models import Host from hosts.utils import clean_tags from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames -from repos.models import Repository -from repos.utils import clean_repos +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from reports.models import Report from reports.tasks import clean_reports_with_no_hosts +from repos.models import Repository +from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import set_verbosity, get_datetime_now +from util import get_datetime_now, set_verbosity from util.logging import info_message diff --git a/security/admin.py b/security/admin.py index 196a9468..aedeaea9 100644 --- a/security/admin.py +++ b/security/admin.py @@ -15,8 +15,8 @@ # along with Patchman. If not, see from django.contrib import admin -from security.models import CWE, CVSS, CVE, Reference +from security.models import CVE, CVSS, CWE, Reference admin.site.register(CWE) admin.site.register(CVSS) diff --git a/security/models.py b/security/models.py index 7f674a0b..0f848260 100644 --- a/security/models.py +++ b/security/models.py @@ -16,14 +16,14 @@ import json import re -from cvss import CVSS2, CVSS3, CVSS4 from time import sleep +from cvss import CVSS2, CVSS3, CVSS4 from django.db import models from django.urls import reverse from security.managers import CVEManager -from util import get_url, fetch_content, tz_aware_datetime, error_message +from util import error_message, fetch_content, get_url, tz_aware_datetime class Reference(models.Model): diff --git a/security/tasks.py b/security/tasks.py index ce60df83..7bff4149 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from security.models import CVE, CWE diff --git a/security/views.py b/security/views.py index 58a686b5..c9e606a6 100644 --- a/security/views.py +++ b/security/views.py @@ -14,17 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from packages.models import Package from operatingsystems.models import OSRelease +from packages.models import Package from security.models import CVE, CWE, Reference -from security.serializers import CVESerializer, CWESerializer, ReferenceSerializer +from security.serializers import ( + CVESerializer, CWESerializer, ReferenceSerializer, +) from util.filterspecs import Filter, FilterBar diff --git a/setup.py b/setup.py index 6ec6d974..8e18eaf9 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ # along with Patchman. If not, see import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup with open('VERSION.txt', 'r', encoding='utf_8') as v: version = v.readline().strip() diff --git a/util/__init__.py b/util/__init__.py index c3dfd6bd..c6c9aa0d 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -15,30 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import requests import bz2 -import magic -import zlib import lzma import os +import zlib + +import magic +import requests + try: # python 3.14+ - can also remove the dependency at that stage from compression import zstd except ImportError: import zstandard as zstd + from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 -from requests.exceptions import HTTPError, Timeout, ConnectionError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from time import time -from tqdm import tqdm -from django.utils.timezone import make_aware -from django.utils.dateparse import parse_datetime from django.conf import settings +from django.utils.dateparse import parse_datetime +from django.utils.timezone import make_aware +from requests.exceptions import ConnectionError, HTTPError, Timeout +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) +from tqdm import tqdm -from util.logging import error_message, info_message, debug_message +from util.logging import debug_message, error_message, info_message pbar = None verbose = None diff --git a/util/filterspecs.py b/util/filterspecs.py index 722b45df..eac0f747 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.utils.safestring import mark_safe -from django.db.models.query import QuerySet from operator import itemgetter +from django.db.models.query import QuerySet +from django.utils.safestring import mark_safe + def get_query_string(qs): new_qs = [f'{k}={v}' for k, v in list(qs.items())] diff --git a/util/logging.py b/util/logging.py index dd79d296..cb00ccce 100644 --- a/util/logging.py +++ b/util/logging.py @@ -14,12 +14,12 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see + from datetime import datetime -from patchman.signals import info_message_s -from patchman.signals import warning_message_s -from patchman.signals import error_message_s -from patchman.signals import debug_message_s +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, warning_message_s, +) def info_message(text): diff --git a/util/tasks.py b/util/tasks.py index f650e3e2..bd76bac6 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -18,7 +18,9 @@ from arch.utils import clean_architectures from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from repos.utils import clean_repos, remove_mirror_trailing_slashes diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 2aea1e5e..674e1721 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,16 +15,15 @@ # along with Patchman. If not, see import re - -from humanize import naturaltime from datetime import datetime, timedelta from urllib.parse import urlencode +from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.utils.html import format_html from django.templatetags.static import static -from django.core.paginator import Paginator +from django.utils.html import format_html +from humanize import naturaltime from util import get_setting_of_type diff --git a/util/views.py b/util/views.py index b66db6b0..fd003bca 100644 --- a/util/views.py +++ b/util/views.py @@ -17,16 +17,16 @@ from datetime import datetime, timedelta -from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F +from django.shortcuts import render from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from repos.models import Repository, Mirror +from operatingsystems.models import OSRelease, OSVariant from packages.models import Package from reports.models import Report +from repos.models import Mirror, Repository from util import get_setting_of_type From 5fa1ef0e3d7484275238b761b05d07782e08a2f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 00:40:08 +0000 Subject: [PATCH 106/120] Bump django from 4.2.25 to 4.2.26 Bumps [django](https://github.com/django/django) from 4.2.25 to 4.2.26. - [Commits](https://github.com/django/django/compare/4.2.25...4.2.26) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.26 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 788f4241..f67fc2ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.25 +Django==4.2.26 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From aac552d7cf22ee1f6037a9604813cc3bfa870571 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:57:03 +0000 Subject: [PATCH 107/120] Bump django from 4.2.26 to 4.2.27 Bumps [django](https://github.com/django/django) from 4.2.26 to 4.2.27. - [Commits](https://github.com/django/django/compare/4.2.26...4.2.27) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.27 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f67fc2ce..08ce4573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.26 +Django==4.2.27 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From da1f44e8723d3feeb2982a2251ac08bc9d2b4940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Sat, 20 Dec 2025 08:24:21 +0000 Subject: [PATCH 108/120] Modified tag handling to preserve case --- hosts/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/models.py b/hosts/models.py index 8ea5e3d5..4689ccc5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -86,7 +86,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {" ".join(self.tags.slugs())}\n' + text += f'Tags : {" ".join(self.tags.names())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' From d8f3ce4203b16f60b6ffdd55edff1ca354f006e9 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 17 Dec 2025 14:11:34 -0500 Subject: [PATCH 109/120] fix same module in different repos --- .../0005_alter_module_unique_together.py | 19 +++++++++++++++++++ modules/models.py | 2 +- modules/utils.py | 5 ++--- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 modules/migrations/0005_alter_module_unique_together.py diff --git a/modules/migrations/0005_alter_module_unique_together.py b/modules/migrations/0005_alter_module_unique_together.py new file mode 100644 index 00000000..046a0c54 --- /dev/null +++ b/modules/migrations/0005_alter_module_unique_together.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.25 on 2025-11-25 16:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('arch', '0001_initial'), + ('repos', '0006_mirror_errata_checksum_mirror_modules_checksum'), + ('modules', '0004_alter_module_options'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='module', + unique_together={('name', 'stream', 'version', 'context', 'arch', 'repo')}, + ), + ] diff --git a/modules/models.py b/modules/models.py index 931a41c3..e1f8071b 100644 --- a/modules/models.py +++ b/modules/models.py @@ -35,7 +35,7 @@ class Module(models.Model): class Meta: verbose_name = 'Module' verbose_name_plural = 'Modules' - unique_together = ['name', 'stream', 'version', 'context', 'arch'] + unique_together = ['name', 'stream', 'version', 'context', 'arch', 'repo'] ordering = ['name', 'stream'] def __str__(self): diff --git a/modules/utils.py b/modules/utils.py index 0d669478..248b8b45 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -25,10 +25,9 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object Returns the module """ - created = False - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + m_arch, _ = PackageArchitecture.objects.get_or_create(name=arch) try: - module, created = Module.objects.get_or_create( + module, _ = Module.objects.get_or_create( name=name, stream=stream, version=version, From 65e88519e1967b68a767d7a38af8e4792e80aa8a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 1 Jan 2026 17:48:35 -0500 Subject: [PATCH 110/120] add priority queues for tasks (#724) * add priority queues for tasks * Update repos/tasks.py Co-authored-by: code-review-doctor[bot] <72320148+code-review-doctor[bot]@users.noreply.github.com> --- errata/tasks.py | 6 ++--- etc/patchman/local_settings.py | 6 ++--- hosts/tasks.py | 6 ++--- patchman/settings.py | 5 ++++ reports/tasks.py | 47 ++++++++++++++++++++++++++-------- repos/tasks.py | 18 ++++++++++--- sbin/patchman | 4 +-- security/tasks.py | 38 +++++++++++++++++++++------ util/tasks.py | 2 +- 9 files changed, 97 insertions(+), 35 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index 8ded3a79..9d1f3ed9 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -29,7 +29,7 @@ from util.logging import error_message, warning_message -@shared_task +@shared_task(priority=1) def update_yum_repo_errata(repo_id=None, force=False): """ Update all yum repos errata """ @@ -41,7 +41,7 @@ def update_yum_repo_errata(repo_id=None, force=False): repo.refresh_errata(force) -@shared_task +@shared_task(priority=1) def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ @@ -85,7 +85,7 @@ def update_errata(erratum_type=None, force=False, repo=None): warning_message('Already updating Errata, skipping task.') -@shared_task +@shared_task(priority=2) def update_errata_and_cves(): """ Task to update all errata """ diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 40dd2efd..15fcb60a 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -56,15 +56,15 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False -# Set the default timeout to e.g. 30 seconds to enable UI caching -# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.redis.RedisCache', 'LOCATION': 'redis://127.0.0.1:6379', - 'TIMEOUT': 0, } } +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time +CACHE_MIDDLEWARE_SECONDS = 0 from datetime import timedelta # noqa diff --git a/hosts/tasks.py b/hosts/tasks.py index 226652f6..f186760f 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -22,7 +22,7 @@ from util.logging import info_message -@shared_task +@shared_task(priority=0) def find_host_updates(host_id): """ Task to find updates for a host """ @@ -30,7 +30,7 @@ def find_host_updates(host_id): host.find_updates() -@shared_task +@shared_task(priority=1) def find_all_host_updates(): """ Task to find updates for all hosts """ @@ -38,7 +38,7 @@ def find_all_host_updates(): find_host_updates.delay(host.id) -@shared_task +@shared_task(priority=1) def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ diff --git a/patchman/settings.py b/patchman/settings.py index 557e8c68..23e932c4 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,6 +13,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', + 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -109,6 +110,10 @@ TAGGIT_CASE_INSENSITIVE = True CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' +CELERY_BROKER_TRANSPORT_OPTIONS = { + 'queue_order_strategy': 'priority', +} +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 LOGIN_REDIRECT_URL = '/patchman/' LOGOUT_REDIRECT_URL = '/patchman/login/' diff --git a/reports/tasks.py b/reports/tasks.py index 07fb3004..6bf16e7c 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -24,25 +24,50 @@ from util.logging import info_message, warning_message -@shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) +@shared_task( + bind=True, + priority=0, + autoretry_for=(OperationalError,), + retry_backoff=True, + retry_kwargs={'max_retries': 5} +) def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - lock_key = f'process_report_lock_{report_id}' - # lock will expire after 1 hour - lock_expire = 60 * 60 + report_id_lock_key = f'process_report_id_lock_{report_id}' + if report.host: + report_host_lock_key = f'process_report_host_lock_{report.host}' + else: + report_host_lock_key = f'process_report_host_lock_{report.report_ip}' + # locks will expire after 2 hours + lock_expire = 60 * 60 * 2 - if cache.add(lock_key, 'true', lock_expire): + if cache.add(report_id_lock_key, 'true', lock_expire): try: - report.process() + processing_report_id = cache.get(report_host_lock_key) + if processing_report_id: + if processing_report_id > report.id: + warning_message(f'Currently processing a newer report for {report.host} or {report.report_ip}, \ + marking report {report.id} as processed.') + report.processed = True + report.save() + else: + warning_message(f'Currently processing an older report for {report.host} or {report.report_ip}, \ + will skip processing this report.') + else: + try: + cache.set(report_host_lock_key, report.id, lock_expire) + report.process() + finally: + cache.delete(report_host_lock_key) finally: - cache.delete(lock_key) + cache.delete(report_id_lock_key) else: warning_message(f'Already processing report {report_id}, skipping task.') -@shared_task +@shared_task(priority=1) def process_reports(): """ Task to process all unprocessed reports """ @@ -51,9 +76,9 @@ def process_reports(): process_report.delay(report.id) -@shared_task -def clean_reports_with_no_hosts(): - """ Task to clean processed reports where the host no longer exists +@shared_task(priority=2) +def remove_reports_with_no_hosts(): + """ Task to remove processed reports where the host no longer exists """ for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): diff --git a/repos/tasks.py b/repos/tasks.py index bc177653..a9fdd5f4 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -21,15 +21,25 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=0) def refresh_repo(repo_id, force=False): """ Refresh metadata for a single repo """ - repo = Repository.objects.get(id=repo_id) - repo.refresh(force) + repo_id_lock_key = f'refresh_repos_{repo_id}_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(repo_id_lock_key, 'true', lock_expire): + try: + repo = Repository.objects.get(id=repo_id) + repo.refresh(force) + finally: + cache.delete(repo_id_lock_key) + else: + warning_message(f'Already refreshing repo {repo_id}, skipping task.') -@shared_task +@shared_task(priority=1) def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ diff --git a/sbin/patchman b/sbin/patchman index c415abec..b76272a2 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from packages.utils import ( clean_packagenames, clean_packages, clean_packageupdates, ) from reports.models import Report -from reports.tasks import clean_reports_with_no_hosts +from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes @@ -156,7 +156,7 @@ def clean_reports(hoststr=None): host.clean_reports() if not hoststr: - clean_reports_with_no_hosts() + remove_reports_with_no_hosts() def host_updates_alt(host=None): diff --git a/security/tasks.py b/security/tasks.py index 7bff4149..0cfbc2f1 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -21,15 +21,26 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=3) def update_cve(cve_id): """ Task to update a CVE """ - cve = CVE.objects.get(id=cve_id) - cve.fetch_cve_data() + cve_id_lock_key = f'update_cve_id_lock_{cve_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cve_id_lock_key, 'true', lock_expire): + try: + cve = CVE.objects.get(id=cve_id) + cve.fetch_cve_data() + finally: + cache.delete(cve_id_lock_key) + else: + warning_message(f'Already updating CVE {cve_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cves(): """ Task to update all CVEs """ @@ -47,15 +58,26 @@ def update_cves(): warning_message('Already updating CVEs, skipping task.') -@shared_task +@shared_task(priority=3) def update_cwe(cwe_id): """ Task to update a CWE """ - cwe = CWE.objects.get(id=cwe_id) - cwe.fetch_cwe_data() + cwe_id_lock_key = f'update_cwe_id_lock_{cwe_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cwe_id_lock_key, 'true', lock_expire): + try: + cwe = CWE.objects.get(id=cwe_id) + cwe.fetch_cwe_data() + finally: + cache.delete(cwe_id_lock_key) + else: + warning_message(f'Already updating CWE {cwe_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cwes(): """ Task to update all CWEs """ diff --git a/util/tasks.py b/util/tasks.py index bd76bac6..12825a8c 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -24,7 +24,7 @@ from repos.utils import clean_repos, remove_mirror_trailing_slashes -@shared_task +@shared_task(priority=1) def clean_database(remove_duplicate_packages=False): """ Task to check the database and remove orphaned objects Runs all clean_* functions to check database consistency From 0338987b7189f89c83634c3a605aa8984342fb75 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 7 Jan 2026 20:46:50 -0500 Subject: [PATCH 111/120] update celery services handling (#726) Signed-off-by: Marcus Furlong --- debian/python3-patchman.install | 3 +- debian/python3-patchman.postinst | 29 +++++++++++++++ etc/patchman/celery.conf | 3 ++ .../system/patchman-celery-beat.service | 19 ++++++++++ .../system/patchman-celery-worker.service | 21 +++++++++++ etc/systemd/system/patchman-celery.service | 14 -------- patchman-client.spec | 17 ++++----- scripts/rpm-post-install.sh | 36 +++++++++++++++---- setup.py | 9 ++++- 9 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 etc/systemd/system/patchman-celery-beat.service create mode 100644 etc/systemd/system/patchman-celery-worker.service delete mode 100644 etc/systemd/system/patchman-celery.service diff --git a/debian/python3-patchman.install b/debian/python3-patchman.install index e13b11ca..71f47b3a 100755 --- a/debian/python3-patchman.install +++ b/debian/python3-patchman.install @@ -1,4 +1,5 @@ #!/usr/bin/dh-exec etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf etc/patchman/local_settings.py etc/patchman -etc/systemd/system/patchman-celery.service => lib/systemd/system/patchman-celery.service +etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service +etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index b64cb816..bce66010 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -25,8 +25,37 @@ if [ "$1" = "configure" ] ; then chown -R www-data:www-data /var/lib/patchman adduser --system --group patchman-celery usermod -a -G www-data patchman-celery + chown root:patchman-celery /etc/patchman/celery.conf + chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + WORKER_COUNT=1 + if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} + fi + + if [ -d /run/systemd/system ]; then + systemctl daemon-reload >/dev/null || true + for i in $(seq 1 "${WORKER_COUNT}"); do + deb-systemd-helper enable "patchman-celery-worker@$i.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-worker@$i.service" >/dev/null || true + done + + active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') + + for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + deb-systemd-invoke stop "$service" >/dev/null || true + deb-systemd-helper disable "$service" >/dev/null || true + fi + done + + deb-systemd-helper enable "patchman-celery-beat.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-beat.service" >/dev/null || true + fi + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/etc/patchman/celery.conf b/etc/patchman/celery.conf index 7afc96ee..2e6f9855 100644 --- a/etc/patchman/celery.conf +++ b/etc/patchman/celery.conf @@ -1,2 +1,5 @@ REDIS_HOST=127.0.0.1 REDIS_PORT=6379 +CELERY_POOL_TYPE=solo +CELERY_WORKER_COUNT=1 +CELERY_CONCURRENCY=1 diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service new file mode 100644 index 00000000..b4ba9f3d --- /dev/null +++ b/etc/systemd/system/patchman-celery-beat.service @@ -0,0 +1,19 @@ +[Unit] +Description=Patchman Celery Beat Scheduler Service +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + beat \ + --loglevel info \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service new file mode 100644 index 00000000..8807cbff --- /dev/null +++ b/etc/systemd/system/patchman-celery-worker.service @@ -0,0 +1,21 @@ +[Unit] +Description=Patchman Celery Worker Service %i +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + worker \ + --task-events \ + --pool ${CELERY_POOL_TYPE:-solo} \ + --concurrency ${CELERY_CONCURRENCY:-1} \ + --hostname patchman-celery-worker%i@%%h + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery.service b/etc/systemd/system/patchman-celery.service deleted file mode 100644 index 6408d818..00000000 --- a/etc/systemd/system/patchman-celery.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Patchman Celery Service -Requires=network-online.target -After=network-onlne.target - -[Service] -Type=simple -User=patchman-celery -Group=patchman-celery -EnvironmentFile=/etc/patchman/celery.conf -ExecStart=/usr/bin/celery --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads - -[Install] -WantedBy=multi-user.target diff --git a/patchman-client.spec b/patchman-client.spec index 68736038..f2f8279a 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -10,7 +10,7 @@ Source: %{expand:%%(pwd)} BuildArch: noarch Requires: curl which coreutils util-linux gawk -%define binary_payload w9.gzdio +%define _binary_payload w9.gzdio %description patchman-client provides a client that uploads reports to a patchman server @@ -20,14 +20,15 @@ find . -mindepth 1 -delete cp -af %{SOURCEURL0}/. . %install -mkdir -p %{buildroot}/usr/sbin -mkdir -p %{buildroot}/etc/patchman -cp ./client/%{name} %{buildroot}/usr/sbin -cp ./client/%{name}.conf %{buildroot}/etc/patchman +mkdir -p %{buildroot}%{_sbindir} +mkdir -p %{buildroot}%{_sysconfdir}/patchman +install -m 755 client/%{name} %{buildroot}%{_sbindir}/%{name} +install -m 644 client/%{name}.conf %{buildroot}%{_sysconfdir}/patchman/%{name}.conf %files -%defattr(755,root,root) -/usr/sbin/patchman-client -%config(noreplace) /etc/patchman/patchman-client.conf +%defattr(-,root,root) +%{_sbindir}/patchman-client +%dir %{_sysconfdir}/patchman +%config(noreplace) %{_sysconfdir}/patchman/patchman-client.conf %changelog diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 24ade8af..798c43ea 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -4,8 +4,9 @@ if [ ! -e /etc/httpd/conf.d/patchman.conf ] ; then cp /etc/patchman/apache.conf.example /etc/httpd/conf.d/patchman.conf fi -if ! grep /usr/lib/python3.9/site-packages /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then - sed -i -e "s/^\(Define patchman_pythonpath\).*/\1 \/usr\/lib\/python3.9\/site-packages/" \ +PYTHON_SITEPACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])") +if ! grep "${PYTHON_SITEPACKAGES}" /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then + sed -i -e "s|^\(Define patchman_pythonpath\).*|\1 ${PYTHON_SITEPACKAGES}|" \ /etc/httpd/conf.d/patchman.conf fi @@ -24,15 +25,38 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -chown -R apache:apache /var/lib/patchman adduser --system --group patchman-celery usermod -a -G apache patchman-celery -chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db -chcon --type httpd_sys_rw_content_t /var/lib/patchman/db/patchman.db -semanage port -a -t http_port_t -p tcp 5672 +chown root:patchman-celery /etc/patchman/celery.conf +chmod 640 /etc/patchman/celery.conf + +chown -R apache:apache /var/lib/patchman +semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" +restorecon -Rv /var/lib/patchman/db setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 +WORKER_COUNT=1 +if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} +fi + +for i in $(seq 1 "${WORKER_COUNT}"); do + systemctl enable --now "patchman-celery-worker@$i.service" +done + +active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') +for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + systemctl stop "$service" + systemctl disable "$service" + fi +done + +systemctl enable --now patchman-celery-beat.service + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/setup.py b/setup.py index 8e18eaf9..a7dbfc68 100755 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ # along with Patchman. If not, see import os +import sys from setuptools import find_packages, setup @@ -29,8 +30,14 @@ with open('requirements.txt', 'r', encoding='utf_8') as rt: install_requires = rt.read().splitlines() - data_files = [] +if 'bdist_rpm' in sys.argv: + data_files.append( + ('/usr/lib/systemd/system', [ + 'etc/systemd/system/patchman-celery-worker@.service', + 'etc/systemd/system/patchman-celery-beat.service' + ]) + ) for dirpath, dirnames, filenames in os.walk('etc'): # Ignore dirnames that start with '.' From ff7f293855183e0c23e55a49d04be0d093738ff7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:10 -0500 Subject: [PATCH 112/120] remove non-present middleware (#729) --- patchman/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/patchman/settings.py b/patchman/settings.py index 23e932c4..c3089caa 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,7 +13,6 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', - 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', From 5e2bb76493da7f9cceb5dbb93f6673857e714175 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:20 -0500 Subject: [PATCH 113/120] fix wsgi so rpm module is only loaded once (#728) --- etc/patchman/apache.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/patchman/apache.conf.example b/etc/patchman/apache.conf.example index 5434cf97..c055eba0 100644 --- a/etc/patchman/apache.conf.example +++ b/etc/patchman/apache.conf.example @@ -1,5 +1,5 @@ Define patchman_pythonpath /srv/patchman/ -WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py +WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py application-group=%{GLOBAL} WSGIPythonPath ${patchman_pythonpath} From 33519f2f465ae2bc9e3b867cc6b3dd630637ff33 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:55:12 -0500 Subject: [PATCH 114/120] give systemd units usable defaults (#727) --- etc/systemd/system/patchman-celery-beat.service | 4 +++- etc/systemd/system/patchman-celery-worker.service | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index b4ba9f3d..e219d38d 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -7,9 +7,11 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ beat \ --loglevel info \ diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 8807cbff..51fa9a0e 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -7,14 +7,18 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" +Environment="CELERY_POOL_TYPE=solo" +Environment="CELERY_CONCURRENCY=1" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ worker \ --task-events \ - --pool ${CELERY_POOL_TYPE:-solo} \ - --concurrency ${CELERY_CONCURRENCY:-1} \ + --pool ${CELERY_POOL_TYPE} \ + --concurrency ${CELERY_CONCURRENCY} \ --hostname patchman-celery-worker%i@%%h [Install] From 6e9b21f804e996da5865c773994bfd25b0e9d899 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 18:38:05 -0500 Subject: [PATCH 115/120] use consistent users/groups on rhel/debian (#730) --- debian/python3-patchman.postinst | 8 ++++---- etc/systemd/system/patchman-celery-beat.service | 4 ++-- etc/systemd/system/patchman-celery-worker.service | 4 ++-- scripts/rpm-post-install.sh | 10 ++++------ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index bce66010..b36a5422 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -22,12 +22,12 @@ if [ "$1" = "configure" ] ; then patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' - chown -R www-data:www-data /var/lib/patchman - adduser --system --group patchman-celery - usermod -a -G www-data patchman-celery - chown root:patchman-celery /etc/patchman/celery.conf + adduser --quiet --system --group patchman + adduser --quiet www-data patchman + chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + chown -R patchman:patchman /var/lib/patchman WORKER_COUNT=1 if [ -f /etc/patchman/celery.conf ]; then diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index e219d38d..c9bce722 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 51fa9a0e..b2d6f6b7 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" Environment="CELERY_POOL_TYPE=solo" diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 798c43ea..451f7d48 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -25,15 +25,13 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -adduser --system --group patchman-celery -usermod -a -G apache patchman-celery -chown root:patchman-celery /etc/patchman/celery.conf +adduser --system --shell /sbin/nologin patchman +usermod -a -G patchman apache +chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf - -chown -R apache:apache /var/lib/patchman +chown -R patchman:patchman /var/lib/patchman semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" restorecon -Rv /var/lib/patchman/db -setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 WORKER_COUNT=1 From d450af7157305ccc760eb4df622b06f7de2f1799 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 15:10:12 -0500 Subject: [PATCH 116/120] fixes for dumping/loading fixtures from sqlite (#731) --- operatingsystems/managers.py | 4 ++-- security/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/operatingsystems/managers.py b/operatingsystems/managers.py index 630484a1..b7b9f24f 100644 --- a/operatingsystems/managers.py +++ b/operatingsystems/managers.py @@ -18,5 +18,5 @@ class OSReleaseManager(models.Manager): - def get_by_natural_key(self, name, codename): - return self.get(name=name, codename=codename) + def get_by_natural_key(self, name, codename, cpe_name): + return self.get(name=name, codename=codename, cpe_name=cpe_name) diff --git a/security/models.py b/security/models.py index 0f848260..405c8db6 100644 --- a/security/models.py +++ b/security/models.py @@ -125,6 +125,8 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] + if isinstance(severity, str): + severity = severity.capitalize() try: cvss, created = CVSS.objects.get_or_create( version=version, From f798882834ed58d2f8e2c9f1c03a7cc472605b57 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 16:54:39 -0500 Subject: [PATCH 117/120] update logging to log to console and celery systemd units (#732) --- etc/patchman/local_settings.py | 17 +++++++++ patchman/receivers.py | 25 ++++++++------ reports/utils.py | 4 ++- repos/repo_types/arch.py | 2 +- repos/repo_types/deb.py | 2 +- repos/repo_types/gentoo.py | 4 +-- repos/repo_types/rpm.py | 2 +- repos/utils.py | 2 +- sbin/patchman | 6 ++-- util/__init__.py | 42 +++-------------------- util/logging.py | 63 +++++++++++++++++++++++++++++----- 11 files changed, 103 insertions(+), 66 deletions(-) diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 15fcb60a..cb19e268 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -96,3 +96,20 @@ 'schedule': timedelta(hours=24), }, } + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + }, + 'loggers': { + 'urllib3': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + 'git': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + } +} diff --git a/patchman/receivers.py b/patchman/receivers.py index 19312ed5..9393d891 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -18,13 +18,13 @@ from colorama import Fore, Style, init from django.conf import settings from django.dispatch import receiver -from tqdm import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, warning_message_s, ) -from util import create_pbar, get_verbosity, update_pbar +from util.logging import create_pbar, get_quiet_mode, logger, update_pbar init(autoreset=True) @@ -53,8 +53,10 @@ def print_info_message(**kwargs): """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.RESET_ALL + Fore.RESET + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(loggers=[logger]): + for line in text.splitlines(): + logger.info(Style.RESET_ALL + Fore.RESET + line) @receiver(warning_message_s) @@ -62,8 +64,9 @@ def print_warning_message(**kwargs): """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.BRIGHT + Fore.YELLOW + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(): + logger.warning(Style.BRIGHT + Fore.YELLOW + text) @receiver(error_message_s) @@ -72,13 +75,15 @@ def print_error_message(**kwargs): """ text = str(kwargs.get('text')) if text: - tqdm.write(Style.BRIGHT + Fore.RED + text) + with logging_redirect_tqdm(): + logger.error(Style.BRIGHT + Fore.RED + text) @receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to handle a debug message, blue text if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if DEBUG is set """ text = str(kwargs.get('text')) - if get_verbosity() and settings.DEBUG and text: - tqdm.write(Style.BRIGHT + Fore.BLUE + text) + if settings.DEBUG and text: + with logging_redirect_tqdm(loggers=[logger]): + logger.debug(Style.BRIGHT + Fore.BLUE + text) diff --git a/reports/utils.py b/reports/utils.py index 8b12f046..1a08cf3c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -34,7 +34,7 @@ from patchman.signals import pbar_start, pbar_update from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo -from util.logging import info_message +from util.logging import debug_message, info_message def process_repos(report, host): @@ -47,6 +47,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): + debug_message(f'Processing report {report.id} repo: {repo_str}') repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) @@ -93,6 +94,7 @@ def process_packages(report, host): packages = parse_packages(report.packages) pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): + debug_message(f'Processing report {report.id} package: {pkg_str}') package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 390b321d..b339ee6a 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -51,7 +51,7 @@ def refresh_arch_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Arch Repo data') if not package_data: continue diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 33d1f2c4..eea6593f 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -95,7 +95,7 @@ def refresh_deb_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Debian Repo data') if not package_data: continue diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index a0584618..61966f7a 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -51,7 +51,7 @@ def refresh_gentoo_main_repo(repo): continue res = get_url(mirror.url + '.md5sum') - data = fetch_content(res, 'Fetching Repo checksum') + data = fetch_content(res, 'Fetching Gentoo Repo checksum') if data is None: mirror.fail() continue @@ -72,7 +72,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Gentoo Repo data') if data is None: mirror.fail() continue diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 5ffbb708..d1482272 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -84,7 +84,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): repo_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching rpm Repo data') if not repo_data: continue diff --git a/repos/utils.py b/repos/utils.py index 29e9cdb3..13cee149 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -155,7 +155,7 @@ def get_mirrorlist_urls(url): return if response_is_valid(res): try: - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Repo data to check for mirrorlist') if data is None: return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) diff --git a/sbin/patchman b/sbin/patchman index b76272a2..06bef981 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -47,8 +47,8 @@ from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import get_datetime_now, set_verbosity -from util.logging import info_message +from util import get_datetime_now +from util.logging import info_message, set_quiet_mode def get_host(host=None, action='Performing action'): @@ -547,7 +547,7 @@ def main(): parser = collect_args() args = parser.parse_args() - set_verbosity(not args.quiet) + set_quiet_mode(args.quiet) showhelp = process_args(args) if showhelp: parser.print_help() diff --git a/util/__init__.py b/util/__init__.py index c6c9aa0d..b85e5e37 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -41,12 +41,14 @@ from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) -from tqdm import tqdm -from util.logging import debug_message, error_message, info_message +from util.logging import ( + create_pbar, debug_message, error_message, info_message, quiet_mode, + update_pbar, +) pbar = None -verbose = None +verbose = not quiet_mode Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') http_proxy = os.getenv('http_proxy') @@ -57,40 +59,6 @@ } -def get_verbosity(): - """ Get the global verbosity level - """ - return verbose - - -def set_verbosity(value): - """ Set the global verbosity level - """ - global verbose - verbose = value - - -def create_pbar(ptext, plength, ljust=35, **kwargs): - """ Create a global progress bar if global verbose is True - """ - global pbar - if verbose and plength > 0: - jtext = str(ptext).ljust(ljust) - pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') - return pbar - - -def update_pbar(index, **kwargs): - """ Update the global progress bar if global verbose is True - """ - global pbar - if verbose and pbar: - pbar.update(n=index-pbar.n) - if index >= pbar.total: - pbar.close() - pbar = None - - def fetch_content(response, text='', ljust=35): """ Display a progress bar to fetch the request content if verbose is True. Otherwise, just return the request content diff --git a/util/logging.py b/util/logging.py index cb00ccce..bf532e54 100644 --- a/util/logging.py +++ b/util/logging.py @@ -15,28 +15,73 @@ # along with Patchman. If not, see -from datetime import datetime +import logging + +from django.conf import settings +from tqdm import tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, warning_message_s, ) +log_format = '[%(asctime)s] %(levelname)s: %(message)s' +if settings.DEBUG: + logging_level = logging.DEBUG +else: + logging_level = logging.INFO +logging.basicConfig(level=logging_level, format=log_format) +logger = logging.getLogger() +logging.getLogger('git.cmd').setLevel(logging.WARNING) + +quiet_mode = False +pbar = None + + +def get_quiet_mode(): + """ Get the global quiet_mode + """ + return quiet_mode + + +def set_quiet_mode(value): + """ Set the global quiet_mode + """ + global quiet_mode + quiet_mode = value + + +def create_pbar(ptext, plength, ljust=35, **kwargs): + """ Create a global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and plength > 0: + jtext = str(ptext).ljust(ljust) + pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') + return pbar + + +def update_pbar(index, **kwargs): + """ Update the global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and pbar: + pbar.update(n=index-pbar.n) + if index >= pbar.total: + pbar.close() + pbar = None + def info_message(text): - ts = datetime.now() - info_message_s.send(sender=None, text=text, ts=ts) + info_message_s.send(sender=None, text=text) def warning_message(text): - ts = datetime.now() - warning_message_s.send(sender=None, text=text, ts=ts) + warning_message_s.send(sender=None, text=text) def debug_message(text): - ts = datetime.now() - debug_message_s.send(sender=None, text=text, ts=ts) + debug_message_s.send(sender=None, text=text) def error_message(text): - ts = datetime.now() - error_message_s.send(sender=None, text=text, ts=ts) + error_message_s.send(sender=None, text=text) From aa43a631bcf87b8c7c108532cd2c832f59174131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 15 Jan 2026 16:38:31 +0000 Subject: [PATCH 118/120] Changed entrypoint script to adapt changes made in 8fa3eb2 --- docker/docker-entrypoint.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index cf0ac72f..97a75152 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -92,17 +92,16 @@ if "${USE_CACHE}"; then redisPort="6379" fi - # Comment DummyCache Block - sed -i '47,51 {/^#/ ! s/\(.*\)/#\1/}' "$conf" - - # Uncomment RedisCache Block - sed -i '55,61 {s/^# //}' "$conf" - - sed -i "58 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" + # Change RedisCache LOCATION + sed -i "62 {s/127.0.0.1:6379/$redisHost:$redisPort/}" "$conf" if [ -n "${CACHE_TIMEOUT}" ]; then - sed -i "59 {s/30/${CACHE_TIMEOUT}/}" "$conf" + sed -i "67 {s/0/${CACHE_TIMEOUT}/}" "$conf" fi +else + # Change RedisCache to DummyCache to avoid ConnectionError and comment LOCATION + sed -i '61 {s/redis.RedisCache/dummy.DummyCache/}' "$conf" + sed -i '62 {/^#/ ! s/\(.*\)/#\1/}' "$conf" fi # Sync database on container first start From f5271450e5a8bc3a0e66830bbc6c39611489cdef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 29 Jan 2026 10:20:42 +0000 Subject: [PATCH 119/120] Branch cleanup --- util/templatetags/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 8f8bc792..0fdeaff1 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -18,7 +18,6 @@ from datetime import timedelta from urllib.parse import urlencode -from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template from django.utils import timezone From 51cf2cd033b832ff6f4db7b138ade5ce4918de86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 5 Mar 2026 14:27:51 +0000 Subject: [PATCH 120/120] Added more configuration options via environment variables --- docker/README.md | 12 ++++++++- docker/docker-entrypoint.sh | 52 +++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3310fccf..b8c7afc5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,12 +23,22 @@ This container is configured using environment variables. The following variable | Variable | Default Value | Description | | :--- | :--- | :--- | +| `DEBUG` | `False` | Debug mode | | `ADMIN_NAME` | `Your Name` | Your name | | `ADMIN_EMAIL` | `you@example.com` | Your e-mail address | | `TIMEZONE` | `America/New_York` | Your timezone | +| `LANGUAGE_CODE` | `en-us` | Your language | +| `SECRET_KEY` | | Unique string not to be shared with anyone. Leave empty to generate one randomly | +| `MAX_MIRRORS` | `2` | Maximum number of mirrors to add or refresh per repo | +| `MAX_MIRROR_FAILURES` | `14` | Maximum number of failures before disabling a mirror, set to `-1` to never disable mirrors | +| `DAYS_WITHOUT_REPORT` | `14` | Number of days to wait before raising that a host has not reported | +| `ERRATA_OS_UPDATES` | `yum, rocky, alma, arch, ubuntu, debian` | List of errata sources to update, remove unwanted ones to improve performance | +| `ALMA_RELEASES` | `8, 9, 10` | List of Alma Linux releases to update | +| `DEBIAN_CODENAMES` | `bookworm, trixie` | List of Debian Linux releases to update | +| `UBUNTU_CODENAMES` | `jammy, noble` | List of Ubuntu Linux releases to update | | `DB_ENGINE` | `SQLite` | Database engine to be used. Choose between `MySQL` or `PostgreSQL`, leave empty to use default `SQLite` | | `DB_HOST` | | Database hostname, IP or container name | -| `DB_PORT` |` | Database port | +| `DB_PORT` | | Database port | | `DB_DATABASE` | | Database name | | `DB_USER` | | Database user | | `DB_PASSWORD` | | Database password | diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 97a75152..d22a17a9 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -2,6 +2,11 @@ conf="/etc/patchman/local_settings.py" +# Configure DEBUG +if "${DEBUG}"; then + sed -i '3 {s/False/True/}' "$conf" +fi + # Configure ADMINS if [ -n "${ADMIN_NAME}" ]; then sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' "$conf" @@ -65,19 +70,62 @@ if [ -n "${DB_ENGINE}" ]; then fi # Configure TIME_ZONE -if [ -n "${TIMEZONE}" ]; then +if [ -n "${TIMEZONE}" ]; then sed -i '22 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' "$conf" fi +# Configure LANGUAGE_CODE +if [ -n "${LANGUAGE_CODE}" ]; then + sed -i '26 {s/en-us/'"${LANGUAGE_CODE}"'/}' "$conf" +fi + # Configure SECRET_KEY if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then if [ -n "${SECRET_KEY}" ]; then - sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" "$conf" + sed -i "29 {s/SECRET_KEY = ''/SECRET_KEY = '${SECRET_KEY}'/}" "$conf" else patchman-set-secret-key fi fi +# Configure MAX_MIRRORS +if [ -n "${MAX_MIRRORS}" ]; then + sed -i '36 {s/2/'"${MAX_MIRRORS}"'/}' "$conf" +fi + +# Configure MAX_MIRROR_FAILURES +if [ -n "${MAX_MIRROR_FAILURES}" ]; then + sed -i '39 {s/14/'"${MAX_MIRROR_FAILURES}"'/}' "$conf" +fi + +# Configure DAYS_WITHOUT_REPORT +if [ -n "${DAYS_WITHOUT_REPORT}" ]; then + sed -i '42 {s/14/'"${DAYS_WITHOUT_REPORT}"'/}' "$conf" +fi + +# Configure ERRATA_OS_UPDATES +if [ -n "${ERRATA_OS_UPDATES}" ]; then + errataOSUpdates="${ERRATA_OS_UPDATES// /}" + sed -i '45 {s/\[.*\]/['"'${errataOSUpdates//,/\', \'}'"']/}' "$conf" +fi + +# Configure ALMA_RELEASES +if [ -n "${ALMA_RELEASES}" ]; then + sed -i '48 {s/\[.*\]/['"${ALMA_RELEASES}"']/}' "$conf" +fi + +# Configure DEBIAN_CODENAMES +if [ -n "${DEBIAN_CODENAMES}" ]; then + debianCodenames="${DEBIAN_CODENAMES// /}" + sed -i '51 {s/\[.*\]/['"'${debianCodenames//,/\', \'}'"']/}' "$conf" +fi + +# Configure UBUNTU_CODENAMES +if [ -n "${UBUNTU_CODENAMES}" ]; then + ubuntuCodenames="${UBUNTU_CODENAMES// /}" + sed -i '54 {s/\[.*\]/['"'${ubuntuCodenames//,/\', \'}'"']/}' "$conf" +fi + # Configure CACHES if "${USE_CACHE}"; then if [ -n "${REDIS_HOST}" ]; then