fork/django-q2
1
0
Fork 0
mirror of https://github.com/django-q2/django-q2.git synced 2026-10-06 16:38:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Replacing format with f-strings

Browse Source
This commit is contained in:
Ilan
2020-02-17 16:59:12 +01:00
parent a1543c8d82
commit 0bca3cbd7a
7 changed files with 683 additions and 426 deletions
Download Patch File Download Diff File Expand all files Collapse all files
django_q/conf.py
+63 -53
View File
@@ -1,15 +1,16 @@
import logging
from copy import deepcopy
from signal import signal
from multiprocessing import cpu_count
# django
from django.utils.translation import gettext_lazy as _
from django.conf import settings
# external
import os
from copy import deepcopy
from multiprocessing import cpu_count
from signal import signal
import pkg_resources
from django.conf import settings
# django
from django.utils.translation import gettext_lazy as _
# local
from django_q.queues import Queue
@@ -25,64 +26,65 @@ class Conf(object):
"""
Configuration class
"""
try:
conf = settings.Q_CLUSTER
except AttributeError:
conf = {}
# Redis server configuration . Follows standard redis keywords
REDIS = conf.get('redis', {})
REDIS = conf.get("redis", {})
# Support for Django-Redis connections
DJANGO_REDIS = conf.get('django_redis', None)
DJANGO_REDIS = conf.get("django_redis", None)
# Disque broker
DISQUE_NODES = conf.get('disque_nodes', None)
DISQUE_NODES = conf.get("disque_nodes", None)
# Optional Authentication
DISQUE_AUTH = conf.get('disque_auth', None)
DISQUE_AUTH = conf.get("disque_auth", None)
# Optional Fast acknowledge
DISQUE_FASTACK = conf.get('disque_fastack', False)
DISQUE_FASTACK = conf.get("disque_fastack", False)
# IronMQ broker
IRON_MQ = conf.get('iron_mq', None)
IRON_MQ = conf.get("iron_mq", None)
# SQS broker
SQS = conf.get('sqs', None)
SQS = conf.get("sqs", None)
# ORM broker
ORM = conf.get('orm', None)
ORM = conf.get("orm", None)
# Custom broker class
BROKER_CLASS = conf.get('broker_class', None)
BROKER_CLASS = conf.get("broker_class", None)
# Database Poll
POLL = conf.get('poll', 0.2)
POLL = conf.get("poll", 0.2)
# MongoDB broker
MONGO = conf.get('mongo', None)
MONGO_DB = conf.get('mongo_db', None)
MONGO = conf.get("mongo", None)
MONGO_DB = conf.get("mongo_db", None)
# Name of the cluster or site. For when you run multiple sites on one redis server
PREFIX = conf.get('name', 'default')
PREFIX = conf.get("name", "default")
# Log output level
LOG_LEVEL = conf.get('log_level', 'INFO')
LOG_LEVEL = conf.get("log_level", "INFO")
# Maximum number of successful tasks kept in the database. 0 saves everything. -1 saves none
# Failures are always saved
SAVE_LIMIT = conf.get('save_limit', 250)
SAVE_LIMIT = conf.get("save_limit", 250)
# Guard loop sleep in seconds. Should be between 0 and 60 seconds.
GUARD_CYCLE = conf.get('guard_cycle', 0.5)
GUARD_CYCLE = conf.get("guard_cycle", 0.5)
# Disable the scheduler
SCHEDULER = conf.get('scheduler', True)
SCHEDULER = conf.get("scheduler", True)
# Number of workers in the pool. Default is cpu count if implemented, otherwise 4.
WORKERS = conf.get('workers', False)
WORKERS = conf.get("workers", False)
if not WORKERS:
try:
WORKERS = cpu_count()
@@ -96,62 +98,62 @@ class Conf(object):
WORKERS = 4
# Option to undaemonize the workers and allow them to spawn child processes
DAEMONIZE_WORKERS = conf.get('daemonize_workers', True)
DAEMONIZE_WORKERS = conf.get("daemonize_workers", True)
# Maximum number of tasks that each cluster can work on
QUEUE_LIMIT = conf.get('queue_limit', int(WORKERS) ** 2)
QUEUE_LIMIT = conf.get("queue_limit", int(WORKERS) ** 2)
# Sets compression of redis packages
COMPRESSED = conf.get('compress', False)
COMPRESSED = conf.get("compress", False)
# Number of tasks each worker can handle before it gets recycled. Useful for releasing memory
RECYCLE = conf.get('recycle', 500)
RECYCLE = conf.get("recycle", 500)
# Number of seconds to wait for a worker to finish.
TIMEOUT = conf.get('timeout', None)
TIMEOUT = conf.get("timeout", None)
# Whether to acknowledge unsuccessful tasks.
# This causes failed tasks to be considered delivered, thereby removing them from
# the task queue. Defaults to False.
ACK_FAILURES = conf.get('ack_failures', False)
ACK_FAILURES = conf.get("ack_failures", False)
# Number of seconds to wait for acknowledgement before retrying a task
# Only works with brokers that guarantee delivery. Defaults to 60 seconds.
RETRY = conf.get('retry', 60)
RETRY = conf.get("retry", 60)
# Sets the amount of tasks the cluster will try to pop off the broker.
# If it supports bulk gets.
BULK = conf.get('bulk', 1)
BULK = conf.get("bulk", 1)
# The Django Admin label for this app
LABEL = conf.get('label', 'Django Q')
LABEL = conf.get("label", "Django Q")
# Sets the number of processors for each worker, defaults to all.
CPU_AFFINITY = conf.get('cpu_affinity', 0)
CPU_AFFINITY = conf.get("cpu_affinity", 0)
# Global sync option to for debugging
SYNC = conf.get('sync', False)
SYNC = conf.get("sync", False)
# The Django cache to use
CACHE = conf.get('cache', 'default')
CACHE = conf.get("cache", "default")
# Use the cache as result backend. Can be 'True' or an integer representing the global cache timeout.
# i.e 'cached: 60' , will make all results go the cache and expire in 60 seconds.
CACHED = conf.get('cached', False)
CACHED = conf.get("cached", False)
# If set to False the scheduler won't execute tasks in the past.
# Instead it will run once and reschedule the next run in the future. Defaults to True.
CATCH_UP = conf.get('catch_up', True)
CATCH_UP = conf.get("catch_up", True)
# Use the secret key for package signing
# Django itself should raise an error if it's not configured
SECRET_KEY = settings.SECRET_KEY
# The redis stats key
Q_STAT = 'django_q:{}:cluster'.format(PREFIX)
Q_STAT = f"django_q:{PREFIX}:cluster"
# Optional error reporting setup
ERROR_REPORTER = conf.get('error_reporter', {})
ERROR_REPORTER = conf.get("error_reporter", {})
# OSX doesn't implement qsize because of missing sem_getvalue()
try:
@@ -160,28 +162,33 @@ class Conf(object):
QSIZE = False
# Getting the signal names
SIGNAL_NAMES = dict((getattr(signal, n), n) for n in dir(signal) if n.startswith('SIG') and '_' not in n)
SIGNAL_NAMES = dict(
(getattr(signal, n), n)
for n in dir(signal)
if n.startswith("SIG") and "_" not in n
)
# Translators: Cluster status descriptions
STARTING = _('Starting')
WORKING = _('Working')
STARTING = _("Starting")
WORKING = _("Working")
IDLE = _("Idle")
STOPPED = _('Stopped')
STOPPING = _('Stopping')
STOPPED = _("Stopped")
STOPPING = _("Stopping")
# to manage workarounds during testing
TESTING = conf.get('testing', False)
TESTING = conf.get("testing", False)
# logger
logger = logging.getLogger('django-q')
logger = logging.getLogger("django-q")
# Set up standard logging handler in case there is none
if not logger.handlers:
logger.setLevel(level=getattr(logging, Conf.LOG_LEVEL))
logger.propagate = False
formatter = logging.Formatter(fmt='%(asctime)s [Q] %(levelname)s %(message)s',
datefmt='%H:%M:%S')
formatter = logging.Formatter(
fmt="%(asctime)s [Q] %(levelname)s %(message)s", datefmt="%H:%M:%S"
)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
@@ -209,7 +216,8 @@ if Conf.ERROR_REPORTER:
# and instantiate an ErrorReporter using the provided config
for name, conf in error_conf.items():
for entry in pkg_resources.iter_entry_points(
'djangoq.errorreporters', name):
"djangoq.errorreporters", name
):
Reporter = entry.load()
reporters.append(Reporter(**conf))
error_reporter = ErrorReporter(reporters)
@@ -221,9 +229,11 @@ else:
# get parent pid compatibility
def get_ppid():
if hasattr(os, 'getppid'):
if hasattr(os, "getppid"):
return os.getppid()
elif psutil:
return psutil.Process(os.getpid()).ppid()
else:
raise OSError('Your OS does not support `os.getppid`. Please install `psutil` as an alternative provider.')
raise OSError(
"Your OS does not support `os.getppid`. Please install `psutil` as an alternative provider."
)