fork/django-q2
1
0
Fork 0
mirror of https://github.com/django-q2/django-q2.git synced 2026-10-11 17:30:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Linting and hints

Browse Source
This commit is contained in:
Ilan Steemers
2020-06-12 12:33:04 +02:00
parent 2f2269c27f
commit ef4c7876ae
11 changed files with 148 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
django_q/cluster.py
+36 -33
View File
@@ -1,5 +1,4 @@
import ast
# Standard
import importlib
import signal
@@ -11,7 +10,6 @@ from time import sleep
# external
import arrow
# Django
from django import db
from django.conf import settings
@@ -20,7 +18,7 @@ from django.utils.translation import gettext_lazy as _
# Local
import django_q.tasks
from django_q.brokers import get_broker
from django_q.brokers import get_broker, Broker
from django_q.conf import Conf, logger, psutil, get_ppid, error_reporter
from django_q.humanhash import humanize
from django_q.models import Task, Success, Schedule
@@ -31,7 +29,7 @@ from django_q.status import Stat, Status
class Cluster:
def __init__(self, broker=None):
def __init__(self, broker: Broker = None):
self.broker = broker or get_broker()
self.sentinel = None
self.stop_event = None
@@ -43,7 +41,7 @@ class Cluster:
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
def start(self):
def start(self) -> int:
# Start Sentinel
self.stop_event = Event()
self.start_event = Event()
@@ -63,7 +61,7 @@ class Cluster:
sleep(0.1)
return self.pid
def stop(self):
def stop(self) -> bool:
if not self.sentinel.is_alive():
return False
logger.info(_(f"Q Cluster {self.name} stopping."))
@@ -83,46 +81,46 @@ class Cluster:
self.stop()
@property
def stat(self):
def stat(self) -> Status:
if self.sentinel:
return Stat.get(pid=self.pid, cluster_id=self.cluster_id)
return Status(pid=self.pid, cluster_id=self.cluster_id)
@property
def name(self):
def name(self) -> str:
return humanize(self.cluster_id.hex)
@property
def is_starting(self):
def is_starting(self) -> bool:
return self.stop_event and self.start_event and not self.start_event.is_set()
@property
def is_running(self):
def is_running(self) -> bool:
return self.stop_event and self.start_event and self.start_event.is_set()
@property
def is_stopping(self):
def is_stopping(self) -> bool:
return (
self.stop_event
and self.start_event
and self.start_event.is_set()
and self.stop_event.is_set()
self.stop_event
and self.start_event
and self.start_event.is_set()
and self.stop_event.is_set()
)
@property
def has_stopped(self):
def has_stopped(self) -> bool:
return self.start_event is None and self.stop_event is None and self.sentinel
class Sentinel:
def __init__(
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -154,7 +152,7 @@ class Sentinel:
self.spawn_cluster()
self.guard()
def status(self):
def status(self) -> str:
if not self.start_event.is_set() and not self.stop_event.is_set():
return Conf.STARTING
elif self.start_event.is_set() and not self.stop_event.is_set():
@@ -166,7 +164,7 @@ class Sentinel:
return Conf.STOPPING
return Conf.STOPPED
def spawn_process(self, target, *args):
def spawn_process(self, target, *args) -> Process:
"""
:type target: function or class
"""
@@ -179,7 +177,7 @@ class Sentinel:
p.start()
return p
def spawn_pusher(self):
def spawn_pusher(self) -> Process:
return self.spawn_process(pusher, self.task_queue, self.event_out, self.broker)
def spawn_worker(self):
@@ -187,7 +185,7 @@ class Sentinel:
worker, self.task_queue, self.result_queue, Value("f", -1), self.timeout
)
def spawn_monitor(self):
def spawn_monitor(self) -> Process:
return self.spawn_process(monitor, self.result_queue, self.broker)
def reincarnate(self, process):
@@ -310,9 +308,10 @@ class Sentinel:
Stat(self).save()
def pusher(task_queue, event, broker=None):
def pusher(task_queue: Queue, event: Event, broker: Broker = None):
"""
Pulls tasks of the broker and puts them in the task queue
:type broker:
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
@@ -545,9 +544,9 @@ def scheduler(broker=None):
try:
with db.transaction.atomic(using=Schedule.objects.db):
for s in (
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
):
args = ()
kwargs = {}
@@ -588,7 +587,11 @@ def scheduler(broker=None):
break
# arrow always returns a tz aware datetime, and we don't want
# this when we explicitly configured django with USE_TZ=False
s.next_run = next_run.datetime if settings.USE_TZ else next_run.datetime.replace(tzinfo=None)
s.next_run = (
next_run.datetime
if settings.USE_TZ
else next_run.datetime.replace(tzinfo=None)
)
s.repeats += -1
# send it to the cluster
q_options["broker"] = broker
@@ -635,7 +638,7 @@ def close_old_django_connections():
db.close_old_connections()
def set_cpu_affinity(n, process_ids, actual=not Conf.TESTING):
def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING):
"""
Sets the cpu affinity for the supplied processes.
Requires the optional psutil module.