fixes connection creation on import

This commit is contained in:
Ilan Steemers
2015-08-31 15:15:24 +02:00
parent 9e04d8c9a8
commit da20ebe734
4 changed files with 29 additions and 14 deletions
+13 -7
View File
@@ -37,8 +37,8 @@ from django_q.brokers import get_broker
class Cluster(object): class Cluster(object):
def __init__(self, broker=get_broker()): def __init__(self, broker=None):
self.broker = broker self.broker = broker or get_broker()
self.sentinel = None self.sentinel = None
self.stop_event = None self.stop_event = None
self.start_event = None self.start_event = None
@@ -106,14 +106,14 @@ class Cluster(object):
class Sentinel(object): class Sentinel(object):
def __init__(self, stop_event, start_event, broker=get_broker(), timeout=Conf.TIMEOUT, start=True): def __init__(self, stop_event, start_event, broker=None, timeout=Conf.TIMEOUT, start=True):
# Make sure we catch signals for the pool # Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.pid = current_process().pid self.pid = current_process().pid
self.parent_pid = get_ppid() self.parent_pid = get_ppid()
self.name = current_process().name self.name = current_process().name
self.broker = broker self.broker = broker or get_broker()
self.reincarnations = 0 self.reincarnations = 0
self.tob = timezone.now() self.tob = timezone.now()
self.stop_event = stop_event self.stop_event = stop_event
@@ -287,12 +287,14 @@ class Sentinel(object):
Stat(self).save() Stat(self).save()
def pusher(task_queue, event, broker=get_broker()): def pusher(task_queue, event, broker=None):
""" """
Pulls tasks of the Redis List and puts them in the task queue Pulls tasks of the Redis List and puts them in the task queue
:type task_queue: multiprocessing.Queue :type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event :type event: multiprocessing.Event
""" """
if not broker:
broker = get_broker()
logger.info(_('{} pushing tasks at {}').format(current_process().name, current_process().pid)) logger.info(_('{} pushing tasks at {}').format(current_process().name, current_process().pid))
while True: while True:
try: try:
@@ -318,11 +320,13 @@ def pusher(task_queue, event, broker=get_broker()):
logger.info(_("{} stopped pushing tasks").format(current_process().name)) logger.info(_("{} stopped pushing tasks").format(current_process().name))
def monitor(result_queue, broker=get_broker()): def monitor(result_queue, broker=None):
""" """
Gets finished tasks from the result queue and saves them to Django Gets finished tasks from the result queue and saves them to Django
:type result_queue: multiprocessing.Queue :type result_queue: multiprocessing.Queue
""" """
if not broker:
broker = get_broker()
name = current_process().name name = current_process().name
logger.info(_("{} monitoring at {}").format(name, current_process().pid)) logger.info(_("{} monitoring at {}").format(name, current_process().pid))
db.close_old_connections() db.close_old_connections()
@@ -413,10 +417,12 @@ def save_task(task):
logger.error(e) logger.error(e)
def scheduler(broker=get_broker()): def scheduler(broker=None):
""" """
Creates a task from a schedule at the scheduled time and schedules next run Creates a task from a schedule at the scheduled time and schedules next run
""" """
if not broker:
broker = get_broker()
try: try:
for s in Schedule.objects.exclude(repeats=0).filter(next_run__lt=timezone.now()): for s in Schedule.objects.exclude(repeats=0).filter(next_run__lt=timezone.now()):
args = () args = ()
+7 -3
View File
@@ -10,13 +10,15 @@ from django.utils import timezone
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
# local # local
from django_q.conf import Conf, redis_client from django_q.conf import Conf
from django_q.status import Stat from django_q.status import Stat
from django_q.brokers import get_broker from django_q.brokers import get_broker
from django_q import models from django_q import models
def monitor(run_once=False, broker=get_broker()): def monitor(run_once=False, broker=None):
if not broker:
broker = get_broker()
term = Terminal() term = Terminal()
broker.ping() broker.ping()
with term.fullscreen(), term.hidden_cursor(), term.cbreak(): with term.fullscreen(), term.hidden_cursor(), term.cbreak():
@@ -85,7 +87,9 @@ def monitor(run_once=False, broker=get_broker()):
val = term.inkey(timeout=1) val = term.inkey(timeout=1)
def info(broker=get_broker()): def info(broker=None):
if not broker:
broker = get_broker()
term = Terminal() term = Terminal()
broker.ping() broker.ping()
stat = Stat.get_all(broker=broker) stat = Stat.get_all(broker=broker)
+6 -2
View File
@@ -72,12 +72,14 @@ class Stat(Status):
return self.done_q_size + self.task_q_size == 0 return self.done_q_size + self.task_q_size == 0
@staticmethod @staticmethod
def get(cluster_id, broker=get_broker()): def get(cluster_id, broker=None):
""" """
gets the current status for the cluster gets the current status for the cluster
:param cluster_id: id of the cluster :param cluster_id: id of the cluster
:return: Stat or Status :return: Stat or Status
""" """
if not broker:
broker = get_broker()
pack = broker.get_stat(Stat.get_key(cluster_id)) pack = broker.get_stat(Stat.get_key(cluster_id))
if pack: if pack:
try: try:
@@ -87,12 +89,14 @@ class Stat(Status):
return Status(cluster_id) return Status(cluster_id)
@staticmethod @staticmethod
def get_all(broker=get_broker()): def get_all(broker=None):
""" """
Get the status for all currently running clusters with the same prefix Get the status for all currently running clusters with the same prefix
and secret key. and secret key.
:return: list of type Stat :return: list of type Stat
""" """
if not broker:
broker = get_broker()
stats = [] stats = []
packs = broker.get_stats('{}:*'.format(Conf.Q_STAT)) or [] packs = broker.get_stats('{}:*'.format(Conf.Q_STAT)) or []
for pack in packs: for pack in packs:
+3 -2
View File
@@ -152,16 +152,17 @@ def delete_group(group_id, tasks=False):
return Task.delete_group(group_id, tasks) return Task.delete_group(group_id, tasks)
def queue_size(broker=get_broker()): def queue_size(broker=None):
""" """
Returns the current queue size. Returns the current queue size.
Note that this doesn't count any tasks currently being processed by workers. Note that this doesn't count any tasks currently being processed by workers.
:param list_key: optional list key
:param broker: optional broker :param broker: optional broker
:return: current queue size :return: current queue size
:rtype: int :rtype: int
""" """
if not broker:
broker = get_broker()
return broker.queue_size() return broker.queue_size()