mirror of
https://github.com/django-q2/django-q2.git
synced 2026-10-08 16:58:12 +08:00
- Added logging with coloredlogs (for now)
- Added task model and admin - Moved pusher into it's own process
This commit is contained in:
Ilan Steemers
+77
-39
@@ -1,56 +1,69 @@
|
||||
import importlib
|
||||
from time import sleep
|
||||
from multiprocessing import Queue, Process, current_process, cpu_count
|
||||
from random import randint
|
||||
from multiprocessing import Queue, Process, Event, current_process, cpu_count
|
||||
import sys
|
||||
import signal
|
||||
from uuid import uuid4
|
||||
import ujson as json
|
||||
import logging
|
||||
from time import sleep
|
||||
|
||||
import jsonpickle as json
|
||||
|
||||
import coloredlogs
|
||||
import redis
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import Task
|
||||
|
||||
from .humanhash import uuid
|
||||
|
||||
r = redis.StrictRedis(decode_responses=True)
|
||||
secret = settings.SECRET_KEY
|
||||
prefix = 'django_q'
|
||||
q_list = '{}:q'.format(prefix)
|
||||
|
||||
logger = logging.getLogger('django-q')
|
||||
coloredlogs.install(level=logging.INFO)
|
||||
|
||||
|
||||
def test():
|
||||
for i in range(4):
|
||||
defer(u'testq.tasks.multiply', 5, i)
|
||||
for i in range(20):
|
||||
defer(u'testq.tasks.multiply', 5, i * randint(2, 100))
|
||||
|
||||
|
||||
def defer(func, *args, **kwargs):
|
||||
pack = json.dumps([uuid4().urn, func, args, kwargs])
|
||||
# [name, func, args, kwargs, started, finished, result]
|
||||
pack = json.dumps([uuid()[0], func, args, kwargs, timezone.now()])
|
||||
r.rpush(q_list, pack)
|
||||
logger.debug('Pushed {}'.format(pack))
|
||||
|
||||
|
||||
class Worker(object):
|
||||
def __init__(self):
|
||||
signal.signal(signal.SIGTERM, self.sig_handler)
|
||||
signal.signal(signal.SIGINT, self.sig_handler)
|
||||
self.running = True
|
||||
self.stable_size = cpu_count() - 2
|
||||
self.stable_size = cpu_count()
|
||||
self.stable = []
|
||||
self.task_queue = Queue()
|
||||
self.done_queue = Queue()
|
||||
self.fail_queue = Queue()
|
||||
self.failure_monitor_pid = None
|
||||
self.success_monitor_pid = None
|
||||
# Spawn work horses
|
||||
for i in range(self.stable_size):
|
||||
self.spawn_horse()
|
||||
# Spawn monitors
|
||||
self.success_monitor_pid = None
|
||||
self.spawn_success_monitor()
|
||||
self.failure_monitor_pid = None
|
||||
self.spawn_failure_monitor()
|
||||
# Attach signal handler
|
||||
signal.signal(signal.SIGTERM, self.sig_handler)
|
||||
signal.signal(signal.SIGINT, self.sig_handler)
|
||||
# Keep popping Redis
|
||||
# Spawn pusher
|
||||
self.pusher_pid = None
|
||||
self.pusher_stop = Event()
|
||||
self.spawn_pusher()
|
||||
# Monitor process health
|
||||
while self.running:
|
||||
self.stable_boy()
|
||||
sleep(0.2)
|
||||
task = r.lpop(q_list)
|
||||
if task:
|
||||
self.task_queue.put(task)
|
||||
sleep(1)
|
||||
|
||||
def spawn_process(self, target, *args):
|
||||
# This is just for PyCharm to not crash. Ignore it.
|
||||
@@ -64,6 +77,9 @@ class Worker(object):
|
||||
p.start()
|
||||
return p.pid
|
||||
|
||||
def spawn_pusher(self):
|
||||
self.pusher_pid = self.spawn_process(self.pusher, self.task_queue, self.pusher_stop)
|
||||
|
||||
def spawn_horse(self):
|
||||
self.spawn_process(self.horse, self.task_queue, self.done_queue, self.fail_queue)
|
||||
|
||||
@@ -76,51 +92,71 @@ class Worker(object):
|
||||
def reincarnate(self, pid):
|
||||
if pid == self.success_monitor_pid:
|
||||
self.spawn_success_monitor()
|
||||
logger.warn("reincarnated success monitor after death of {}".format(pid))
|
||||
elif pid == self.failure_monitor_pid:
|
||||
self.spawn_failure_monitor()
|
||||
logger.warn("reincarnated failure monitor after death of {}".format(pid))
|
||||
else:
|
||||
self.spawn_horse()
|
||||
logger.warn("reincarnated work horse after death of {}".format(pid))
|
||||
|
||||
@staticmethod
|
||||
def pusher(task_queue, e):
|
||||
logger.info('{} pushing tasks at {}'.format(current_process().name, current_process().pid))
|
||||
while not e.is_set():
|
||||
task = r.blpop(q_list, 1)
|
||||
if task:
|
||||
task_queue.put(task[1])
|
||||
logger.debug('queueing {}'.format(task[1]))
|
||||
|
||||
@staticmethod
|
||||
def success_monitor(done_queue):
|
||||
name = current_process().name
|
||||
print("{} monitoring successes at {}".format(name, current_process().pid))
|
||||
for result in iter(done_queue.get, 'STOP'):
|
||||
task = result[0]
|
||||
res = result[1]
|
||||
print("Success [{}:{} - {}]".format(task[1], task[0], res))
|
||||
print("{} stopped".format(name))
|
||||
logger.info("{} monitoring successes at {}".format(name, current_process().pid))
|
||||
for task in iter(done_queue.get, 'STOP'):
|
||||
logger.info("Finished [{}:{}]".format(task[1], task[0]))
|
||||
Task.objects.create(name=task[0], func=task[1], task=json.dumps(task),
|
||||
started=task[4],
|
||||
stopped=task[5])
|
||||
logger.info("{} stopped".format(name))
|
||||
|
||||
@staticmethod
|
||||
def failure_monitor(fail_queue):
|
||||
name = current_process().name
|
||||
print("{} monitoring failures at {}".format(name, current_process().pid))
|
||||
for result in iter(fail_queue.get, 'STOP'):
|
||||
task = result[0]
|
||||
e = result[1]
|
||||
print("Failure [{}:{} - {}]".format(task[1], task[0], e))
|
||||
print("{} stopped".format(name))
|
||||
logger.info("{} monitoring failures at {}".format(name, current_process().pid))
|
||||
for task in iter(fail_queue.get, 'STOP'):
|
||||
logger.error("Failure [{}:{} - {}]".format(task[1], task[0], task[6]))
|
||||
Task.objects.create(name=task[0], func=task[1], task=json.dumps(task),
|
||||
started=task[4],
|
||||
stopped=task[5], success=False)
|
||||
logger.info("{} stopped".format(name))
|
||||
|
||||
@staticmethod
|
||||
def horse(task_queue, done_queue, fail_queue):
|
||||
name = current_process().name
|
||||
print('{} ready for work at {}'.format(name, current_process().pid))
|
||||
logger.info('{} ready for work at {}'.format(name, current_process().pid))
|
||||
for pack in iter(task_queue.get, 'STOP'):
|
||||
task = json.loads(pack)
|
||||
uid = task[0]
|
||||
try:
|
||||
task = json.loads(pack)
|
||||
except TypeError as e:
|
||||
logger.error(e)
|
||||
continue
|
||||
func = task[1]
|
||||
module, func = func.rsplit('.', 1)
|
||||
args = task[2]
|
||||
kwargs = task[3]
|
||||
print(name, 'Processing [{}:{}]'.format(func, uid))
|
||||
logger.info('{} processing [{}:{}]'.format(name, func, task[0]))
|
||||
task.append(timezone.now())
|
||||
try:
|
||||
m = importlib.import_module(module)
|
||||
f = getattr(m, func)
|
||||
result = f(*args, **kwargs)
|
||||
done_queue.put((task, result))
|
||||
task.append(result)
|
||||
done_queue.put(task)
|
||||
except TypeError as e:
|
||||
fail_queue.put((task, e))
|
||||
print(name, 'Stopped')
|
||||
task.append(e)
|
||||
fail_queue.put(task)
|
||||
logger.info('{} Stopped'.format(name))
|
||||
|
||||
def stable_boy(self):
|
||||
# Check if all the horses are alive
|
||||
@@ -135,18 +171,20 @@ class Worker(object):
|
||||
def stop(self):
|
||||
# Send the STOP signal to the stable
|
||||
self.running = False
|
||||
print('Stopping')
|
||||
logger.info('Stopping')
|
||||
# Wait for all the workers to finish the queue
|
||||
for p in self.stable:
|
||||
if p.pid == self.failure_monitor_pid:
|
||||
self.fail_queue.put('STOP')
|
||||
elif p.pid == self.success_monitor_pid:
|
||||
self.done_queue.put('STOP')
|
||||
elif p.pid == self.pusher_pid:
|
||||
self.pusher_stop.set()
|
||||
else:
|
||||
self.task_queue.put('STOP')
|
||||
p.join()
|
||||
|
||||
print('Goodbye. Have a wonderful time.')
|
||||
logger.info('Goodbye. Have a wonderful time.')
|
||||
|
||||
def sig_handler(self, signum, frame):
|
||||
self.stop()
|
||||
|
||||
Reference in New Issue
Block a user