diff --git a/django_q/cluster.py b/django_q/cluster.py index 5803e89..7329f81 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -22,12 +22,6 @@ from multiprocessing import Queue, Event, Process, Value, current_process # external import arrow -# optional -try: - import psutil -except ImportError: - psutil = None - # Django from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -37,7 +31,7 @@ from django import db import signing import tasks -from django_q.conf import Conf, redis_client, logger +from django_q.conf import Conf, redis_client, logger, psutil from django_q.models import Task, Success, Schedule from django_q.monitor import Status, Stat diff --git a/django_q/conf.py b/django_q/conf.py index fe142cf..9c57b56 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -2,10 +2,19 @@ import logging from signal import signal from multiprocessing import cpu_count, Queue +# django from django.utils.translation import ugettext_lazy as _ from django.conf import settings + +# external import redis +# optional +try: + import psutil +except ImportError: + psutil = None + class Conf(object): """ @@ -36,8 +45,19 @@ class Conf(object): # Maximum number of tasks that each cluster can work on QUEUE_LIMIT = conf.get('queue_limit', None) - # Number of workers in the pool. Default is cpu count. +2 for monitor and pusher - WORKERS = conf.get('workers', cpu_count()) + # Number of workers in the pool. Default is cpu count if implemented, otherwise 4. + WORKERS = conf.get('workers', False) + if not WORKERS: + try: + WORKERS = cpu_count() + # in rare cases this might fail + except NotImplementedError: + # try psutil + if psutil: + WORKERS = psutil.cpu_count() or 4 + else: + # sensible default + WORKERS = 4 # Sets compression of redis packages COMPRESSED = conf.get('compress', False)