From ae8745ab39dd290d2d32be0bb20da06174f275d6 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 16:28:19 +0200 Subject: [PATCH] Checks if cpu_count is implemented On rare occasions, usually on win32, cpu_count will raise a NotImplemented error. If psutil is installed we will use this to get the correct count, otherwise we default to 4 --- django_q/cluster.py | 8 +------- django_q/conf.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 5803e89..7329f81 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -22,12 +22,6 @@ from multiprocessing import Queue, Event, Process, Value, current_process # external import arrow -# optional -try: - import psutil -except ImportError: - psutil = None - # Django from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -37,7 +31,7 @@ from django import db import signing import tasks -from django_q.conf import Conf, redis_client, logger +from django_q.conf import Conf, redis_client, logger, psutil from django_q.models import Task, Success, Schedule from django_q.monitor import Status, Stat diff --git a/django_q/conf.py b/django_q/conf.py index fe142cf..9c57b56 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -2,10 +2,19 @@ import logging from signal import signal from multiprocessing import cpu_count, Queue +# django from django.utils.translation import ugettext_lazy as _ from django.conf import settings + +# external import redis +# optional +try: + import psutil +except ImportError: + psutil = None + class Conf(object): """ @@ -36,8 +45,19 @@ class Conf(object): # Maximum number of tasks that each cluster can work on QUEUE_LIMIT = conf.get('queue_limit', None) - # Number of workers in the pool. Default is cpu count. +2 for monitor and pusher - WORKERS = conf.get('workers', cpu_count()) + # Number of workers in the pool. Default is cpu count if implemented, otherwise 4. + WORKERS = conf.get('workers', False) + if not WORKERS: + try: + WORKERS = cpu_count() + # in rare cases this might fail + except NotImplementedError: + # try psutil + if psutil: + WORKERS = psutil.cpu_count() or 4 + else: + # sensible default + WORKERS = 4 # Sets compression of redis packages COMPRESSED = conf.get('compress', False)