From 2d0820e80b8d99f3a66d48c697bff313efd800ca Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 19:53:13 +0200 Subject: [PATCH 1/6] Adds cpu affinity for the workers. Courtesy of the optional psutil module. --- django_q/cluster.py | 41 ++++++++++++++++++++++++++++++++++++ django_q/conf.py | 3 +++ django_q/monitor.py | 3 +-- django_q/tests/settings.py | 3 ++- docs/install.rst | 43 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 364e1a7..10e4f55 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -22,6 +22,12 @@ from multiprocessing import Queue, Event, Process, Value, current_process # external import arrow +# optional +try: + import psutil +except ImportError: + psutil = None + # Django from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -203,10 +209,15 @@ class Sentinel(object): def spawn_cluster(self): self.pool = [] Stat(self).save() + # spawn worker pool for i in range(self.pool_size): self.spawn_worker() + # spawn auxiliary self.monitor = self.spawn_monitor() self.pusher = self.spawn_pusher() + # set worker cpu affinity if needed + if psutil and Conf.CPU_AFFINITY: + set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool]) def guard(self): logger.info(_('{} guarding cluster at {}').format(current_process().name, self.pid)) @@ -460,3 +471,33 @@ def scheduler(list_key=Conf.Q_LIST): s.repeats = 0 # save the schedule s.save() + + +def set_cpu_affinity(n, process_ids): + """ + Sets the cpu affinity for the supplied processes. + Requires the optional psutil module. + :param int n: + :param list process_ids: a list of pids + """ + # check if we have the psutil module + if not psutil: + return + # get the available processors + cpu_list = list(range(psutil.cpu_count())) + # affinities of 0 or gte cpu_count, equals to no affinity + if not n or n >= len(cpu_list): + return + # spread the workers over the available processors. + index = 0 + for pid in process_ids: + affinity = [] + for k in range(n): + if index == len(cpu_list): + index = 0 + affinity.append(cpu_list[index]) + index += 1 + if psutil.pid_exists(pid): + p = psutil.Process(pid) + p.cpu_affinity(affinity) + logger.info('{} will use cpu {}'.format(pid, affinity)) \ No newline at end of file diff --git a/django_q/conf.py b/django_q/conf.py index 37e04bf..9606674 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -45,6 +45,9 @@ class Conf(object): # The Django Admin label for this app LABEL = conf.get('label', 'Django Q') + # Sets the number of processors for each worker, defaults to all. + CPU_AFFINITY = conf.get('cpu_affinity', 0) + # Use the secret key for package signing # Django itself should raise an error if it's not configured SECRET_KEY = settings.SECRET_KEY diff --git a/django_q/monitor.py b/django_q/monitor.py index e2b6639..02d6a59 100644 --- a/django_q/monitor.py +++ b/django_q/monitor.py @@ -123,8 +123,7 @@ class Stat(Status): self.task_q_size = sentinel.task_queue.qsize() if sentinel.pusher: self.pusher = sentinel.pusher.pid - for w in sentinel.pool: - self.workers.append(w.pid) + self.workers = [w.pid for w in sentinel.pool] def uptime(self): return (timezone.now() - self.tob).total_seconds() diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py index cf2e3c8..828195d 100644 --- a/django_q/tests/settings.py +++ b/django_q/tests/settings.py @@ -102,4 +102,5 @@ LOGGING = { STATIC_URL = '/static/' # Django Q specific -Q_CLUSTER = {'name': 'django_q_test'} +Q_CLUSTER = {'name': 'django_q_test', + 'cpu_affinity': 1} diff --git a/docs/install.rst b/docs/install.rst index 57d4c18..8abebe3 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -132,6 +132,49 @@ of the cache connection you want to use:: .. tip:: Django Q uses your `SECRET_KEY` to encrypt task packages and prevent task crossover. So make sure you have it set up in your Django settings. +cpu_affinity +~~~~~~~~~~~~ + +Sets the number of processor each worker can use. This does not affect auxiliary process like the sentinel or monitor and is only useful for tweaking the performance of very high traffic clusters. +The default is to use all available processors for all workers. The affinity number has to be higher than zero and less than the total number of processors to have any effect:: + + # processor affinity example. + + 4 processors, 4 workers, cpu_affinity: 1 + + worker 1 cpu [0] + worker 2 cpu [1] + worker 3 cpu [2] + worker 4 cpu [3] + + 4 processors, 4 workers, cpu_affinity: 2 + + worker 1 cpu [0, 1] + worker 2 cpu [2, 3] + worker 3 cpu [0, 1] + worker 4 cpu [2, 3] + + 8 processors, 8 workers, cpu_affinity: 3 + + worker 1 cpu [0, 1, 2] + worker 2 cpu [3, 4, 5] + worker 3 cpu [6, 7, 0] + worker 4 cpu [1, 2, 3] + worker 5 cpu [4, 5, 6] + worker 6 cpu [7, 0, 1] + worker 7 cpu [2, 3, 4] + worker 8 cpu [5, 6, 7] + + +In some cases, setting the cpu affinity for your workers can lead to performance improvements, especially if the load is high and consists of many repeating small tasks. +Start with an affinity of 1 and work your way up. You will have to experiment with what works best for you. +Usually a low affinity number has the most effect. Which could be positive or negative depending on your particular project. + +.. note:: + + The `cpu_affinity` setting requires the optional `psutil `__ module by Giampaolo Rodola'. + You can install it with `pip install psutil`. + Requirements ------------ diff --git a/requirements.txt b/requirements.txt index 8a95852..144d181 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ Django>=1.7.8 future==0.14.3 hiredis==0.2.0 redis==2.10.3 +psutil==3.1.1 + From 2bef73c22f98f84ad29c85b900e92019937188e6 Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 20:01:55 +0200 Subject: [PATCH 2/6] Tweaked cpu affinity docs --- docs/install.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 8abebe3..3125b99 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -136,7 +136,7 @@ cpu_affinity ~~~~~~~~~~~~ Sets the number of processor each worker can use. This does not affect auxiliary process like the sentinel or monitor and is only useful for tweaking the performance of very high traffic clusters. -The default is to use all available processors for all workers. The affinity number has to be higher than zero and less than the total number of processors to have any effect:: +The affinity number has to be higher than zero and less than the total number of processors to have any effect. Defaults to using all processors:: # processor affinity example. @@ -168,12 +168,13 @@ The default is to use all available processors for all workers. The affinity num In some cases, setting the cpu affinity for your workers can lead to performance improvements, especially if the load is high and consists of many repeating small tasks. Start with an affinity of 1 and work your way up. You will have to experiment with what works best for you. -Usually a low affinity number has the most effect. Which could be positive or negative depending on your particular project. +Usually a low affinity number has the most effect. Which could be a positive or negative one, depending on your particular project. .. note:: The `cpu_affinity` setting requires the optional `psutil `__ module by Giampaolo Rodola'. - You can install it with `pip install psutil`. + You can install it with: + ``pip install psutil`` Requirements ------------ From 1f28bf1c6874889a1a7f6e2e2d933e3e4c0fca0f Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 20:15:46 +0200 Subject: [PATCH 3/6] Travis doesn't like cpu affinity. I'm guessing --- django_q/tests/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py index 828195d..4cfc90e 100644 --- a/django_q/tests/settings.py +++ b/django_q/tests/settings.py @@ -103,4 +103,4 @@ STATIC_URL = '/static/' # Django Q specific Q_CLUSTER = {'name': 'django_q_test', - 'cpu_affinity': 1} + 'cpu_affinity': 0} From 464255fa881f7d1731f51f06354007ba479f7eb8 Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 22:06:17 +0200 Subject: [PATCH 4/6] Changed cpu affinity docs slightly --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index 3125b99..7f8d4bb 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -168,7 +168,7 @@ The affinity number has to be higher than zero and less than the total number of In some cases, setting the cpu affinity for your workers can lead to performance improvements, especially if the load is high and consists of many repeating small tasks. Start with an affinity of 1 and work your way up. You will have to experiment with what works best for you. -Usually a low affinity number has the most effect. Which could be a positive or negative one, depending on your particular project. +As a rule of thumb; cpu_affinity 1 favors repetitive short running tasks,while no affinity benefits longer running tasks. .. note:: From a5580c75a199f276af6639d159a9d7ef9958d663 Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 22:17:13 +0200 Subject: [PATCH 5/6] Workaround for Travis Travis doesn't support cpu affinity in their virtuals so I've added a setting TESTING that will bypass the actual affinity setting, but will still cover most of the code. Might be useful for future workarounds. --- django_q/cluster.py | 6 ++++-- django_q/conf.py | 3 +++ django_q/tests/settings.py | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 10e4f55..3470ccb 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -473,12 +473,13 @@ def scheduler(list_key=Conf.Q_LIST): s.save() -def set_cpu_affinity(n, process_ids): +def set_cpu_affinity(n, process_ids, actual=not Conf.TESTING): """ Sets the cpu affinity for the supplied processes. Requires the optional psutil module. :param int n: :param list process_ids: a list of pids + :param bool actual: Test workaround for Travis not supporting cpu affinity """ # check if we have the psutil module if not psutil: @@ -499,5 +500,6 @@ def set_cpu_affinity(n, process_ids): index += 1 if psutil.pid_exists(pid): p = psutil.Process(pid) - p.cpu_affinity(affinity) + if actual: + p.cpu_affinity(affinity) logger.info('{} will use cpu {}'.format(pid, affinity)) \ No newline at end of file diff --git a/django_q/conf.py b/django_q/conf.py index 9606674..65f44c3 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -67,6 +67,9 @@ class Conf(object): STOPPED = _('Stopped') STOPPING = _('Stopping') + # to manage workarounds during testing + TESTING = conf.get('testings', False) + # logger logger = logging.getLogger('django-q') diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py index 4cfc90e..7999a78 100644 --- a/django_q/tests/settings.py +++ b/django_q/tests/settings.py @@ -103,4 +103,5 @@ STATIC_URL = '/static/' # Django Q specific Q_CLUSTER = {'name': 'django_q_test', - 'cpu_affinity': 0} + 'cpu_affinity': 1, + 'testing': True} From 9857f33e3e2b0a6438982724fa89ac526eaa0300 Mon Sep 17 00:00:00 2001 From: Ilan Date: Wed, 15 Jul 2015 22:34:40 +0200 Subject: [PATCH 6/6] Workaround for Travis fixed typo --- django_q/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/conf.py b/django_q/conf.py index 65f44c3..b6b9fd1 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -68,7 +68,7 @@ class Conf(object): STOPPING = _('Stopping') # to manage workarounds during testing - TESTING = conf.get('testings', False) + TESTING = conf.get('testing', False) # logger