Merge pull request #261 from danielwelch/error-reporter

Error reporter

django_q/cluster.py

@@ -25,7 +25,7 @@ import signing
 import tasks
 
 from django_q.compat import range
-from django_q.conf import Conf, logger, psutil, get_ppid, rollbar
+from django_q.conf import Conf, logger, psutil, get_ppid, error_reporter, rollbar
 from django_q.models import Task, Success, Schedule
 from django_q.status import Stat, Status
 from django_q.brokers import get_broker
@@ -365,6 +365,8 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
                 f = getattr(m, func)
             except (ValueError, ImportError, AttributeError) as e:
                 result = (e, False)
+                if error_reporter:
+                    error_reporter.report()
                 if rollbar:
                     rollbar.report_exc_info()
         # We're still going
@@ -380,6 +382,8 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
                 result = (res, True)
             except Exception as e:
                 result = ('{}'.format(e), False)
+                if error_reporter:
+                    error_reporter.report()
                 if rollbar:
                     rollbar.report_exc_info()
         # Process result

django_q/conf.py

@@ -9,6 +9,7 @@ from django.conf import settings
 
 # external
 import os
+import pkg_resources
 
 # optional
 try:
@@ -141,9 +142,12 @@ class Conf(object):
     # The redis stats key
     Q_STAT = 'django_q:{}:cluster'.format(PREFIX)
 
-    # Optional Rollbar key
+    # Optional rollbar key
     ROLLBAR = conf.get('rollbar', {})
 
+    # Optional error reporting setup
+    ERROR_REPORTER = conf.get('error_reporter', {})
+
     # OSX doesn't implement qsize because of missing sem_getvalue()
     try:
         QSIZE = Queue().qsize() == 0
@@ -177,6 +181,7 @@ if not logger.handlers:
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 
+
 # rollbar
 if Conf.ROLLBAR:
     rollbar_conf = deepcopy(Conf.ROLLBAR)
@@ -190,6 +195,38 @@ else:
     rollbar = None
 
 
+# Error Reporting Interface
+class ErrorReporter(object):
+
+    # initialize with iterator of reporters (better name, targets?)
+    def __init__(self, reporters):
+        self.targets = [target for target in reporters]
+
+    # report error to all configured targets
+    def report(self):
+        for t in self.targets:
+            t.report()
+
+
+# error reporting setup (sentry or rollbar)
+if Conf.ERROR_REPORTER:
+    error_conf = deepcopy(Conf.ERROR_REPORTER)
+    try:
+        reporters = []
+        # iterate through the configured error reporters,
+        # and instantiate an ErrorReporter using the provided config
+        for name, conf in error_conf.items():
+            Reporter = pkg_resources.iter_entry_points(
+                    'djangoq.errorreporters', name).load()
+            e = Reporter(**conf)
+            reporters.append(e)
+        error_reporter = ErrorReporter(reporters)
+    except ImportError:
+        error_reporter = None
+else:
+    error_reporter = None
+
+
 # get parent pid compatibility
 def get_ppid():
     if hasattr(os, 'getppid'):

docs/configure.rst

@@ -353,6 +353,26 @@ scheduler
 You can disable the scheduler by setting this option to ``False``. This will reduce a little overhead if you're not using schedules, but is most useful if you want to temporarily disable all schedules.
 Defaults to ``True``
 
+.. _error_reporter:
+
+error_reporter
+~~~~~~~~~~~~~~
+You can redirect worker exceptions directly to various error reportes (for example, `Rollbar <https://rollbar.com/>` or `Sentry <https://docs.sentry.io/>`) by installing Django Q with the necessary `extras <https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies>`.
+
+To enable installed error reporters, you must provide the configuration settings required by an error reporter extension::
+
+    # error_reporter config--rollbar example
+    Q_CLUSTER = {
+        'error_reporter': {
+            'rollbar': {
+                'access_token': '32we33a92a5224jiww8982',
+                'environment': 'Django-Q'
+            }
+        } 
+    }
+
+For more information on error reporters and developing error reporting plugins for Django Q, see :doc:`errors<errors>`.
+
 rollbar
 ~~~~~~~
 You can redirect worker exceptions directly to your `Rollbar <https://rollbar.com/>`__ dashboard by installing the python notifier with ``pip install rollbar`` and adding this configuration dictionary to your config::
@@ -368,6 +388,10 @@ You can redirect worker exceptions directly to your `Rollbar <https://rollbar.co
 Please check the Pyrollbar `configuration reference <https://github.com/rollbar/pyrollbar#configuration-reference>`__ for more options.
 Note that you will need a `Rollbar <https://rollbar.com/>`__ account and access token to use this feature.
 
+
+.. note::
+    The ``rollbar`` setting is included for backwards compatibility, for those who utilized rollbar configuration before the ``error_reporter`` interface was introduced. Note that Rollbar support can be configured either via the ``rollbar`` setting, or via the ``django-q-rollbar`` package and enabled via the ``error_reporter`` setting above.
+
 cpu_affinity
 ~~~~~~~~~~~~
 

docs/errors.rst

@@ -0,0 +1,9 @@
+Errors
+------
+.. py:currentmodule:: django_q 
+
+Django Q uses a pluggable error reporter system based upon python `extras <https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies>`, allowing anyone to develop plugins for error reporting and monitoring integration. Currently implemented examples include `Rollbar <https://rollbar.com/>` and `Sentry <https://docs.sentry.io/>`).
+
+Error reporting plugins register a class which implements a ``report`` method, which is invoked when a Django Q cluster encounters an error, pasing information to the particular service. Error reporters must be :ref:`configured<error_reporter>` via the ``Q_CLUSTER`` dictionary in your :file:`settings.py`. These settings are passed as kwargs upon initiation of the Error Reporter. Therefore, in order to implement a new plugin, a package must expose a class which will be instantiated with the necessary information via the ``Q_CLUSTER`` settings and implements a single ``report`` method.
+
+For example implementations, see `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>` and `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`

docs/index.rst

@@ -42,6 +42,7 @@ Contents:
     Cluster <cluster>
     Monitor <monitor>
     Admin <admin>
+    Errors <errors>
     Signals <signals>
     Architecture <architecture>
     Examples <examples>

setup.py

@@ -56,5 +56,15 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Python Modules',
-    ]
+    ],
+    entry_points={
+        'djangoq.errorreporters': [
+            'rollbar = django_q_rollbar.Rollbar',
+            'sentry = django_q_sentry.Sentry',
+        ]
+    },
+    extras_require={
+        'rollbar': ["django-q-rollbar>=0.1"],
+        'sentry': ["django-q-sentry>=0.1"],
+    }
 )