Multiple queue, multiple cluster in one site (#71)

* Support for multi-queue, multi-cluster configuration. API changes include:
  * Adding `cluster` to async_task() parameters and Task model
  * Adding argument --name to qcluster command
  * Necessary adjustments to Conf and Broker classes
  * Some admin improvements

* Add settings.Q_CLUSTER['ALT_CLUSTERS']: q_cluster config overrides for alternative clusters;
Add Conf.CLUSTER_NAME: separate usage from Conf.PREFIX;
QueueAdmin/OrmQ detail page enhanced: now displaying args/kwargs/q_options instead of encrypted payload.

* if `cluster` argument is not set (the default), async_task() and schedule() will be handled by the default cluster; Documentation update.

* Text cleanup

* Documentation update.

* Fix TIMEOUT setting in Windows for non-default cluster

---------

Co-authored-by: Stan Triepels <1939656+GDay@users.noreply.github.com>
This commit is contained in:
sinowood
2023-04-02 16:36:17 +02:00
committed by GitHub
co-authored by Stan Triepels
parent f8520c9cda
commit 52c04217e9
17 changed files with 215 additions and 47 deletions
+27 -8
View File
@@ -11,6 +11,7 @@ from django.utils import timezone
from django.utils.timezone import is_aware
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
from django.utils.functional import cached_property
# External
from picklefield import PickledObjectField
@@ -33,6 +34,7 @@ class Task(models.Model):
kwargs = PickledObjectField(null=True, protocol=-1)
result = PickledObjectField(null=True, protocol=-1)
group = models.CharField(max_length=100, editable=False, null=True)
cluster = models.CharField(max_length=100, default=None, null=True, blank=True)
started = models.DateTimeField(editable=False)
stopped = models.DateTimeField(editable=False)
success = models.BooleanField(default=True, editable=False)
@@ -215,7 +217,10 @@ class Schedule(models.Model):
help_text=_("Cron expression"),
)
task = models.CharField(max_length=100, null=True, editable=False)
cluster = models.CharField(max_length=100, default=None, null=True, blank=True)
cluster = models.CharField(
max_length=100, default=None, null=True, blank=True,
help_text=_("Name of the target cluster")
)
intended_date_kwarg = models.CharField(
max_length=100,
null=True,
@@ -302,24 +307,38 @@ class Schedule(models.Model):
class OrmQ(models.Model):
key = models.CharField(max_length=100)
key = models.CharField(max_length=100, help_text=_("Name of the target cluster"))
payload = models.TextField()
lock = models.DateTimeField(null=True)
lock = models.DateTimeField(null=True, help_text=_("Prevent any cluster from pulling until"))
@cached_property
def task(self):
return SignedPackage.loads(self.payload)
try:
return SignedPackage.loads(self.payload)
except Exception as e:
return {"id": "*" + e.__class__.__name__}
def func(self):
return get_func_repr(self.task()["func"])
return get_func_repr(self.task.get("func"))
def task_id(self):
return self.task()["id"]
return self.task.get("id")
def name(self):
return self.task()["name"]
return self.task.get("name")
def group(self):
return self.task().get("group")
return self.task.get("group")
def args(self):
return self.task.get("args")
def kwargs(self):
return self.task.get("kwargs")
def q_options(self):
exclude = {"id", "name", "group", "func", "args", "kwargs"}
return {k: v for k, v in self.task.items() if k not in exclude}
class Meta:
app_label = "django_q"