mirror of
https://github.com/django-q2/django-q2.git
synced 2026-10-05 06:08:12 +08:00
Move worker, scheduler, pusher and monitor to separate files (#100)
This commit is contained in:
Stan Triepels
+181
-493
@@ -1,510 +1,198 @@
|
||||
from datetime import timedelta
|
||||
from multiprocessing.process import current_process
|
||||
from multiprocessing.queues import Queue
|
||||
|
||||
# django
|
||||
from django.db import connection
|
||||
from django.db.models import F, Sum
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext as _
|
||||
from django import db
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from django_q import VERSION, models
|
||||
from django_q.brokers import get_broker
|
||||
import django_q.tasks
|
||||
from django_q.brokers import Broker, get_broker
|
||||
from django_q.conf import Conf, logger, setproctitle
|
||||
from django_q.models import Success, Task
|
||||
from django_q.signals import post_execute
|
||||
from django_q.signing import SignedPackage
|
||||
from django_q.utils import close_old_django_connections, get_func_repr
|
||||
|
||||
# local
|
||||
from django_q.conf import Conf
|
||||
from django_q.status import Stat
|
||||
|
||||
# optional
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
psutil = None
|
||||
import setproctitle
|
||||
except ModuleNotFoundError:
|
||||
setproctitle = None
|
||||
|
||||
|
||||
def get_process_mb(pid):
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
mb_used = round(process.memory_info().rss / 1024**2, 2)
|
||||
except psutil.NoSuchProcess:
|
||||
mb_used = "NO_PROCESS_FOUND"
|
||||
return mb_used
|
||||
|
||||
|
||||
BLESSED_INSTALL_MESSAGE = (
|
||||
"Blessed is not installed. Please install blessed to use this: "
|
||||
"https://pypi.org/project/blessed/"
|
||||
)
|
||||
|
||||
|
||||
def monitor(run_once=False, broker=None):
|
||||
def monitor(result_queue: Queue, broker: Broker = None):
|
||||
"""
|
||||
Gets finished tasks from the result queue and saves them to Django
|
||||
:type broker: brokers.Broker
|
||||
:type result_queue: multiprocessing.Queue
|
||||
"""
|
||||
if not broker:
|
||||
broker = get_broker()
|
||||
try:
|
||||
from blessed import Terminal
|
||||
proc_name = current_process().name
|
||||
if setproctitle:
|
||||
setproctitle.setproctitle(f"qcluster {proc_name} monitor")
|
||||
logger.info(
|
||||
_("%(name)s monitoring at %(id)s")
|
||||
% {"name": proc_name, "id": current_process().pid}
|
||||
)
|
||||
for task in iter(result_queue.get, "STOP"):
|
||||
# save the result
|
||||
if task.get("cached", False):
|
||||
save_cached(task, broker)
|
||||
else:
|
||||
save_task(task, broker)
|
||||
# acknowledge result
|
||||
ack_id = task.pop("ack_id", False)
|
||||
if ack_id and (task["success"] or task.get("ack_failure", False)):
|
||||
broker.acknowledge(ack_id)
|
||||
# signal execution done
|
||||
post_execute.send(sender="django_q", task=task)
|
||||
# log the result
|
||||
info_name = get_func_repr(task["func"])
|
||||
if task["success"]:
|
||||
# log success
|
||||
logger.info(
|
||||
_("Processed '%(info_name)s' (%(task_name)s)")
|
||||
% {"info_name": info_name, "task_name": task["name"]}
|
||||
)
|
||||
else:
|
||||
# log failure
|
||||
logger.error(
|
||||
_("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
|
||||
% {
|
||||
"info_name": info_name,
|
||||
"task_name": task["name"],
|
||||
"task_result": task["result"],
|
||||
}
|
||||
)
|
||||
logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
|
||||
|
||||
term = Terminal()
|
||||
except ImportError:
|
||||
print(BLESSED_INSTALL_MESSAGE)
|
||||
|
||||
def save_task(task, broker: Broker):
|
||||
"""
|
||||
Saves the task package to Django or the cache
|
||||
:param task: the task package
|
||||
:type broker: brokers.Broker
|
||||
"""
|
||||
# SAVE LIMIT < 0 : Don't save success
|
||||
if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
|
||||
return
|
||||
# enqueues next in a chain
|
||||
if task.get("chain", None):
|
||||
django_q.tasks.async_chain(
|
||||
task["chain"],
|
||||
group=task["group"],
|
||||
cached=task["cached"],
|
||||
sync=task["sync"],
|
||||
broker=broker,
|
||||
)
|
||||
# SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
|
||||
close_old_django_connections()
|
||||
|
||||
broker.ping()
|
||||
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
|
||||
val = None
|
||||
start_width = int(term.width / 8)
|
||||
while val not in (
|
||||
"q",
|
||||
"Q",
|
||||
try:
|
||||
filters = {}
|
||||
if (
|
||||
Conf.SAVE_LIMIT_PER
|
||||
and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
|
||||
and Conf.SAVE_LIMIT_PER in task
|
||||
):
|
||||
col_width = int(term.width / 8)
|
||||
# In case of resize
|
||||
if col_width != start_width:
|
||||
print(term.clear())
|
||||
start_width = col_width
|
||||
print(
|
||||
term.move(0, 0)
|
||||
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
|
||||
value = task[Conf.SAVE_LIMIT_PER]
|
||||
if Conf.SAVE_LIMIT_PER == "func":
|
||||
value = get_func_repr(value)
|
||||
filters[Conf.SAVE_LIMIT_PER] = value
|
||||
|
||||
with db.transaction.atomic(using=db.router.db_for_write(Success)):
|
||||
last = Success.objects.filter(**filters).select_for_update().last()
|
||||
if (
|
||||
task["success"]
|
||||
and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
|
||||
):
|
||||
last.delete()
|
||||
|
||||
# check if this task has previous results
|
||||
try:
|
||||
existing_task = Task.objects.get(id=task["id"], name=task["name"])
|
||||
# only update the result if it hasn't succeeded yet
|
||||
if not existing_task.success:
|
||||
existing_task.stopped = task["stopped"]
|
||||
existing_task.result = task["result"]
|
||||
existing_task.success = task["success"]
|
||||
existing_task.attempt_count = existing_task.attempt_count + 1
|
||||
existing_task.save()
|
||||
|
||||
if (
|
||||
Conf.MAX_ATTEMPTS > 0
|
||||
and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
|
||||
):
|
||||
broker.acknowledge(task["ack_id"])
|
||||
|
||||
except Task.DoesNotExist:
|
||||
# convert func to string
|
||||
func = get_func_repr(task["func"])
|
||||
Task.objects.create(
|
||||
id=task["id"],
|
||||
name=task["name"],
|
||||
func=func,
|
||||
hook=task.get("hook"),
|
||||
args=task["args"],
|
||||
kwargs=task["kwargs"],
|
||||
cluster=task.get("cluster"),
|
||||
started=task["started"],
|
||||
stopped=task["stopped"],
|
||||
result=task["result"],
|
||||
group=task.get("group"),
|
||||
success=task["success"],
|
||||
attempt_count=1,
|
||||
)
|
||||
print(
|
||||
term.move(0, 1 * col_width)
|
||||
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 2 * col_width)
|
||||
+ term.black_on_green(term.center(_("State"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 3 * col_width)
|
||||
+ term.black_on_green(term.center(_("Pool"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 4 * col_width)
|
||||
+ term.black_on_green(term.center(_("TQ"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 5 * col_width)
|
||||
+ term.black_on_green(term.center(_("RQ"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 6 * col_width)
|
||||
+ term.black_on_green(term.center(_("RC"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 7 * col_width)
|
||||
+ term.black_on_green(term.center(_("Up"), width=col_width - 1))
|
||||
)
|
||||
i = 2
|
||||
stats = Stat.get_all(broker=broker)
|
||||
print(term.clear_eos())
|
||||
for stat in stats:
|
||||
status = stat.status
|
||||
# color status
|
||||
if stat.status == Conf.WORKING:
|
||||
status = term.green(str(Conf.WORKING))
|
||||
elif stat.status == Conf.STOPPING:
|
||||
status = term.yellow(str(Conf.STOPPING))
|
||||
elif stat.status == Conf.STOPPED:
|
||||
status = term.red(str(Conf.STOPPED))
|
||||
elif stat.status == Conf.IDLE:
|
||||
status = str(Conf.IDLE)
|
||||
# color q's
|
||||
tasks = str(stat.task_q_size)
|
||||
if stat.task_q_size > 0:
|
||||
tasks = term.cyan(str(stat.task_q_size))
|
||||
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
|
||||
tasks = term.green(str(stat.task_q_size))
|
||||
results = stat.done_q_size
|
||||
if results > 0:
|
||||
results = term.cyan(str(results))
|
||||
# color workers
|
||||
workers = len(stat.workers)
|
||||
if workers < Conf.WORKERS:
|
||||
workers = term.yellow(str(workers))
|
||||
# format uptime
|
||||
uptime = (timezone.now() - stat.tob).total_seconds()
|
||||
hours, remainder = divmod(uptime, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
|
||||
# print to the terminal
|
||||
print(
|
||||
term.move(i, 0)
|
||||
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 1 * col_width)
|
||||
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 2 * col_width)
|
||||
+ term.center(status, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 3 * col_width)
|
||||
+ term.center(workers, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 4 * col_width)
|
||||
+ term.center(tasks, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 5 * col_width)
|
||||
+ term.center(results, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 6 * col_width)
|
||||
+ term.center(stat.reincarnations, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(i, 7 * col_width)
|
||||
+ term.center(uptime, width=col_width - 1)
|
||||
)
|
||||
i += 1
|
||||
# bottom bar
|
||||
i += 1
|
||||
queue_size = broker.queue_size()
|
||||
lock_size = broker.lock_size()
|
||||
if lock_size:
|
||||
queue_size = f"{queue_size}({lock_size})"
|
||||
print(
|
||||
term.move(i, 0)
|
||||
+ term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
|
||||
)
|
||||
print(
|
||||
term.move(i, 2 * col_width)
|
||||
+ term.black_on_cyan(term.center(_("Queued"), width=col_width))
|
||||
)
|
||||
print(
|
||||
term.move(i, 3 * col_width)
|
||||
+ term.white_on_cyan(term.center(queue_size, width=col_width))
|
||||
)
|
||||
print(
|
||||
term.move(i, 4 * col_width)
|
||||
+ term.black_on_cyan(term.center(_("Success"), width=col_width))
|
||||
)
|
||||
print(
|
||||
term.move(i, 5 * col_width)
|
||||
+ term.white_on_cyan(
|
||||
term.center(models.Success.objects.count(), width=col_width)
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(i, 6 * col_width)
|
||||
+ term.black_on_cyan(term.center(_("Failures"), width=col_width))
|
||||
)
|
||||
print(
|
||||
term.move(i, 7 * col_width)
|
||||
+ term.white_on_cyan(
|
||||
term.center(models.Failure.objects.count(), width=col_width)
|
||||
)
|
||||
)
|
||||
# for testing
|
||||
if run_once:
|
||||
return Stat.get_all(broker=broker)
|
||||
print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
|
||||
val = term.inkey(timeout=1)
|
||||
except Exception:
|
||||
logger.exception("Could not save task result")
|
||||
|
||||
|
||||
def info(broker=None):
|
||||
if not broker:
|
||||
broker = get_broker()
|
||||
def save_cached(task, broker: Broker):
|
||||
task_key = f'{broker.list_key}:{task["id"]}'
|
||||
timeout = task["cached"]
|
||||
if timeout is True:
|
||||
timeout = None
|
||||
try:
|
||||
from blessed import Terminal
|
||||
|
||||
term = Terminal()
|
||||
except ImportError:
|
||||
print(BLESSED_INSTALL_MESSAGE)
|
||||
return
|
||||
|
||||
broker.ping()
|
||||
stat = Stat.get_all(broker=broker)
|
||||
# general stats
|
||||
clusters = len(stat)
|
||||
workers = 0
|
||||
reincarnations = 0
|
||||
for cluster in stat:
|
||||
workers += len(cluster.workers)
|
||||
reincarnations += cluster.reincarnations
|
||||
# calculate tasks pm and avg exec time
|
||||
tasks_per = 0
|
||||
per = _("day")
|
||||
exec_time = 0
|
||||
last_tasks = models.Success.objects.filter(
|
||||
stopped__gte=timezone.now() - timedelta(hours=24)
|
||||
)
|
||||
tasks_per_day = last_tasks.count()
|
||||
if tasks_per_day > 0:
|
||||
# average execution time over the last 24 hours
|
||||
if connection.vendor != "sqlite":
|
||||
exec_time = last_tasks.aggregate(
|
||||
time_taken=Sum(F("stopped") - F("started"))
|
||||
)
|
||||
exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
|
||||
else:
|
||||
# can't sum timedeltas on sqlite
|
||||
for t in last_tasks:
|
||||
exec_time += t.time_taken()
|
||||
exec_time = exec_time / tasks_per_day
|
||||
# tasks per second/minute/hour/day in the last 24 hours
|
||||
if tasks_per_day > 24 * 60 * 60:
|
||||
tasks_per = tasks_per_day / (24 * 60 * 60)
|
||||
per = _("second")
|
||||
elif tasks_per_day > 24 * 60:
|
||||
tasks_per = tasks_per_day / (24 * 60)
|
||||
per = _("minute")
|
||||
elif tasks_per_day > 24:
|
||||
tasks_per = tasks_per_day / 24
|
||||
per = _("hour")
|
||||
else:
|
||||
tasks_per = tasks_per_day
|
||||
# print to terminal
|
||||
print(term.clear_eos())
|
||||
col_width = int(term.width / 6)
|
||||
print(
|
||||
term.black_on_green(
|
||||
term.center(
|
||||
_("-- %(prefix)s %(version)s on %(info)s --")
|
||||
% {
|
||||
"prefix": Conf.PREFIX.capitalize(),
|
||||
"version": ".".join(str(v) for v in VERSION),
|
||||
"info": broker.info(),
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.cyan(_("Clusters"))
|
||||
+ term.move_x(1 * col_width)
|
||||
+ term.white(str(clusters))
|
||||
+ term.move_x(2 * col_width)
|
||||
+ term.cyan(_("Workers"))
|
||||
+ term.move_x(3 * col_width)
|
||||
+ term.white(str(workers))
|
||||
+ term.move_x(4 * col_width)
|
||||
+ term.cyan(_("Restarts"))
|
||||
+ term.move_x(5 * col_width)
|
||||
+ term.white(str(reincarnations))
|
||||
)
|
||||
print(
|
||||
term.cyan(_("Queued"))
|
||||
+ term.move_x(1 * col_width)
|
||||
+ term.white(str(broker.queue_size()))
|
||||
+ term.move_x(2 * col_width)
|
||||
+ term.cyan(_("Successes"))
|
||||
+ term.move_x(3 * col_width)
|
||||
+ term.white(str(models.Success.objects.count()))
|
||||
+ term.move_x(4 * col_width)
|
||||
+ term.cyan(_("Failures"))
|
||||
+ term.move_x(5 * col_width)
|
||||
+ term.white(str(models.Failure.objects.count()))
|
||||
)
|
||||
print(
|
||||
term.cyan(_("Schedules"))
|
||||
+ term.move_x(1 * col_width)
|
||||
+ term.white(str(models.Schedule.objects.count()))
|
||||
+ term.move_x(2 * col_width)
|
||||
+ term.cyan(_("Tasks/%(per)s") % {"per": per})
|
||||
+ term.move_x(3 * col_width)
|
||||
+ term.white(f"{tasks_per:.2f}")
|
||||
+ term.move_x(4 * col_width)
|
||||
+ term.cyan(_("Avg time"))
|
||||
+ term.move_x(5 * col_width)
|
||||
+ term.white(f"{exec_time:.4f}")
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def memory(run_once=False, workers=False, broker=None):
|
||||
if not broker:
|
||||
broker = get_broker()
|
||||
try:
|
||||
from blessed import Terminal
|
||||
|
||||
term = Terminal()
|
||||
except ImportError:
|
||||
print(BLESSED_INSTALL_MESSAGE)
|
||||
return
|
||||
broker.ping()
|
||||
if not psutil:
|
||||
print(term.clear_eos())
|
||||
print(
|
||||
term.white_on_red(
|
||||
'Cannot start "qmemory" command. Missing "psutil" library.'
|
||||
)
|
||||
)
|
||||
return
|
||||
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
|
||||
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
|
||||
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
|
||||
cols = 8
|
||||
val = None
|
||||
start_width = int(term.width / cols)
|
||||
while val not in ["q", "Q"]:
|
||||
col_width = int(term.width / cols)
|
||||
# In case of resize
|
||||
if col_width != start_width:
|
||||
print(term.clear())
|
||||
start_width = col_width
|
||||
# sentinel, monitor and workers memory usage
|
||||
print(
|
||||
term.move(0, 0 * col_width)
|
||||
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 1 * col_width)
|
||||
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 2 * col_width)
|
||||
+ term.black_on_green(
|
||||
term.center(_("Available (%)"), width=col_width - 1)
|
||||
group = task.get("group", None)
|
||||
iter_count = task.get("iter_count", 0)
|
||||
# if it's a group append to the group list
|
||||
if group:
|
||||
group_key = f"{broker.list_key}:{group}:keys"
|
||||
group_list = broker.cache.get(group_key) or []
|
||||
# if it's an iter group, check if we are ready
|
||||
if iter_count and len(group_list) == iter_count - 1:
|
||||
group_args = f"{broker.list_key}:{group}:args"
|
||||
# collate the results into a Task result
|
||||
results = [
|
||||
SignedPackage.loads(broker.cache.get(k))["result"]
|
||||
for k in group_list
|
||||
]
|
||||
results.append(task["result"])
|
||||
task["result"] = results
|
||||
task["id"] = group
|
||||
task["args"] = SignedPackage.loads(broker.cache.get(group_args))
|
||||
task.pop("iter_count", None)
|
||||
task.pop("group", None)
|
||||
if task.get("iter_cached", None):
|
||||
task["cached"] = task.pop("iter_cached", None)
|
||||
save_cached(task, broker=broker)
|
||||
else:
|
||||
save_task(task, broker)
|
||||
broker.cache.delete_many(group_list)
|
||||
broker.cache.delete_many([group_key, group_args])
|
||||
return
|
||||
# save the group list
|
||||
group_list.append(task_key)
|
||||
broker.cache.set(group_key, group_list, timeout)
|
||||
# async_task next in a chain
|
||||
if task.get("chain", None):
|
||||
django_q.tasks.async_chain(
|
||||
task["chain"],
|
||||
group=group,
|
||||
cached=task["cached"],
|
||||
sync=task["sync"],
|
||||
broker=broker,
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(0, 3 * col_width)
|
||||
+ term.black_on_green(
|
||||
term.center(_("Available (MB)"), width=col_width - 1)
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(0, 4 * col_width)
|
||||
+ term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
|
||||
)
|
||||
print(
|
||||
term.move(0, 5 * col_width)
|
||||
+ term.black_on_green(
|
||||
term.center(_("Sentinel (MB)"), width=col_width - 1)
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(0, 6 * col_width)
|
||||
+ term.black_on_green(
|
||||
term.center(_("Monitor (MB)"), width=col_width - 1)
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(0, 7 * col_width)
|
||||
+ term.black_on_green(
|
||||
term.center(_("Workers (MB)"), width=col_width - 1)
|
||||
)
|
||||
)
|
||||
row = 2
|
||||
stats = Stat.get_all(broker=broker)
|
||||
print(term.clear_eos())
|
||||
for stat in stats:
|
||||
# memory available (%)
|
||||
memory_available_percentage = round(
|
||||
psutil.virtual_memory().available
|
||||
* 100
|
||||
/ psutil.virtual_memory().total,
|
||||
2,
|
||||
)
|
||||
# memory available (MB)
|
||||
memory_available = round(
|
||||
psutil.virtual_memory().available / 1024**2, 2
|
||||
)
|
||||
if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
|
||||
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
|
||||
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
|
||||
print(
|
||||
term.move(row, 0 * col_width)
|
||||
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(row, 1 * col_width)
|
||||
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(row, 2 * col_width)
|
||||
+ term.center(memory_available_percentage, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(row, 3 * col_width)
|
||||
+ term.center(memory_available, width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(row, 4 * col_width)
|
||||
+ term.center(
|
||||
round(psutil.virtual_memory().total / 1024**2, 2),
|
||||
width=col_width - 1,
|
||||
)
|
||||
)
|
||||
print(
|
||||
term.move(row, 5 * col_width)
|
||||
+ term.center(get_process_mb(stat.sentinel), width=col_width - 1)
|
||||
)
|
||||
print(
|
||||
term.move(row, 6 * col_width)
|
||||
+ term.center(
|
||||
get_process_mb(getattr(stat, "monitor", None)),
|
||||
width=col_width - 1,
|
||||
)
|
||||
)
|
||||
workers_mb = 0
|
||||
for worker_pid in stat.workers:
|
||||
result = get_process_mb(worker_pid)
|
||||
if isinstance(result, str):
|
||||
result = 0
|
||||
workers_mb += result
|
||||
print(
|
||||
term.move(row, 7 * col_width)
|
||||
+ term.center(
|
||||
workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
|
||||
)
|
||||
)
|
||||
row += 1
|
||||
# each worker's memory usage
|
||||
if workers:
|
||||
row += 2
|
||||
col_width = int(term.width / (1 + Conf.WORKERS))
|
||||
print(
|
||||
term.move(row, 0 * col_width)
|
||||
+ term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
|
||||
)
|
||||
for worker_num in range(Conf.WORKERS):
|
||||
print(
|
||||
term.move(row, (worker_num + 1) * col_width)
|
||||
+ term.black_on_cyan(
|
||||
term.center(
|
||||
"Worker #{} (MB)".format(worker_num + 1),
|
||||
width=col_width - 1,
|
||||
)
|
||||
)
|
||||
)
|
||||
row += 2
|
||||
for stat in stats:
|
||||
print(
|
||||
term.move(row, 0 * col_width)
|
||||
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
||||
)
|
||||
for idx, worker_pid in enumerate(stat.workers):
|
||||
mb_used = get_process_mb(worker_pid)
|
||||
print(
|
||||
term.move(row, (idx + 1) * col_width)
|
||||
+ term.center(mb_used, width=col_width - 1)
|
||||
)
|
||||
row += 1
|
||||
row += 1
|
||||
print(
|
||||
term.move(row, 0)
|
||||
+ _("Available lowest (): %(memory_percent)s ((at)s)")
|
||||
% {
|
||||
"memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
|
||||
"at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
|
||||
"%Y-%m-%d %H:%M:%S+00:00"
|
||||
),
|
||||
}
|
||||
)
|
||||
# for testing
|
||||
if run_once:
|
||||
return Stat.get_all(broker=broker)
|
||||
print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
|
||||
val = term.inkey(timeout=1)
|
||||
|
||||
|
||||
def get_ids():
|
||||
# prints id (PID) of running clusters
|
||||
stat = Stat.get_all()
|
||||
if stat:
|
||||
for s in stat:
|
||||
print(s.cluster_id)
|
||||
else:
|
||||
print(_("No clusters appear to be running."))
|
||||
return True
|
||||
# save the task
|
||||
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
|
||||
except Exception:
|
||||
logger.exception("Could not save task result")
|
||||
|
||||
Reference in New Issue
Block a user