mirror of
https://github.com/django-q2/django-q2.git
synced 2026-09-15 13:37:56 +08:00
509 lines
18 KiB
Python
509 lines
18 KiB
Python
from datetime import timedelta
|
|
|
|
# django
|
|
from django.db import connection
|
|
from django.db.models import F, Sum
|
|
from django.utils import timezone
|
|
from django.utils.translation import gettext as _
|
|
|
|
from django_q import VERSION, models
|
|
from django_q.brokers import get_broker
|
|
|
|
# local
|
|
from django_q.conf import Conf
|
|
from django_q.status import Stat
|
|
|
|
# optional
|
|
try:
|
|
import psutil
|
|
except ImportError:
|
|
psutil = None
|
|
|
|
|
|
def get_process_mb(pid):
|
|
try:
|
|
process = psutil.Process(pid)
|
|
mb_used = round(process.memory_info().rss / 1024**2, 2)
|
|
except psutil.NoSuchProcess:
|
|
mb_used = "NO_PROCESS_FOUND"
|
|
return mb_used
|
|
|
|
|
|
BLESSED_INSTALL_MESSAGE = (
|
|
"Blessed is not installed. Please install blessed to use this: "
|
|
"https://pypi.org/project/blessed/"
|
|
)
|
|
|
|
|
|
def monitor(run_once=False, broker=None):
|
|
if not broker:
|
|
broker = get_broker()
|
|
try:
|
|
from blessed import Terminal
|
|
|
|
term = Terminal()
|
|
except ImportError:
|
|
print(BLESSED_INSTALL_MESSAGE)
|
|
return
|
|
|
|
broker.ping()
|
|
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
|
|
val = None
|
|
start_width = int(term.width / 8)
|
|
while val not in (
|
|
"q",
|
|
"Q",
|
|
):
|
|
col_width = int(term.width / 8)
|
|
# In case of resize
|
|
if col_width != start_width:
|
|
print(term.clear())
|
|
start_width = col_width
|
|
print(
|
|
term.move(0, 0)
|
|
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 1 * col_width)
|
|
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 2 * col_width)
|
|
+ term.black_on_green(term.center(_("State"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 3 * col_width)
|
|
+ term.black_on_green(term.center(_("Pool"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 4 * col_width)
|
|
+ term.black_on_green(term.center(_("TQ"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 5 * col_width)
|
|
+ term.black_on_green(term.center(_("RQ"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 6 * col_width)
|
|
+ term.black_on_green(term.center(_("RC"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 7 * col_width)
|
|
+ term.black_on_green(term.center(_("Up"), width=col_width - 1))
|
|
)
|
|
i = 2
|
|
stats = Stat.get_all(broker=broker)
|
|
print(term.clear_eos())
|
|
for stat in stats:
|
|
status = stat.status
|
|
# color status
|
|
if stat.status == Conf.WORKING:
|
|
status = term.green(str(Conf.WORKING))
|
|
elif stat.status == Conf.STOPPING:
|
|
status = term.yellow(str(Conf.STOPPING))
|
|
elif stat.status == Conf.STOPPED:
|
|
status = term.red(str(Conf.STOPPED))
|
|
elif stat.status == Conf.IDLE:
|
|
status = str(Conf.IDLE)
|
|
# color q's
|
|
tasks = str(stat.task_q_size)
|
|
if stat.task_q_size > 0:
|
|
tasks = term.cyan(str(stat.task_q_size))
|
|
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
|
|
tasks = term.green(str(stat.task_q_size))
|
|
results = stat.done_q_size
|
|
if results > 0:
|
|
results = term.cyan(str(results))
|
|
# color workers
|
|
workers = len(stat.workers)
|
|
if workers < Conf.WORKERS:
|
|
workers = term.yellow(str(workers))
|
|
# format uptime
|
|
uptime = (timezone.now() - stat.tob).total_seconds()
|
|
hours, remainder = divmod(uptime, 3600)
|
|
minutes, seconds = divmod(remainder, 60)
|
|
uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
|
|
# print to the terminal
|
|
print(
|
|
term.move(i, 0)
|
|
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 1 * col_width)
|
|
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 2 * col_width)
|
|
+ term.center(status, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 3 * col_width)
|
|
+ term.center(workers, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 4 * col_width)
|
|
+ term.center(tasks, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 5 * col_width)
|
|
+ term.center(results, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 6 * col_width)
|
|
+ term.center(stat.reincarnations, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(i, 7 * col_width)
|
|
+ term.center(uptime, width=col_width - 1)
|
|
)
|
|
i += 1
|
|
# bottom bar
|
|
i += 1
|
|
queue_size = broker.queue_size()
|
|
lock_size = broker.lock_size()
|
|
if lock_size:
|
|
queue_size = f"{queue_size}({lock_size})"
|
|
print(
|
|
term.move(i, 0)
|
|
+ term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
|
|
)
|
|
print(
|
|
term.move(i, 2 * col_width)
|
|
+ term.black_on_cyan(term.center(_("Queued"), width=col_width))
|
|
)
|
|
print(
|
|
term.move(i, 3 * col_width)
|
|
+ term.white_on_cyan(term.center(queue_size, width=col_width))
|
|
)
|
|
print(
|
|
term.move(i, 4 * col_width)
|
|
+ term.black_on_cyan(term.center(_("Success"), width=col_width))
|
|
)
|
|
print(
|
|
term.move(i, 5 * col_width)
|
|
+ term.white_on_cyan(
|
|
term.center(models.Success.objects.count(), width=col_width)
|
|
)
|
|
)
|
|
print(
|
|
term.move(i, 6 * col_width)
|
|
+ term.black_on_cyan(term.center(_("Failures"), width=col_width))
|
|
)
|
|
print(
|
|
term.move(i, 7 * col_width)
|
|
+ term.white_on_cyan(
|
|
term.center(models.Failure.objects.count(), width=col_width)
|
|
)
|
|
)
|
|
# for testing
|
|
if run_once:
|
|
return Stat.get_all(broker=broker)
|
|
print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
|
|
val = term.inkey(timeout=1)
|
|
|
|
|
|
def info(broker=None):
|
|
if not broker:
|
|
broker = get_broker()
|
|
try:
|
|
from blessed import Terminal
|
|
|
|
term = Terminal()
|
|
except ImportError:
|
|
print(BLESSED_INSTALL_MESSAGE)
|
|
return
|
|
|
|
broker.ping()
|
|
stat = Stat.get_all(broker=broker)
|
|
# general stats
|
|
clusters = len(stat)
|
|
workers = 0
|
|
reincarnations = 0
|
|
for cluster in stat:
|
|
workers += len(cluster.workers)
|
|
reincarnations += cluster.reincarnations
|
|
# calculate tasks pm and avg exec time
|
|
tasks_per = 0
|
|
per = _("day")
|
|
exec_time = 0
|
|
last_tasks = models.Success.objects.filter(
|
|
stopped__gte=timezone.now() - timedelta(hours=24)
|
|
)
|
|
tasks_per_day = last_tasks.count()
|
|
if tasks_per_day > 0:
|
|
# average execution time over the last 24 hours
|
|
if connection.vendor != "sqlite":
|
|
exec_time = last_tasks.aggregate(
|
|
time_taken=Sum(F("stopped") - F("started"))
|
|
)
|
|
exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
|
|
else:
|
|
# can't sum timedeltas on sqlite
|
|
for t in last_tasks:
|
|
exec_time += t.time_taken()
|
|
exec_time = exec_time / tasks_per_day
|
|
# tasks per second/minute/hour/day in the last 24 hours
|
|
if tasks_per_day > 24 * 60 * 60:
|
|
tasks_per = tasks_per_day / (24 * 60 * 60)
|
|
per = _("second")
|
|
elif tasks_per_day > 24 * 60:
|
|
tasks_per = tasks_per_day / (24 * 60)
|
|
per = _("minute")
|
|
elif tasks_per_day > 24:
|
|
tasks_per = tasks_per_day / 24
|
|
per = _("hour")
|
|
else:
|
|
tasks_per = tasks_per_day
|
|
# print to terminal
|
|
print(term.clear_eos())
|
|
col_width = int(term.width / 6)
|
|
print(
|
|
term.black_on_green(
|
|
term.center(
|
|
_("-- %(prefix)s %(version)s on %(info)s --")
|
|
% {
|
|
"prefix": Conf.PREFIX.capitalize(),
|
|
"version": ".".join(str(v) for v in VERSION),
|
|
"info": broker.info(),
|
|
}
|
|
)
|
|
)
|
|
)
|
|
print(
|
|
term.cyan(_("Clusters"))
|
|
+ term.move_x(1 * col_width)
|
|
+ term.white(str(clusters))
|
|
+ term.move_x(2 * col_width)
|
|
+ term.cyan(_("Workers"))
|
|
+ term.move_x(3 * col_width)
|
|
+ term.white(str(workers))
|
|
+ term.move_x(4 * col_width)
|
|
+ term.cyan(_("Restarts"))
|
|
+ term.move_x(5 * col_width)
|
|
+ term.white(str(reincarnations))
|
|
)
|
|
print(
|
|
term.cyan(_("Queued"))
|
|
+ term.move_x(1 * col_width)
|
|
+ term.white(str(broker.queue_size()))
|
|
+ term.move_x(2 * col_width)
|
|
+ term.cyan(_("Successes"))
|
|
+ term.move_x(3 * col_width)
|
|
+ term.white(str(models.Success.objects.count()))
|
|
+ term.move_x(4 * col_width)
|
|
+ term.cyan(_("Failures"))
|
|
+ term.move_x(5 * col_width)
|
|
+ term.white(str(models.Failure.objects.count()))
|
|
)
|
|
print(
|
|
term.cyan(_("Schedules"))
|
|
+ term.move_x(1 * col_width)
|
|
+ term.white(str(models.Schedule.objects.count()))
|
|
+ term.move_x(2 * col_width)
|
|
+ term.cyan(_("Tasks/%(per)s") % {"per": per})
|
|
+ term.move_x(3 * col_width)
|
|
+ term.white(f"{tasks_per:.2f}")
|
|
+ term.move_x(4 * col_width)
|
|
+ term.cyan(_("Avg time"))
|
|
+ term.move_x(5 * col_width)
|
|
+ term.white(f"{exec_time:.4f}")
|
|
)
|
|
return True
|
|
|
|
|
|
def memory(run_once=False, workers=False, broker=None):
|
|
if not broker:
|
|
broker = get_broker()
|
|
try:
|
|
from blessed import Terminal
|
|
|
|
term = Terminal()
|
|
except ImportError:
|
|
print(BLESSED_INSTALL_MESSAGE)
|
|
return
|
|
broker.ping()
|
|
if not psutil:
|
|
print(term.clear_eos())
|
|
print(
|
|
term.white_on_red(
|
|
'Cannot start "qmemory" command. Missing "psutil" library.'
|
|
)
|
|
)
|
|
return
|
|
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
|
|
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
|
|
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
|
|
cols = 8
|
|
val = None
|
|
start_width = int(term.width / cols)
|
|
while val not in ["q", "Q"]:
|
|
col_width = int(term.width / cols)
|
|
# In case of resize
|
|
if col_width != start_width:
|
|
print(term.clear())
|
|
start_width = col_width
|
|
# sentinel, monitor and workers memory usage
|
|
print(
|
|
term.move(0, 0 * col_width)
|
|
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 1 * col_width)
|
|
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 2 * col_width)
|
|
+ term.black_on_green(
|
|
term.center(_("Available (%)"), width=col_width - 1)
|
|
)
|
|
)
|
|
print(
|
|
term.move(0, 3 * col_width)
|
|
+ term.black_on_green(
|
|
term.center(_("Available (MB)"), width=col_width - 1)
|
|
)
|
|
)
|
|
print(
|
|
term.move(0, 4 * col_width)
|
|
+ term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
|
|
)
|
|
print(
|
|
term.move(0, 5 * col_width)
|
|
+ term.black_on_green(
|
|
term.center(_("Sentinel (MB)"), width=col_width - 1)
|
|
)
|
|
)
|
|
print(
|
|
term.move(0, 6 * col_width)
|
|
+ term.black_on_green(
|
|
term.center(_("Monitor (MB)"), width=col_width - 1)
|
|
)
|
|
)
|
|
print(
|
|
term.move(0, 7 * col_width)
|
|
+ term.black_on_green(
|
|
term.center(_("Workers (MB)"), width=col_width - 1)
|
|
)
|
|
)
|
|
row = 2
|
|
stats = Stat.get_all(broker=broker)
|
|
print(term.clear_eos())
|
|
for stat in stats:
|
|
# memory available (%)
|
|
memory_available_percentage = round(
|
|
psutil.virtual_memory().available
|
|
* 100
|
|
/ psutil.virtual_memory().total,
|
|
2,
|
|
)
|
|
# memory available (MB)
|
|
memory_available = round(psutil.virtual_memory().available / 1024**2, 2)
|
|
if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
|
|
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
|
|
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
|
|
print(
|
|
term.move(row, 0 * col_width)
|
|
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(row, 1 * col_width)
|
|
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(row, 2 * col_width)
|
|
+ term.center(memory_available_percentage, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(row, 3 * col_width)
|
|
+ term.center(memory_available, width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(row, 4 * col_width)
|
|
+ term.center(
|
|
round(psutil.virtual_memory().total / 1024**2, 2),
|
|
width=col_width - 1,
|
|
)
|
|
)
|
|
print(
|
|
term.move(row, 5 * col_width)
|
|
+ term.center(get_process_mb(stat.sentinel), width=col_width - 1)
|
|
)
|
|
print(
|
|
term.move(row, 6 * col_width)
|
|
+ term.center(
|
|
get_process_mb(getattr(stat, "monitor", None)),
|
|
width=col_width - 1,
|
|
)
|
|
)
|
|
workers_mb = 0
|
|
for worker_pid in stat.workers:
|
|
result = get_process_mb(worker_pid)
|
|
if isinstance(result, str):
|
|
result = 0
|
|
workers_mb += result
|
|
print(
|
|
term.move(row, 7 * col_width)
|
|
+ term.center(
|
|
workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
|
|
)
|
|
)
|
|
row += 1
|
|
# each worker's memory usage
|
|
if workers:
|
|
row += 2
|
|
col_width = int(term.width / (1 + Conf.WORKERS))
|
|
print(
|
|
term.move(row, 0 * col_width)
|
|
+ term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
|
|
)
|
|
for worker_num in range(Conf.WORKERS):
|
|
print(
|
|
term.move(row, (worker_num + 1) * col_width)
|
|
+ term.black_on_cyan(
|
|
term.center(
|
|
"Worker #{} (MB)".format(worker_num + 1),
|
|
width=col_width - 1,
|
|
)
|
|
)
|
|
)
|
|
row += 2
|
|
for stat in stats:
|
|
print(
|
|
term.move(row, 0 * col_width)
|
|
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
|
|
)
|
|
for idx, worker_pid in enumerate(stat.workers):
|
|
mb_used = get_process_mb(worker_pid)
|
|
print(
|
|
term.move(row, (idx + 1) * col_width)
|
|
+ term.center(mb_used, width=col_width - 1)
|
|
)
|
|
row += 1
|
|
row += 1
|
|
print(
|
|
term.move(row, 0)
|
|
+ _("Available lowest (): %(memory_percent)s ((at)s)")
|
|
% {
|
|
"memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
|
|
"at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
|
|
"%Y-%m-%d %H:%M:%S+00:00"
|
|
),
|
|
}
|
|
)
|
|
# for testing
|
|
if run_once:
|
|
return Stat.get_all(broker=broker)
|
|
print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
|
|
val = term.inkey(timeout=1)
|
|
|
|
|
|
def get_ids():
|
|
# prints id (PID) of running clusters
|
|
stat = Stat.get_all()
|
|
if stat:
|
|
for s in stat:
|
|
print(s.cluster_id)
|
|
else:
|
|
print(_("No clusters appear to be running."))
|
|
return True
|