mirror of
https://github.com/chatmail/relay.git
synced 2026-05-10 16:04:37 +00:00
feat(chatmaild/fsreport): add Prometheus textfile output, count files
- Count files in report
- Extend size buckets to bigger messages (5, 10 MiB)
- Two textfile exporters:
- Full, bucketed size statistics with --textfile option
- Account count only matching metrics.py format with --legacy-metrics
option (filename defaults to /var/www/html/metrics)
- Improve option help texts
This commit is contained in:
j4n
@@ -13,9 +13,20 @@ to show storage summaries only for first 1000 mailboxes
|
||||
|
||||
python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
|
||||
|
||||
to write Prometheus textfile for node_exporter
|
||||
|
||||
python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/
|
||||
|
||||
writes to /var/lib/prometheus/node-exporter/fsreport.prom
|
||||
|
||||
to also write legacy metrics.py style output (default: /var/www/html/metrics):
|
||||
|
||||
python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/ --legacy-metrics
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from argparse import ArgumentParser
|
||||
from datetime import datetime
|
||||
|
||||
@@ -48,7 +59,19 @@ class Report:
|
||||
self.num_ci_logins = self.num_all_logins = 0
|
||||
self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
|
||||
|
||||
self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
|
||||
KiB = 1024
|
||||
MiB = 1024 * KiB
|
||||
self.message_size_thresholds = (
|
||||
0,
|
||||
100 * KiB,
|
||||
MiB // 2,
|
||||
1 * MiB,
|
||||
2 * MiB,
|
||||
5 * MiB,
|
||||
10 * MiB,
|
||||
)
|
||||
self.message_buckets = {x: 0 for x in self.message_size_thresholds}
|
||||
self.message_count_buckets = {x: 0 for x in self.message_size_thresholds}
|
||||
|
||||
def process_mailbox_stat(self, mailbox):
|
||||
# categorize login times
|
||||
@@ -71,6 +94,7 @@ class Report:
|
||||
if self.mdir and f"/{self.mdir}/" not in msg.path:
|
||||
continue
|
||||
self.message_buckets[size] += msg.size
|
||||
self.message_count_buckets[size] += 1
|
||||
|
||||
self.size_messages += sum(entry.size for entry in mailbox.messages)
|
||||
self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
|
||||
@@ -93,9 +117,10 @@ class Report:
|
||||
|
||||
pref = f"[{self.mdir}] " if self.mdir else ""
|
||||
for minsize, sumsize in self.message_buckets.items():
|
||||
count = self.message_count_buckets[minsize]
|
||||
percent = (sumsize / all_messages * 100) if all_messages else 0
|
||||
print(
|
||||
f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
|
||||
f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%), {count} msgs"
|
||||
)
|
||||
|
||||
user_logins = self.num_all_logins - self.num_ci_logins
|
||||
@@ -111,6 +136,75 @@ class Report:
|
||||
for days, active in self.login_buckets.items():
|
||||
print(f"last {days:3} days: {HSize(active)} {p(active)}")
|
||||
|
||||
def _write_atomic(self, filepath, content):
|
||||
"""Atomically write content to filepath via tmp+rename."""
|
||||
dirpath = os.path.dirname(os.path.abspath(filepath))
|
||||
fd, tmppath = tempfile.mkstemp(dir=dirpath, suffix=".tmp")
|
||||
try:
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(content)
|
||||
os.chmod(tmppath, 0o644)
|
||||
os.rename(tmppath, filepath)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmppath)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def dump_textfile(self, filepath):
|
||||
"""Dump metrics in Prometheus exposition format."""
|
||||
lines = []
|
||||
|
||||
lines.append("# HELP chatmail_storage_bytes Mailbox storage in bytes.")
|
||||
lines.append("# TYPE chatmail_storage_bytes gauge")
|
||||
lines.append(f'chatmail_storage_bytes{{kind="messages"}} {self.size_messages}')
|
||||
lines.append(f'chatmail_storage_bytes{{kind="extra"}} {self.size_extra}')
|
||||
total = self.size_extra + self.size_messages
|
||||
lines.append(f'chatmail_storage_bytes{{kind="total"}} {total}')
|
||||
|
||||
lines.append("# HELP chatmail_messages_bytes Sum of msg bytes >= threshold.")
|
||||
lines.append("# TYPE chatmail_messages_bytes gauge")
|
||||
for minsize, sumsize in self.message_buckets.items():
|
||||
lines.append(f'chatmail_messages_bytes{{min_size="{minsize}"}} {sumsize}')
|
||||
|
||||
lines.append("# HELP chatmail_messages_count Number of msgs >= size threshold.")
|
||||
lines.append("# TYPE chatmail_messages_count gauge")
|
||||
for minsize, count in self.message_count_buckets.items():
|
||||
lines.append(f'chatmail_messages_count{{min_size="{minsize}"}} {count}')
|
||||
|
||||
lines.append("# HELP chatmail_accounts Number of accounts.")
|
||||
lines.append("# TYPE chatmail_accounts gauge")
|
||||
user_logins = self.num_all_logins - self.num_ci_logins
|
||||
lines.append(f'chatmail_accounts{{kind="all"}} {self.num_all_logins}')
|
||||
lines.append(f'chatmail_accounts{{kind="ci"}} {self.num_ci_logins}')
|
||||
lines.append(f'chatmail_accounts{{kind="user"}} {user_logins}')
|
||||
|
||||
lines.append(
|
||||
"# HELP chatmail_accounts_active Non-CI accounts active within N days."
|
||||
)
|
||||
lines.append("# TYPE chatmail_accounts_active gauge")
|
||||
for days, active in self.login_buckets.items():
|
||||
lines.append(f'chatmail_accounts_active{{days="{days}"}} {active}')
|
||||
|
||||
self._write_atomic(filepath, "\n".join(lines) + "\n")
|
||||
|
||||
def dump_compat_textfile(self, filepath):
|
||||
"""Dump legacy metrics.py style metrics."""
|
||||
user_logins = self.num_all_logins - self.num_ci_logins
|
||||
lines = [
|
||||
"# HELP total number of accounts",
|
||||
"# TYPE accounts gauge",
|
||||
f"accounts {self.num_all_logins}",
|
||||
"# HELP number of CI accounts",
|
||||
"# TYPE ci_accounts gauge",
|
||||
f"ci_accounts {self.num_ci_logins}",
|
||||
"# HELP number of non-CI accounts",
|
||||
"# TYPE nonci_accounts gauge",
|
||||
f"nonci_accounts {user_logins}",
|
||||
]
|
||||
self._write_atomic(filepath, "\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def main(args=None):
|
||||
"""Report about filesystem storage usage of all mailboxes and messages"""
|
||||
@@ -127,19 +221,21 @@ def main(args=None):
|
||||
"--days",
|
||||
default=0,
|
||||
action="store",
|
||||
help="assume date to be days older than now",
|
||||
help="assume date to be DAYS older than now",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-login-age",
|
||||
default=0,
|
||||
metavar="DAYS",
|
||||
dest="min_login_age",
|
||||
action="store",
|
||||
help="only sum up message size if last login is at least min-login-age days old",
|
||||
help="only sum up message size if last login is at least DAYS days old",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mdir",
|
||||
metavar="{cur,new,tmp}",
|
||||
action="store",
|
||||
help="only consider 'cur' or 'new' or 'tmp' messages for summary",
|
||||
help="only consider messages in specified Maildir subdirectory for summary",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
@@ -148,6 +244,21 @@ def main(args=None):
|
||||
action="store",
|
||||
help="maximum number of mailboxes to iterate on",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--textfile",
|
||||
metavar="PATH",
|
||||
default=None,
|
||||
help="write Prometheus textfile to PATH (directory or file); "
|
||||
"if PATH is a directory, writes 'fsreport.prom' inside it",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--legacy-metrics",
|
||||
metavar="FILENAME",
|
||||
nargs="?",
|
||||
const="/var/www/html/metrics",
|
||||
default=None,
|
||||
help="write legacy metrics.py textfile (default: /var/www/html/metrics)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(args)
|
||||
|
||||
@@ -161,6 +272,14 @@ def main(args=None):
|
||||
rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
|
||||
for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
|
||||
rep.process_mailbox_stat(mbox)
|
||||
if args.textfile:
|
||||
path = args.textfile
|
||||
if os.path.isdir(path):
|
||||
path = os.path.join(path, "fsreport.prom")
|
||||
rep.dump_textfile(path)
|
||||
if args.legacy_metrics:
|
||||
rep.dump_compat_textfile(args.legacy_metrics)
|
||||
if not args.textfile and not args.legacy_metrics:
|
||||
rep.dump_summary()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user