cqrenet/relay
3
0
Fork 0
mirror of https://github.com/chatmail/relay.git synced 2026-05-15 10:24:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename deploy_chatmail to cmdeploy

Browse Source
This commit is contained in:
holger krekel
2023-12-11 17:12:47 +01:00
committed by missytake
parent cad1d32682
commit d6df0f0604
32 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

494
cmdeploy/src/cmdeploy/__init__.py Normal file
View File

@@ -0,0 +1,494 @@
"""
Chat Mail pyinfra deploy.
"""
import sys
import importlib.resources
import subprocess
import shutil
import io
from pathlib import Path
from pyinfra import host
from pyinfra.operations import apt, files, server, systemd, pip
from pyinfra.facts.files import File
from pyinfra.facts.systemd import SystemdEnabled
from .acmetool import deploy_acmetool
from chatmaild.config import read_config, Config
def _build_chatmaild(dist_dir) -> None:
dist_dir = Path(dist_dir).resolve()
if dist_dir.exists():
shutil.rmtree(dist_dir)
dist_dir.mkdir()
subprocess.check_output(
[sys.executable, "-m", "build", "-n"]
+ ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
)
entries = list(dist_dir.iterdir())
assert len(entries) == 1
return entries[0]
def remove_legacy_artifacts():
# disable legacy doveauth-dictproxy.service
if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
systemd.service(
name="Disable legacy doveauth-dictproxy.service",
service="doveauth-dictproxy.service",
running=False,
enabled=False,
)
def _install_remote_venv_with_chatmaild(config) -> None:
remove_legacy_artifacts()
dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
remote_base_dir = "/usr/local/lib/chatmaild"
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
remote_venv_dir = f"{remote_base_dir}/venv"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
apt.packages(
name="apt install python3-virtualenv",
packages=["python3-virtualenv"],
)
files.put(
name="Upload chatmaild source package",
src=dist_file.open("rb"),
dest=remote_dist_file,
create_remote_dir=True,
**root_owned,
)
files.put(
name=f"Upload {remote_chatmail_inipath}",
src=config._getbytefile(),
dest=remote_chatmail_inipath,
**root_owned,
)
pip.virtualenv(
name=f"chatmaild virtualenv {remote_venv_dir}",
path=remote_venv_dir,
always_copy=True,
)
server.shell(
name=f"forced pip-install {dist_file.name}",
commands=[
f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
],
)
# install systemd units
for fn in (
"doveauth",
"filtermail",
):
params = dict(
execpath=f"{remote_venv_dir}/bin/{fn}",
config_path=remote_chatmail_inipath,
)
source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
content = source_path.read_text().format(**params).encode()
files.put(
name=f"Upload {fn}.service",
src=io.BytesIO(content),
dest=f"/etc/systemd/system/{fn}.service",
**root_owned,
)
systemd.service(
name=f"Setup {fn} service",
service=f"{fn}.service",
running=True,
enabled=True,
restarted=True,
daemon_reload=True,
)
def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
"""Configures OpenDKIM"""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
dest="/etc/opendkim.conf",
user="root",
group="root",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= main_config.changed
files.directory(
name="Add opendkim directory to /etc",
path="/etc/opendkim",
user="opendkim",
group="opendkim",
mode="750",
present=True,
)
keytable = files.template(
src=importlib.resources.files(__package__).joinpath("opendkim/KeyTable"),
dest="/etc/dkimkeys/KeyTable",
user="opendkim",
group="opendkim",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= keytable.changed
signing_table = files.template(
src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
dest="/etc/dkimkeys/SigningTable",
user="opendkim",
group="opendkim",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= signing_table.changed
files.directory(
name="Add opendkim socket directory to /var/spool/postfix",
path="/var/spool/postfix/opendkim",
user="opendkim",
group="opendkim",
mode="750",
present=True,
)
if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
server.shell(
name="Generate OpenDKIM domain keys",
commands=[
f"opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
],
_sudo=True,
_sudo_user="opendkim",
)
return need_restart
def _install_mta_sts_daemon() -> bool:
need_restart = False
config = files.put(
name="upload postfix-mta-sts-resolver config",
src=importlib.resources.files(__package__).joinpath(
"postfix/mta-sts-daemon.yml"
),
dest="/etc/mta-sts-daemon.yml",
user="root",
group="root",
mode="644",
)
need_restart |= config.changed
server.shell(
name="install postfix-mta-sts-resolver with pip",
commands=[
"python3 -m virtualenv /usr/local/lib/postfix-mta-sts-resolver",
"/usr/local/lib/postfix-mta-sts-resolver/bin/pip install postfix-mta-sts-resolver",
],
)
systemd_unit = files.put(
name="upload mta-sts-daemon systemd unit",
src=importlib.resources.files(__package__).joinpath(
"postfix/mta-sts-daemon.service"
),
dest="/etc/systemd/system/mta-sts-daemon.service",
user="root",
group="root",
mode="644",
)
need_restart |= systemd_unit.changed
return need_restart
def _configure_postfix(config: Config, debug: bool = False) -> bool:
"""Configures Postfix SMTP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
dest="/etc/postfix/main.cf",
user="root",
group="root",
mode="644",
config=config,
)
need_restart |= main_config.changed
master_config = files.template(
src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
dest="/etc/postfix/master.cf",
user="root",
group="root",
mode="644",
debug=debug,
config=config,
)
need_restart |= master_config.changed
return need_restart
def _configure_dovecot(mail_server: str, debug: bool = False) -> bool:
"""Configures Dovecot IMAP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
dest="/etc/dovecot/dovecot.conf",
user="root",
group="root",
mode="644",
config={"hostname": mail_server},
debug=debug,
)
need_restart |= main_config.changed
auth_config = files.put(
src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
dest="/etc/dovecot/auth.conf",
user="root",
group="root",
mode="644",
)
need_restart |= auth_config.changed
files.put(
src=importlib.resources.files(__package__)
.joinpath("dovecot/expunge.cron")
.open("rb"),
dest="/etc/cron.d/expunge",
user="root",
group="root",
mode="644",
)
# as per https://doc.dovecot.org/configuration_manual/os/
# it is recommended to set the following inotify limits
for name in ("max_user_instances", "max_user_watches"):
key = f"fs.inotify.{name}"
server.sysctl(
name=f"Change {key}",
key=key,
value=65535,
persist=True,
)
return need_restart
def _configure_nginx(domain: str, debug: bool = False) -> bool:
"""Configures nginx HTTP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
dest="/etc/nginx/nginx.conf",
user="root",
group="root",
mode="644",
config={"domain_name": domain},
)
need_restart |= main_config.changed
autoconfig = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
user="root",
group="root",
mode="644",
config={"domain_name": domain},
)
need_restart |= autoconfig.changed
mta_sts_config = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
dest="/var/www/html/.well-known/mta-sts.txt",
user="root",
group="root",
mode="644",
config={"domain_name": domain},
)
need_restart |= mta_sts_config.changed
# install CGI newemail script
#
cgi_dir = "/usr/lib/cgi-bin"
files.directory(
name=f"Ensure {cgi_dir} exists",
path=cgi_dir,
user="root",
group="root",
)
files.put(
name="Upload cgi newemail.py script",
src=importlib.resources.files("chatmaild").joinpath("newemail.py").open("rb"),
dest=f"{cgi_dir}/newemail.py",
user="root",
group="root",
mode="755",
)
return need_restart
def check_config(config):
mail_domain = config.mail_domain
if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
blocked_words = "merlinux schmieder testrun.org".split()
for value in config.__dict__.values():
if any(x in value for x in blocked_words):
raise ValueError(
f"please set your own privacy contacts/addresses in {config._inipath}"
)
return config
def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
"""Deploy a chat-mail instance.
:param mail_domain: domain part of your future email addresses
:param mail_server: the DNS name under which your mail server is reachable
:param dkim_selector:
"""
from .www import build_webpages
apt.update(name="apt update", cache_time=24 * 3600)
server.group(name="Create vmail group", group="vmail", system=True)
server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
server.group(name="Create opendkim group", group="opendkim", system=True)
server.user(
name="Add postfix user to opendkim group for socket access",
user="postfix",
groups=["opendkim"],
system=True,
)
# Deploy acmetool to have TLS certificates.
deploy_acmetool(nginx_hook=True, domains=[mail_server, f"mta-sts.{mail_server}"])
apt.packages(
name="Install Postfix",
packages="postfix",
)
apt.packages(
name="Install Dovecot",
packages=["dovecot-imapd", "dovecot-lmtpd"],
)
apt.packages(
name="Install OpenDKIM",
packages=[
"opendkim",
"opendkim-tools",
],
)
apt.packages(
name="Install nginx",
packages=["nginx"],
)
apt.packages(
name="Install fcgiwrap",
packages=["fcgiwrap"],
)
pkg_root = importlib.resources.files(__package__)
chatmail_ini = pkg_root.joinpath("../../../chatmail.ini").resolve()
config = read_config(chatmail_ini)
check_config(config)
www_path = pkg_root.joinpath("../../../www").resolve()
build_dir = www_path.joinpath("build")
src_dir = www_path.joinpath("src")
build_webpages(src_dir, build_dir, config)
files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
_install_remote_venv_with_chatmaild(config)
debug = False
dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
postfix_need_restart = _configure_postfix(config, debug=debug)
opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)
mta_sts_need_restart = _install_mta_sts_daemon()
nginx_need_restart = _configure_nginx(mail_domain)
systemd.service(
name="Start and enable OpenDKIM",
service="opendkim.service",
running=True,
enabled=True,
restarted=opendkim_need_restart,
)
systemd.service(
name="Start and enable MTA-STS daemon",
service="mta-sts-daemon.service",
daemon_reload=True,
running=True,
enabled=True,
restarted=mta_sts_need_restart,
)
systemd.service(
name="Start and enable Postfix",
service="postfix.service",
running=True,
enabled=True,
restarted=postfix_need_restart,
)
systemd.service(
name="Start and enable Dovecot",
service="dovecot.service",
running=True,
enabled=True,
restarted=dovecot_need_restart,
)
systemd.service(
name="Start and enable nginx",
service="nginx.service",
running=True,
enabled=True,
restarted=nginx_need_restart,
)
# This file is used by auth proxy.
# https://wiki.debian.org/EtcMailName
server.shell(
name="Setup /etc/mailname",
commands=[f"echo {mail_domain} >/etc/mailname; chmod 644 /etc/mailname"],
)
journald_conf = files.put(
name="Configure journald",
src=importlib.resources.files(__package__).joinpath("journald.conf"),
dest="/etc/systemd/journald.conf",
user="root",
group="root",
mode="644",
)
systemd.service(
name="Start and enable journald",
service="systemd-journald.service",
running=True,
enabled=True,
restarted=journald_conf,
)

52
cmdeploy/src/cmdeploy/acmetool/__init__.py Normal file
View File

@@ -0,0 +1,52 @@
import importlib.resources
from pyinfra.operations import apt, files, server
def deploy_acmetool(nginx_hook=False, email="", domains=[]):
"""Deploy acmetool."""
apt.packages(
name="Install acmetool",
packages=["acmetool"],
)
files.put(
src=importlib.resources.files(__package__).joinpath("acmetool.cron").open("rb"),
dest="/etc/cron.d/acmetool",
user="root",
group="root",
mode="644",
)
if nginx_hook:
files.put(
src=importlib.resources.files(__package__)
.joinpath("acmetool.hook")
.open("rb"),
dest="/usr/lib/acme/hooks/nginx",
user="root",
group="root",
mode="744",
)
files.template(
src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
dest="/var/lib/acme/conf/responses",
user="root",
group="root",
mode="644",
email=email,
)
files.template(
src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
dest="/var/lib/acme/conf/target",
user="root",
group="root",
mode="644",
)
server.shell(
name=f"Request certificate for: { ', '.join(domains) }",
commands=[f"acmetool want { ' '.join(domains)}"],
)

4
cmdeploy/src/cmdeploy/acmetool/acmetool.cron Normal file
View File

@@ -0,0 +1,4 @@
SHELL=/bin/sh
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
MAILTO=root
20 16 * * * root /usr/bin/acmetool --batch reconcile

5
cmdeploy/src/cmdeploy/acmetool/acmetool.hook Normal file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
set -e
EVENT_NAME="$1"
[ "$EVENT_NAME" = "live-updated" ] || exit 42
systemctl restart nginx.service

2
cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2 Normal file
View File

@@ -0,0 +1,2 @@
"acme-enter-email": "{{ email }}"
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf": true

7
cmdeploy/src/cmdeploy/acmetool/target.yaml.j2 Normal file
View File

@@ -0,0 +1,7 @@
request:
provider: https://acme-v02.api.letsencrypt.org/directory
key:
type: rsa
challenge:
webroot-paths:
- /var/www/html/.well-known/acme-challenge

12
cmdeploy/src/cmdeploy/chatmail.zone.f Normal file
View File

@@ -0,0 +1,12 @@
{chatmail_domain}. MX 10 {chatmail_domain}.
_submission._tcp.{chatmail_domain}. SRV 0 1 587 {chatmail_domain}.
_submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
_imap._tcp.{chatmail_domain}. SRV 0 1 143 {chatmail_domain}.
_imaps._tcp.{chatmail_domain}. SRV 0 1 993 {chatmail_domain}.
{chatmail_domain}. IN CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
{chatmail_domain}. TXT "v=spf1 a:{chatmail_domain} -all"
_dmarc.{chatmail_domain}. TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
_mta-sts.{chatmail_domain}. IN TXT "v=STSv1; id={sts_id}"
mta-sts.{chatmail_domain}. IN CNAME {chatmail_domain}.
_smtp._tls.{chatmail_domain}. IN TXT "v=TLSRPTv1;rua=mailto:{email}"
{dkim_entry}

301
cmdeploy/src/cmdeploy/cmdeploy.py Normal file
View File

@@ -0,0 +1,301 @@
"""
Provides the `cmdeploy` entry point function,
along with command line option and subcommand parsing.
"""
import argparse
import datetime
import shutil
import subprocess
import importlib.resources
import importlib.util
import os
import sys
from pathlib import Path
from termcolor import colored
from chatmaild.config import read_config, write_initial_config
#
# cmdeploy sub commands and options
#
def init_cmd_options(parser):
parser.add_argument(
"chatmail_domain",
action="store",
help="fully qualified DNS domain name for your chatmail instance",
)
def init_cmd(args, out):
"""Initialize chatmail config file."""
if args.inipath.exists():
out.red(f"Path exists, not modifying: {args.inipath}")
raise SystemExit(1)
write_initial_config(args.inipath, args.chatmail_domain)
out.green(f"created config file for {args.chatmail_domain} in {args.inipath}")
def run_cmd_options(parser):
parser.add_argument(
"--dry-run",
dest="dry_run",
action="store_true",
help="don't actually modify the server",
)
def run_cmd(args, out):
"""Deploy chatmail services on the remote server."""
env = os.environ.copy()
env["CHATMAIL_DOMAIN"] = args.config.mail_domain
deploy_path = "cmdeploy/src/cmdeploy/deploy.py"
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
out.check_call(cmd, env=env)
def dns_cmd(args, out):
"""Generate dns zone file."""
template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
ssh = f"ssh root@{args.config.mail_domain}"
def read_dkim_entries(entry):
lines = []
for line in entry.split("\n"):
if line.startswith(";") or not line.strip():
continue
line = line.replace("\t", " ")
lines.append(line)
return "\n".join(lines)
acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
out(
f"[writing {args.config.mail_domain} zone data (using space as separator) to stdout output]",
green=True,
)
print(
template.read_text()
.format(
acme_account_url=acme_account_url,
email=f"root@{args.config.mail_domain}",
sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
chatmail_domain=args.config.mail_domain,
dkim_entry=dkim_entry,
)
.strip()
)
def status_cmd(args, out):
"""Display status for online chatmail instance."""
ssh = f"ssh root@{args.config.mail_domain}"
out.green(f"chatmail domain: {args.config.mail_domain}")
if args.config.privacy_mail:
out.green("privacy settings: present")
else:
out.red("no privacy settings")
s1 = "systemctl --type=service --state=running"
for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
if line.startswith(" "):
print(line)
def test_cmd(args, out):
"""Run local and online tests for chatmail deployment.
This will automatically pip-install 'deltachat' if it's not available.
"""
x = importlib.util.find_spec("deltachat")
if x is None:
out.check_call(f"{sys.executable} -m pip install deltachat")
pytest_path = shutil.which("pytest")
ret = out.run_ret(
[pytest_path, "tests/", "-n4", "-rs", "-x", "-vrx", "--durations=5"]
)
return ret
def fmt_cmd_options(parser):
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store_true",
help="provide information on invocations",
)
parser.add_argument(
"--check",
"-c",
action="store_true",
help="only check but don't fix problems",
)
def fmt_cmd(args, out):
"""Run formattting fixes (fuff and black) on all chatmail source code."""
chatmaild = importlib.resources.files("chatmaild")
cmdeploy = importlib.resources.files("cmdeploy")
tests = cmdeploy.joinpath("../../../tests")
sources = list(str(x) for x in [chatmaild, cmdeploy, tests])
black_args = [shutil.which("black")]
ruff_args = [shutil.which("ruff")]
if args.check:
black_args.append("--check")
else:
ruff_args.append("--fix")
if not args.verbose:
black_args.append("-q")
ruff_args.append("-q")
black_args.extend(sources)
ruff_args.extend(sources)
out.check_call(" ".join(black_args), quiet=not args.verbose)
out.check_call(" ".join(ruff_args), quiet=not args.verbose)
return 0
def bench_cmd(args, out):
"""Run benchmarks against an online chatmail instance."""
pytest_path = shutil.which("pytest")
benchmark = "tests/online/benchmark.py"
subprocess.check_call([pytest_path, benchmark, "-vrx"])
def webdev_cmd(args, out):
"""Run local web development loop for static web pages."""
from .www import main
main()
#
# Parsing command line options and starting commands
#
class Out:
"""Convenience output printer providing coloring."""
def red(self, msg, file=sys.stderr):
print(colored(msg, "red"), file=file)
def green(self, msg, file=sys.stderr):
print(colored(msg, "green"), file=file)
def __call__(self, msg, red=False, green=False, file=sys.stdout):
color = "red" if red else ("green" if green else None)
print(colored(msg, color), file=file)
def shell_output(self, arg):
self(f"[$ {arg}]", file=sys.stderr)
return subprocess.check_output(arg, shell=True).decode()
def check_call(self, arg, env=None, quiet=False):
if not quiet:
self(f"[$ {arg}]", file=sys.stderr)
return subprocess.check_call(arg, shell=True, env=env)
def run_ret(self, args, env=None, quiet=False):
if not quiet:
cmdstring = " ".join(args)
self(f"[$ {cmdstring}]", file=sys.stderr)
proc = subprocess.run(args, env=env)
return proc.returncode
def add_config_option(parser):
parser.add_argument(
"--config",
dest="inipath",
action="store",
default=Path("chatmail.ini"),
type=Path,
help="path to the chatmail.ini file",
)
def add_subcommand(subparsers, func):
name = func.__name__
assert name.endswith("_cmd")
name = name[:-4]
doc = func.__doc__.strip()
help = doc.split("\n")[0].strip(".")
p = subparsers.add_parser(name, description=doc, help=help)
p.set_defaults(func=func)
add_config_option(p)
return p
description = """
Setup your chatmail server configuration and
deploy it via SSH to your remote location.
"""
def get_parser():
"""Return an ArgumentParser for the 'cmdeploy' CLI"""
parser = argparse.ArgumentParser(description=description.strip())
subparsers = parser.add_subparsers(title="subcommands")
# find all subcommands in the module namespace
glob = globals()
for name, func in glob.items():
if name.endswith("_cmd"):
subparser = add_subcommand(subparsers, func)
addopts = glob.get(name + "_options")
if addopts is not None:
addopts(subparser)
return parser
def main(args=None):
"""Provide main entry point for 'xdcget' CLI invocation."""
parser = get_parser()
args = parser.parse_args(args=args)
if not hasattr(args, "func"):
return parser.parse_args(["-h"])
out = Out()
kwargs = {}
if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
if not args.inipath.exists():
out.red(f"expecting {args.inipath} to exist, run init first?")
raise SystemExit(1)
try:
args.config = read_config(args.inipath)
except Exception as ex:
out.red(ex)
raise SystemExit(1)
try:
res = args.func(args, out, **kwargs)
if res is None:
res = 0
return res
except KeyboardInterrupt:
out.red("KeyboardInterrupt")
sys.exit(130)
if __name__ == "__main__":
main()

BIN
cmdeploy/src/cmdeploy/data/delta-chat-bw.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

BIN
cmdeploy/src/cmdeploy/data/opensans-regular.ttf Normal file
View File

Binary file not shown.

19
cmdeploy/src/cmdeploy/deploy.py Normal file
View File

@@ -0,0 +1,19 @@
import os
import pyinfra
from cmdeploy import deploy_chatmail
def main():
mail_domain = os.getenv("CHATMAIL_DOMAIN")
mail_server = os.getenv("CHATMAIL_SERVER", mail_domain)
dkim_selector = os.getenv("CHATMAIL_DKIM_SELECTOR", "dkim")
assert mail_domain
assert mail_server
assert dkim_selector
deploy_chatmail(mail_domain, mail_server, dkim_selector)
if pyinfra.is_cli:
main()

5
cmdeploy/src/cmdeploy/dovecot/auth.conf Normal file
View File

@@ -0,0 +1,5 @@
uri = proxy:/run/dovecot/doveauth.socket:auth
iterate_disable = yes
default_pass_scheme = plain
password_key = passdb/%w/%u
user_key = userdb/%u

144
cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 Normal file
View File

@@ -0,0 +1,144 @@
## Dovecot configuration file
protocols = imap lmtp
auth_mechanisms = plain
{% if debug == true %}
auth_verbose = yes
auth_debug = yes
auth_debug_passwords = yes
auth_verbose_passwords = plain
auth_cache_size = 100M
mail_debug = yes
{% endif %}
mail_plugins = quota
# these are the capabilities Delta Chat cares about actually
# so let's keep the network overhead per login small
# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY
# Authentication for system users.
passdb {
driver = dict
args = /etc/dovecot/auth.conf
}
userdb {
driver = dict
args = /etc/dovecot/auth.conf
}
##
## Mailbox locations and namespaces
##
# Mailboxes are stored in the "mail" directory of the vmail user home.
mail_location = maildir:/home/vmail/mail/%d/%u
namespace inbox {
inbox = yes
mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Trash {
special_use = \Trash
}
# For \Sent mailboxes there are two widely used names. We'll mark both of
# them as \Sent. User typically deletes one of them if duplicates are created.
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
}
mail_uid = vmail
mail_gid = vmail
mail_privileged_group = vmail
##
## Mail processes
##
# Enable IMAP COMPRESS (RFC 4978).
# <https://datatracker.ietf.org/doc/html/rfc4978.html>
protocol imap {
mail_plugins = $mail_plugins imap_zlib imap_quota
}
protocol lmtp {
mail_plugins = $mail_plugins quota
}
plugin {
imap_compress_deflate_level = 6
}
plugin {
# for now we define static quota-rules for all users
quota = maildir:User quota
quota_rule = *:storage=100M
quota_max_mail_size=30M
quota_grace = 0
# quota_over_flag_value = TRUE
}
service lmtp {
user=vmail
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service auth {
unix_listener /var/spool/postfix/private/auth {
mode = 0660
user = postfix
group = postfix
}
}
service auth-worker {
# Default is root.
# Drop privileges we don't need.
user = vmail
}
service imap-login {
# High-security mode.
# Each process serves a single connection and exits afterwards.
# This is the default, but we set it explicitly to be sure.
# See <https://doc.dovecot.org/admin_manual/login_processes/#high-security-mode> for details.
service_count = 1
# Inrease the number of simultaneous connections.
#
# As of Dovecot 2.3.19.1 the default is 100 processes.
# Combined with `service_count = 1` it means only 100 connections
# can be handled simultaneously.
process_limit = 10000
# Avoid startup latency for new connections.
process_min_avail = 10
}
ssl = required
ssl_cert = </var/lib/acme/live/{{ config.hostname }}/fullchain
ssl_key = </var/lib/acme/live/{{ config.hostname }}/privkey
ssl_dh = </usr/share/dovecot/dh.pem
ssl_min_protocol = TLSv1.2
ssl_prefer_server_ciphers = yes

4
cmdeploy/src/cmdeploy/dovecot/expunge.cron Normal file
View File

@@ -0,0 +1,4 @@
2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d INBOX
2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d Deltachat
2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d Trash
2 30 * * * dovecot doveadm purge -A

87
cmdeploy/src/cmdeploy/genqr.py Normal file
View File

@@ -0,0 +1,87 @@
import importlib
import qrcode
import os
from PIL import ImageFont, ImageDraw, Image
import io
def gen_qr_png_data(maildomain):
url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
image = gen_qr(maildomain, url)
temp = io.BytesIO()
image.save(temp, format="png")
temp.seek(0)
return temp
def gen_qr(maildomain, url):
# taken and modified from
# https://github.com/deltachat/mailadm/blob/master/src/mailadm/gen_qr.py
# info = f"{maildomain} invite code"
info = ""
# load QR code
qr = qrcode.QRCode(
version=1,
error_correction=qrcode.constants.ERROR_CORRECT_H,
box_size=1,
border=1,
)
qr.add_data(url)
qr.make(fit=True)
qr_img = qr.make_image(fill_color="black", back_color="white")
# paint all elements
ttf_path = str(
importlib.resources.files(__package__).joinpath("data/opensans-regular.ttf")
)
logo_red_path = str(
importlib.resources.files(__package__).joinpath("data/delta-chat-bw.png")
)
assert os.path.exists(ttf_path), ttf_path
font_size = 16
font = ImageFont.truetype(font=ttf_path, size=font_size)
num_lines = ((info).count("\n") + 1) if info else 0
size = width = 384
qr_padding = 6
text_height = font_size * num_lines
height = size + text_height
image = Image.new("RGBA", (width, height), "white")
qr_final_size = width - (qr_padding * 2)
if num_lines:
draw = ImageDraw.Draw(image)
# draw text
if hasattr(font, "getsize"):
info_pos = (width - font.getsize(info.strip())[0]) // 2
else:
info_pos = (width - font.getbbox(info.strip())[3]) // 2
draw.multiline_text(
(info_pos, size - qr_padding // 2),
info,
font=font,
fill="black",
align="right",
)
# paste QR code
image.paste(
qr_img.resize((qr_final_size, qr_final_size), resample=Image.NEAREST),
(qr_padding, qr_padding),
)
# background delta logo
logo2_img = Image.open(logo_red_path)
logo2_width = int(size / 6)
logo2 = logo2_img.resize((logo2_width, logo2_width), resample=Image.NEAREST)
pos = int((size / 2) - (logo2_width / 2))
image.paste(logo2, (pos, pos), mask=logo2)
return image

2
cmdeploy/src/cmdeploy/journald.conf Normal file
View File

@@ -0,0 +1,2 @@
[Journal]
MaxRetentionSec=3d

37
cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2 Normal file
View File

@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<clientConfig version="1.1">
<emailProvider id="{{ config.domain_name }}">
<domain>{{ config.domain_name }}</domain>
<displayName>{{ config.domain_name }} chatmail</displayName>
<displayShortName>{{ config.domain_name }}</displayShortName>
<incomingServer type="imap">
<hostname>{{ config.domain_name }}</hostname>
<port>993</port>
<socketType>SSL</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</incomingServer>
<incomingServer type="imap">
<hostname>{{ config.domain_name }}</hostname>
<port>143</port>
<socketType>STARTTLS</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</incomingServer>
<outgoingServer type="smtp">
<hostname>{{ config.domain_name }}</hostname>
<port>465</port>
<socketType>SSL</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</outgoingServer>
<outgoingServer type="smtp">
<hostname>{{ config.domain_name }}</hostname>
<port>587</port>
<socketType>STARTTLS</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</outgoingServer>
</emailProvider>
</clientConfig>

4
cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2 Normal file
View File

@@ -0,0 +1,4 @@
version: STSv1
mode: enforce
mx: {{ config.domain_name }}
max_age: 2419200

55
cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 Normal file
View File

@@ -0,0 +1,55 @@
user www-data;
worker_processes auto;
pid /run/nginx.pid;
error_log /var/log/nginx/error.log;
events {
worker_connections 768;
# multi_accept on;
}
http {
sendfile on;
tcp_nopush on;
# Do not emit nginx version on error pages.
server_tokens off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
gzip on;
server {
listen 443 ssl default_server;
listen [::]:443 ssl default_server;
root /var/www/html;
index index.html index.htm;
server_name _;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
try_files $uri $uri/ =404;
}
# add cgi-bin support
include /usr/share/doc/fcgiwrap/examples/nginx.conf;
}
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
return 301 https://$host$request_uri;
}
}

1
cmdeploy/src/cmdeploy/opendkim/KeyTable Normal file
View File

@@ -0,0 +1 @@
dkim._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/dkim.private

1
cmdeploy/src/cmdeploy/opendkim/SigningTable Normal file
View File

@@ -0,0 +1 @@
*@{{ config.domain_name }} {{ config.opendkim_selector }}._domainkey.{{ config.domain_name }}

50
cmdeploy/src/cmdeploy/opendkim/opendkim.conf Normal file
View File

@@ -0,0 +1,50 @@
# OpenDKIM configuration.
Syslog yes
SyslogSuccess yes
#LogWhy no
# Common signing and verification parameters. In Debian, the "From" header is
# oversigned, because it is often the identity key used by reputation systems
# and thus somewhat security sensitive.
Canonicalization relaxed/simple
#Mode sv
#SubDomains no
OversignHeaders From
# Signing domain, selector, and key (required). For example, perform signing
# for domain "example.com" with selector "2020" (2020._domainkey.example.com),
# using the private key stored in /etc/dkimkeys/example.private. More granular
# setup options can be found in /usr/share/doc/opendkim/README.opendkim.
Domain {{ config.domain_name }}
Selector {{ config.opendkim_selector }}
KeyFile /etc/dkimkeys/{{ config.opendkim_selector }}.private
KeyTable /etc/dkimkeys/KeyTable
SigningTable refile:/etc/dkimkeys/SigningTable
# In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
# using a local socket with MTAs that access the socket as a non-privileged
# user (for example, Postfix). You may need to add user "postfix" to group
# "opendkim" in that case.
UserID opendkim
UMask 007
# Socket for the MTA connection (required). If the MTA is inside a chroot jail,
# it must be ensured that the socket is accessible. In Debian, Postfix runs in
# a chroot in /var/spool/postfix, therefore a Unix socket would have to be
# configured as shown on the last line below.
#Socket local:/run/opendkim/opendkim.sock
#Socket inet:8891@localhost
#Socket inet:8891
Socket local:/var/spool/postfix/opendkim/opendkim.sock
PidFile /run/opendkim/opendkim.pid
# Hosts for which to sign rather than verify, default is 127.0.0.1. See the
# OPERATION section of opendkim(8) for more information.
#InternalHosts 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12
# The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
# by the package dns-root-data.
TrustAnchorFile /usr/share/dns/root.key
#Nameservers 127.0.0.1

51
cmdeploy/src/cmdeploy/postfix/main.cf.j2 Normal file
View File

@@ -0,0 +1,51 @@
myorigin = {{ config.mail_domain }}
smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
biff = no
# appending .domain is the MUA's job.
append_dot_mydomain = no
# Uncomment the next line to generate "delayed mail" warnings
#delay_warning_time = 4h
readme_directory = no
# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 2 on
# fresh installs.
compatibility_level = 2
# TLS parameters
smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
smtpd_tls_security_level=may
smtp_tls_CApath=/etc/ssl/certs
smtp_tls_security_level=may
smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
myhostname = {{ config.mail_domain }}
alias_maps = hash:/etc/aliases
alias_database = hash:/etc/aliases
# Postfix does not deliver mail for any domain by itself.
# Primary domain is listed in `virtual_mailbox_domains` instead
# and handed over to Dovecot.
mydestination =
relayhost =
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
mailbox_size_limit = 0
# maximum 30MB sized messages
message_size_limit = 31457280
recipient_delimiter = +
inet_interfaces = all
inet_protocols = all
virtual_transport = lmtp:unix:private/dovecot-lmtp
virtual_mailbox_domains = {{ config.mail_domain }}
smtpd_milters = unix:opendkim/opendkim.sock
non_smtpd_milters = $smtpd_milters

82
cmdeploy/src/cmdeploy/postfix/master.cf.j2 Normal file
View File

@@ -0,0 +1,82 @@
#
# Postfix master process configuration file. For details on the format
# of the file, see the master(5) manual page (command: "man 5 master" or
# on-line: http://www.postfix.org/master.5.html).
#
# Do not forget to execute "postfix reload" after editing this file.
#
# ==========================================================================
# service type private unpriv chroot wakeup maxproc command + args
# (yes) (yes) (no) (never) (100)
# ==========================================================================
{% if debug == true %}
smtp inet n - y - - smtpd -v
{% else %}
smtp inet n - y - - smtpd
{% endif %}
#smtp inet n - y - 1 postscreen
#smtpd pass - - y - - smtpd
#dnsblog unix - - y - 0 dnsblog
#tlsproxy unix - - y - 0 tlsproxy
submission inet n - y - - smtpd
-o syslog_name=postfix/submission
-o smtpd_tls_security_level=encrypt
-o smtpd_sasl_auth_enable=yes
-o smtpd_sasl_type=dovecot
-o smtpd_sasl_path=private/auth
-o smtpd_tls_auth_only=yes
-o smtpd_reject_unlisted_recipient=no
-o smtpd_client_restrictions=$mua_client_restrictions
-o smtpd_helo_restrictions=$mua_helo_restrictions
-o smtpd_sender_restrictions=$mua_sender_restrictions
-o smtpd_recipient_restrictions=
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_client_connection_count_limit=1000
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
smtps inet n - y - - smtpd
-o syslog_name=postfix/smtps
-o smtpd_tls_wrappermode=yes
-o smtpd_tls_security_level=encrypt
-o smtpd_sasl_auth_enable=yes
-o smtpd_sasl_type=dovecot
-o smtpd_sasl_path=private/auth
-o smtpd_reject_unlisted_recipient=no
-o smtpd_client_restrictions=$mua_client_restrictions
-o smtpd_helo_restrictions=$mua_helo_restrictions
-o smtpd_sender_restrictions=$mua_sender_restrictions
-o smtpd_recipient_restrictions=
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-o smtpd_client_connection_count_limit=1000
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
#628 inet n - y - - qmqpd
pickup unix n - y 60 1 pickup
cleanup unix n - y - 0 cleanup
qmgr unix n - n 300 1 qmgr
#qmgr unix n - n 300 1 oqmgr
tlsmgr unix - - y 1000? 1 tlsmgr
rewrite unix - - y - - trivial-rewrite
bounce unix - - y - 0 bounce
defer unix - - y - 0 bounce
trace unix - - y - 0 bounce
verify unix - - y - 1 verify
flush unix n - y 1000? 0 flush
proxymap unix - - n - - proxymap
proxywrite unix - - n - 1 proxymap
smtp unix - - y - - smtp
relay unix - - y - - smtp
-o syslog_name=postfix/$service_name
# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5
showq unix n - y - - showq
error unix - - y - - error
retry unix - - y - - error
discard unix - - y - - discard
lmtp unix - - y - - lmtp
anvil unix - - y - 1 anvil
scache unix - - y - 1 scache
postlog unix-dgram n - n - 1 postlogd
filter unix - n n - - lmtp
# Local SMTP server for reinjecting filered mail.
localhost:{{ config.postfix_reinject_port }} inet n - n - 10 smtpd
-o syslog_name=postfix/reinject

10
cmdeploy/src/cmdeploy/postfix/mta-sts-daemon.service Normal file
View File

@@ -0,0 +1,10 @@
[Unit]
Description=Postfix MTA-STS resolver daemon
[Service]
ExecStart=/usr/local/lib/postfix-mta-sts-resolver/bin/mta-sts-daemon
Restart=always
RestartSec=30
[Install]
WantedBy=multi-user.target

13
cmdeploy/src/cmdeploy/postfix/mta-sts-daemon.yml Normal file
View File

@@ -0,0 +1,13 @@
host: 127.0.0.1
port: 8461
reuse_port: true
shutdown_timeout: 20
cache:
type: internal
options:
cache_size: 10000
proactive_policy_fetching:
enabled: true
default_zone:
strict_testing: false
timeout: 4

110
cmdeploy/src/cmdeploy/www.py Normal file
View File

@@ -0,0 +1,110 @@
import importlib.resources
import webbrowser
import hashlib
import time
import traceback
import markdown
from jinja2 import Template
from .genqr import gen_qr_png_data
from chatmaild.config import read_config
def snapshot_dir_stats(somedir):
d = {}
for path in somedir.iterdir():
if path.is_file() and path.name[0] != "." and path.suffix != ".swp":
mtime = path.stat().st_mtime
hash = hashlib.md5(path.read_bytes()).hexdigest()
d[path] = (mtime, hash)
return d
def prepare_template(source):
assert source.exists(), source
render_vars = {}
render_vars["pagename"] = "home" if source.stem == "index" else source.stem
render_vars["markdown_html"] = markdown.markdown(source.read_text())
page_layout = source.with_name("page-layout.html").read_text()
return render_vars, page_layout
def build_webpages(src_dir, build_dir, config):
try:
_build_webpages(src_dir, build_dir, config)
except Exception:
print(traceback.format_exc())
def _build_webpages(src_dir, build_dir, config):
mail_domain = config.mail_domain
assert src_dir.exists(), src_dir
if not build_dir.exists():
build_dir.mkdir()
qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png")
qr_path.write_bytes(gen_qr_png_data(mail_domain).read())
for path in src_dir.iterdir():
if path.suffix == ".md":
render_vars, content = prepare_template(path)
target = build_dir.joinpath(path.stem + ".html")
# recursive jinja2 rendering
while 1:
new = Template(content).render(config=config, **render_vars)
if new == content:
break
content = new
with target.open("w") as f:
f.write(content)
elif path.name != "page-layout.html":
target = build_dir.joinpath(path.name)
target.write_bytes(path.read_bytes())
return build_dir
def main():
path = importlib.resources.files(__package__)
reporoot = path.joinpath("../../../").resolve()
inipath = reporoot.joinpath("chatmail.ini")
config = read_config(inipath)
config.webdev = True
assert config.mail_domain
www_path = reporoot.joinpath("www")
src_path = www_path.joinpath("src")
stats = None
build_dir = www_path.joinpath("build")
src_dir = www_path.joinpath("src")
index_path = build_dir.joinpath("index.html")
# start web page generation, open a browser and wait for changes
build_webpages(src_dir, build_dir, config)
webbrowser.open(str(index_path))
stats = snapshot_dir_stats(src_path)
print(f"\nOpened URL: file://{index_path.resolve()}\n")
print(f"watching {src_path} directory for changes")
changenum = 0
for count in range(0, 1000000):
newstats = snapshot_dir_stats(src_path)
if newstats == stats and count % 60 != 0:
count += 1
time.sleep(1.0)
continue
for key in newstats:
if stats[key] != newstats[key]:
print(f"*** CHANGED: {key}")
changenum += 1
stats = newstats
build_webpages(src_dir, build_dir, config)
print(f"[{changenum}] regenerated web pages at: {index_path}")
print(f"URL: file://{index_path.resolve()}\n\n")
count = 0
if __name__ == "__main__":
main()