Compare commits

..

5 Commits

Author SHA1 Message Date
missytake b8dcdcba91 docs: add paragraph on how to migrate your local repository to the server 2026-06-03 17:09:54 +02:00
missytake 6a0bf5abe7 docs: webdev needs to be exposed via nginx if run on the relay 2026-06-03 17:09:54 +02:00
missytake e7829672f8 docs: use ssh_host = localhost in getting started docs
remove mentions of the build machine / deployment server separation
2026-06-03 17:09:54 +02:00
missytake 62d19fd910 docs: cmdeploy dns + test are kind of necessary 2026-06-03 17:09:54 +02:00
missytake e1a8d798d3 cmdeploy: add ssh_host chatmail.ini option to deploy remotely 2026-06-03 17:09:54 +02:00
12 changed files with 123 additions and 143 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ name: Trigger Docker build
on:
push:
branches: [main, j4n/dovecot-multidist]
branches: [main]
tags: ['[0-9]+.[0-9]+.[0-9]+']
workflow_dispatch:
+1
View File
@@ -10,6 +10,7 @@ dependencies = [
"filelock",
"requests",
"crypt-r >= 3.13.1 ; python_version >= '3.11'",
"domain-validator",
]
[tool.setuptools]
+3
View File
@@ -2,6 +2,7 @@ import ipaddress
from pathlib import Path
import iniconfig
from domain_validator import DomainValidator
from chatmaild.user import User
@@ -18,12 +19,14 @@ class Config:
params = dict(params)
raw_domain = params.pop("mail_domain")
self.mail_domain_bare = raw_domain
self.ssh_host = params.pop("ssh_host", raw_domain)
if is_valid_ipv4(raw_domain):
self.ipv4_relay = raw_domain
self.mail_domain = f"[{raw_domain}]"
self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
else:
DomainValidator().validate_domain_re(raw_domain)
self.ipv4_relay = None
self.mail_domain = raw_domain
self.postfix_myhostname = raw_domain
@@ -3,6 +3,9 @@
# mail domain (MUST be set to fully qualified chat mail domain)
mail_domain = {mail_domain}
# Where to deploy the relay - if unspecified, mail_domain will be used.
ssh_host = localhost
#
# If you only do private test deploys, you don't need to modify any settings below
#
+4 -4
View File
@@ -96,7 +96,7 @@ def _warn_unused_settings(unused_keys, out):
def run_cmd(args, out):
"""Deploy chatmail services on the remote server."""
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
sshexec = get_sshexec(ssh_host)
require_iroh = args.config.enable_iroh_relay
strict_tls = args.config.tls_cert_mode == "acme"
@@ -116,7 +116,7 @@ def run_cmd(args, out):
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
if ssh_host == "localhost":
if ssh_host in ["localhost", "@local"]:
cmd = f"{pyinf} @local {deploy_path} -y"
if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -158,7 +158,7 @@ def dns_cmd(args, out):
ipv4 = args.config.ipv4_relay
print(f"[WARNING] {ipv4} is not a domain, skipping DNS checks.")
return 0
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
tls_cert_mode = args.config.tls_cert_mode
strict_tls = tls_cert_mode == "acme"
@@ -195,7 +195,7 @@ def status_cmd_options(parser):
def status_cmd(args, out):
"""Display status for online chatmail instance."""
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
out.green(f"chatmail domain: {args.config.mail_domain}")
-6
View File
@@ -389,12 +389,6 @@ class ChatmailDeployer(Deployer):
src=BytesIO(b'APT::Install-Recommends "false";\n'),
dest="/etc/apt/apt.conf.d/00InstallRecommends",
)
# Pin dovecot-* to priority -1 before any apt operation, apt should
# never manage dovecot as our version might be lower than the distro's.
self.put_file(
src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
dest="/etc/apt/preferences.d/pin-dovecot",
)
apt.update(name="apt update", cache_time=24 * 3600)
apt.upgrade(name="upgrade apt packages", auto_remove=True)
+29 -33
View File
@@ -1,3 +1,4 @@
import io
import urllib.request
from chatmaild.config import Config
@@ -18,18 +19,12 @@ DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
DOVECOT_SHA256 = {
("amd64", "bookworm", "core"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
("arm64", "bookworm", "core"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
("amd64", "bookworm", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("arm64", "bookworm", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("amd64", "bookworm", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("arm64", "bookworm", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
("amd64", "trixie", "core"): "406d3781ed81e0913c472077dcf62cb1106e3855983efa6e44ddf43b4b0c9be1",
("arm64", "trixie", "core"): "c75b0d9df11a77d07ebd8522920380c167fa47330ddefebe10575d99d0ecdf7f",
("amd64", "trixie", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("arm64", "trixie", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("amd64", "trixie", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("arm64", "trixie", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
}
@@ -43,32 +38,34 @@ class DovecotDeployer(Deployer):
def install(self):
arch = host.get_fact(Arch)
codename = (host.get_fact(Command, "grep '^VERSION_CODENAME=' /etc/os-release | cut -d= -f2") or "").strip()
if codename not in {key[1] for key in DOVECOT_SHA256}:
raise ValueError(f"Unsupported Debian codename: {codename!r}")
with blocked_service_startup():
debs = []
for pkg in ("core", "imapd", "lmtpd"):
deb, changed = _download_dovecot_package(pkg, arch, codename)
deb, changed = _download_dovecot_package(pkg, arch)
self.need_restart |= changed
if deb:
debs.append(deb)
if debs:
deb_list = " ".join(debs)
# apt-get install with local .deb paths resolves depends
# against the configured repos (e.g. pulls libwrap0),
# The pin file written earlier by ChatmailDeployer prevents apt
# from installing a 'wrong' version
# First dpkg may fail on missing dependencies (stderr suppressed);
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
server.shell(
name="Install dovecot packages",
commands=[
"DEBIAN_FRONTEND=noninteractive apt-get install -y "
'-o Dpkg::Options::="--force-confdef" '
'-o Dpkg::Options::="--force-confold" '
f"--allow-downgrades {deb_list}",
f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
"DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
f"dpkg --force-confdef --force-confold -i {deb_list}",
],
)
self.need_restart = True
self.put_file(
src=io.StringIO(
"Package: dovecot-*\n"
"Pin: version *\n"
"Pin-Priority: -1\n"
),
dest="/etc/apt/preferences.d/pin-dovecot",
)
def configure(self):
configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -81,7 +78,7 @@ class DovecotDeployer(Deployer):
if not self.disable_mail and not self.need_restart:
stale = host.get_fact(
Command,
"pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
" && echo STALE || true",
)
@@ -105,13 +102,13 @@ def _pick_url(primary, fallback):
return fallback
def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[str | None, bool]:
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
"""Download a dovecot .deb if needed, return (path, changed)."""
arch = "amd64" if arch == "x86_64" else arch
arch = "arm64" if arch == "aarch64" else arch
pkg_name = f"dovecot-{package}"
sha256 = DOVECOT_SHA256.get((arch, codename, package))
sha256 = DOVECOT_SHA256.get((package, arch))
if sha256 is None:
op = apt.packages(packages=[pkg_name])
return None, bool(getattr(op, "changed", False))
@@ -122,10 +119,8 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{codename}/{url_version}/{deb_base}"
upstream_version = DOVECOT_ARCHIVE_VERSION.rsplit("-", 1)[0].replace("+", "%2B")
fallback_deb = f"{pkg_name}_{url_version}_{arch}_{codename}.deb"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{upstream_version}/{fallback_deb}"
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
url = _pick_url(primary_url, fallback_url)
deb_filename = f"/root/{deb_base}"
@@ -139,7 +134,6 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
return deb_filename, True
def _configure_dovecot(deployer, config: Config, debug: bool = False):
"""Configures Dovecot IMAP server."""
deployer.put_template(
@@ -150,7 +144,9 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
disable_ipv6=config.disable_ipv6,
)
deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
deployer.put_file(
"dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
)
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
# it is recommended to set the following inotify limits
@@ -1,3 +1,23 @@
/^DKIM-Signature:/ IGNORE
/^Authentication-Results:/ IGNORE
/^Received:/ IGNORE
# List of headers for incoming messages
# that must be retained for functionality and compatibility reasons
/^From:/ DUNNO
/^Message-Id:/ DUNNO
/^Chat-/ DUNNO
/^Content-Type:/ DUNNO
# For receiving clear-text messages (still supported in May 2026)
/^Subject:/ DUNNO
/^Date:/ DUNNO
/^To:/ DUNNO
/^CC:/ DUNNO
/^References:/ DUNNO
/^In-Reply-To:/ DUNNO
# Senders might support Autocrypt 1 but not RFC9788 (Header Protection)
/^Autocrypt:/ DUNNO
# SecureJoin V2 protocol headers (for backward compatibility)
/^Secure-Join/ DUNNO
# Ignore all other headers
/.*/ IGNORE
+4 -2
View File
@@ -10,11 +10,13 @@ from pathlib import Path
import pytest
from chatmaild.config import is_valid_ipv4, read_config
from domain_validator import DomainValidator
def format_mail_domain(raw_domain: str) -> str:
if is_valid_ipv4(raw_domain):
return f"[{raw_domain}]"
DomainValidator().validate_domain_re(raw_domain)
return raw_domain
@@ -60,8 +62,8 @@ def maildomain(chatmail_config):
@pytest.fixture(scope="session")
def sshdomain(maildomain):
return os.environ.get("CHATMAIL_SSH", maildomain)
def sshdomain(chatmail_config):
return os.environ.get("CHATMAIL_SSH", chatmail_config.ssh_host)
@pytest.fixture
@@ -3,7 +3,6 @@ from types import SimpleNamespace
import pytest
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Command
from cmdeploy.dovecot import deployer as dovecot_deployer
@@ -20,7 +19,7 @@ def make_host(*fact_pairs):
"""
facts = dict(fact_pairs)
def get_fact(cls, *args):
def get_fact(cls):
if cls not in facts:
registered = ", ".join(c.__name__ for c in facts)
raise LookupError(
@@ -83,9 +82,7 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package(
"core", "amd64", codename="bookworm"
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
assert changed is False, "should not flag changed when version already installed"
@@ -112,9 +109,7 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package(
"core", "amd64", codename="bookworm"
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
@@ -144,7 +139,6 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
},
),
(dovecot_deployer.Arch, "x86_64"),
(Command, "bookworm"),
),
)
downloads = []
@@ -166,13 +160,11 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
def test_install_unsupported_arch_falls_back_to_apt(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
# For unsupported architectures, all fact lookups return the arch string.
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.Arch, "riscv64"),
(Command, "bookworm"),
),
SimpleNamespace(get_fact=lambda cls: "riscv64"),
)
apt_calls = []
@@ -206,7 +198,6 @@ def test_install_runs_dpkg_when_packages_need_download(
make_host(
(dovecot_deployer.DebPackages, {}),
(dovecot_deployer.Arch, "x86_64"),
(Command, "bookworm"),
),
)
monkeypatch.setattr(
@@ -226,12 +217,10 @@ def test_install_runs_dpkg_when_packages_need_download(
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
)
cmds = track_shell[0]["commands"]
assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
assert "apt-get install -y" in cmds[0]
assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
assert "--allow-downgrades" in cmds[0]
assert ".deb" in cmds[0]
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
assert "apt-get -y --fix-broken install" in cmds[1]
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
assert deployer.need_restart is True, (
"need_restart should be True after dpkg install"
)
@@ -246,19 +235,3 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
assert result == "http://fallback", (
f"should fall back when primary fails, got {result!r}"
)
def test_install_fails_on_unsupported_debian_version(
deployer, patch_blocked, monkeypatch
):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.Arch, "x86_64"),
(Command, "sid"),
),
)
with pytest.raises(ValueError, match="Unsupported Debian codename"):
deployer.install()
+30 -56
View File
@@ -14,21 +14,14 @@ Minimal requirements and prerequisites
You will need the following:
- A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
- Control over a domain through a DNS provider of your choice.
(there is experimental support for :ref:`IP-only relays <iponly>`).
- A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
IPv6 is encouraged if available. Chatmail relay servers only require
1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
chatmail addresses.
- A Linux or Unix **build machine** with key-based SSH access to the root
user of the deployment server.
You must add a passphrase-protected private key to your local ssh-agent because you
cant type in your passphrase during deployment.
(An ed25519 private key is required due to an `upstream bug in
paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
- Control over a domain through a DNS provider of your choice
(there is experimental support for :ref:`IP-only relays <iponly>`).
.. _setup:
@@ -38,7 +31,7 @@ Setup with ``scripts/cmdeploy``
We use ``chat.example.org`` as the chatmail domain in the following
steps. Please substitute it with your own domain.
1. Setup the initial DNS records for your deployment server.
1. Setup the initial DNS records for your relay.
The following is an example in the
familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
Please substitute your domain and IP addresses.
@@ -58,22 +51,25 @@ steps. Please substitute it with your own domain.
The ``mta-sts`` CNAME and ``_mta-sts`` TXT records
are not needed for such domains.
2. On your local PC, clone the repository and bootstrap the Python
2. Login to the server with SSH, clone the repository and bootstrap the Python
virtualenv.
::
ssh root@chat.example.org
git clone https://github.com/chatmail/relay
cd relay
scripts/initenv.sh
3. On your local build machine (PC), create a chatmail configuration file
3. Then, create a chatmail configuration file
``chatmail.ini``:
::
scripts/cmdeploy init chat.example.org # <-- use your domain
.. note::
To use self-signed TLS certificates
instead of Let's Encrypt,
use a domain name starting with ``_``
@@ -84,13 +80,7 @@ steps. Please substitute it with your own domain.
See the :doc:`overview`
for details on certificate provisioning.
4. Verify that SSH root login to the deployment server server works:
::
ssh root@chat.example.org # <-- use your domain
5. From your local build machine, setup and configure the remote deployment server:
4. Now run the deployment script to install the relay to the server:
::
@@ -102,7 +92,6 @@ steps. Please substitute it with your own domain.
public).
Docker installation
-------------------
@@ -110,27 +99,33 @@ There is experimental support for running chatmail via Docker.
A monolithic image based on the above cmdeploy method is available `through a separate repository <https://github.com/chatmail/docker/pkgs/container/docker>`_.
See the `chatmail/docker README <https://github.com/chatmail/docker>`_ for full setup instructions.
Other helpful commands
----------------------
To check the status of your deployment server running the chatmail service:
Next Steps
----------
::
scripts/cmdeploy status
To display and check all recommended DNS records:
Now you should display and check all recommended DNS records
to enable federation with other relays:
::
scripts/cmdeploy dns
To test whether your chatmail service is working correctly:
You should also test whether your chatmail service is working correctly:
::
scripts/cmdeploy test
Other Helpful Commands
----------------------
To check the status of your chatmail relay:
::
scripts/cmdeploy status
To measure the performance of your chatmail service:
::
@@ -171,8 +166,9 @@ This starts a local live development cycle for chatmail web pages:
directory and generating HTML files and copying assets to the
``www/build`` directory.
- Starts a browser window automatically where you can “refresh” as
needed.
- if you are running scripts/cmdeploy webdev on the relay itself,
you need to configure a route in /etc/nginx/nginx.conf
to expose the build directory.
Custom web pages
----------------
@@ -190,7 +186,7 @@ Disable automatic address creation
--------------------------------------------------------
If you need to stop address creation, e.g. because some script is wildly
creating addresses, login with ssh to the deployment machine and run:
creating addresses, login with ssh to the relay and run:
::
@@ -246,25 +242,3 @@ The deploy will verify that both files exist on the server.
If you use such a setup, you must trigger the reload explicitly after renewal::
systemctl start tls-cert-reload.service
Migrating to a new build machine
----------------------------------
To move or add a build machine,
clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
Make sure ``rsync`` is installed, then initialize the environment:
::
./scripts/initenv.sh
Run safety checks before a new deployment:
::
./scripts/cmdeploy dns
./scripts/cmdeploy status
If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
them.
+16 -2
View File
@@ -1,6 +1,6 @@
Migrating to a new machine
===========================
Migrating the relay to a new server
===================================
This migration tutorial provides a step-wise approach
to safely migrate a chatmail relay from one remote machine to another.
@@ -96,3 +96,17 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
If you have lowered the Time-to-Live for DNS records in step 1,
better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
Migrating a local chatmail/relay repository to the server
=========================================================
To move the directory with your local chatmail/relay repository and ``chatmail.ini`` file,
clone the `relay repository <https://github.com/chatmail/relay/>` to the server where the relay is running,
and copy the ``chatmail.ini`` file from your local chatmail/relay repository there.
If you made local changes to the repository,
you can store them in a file with ``git diff origin/main > local-changes.patch``,
copy the file to the repository on the server,
and run ``git apply local-changes.patch``.
Then you can proceed with the `installation steps <setup>`.