Compare commits

..

3 Commits

Author SHA1 Message Date
holger krekel
c849036d0b fix tar commands 2025-12-18 23:40:20 +01:00
holger krekel
bf371e7b6d use $OLD_IP4 and $NEW_IP4 to make docs more readable. Also streamline "set TTL to 5 minute" phrasing a bit. 2025-12-18 16:58:05 +01:00
missytake
35867153af docs: update migration guide after nine migration 2025-12-18 09:32:25 +01:00
15 changed files with 136 additions and 189 deletions

View File

@@ -19,8 +19,13 @@ jobs:
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency: staging-ipv4.testrun.org
concurrency:
group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
steps:
- uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/checkout@v4
- name: prepare SSH
@@ -88,7 +93,7 @@ jobs:
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
- name: cmdeploy dns
run: cmdeploy dns -v

View File

@@ -19,8 +19,13 @@ jobs:
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency: staging2.testrun.org
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
steps:
- uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/checkout@v4
- name: prepare SSH
@@ -89,7 +94,7 @@ jobs:
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
- name: cmdeploy dns
run: cmdeploy dns -v

View File

@@ -1,31 +1,5 @@
# Changelog for chatmail deployment
## 1.9.0 2025-12-18
### Documentation
- Add RELEASE.md and CONTRIBUTING.md
- README update, mention Chatmail Cookbook project
### Bug Fixes
- Expire messages also from IMAP subfolders
- Use absolute path instead of relative path in message expiration script
- Restart Postfix and Dovecot automatically on failure
- acmetool: Use a fixed name and `reconcile` instead of `want`
### Features
- Report DKIM error code in SMTP response
- Remove development notice from the web pages
### Miscellaneous Tasks
- Update the heading in the CHANGELOG.md
- Setup git-cliff
- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
## 1.8.0 2025-12-12
- Add imap_compress option to chatmail.ini

View File

@@ -14,7 +14,7 @@ from stat import S_ISREG
from chatmaild.config import read_config
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
def iter_mailboxes(basedir, maxnum):
@@ -51,27 +51,33 @@ class MailboxStat:
def __init__(self, basedir):
self.basedir = str(basedir)
# all detected messages in cur/new/tmp folders
self.messages = []
self.extrafiles = []
self.scandir(self.basedir)
def scandir(self, folderdir):
for name in os_listdir_if_exists(folderdir):
path = f"{folderdir}/{name}"
# all detected files in mailbox top dir
self.extrafiles = []
# scan all relevant files (without recursion)
old_cwd = os.getcwd()
try:
os.chdir(self.basedir)
except FileNotFoundError:
return
for name in os_listdir_if_exists("."):
if name in ("cur", "new", "tmp"):
for msg_name in os_listdir_if_exists(path):
entry = get_file_entry(f"{path}/{msg_name}")
for msg_name in os_listdir_if_exists(name):
entry = get_file_entry(f"{name}/{msg_name}")
if entry is not None:
self.messages.append(entry)
elif os.path.isdir(path):
self.scandir(path)
else:
entry = get_file_entry(path)
entry = get_file_entry(name)
if entry is not None:
self.extrafiles.append(entry)
if name == "password":
self.last_login = entry.mtime
self.extrafiles.sort(key=lambda x: -x.size)
os.chdir(old_cwd)
def print_info(msg):
@@ -124,6 +130,13 @@ class Expiry:
self.remove_mailbox(mbox.basedir)
return
# all to-be-removed files are relative to the mailbox basedir
try:
os.chdir(mbox.basedir)
except FileNotFoundError:
print_info(f"mailbox not found/vanished {mbox.basedir}")
return
mboxname = os.path.basename(mbox.basedir)
if self.verbose:
date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -134,12 +147,11 @@ class Expiry:
self.all_files += len(mbox.messages)
for message in mbox.messages:
if message.mtime < cutoff_mails:
self.remove_file(message.path, mtime=message.mtime)
self.remove_file(message.relpath, mtime=message.mtime)
elif message.size > 200000 and message.mtime < cutoff_large_mails:
# we only remove noticed large files (not unnoticed ones in new/)
parts = message.path.split("/")
if len(parts) >= 2 and parts[-2] == "cur":
self.remove_file(message.path, mtime=message.mtime)
if message.relpath.startswith("cur/"):
self.remove_file(message.relpath, mtime=message.mtime)
else:
continue
changed = True

View File

@@ -307,9 +307,12 @@ class IncomingBeforeQueueHandler:
return error
log_info("re-injecting the mail that passed checks")
# the smtp daemon on reinject_port_incoming gives it to dkim milter
# which looks at source address to determine whether to verify or sign
client = SMTPClient(
"localhost",
self.config.postfix_reinject_port_incoming,
source_address=("127.0.0.2", 0),
)
client.sendmail(
envelope.mail_from, envelope.rcpt_tos, envelope.original_content

View File

@@ -15,7 +15,7 @@ mail_domain = {mail_domain}
max_user_send_per_minute = 60
# maximum mailbox size of a chatmail address
max_mailbox_size = 500M
max_mailbox_size = 100M
# maximum message size for an e-mail in bytes
max_message_size = 31457280

View File

@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
assert config.filtermail_smtp_port == 10080
assert config.postfix_reinject_port == 10025
assert config.max_user_send_per_minute == 60
assert config.max_mailbox_size == "500M"
assert config.max_mailbox_size == "100M"
assert config.delete_mails_after == "20"
assert config.delete_large_after == "7"
assert config.username_min_length == 9

View File

@@ -17,17 +17,19 @@ from chatmaild.expire import main as expiry_main
from chatmaild.fsreport import main as report_main
def fill_mbox(folderdir):
password = folderdir.joinpath("password")
def fill_mbox(basedir):
basedir1 = basedir.joinpath("mailbox1@example.org")
basedir1.mkdir()
password = basedir1.joinpath("password")
password.write_text("xxx")
folderdir.joinpath("maildirsize").write_text("xxx")
basedir1.joinpath("maildirsize").write_text("xxx")
garbagedir = folderdir.joinpath("garbagedir")
garbagedir = basedir1.joinpath("garbagedir")
garbagedir.mkdir()
garbagedir.joinpath("bimbum").write_text("hello")
create_new_messages(folderdir, ["cur/msg1"], size=500)
create_new_messages(folderdir, ["new/msg2"], size=600)
create_new_messages(basedir1, ["cur/msg1"], size=500)
create_new_messages(basedir1, ["new/msg2"], size=600)
return basedir1
def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -43,21 +45,8 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
@pytest.fixture
def mbox1(example_config):
mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
mboxdir.mkdir()
fill_mbox(mboxdir)
return MailboxStat(mboxdir)
def test_deltachat_folder(example_config):
"""Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
mboxdir.mkdir()
mbox2dir = mboxdir.joinpath(".DeltaChat")
mbox2dir.mkdir()
fill_mbox(mbox2dir)
mb = MailboxStat(mboxdir)
assert len(mb.messages) == 2
basedir1 = fill_mbox(example_config.mailboxes_dir)
return MailboxStat(basedir1)
def test_filentry_ordering(tmp_path):
@@ -87,7 +76,7 @@ def test_stats_mailbox(mbox1):
create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
create_new_messages(mbox1.basedir, ["index-something"], size=3)
mbox2 = MailboxStat(mbox1.basedir)
assert len(mbox2.extrafiles) == 5
assert len(mbox2.extrafiles) == 4
assert mbox2.extrafiles[0].size == 1000
# cope well with mailbox dirs that have no password (for whatever reason)

View File

@@ -1,5 +1,4 @@
mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
if mtaname == "ORIGINATING" then
if odkim.internal_ip(ctx) == 1 then
-- Outgoing message will be signed,
-- no need to look for signatures.
return nil
@@ -34,15 +33,6 @@ if valid then
for i = nsigs, 1, -1 do
odkim.del_header(ctx, "DKIM-Signature", i)
end
-- Delete first and presumably only occurence
odkim.del_header(ctx, "Authentication-Results", 0)
else
odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
-- Delete in reverse order to avoid index shifting.
for i = nsigs, 1, -1 do
odkim.del_header(ctx, "DKIM-Signature", i)
end
else
odkim.set_reply(ctx, "554", "5.7.1", error_msg)
odkim.set_result(ctx, SMFIS_REJECT)

View File

@@ -65,9 +65,3 @@ PidFile /run/opendkim/opendkim.pid
# The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
# by the package dns-root-data.
TrustAnchorFile /usr/share/dns/root.key
# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
MTA ORIGINATING
# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
InternalHosts -

View File

@@ -31,6 +31,7 @@ submission inet n - y - 5000 smtpd
-o smtpd_sender_restrictions=$mua_sender_restrictions
-o smtpd_recipient_restrictions=
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_client_connection_count_limit=1000
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
smtps inet n - y - 5000 smtpd
@@ -48,6 +49,7 @@ smtps inet n - y - 5000 smtpd
-o smtpd_recipient_restrictions=
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-o smtpd_client_connection_count_limit=1000
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
#628 inet n - y - - qmqpd
pickup unix n - y 60 1 pickup
@@ -79,7 +81,6 @@ filter unix - n n - - lmtp
# Local SMTP server for reinjecting outgoing filtered mail.
127.0.0.1:{{ config.postfix_reinject_port }} inet n - n - 100 smtpd
-o syslog_name=postfix/reinject
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_milters=unix:opendkim/opendkim.sock
-o cleanup_service_name=authclean

View File

@@ -16,16 +16,15 @@ You will need the following:
- Control over a domain through a DNS provider of your choice.
- A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
- A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
IPv6 is encouraged if available. Chatmail relay servers only require
1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
chatmail addresses.
- A Linux or Unix **build machine** with key-based SSH access to the root
user of the deployment server.
You must add a passphrase-protected private key to your local ssh-agent because you
cant type in your passphrase during deployment.
(An ed25519 private key is required due to an `upstream bug in
- Key-based SSH authentication to the root user. You must add a
passphrase-protected private key to your local ssh-agent because you
cant type in your passphrase during deployment. (An ed25519 private
key is required due to an `upstream bug in
paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
@@ -35,17 +34,16 @@ Setup with ``scripts/cmdeploy``
We use ``chat.example.org`` as the chatmail domain in the following
steps. Please substitute it with your own domain.
1. Setup the initial DNS records for your deployment server.
The following is an example in the
1. Setup the initial DNS records. The following is an example in the
familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
Please substitute your domain and IP addresses.
::
chat.example.org. 3600 IN A 198.51.100.5
chat.example.org. 3600 IN AAAA 2001:db8::5
www.chat.example.org. 3600 IN CNAME chat.example.org.
mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
chat.example.com. 3600 IN A 198.51.100.5
chat.example.com. 3600 IN AAAA 2001:db8::5
www.chat.example.com. 3600 IN CNAME chat.example.com.
mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
2. On your local PC, clone the repository and bootstrap the Python
virtualenv.
@@ -56,20 +54,20 @@ steps. Please substitute it with your own domain.
cd relay
scripts/initenv.sh
3. On your local build machine (PC), create a chatmail configuration file
3. On your local PC, create chatmail configuration file
``chatmail.ini``:
::
scripts/cmdeploy init chat.example.org # <-- use your domain
4. Verify that SSH root login to the deployment server server works:
4. Verify that SSH root login to your remote server works:
::
ssh root@chat.example.org # <-- use your domain
5. From your local build machine, setup and configure the remote deployment server:
5. From your local PC, deploy the remote chatmail relay server:
::
@@ -83,7 +81,7 @@ steps. Please substitute it with your own domain.
Other helpful commands
----------------------
To check the status of your deployment server running the chatmail service:
To check the status of your remotely running chatmail service:
::
@@ -160,7 +158,7 @@ Disable automatic address creation
--------------------------------------------------------
If you need to stop address creation, e.g. because some script is wildly
creating addresses, login with ssh to the deployment machine and run:
creating addresses, login with ssh and run:
::
@@ -169,23 +167,3 @@ creating addresses, login with ssh to the deployment machine and run:
Chatmail address creation will be denied while this file is present.
Migrating to a new build machine
----------------------------------
To move or add a build machine,
clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
Make sure ``rsync`` is installed, then initialize the environment:
::
./scripts/initenv.sh
Run safety checks before a new deployment:
::
./scripts/cmdeploy dns
./scripts/cmdeploy status
If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
them.

View File

@@ -1,72 +1,77 @@
Migrating to a new machine
===========================
Migrating to a new host
-----------------------
This migration tutorial provides a step-wise approach
to safely migrate a chatmail relay from one remote machine to another.
If you want to migrate chatmail relay from an old machine to a new
machine, you can use these steps. They were tested with a Linux laptop;
you might need to adjust some of the steps to your environment.
Preliminary notes and assumptions
---------------------------------
Lets assume that your ``mail_domain`` is ``mail.example.org``, all
involved machines run Debian 12, your old sites IP version 4 address is
``$OLD_IP4``, and your new sites IP4 address is ``$NEW_IP4``.
- If the migration is a planned move,
it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
at best much earlier than the actual planned migration.
This speeds up propagation of DNS changes in the Internet after the migration is complete.
First of all, you should lower the Time To Live (TTL) of your DNS records
to a value such as 300 (5 minutes).
Short TTL values allow to change DNS records during the migration more timely.
- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
During the guide you might get a warning about changed SSH Host keys; in
this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
- Your ``mail_domain`` is ``mail.example.org``.
1. First, to make the downtime during the migration shorter,
let's transfer the current state of the mailboxes.
Login to your old machine (while forwarding your ssh-agent with ``ssh -A``)
so you can copy directly from the old to the new site with your SSH
key:
- All remote machines run Debian 12.
- The old sites IP version 4 address is ``$OLD_IP4``.
- The new sites IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
The six steps to migrate
------------------------
Note that during some of the following steps you might get a warning about changed SSH Host keys;
in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
1. **Initially transfer mailboxes from old to new site.**
Login to old site, forwarding your ssh-agent with ``ssh -A``
to allow using ssh to directly copy files from old to new site.
::
ssh -A root@$OLD_IP4
tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
This saves us time during the downtime,
at least the mailboxes are there already.
They contain user passwords, encrypted push notification tokens,
messages which might not have been fetched by all devices of the user yet,
and dovecot indexes which track the state of the mailbox.
2. **Pre-configure the new site but keep it inactive until step 6**
::
CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
3. **It's getting serious: disable mail services on the old site.**
Users will not be able to send or receive messages until all steps are completed.
Other relays and mail servers will retry delivering messages from time to time,
so nothing is lost for users.
2. Then, from your local machine, install chatmail on the new machine, but don't activate it yet:
::
scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
CMDEPLOY_STAGES=install,configure cmdeploy run --ssh-host $NEW_IP4
The services are disabled for now; we will enable them later.
We first need to make the new site fully operational.
3. Now it's getting serious: disable the mail services on the old site.
::
cmdeploy run --disable-mail --ssh-host $OLD_IP4
Your users will start to notice the migration and will not be able to send
or receive messages until the migration is completed.
Other relays and mail servers will wait with delivering messages
until your relay is reachable again.
4. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
``/etc/dkimkeys``, and ``/var/spool/postfix`` to
the new site. Let's forward the SSH agent again to copy the files directly.
This time, we copy ``/home/vmail/mail`` with rsync to only copy the recent changes:
4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
from the old to the new site.
::
ssh -A root@$OLD_IP4
tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
Login to the new site and ensure file ownerships are correctly set:
This transfers all messages which have not been fetched yet, the TLS certificate,
and DKIM keys (so DKIM DNS record remains valid).
It also preserves the Postfix mail spool so any messages
pending delivery will still be delivered.
5. Now login to the new site and run the following to ensure the ownership is correct
in case UIDs/GIDs changed:
::
@@ -75,8 +80,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
chown opendkim: -R /etc/dkimkeys
chown vmail: -R /home/vmail/mail
5. **Update the DNS entries to point to the new site.**
6. Now, update the DNS entries.
You only need to change the ``A`` and ``AAAA`` records, for example:
::
@@ -84,15 +88,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
mail.example.org. IN A $NEW_IP4
mail.example.org. IN AAAA $NEW_IP6
6. **Activate chatmail relay on new site.**
::
CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
Voilà!
Users will be able to use the relay as soon as the DNS changes have propagated.
If you have lowered the Time-to-Live for DNS records in step 1,
better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
7. Finally, you can execute ``CMDEPLOY_STAGES=activate cmdeploy run --ssh-host $NEW_IP4`` to
turn on chatmail on the new relay. Your users will be able to use the
chatmail relay as soon as the DNS changes have propagated. Voilà!

View File

@@ -272,8 +272,8 @@ by OpenDKIM screen policy script before validating the signatures. This
corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
If there is no valid DKIM signature on the incoming email, the
sender receives a “5.7.1 No valid DKIM signature found” error.
After validating the DKIM signature,
the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
After validating the DKIM signature,
the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
Note that chatmail relays

View File

@@ -14,10 +14,10 @@ We know of three work-in-progress alternative implementation efforts:
it to support all of the features and configuration settings required
to operate as a chatmail relay.
- `Madmail <https://github.com/omidz4t/madmail>`_: an
experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
for chatmail deployments. It provides a single binary solution
for running a chatmail relay.
- `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
plugin for the `Maddy email server <https://maddy.email/>`_ which
aims to implement the chatmail relay features and configuration
options.
- `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
A Chef Cookbook implementing a relay server. The project follows the