Document how to migrate the server

CHANGELOG.md

@@ -2,16 +2,6 @@
 
 ## untagged
 
-- add mtail support (new optional `mail_address` ini value)
-  This defines the address on which [`mtail`](https://google.github.io/mtail/)
-  exposes its metrics collected from the logs.
-  If you want to collect the metrics with Prometheus,
-  setup a private network (e.g. WireGuard interface)
-  and assign an IP address from this network to the host.
-  If you do not plan to collect metrics,
-  keep this setting unset.
-  ([#388](https://github.com/deltachat/chatmail/pull/388))
-
 - fix checking for required DNS records
   ([#412](https://github.com/deltachat/chatmail/pull/412))
 

README.md

@@ -188,120 +188,112 @@ and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch
 `From:` header must correspond to envelope MAIL FROM,
 this is ensured by `filtermail` proxy.
 
-## Setting up a reverse proxy
+## Migrating chatmail server to a new host
 
-A chatmail server does not depend on the client IP address
-for its operation, so it can be run behind a reverse proxy.
-This will not even affect incoming mail authentication
-as DKIM only checks the cryptographic signature
-of the message and does not use the IP address as the input.
+If you want to migrate your chatmail server to a new host,
+follow these steps:
 
-For example, you may want to self-host your chatmail server
-and only use hosted VPS to provide a public IP address
-for client connections and incoming mail.
-You can connect chatmail server to VPS
-using a tunnel protocol
-such as [WireGuard](https://www.wireguard.com/)
-and setup a reverse proxy on a VPS
-to forward connections to the chatmail server
-over the tunnel.
-You can also setup multiple reverse proxies
-for your chatmail server in different networks
-to ensure your server is reachable even when
-one of the IPs becomes inaccessible due to
-hosting or routing problems.
+1. Block all ports except 80 and 22 with firewall on a new server.
 
-Note that your server still needs
-to be able to make outgoing connections on port 25
-to send messages outside.
-
-To setup a reverse proxy
-(or rather Destination NAT, DNAT)
-for your chatmail server,
-put the following configuration in `/etc/nftables.conf`:
+   To do this, add the following config to `/etc/nftables.conf`:
 ```
 #!/usr/sbin/nft -f
 
 flush ruleset
 
-define wan = eth0
+table inet filter {
+    chain input {
+        type filter hook input priority filter; policy drop;
 
-# Which ports to proxy.
-#
-# Note that SSH is not proxied
-# so it is possible to log into the proxy server
-# and not the original one.
-define ports = { smtp, http, https, imap, imaps, submission, submissions }
+        # Accept ICMP.
+        # It is especially important to accept ICMPv6 ND messages,
+        # otherwise IPv6 connectivity breaks.
+        icmp type { echo-request } accept
+        icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
 
-# The host we want to proxy to.
-define ipv4_address = AAA.BBB.CCC.DDD
-define ipv6_address = [XXX::1]
+        tcp dport { ssh, http } accept
 
-table ip nat {
-        chain prerouting {
-                type nat hook prerouting priority dstnat; policy accept;
-                iif $wan tcp dport $ports dnat to $ipv4_address
-        }
-
-        chain postrouting {
-                type nat hook postrouting priority 0;
-
-                oifname $wan masquerade
-        }
+        ct state established accept
+    }
+    chain forward {
+        type filter hook forward priority filter;
+    }
+    chain output {
+        type filter hook output priority filter;
+    }
 }
+```
+   Then execute `nft -f /etc/nftables.conf` as root.
 
-table ip6 nat {
-        chain prerouting {
-                type nat hook prerouting priority dstnat; policy accept;
-                iif $wan tcp dport $ports dnat to $ipv6_address
-        }
+   This will ensure users will not connect to the new server
+   and mails will not be delivered to the new server
+   before you finish the setup.
 
-        chain postrouting {
-                type nat hook postrouting priority 0;
+   Port 22 is needed for SSH access
+   and port 80 is needed to get a TLS certificate.
+   They are not used by Delta Chat
+   or by other email servers trying to deliver the messages.
 
-                oifname $wan masquerade
-        }
-}
+2. Point DNS to the new IP addresses.
+
+   You can already remove the old IP addresses from DNS.
+   Existing Delta Chat users will still be able to connect
+   to the old server, send and receive messages,
+   but new users will fail to create new profiles
+   with your chatmail server.
+
+3. Setup the new server with `cmdeploy`.
+
+   This step is similar to initial setup.
+   However, because ports Delta Chat uses are blocked,
+   new server will not become usable immediately.
+   If other servers try to deliver messages to your new server they will fail,
+   but normally email servers will retry delivering messages
+   for at least a week, so messages will not be lost.
+
+4. Firewall all ports except `ssh` (22) on the old server.
+   Existing users will not be able to connect from now on
+   and no more messages will be delivered to your old chatmail server.
+
+   Blocking users from connecting to the new server
+   until mailboxes are migrated is needed to avoid UID validity change.
+   If Delta Chat connects to the new server before it is fully set up,
+   it will lose track of the IMAP message UID
+   and miss messages that arrived during migration.
+
+   Same for SMTP port 25, you want it blocked during migration so no new mails arrive
+   while the server is moving.
+
+5. Use `rsync -avz` over SSH to copy /home/vmail/mail from the old server to the new one
+   preserving file permissions and timestamps.
+
+6. Unblock ports used by Delta Chat and SMTP message exchange.
+   For that you can modify `/etc/nftables.conf` as follows:
+```
+#!/usr/sbin/nft -f
+
+flush ruleset
 
 table inet filter {
-        chain input {
-                type filter hook input priority filter; policy drop;
+    chain input {
+        type filter hook input priority filter; policy drop;
 
-                # Accept ICMP.
-                # It is especially important to accept ICMPv6 ND messages,
-                # otherwise IPv6 connectivity breaks.
-                icmp type { echo-request } accept
-                icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
+        # Accept ICMP.
+        # It is especially important to accept ICMPv6 ND messages,
+        # otherwise IPv6 connectivity breaks.
+        icmp type { echo-request } accept
+        icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
 
-                # Allow incoming SSH connections.
-                tcp dport { ssh } accept
+        tcp dport { ssh, smtp, http, https, imap, imaps, submission, submissions } accept
 
-                ct state established accept
-        }
-        chain forward {
-                type filter hook forward priority filter; policy drop;
-
-                ct state established accept
-                ip daddr $ipv4_address counter accept
-                ip6 daddr $ipv6_address counter accept
-        }
-        chain output {
-                type filter hook output priority filter;
-        }
+        ct state established accept
+    }
+    chain forward {
+        type filter hook forward priority filter;
+    }
+    chain output {
+        type filter hook output priority filter;
+    }
 }
 ```
-
-Run `systemctl enable nftables.service`
-to ensure configuration is reloaded when the proxy server reboots.
-
-Uncomment in `/etc/sysctl.conf` the following two lines:
-
-```
-net.ipv4.ip_forward=1
-net.ipv6.conf.all.forwarding=1
-```
-
-Then reboot the server or do `sysctl -p` and `nft -f /etc/nftables.conf`.
-
-Once proxy server is set up,
-you can add its IP address to the DNS.
+Execute `nft -f /etc/nftables.conf` as root to apply the changes.

chatmaild/src/chatmaild/config.py

@@ -30,7 +30,6 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
-        self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")

chatmaild/src/chatmaild/filtermail.py

@@ -183,29 +183,15 @@ class BeforeQueueHandler:
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
-        envelope_from_domain = from_addr.split("@").pop()
-
         logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-        if mail_encrypted:
-            print("Filtering encrypted mail.", file=sys.stderr)
-        else:
-            print("Filtering unencrypted mail.", file=sys.stderr)
-
         if envelope.mail_from in self.config.passthrough_senders:
             return
 
         passthrough_recipients = self.config.passthrough_recipients
-
-        is_securejoin = message.get("secure-join") in [
-            "vc-request",
-            "vg-request",
-        ]
-        if is_securejoin:
-            return
-
+        envelope_from_domain = from_addr.split("@").pop()
         for recipient in envelope.rcpt_tos:
             if envelope.mail_from == recipient:
                 # Always allow sending emails to self.
@@ -219,8 +205,12 @@ class BeforeQueueHandler:
 
             is_outgoing = recipient_domain != envelope_from_domain
             if is_outgoing and not mail_encrypted:
-                print("Rejected unencrypted mail.", file=sys.stderr)
-                return f"500 Invalid unencrypted mail to <{recipient}>"
+                is_securejoin = message.get("secure-join") in [
+                    "vc-request",
+                    "vg-request",
+                ]
+                if not is_securejoin:
+                    return f"500 Invalid unencrypted mail to <{recipient}>"
 
 
 class SendRateLimiter:

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -55,22 +55,6 @@ postfix_reinject_port = 10025
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
-# Address on which `mtail` listens,
-# e.g. 127.0.0.1 or some private network
-# address like 192.168.10.1.
-# You can point Prometheus
-# or some other OpenMetrics-compatible
-# collector to
-# http://{{mtail_address}}:3903/metrics
-# and display collected metrics with Grafana.
-#
-# WARNING: do not expose this service
-# to the public IP address.
-#
-# `mtail is not running if the setting is not set.
-
-# mtail_address = 127.0.0.1
-
 #
 # Debugging options 
 #

cmdeploy/src/cmdeploy/__init__.py

@@ -441,44 +441,6 @@ def check_config(config):
     return config
 
 
-def deploy_mtail(config):
-    apt.packages(
-        name="Install mtail",
-        packages=["mtail"],
-    )
-
-    # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
-    # This allows to read from journalctl instead of log files.
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"),
-        dest="/etc/systemd/system/mtail.service",
-        user="root",
-        group="root",
-        mode="644",
-        address=config.mtail_address or "127.0.0.1",
-        port=3903,
-    )
-
-    mtail_conf = files.put(
-        name="Mtail configuration",
-        src=importlib.resources.files(__package__).joinpath(
-            "mtail/delivered_mail.mtail"
-        ),
-        dest="/etc/mtail/delivered_mail.mtail",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    systemd.service(
-        name="Start and enable mtail",
-        service="mtail.service",
-        running=bool(config.mtail_address),
-        enabled=bool(config.mtail_address),
-        restarted=mtail_conf.changed,
-    )
-
-
 def deploy_chatmail(config_path: Path) -> None:
     """Deploy a chat-mail instance.
 
@@ -674,5 +636,3 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
-
-    deploy_mtail(config)

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -74,7 +74,7 @@ def run_cmd(args, out):
     retcode = out.check_call(cmd, env=env)
     if retcode == 0:
         out.green("Deploy completed, call `cmdeploy dns` next.")
-    elif not remote_data.get("acme_account_url"):
+    elif not remote_data["acme_account_url"]:
         out.red("Deploy completed but letsencrypt not configured")
         out.red("Run 'cmdeploy run' again")
         retcode = 0
@@ -100,7 +100,7 @@ def dns_cmd(args, out):
     if not remote_data:
         return 1
 
-    if not remote_data.get("acme_account_url"):
+    if not remote_data["acme_account_url"]:
         out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
         return 1
 

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -1,64 +0,0 @@
-counter delivered_mail
-/saved mail to INBOX$/ {
-  delivered_mail++
-}
-
-counter quota_exceeded
-/Quota exceeded \(mailbox for user is full\)$/ {
-  quota_exceeded++
-}
-
-# Essentially the number of outgoing messages.
-counter dkim_signed
-/DKIM-Signature field added/ {
-  dkim_signed++
-}
-
-counter created_accounts
-counter created_ci_accounts
-counter created_nonci_accounts
-
-/: Created address: (?P<addr>.*)$/ {
-  created_accounts++
-
-  $addr =~ /ci-/ {
-    created_ci_accounts++
-  } else {
-    created_nonci_accounts++
-  }
-}
-
-counter postfix_timeouts
-/timeout after DATA/ {
-  postfix_timeouts++
-}
-
-counter postfix_noqueue
-/postfix\/.*NOQUEUE/ {
-  postfix_noqueue++
-}
-
-counter warning_count
-/warning/ {
-  warning_count++
-}
-
-
-counter filtered_mail_count
-
-counter encrypted_mail_count
-/Filtering encrypted mail\./ {
-  encrypted_mail_count++
-  filtered_mail_count++
-}
-
-counter unencrypted_mail_count
-/Filtering unencrypted mail\./ {
-  unencrypted_mail_count++
-  filtered_mail_count++
-}
-
-counter rejected_unencrypted_mail_count
-/Rejected unencrypted mail\./ {
-  rejected_unencrypted_mail_count++
-}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -1,10 +0,0 @@
-[Unit]
-Description=mtail
-
-[Service]
-Type=simple
-ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target