postfix: do lmtp via local instead of virtual transport

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

README.md

@@ -1,20 +1,25 @@
 
-# Chatmail relays for end-to-end encrypted email
+# No-DNS Chatmail relay
 
-Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for: 
+With this branch, you don't need DNS at all,
+just a VPS with an IPv4 address,
+let's take `77.42.80.106` as an example.
+First, choose a random domain name (it doesn't need working DNS)
+and create a chatmail.ini config file:
 
--  **Zero State:** no private data or metadata collected, messages are auto-deleted, low disk usage
+```
+cmdeploy init [77.42.80.106]
+```
 
--  **Instant/Realtime:** sub-second message delivery, realtime P2P
-   streaming, privacy-preserving Push Notifications for Apple, Google, and Huawei;
+Then, in `cmdeploy/src/cmdeploy/postfix/transport`,
+remove the line corresponding to your relay,
+and add other for relays you know.
+Now you can deploy the relay to your IP address:
 
--  **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
+```
+cmdeploy run --skip-dns-check --ssh-host 77.42.80.106
+```
 
--  **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
-   depends on established IETF standards and protocols.
-
-This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host. 
-For getting started and more information please refer to the web version of this repositories' documentation at
-
-[https://chatmail.at/doc/relay](https://chatmail.at/doc/relay)
+Finally, you can login with a `dclogin://` code like this, with the correct "domain name" and IP address:
 
+`dclogin:s0mer4nd0@[77.42.80.106]?p=w7i8da7h8uads92ycc2rufyl&v=1&ih=77.42.80.106&sh=77.42.80.106&sp=443&ip=443&ic=3&sc=3`

chatmaild/src/chatmaild/config.py

@@ -20,8 +20,7 @@ class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
-        self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
+        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
@@ -46,7 +45,6 @@ class Config:
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
-        self.turn_socket_path = params.get("turn_socket_path", "/run/chatmail-turn/turn.socket")
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -11,12 +11,9 @@ mail_domain = {mail_domain}
 # Restrictions on user addresses
 #
 
-# email sending rate per user and minute
+# how many mails a user can send out per minute
 max_user_send_per_minute = 60
 
-# per-user max burst size for sending rate limiting (GCRA bucket capacity)
-max_user_send_burst_size = 10
-
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 500M
 
@@ -55,10 +52,7 @@ passthrough_recipients =
 # Deployment Details
 #
 
-# Path to the TURN server Unix socket
-turn_socket_path = /run/chatmail-turn/turn.socket
-
-# SMTP outgoing filtermail and reinjection
+# SMTP outgoing filtermail and reinjection 
 filtermail_smtp_port = 10080
 postfix_reinject_port = 10025
 

chatmaild/src/chatmaild/metadata.py

@@ -76,13 +76,12 @@ class Metadata:
 
 
 class MetadataDictProxy(DictProxy):
-    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None, config=None):
+    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None):
         super().__init__()
         self.notifier = notifier
         self.metadata = metadata
         self.iroh_relay = iroh_relay
         self.turn_hostname = turn_hostname
-        self.config = config
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
@@ -102,7 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                res = turn_credentials(self.config)
+                res = turn_credentials()
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 
@@ -147,7 +146,6 @@ def main():
         metadata=metadata,
         iroh_relay=iroh_relay,
         turn_hostname=mail_domain,
-        config=config,
     )
 
     dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,120 +0,0 @@
-"""Tests for turnserver functionality, particularly metadata integration."""
-
-import socket
-import tempfile
-import threading
-from pathlib import Path
-
-from chatmaild.config import read_config, write_initial_config
-from chatmaild.metadata import MetadataDictProxy, Metadata
-from chatmaild.notifier import Notifier
-from chatmaild.turnserver import turn_credentials
-
-
-def test_turn_credentials_function_with_custom_socket():
-    """Test that turn_credentials function works with a custom socket path from config."""
-    # Create a temporary directory and socket file
-    temp_dir = Path(tempfile.mkdtemp())
-    temp_socket_path = temp_dir / "test_turn.socket"
-
-    # Create a mock TURN credentials server
-    def mock_server():
-        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        server_sock.bind(str(temp_socket_path))
-        server_sock.listen(1)
-
-        # Accept connection and send mock credentials
-        conn, addr = server_sock.accept()
-        with conn:
-            conn.send(b"mock_turn_credentials_abc123\n")
-        server_sock.close()
-
-    # Start server in a background thread
-    server_thread = threading.Thread(target=mock_server, daemon=True)
-    server_thread.start()
-
-    # Create a config with custom socket path
-    config_path = temp_dir / "chatmail.ini"
-    write_initial_config(config_path, "test.example.org", {
-        "turn_socket_path": str(temp_socket_path)
-    })
-    config = read_config(config_path)
-
-    # Allow time for server to start
-    import time
-    time.sleep(0.01)
-
-    # Test that turn_credentials can connect using the config
-    credentials = turn_credentials(config)
-    assert credentials == "mock_turn_credentials_abc123"
-
-    server_thread.join(timeout=1)  # Clean up thread
-
-
-def test_metadata_turn_lookup_integration(tmp_path):
-    """Test that metadata service properly handles TURN metadata lookups."""
-    # Create mock config with custom turn socket path
-    config_path = tmp_path / "chatmail.ini"
-    socket_path = tmp_path / "test_turn.socket"
-    write_initial_config(config_path, "example.org", {
-        "turn_socket_path": str(socket_path)
-    })
-    config = read_config(config_path)
-
-    # Create mock TURN server to return credentials
-    def mock_turn_server():
-        import os
-        os.makedirs(socket_path.parent, exist_ok=True)  # Ensure parent directory exists
-
-        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        server_sock.bind(str(socket_path))
-        server_sock.listen(1)
-
-        # Accept connection and send mock credentials
-        conn, addr = server_sock.accept()
-        with conn:
-            conn.send(b"test_creds_12345\n")
-        server_sock.close()
-
-    server_thread = threading.Thread(target=mock_turn_server, daemon=True)
-    server_thread.start()
-
-    import time
-    time.sleep(0.01)  # Allow server to start
-
-    # Create a MetadataDictProxy with config
-    queue_dir = tmp_path / "queue"
-    queue_dir.mkdir()
-    notifier = Notifier(queue_dir)
-    metadata = Metadata(tmp_path / "vmail")
-
-    dict_proxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        iroh_relay="https://example.org",
-        turn_hostname="example.org",
-        config=config
-    )
-
-    # Simulate a lookup for TURN credentials using the correct format
-    # Input: "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    # After parts[0].split("/", 2):
-    # - keyparts[0] = "shared"
-    # - keyparts[1] = "0123"
-    # - keyparts[2] = "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    # So keyname = keyparts[2] should match "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    parts = [
-        "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn",
-        "dummy@user.org"
-    ]
-
-    # Call handle_lookup directly
-    result = dict_proxy.handle_lookup(parts)
-
-    # Verify the response format is correct for TURN credentials
-    assert result.startswith("O")  # Output response starts with 'O'
-    assert ":3478:" in result  # Contains port 3478
-    assert "test_creds_12345" in result  # Contains credentials returned by mock server
-    assert "example.org:3478:test_creds_12345" in result
-
-    server_thread.join(timeout=1)  # Clean up thread

chatmaild/src/chatmaild/turnserver.py

@@ -2,8 +2,8 @@
 import socket
 
 
-def turn_credentials(config) -> str:
+def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.connect(config.turn_socket_path)
+        client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -89,6 +89,7 @@ def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
+    ssh_host = ssh_host.strip("[").strip("]")
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     if not args.dns_check_disabled:

cmdeploy/src/cmdeploy/deployers.py

@@ -17,7 +17,6 @@ from pyinfra.operations import apt, files, pip, server, systemd
 
 from cmdeploy.cmdeploy import Out
 
-from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
@@ -141,10 +140,6 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
 
 
 class UnboundDeployer(Deployer):
-    def __init__(self, config):
-        self.config = config
-        self.need_restart = False
-
     def install(self):
         # Run local DNS resolver `unbound`.
         # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -181,27 +176,6 @@ class UnboundDeployer(Deployer):
                 "unbound-anchor -a /var/lib/unbound/root.key || true",
             ],
         )
-        if self.config.disable_ipv6:
-            files.directory(
-                path="/etc/unbound/unbound.conf.d",
-                present=True,
-                user="root",
-                group="root",
-                mode="755",
-            )
-            conf = files.put(
-                src=get_resource("unbound/unbound.conf.j2"),
-                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
-                user="root",
-                group="root",
-                mode="644",
-            )
-        else:
-            conf = files.file(
-                path="/etc/unbound/unbound.conf.d/chatmail.conf",
-                present=False,
-            )
-        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -216,7 +190,6 @@ class UnboundDeployer(Deployer):
             service="unbound.service",
             running=True,
             enabled=True,
-            restarted=self.need_restart,
         )
 
 
@@ -553,14 +526,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
             path="/etc/resolv.conf",
-            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
-            line="\nnameserver 9.9.9.9",
+            line="nameserver 9.9.9.9",
         )
 
     port_services = [
         (["master", "smtpd"], 25),
         ("unbound", 53),
-        ("acmetool", 80),
         (["imap-login", "dovecot"], 143),
         ("nginx", 443),
         (["master", "smtpd"], 465),
@@ -592,10 +563,9 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),
-        UnboundDeployer(config),
+        UnboundDeployer(),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        AcmetoolDeployer(config.acme_email, tls_domains),
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,12 +1,13 @@
 ## Dovecot configuration file
 
 {% if disable_ipv6 %}
-listen = 0.0.0.0
+listen = *
 {% endif %}
 
 protocols = imap lmtp
 
 auth_mechanisms = plain
+auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@[]
 
 {% if debug == true %}
 auth_verbose = yes
@@ -228,8 +229,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
-ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
+ssl_cert = </etc/ssl/certs/ssl-cert-snakeoil.pem
+ssl_key = </etc/ssl/private/ssl-cert-snakeoil.key
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
+            "x86_64": "de7de6e011ffc06881d3a05fc9788e327ba2389219e77280ace38b429e11a5ce",
+            "aarch64": "a78fcdfb81eb3d9c8a8b6f84f6c0a75519b8be01aa25bd4617d72aae543992b4",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -53,8 +53,8 @@ http {
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
-	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
+	ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
+	ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
 
 	gzip on;
 

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -60,7 +60,19 @@ class PostfixDeployer(Deployer):
             mode="644",
         )
         need_restart |= lmtp_header_cleanup.changed
-
+        # Transport map that discards messages to nine.testrun.org
+        transport_map = files.put(
+            src=get_resource("postfix/transport"),
+            dest="/etc/postfix/transport",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= transport_map.changed
+        if transport_map.changed:
+            server.shell(
+                commands=["postmap /etc/postfix/transport"],
+            )
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,12 +15,12 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
+smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level=encrypt
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -54,21 +54,18 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
 tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.mail_domain }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
 # Postfix does not deliver mail for any domain by itself.
 # Primary domain is listed in `virtual_mailbox_domains` instead
 # and handed over to Dovecot.
-mydestination =
+mydestination = {{ config.mail_domain }}
+local_transport = lmtp:unix:private/dovecot-lmtp
+local_recipient_maps =
 
 relayhost = 
-{% if disable_ipv6 %}
-mynetworks = 127.0.0.0/8
-{% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +
@@ -79,14 +76,15 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-virtual_transport = lmtp:unix:private/dovecot-lmtp
-virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
 mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
 mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
 
+# Discard messages to nine.testrun.org
+transport_maps = hash:/etc/postfix/transport
+
 # 1:1 map MAIL FROM to SASL login name.
 smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -86,7 +86,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject_incoming
-  -o smtpd_milters=unix:opendkim/opendkim.sock
+#  -o smtpd_milters=unix:opendkim/opendkim.sock
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.

cmdeploy/src/cmdeploy/postfix/transport

@@ -0,0 +1,2 @@
+nine.testrun.org discard:
+* :

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -190,18 +190,22 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
         "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
     ).as_string()
 
-    start = time.time()
-    for i in range(chatmail_config.max_user_send_per_minute * 3):
-        print("Sending mail", str(i + 1), "at", time.time() - start, "s.")
+    timestamps = []
+    i = 0
+    while len(timestamps) <= chatmail_config.max_user_send_per_minute * 1.7:
+        print("Sending mail", str(i))
+        i += 1
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
+            timestamps.append(time.time())
         except smtplib.SMTPException as e:
-            if i < chatmail_config.max_user_send_burst_size:
+            if len(timestamps) < chatmail_config.max_user_send_per_minute:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
+        timestamps[:] = [ts for ts in timestamps if ts >= (time.time() - 60)]
     pytest.fail("Rate limit was not exceeded")
 
 

cmdeploy/src/cmdeploy/unbound/unbound.conf.j2

@@ -1,4 +0,0 @@
-# Managed by cmdeploy: disable IPv6 in unbound.
-server:
-  interface: 127.0.0.1
-  do-ip6: no

doc/source/overview.rst

@@ -42,11 +42,6 @@ The deployed system components of a chatmail relay are:
 -  Dovecot_ is the Mail Delivery Agent (MDA) and
    stores messages for users until they download them
 
--  `filtermail <https://github.com/chatmail/filtermail>`_
-   prevents unencrypted email from leaving or entering the chatmail
-   service and is integrated into Postfix’s outbound and inbound mail
-   pipelines.
-
 -  Nginx_ shows the web page with privacy policy and additional information
 
 -  `acmetool <https://hlandau.github.io/acmetool/>`_ manages TLS
@@ -90,6 +85,11 @@ short overview of ``chatmaild`` services:
    <https://doc.dovecot.org/2.3/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket>`_
    to authenticate logins.
 
+-  `filtermail <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/filtermail.py>`_
+   prevents unencrypted email from leaving or entering the chatmail
+   service and is integrated into Postfix’s outbound and inbound mail
+   pipelines.
+
 -  `chatmail-metadata <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metadata.py>`_
    is contacted by a `Dovecot lua
    script <https://github.com/chatmail/relay/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua>`_