increase number of login connections

.github/workflows/ci.yaml

@@ -10,10 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        # Checkout pull request HEAD commit instead of merge commit
-        # Otherwise `test_deployed_state` will be unhappy.
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: run chatmaild tests 
         working-directory: chatmaild

CHANGELOG.md

@@ -2,23 +2,8 @@
 
 ## untagged
 
-- Check whether GCC is installed in initenv.sh
+- Increase dovecot imap-login limits
-  ([#608](https://github.com/chatmail/relay/pull/608))
+  ([#577](https://github.com/chatmail/relay/pull/577))
-
-- Expire push notification tokens after 90 days
-  ([#583](https://github.com/chatmail/relay/pull/583))
-
-- Use official `mtail` binary instead of `mtail` package
-  ([#581](https://github.com/chatmail/relay/pull/581))
-
-- dovecot: install from download.delta.chat instead of openSUSE Build Service
-  ([#590](https://github.com/chatmail/relay/pull/590))
-
-- Reconfigure Dovecot imap-login service to high-performance mode
-  ([#578](https://github.com/chatmail/relay/pull/578))
-
-- Set timezone to improve dovecot performance
-  ([#584](https://github.com/chatmail/relay/pull/584))
 
 - Increase nginx connection limits
   ([#576](https://github.com/chatmail/relay/pull/576))

README.md

@@ -69,7 +69,7 @@ Please substitute it with your own domain.
     mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
    ```
 
-2. On your local PC, clone the repository and bootstrap the Python virtualenv.
+2. Clone the repository and bootstrap the Python virtualenv.
 
    ```
     git clone https://github.com/chatmail/relay
@@ -77,29 +77,30 @@ Please substitute it with your own domain.
     scripts/initenv.sh
    ```
 
-3. On your local PC, create chatmail configuration file `chatmail.ini`:
+3. Create chatmail configuration file `chatmail.ini`:
 
    ```
     scripts/cmdeploy init chat.example.org  # <-- use your domain 
    ```
 
-4. Verify that SSH root login to your remote server works:
+4. Verify that SSH root login works:
 
    ```
-    ssh root@chat.example.org  # <-- use your domain 
+    ssh root@chat.example.org   # <-- use your domain 
    ```
 
-5. From your local PC, deploy the remote chatmail relay server:
+
+5. Deploy the remote chatmail relay server:
 
    ```
     scripts/cmdeploy run
    ```
-   This script will also check that you have all necessary DNS records.
+   This script will check that you have all necessary DNS records.
    If DNS records are missing, it will recommend
    which you should configure at your DNS provider
    (it can take some time until they are public).
 
-### Other helpful commands
+### Other helpful commands:
 
 To check the status of your remotely running chatmail service:
 
@@ -541,6 +542,3 @@ Here are some related projects that you may be interested in:
   progress](https://github.com/mjl-/mox/issues/251) to modify it to support all
   of the features and configuration settings required to operate as a chatmail
   relay.
-- [Maddy-Chatmail](https://github.com/sadraiiali/maddy_chatmail): a plugin for the
-  [Maddy email server](https://maddy.email/) which aims to implement the
-  chatmail relay features and configuration options.

chatmaild/pyproject.toml

@@ -48,9 +48,6 @@ lint.select = [
   "PLE", # Pylint Error
   "PLW", # Pylint Warning
 ]
-lint.ignore = [
-  "PLC0415" # import-outside-top-level
-]
 
 [tool.tox]
 legacy_tox_ini = """

chatmaild/src/chatmaild/metadata.py

@@ -1,7 +1,5 @@
 import logging
 import sys
-import time
-from contextlib import contextmanager
 
 from .config import read_config
 from .dictproxy import DictProxy
@@ -9,15 +7,8 @@ from .filedict import FileDict
 from .notifier import Notifier
 
 
-def _is_valid_token_timestamp(timestamp, now):
-    # Token if invalid after 90 days
-    # or if the timestamp is in the future.
-    return timestamp > now - 3600 * 24 * 90 and timestamp < now + 60
-
-
 class Metadata:
-    # each SETMETADATA on this key appends to dictionary
+    # each SETMETADATA on this key appends to a list of unique device tokens
-    # mapping of unique device tokens
     # which only ever get removed if the upstream indicates the token is invalid
     DEVICETOKEN_KEY = "devicetoken"
 
@@ -27,51 +18,21 @@ class Metadata:
     def get_metadata_dict(self, addr):
         return FileDict(self.vmail_dir / addr / "metadata.json")
 
-    @contextmanager
-    def _modify_tokens(self, addr):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.setdefault(self.DEVICETOKEN_KEY, {})
-            now = int(time.time())
-            if isinstance(tokens, list):
-                data[self.DEVICETOKEN_KEY] = tokens = {t: now for t in tokens}
-
-            expired_tokens = [
-                token
-                for token, timestamp in tokens.items()
-                if not _is_valid_token_timestamp(tokens[token], now)
-            ]
-            for expired_token in expired_tokens:
-                del tokens[expired_token]
-
-            yield tokens
-
     def add_token_to_addr(self, addr, token):
-        with self._modify_tokens(addr) as tokens:
+        with self.get_metadata_dict(addr).modify() as data:
-            tokens[token] = int(time.time())
+            tokens = data.setdefault(self.DEVICETOKEN_KEY, [])
+            if token not in tokens:
+                tokens.append(token)
 
     def remove_token_from_addr(self, addr, token):
-        with self._modify_tokens(addr) as tokens:
+        with self.get_metadata_dict(addr).modify() as data:
+            tokens = data.get(self.DEVICETOKEN_KEY, [])
             if token in tokens:
-                del tokens[token]
+                tokens.remove(token)
 
     def get_tokens_for_addr(self, addr):
         mdict = self.get_metadata_dict(addr).read()
-        tokens = mdict.get(self.DEVICETOKEN_KEY, {})
+        return mdict.get(self.DEVICETOKEN_KEY, [])
-
-        now = int(time.time())
-        if isinstance(tokens, dict):
-            token_list = [
-                token
-                for token, timestamp in tokens.items()
-                if _is_valid_token_timestamp(timestamp, now)
-            ]
-            if len(token_list) < len(tokens):
-                # Some tokens have expired, remove them.
-                with self._modify_tokens(addr) as _tokens:
-                    pass
-        else:
-            token_list = []
-        return token_list
 
 
 class MetadataDictProxy(DictProxy):

chatmaild/src/chatmaild/notifier.py

@@ -17,11 +17,11 @@ and which are scheduled for retry using exponential back-off timing.
 If a token notification would be scheduled more than DROP_DEADLINE seconds
 after its first attempt, it is dropped with a log error.
 
-Note that tokens are opaque to the notification machinery here
+Note that tokens are completely opaque to the notification machinery here
-and are encrypted foreclosing all ability to distinguish
+and will in the future be encrypted foreclosing all ability to distinguish
 which device token ultimately goes to which phone-provider notification service,
 or to understand the relation of "device tokens" and chatmail addresses.
-The meaning and format of tokens is basically a matter of chatmail Core and
+The meaning and format of tokens is basically a matter of Delta-Chat Core and
 the `notification.delta.chat` service.
 """
 

cmdeploy/pyproject.toml

@@ -41,6 +41,3 @@ lint.select = [
   "PLE", # Pylint Error
   "PLW", # Pylint Warning
 ]
-lint.ignore = [
-  "PLC0415" # import-outside-top-level
-]

cmdeploy/src/cmdeploy/__init__.py

@@ -318,40 +318,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
-def _install_dovecot_package(package: str, arch: str):
-    arch = "amd64" if arch == "x86_64" else arch
-    arch = "arm64" if arch == "aarch64" else arch
-    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    deb_filename = "/root/" + url.split("/")[-1]
-
-    match (package, arch):
-        case ("core", "amd64"):
-            sha256 = "43f593332e22ac7701c62d58b575d2ca409e0f64857a2803be886c22860f5587"
-        case ("core", "arm64"):
-            sha256 = "4d21eba1a83f51c100f08f2e49f0c9f8f52f721ebc34f75018e043306da993a7"
-        case ("imapd", "amd64"):
-            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
-        case ("imapd", "arm64"):
-            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
-        case ("lmtpd", "amd64"):
-            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
-        case ("lmtpd", "arm64"):
-            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
-        case _:
-            apt.packages(packages=[f"dovecot-{package}"])
-            return
-
-    files.download(
-        name=f"Download dovecot-{package}",
-        src=url,
-        dest=deb_filename,
-        sha256sum=sha256,
-        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
-    )
-
-    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
-
-
 def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     """Configures Dovecot IMAP server."""
     need_restart = False
@@ -410,13 +376,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
             persist=True,
         )
 
-    timezone_env = files.line(
-        name="Set TZ environment variable",
-        path="/etc/environment",
-        line="TZ=:/etc/localtime",
-    )
-    need_restart |= timezone_env.changed
-
     return need_restart
 
 
@@ -498,26 +457,9 @@ def check_config(config):
 
 
 def deploy_mtail(config):
-    # Uninstall mtail package, we are going to install a static binary.
+    apt.packages(
-    apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
+        name="Install mtail",
-
+        packages=["mtail"],
-    (url, sha256sum) = {
-        "x86_64": (
-            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
-            "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
-        ),
-        "aarch64": (
-            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
-            "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
-        ),
-    }[host.get_fact(facts.server.Arch)]
-
-    server.shell(
-        name="Download mtail",
-        commands=[
-            f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
-            "chmod 755 /usr/local/bin/mtail",
-        ],
     )
 
     # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
@@ -653,7 +595,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         path="/etc/apt/sources.list",
         line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
         escape_regex_characters=True,
-        present=False,
+        ensure_newline=True,
     )
 
     if host.get_fact(Port, port=53) != "unbound":
@@ -716,10 +658,10 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         packages="postfix",
     )
 
-    if not "dovecot.service" in host.get_fact(SystemdEnabled):
+    apt.packages(
-        _install_dovecot_package("core", host.get_fact(facts.server.Arch))
+        name="Install Dovecot",
-        _install_dovecot_package("imapd", host.get_fact(facts.server.Arch))
+        packages=["dovecot-imapd", "dovecot-lmtpd"],
-        _install_dovecot_package("lmtpd", host.get_fact(facts.server.Arch))
+    )
 
     apt.packages(
         name="Install nginx",
@@ -816,14 +758,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
-    try:
+    git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-        git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+    git_diff = subprocess.check_output(["git", "diff"]).decode()
-    except Exception:
-        git_hash = "unknown\n"
-    try:
-        git_diff = subprocess.check_output(["git", "diff"]).decode()
-    except Exception:
-        git_diff = ""
     files.put(
         name="Upload chatmail relay git commiit hash",
         src=StringIO(git_hash + git_diff),

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -19,7 +19,7 @@ from packaging import version
 from termcolor import colored
 
 from . import dns, remote
-from .sshexec import SSHExec, Local
+from .sshexec import SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -62,18 +62,13 @@ def run_cmd_options(parser):
         "--ssh-host",
         dest="ssh_host",
         help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default",
-        default=None,
     )
 
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
+    sshexec = args.get_sshexec()
-    if ssh_host == "localhost":
-        sshexec = Local(ssh_host)
-    else:
-        sshexec = args.get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
     if not dns.check_initial_remote_data(remote_data, print=out.red):
@@ -85,7 +80,7 @@ def run_cmd(args, out):
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
-    ssh_host = "@local" if ssh_host == "localhost" else f"--ssh-host {ssh_host}"
+    ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
     if version.parse(pyinfra.__version__) < version.parse("3"):
         out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
@@ -335,9 +330,9 @@ def main(args=None):
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
 
-    def get_sshexec(host):
+    def get_sshexec():
-        print(f"[ssh] login to {host}")
+        print(f"[ssh] login to {args.config.mail_domain}")
-        return SSHExec(host, verbose=args.verbose)
+        return SSHExec(args.config.mail_domain, verbose=args.verbose)
 
     args.get_sshexec = get_sshexec
 

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -177,34 +177,23 @@ service auth-worker {
 }
 
 service imap-login {
-    # High-performance mode as described in
+    # We are not using "high-security-mode" because even if dovecot
-    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-performance-mode>
+    # login would be compromised, there are only E2EE messages to be
-    #
+    # found or sent, and an attacker doesn't have the key used with this address. 
-    # So-called high-security mode described in
+    # See <https://doc.dovecot.org/admin_manual/login_processes/#high-security-mode> for details.
-    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-security-mode>
-    # and enabled by default with `service_count = 1` starts one process per connection
-    # and has problems logging in thousands of users after Dovecot restart.
-    service_count = 0
     
-    # Increase virtual memory size limit.
+    # Performance Mode. Each process serves up to 100 connections and exits afterwards. 
-    # Since imap-login processes handle TLS connections
+    service_count = 100
-    # even after logging users in
+
-    # and many connections are handled by each process,
+    # Inrease the number of simultaneous connections.
-    # memory size limit should be increased.
     #
-    # Otherwise the whole process eventually dies
+    # As of Dovecot 2.3.19.1 the default is 100 processes.
-    # with an error similar to
+    # Combined with `service_count = 1` it means only 100 connections
-    #   imap-login: Fatal: master: service(imap-login):
+    # can be handled simultaneously.
-    #   child 1422951 returned error 83
+    # We allow up to 5000 * 100 = 500K connections 
-    #   (Out of memory (service imap-login { vsz_limit=256 MB },
+    process_limit = 10000
-    #    you may need to increase it)
-    # and takes down all its TLS connections at once.
-    vsz_limit = 1G
 
     # Avoid startup latency for new connections.
-    #
-    # Should be set to at least the number of CPU cores
-    # according to the documentation.
     process_min_avail = 10
 }
 

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -3,7 +3,7 @@ Description=mtail
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs /dev/stdin"
 Restart=on-failure
 
 [Install]

cmdeploy/src/cmdeploy/sshexec.py

@@ -1,6 +1,5 @@
 import inspect
 import os
-import subprocess
 import sys
 from queue import Queue
 
@@ -45,16 +44,30 @@ def print_stderr(item="", end="\n"):
     print(item, file=sys.stderr, end=end)
 
 
-class Exec:
+class SSHExec:
+    RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, verbose, timeout):
+    def __init__(self, host, verbose=False, python="python3", timeout=60):
-        self.host = host
+        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+        self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
 
     def __call__(self, call, kwargs=None, log_callback=None):
-        return subprocess.check_output(call)
+        if kwargs is None:
+            kwargs = {}
+        assert call.__module__.startswith("cmdeploy.remote")
+        modname = call.__module__.replace("cmdeploy.", "")
+        self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
+        while 1:
+            code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
+            if log_callback is not None and code == "log":
+                log_callback(data)
+            elif code == "finish":
+                return data
+            elif code == "error":
+                raise self.FuncError(data)
 
     def logged(self, call, kwargs):
         def log_progress(data):
@@ -72,33 +85,3 @@ class Exec:
             res = self(call, kwargs, log_callback=log_progress)
             print_stderr()
             return res
-
-
-class Local(Exec):
-
-    def __init__(self, host, verbose=False, timeout=60):
-        super().__init__(host, verbose, timeout)
-
-
-class SSHExec(Exec):
-    RemoteError = execnet.RemoteError
-
-    def __init__(self, host, verbose=False, timeout=60):
-        super().__init__(host, verbose, timeout)
-        self.gateway = execnet.makegateway(f"ssh=root@{host}//python=python3")
-        self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
-
-    def __call__(self, call, kwargs=None, log_callback=None):
-        if kwargs is None:
-            kwargs = {}
-        assert call.__module__.startswith("cmdeploy.remote")
-        modname = call.__module__.replace("cmdeploy.", "")
-        self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
-        while 1:
-            code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
-            if log_callback is not None and code == "log":
-                log_callback(data)
-            elif code == "finish":
-                return data
-            elif code == "error":
-                raise self.FuncError(data)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,6 +1,5 @@
 import datetime
 import smtplib
-import socket
 import subprocess
 
 import pytest
@@ -65,14 +64,6 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
-def test_timezone_env(remote):
-    for line in remote.iter_output("env"):
-        print(line)
-        if line == "tz=:/etc/localtime":
-            return True
-    pytest.fail("TZ is not set")
-
-
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
     imap_or_smtp.connect()
@@ -127,19 +118,14 @@ def test_authenticated_from(cmsetup, maildata):
 
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
-    domain = cmsetup.maildomain
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.settimeout(10)
-    try:
-        sock.connect((domain, 25))
-    except socket.timeout:
-        pytest.skip(f"port 25 not reachable for {domain}")
-
     recipient = cmsetup.gen_users(1)[0]
     msg = maildata(
         "encrypted.eml", from_addr=from_addr, to_addr=recipient.addr
     ).as_string()
-    conn = smtplib.SMTP(cmsetup.maildomain, 25, timeout=10)
+    try:
+        conn = smtplib.SMTP(cmsetup.maildomain, 25, timeout=10)
+    except TimeoutError:
+        pytest.skip(f"port 25 not reachable for {cmsetup.maildomain}")
 
     with conn as s:
         with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):

scripts/initenv.sh

@@ -9,11 +9,6 @@ if command -v lsb_release 2>&1 >/dev/null; then
         echo "You need to install python3-dev for installing the other dependencies."
         exit 1
       fi
-      if ! gcc --version 2>&1 >/dev/null
-      then
-        echo "You need to install gcc for building Python dependencies."
-        exit 1
-      fi
       ;;
   esac
 fi