Add script to remove old seen messages

.github/workflows/test-and-deploy.yaml

@@ -15,14 +15,10 @@ jobs:
   deploy:
     name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     concurrency:
-      group: ci-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+      group: staging-deploy
+      cancel-in-progress: true
     steps:
-      - uses: jsok/serialize-workflow-action@v1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -76,12 +72,12 @@ jobs:
 
       - run: cmdeploy init staging2.testrun.org
 
-      - run: cmdeploy run --verbose
+      - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone --verbose
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
           scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
@@ -92,5 +88,5 @@ jobs:
         run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns -v || cmdeploy dns -v || cmdeploy dns -v
+        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
 

CHANGELOG.md

@@ -2,36 +2,6 @@
 
 ## untagged
 
-- BREAKING: new required chatmail.ini values:
-
-  mailboxes_dir = /home/vmail/mail/{mail_domain}
-  passdb = /home/vmail/passdb.sqlite
-  
-  reducing hardcoding these two paths all over the files, also improving testability. 
-  ([#351](https://github.com/deltachat/chatmail/pull/351))
-
-- BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
-  which removes users from database and mails after 100 days without any login. 
-  ([#350](https://github.com/deltachat/chatmail/pull/350))
-
-- reload nginx in the acmetool cronjob
-  ([#360](https://github.com/deltachat/chatmail/pull/360))
-
-- remove checking of reverse-DNS PTR records.  Chatmail-servers don't
-  depend on it and even in the wider e-mail system it's not common anymore. 
-  If it's an issue, a chatmail operator can still care to properly set reverse DNS. 
-  ([#348](https://github.com/deltachat/chatmail/pull/348))
-
-- Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
-  also introducing a generic mechanism for rapid remote ssh-based python function execution. 
-  ([#346](https://github.com/deltachat/chatmail/pull/346))
-
-- Don't fix file owner ship of /home/vmail 
-  ([#345](https://github.com/deltachat/chatmail/pull/345))
-
-- Support iterating over all users with doveadm commands 
-  ([#344](https://github.com/deltachat/chatmail/pull/344))
-
 - Test and fix for attempts to create inadmissible accounts 
   ([#333](https://github.com/deltachat/chatmail/pull/321))
 
@@ -51,24 +21,6 @@
 - filtermail: do not allow ASCII armor without actual payload
   ([#325](https://github.com/deltachat/chatmail/pull/325))
 
-- Remove sieve to enable hardlink deduplication in LMTP
-  ([#343](https://github.com/deltachat/chatmail/pull/343))
-
-- dovecot: enable gzip compression on disk
-  ([#341](https://github.com/deltachat/chatmail/pull/341))
-
-- DKIM-sign Content-Type and oversign all signed headers
-  ([#296](https://github.com/deltachat/chatmail/pull/296))
-
-- Add nonci_accounts metric
-  ([#347](https://github.com/deltachat/chatmail/pull/347))
-
-- doveauth: log when a new account is created
-  ([#349](https://github.com/deltachat/chatmail/pull/349))
-
-- Multiplex HTTPS, IMAP and SMTP on port 443
-  ([#357](https://github.com/deltachat/chatmail/pull/357))
-
 ## 1.3.0 - 2024-06-06
 
 - don't check necessary DNS records on cmdeploy init anymore

README.md

@@ -155,8 +155,7 @@ While this file is present, account creation will be blocked.
 
 [Postfix](http://www.postfix.org/) listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 [Dovecot](https://www.dovecot.org/) listens on ports 143 (imap) and 993 (imaps).
-[nginx](https://www.nginx.com/) listens on port 8443 (https-alt) and 443 (https).
-Port 443 multiplexes HTTPS, IMAP and SMTP using ALPN to redirect connections to ports 8443, 465 or 993.
+[nginx](https://www.nginx.com/) listens on port 443 (https).
 [acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
 
 Delta Chat apps will, however, discover all ports and configurations

chatmaild/pyproject.toml

@@ -26,7 +26,6 @@ chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
-delete_inactive_users = "chatmaild.delete_inactive_users:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/config.py

@@ -1,13 +1,9 @@
-from pathlib import Path
-
 import iniconfig
 
 
 def read_config(inipath):
-    assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
-    params = cfg.sections["params"]
-    return Config(inipath, params=params)
+    return Config(inipath, params=cfg.sections["params"])
 
 
 class Config:
@@ -17,14 +13,11 @@ class Config:
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
         self.delete_mails_after = params["delete_mails_after"]
-        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
-        self.mailboxes_dir = Path(params["mailboxes_dir"].strip())
-        self.passdb_path = Path(params["passdb_path"].strip())
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.iroh_relay = params.get("iroh_relay")
@@ -36,36 +29,14 @@ class Config:
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
-    def get_user_maildir(self, addr):
-        if addr and addr != "." and "/" not in addr:
-            res = self.mailboxes_dir.joinpath(addr).resolve()
-            if res.is_relative_to(self.mailboxes_dir):
-                return res
-        raise ValueError(f"invalid address {addr!r}")
 
-
-def write_initial_config(inipath, mail_domain, overrides):
-    """Write out default config file, using the specified config value overrides."""
+def write_initial_config(inipath, mail_domain):
     from importlib.resources import files
 
     inidir = files(__package__).joinpath("ini")
-    source_inipath = inidir.joinpath("chatmail.ini.f")
-    content = source_inipath.read_text().format(mail_domain=mail_domain)
-
-    # apply config overrides
-    new_lines = []
-    for line in content.split("\n"):
-        new_line = line.strip()
-        if new_line and new_line[0] not in "#[":
-            name, value = map(str.strip, new_line.split("=", maxsplit=1))
-            value = overrides.get(name, value)
-            new_line = f"{name} = {value}"
-        new_lines.append(new_line)
-
-    content = "\n".join(new_lines)
-
-    # apply testrun privacy overrides
-
+    content = (
+        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
+    )
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
         privacy = iniconfig.IniConfig(override_inipath)["privacy"]

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -1,33 +0,0 @@
-"""
-Remove inactive users
-"""
-
-import shutil
-import sys
-import time
-
-from .config import read_config
-from .database import Database
-from .doveauth import iter_userdb_lastlogin_before
-
-
-def delete_inactive_users(db, config, CHUNK=100):
-    cutoff_date = time.time() - config.delete_inactive_users_after * 86400
-
-    old_users = iter_userdb_lastlogin_before(db, cutoff_date)
-    chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK))
-    for sublist in chunks:
-        for user in sublist:
-            user_mail_dir = config.get_user_maildir(user)
-            shutil.rmtree(user_mail_dir, ignore_errors=True)
-
-        with db.write_transaction() as conn:
-            for user in sublist:
-                conn.execute("DELETE FROM users WHERE addr = ?", (user,))
-
-
-def main():
-    (cfgpath,) = sys.argv[1:]
-    config = read_config(cfgpath)
-    db = Database(config.passdb_path)
-    delete_inactive_users(db, config)

chatmaild/src/chatmaild/doveauth.py

@@ -68,7 +68,7 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
 def get_user_data(db, config: Config, user):
     if user == f"echo@{config.mail_domain}":
         return dict(
-            home=str(config.get_user_maildir(user)),
+            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
             uid="vmail",
             gid="vmail",
         )
@@ -76,7 +76,7 @@ def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
-        result["home"] = str(config.get_user_maildir(user))
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
@@ -86,7 +86,7 @@ def lookup_userdb(db, config: Config, user):
     return get_user_data(db, config, user)
 
 
-def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None):
+def lookup_passdb(db, config: Config, user, cleartext_password):
     if user == f"echo@{config.mail_domain}":
         # Echobot writes password it wants to log in with into /run/echobot/password
         try:
@@ -96,25 +96,21 @@ def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None)
             return None
 
         return dict(
-            home=str(config.get_user_maildir(user)),
+            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
             uid="vmail",
             gid="vmail",
             password=encrypt_password(password),
         )
 
-    if last_login is None:
-        last_login = time.time()
-    last_login = int(last_login)
-
     with db.write_transaction() as conn:
         userdata = conn.get_user(user)
         if userdata:
             # Update last login time.
             conn.execute(
-                "UPDATE users SET last_login=? WHERE addr=?", (last_login, user)
+                "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
-            userdata["home"] = str(config.get_user_maildir(user))
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -124,34 +120,15 @@ def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None)
         encrypted_password = encrypt_password(cleartext_password)
         q = """INSERT INTO users (addr, password, last_login)
                VALUES (?, ?, ?)"""
-        conn.execute(q, (user, encrypted_password, last_login))
-        print(f"Created address: {user}", file=sys.stderr)
+        conn.execute(q, (user, encrypted_password, int(time.time())))
         return dict(
-            home=str(config.get_user_maildir(user)),
+            home=f"/home/vmail/mail/{config.mail_domain}/{user}",
             uid="vmail",
             gid="vmail",
             password=encrypted_password,
         )
 
 
-def iter_userdb(db) -> list:
-    """Get a list of all user addresses."""
-    with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr from users",
-        ).fetchall()
-    return [x[0] for x in rows]
-
-
-def iter_userdb_lastlogin_before(db, cutoff_date):
-    """Get a list of users where last login was before cutoff_date."""
-    with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,)
-        ).fetchall()
-    return [x[0] for x in rows]
-
-
 def split_and_unescape(s):
     """Split strings using double quote as a separator and backslash as escape character
     into parts."""
@@ -215,13 +192,6 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    elif short_command == "I":  # ITERATE
-        # example: I0\t0\tshared/userdb/
-        parts = msg[1:].split("\t")
-        if parts[2] == "shared/userdb/":
-            result = "".join(f"Oshared/userdb/{user}\t\n" for user in iter_userdb(db))
-            return f"{result}\n"
-
     raise UnknownCommand(msg)
 
 
@@ -245,9 +215,9 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, cfgpath = sys.argv[1:]
-    config = read_config(cfgpath)
-    db = Database(config.passdb_path)
+    socket = sys.argv[1]
+    db = Database(sys.argv[2])
+    config = read_config(sys.argv[3])
 
     class Handler(StreamRequestHandler):
         def handle(self):

chatmaild/src/chatmaild/filtermail.py

@@ -111,7 +111,7 @@ def check_encrypted(message):
     """
     if not message.is_multipart():
         return False
-    if message.get("subject") not in {"...", "[...]"}:
+    if message.get("subject") != "...":
         return False
     if message.get_content_type() != "multipart/encrypted":
         return False
@@ -152,7 +152,7 @@ class BeforeQueueHandler:
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        logging.info("handle_MAIL from %s", address)
+        logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
         max_sent = self.config.max_user_send_per_minute
         if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
@@ -176,13 +176,13 @@ class BeforeQueueHandler:
 
     def check_DATA(self, envelope):
         """the central filtering function for e-mails."""
-        logging.info("Processing DATA message from %s", envelope.mail_from)
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
         message = BytesParser(policy=policy.default).parsebytes(envelope.content)
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
-        logging.info("mime-from: %s envelope-from: %r", from_addr, envelope.mail_from)
+        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -8,21 +8,18 @@ mail_domain = {mail_domain}
 #
 
 #
-# Restrictions on user addresses
+# Account Restrictions
 #
 
 # how many mails a user can send out per minute
 max_user_send_per_minute = 60
 
-# maximum mailbox size of a chatmail address
+# maximum mailbox size of a chatmail account
 max_mailbox_size = 100M
 
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
-# days after which users without a login are deleted (database and mails)
-delete_inactive_users_after = 100
-
 # minimum length a username must have
 username_min_length = 9
 
@@ -32,7 +29,7 @@ username_max_length = 9
 # minimum length a password must have
 password_min_length = 9
 
-# list of chatmail addresses which can send outbound un-encrypted mail
+# list of chatmail accounts which can send outbound un-encrypted mail
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
@@ -42,12 +39,6 @@ passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
 # Deployment Details
 #
 
-# Directory where user mailboxes are stored
-mailboxes_dir = /home/vmail/mail/{mail_domain}
-
-# user address sqlite database path
-passdb_path = /home/vmail/passdb.sqlite
-
 # where the filtermail SMTP service listens
 filtermail_smtp_port = 10080
 
@@ -69,3 +60,4 @@ privacy_pdo =
 
 # postal address of the privacy supervisor
 privacy_supervisor =
+

chatmaild/src/chatmaild/metadata.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import sys
+from pathlib import Path
 from socketserver import (
     StreamRequestHandler,
     ThreadingMixIn,
@@ -127,12 +128,12 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, config_path = sys.argv[1:]
+    socket, vmail_dir, config_path = sys.argv[1:]
 
     config = read_config(config_path)
     iroh_relay = config.iroh_relay
 
-    vmail_dir = config.mailboxes_dir
+    vmail_dir = Path(vmail_dir)
     if not vmail_dir.exists():
         logging.error("vmail dir does not exist: %r", vmail_dir)
         return 1

chatmaild/src/chatmaild/metrics.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import sys
+import time
 from pathlib import Path
 
 
@@ -15,15 +16,9 @@ def main(vmail_dir=None):
         if path.name[:3] in ("ci-", "ac_"):
             ci_accounts += 1
 
-    print("# HELP total number of accounts")
-    print("# TYPE accounts gauge")
-    print(f"accounts {accounts}")
-    print("# HELP number of CI accounts")
-    print("# TYPE ci_accounts gauge")
-    print(f"ci_accounts {ci_accounts}")
-    print("# HELP number of non-CI accounts")
-    print("# TYPE nonci_accounts gauge")
-    print(f"nonci_accounts {accounts - ci_accounts}")
+    timestamp = int(time.time() * 1000)
+    print(f"accounts {accounts} {timestamp}")
+    print(f"ci_accounts {ci_accounts} {timestamp}")
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/mail-data/encrypted.eml

@@ -1,6 +1,6 @@
 From: {from_addr}
 To: {to_addr}
-Subject: {subject}
+Subject: ...
 Date: Sun, 15 Oct 2023 16:43:21 +0000
 Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
 In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>

chatmaild/src/chatmaild/tests/plugin.py

@@ -16,11 +16,7 @@ def make_config(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
 
     def make_conf(mail_domain):
-        basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
-        basedir.mkdir(parents=True, exist_ok=True)
-        passdb = tmp_path.joinpath("vmail/passdb.sqlite")
-        overrides = dict(mailboxes_dir=str(basedir), passdb_path=str(passdb))
-        write_initial_config(inipath, mail_domain, overrides=overrides)
+        write_initial_config(inipath, mail_domain=mail_domain)
         return read_config(inipath)
 
     return make_conf
@@ -71,11 +67,11 @@ def maildata(request):
 
     assert datadir.exists(), datadir
 
-    def maildata(name, from_addr, to_addr, subject="..."):
+    def maildata(name, from_addr, to_addr):
         # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
         data = datadir.joinpath(name).read_bytes().decode()
 
-        text = data.format(from_addr=from_addr, to_addr=to_addr, subject=subject)
+        text = data.format(from_addr=from_addr, to_addr=to_addr)
         return BytesParser(policy=policy.default).parsebytes(text.encode())
 
     return maildata

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,4 +1,3 @@
-import pytest
 from chatmaild.config import read_config
 
 
@@ -31,31 +30,3 @@ def test_read_config_testrun(make_config):
     assert config.password_min_length == 9
     assert "privacy@testrun.org" in config.passthrough_recipients
     assert config.passthrough_senders == []
-
-
-def test_config_userstate_paths(make_config, tmp_path):
-    config = make_config("something.testrun.org")
-    mailboxes_dir = config.mailboxes_dir
-    passdb_path = config.passdb_path
-    assert mailboxes_dir.name == "something.testrun.org"
-    assert passdb_path.name == "passdb.sqlite"
-    assert passdb_path.is_relative_to(tmp_path)
-    assert config.mail_domain == "something.testrun.org"
-    path = config.get_user_maildir("user1@something.testrun.org")
-    assert not path.exists()
-    assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
-
-    with pytest.raises(ValueError):
-        config.get_user_maildir("")
-
-    with pytest.raises(ValueError):
-        config.get_user_maildir(None)
-
-    with pytest.raises(ValueError):
-        config.get_user_maildir("../some@something.testrun.org")
-
-    with pytest.raises(ValueError):
-        config.get_user_maildir("..")
-
-    with pytest.raises(ValueError):
-        config.get_user_maildir(".")

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,51 +0,0 @@
-import time
-
-from chatmaild.delete_inactive_users import delete_inactive_users
-from chatmaild.doveauth import lookup_passdb
-
-
-def test_remove_stale_users(db, example_config):
-    new = time.time()
-    old = new - (example_config.delete_inactive_users_after * 86400) - 1
-
-    def create_user(addr, last_login):
-        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login)
-        md = example_config.get_user_maildir(addr)
-        md.mkdir(parents=True)
-        md.joinpath("cur").mkdir()
-        md.joinpath("cur", "something").mkdir()
-
-    # create some stale and some new accounts
-    to_remove = []
-    for i in range(150):
-        addr = f"oldold{i:03}@chat.example.org"
-        create_user(addr, last_login=old)
-        with db.read_connection() as conn:
-            assert conn.get_user(addr)
-        to_remove.append(addr)
-
-    remain = []
-    for i in range(5):
-        addr = f"newnew{i:03}@chat.example.org"
-        create_user(addr, last_login=new)
-        remain.append(addr)
-
-    # check pre and post-conditions for delete_inactive_users()
-
-    for addr in to_remove:
-        assert example_config.get_user_maildir(addr).exists()
-
-    delete_inactive_users(db, example_config)
-
-    for p in example_config.mailboxes_dir.iterdir():
-        assert not p.name.startswith("old")
-
-    for addr in to_remove:
-        assert not example_config.get_user_maildir(addr).exists()
-        with db.read_connection() as conn:
-            assert not conn.get_user(addr)
-
-    for addr in remain:
-        assert example_config.get_user_maildir(addr).exists()
-        with db.read_connection() as conn:
-            assert conn.get_user(addr)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -12,8 +12,6 @@ from chatmaild.doveauth import (
     handle_dovecot_protocol,
     handle_dovecot_request,
     is_allowed_to_create,
-    iter_userdb,
-    iter_userdb_lastlogin_before,
     lookup_passdb,
 )
 from chatmaild.newemail import create_newemail_dict
@@ -29,35 +27,6 @@ def test_basic(db, example_config):
     assert data == data2
 
 
-def test_iterate_addresses(db, example_config):
-    addresses = []
-
-    for i in range(10):
-        addresses.append(f"asdf1234{i}@chat.example.org")
-        lookup_passdb(db, example_config, addresses[-1], "q9mr3faue")
-    res = iter_userdb(db)
-    assert res == addresses
-
-
-def test_iterate_addresses_lastlogin_before(db, example_config):
-    addresses = []
-
-    cutoff_date = 1000
-    for i in range(10):
-        addr = f"oldold{i:03}@chat.example.org"
-        lookup_passdb(
-            db, example_config, addr, "q9mr3faue", last_login=cutoff_date - 10
-        )
-        addresses.append(addr)
-
-    for i in range(5):
-        addr = f"newnew{i:03}@chat.example.org"
-        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=cutoff_date + i)
-
-    res = iter_userdb_lastlogin_before(db, cutoff_date)
-    assert sorted(res) == sorted(addresses)
-
-
 def test_invalid_username_length(example_config):
     config = example_config
     config.username_min_length = 6
@@ -114,7 +83,10 @@ def test_handle_dovecot_request(db, example_config):
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
-    assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org")
+    assert (
+        userdata["home"]
+        == "/home/vmail/mail/chat.example.org/some42123@chat.example.org"
+    )
     assert userdata["uid"] == userdata["gid"] == "vmail"
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
@@ -136,18 +108,6 @@ def test_handle_dovecot_protocol(db, example_config):
     assert wfile.getvalue() == b"N\n"
 
 
-def test_handle_dovecot_protocol_iterate(db, gencreds, example_config):
-    lookup_passdb(db, example_config, "asdf00000@chat.example.org", "q9mr3faue")
-    lookup_passdb(db, example_config, "asdf11111@chat.example.org", "q9mr3faue")
-    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    lines = wfile.getvalue().decode("ascii").split("\n")
-    assert lines[0] == "Oshared/userdb/asdf00000@chat.example.org\t"
-    assert lines[1] == "Oshared/userdb/asdf11111@chat.example.org\t"
-    assert not lines[2]
-
-
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -54,16 +54,10 @@ def test_filtermail_no_encryption_detection(maildata):
 
 
 def test_filtermail_encryption_detection(maildata):
-    for subject in ("...", "[...]"):
-        msg = maildata(
-            "encrypted.eml",
-            from_addr="1@example.org",
-            to_addr="2@example.org",
-            subject=subject,
-        )
-        assert check_encrypted(msg)
+    msg = maildata("encrypted.eml", from_addr="1@example.org", to_addr="2@example.org")
+    assert check_encrypted(msg)
 
-    # if the subject is not a known encrypted subject value, it is not considered ac-encrypted
+    # if the subject is not "..." it is not considered ac-encrypted
     msg.replace_header("Subject", "Click this link")
     assert not check_encrypted(msg)
 
@@ -78,7 +72,7 @@ def test_filtermail_unencrypted_mdn(maildata, gencreds):
     """Unencrypted MDNs should not pass."""
     from_addr = gencreds()[0]
     to_addr = gencreds()[0] + ".other"
-    msg = maildata("mdn.eml", from_addr=from_addr, to_addr=to_addr)
+    msg = maildata("mdn.eml", from_addr, to_addr)
 
     assert not check_encrypted(msg)
 
@@ -101,7 +95,7 @@ def test_excempt_privacy(maildata, gencreds, handler):
     handler.config.passthrough_recipients = [to_addr]
     false_to = "privacy@something.org"
 
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+    msg = maildata("plain.eml", from_addr, to_addr)
 
     class env:
         mail_from = from_addr
@@ -124,7 +118,7 @@ def test_passthrough_senders(gencreds, handler, maildata):
     to_addr = "recipient@something.org"
     handler.config.passthrough_senders = [acc1]
 
-    msg = maildata("plain.eml", from_addr=acc1, to_addr=to_addr)
+    msg = maildata("plain.eml", acc1, to_addr)
 
     class env:
         mail_from = acc1

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -8,10 +8,9 @@ def test_main(tmp_path, capsys):
     out, _ = capsys.readouterr()
     d = {}
     for line in out.split("\n"):
-        if line.strip() and not line.startswith("#"):
-            name, num = line.split()
+        if line.strip():
+            name, num, _ = line.split()
             d[name] = int(num)
 
     assert d["accounts"] == 4
     assert d["ci_accounts"] == 3
-    assert d["nonci_accounts"] == 1

cmdeploy/pyproject.toml

@@ -18,7 +18,6 @@ dependencies = [
   "ruff",
   "pytest",
   "pytest-xdist", 
-  "execnet",
   "imap_tools",
 ]
 

cmdeploy/src/cmdeploy/__init__.py

@@ -92,7 +92,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         group="root",
         mode="644",
         config={
-            "mailboxes_dir": config.mailboxes_dir,
+            "mail_domain": config.mail_domain,
             "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
         },
     )
@@ -338,6 +338,20 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= lua_push_notification_script.changed
 
+    sieve_script = files.put(
+        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
+        dest="/etc/dovecot/default.sieve",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= sieve_script.changed
+    if sieve_script.changed:
+        server.shell(
+            name="compile sieve script",
+            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
+        )
+
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
         dest="/etc/cron.d/expunge",
@@ -347,6 +361,14 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
     )
 
+    files.put(
+            src=importlib.resources.files(__package__).joinpath("dovecot/remove-seen.py"),
+            dest="/usr/local/bin/remove-seen.py",
+            user="root",
+            group="root",
+            mode="755"
+    )
+
     # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
@@ -465,6 +487,11 @@ def deploy_chatmail(config_path: Path) -> None:
     )
     server.user(name="Create echobot user", user="echobot", system=True)
 
+    server.shell(
+        name="Fix file owner in /home/vmail",
+        commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
+    )
+
     # Add our OBS repository for dovecot_no_delay
     files.put(
         name="Add Deltachat OBS GPG key to apt keyring",
@@ -529,12 +556,12 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
     )
 
     apt.packages(
         name="Install nginx",
-        packages=["nginx", "libnginx-mod-stream"],
+        packages=["nginx"],
     )
 
     apt.packages(
@@ -630,3 +657,5 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
+
+

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix && systemctl reload nginx
+20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -0,0 +1,15 @@
+{chatmail_domain}.                   A     {ipv4}
+{chatmail_domain}.                   AAAA  {ipv6}
+{chatmail_domain}.                   MX 10 {chatmail_domain}.
+_submission._tcp.{chatmail_domain}.  SRV 0 1 587 {chatmail_domain}.
+_submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
+_imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
+_imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
+{chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
+_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+_mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
+mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
+www.{chatmail_domain}.               CNAME {chatmail_domain}.
+{dkim_entry}
+_adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,21 +0,0 @@
-{% if ipv4 %}
-{{ chatmail_domain }}.                   A     {{ ipv4 }}
-{% endif %}
-{% if ipv6 %}
-{{ chatmail_domain }}.                   AAAA  {{ ipv6 }}
-{% endif %}
-{{ chatmail_domain }}.                   MX 10 {{ chatmail_domain }}.
-_submission._tcp.{{ chatmail_domain }}.  SRV 0 1 587 {{ chatmail_domain }}.
-_submissions._tcp.{{ chatmail_domain }}. SRV 0 1 465 {{ chatmail_domain }}.
-_imap._tcp.{{ chatmail_domain }}.        SRV 0 1 143 {{ chatmail_domain }}.
-_imaps._tcp.{{ chatmail_domain }}.       SRV 0 1 993 {{ chatmail_domain }}.
-{% if acme_account_url %}
-{{ chatmail_domain }}.                   CAA 128 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
-{% endif %}
-{{ chatmail_domain }}.                   TXT "v=spf1 a:{{ chatmail_domain }} ~all"
-_dmarc.{{ chatmail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_mta-sts.{{ chatmail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-mta-sts.{{ chatmail_domain }}.           CNAME {{ chatmail_domain }}.
-www.{{ chatmail_domain }}.               CNAME {{ chatmail_domain }}.
-{{ dkim_entry }}
-_adsp._domainkey.{{ chatmail_domain }}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,8 +15,7 @@ from pathlib import Path
 from chatmaild.config import read_config, write_initial_config
 from termcolor import colored
 
-from . import dns, remote_funcs
-from .sshexec import SSHExec
+from cmdeploy.dns import check_necessary_dns, show_dns
 
 #
 # cmdeploy sub commands and options
@@ -36,9 +35,8 @@ def init_cmd(args, out):
     mail_domain = args.chatmail_domain
     if args.inipath.exists():
         print(f"Path exists, not modifying: {args.inipath}")
-        return 1
     else:
-        write_initial_config(args.inipath, mail_domain, overrides={})
+        write_initial_config(args.inipath, mail_domain)
         out.green(f"created config file for {mail_domain} in {args.inipath}")
 
 
@@ -53,10 +51,12 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
-
-    remote_data = dns.get_initial_remote_data(args, out)
-    if not dns.check_initial_remote_data(remote_data, print=out.red):
-        return 1
+    mail_domain = args.config.mail_domain
+    if not check_necessary_dns(
+        out,
+        mail_domain,
+    ):
+        sys.exit(1)
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
@@ -64,16 +64,8 @@ def run_cmd(args, out):
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
     cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
 
-    retcode = out.check_call(cmd, env=env)
-    if retcode == 0:
-        out.green("Deploy completed, call `cmdeploy dns` next.")
-    elif not remote_data["acme_account_url"]:
-        out.red("Deploy completed but letsencrypt not configured")
-        out.red("Run 'cmdeploy run' again")
-        retcode = 0
-    else:
-        out.red("Deploy failed")
-    return retcode
+    out.check_call(cmd, env=env)
+    print("Deploy completed, call `cmdeploy dns` next.")
 
 
 def dns_cmd_options(parser):
@@ -85,18 +77,15 @@ def dns_cmd_options(parser):
 
 
 def dns_cmd(args, out):
-    """Check DNS entries and optionally generate dns zone file."""
-    remote_data = dns.get_initial_remote_data(args, out)
-    if not remote_data:
-        return 1
-    retcode = dns.show_dns(args, out, remote_data)
-    return retcode
+    """Generate dns zone file."""
+    exit_code = show_dns(args, out)
+    exit(exit_code)
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    sshexec = args.get_sshexec()
+    ssh = f"ssh root@{args.config.mail_domain}"
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -104,8 +93,10 @@ def status_cmd(args, out):
     else:
         out.red("no privacy settings")
 
-    for line in sshexec(remote_funcs.get_systemd_running):
-        print(line)
+    s1 = "systemctl --type=service --state=running"
+    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
+        if line.startswith("  "):
+            print(line)
 
 
 def test_cmd_options(parser):
@@ -134,7 +125,7 @@ def test_cmd(args, out):
         "-n4",
         "-rs",
         "-x",
-        "-v",
+        "-vrx",
         "--durations=5",
     ]
     if args.slow:
@@ -144,6 +135,14 @@ def test_cmd(args, out):
 
 
 def fmt_cmd_options(parser):
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        action="store_true",
+        help="provide information on invocations",
+    )
+
     parser.add_argument(
         "--check",
         "-c",
@@ -173,6 +172,7 @@ def fmt_cmd(args, out):
 
     out.check_call(" ".join(format_args), quiet=not args.verbose)
     out.check_call(" ".join(check_args), quiet=not args.verbose)
+    return 0
 
 
 def bench_cmd(args, out):
@@ -208,6 +208,16 @@ class Out:
         color = "red" if red else ("green" if green else None)
         print(colored(msg, color), file=file)
 
+    def shell_output(self, arg, no_print=False, timeout=10):
+        if not no_print:
+            self(f"[$ {arg}]", file=sys.stderr)
+            output = subprocess.STDOUT
+        else:
+            output = subprocess.DEVNULL
+        return subprocess.check_output(
+            arg, shell=True, timeout=timeout, stderr=output
+        ).decode()
+
     def check_call(self, arg, env=None, quiet=False):
         if not quiet:
             self(f"[$ {arg}]", file=sys.stderr)
@@ -230,14 +240,6 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        default=False,
-        help="provide verbose logging",
-    )
 
 
 def add_subcommand(subparsers, func):
@@ -277,23 +279,11 @@ def get_parser():
 
 
 def main(args=None):
-    """Provide main entry point for 'cmdeploy' CLI invocation."""
+    """Provide main entry point for 'xdcget' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
-
-    ssh_cache = []
-
-    def get_sshexec():
-        if not ssh_cache:
-            print(f"[ssh] login to {args.config.mail_domain}")
-            ssh = SSHExec(args.config.mail_domain, remote_funcs, verbose=args.verbose)
-            ssh_cache.append(ssh)
-        return ssh_cache[0]
-
-    args.get_sshexec = get_sshexec
-
     out = Out()
     kwargs = {}
     if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
@@ -311,6 +301,7 @@ def main(args=None):
         if res is None:
             res = 0
         return res
+
     except KeyboardInterrupt:
         out.red("KeyboardInterrupt")
         sys.exit(130)

cmdeploy/src/cmdeploy/dns.py

@@ -1,77 +1,209 @@
 import datetime
 import importlib
+import subprocess
+import sys
 
-from jinja2 import Template
-
-from . import remote_funcs
+import requests
 
 
-def get_initial_remote_data(args, out):
-    sshexec = args.get_sshexec()
+class DNS:
+    def __init__(self, out, mail_domain):
+        self.session = requests.Session()
+        self.out = out
+        self.ssh = f"ssh root@{mail_domain} -- "
+        self.out.shell_output(
+            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
+            timeout=60,
+            no_print=True,
+        )
+        try:
+            self.shell(f"unbound-control flush_zone {mail_domain}")
+        except subprocess.CalledProcessError:
+            pass
+
+    def shell(self, cmd):
+        try:
+            return self.out.shell_output(f"{self.ssh}{cmd}", no_print=True)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            if "exit status 255" in str(e) or "timed out" in str(e):
+                self.out.red(f"Error: can't reach the server with: {self.ssh[:-4]}")
+                sys.exit(1)
+            else:
+                raise
+
+    def get_ipv4(self):
+        cmd = "ip a | grep 'inet ' | grep 'scope global' | grep -oE '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' | head -1"
+        return self.shell(cmd).strip()
+
+    def get_ipv6(self):
+        cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
+        return self.shell(cmd).strip()
+
+    def get(self, typ: str, domain: str) -> str:
+        """Get a DNS entry or empty string if there is none."""
+        dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
+        line = dig_result.partition("\n")[0]
+        return line
+
+    def check_ptr_record(self, ip: str, mail_domain) -> bool:
+        """Check the PTR record for an IPv4 or IPv6 address."""
+        result = self.shell(f"dig -r -x {ip} +short").rstrip()
+        return result == f"{mail_domain}."
+
+
+def show_dns(args, out) -> int:
+    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
-    return sshexec.logged(
-        call=remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
+    ssh = f"ssh root@{mail_domain}"
+    dns = DNS(out, mail_domain)
+
+    print("Checking your DKIM keys and DNS entries...")
+    try:
+        acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
+    except subprocess.CalledProcessError:
+        print("Please run `cmdeploy run` first.")
+        return 1
+
+    dkim_selector = "opendkim"
+    dkim_pubkey = out.shell_output(
+        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
+        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
     )
+    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
+    dkim_entry_str = ""
+    while len(dkim_entry_value) >= 255:
+        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
+        dkim_entry_value = dkim_entry_value[255:]
+    dkim_entry_str += '"' + dkim_entry_value + '"'
+    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
 
+    ipv6 = dns.get_ipv6()
+    reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
+    ipv4 = dns.get_ipv4()
+    reverse_ipv4 = dns.check_ptr_record(ipv4, mail_domain)
+    to_print = []
 
-def check_initial_remote_data(remote_data, print=print):
-    mail_domain = remote_data["mail_domain"]
-    if not remote_data["A"] and not remote_data["AAAA"]:
-        print("Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif not remote_data["MTA_STS"]:
-        print("Missing MTA-STS CNAME record:")
-        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}")
+    with open(template, "r") as f:
+        zonefile = (
+            f.read()
+            .format(
+                acme_account_url=acme_account_url,
+                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+                chatmail_domain=args.config.mail_domain,
+                dkim_entry=dkim_entry,
+                ipv6=ipv6,
+                ipv4=ipv4,
+            )
+            .strip()
+        )
+        try:
+            with open(args.zonefile, "w+") as zf:
+                zf.write(zonefile)
+                print(f"DNS records successfully written to: {args.zonefile}")
+            return 0
+        except TypeError:
+            pass
+        for raw_line in zonefile.splitlines():
+            line = raw_line.format(
+                acme_account_url=acme_account_url,
+                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+                chatmail_domain=args.config.mail_domain,
+                dkim_entry=dkim_entry,
+                ipv6=ipv6,
+            ).strip()
+            for typ in ["A", "AAAA", "CNAME", "CAA"]:
+                if f" {typ} " in line:
+                    domain, value = line.split(f" {typ} ")
+                    current = dns.get(typ, domain.strip()[:-1])
+                    if current != value.strip():
+                        to_print.append(line)
+            if " MX " in line:
+                domain, typ, prio, value = line.split()
+                current = dns.get(typ, domain[:-1])
+                if not current:
+                    to_print.append(line)
+                elif current.split()[1] != value:
+                    print(line.replace(prio, str(int(current[0]) + 1)))
+            if " SRV " in line:
+                domain, typ, prio, weight, port, value = line.split()
+                current = dns.get("SRV", domain[:-1])
+                if current != f"{prio} {weight} {port} {value}":
+                    to_print.append(line)
+            if " TXT " in line:
+                domain, value = line.split(" TXT ")
+                current = dns.get("TXT", domain.strip()[:-1])
+                if domain.startswith("_mta-sts."):
+                    if current:
+                        if current.split("id=")[0] == value.split("id=")[0]:
+                            continue
+
+                # TXT records longer than 255 bytes
+                # are split into multiple <character-string>s.
+                # This typically happens with DKIM record
+                # which contains long RSA key.
+                #
+                # Removing `" "` before comparison
+                # to get back a single string.
+                if current.replace('" "', "") != value.replace('" "', ""):
+                    to_print.append(line)
+
+    exit_code = 0
+    if to_print:
+        to_print.insert(
+            0, "You should configure the following DNS entries at your provider:\n"
+        )
+        to_print.append(
+            "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
+        )
+        print("\n".join(to_print))
+        exit_code = 1
     else:
-        return remote_data
+        out.green("Great! All your DNS entries are correct.")
+
+    to_print = []
+    if not reverse_ipv4:
+        to_print.append(f"\tIPv4:\t{ipv4}\t{args.config.mail_domain}")
+    if not reverse_ipv6:
+        to_print.append(f"\tIPv6:\t{ipv6}\t{args.config.mail_domain}")
+    if len(to_print) > 0:
+        if len(to_print) == 1:
+            warning = "You should add the following PTR/reverse DNS entry:"
+        else:
+            warning = "You should add the following PTR/reverse DNS entries:"
+        out.red(warning)
+        for entry in to_print:
+            print(entry)
+        print(
+            "You can do so at your hosting provider (maybe this isn't your DNS provider)."
+        )
+        exit_code = 1
+    return exit_code
 
 
-def show_dns(args, out, remote_data) -> int:
-    """Check existing DNS records, optionally write them to zone file
-    and return (exitcode, remote_data) tuple."""
-
-    sshexec = args.get_sshexec()
-
-    if not remote_data["acme_account_url"]:
-        out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
-        return 1
-
-    if not remote_data["dkim_entry"]:
-        out.red("could not determine dkim_entry, please run 'cmdeploy run'")
-        return 1
-
-    sts_id = remote_data.get("sts_id")
-    if not sts_id:
-        sts_id = datetime.datetime.now().strftime("%Y%m%d%H%M")
-
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(
-        acme_account_url=remote_data.get("acme_account_url"),
-        dkim_entry=remote_data["dkim_entry"],
-        ipv4=remote_data["A"],
-        ipv6=remote_data["AAAA"],
-        sts_id=sts_id,
-        chatmail_domain=args.config.mail_domain,
-    )
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
-    lines.append("")
-    zonefile = "\n".join(lines)
-
-    diff_records = sshexec.logged(
-        remote_funcs.check_zonefile, kwargs=dict(zonefile=zonefile)
-    )
-
-    if getattr(args, "zonefile", None):
-        with open(args.zonefile, "w+") as zf:
-            zf.write(zonefile)
-        out.green(f"DNS records successfully written to: {args.zonefile}")
-        return 0
-
-    if diff_records:
-        out.red("Please set the following DNS entries at your DNS provider:\n")
-        for line in diff_records:
-            out(line)
-        return 1
+def check_necessary_dns(out, mail_domain):
+    """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
+    print("Checking necessary DNS records... ")
+    dns = DNS(out, mail_domain)
+    ipv4 = dns.get("A", mail_domain)
+    ipv6 = dns.get("AAAA", mail_domain)
+    mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
+    www_entry = dns.get("CNAME", "www." + mail_domain)
+    to_print = []
+    if not (ipv4 or ipv6):
+        to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
+    if mta_entry != mail_domain + ".":
+        to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
+    if www_entry != mail_domain + ".":
+        to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
+    if to_print:
+        to_print.insert(
+            0,
+            "\nFor chatmail to work, you need to configure this at your DNS provider:\n",
+        )
+        for line in to_print:
+            print(line)
+        print()
     else:
-        out.green("Great! All your DNS entries are verified and correct.")
-        return 0
+        dns.out.green("All necessary DNS records seem to be set.")
+        return True

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -1,7 +1,5 @@
 uri = proxy:/run/doveauth/doveauth.socket:auth
-iterate_disable = no
-iterate_prefix = userdb/
-
+iterate_disable = yes
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
 # See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -0,0 +1,7 @@
+require ["imap4flags"];
+
+# flag the message so it doesn't cause a push notification
+
+if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
+	addflag "$Auto";
+}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -38,14 +38,7 @@ service imap {
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
-# `zlib` enables compressing messages stored in the maildir.
-# See
-# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
-# for documentation.
-#
-# quota plugin documentation:
-# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
-mail_plugins = zlib quota
+mail_plugins = quota
 
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
@@ -67,7 +60,7 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
-mail_location = maildir:{{ config.mailboxes_dir }}/%u
+mail_location = maildir:/home/vmail/mail/%d/%u
 
 namespace inbox {
   inbox = yes
@@ -103,7 +96,7 @@ mail_privileged_group = vmail
 # Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
 mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 
-# `imap_zlib` enables IMAP COMPRESS (RFC 4978).
+# Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
   mail_plugins = $mail_plugins imap_zlib imap_quota
@@ -111,6 +104,9 @@ protocol imap {
 }
 
 protocol lmtp {
+  # quota plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+  #
   # notify plugin is a dependency of push_notification plugin:
   # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
   #
@@ -119,11 +115,10 @@ protocol lmtp {
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
   # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
-  mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
-}
-
-plugin {
-  zlib_save = gz
+  #
+  # Sieve to mark messages that should not be notified as \Seen
+  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
+  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
 }
 
 plugin {
@@ -145,6 +140,10 @@ plugin {
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
+plugin {
+  sieve_default = file:/etc/dovecot/default.sieve
+}
+
 service lmtp {
   user=vmail
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,12 +1,12 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete
-4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/delete_inactive_users /usr/local/lib/chatmaild/chatmail.ini 
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
+4 0 * * * vmail /usr/local/bin/remove-seen.py /home/vmail/mail/{{ config.mail_domain }}

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -17,8 +17,12 @@ function dovecot_lua_notify_event_message_new(user, event)
 
   if user.username ~= event.from_address then
     -- Incoming message
-    -- Notify METADATA server about new message.
-    mbox:metadata_set("/private/messagenew", "")
+    if not contains(event.keywords, "$Auto") then
+      -- Not an Auto-Submitted message, notifying.
+
+      -- Notify METADATA server about new message.
+      mbox:metadata_set("/private/messagenew", "")
+    end
   end
 
   mbox:free()

cmdeploy/src/cmdeploy/dovecot/remove-seen.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Remove seen messages that are older than two days
+if maildir has more than 80 MB of messages."""
+import sys
+import time
+from pathlib import Path
+
+
+def getdirsize(path):
+    return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file())
+
+
+def parse_dovecot_seen(path):
+    return "S" in path.name.split(":2,")[-1]
+
+
+def main():
+    now = time.time()
+
+    mailhome = Path(sys.argv[1])
+
+    for p in mailhome.iterdir():
+        dirsize = getdirsize(p / "cur") + getdirsize(p / "new")
+        if dirsize < 80000000:
+            continue
+
+        removed_bytes = 0
+        for mailpath in (p / "cur").iterdir():
+            seen = parse_dovecot_seen(mailpath)
+            stat = mailpath.stat()
+            size = stat.st_size
+            if seen and now > stat.st_mtime + 2 * 24 * 3600:
+                removed_bytes += size
+                mailpath.unlink(missing_ok=True)
+
+        if removed_bytes > 0:
+            (p / "maildirsize").unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    main()

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -1 +1 @@
-*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics
+*/5 * * * * root {{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -19,13 +19,6 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
-    <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
-      <port>443</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
@@ -40,12 +33,5 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
-    <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
-      <port>443</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
   </emailProvider>
 </clientConfig>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -1,5 +1,3 @@
-load_module modules/ngx_stream_module.so;
-
 user www-data;
 worker_processes auto;
 pid /run/nginx.pid;
@@ -10,21 +8,6 @@ events {
 	# multi_accept on;
 }
 
-stream {
-        map $ssl_preread_alpn_protocols $proxy {
-            default 127.0.0.1:8443;
-            ~\bsmtp\b 127.0.0.1:submissions;
-            ~\bimap\b 127.0.0.1:imaps;
-        }
-
-        server {
-                listen 443;
-                listen [::]:443;
-                proxy_pass $proxy;
-                ssl_preread on;
-        }
-}
-
 http {
 	sendfile on;
 	tcp_nopush on;
@@ -43,8 +26,8 @@ http {
 	gzip on;
 
 	server {
-		listen 8443 ssl default_server;
-		listen [::]:8443 ssl default_server;
+		listen 443 ssl default_server;
+		listen [::]:443 ssl default_server;
 
 		root /var/www/html;
 
@@ -95,8 +78,8 @@ http {
 
 	# Redirect www. to non-www
 	server {
-		listen 8443 ssl;
-		listen [::]:8443 ssl;
+		listen 443 ssl;
+		listen [::]:443 ssl;
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -25,24 +25,7 @@ KeyTable          /etc/dkimkeys/KeyTable
 SigningTable      refile:/etc/dkimkeys/SigningTable
 
 # Sign Autocrypt header in addition to the default specified in RFC 6376.
-#
-# Default list is here:
-# <https://github.com/trusteddomainproject/OpenDKIM/blob/5c539587561785a66c1f67f720f2fb741f320785/libopendkim/dkim.c#L221-L245>
-SignHeaders *,+autocrypt,+content-type
-
-# Prevent addition of second Content-Type header
-# and other important headers that should not be added
-# after signing the message.
-# See
-# <https://www.zone.eu/blog/2024/05/17/bimi-and-dmarc-cant-save-you/>
-# and RFC 6376 (page 41) for reference.
-#
-# We don't use "l=" body length so the problem described in RFC 6376
-# is not applicable, but adding e.g. a second "From" header
-# or second "Autocrypt" header is better prevented in any case.
-#
-# Default is empty.
-OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
+SignHeaders *,+autocrypt
 
 # Script to ignore signatures that do not correspond to the From: domain.
 ScreenPolicyScript /etc/opendkim/screen.lua

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -77,7 +77,3 @@ mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_
 
 # 1:1 map MAIL FROM to SASL login name.
 smtpd_sender_login_maps = regexp:/etc/postfix/login_map
-
-# Do not lookup SMTP client hostnames to reduce delays
-# and avoid unnecessary DNS requests.
-smtpd_peername_lookup = no

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -15,7 +15,7 @@ smtp      inet  n       -       y       -       -       smtpd -v
 smtp      inet  n       -       y       -       -       smtpd 
 {%- endif %}
   -o smtpd_milters=unix:opendkim/opendkim.sock
-submission inet n       -       y       -       5000    smtpd
+submission inet n       -       y       -       -       smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
   -o smtpd_sasl_auth_enable=yes
@@ -32,7 +32,7 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
   -o cleanup_service_name=authclean
-smtps     inet  n       -       y       -       5000    smtpd
+smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
   -o smtpd_tls_security_level=encrypt

cmdeploy/src/cmdeploy/remote_funcs.py

@@ -1,125 +0,0 @@
-"""
-Pure python functions which execute remotely in a system Python interpreter.
-
-All functions of this module
-
-- need to get and and return Python builtin data types only,
-
-- can only use standard library dependencies,
-
-- can freely call each other.
-"""
-
-import re
-import traceback
-from subprocess import CalledProcessError, check_output
-
-
-def shell(command, fail_ok=False):
-    print(f"$ {command}")
-    try:
-        return check_output(command, shell=True).decode().rstrip()
-    except CalledProcessError:
-        if not fail_ok:
-            raise
-        return ""
-
-
-def get_systemd_running():
-    lines = shell("systemctl --type=service --state=running").split("\n")
-    return [line for line in lines if line.startswith("  ")]
-
-
-def perform_initial_checks(mail_domain):
-    """Collecting initial DNS zone content."""
-    assert mail_domain
-    A = query_dns("A", mail_domain)
-    AAAA = query_dns("AAAA", mail_domain)
-    MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
-
-    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS)
-    if not MTA_STS or (not A and not AAAA):
-        return res
-
-    res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
-    if not shell("dig", fail_ok=True):
-        shell("apt-get install -y dnsutils")
-    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
-    res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
-
-    # parse out sts-id if exists, example: "v=STSv1; id=2090123"
-    parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
-    res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
-    return res
-
-
-def get_dkim_entry(mail_domain, dkim_selector):
-    try:
-        dkim_pubkey = shell(
-            f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
-            "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-        )
-    except CalledProcessError:
-        return
-    dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
-    return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
-
-
-def query_dns(typ, domain):
-    res = shell(f"dig -r -q {domain} -t {typ} +short")
-    print(res)
-    if res:
-        return res.split("\n")[0]
-    return ""
-
-
-def check_zonefile(zonefile):
-    """Check expected zone file entries."""
-    diff = []
-
-    for zf_line in zonefile.splitlines():
-        print(f"dns-checking {zf_line!r}")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        query_value = query_dns(zf_typ, zf_domain)
-        if zf_value != query_value:
-            assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
-            diff.append(zf_line)
-
-    return diff
-
-
-## Function Execution server
-
-
-def _run_loop(cmd_channel):
-    while 1:
-        cmd = cmd_channel.receive()
-        if cmd is None:
-            break
-
-        cmd_channel.send(_handle_one_request(cmd))
-
-
-def _handle_one_request(cmd):
-    func_name, kwargs = cmd
-    try:
-        res = globals()[func_name](**kwargs)
-        return ("finish", res)
-    except:
-        data = traceback.format_exc()
-        return ("error", data)
-
-
-# check if this module is executed remotely
-# and setup a simple serialized function-execution loop
-
-if __name__ == "__channelexec__":
-    channel = channel  # noqa (channel object gets injected)
-
-    # enable simple "print" debugging for anyone changing this module
-    globals()["print"] = lambda x="": channel.send(("log", x))
-
-    _run_loop(channel)

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/doveauth/doveauth.socket {config_path}
+ExecStart={execpath} /run/doveauth/doveauth.socket /home/vmail/passdb.sqlite {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/sshexec.py

@@ -1,48 +0,0 @@
-import sys
-
-import execnet
-
-
-class FuncError(Exception):
-    pass
-
-
-class SSHExec:
-    RemoteError = execnet.RemoteError
-    FuncError = FuncError
-
-    def __init__(self, host, remote_funcs, verbose=False, python="python3", timeout=60):
-        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
-        self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
-        self.timeout = timeout
-        self.verbose = verbose
-
-    def __call__(self, call, kwargs=None, log_callback=None):
-        if kwargs is None:
-            kwargs = {}
-        self._remote_cmdloop_channel.send((call.__name__, kwargs))
-        while 1:
-            code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
-            if log_callback is not None and code == "log":
-                log_callback(data)
-            elif code == "finish":
-                return data
-            elif code == "error":
-                raise self.FuncError(data)
-
-    def logged(self, call, kwargs):
-        def log_progress(data):
-            sys.stdout.write(".")
-            sys.stdout.flush()
-
-        title = call.__doc__
-        if not title:
-            title = call.__name__
-        if self.verbose:
-            print("[ssh] " + title)
-            return self(call, kwargs, log_callback=print)
-        else:
-            print(title, end="")
-            res = self(call, kwargs, log_callback=log_progress)
-            print()
-            return res

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -2,56 +2,6 @@ import smtplib
 
 import pytest
 
-from cmdeploy import remote_funcs
-from cmdeploy.sshexec import SSHExec
-
-
-class TestSSHExecutor:
-    @pytest.fixture(scope="class")
-    def sshexec(self, sshdomain):
-        return SSHExec(sshdomain, remote_funcs)
-
-    def test_ls(self, sshexec):
-        out = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
-        out2 = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
-        assert out == out2
-
-    def test_perform_initial(self, sshexec, maildomain):
-        res = sshexec(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
-        )
-        assert res["A"] or res["AAAA"]
-
-    def test_logged(self, sshexec, maildomain, capsys):
-        sshexec.logged(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
-        )
-        out, err = capsys.readouterr()
-        assert out.startswith("Collecting")
-        assert out.endswith("....\n")
-        assert out.count("\n") == 1
-
-        sshexec.verbose = True
-        sshexec.logged(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
-        )
-        out, err = capsys.readouterr()
-        lines = out.split("\n")
-        assert len(lines) > 4
-        assert remote_funcs.perform_initial_checks.__doc__ in lines[0]
-
-    def test_exception(self, sshexec, capsys):
-        try:
-            sshexec.logged(
-                remote_funcs.perform_initial_checks,
-                kwargs=dict(mail_domain=None),
-            )
-        except sshexec.FuncError as e:
-            assert "remote_funcs.py" in str(e)
-            assert "AssertionError" in str(e)
-        else:
-            pytest.fail("didn't raise exception")
-
 
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
@@ -140,12 +90,12 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
     ]
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -35,7 +35,7 @@ def pytest_runtest_setup(item):
             pytest.skip("skipping slow test, use --slow to run")
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def chatmail_config(pytestconfig):
     current = basedir = Path().resolve()
     while 1:
@@ -49,12 +49,12 @@ def chatmail_config(pytestconfig):
     pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def maildomain(chatmail_config):
     return chatmail_config.mail_domain
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def sshdomain(maildomain):
     return os.environ.get("CHATMAIL_SSH", maildomain)
 

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -21,9 +21,8 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys):
-        assert main(["init", "chat.example.org"]) == 0
-        capsys.readouterr()
-        assert main(["init", "chat.example.org"]) == 1
-        out, err = capsys.readouterr()
-        assert "path exists" in out.lower()
+    @pytest.mark.xfail(reason="init doesn't exit anymore, check for CLI output instead")
+    def test_init_not_overwrite(self):
+        main(["init", "chat.example.org"])
+        with pytest.raises(SystemExit):
+            main(["init", "chat.example.org"])

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -1,50 +0,0 @@
-import pytest
-
-from cmdeploy import remote_funcs
-from cmdeploy.dns import check_initial_remote_data
-
-
-class TestPerformInitialChecks:
-    @pytest.fixture
-    def mockdns(self, monkeypatch):
-        qdict = {
-            "A": {"some.domain": "1.1.1.1"},
-            "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
-            "CNAME": {"mta-sts.some.domain": "some.domain"},
-        }.copy()
-
-        def query_dns(typ, domain):
-            try:
-                return qdict[typ][domain]
-            except KeyError:
-                return ""
-
-        monkeypatch.setattr(remote_funcs, query_dns.__name__, query_dns)
-        return qdict
-
-    def test_perform_initial_checks_ok1(self, mockdns):
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 7
-
-    @pytest.mark.parametrize("drop", ["A", "AAAA"])
-    def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop):
-        del mockdns[drop]
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 7
-        assert not remote_data[drop]
-
-        l = []
-        res = check_initial_remote_data(remote_data, print=l.append)
-        assert res
-        assert not l
-
-    def test_perform_initial_checks_no_mta_sts(self, mockdns):
-        del mockdns["CNAME"]
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 4
-        assert not remote_data["MTA_STS"]
-
-        l = []
-        res = check_initial_remote_data(remote_data, print=l.append)
-        assert not res
-        assert len(l) == 2