upgrade Android app version to 2.67.0

nginx listens on 127.0.0.1:8443 behind the stream proxy on port 443.
For a directory requested without a trailing slash, nginx built an
absolute Location header with its own port, e.g.
https://example.org:8443/apps/, which is unreachable from outside.
With absolute_redirect off, nginx sends a relative Location: /apps/.

chatmaild/src/chatmaild/defaults/appversions.json

@@ -5,9 +5,9 @@
       "sources": [
         {
           "sourceId": "gplay",
-          "versionInteger": 757,
+          "versionInteger": 760,
-          "versionString": "2.59.1",
+          "versionString": "2.67.0",
-          "downloadUrl": "https://github.com/deltachat/deltachat-android/releases/download/v2.59.1/deltachat-gplay-release-2.59.1.apk"
+          "downloadUrl": "https://github.com/deltachat/deltachat-android/releases/download/v2.67.0/deltachat-gplay-release-2.67.0.apk"
         }
       ]
     }

cmdeploy/src/cmdeploy/deployers.py

@@ -12,6 +12,7 @@ from chatmaild.config import read_config
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
 from pyinfra.facts import hardware
+from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
@@ -38,6 +39,8 @@ from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
 from .www import build_webpages, find_merge_conflict, get_paths
 
+OLD_JOURNALD_SHA256 = "107c678516c972d5293a67386458fd4785d467cfa2f2b7174185e9a74e034be5"
+
 
 class Port(FactBase):
     """
@@ -354,7 +357,15 @@ class IrohDeployer(Deployer):
 
 class JournaldDeployer(Deployer):
     def configure(self):
-        self.put_file("journald.conf", "/etc/systemd/journald.conf")
+        self.put_file("90-volatile-logging.conf", "/etc/systemd/journald.conf.d/90-volatile-logging.conf")
+        if host.get_fact(Sha256File, "/etc/systemd/journald.conf") == OLD_JOURNALD_SHA256:
+            server.shell(
+                name="Restore Debian's journald.conf",
+                commands=[
+                    "rm /etc/systemd/journald.conf",
+                    "DEBIAN_FRONTEND=noninteractive apt-get install -y --reinstall -o Dpkg::Options::=--force-confmiss systemd",
+                ],
+            )
 
     def activate(self):
         self.ensure_service("systemd-journald.service")

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -133,6 +133,9 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
         config=config,
         debug=debug,
         disable_ipv6=config.disable_ipv6,
+        config_dir="/etc/dovecot",
+        dh_path="/usr/share/dovecot/dh.pem",
+        quota_expire_bin="/usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire",
     )
     deployer.put_template("dovecot/auth.lua.j2", "/etc/dovecot/auth.lua", config=config)
     deployer.remove_file("/etc/dovecot/auth.conf")

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -62,11 +62,11 @@ imap_capability = +XDELTAPUSH XCHATMAIL
 # Authentication for system users.
 passdb {
   driver = lua
-  args = file=/etc/dovecot/auth.lua blocking=yes
+  args = file={{ config_dir }}/auth.lua blocking=yes
 }
 userdb {
   driver = lua
-  args = file=/etc/dovecot/auth.lua blocking=yes
+  args = file={{ config_dir }}/auth.lua blocking=yes
 }
 ##
 ## Mailbox locations and namespaces
@@ -168,7 +168,7 @@ plugin {
 }
 
 service quota-warning {
-  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
+  executable = script {{ quota_expire_bin }}
   user = vmail
   unix_listener quota-warning {
      user = vmail
@@ -179,7 +179,7 @@ service quota-warning {
 # push_notification configuration
 plugin {
   # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
-  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
+  push_notification_driver = lua:file={{ config_dir }}/push_notification.lua
 }
 
 service lmtp {
@@ -254,7 +254,7 @@ service anvil {
 ssl = required
 ssl_cert = <{{ config.tls_cert_path }}
 ssl_key = <{{ config.tls_key_path }}
-ssl_dh = </usr/share/dovecot/dh.pem
+ssl_dh = <{{ dh_path }}
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
 

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -55,6 +55,10 @@ def _configure_nginx(deployer, config: Config, debug: bool = False):
         "/etc/nginx/nginx.conf",
         config=config,
         disable_ipv6=config.disable_ipv6,
+        config_dir="/etc/nginx",
+        stream_module="modules/ngx_stream_module.so",
+        www_root="/var/www/html",
+        cgi_dir="/usr/lib/cgi-bin",
     )
 
     deployer.put_template(

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -1,4 +1,4 @@
-load_module modules/ngx_stream_module.so;
+{% if stream_module %}load_module {{ stream_module }};{% endif %}
 
 user www-data;
 worker_processes auto;
@@ -54,7 +54,7 @@ http {
 	# Do not emit nginx version on error pages.
 	server_tokens off;
 
-	include /etc/nginx/mime.types;
+	include {{ config_dir }}/mime.types;
 	default_type application/octet-stream;
 
 	ssl_protocols TLSv1.2 TLSv1.3;
@@ -68,13 +68,19 @@ http {
   
 		listen 127.0.0.1:8443 ssl default_server;
 
-		root /var/www/html;
+		# nginx is reached through the stream proxy on port 443,
+		# so redirects it generates itself (e.g. /dir -> /dir/)
+		# must not contain the internal port 8443.
+		absolute_redirect off;
+
+		root {{ www_root }};
 
 		index index.html index.htm;
 
 		server_name {{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
 
 		location /mxdeliv {
+		    client_max_body_size {{ config.max_message_size }};
 		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
 		}
 
@@ -96,8 +102,8 @@ http {
 {% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
-			include /etc/nginx/fastcgi_params;
+			include {{ config_dir }}/fastcgi_params;
-			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
+			fastcgi_param SCRIPT_FILENAME {{ cgi_dir }}/newemail.py;
 		}
 
 		# Old URL for compatibility with e.g. printed QR codes.
@@ -114,8 +120,8 @@ http {
 {% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
-			include /etc/nginx/fastcgi_params;
+			include {{ config_dir }}/fastcgi_params;
-			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
+			fastcgi_param SCRIPT_FILENAME {{ cgi_dir }}/newemail.py;
 		}
 
                 # Proxy to iroh-relay service.

cmdeploy/src/cmdeploy/opendkim/KeyTable

@@ -1 +1 @@
-{{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/{{ config.opendkim_selector }}.private
+{{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:{{ keys_dir }}/{{ config.opendkim_selector }}.private

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -30,6 +30,8 @@ class OpendkimDeployer(Deployer):
             "opendkim/opendkim.conf",
             "/etc/opendkim.conf",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
+            keys_dir="/etc/dkimkeys",
+            trust_anchor="/usr/share/dns/root.key",
         )
 
         self.remove_file("/etc/opendkim/screen.lua")
@@ -46,6 +48,7 @@ class OpendkimDeployer(Deployer):
             "/etc/dkimkeys/KeyTable",
             owner="opendkim",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
+            keys_dir="/etc/dkimkeys",
         )
 
         self.put_template(

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -21,9 +21,9 @@ DNSTimeout          60
 # setup options can be found in /usr/share/doc/opendkim/README.opendkim.
 Domain			{{ config.domain_name }}
 Selector		{{ config.opendkim_selector }}
-KeyFile		  /etc/dkimkeys/{{ config.opendkim_selector }}.private
+KeyFile		  {{ keys_dir }}/{{ config.opendkim_selector }}.private
-KeyTable          /etc/dkimkeys/KeyTable
+KeyTable          {{ keys_dir }}/KeyTable
-SigningTable      refile:/etc/dkimkeys/SigningTable
+SigningTable      refile:{{ keys_dir }}/SigningTable
 
 # Sign Autocrypt header in addition to the default specified in RFC 6376.
 #
@@ -58,7 +58,7 @@ PidFile			/run/opendkim/opendkim.pid
 
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
-TrustAnchorFile		/usr/share/dns/root.key
+TrustAnchorFile		{{ trust_anchor }}
 
 # Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
 MTA ORIGINATING

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -24,6 +24,8 @@ class PostfixDeployer(Deployer):
             "/etc/postfix/main.cf",
             config=config,
             disable_ipv6=config.disable_ipv6,
+            config_dir="/etc/postfix",
+            ca_path="/etc/ssl/certs",
         )
 
         self.put_template(
@@ -31,6 +33,7 @@ class PostfixDeployer(Deployer):
             "/etc/postfix/master.cf",
             debug=False,
             config=config,
+            config_dir="/etc/postfix",
         )
 
         self.put_file(

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -19,13 +19,13 @@ smtpd_tls_cert_file={{ config.tls_cert_path }}
 smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
-smtp_tls_CApath=/etc/ssl/certs
+smtp_tls_CApath={{ ca_path }}
 smtp_tls_security_level=verify
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = regexp:/etc/postfix/smtp_tls_policy_map
+smtp_tls_policy_maps = regexp:{{ config_dir }}/smtp_tls_policy_map
 smtp_tls_protocols = >=TLSv1.2
 smtp_tls_mandatory_protocols = >=TLSv1.2
 
@@ -83,7 +83,7 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
+lmtp_header_checks = regexp:{{ config_dir }}/lmtp_header_cleanup
 
 # Do not apply header checks to MIME headers
 # and other headers that are actually part of the message body.
@@ -98,7 +98,7 @@ mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticate
 mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
 
 # 1:1 map MAIL FROM to SASL login name.
-smtpd_sender_login_maps = regexp:/etc/postfix/login_map
+smtpd_sender_login_maps = regexp:{{ config_dir }}/login_map
 
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
@@ -127,6 +127,7 @@ lmtp-filtermail_destination_concurrency_limit=10000
 
 # Do not try to deliver messages for more than 2 days.
 maximal_queue_lifetime = 2d
+bounce_queue_lifetime = 2d
 
 {% if not config.ipv4_relay %}
 # DKIM-sign locally generated mail (bounces, DSNs).

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -102,7 +102,7 @@ postlog   unix-dgram n  -       n       -       1       postlogd
 # to make sure the users
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
-  -o header_checks=regexp:/etc/postfix/submission_header_cleanup
+  -o header_checks=regexp:{{ config_dir }}/submission_header_cleanup
 
 # Reducing `maxproc` here may result in a head of line blocking
 # when there are many messages sent to unreachable destinations

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -91,7 +91,7 @@ def test_concurrent_logins_same_account(
 
 def test_no_vrfy(cmfactory, chatmail_config, maildomain):
     ac = cmfactory.get_online_account()
-    addr = ac.get_config("addr")
+    addr = ac.list_transports()[0]["addr"]
 
     s = smtplib.SMTP(maildomain)
     s.starttls()

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -5,6 +5,7 @@ import subprocess
 import time
 
 import pytest
+import requests
 from chatmaild.config import is_valid_ipv4
 
 from cmdeploy import remote
@@ -132,8 +133,8 @@ def test_use_two_chatmailservers(cmfactory, maildomain2):
     ac1 = cmfactory.get_online_account()
     ac2 = cmfactory.get_online_account(domain=maildomain2)
     cmfactory.get_accepted_chat(ac1, ac2)
-    domain1 = ac1.get_config("addr").split("@")[1]
+    domain1 = ac1.list_transports()[0]["addr"].split("@")[1]
-    domain2 = ac2.get_config("addr").split("@")[1]
+    domain2 = ac2.list_transports()[0]["addr"].split("@")[1]
     assert domain1 != domain2
 
 
@@ -326,3 +327,14 @@ def test_nginx_access_log_only_defined_once(sshdomain):
     assert len(access_logs) == 1, (
         f"expected 1 access_log, found {len(access_logs)}: {access_logs}"
     )
+
+
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_mxdeliv_respects_max_message_size(maildomain, chatmail_config):
+    url = f"https://{maildomain}/mxdeliv"
+    verify = chatmail_config.tls_cert_mode == "acme"
+    size = chatmail_config.max_message_size
+    res = requests.post(url, data=b"x" * size, verify=verify)
+    assert res.text == "500 Invalid DATA"
+    res = requests.post(url, data=b"x" * (size + 1), verify=verify)
+    assert res.status_code == 413

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -15,8 +15,10 @@ from cmdeploy.remote import rshell
 @pytest.fixture
 def imap_mailbox(cmfactory, ssl_context):
     (ac1,) = cmfactory.get_online_accounts(1)
-    user = ac1.get_config("addr")
+    # Delta Chat 2.63 stopped writing the legacy addr and mail_pw config keys.
-    password = ac1.get_config("mail_pw")
+    transport = ac1.list_transports()[0]
+    user = transport["addr"]
+    password = transport["password"]
     host = user.split("@")[1].strip("[").strip("]")
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
@@ -88,7 +90,7 @@ class TestEndToEndDeltaChat:
         ac1, ac2 = cmfactory.get_online_accounts(2)
         chat = cmfactory.get_accepted_chat(ac1, ac2)
 
-        user = ac2.get_config("configured_addr")
+        user = ac2.list_transports()[0]["addr"]
 
         def parse_size_limit(limit: str) -> int:
             """Parse a size limit and return the number of bytes as integer.
@@ -190,9 +192,10 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
 
     chat.send_text("testing submission header cleanup")
     user2.wait_for_incoming_msg()
-    addr = user2.get_config("addr")
+    transport = user2.list_transports()[0]
+    addr = transport["addr"]
     host = addr.split("@")[1].strip("[").strip("]")
-    pw = user2.get_config("mail_pw")
+    pw = transport["password"]
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(addr, pw)
     msgs = list(mailbox.fetch(mark_seen=False))

doc/source/faq.rst

@@ -63,9 +63,17 @@ and run the following commands:
        scripts/initenv.sh
        scripts/cmdeploy run
 
-If you don't want the latest development version,
+To upgrade to the latest tag,
-but a specific tagged release like `1.10.0 <https://github.com/chatmail/relay/releases/tag/1.10.0>`_,
+``cd`` into your local checkout of https://github.com/chatmail/relay/
-run ``git pull origin 1.10.0`` instead.
+and run the following commands:
+
+   ::
+
+       git fetch --tags
+       latestTag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+       git checkout $latestTag
+       scripts/initenv.sh
+       scripts/cmdeploy run
 
 If you made local changes for your setup,
 they will be reapplied as long as they don't conflict with the upgrade.