Prioritize port 443

Port 443 has more chance to work
in networks where ports other than 80 and 443
are dropped.
Otherwise user has to wait for other ports
to time out before trying port 443.

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog for chatmail deployment 
 
+## untagged
+
+- avoid nginx listening on ipv6 if v6 is dsiabled 
+  ([#402](https://github.com/deltachat/chatmail/pull/402))
+
+- trigger "apt upgrade" during "cmdeploy run" 
+  ([#398](https://github.com/deltachat/chatmail/pull/398))
+
+- drop hispanilandia passthrough address
+  ([#401](https://github.com/deltachat/chatmail/pull/401))
+
+
+## 1.4.1 2024-07-31
+
+- fix metadata dictproxy which would confuse transactions
+  resulting in missed notifications and other issues. 
+  ([#393](https://github.com/deltachat/chatmail/pull/393))
+  ([#394](https://github.com/deltachat/chatmail/pull/394))
+
+- add optional "imap_rawlog" config option. If true, 
+  .in/.out files are created in user home dirs 
+  containing the imap protocol messages. 
+  ([#389](https://github.com/deltachat/chatmail/pull/389))
+
 ## 1.4.0 2024-07-28
 
 - Add `disable_ipv6` config option to chatmail.ini.

chatmaild/src/chatmaild/config.py

@@ -31,6 +31,7 @@ class Config:
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")

chatmaild/src/chatmaild/dictproxy.py

@@ -4,21 +4,24 @@ from socketserver import StreamRequestHandler, ThreadingUnixStreamServer
 
 
 class DictProxy:
-    def __init__(self):
-        self.transactions = {}
-
     def loop_forever(self, rfile, wfile):
+        # Transaction storage is local to each handler loop.
+        # Dovecot reuses transaction IDs across connections,
+        # starting transaction with the name `1`
+        # on two different connections to the same proxy sometimes.
+        transactions = {}
+
         while True:
             msg = rfile.readline().strip().decode()
             if not msg:
                 break
 
-            res = self.handle_dovecot_request(msg)
+            res = self.handle_dovecot_request(msg, transactions)
             if res:
                 wfile.write(res.encode("ascii"))
                 wfile.flush()
 
-    def handle_dovecot_request(self, msg):
+    def handle_dovecot_request(self, msg, transactions):
         # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
@@ -37,11 +40,14 @@ class DictProxy:
         transaction_id = parts[0]
 
         if short_command == "B":
-            return self.handle_begin_transaction(transaction_id, parts)
+            return self.handle_begin_transaction(transaction_id, parts, transactions)
         elif short_command == "C":
-            return self.handle_commit_transaction(transaction_id, parts)
+            return self.handle_commit_transaction(transaction_id, parts, transactions)
         elif short_command == "S":
-            return self.handle_set(transaction_id, parts)
+            addr = transactions[transaction_id]["addr"]
+            if not self.handle_set(addr, parts):
+                transactions[transaction_id]["res"] = "F\n"
+                logging.error(f"dictproxy-set failed for {addr!r}: {msg!r}")
 
     def handle_lookup(self, parts):
         logging.warning(f"lookup ignored: {parts!r}")
@@ -52,19 +58,18 @@ class DictProxy:
         # If we don't return empty line Dovecot will timeout.
         return "\n"
 
-    def handle_begin_transaction(self, transaction_id, parts):
+    def handle_begin_transaction(self, transaction_id, parts, transactions):
         addr = parts[1]
-        self.transactions[transaction_id] = dict(addr=addr, res="O\n")
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+        return False
 
-        self.transactions[transaction_id]["res"] = "F\n"
-
-    def handle_commit_transaction(self, transaction_id, parts):
+    def handle_commit_transaction(self, transaction_id, parts, transactions):
         # return whatever "set" command(s) set as result.
-        return self.transactions.pop(transaction_id)["res"]
+        return transactions.pop(transaction_id)["res"]
 
     def serve_forever_from_socket(self, socket):
         dictproxy = self

chatmaild/src/chatmaild/doveauth.py

@@ -141,7 +141,7 @@ class AuthDictProxy(DictProxy):
             return
 
         user.set_password(encrypt_password(cleartext_password))
-        print(f"Created address: {user}", file=sys.stderr)
+        print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -39,7 +39,8 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
+# (space-separated)
+passthrough_recipients = xstore@testrun.org 
 
 #
 # Deployment Details
@@ -54,6 +55,17 @@ postfix_reinject_port = 10025
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+#
+# Debugging options 
+#
+
+# set to True if you want to track imap protocol execution
+# in per-maildir ".in/.out" files. 
+# Note that you need to manually cleanup these files
+# so use this option with caution on production servers. 
+imap_rawlog = false 
+
+
 #
 # Privacy Policy
 #

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net
+passthrough_recipients = privacy@testrun.org xstore@testrun.org
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/lastlogin.py

@@ -9,20 +9,19 @@ class LastLoginDictProxy(DictProxy):
         super().__init__()
         self.config = config
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = self.transactions[transaction_id]["addr"]
         if keyname[0] == "shared" and keyname[1] == "last-login":
             if addr.startswith("echo@"):
-                return
+                return True
             addr = keyname[2]
             timestamp = int(value)
             user = self.config.get_user(addr)
             user.set_last_login_timestamp(timestamp)
-        else:
-            # Transaction failed.
-            self.transactions[transaction_id]["res"] = "F\n"
+            return True
+
+        return False
 
 
 def main():

chatmaild/src/chatmaild/metadata.py

@@ -62,24 +62,19 @@ class MetadataDictProxy(DictProxy):
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = self.transactions[transaction_id]["addr"]
         if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
             self.metadata.add_token_to_addr(addr, value)
+            return True
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
             self.notifier.new_message_for_addr(addr, self.metadata)
-        else:
-            # Transaction failed.
-            try:
-                self.transactions[transaction_id]["res"] = "F\n"
-            except KeyError:
-                logging.error(
-                    f"could not mark tx as failed: {transaction_id} {self.transactions}"
-                )
+            return True
+
+        return False
 
 
 def main():

chatmaild/src/chatmaild/tests/plugin.py

@@ -7,6 +7,7 @@ from email.parser import BytesParser
 from pathlib import Path
 
 import pytest
+
 from chatmaild.config import read_config, write_initial_config
 
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,4 +1,5 @@
 import pytest
+
 from chatmaild.config import read_config
 
 

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -4,8 +4,9 @@ import queue
 import threading
 import traceback
 
-import chatmaild.doveauth
 import pytest
+
+import chatmaild.doveauth
 from chatmaild.doveauth import (
     AuthDictProxy,
     is_allowed_to_create,
@@ -72,12 +73,13 @@ def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
 
 
 def test_handle_dovecot_request(dictproxy):
+    transactions = {}
     # Test that password can contain ", ', \ and /
     msg = (
         'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
         "some42123@chat.example.org\tsome42123@chat.example.org"
     )
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,4 +1,5 @@
 import pytest
+
 from chatmaild.filtermail import (
     BeforeQueueHandler,
     SendRateLimiter,

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -12,28 +12,30 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     authproxy = AuthDictProxy(config=example_config)
     authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
 
+    dictproxy_transactions = {}
+
     # Begin transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert not res
-    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert dictproxy_transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     # set last-login info for user
     user = dictproxy.config.get_user(testaddr)
     timestamp = int(time.time())
     msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert not res
-    assert len(dictproxy.transactions) == 1
+    assert len(dictproxy_transactions) == 1
     read_timestamp = user.get_last_login_timestamp()
     assert read_timestamp == timestamp // 86400 * 86400
 
     # finish transaction
     msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
-    assert len(dictproxy.transactions) == 0
+    assert len(dictproxy_transactions) == 0
 
 
 def test_handle_dovecot_request_last_login_echobot(example_config):
@@ -44,17 +46,19 @@ def test_handle_dovecot_request_last_login_echobot(example_config):
     authproxy.lookup_passdb(testaddr, "ignore")
     user = dictproxy.config.get_user(testaddr)
 
+    transactions = {}
+
     # set last-login info for user
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
-    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     timestamp = int(time.time())
     msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
-    assert len(dictproxy.transactions) == 1
+    assert len(transactions) == 1
     read_timestamp = user.get_last_login_timestamp()
     assert read_timestamp is None

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -3,6 +3,7 @@ import time
 
 import pytest
 import requests
+
 from chatmaild.metadata import (
     Metadata,
     MetadataDictProxy,
@@ -88,42 +89,45 @@ def test_notifier_remove_without_set(metadata, testaddr):
 
 
 def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
-    res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}")
+    transactions = {}
+    res = dictproxy.handle_dovecot_request(
+        f"Lpriv/123/chatmail\t{testaddr}", transactions
+    )
     assert res == "N\n"
 
 
 def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     metadata = dictproxy.metadata
-    transactions = dictproxy.transactions
+    transactions = {}
     notifier = dictproxy.notifier
 
     # set device token in a transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res and not metadata.get_tokens_for_addr(testaddr)
     assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
     assert len(transactions) == 1
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res == "O\n"
     assert len(transactions) == 0
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     # trigger notification for incoming message
     tx2 = "2222"
-    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}") is None
+    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions) is None
     msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert dictproxy.handle_dovecot_request(msg) is None
+    assert dictproxy.handle_dovecot_request(msg, transactions) is None
     queue_item = notifier.retry_queues[0].get()[1]
     assert queue_item.token == token
-    assert dictproxy.handle_dovecot_request(f"C{tx2}") == "O\n"
+    assert dictproxy.handle_dovecot_request(f"C{tx2}", transactions) == "O\n"
     assert not transactions
     assert queue_item.path.exists()
 

chatmaild/src/chatmaild/tests/test_user.py

@@ -12,11 +12,11 @@ def test_get_user_dict_not_set(testaddr, example_config, caplog):
     user = example_config.get_user(testaddr)
     assert not caplog.records
     assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 1
+    assert len(caplog.records) == 0
 
     user.set_password("")
     assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 2
+    assert len(caplog.records) == 1
 
 
 def test_get_user_dict(make_config, tmp_path):

chatmaild/src/chatmaild/user.py

@@ -26,7 +26,6 @@ class User:
         try:
             pw = self.password_path.read_text()
         except FileNotFoundError:
-            logging.error(f"password not set for: {self.addr}")
             return {}
 
         if not pw:

cmdeploy/src/cmdeploy/__init__.py

@@ -489,6 +489,7 @@ def deploy_chatmail(config_path: Path) -> None:
     )
 
     apt.update(name="apt update", cache_time=24 * 3600)
+    apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
     apt.packages(
         name="Install rsync",

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -12,11 +12,10 @@ import shutil
 import subprocess
 import sys
 from pathlib import Path
-from packaging import version
 
 import pyinfra
-
 from chatmaild.config import read_config, write_initial_config
+from packaging import version
 from termcolor import colored
 
 from . import dns, remote_funcs

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -203,3 +203,24 @@ ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.2
 ssl_prefer_server_ciphers = yes
+
+
+{% if config.imap_rawlog %}
+service postlogin {
+  executable = script-login -d rawlog
+  unix_listener postlogin {
+ }  
+} 
+service imap {
+  executable = imap postlogin 
+} 
+  
+protocol imap { 
+  #rawlog_dir = /tmp/rawlog/%u
+  # Put .in and .out imap protocol logging files into per-user homedir 
+  # You can use a command like this to combine into one protocol stream:
+  # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
+
+  rawlog_dir = %h 
+} 
+{% endif %}

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -5,6 +5,13 @@
     <domain>{{ config.domain_name }}</domain>
     <displayName>{{ config.domain_name }} chatmail</displayName>
     <displayShortName>{{ config.domain_name }}</displayShortName>
+    <incomingServer type="imap">
+      <hostname>{{ config.domain_name }}</hostname>
+      <port>443</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
     <incomingServer type="imap">
       <hostname>{{ config.domain_name }}</hostname>
       <port>993</port>
@@ -19,13 +26,13 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
-    <incomingServer type="imap">
+    <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
-    </incomingServer>
+    </outgoingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
@@ -40,12 +47,5 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
-    <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
-      <port>443</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
   </emailProvider>
 </clientConfig>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -19,7 +19,9 @@ stream {
 
         server {
                 listen 443;
+                {% if not disable_ipv6 %}
                 listen [::]:443;
+                {% endif %}
                 proxy_pass $proxy;
                 ssl_preread on;
         }

www/src/index.md

@@ -11,7 +11,11 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
+<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
+
+If you are viewing this page on a different device
+without a Delta Chat app,
+you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -8,11 +8,9 @@ for the usage in chats, especially DeltaChat.
 
 ### Choosing a chatmail address instead of using a random one
 
-In the Delta Chat account setup 
-you may tap `I already have a profile`
-and fill the two fields like this: 
+In the Delta Chat account setup you may tap `Create a profile` then `Use other server` and choose `Classic e-mail login`. Here fill the two fields like this: 
 
-- `Address`: invent a word with
+- `E-Mail Address`: invent a word with
 {% if username_min_length == username_max_length %}
   *exactly* {{ username_min_length }}
 {% else %}
@@ -26,7 +24,7 @@ and fill the two fields like this:
   characters
   and append `@{{config.mail_domain}}` to it.
 
-- `Password`: invent at least {{ password_min_length }} characters.
+- `Existing Password`: invent at least {{ password_min_length }} characters.
 
 If the e-mail address is not yet taken, you'll get that account. 
 The first login sets your password. 

www/src/main.css

@@ -72,3 +72,15 @@ code {
   color: red;
   font-weight: bold;
 }
+
+.cta-button, .cta-button:hover, .cta-button:visited {
+    border: 1.5px solid #a4c2d0;
+    border-radius: 5px;
+    padding: 10px;
+    display: inline-block;
+    margin: 10px 0;
+
+    background: linear-gradient(120deg, #77888f, #364e59);
+    color: white !important;
+    font-weight: bold;
+}