make another release

DictProxy can have transactions with the same name
(most frequently `1`) processed in parallel.
Dovecot expects that transaction names on each connection
are independent.

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog for chatmail deployment 
 
+## 1.4.1 2024-07-31
+
+- fix metadata dictproxy which would confuse transactions
+  resulting in missed notifications and other issues. 
+  ([#393](https://github.com/deltachat/chatmail/pull/388))
+  ([#394](https://github.com/deltachat/chatmail/pull/389))
+
+- add optional "imap_rawlog" config option. If true, 
+  .in/.out files are created in user home dirs 
+  containing the imap protocol messages. 
+  ([#389](https://github.com/deltachat/chatmail/pull/389))
+
 ## 1.4.0 2024-07-28
 
 - Add `disable_ipv6` config option to chatmail.ini.

chatmaild/src/chatmaild/config.py

@@ -31,6 +31,7 @@ class Config:
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")

chatmaild/src/chatmaild/dictproxy.py

@@ -4,21 +4,24 @@ from socketserver import StreamRequestHandler, ThreadingUnixStreamServer
 
 
 class DictProxy:
-    def __init__(self):
-        self.transactions = {}
-
     def loop_forever(self, rfile, wfile):
+        # Transaction storage is local to each handler loop.
+        # Dovecot reuses transaction IDs across connections,
+        # starting transaction with the name `1`
+        # on two different connections to the same proxy sometimes.
+        transactions = {}
+
         while True:
             msg = rfile.readline().strip().decode()
             if not msg:
                 break
 
-            res = self.handle_dovecot_request(msg)
+            res = self.handle_dovecot_request(msg, transactions)
             if res:
                 wfile.write(res.encode("ascii"))
                 wfile.flush()
 
-    def handle_dovecot_request(self, msg):
+    def handle_dovecot_request(self, msg, transactions):
         # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
@@ -37,11 +40,14 @@ class DictProxy:
         transaction_id = parts[0]
 
         if short_command == "B":
-            return self.handle_begin_transaction(transaction_id, parts)
+            return self.handle_begin_transaction(transaction_id, parts, transactions)
         elif short_command == "C":
-            return self.handle_commit_transaction(transaction_id, parts)
+            return self.handle_commit_transaction(transaction_id, parts, transactions)
         elif short_command == "S":
-            return self.handle_set(transaction_id, parts)
+            addr = transactions[transaction_id]["addr"]
+            if not self.handle_set(addr, parts):
+                transactions[transaction_id]["res"] = "F\n"
+                logging.error(f"dictproxy-set failed for {addr!r}: {msg!r}")
 
     def handle_lookup(self, parts):
         logging.warning(f"lookup ignored: {parts!r}")
@@ -52,19 +58,18 @@ class DictProxy:
         # If we don't return empty line Dovecot will timeout.
         return "\n"
 
-    def handle_begin_transaction(self, transaction_id, parts):
+    def handle_begin_transaction(self, transaction_id, parts, transactions):
         addr = parts[1]
-        self.transactions[transaction_id] = dict(addr=addr, res="O\n")
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+        return False
 
-        self.transactions[transaction_id]["res"] = "F\n"
+    def handle_commit_transaction(self, transaction_id, parts, transactions):
-
-    def handle_commit_transaction(self, transaction_id, parts):
         # return whatever "set" command(s) set as result.
-        return self.transactions.pop(transaction_id)["res"]
+        return transactions.pop(transaction_id)["res"]
 
     def serve_forever_from_socket(self, socket):
         dictproxy = self

chatmaild/src/chatmaild/doveauth.py

@@ -141,7 +141,7 @@ class AuthDictProxy(DictProxy):
             return
 
         user.set_password(encrypt_password(cleartext_password))
-        print(f"Created address: {user}", file=sys.stderr)
+        print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -54,6 +54,17 @@ postfix_reinject_port = 10025
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+#
+# Debugging options 
+#
+
+# set to True if you want to track imap protocol execution
+# in per-maildir ".in/.out" files. 
+# Note that you need to manually cleanup these files
+# so use this option with caution on production servers. 
+imap_rawlog = false 
+
+
 #
 # Privacy Policy
 #

chatmaild/src/chatmaild/lastlogin.py

@@ -9,20 +9,19 @@ class LastLoginDictProxy(DictProxy):
         super().__init__()
         self.config = config
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = self.transactions[transaction_id]["addr"]
         if keyname[0] == "shared" and keyname[1] == "last-login":
             if addr.startswith("echo@"):
-                return
+                return True
             addr = keyname[2]
             timestamp = int(value)
             user = self.config.get_user(addr)
             user.set_last_login_timestamp(timestamp)
-        else:
+            return True
-            # Transaction failed.
+
-            self.transactions[transaction_id]["res"] = "F\n"
+        return False
 
 
 def main():

chatmaild/src/chatmaild/metadata.py

@@ -62,24 +62,19 @@ class MetadataDictProxy(DictProxy):
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
 
-    def handle_set(self, transaction_id, parts):
+    def handle_set(self, addr, parts):
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = self.transactions[transaction_id]["addr"]
         if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
             self.metadata.add_token_to_addr(addr, value)
+            return True
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
             self.notifier.new_message_for_addr(addr, self.metadata)
-        else:
+            return True
-            # Transaction failed.
+
-            try:
+        return False
-                self.transactions[transaction_id]["res"] = "F\n"
-            except KeyError:
-                logging.error(
-                    f"could not mark tx as failed: {transaction_id} {self.transactions}"
-                )
 
 
 def main():

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -72,12 +72,13 @@ def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
 
 
 def test_handle_dovecot_request(dictproxy):
+    transactions = {}
     # Test that password can contain ", ', \ and /
     msg = (
         'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
         "some42123@chat.example.org\tsome42123@chat.example.org"
     )
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -12,28 +12,30 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     authproxy = AuthDictProxy(config=example_config)
     authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
 
+    dictproxy_transactions = {}
+
     # Begin transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert not res
-    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert dictproxy_transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     # set last-login info for user
     user = dictproxy.config.get_user(testaddr)
     timestamp = int(time.time())
     msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert not res
-    assert len(dictproxy.transactions) == 1
+    assert len(dictproxy_transactions) == 1
     read_timestamp = user.get_last_login_timestamp()
     assert read_timestamp == timestamp // 86400 * 86400
 
     # finish transaction
     msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
-    assert len(dictproxy.transactions) == 0
+    assert len(dictproxy_transactions) == 0
 
 
 def test_handle_dovecot_request_last_login_echobot(example_config):
@@ -44,17 +46,19 @@ def test_handle_dovecot_request_last_login_echobot(example_config):
     authproxy.lookup_passdb(testaddr, "ignore")
     user = dictproxy.config.get_user(testaddr)
 
+    transactions = {}
+
     # set last-login info for user
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
-    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     timestamp = int(time.time())
     msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
-    assert len(dictproxy.transactions) == 1
+    assert len(transactions) == 1
     read_timestamp = user.get_last_login_timestamp()
     assert read_timestamp is None

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -88,42 +88,45 @@ def test_notifier_remove_without_set(metadata, testaddr):
 
 
 def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
-    res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}")
+    transactions = {}
+    res = dictproxy.handle_dovecot_request(
+        f"Lpriv/123/chatmail\t{testaddr}", transactions
+    )
     assert res == "N\n"
 
 
 def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     metadata = dictproxy.metadata
-    transactions = dictproxy.transactions
+    transactions = {}
     notifier = dictproxy.notifier
 
     # set device token in a transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res and not metadata.get_tokens_for_addr(testaddr)
     assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
     assert len(transactions) == 1
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res == "O\n"
     assert len(transactions) == 0
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     # trigger notification for incoming message
     tx2 = "2222"
-    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}") is None
+    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions) is None
     msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert dictproxy.handle_dovecot_request(msg) is None
+    assert dictproxy.handle_dovecot_request(msg, transactions) is None
     queue_item = notifier.retry_queues[0].get()[1]
     assert queue_item.token == token
-    assert dictproxy.handle_dovecot_request(f"C{tx2}") == "O\n"
+    assert dictproxy.handle_dovecot_request(f"C{tx2}", transactions) == "O\n"
     assert not transactions
     assert queue_item.path.exists()
 

chatmaild/src/chatmaild/tests/test_user.py

@@ -12,11 +12,11 @@ def test_get_user_dict_not_set(testaddr, example_config, caplog):
     user = example_config.get_user(testaddr)
     assert not caplog.records
     assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 1
+    assert len(caplog.records) == 0
 
     user.set_password("")
     assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 2
+    assert len(caplog.records) == 1
 
 
 def test_get_user_dict(make_config, tmp_path):

chatmaild/src/chatmaild/user.py

@@ -26,7 +26,6 @@ class User:
         try:
             pw = self.password_path.read_text()
         except FileNotFoundError:
-            logging.error(f"password not set for: {self.addr}")
             return {}
 
         if not pw:

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -12,11 +12,10 @@ import shutil
 import subprocess
 import sys
 from pathlib import Path
-from packaging import version
 
 import pyinfra
-
 from chatmaild.config import read_config, write_initial_config
+from packaging import version
 from termcolor import colored
 
 from . import dns, remote_funcs

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -203,3 +203,24 @@ ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.2
 ssl_prefer_server_ciphers = yes
+
+
+{% if config.imap_rawlog %}
+service postlogin {
+  executable = script-login -d rawlog
+  unix_listener postlogin {
+ }  
+} 
+service imap {
+  executable = imap postlogin 
+} 
+  
+protocol imap { 
+  #rawlog_dir = /tmp/rawlog/%u
+  # Put .in and .out imap protocol logging files into per-user homedir 
+  # You can use a command like this to combine into one protocol stream:
+  # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
+
+  rawlog_dir = %h 
+} 
+{% endif %}

www/src/index.md

@@ -11,7 +11,11 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
+<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
+
+If you are viewing this page on a different device
+without a Delta Chat app,
+you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -8,11 +8,9 @@ for the usage in chats, especially DeltaChat.
 
 ### Choosing a chatmail address instead of using a random one
 
-In the Delta Chat account setup 
+In the Delta Chat account setup you may tap `Create a profile` then `Use other server` and choose `Classic e-mail login`. Here fill the two fields like this: 
-you may tap `I already have a profile`
-and fill the two fields like this: 
 
-- `Address`: invent a word with
+- `E-Mail Address`: invent a word with
 {% if username_min_length == username_max_length %}
   *exactly* {{ username_min_length }}
 {% else %}
@@ -26,7 +24,7 @@ and fill the two fields like this:
   characters
   and append `@{{config.mail_domain}}` to it.
 
-- `Password`: invent at least {{ password_min_length }} characters.
+- `Existing Password`: invent at least {{ password_min_length }} characters.
 
 If the e-mail address is not yet taken, you'll get that account. 
 The first login sets your password. 

www/src/main.css

@@ -72,3 +72,15 @@ code {
   color: red;
   font-weight: bold;
 }
+
+.cta-button, .cta-button:hover, .cta-button:visited {
+    border: 1.5px solid #a4c2d0;
+    border-radius: 5px;
+    padding: 10px;
+    display: inline-block;
+    margin: 10px 0;
+
+    background: linear-gradient(120deg, #77888f, #364e59);
+    color: white !important;
+    font-weight: bold;
+}