Compare commits

..

7 Commits

Author SHA1 Message Date
Christian Hagenest
8349416024 wip 2024-05-27 13:03:14 +02:00
Christian Hagenest
6dd54fc0ed wip expunge timer 2024-05-19 22:17:11 +02:00
Christian Hagenest
2aae3a1c2e wip 2024-05-19 17:17:30 +02:00
Christian Hagenest
0023af4d79 changelog 2024-05-19 11:49:12 +02:00
Christian Hagenest
f40e04f9dc missing comma 2024-05-19 11:44:26 +02:00
Christian Hagenest
84ec0c13cc systemd-metrics ready to test 2024-05-19 11:42:57 +02:00
Christian Hagenest
50f0be2e1d WIP systemd-metrics 2024-05-19 11:42:57 +02:00
32 changed files with 206 additions and 573 deletions

View File

@@ -9,7 +9,7 @@ jobs:
name: isolated chatmaild tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: run chatmaild tests
working-directory: chatmaild
@@ -19,7 +19,7 @@ jobs:
name: deploy-chatmail tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: initenv
run: scripts/initenv.sh

View File

@@ -1,6 +1,6 @@
;; Zone file for staging2.testrun.org
;; Zone file for staging.testrun.org
$ORIGIN staging2.testrun.org.
$ORIGIN staging.testrun.org.
$TTL 300
@ IN SOA ns.testrun.org. root.nine.testrun.org (
@@ -15,7 +15,6 @@ $TTL 300
@ IN NS ns.testrun.org.
;; DNS records.
@ IN A 37.27.24.139
mta-sts.staging2.testrun.org. CNAME staging2.testrun.org.
www.staging2.testrun.org. CNAME staging2.testrun.org.
@ IN A 37.27.37.98
mta-sts.staging.testrun.org. CNAME staging.testrun.org.
www.staging.testrun.org. CNAME staging.testrun.org.

View File

@@ -1,4 +1,4 @@
name: deploy on staging2.testrun.org, and run tests
name: deploy on staging.testrun.org, and run tests
on:
push:
@@ -13,31 +13,28 @@ on:
jobs:
deploy:
name: deploy on staging2.testrun.org, and run tests
name: deploy on staging.testrun.org, and run tests
runs-on: ubuntu-latest
concurrency:
group: staging-deploy
cancel-in-progress: true
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
rsync -avz root@staging.testrun.org:/var/lib/acme . || true
rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
if [ -z "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
- name: rebuild staging2.testrun.org to have a clean VPS
- name: rebuild staging.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
@@ -52,17 +49,17 @@ jobs:
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
echo " --- wait until staging.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true
rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
# restore acme & dkim state to staging.testrun.org
rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
- name: run formatting checks
run: cmdeploy fmt -v
@@ -70,18 +67,18 @@ jobs:
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- run: cmdeploy init staging2.testrun.org
- run: cmdeploy init staging.testrun.org
- run: cmdeploy run
- name: set DNS entries
run: |
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
cmdeploy dns --zonefile staging-generated.zone
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
cat .github/workflows/staging.testrun.org-default.zone
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test

View File

@@ -2,42 +2,14 @@
## untagged
- Test and fix for attempts to create inadmissible accounts
([#333](https://github.com/deltachat/chatmail/pull/321))
- check that OpenPGP has only PKESK, SKESK and SEIPD packets
([#323](https://github.com/deltachat/chatmail/pull/323),
[#324](https://github.com/deltachat/chatmail/pull/324))
- improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
([#320](https://github.com/deltachat/chatmail/pull/320))
- replace `bash` with `/bin/sh`
([#334](https://github.com/deltachat/chatmail/pull/334))
- Increase number of logged in IMAP sessions to 50000
([#335](https://github.com/deltachat/chatmail/pull/335))
- filtermail: do not allow ASCII armor without actual payload
([#325](https://github.com/deltachat/chatmail/pull/325))
## 1.3.0 - 2024-06-06
- don't check necessary DNS records on cmdeploy init anymore
([#316](https://github.com/deltachat/chatmail/pull/316))
- ensure cron and acl are installed
([#293](https://github.com/deltachat/chatmail/pull/293),
[#310](https://github.com/deltachat/chatmail/pull/310))
- run metrics generation with systemd-timer instead of cron
([#304](https://github.com/deltachat/chatmail/pull/304))
- change default for delete_mails_after from 40 to 20 days
([#300](https://github.com/deltachat/chatmail/pull/300))
- save journald logs only to memory and save nginx logs to journald instead of file
([#299](https://github.com/deltachat/chatmail/pull/299))
([#300]https://github.com/deltachat/chatmail/pull/300)
- fix writing of multiple obs repositories in `/etc/apt/sources.list`
([#290](https://github.com/deltachat/chatmail/pull/290))
([#272](https://github.com/deltachat/chatmail/issues/272))
- metadata: add support for `/shared/vendor/deltachat/irohrelay`
([#284](https://github.com/deltachat/chatmail/pull/284))

View File

@@ -60,7 +60,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
config.username_min_length,
config.username_max_length,
)
return False
return True

View File

@@ -1,7 +1,5 @@
#!/usr/bin/env python3
import asyncio
import base64
import binascii
import logging
import sys
import time
@@ -15,100 +13,8 @@ from aiosmtpd.controller import Controller
from .config import read_config
def check_openpgp_payload(payload: bytes):
"""Checks the OpenPGP payload.
OpenPGP payload must consist only of PKESK and SKESK packets
terminated by a single SEIPD packet.
Returns True if OpenPGP payload is correct,
False otherwise.
May raise IndexError while trying to read OpenPGP packet header
if it is truncated.
"""
i = 0
while i < len(payload):
# Only OpenPGP format is allowed.
if payload[i] & 0xC0 != 0xC0:
return False
packet_type_id = payload[i] & 0x3F
i += 1
if payload[i] < 192:
# One-octet length.
body_len = payload[i]
i += 1
elif payload[i] < 224:
# Two-octet length.
body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
i += 2
elif payload[i] == 255:
# Five-octet length.
body_len = (
(payload[i + 1] << 24)
| (payload[i + 2] << 16)
| (payload[i + 3] << 8)
| payload[i + 4]
)
i += 5
else:
# Partial body length is not allowed.
return False
i += body_len
if i == len(payload):
if packet_type_id == 18:
# Last packet should be
# Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
return True
elif packet_type_id not in [1, 3]:
# All packets except the last one must be either
# Public-Key Encrypted Session Key Packet (PKESK)
# or
# Symmetric-Key Encrypted Session Key Packet (SKESK)
return False
if i == 0:
return False
if i > len(payload):
# Payload is truncated.
return False
return True
def check_armored_payload(payload: str):
prefix = "-----BEGIN PGP MESSAGE-----\r\n\r\n"
if not payload.startswith(prefix):
return False
payload = payload.removeprefix(prefix)
suffix = "-----END PGP MESSAGE-----\r\n\r\n"
if not payload.endswith(suffix):
return False
payload = payload.removesuffix(suffix)
# Remove CRC24.
payload = payload.rpartition("=")[0]
try:
payload = base64.b64decode(payload)
except binascii.Error:
return False
try:
return check_openpgp_payload(payload)
except IndexError:
return False
def check_encrypted(message):
"""Check that the message is an OpenPGP-encrypted message.
MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
"""
"""Check that the message is an OpenPGP-encrypted message."""
if not message.is_multipart():
return False
if message.get("subject") != "...":
@@ -117,30 +23,46 @@ def check_encrypted(message):
return False
parts_count = 0
for part in message.iter_parts():
# We explicitly check Content-Type of each part later,
# but this is to be absolutely sure `get_payload()` returns string and not list.
if part.is_multipart():
return False
if parts_count == 0:
if part.get_content_type() != "application/pgp-encrypted":
return False
payload = part.get_payload()
if payload.strip() != "Version: 1":
return False
elif parts_count == 1:
if part.get_content_type() != "application/octet-stream":
return False
if not check_armored_payload(part.get_payload()):
return False
else:
return False
parts_count += 1
return True
def check_mdn(message, envelope):
if len(envelope.rcpt_tos) != 1:
return False
for name in ["auto-submitted", "chat-version"]:
if not message.get(name):
return False
if message.get_content_type() != "multipart/report":
return False
body = message.get_body()
if body.get_content_type() != "text/plain":
return False
if list(body.iter_attachments()) or list(body.iter_parts()):
return False
# even with all mime-structural checks an attacker
# could try to abuse the subject or body to contain links or other
# annoyance -- we skip on checking subject/body for now as Delta Chat
# should evolve to create E2E-encrypted read receipts anyway.
# and then MDNs are just encrypted mail and can pass the border
# to other instances.
return True
async def asyncmain_beforequeue(config):
port = config.filtermail_smtp_port
Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
@@ -186,6 +108,9 @@ class BeforeQueueHandler:
if envelope.mail_from.lower() != from_addr.lower():
return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
if not mail_encrypted and check_mdn(message, envelope):
return
if envelope.mail_from in self.config.passthrough_senders:
return

View File

@@ -1,44 +0,0 @@
From: {from_addr}
To: {to_addr}
Subject: ...
Date: Sun, 15 Oct 2023 16:43:21 +0000
Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
Chat-Version: 1.0
Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
MIME-Version: 1.0
Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"

View File

@@ -68,9 +68,7 @@ def maildata(request):
assert datadir.exists(), datadir
def maildata(name, from_addr, to_addr):
# Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
data = datadir.joinpath(name).read_bytes().decode()
data = datadir.joinpath(name).read_text()
text = data.format(from_addr=from_addr, to_addr=to_addr)
return BytesParser(policy=policy.default).parsebytes(text.encode())

View File

@@ -11,10 +11,8 @@ from chatmaild.doveauth import (
get_user_data,
handle_dovecot_protocol,
handle_dovecot_request,
is_allowed_to_create,
lookup_passdb,
)
from chatmaild.newemail import create_newemail_dict
def test_basic(db, example_config):
@@ -27,20 +25,6 @@ def test_basic(db, example_config):
assert data == data2
def test_invalid_username_length(example_config):
config = example_config
config.username_min_length = 6
config.username_max_length = 10
password = create_newemail_dict(config)["password"]
assert not is_allowed_to_create(config, f"a1234@{config.mail_domain}", password)
assert is_allowed_to_create(config, f"012345@{config.mail_domain}", password)
assert is_allowed_to_create(config, f"0123456@{config.mail_domain}", password)
assert is_allowed_to_create(config, f"0123456789@{config.mail_domain}", password)
assert not is_allowed_to_create(
config, f"0123456789x@{config.mail_domain}", password
)
def test_dont_overwrite_password_on_wrong_login(db, example_config):
"""Test that logging in with a different password doesn't create a new user"""
res = lookup_passdb(

View File

@@ -2,8 +2,8 @@ import pytest
from chatmaild.filtermail import (
BeforeQueueHandler,
SendRateLimiter,
check_armored_payload,
check_encrypted,
check_mdn,
)
@@ -62,19 +62,34 @@ def test_filtermail_encryption_detection(maildata):
assert not check_encrypted(msg)
def test_filtermail_no_literal_packets(maildata):
"""Test that literal OpenPGP packet is not considered an encrypted mail."""
msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
assert not check_encrypted(msg)
def test_filtermail_unencrypted_mdn(maildata, gencreds):
"""Unencrypted MDNs should not pass."""
def test_filtermail_is_mdn(maildata, gencreds, handler):
from_addr = gencreds()[0]
to_addr = gencreds()[0] + ".other"
msg = maildata("mdn.eml", from_addr, to_addr)
assert not check_encrypted(msg)
class env:
mail_from = from_addr
rcpt_tos = [to_addr]
content = msg.as_bytes()
assert check_mdn(msg, env)
print(msg.as_string())
assert not handler.check_DATA(env)
def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
from_addr = gencreds()[0]
to_addr = gencreds()[0] + ".other"
thirdaddr = gencreds()[0]
msg = maildata("mdn.eml", from_addr, to_addr)
class env:
mail_from = from_addr
rcpt_tos = [to_addr, thirdaddr]
content = msg.as_bytes()
assert not check_mdn(msg, env)
def test_send_rate_limiter():
@@ -127,59 +142,3 @@ def test_passthrough_senders(gencreds, handler, maildata):
# assert that None/no error is returned
assert not handler.check_DATA(envelope=env)
def test_check_armored_payload():
payload = """-----BEGIN PGP MESSAGE-----\r
\r
wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
=b5Kp\r
-----END PGP MESSAGE-----\r
\r
"""
assert check_armored_payload(payload) == True
payload = """-----BEGIN PGP MESSAGE-----\r
\r
HELLOWORLD
-----END PGP MESSAGE-----\r
\r
"""
assert check_armored_payload(payload) == False
payload = """-----BEGIN PGP MESSAGE-----\r
\r
=njUN
-----END PGP MESSAGE-----\r
\r
"""
assert check_armored_payload(payload) == False

View File

@@ -17,6 +17,7 @@ from pyinfra.operations import apt, files, pip, server, systemd
from .acmetool import deploy_acmetool
root_owned = dict(user="root", group="root", mode="644")
def _build_chatmaild(dist_dir) -> None:
dist_dir = Path(dist_dir).resolve()
@@ -50,7 +51,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
remote_venv_dir = f"{remote_base_dir}/venv"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
apt.packages(
name="apt install python3-virtualenv",
@@ -85,9 +85,19 @@ def _install_remote_venv_with_chatmaild(config) -> None:
],
)
# create metrics every 5 minutes via systemd
files.put(
name="Upload metrics.timer",
src=importlib.resources.files(__package__).joinpath("service/metrics.timer"),
dest=f"/etc/systemd/system/metrics.timer",
**root_owned,
)
files.template(
src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
dest="/etc/cron.d/chatmail-metrics",
name="upload metrics.service",
src=importlib.resources.files(__package__).joinpath("service/metrics.service.j2"),
dest="/etc/systemd/system/metrics.service",
user="root",
group="root",
mode="644",
@@ -97,6 +107,15 @@ def _install_remote_venv_with_chatmaild(config) -> None:
},
)
systemd.service(
name=f"Setup metrics timer",
service="metrics.timer",
running=True,
enabled=True,
restarted=True,
daemon_reload=True,
)
# install systemd units
for fn in (
"doveauth",
@@ -352,6 +371,23 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
)
files.template(
src=importlib.resources.files(__package__).joinpath("service/expunge.service.j2"),
dest="/etc/systemd/system/expunge.service",
config={
"mail_domain": config.mail_domain,
"delete_mails_after": config.delete_mails_after,
},
**root_owned,
)
files.put(
name="Upload expunge.timer",
src=importlib.resources.files(__package__).joinpath("service/expunge.timer"),
dest=f"/etc/systemd/system/expunge.timer",
**root_owned,
)
files.template(
src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
dest="/etc/cron.d/expunge",
@@ -361,14 +397,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
config=config,
)
files.put(
src=importlib.resources.files(__package__).joinpath("dovecot/remove-seen.py"),
dest="/usr/local/bin/remove-seen.py",
user="root",
group="root",
mode="755"
)
# as per https://doc.dovecot.org/configuration_manual/os/
# it is recommended to set the following inotify limits
for name in ("max_user_instances", "max_user_watches"):
@@ -531,6 +559,7 @@ def deploy_chatmail(config_path: Path) -> None:
"systemctl reset-failed unbound.service",
],
)
systemd.service(
name="Start and enable unbound",
service="unbound.service",
@@ -650,12 +679,5 @@ def deploy_chatmail(config_path: Path) -> None:
service="systemd-journald.service",
running=True,
enabled=True,
restarted=journald_conf.changed,
restarted=journald_conf,
)
apt.packages(
name="Ensure cron is installed",
packages=["cron"],
)

View File

@@ -69,7 +69,8 @@ def deploy_acmetool(email="", domains=[]):
restarted=service_file.changed,
)
server.shell(
name=f"Request certificate for: { ', '.join(domains) }",
commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
)
if str(host) != "staging.testrun.org":
server.shell(
name=f"Request certificate for: { ', '.join(domains) }",
commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
)

View File

@@ -38,6 +38,10 @@ def init_cmd(args, out):
else:
write_initial_config(args.inipath, mail_domain)
out.green(f"created config file for {mail_domain} in {args.inipath}")
check_necessary_dns(
out,
mail_domain,
)
def run_cmd_options(parser):

View File

@@ -183,7 +183,6 @@ def show_dns(args, out) -> int:
def check_necessary_dns(out, mail_domain):
"""Check whether $mail_domain and mta-sts.$mail_domain resolve."""
print("Checking necessary DNS records... ")
dns = DNS(out, mail_domain)
ipv4 = dns.get("A", mail_domain)
ipv6 = dns.get("AAAA", mail_domain)
@@ -205,5 +204,5 @@ def check_necessary_dns(out, mail_domain):
print(line)
print()
else:
dns.out.green("All necessary DNS records seem to be set.")
dns.out.green("\nAll necessary DNS entries seem to be set.")
return True

View File

@@ -19,22 +19,6 @@ mail_debug = yes
# master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
default_client_limit = 20000
# Increase number of logged in IMAP connections.
# Each connection is handled by a separate `imap` process.
# `imap` process should have `client_limit=1` as described in
# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
# so each logged in IMAP session will need its own `imap` process.
#
# If this limit is reached,
# users will fail to LOGIN as `imap-login` process
# will accept them logging in but fail to transfer logged in
# connection to `imap` process until someone logs out and
# the following warning will be logged:
# Warning: service(imap): process_limit (1024) reached, client connections are being dropped
service imap {
process_limit = 50000
}
mail_server_admin = mailto:root@{{ config.mail_domain }}
mail_server_comment = Chatmail server

View File

@@ -9,4 +9,3 @@
2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
4 0 * * * vmail /usr/local/bin/remove-seen.py /home/vmail/mail/{{ config.mail_domain }}

View File

@@ -1,41 +0,0 @@
#!/usr/bin/env python3
"""Remove seen messages that are older than two days
if maildir has more than 80 MB of messages."""
import sys
import time
from pathlib import Path
def getdirsize(path):
return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file())
def parse_dovecot_seen(path):
return "S" in path.name.split(":2,")[-1]
def main():
now = time.time()
mailhome = Path(sys.argv[1])
for p in mailhome.iterdir():
dirsize = getdirsize(p / "cur") + getdirsize(p / "new")
if dirsize < 80000000:
continue
removed_bytes = 0
for mailpath in (p / "cur").iterdir():
seen = parse_dovecot_seen(mailpath)
stat = mailpath.stat()
size = stat.st_size
if seen and now > stat.st_mtime + 2 * 24 * 3600:
removed_bytes += size
mailpath.unlink(missing_ok=True)
if removed_bytes > 0:
(p / "maildirsize").unlink(missing_ok=True)
if __name__ == "__main__":
main()

View File

@@ -1,3 +1,2 @@
[Journal]
MaxRetentionSec=3d
Storage=volatile

View File

@@ -1,7 +1,7 @@
user www-data;
worker_processes auto;
pid /run/nginx.pid;
error_log syslog:server=unix:/dev/log,facility=local3;
error_log /var/log/nginx/error.log;
events {
worker_connections 768;
@@ -35,8 +35,6 @@ http {
server_name _;
access_log syslog:server=unix:/dev/log,facility=local7;
location / {
# First attempt to serve request as file, then
# as directory, then fall back to displaying a 404.
@@ -82,6 +80,5 @@ http {
listen [::]:443 ssl;
server_name www.{{ config.domain_name }};
return 301 $scheme://{{ config.domain_name }}$request_uri;
access_log syslog:server=unix:/dev/log,facility=local7;
}
}

View File

@@ -0,0 +1,16 @@
[Unit]
Description=Expunge old mails after {{ config.delete_mails_after }} days
[Service]
Type=oneshot
# delete all mails after {{ config.delete_mails_after }} days, in the Inbox
ExecStart=/home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
# or in any IMAP subfolder
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
# even if they are unseen
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

View File

@@ -0,0 +1,9 @@
[Unit]
Description=Run expunge.service daily
[Timer]
OnCalendar=weekly
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,5 @@
[Unit]
Description=Generate metrics in /var/www/html/metrics
[Service]
ExecStart={{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} > /var/www/html/metrics

View File

@@ -0,0 +1,9 @@
[Unit]
Description=Run metrics.service every 5 minutes
[Timer]
OnBootSec=5min
OnUnitActiveSec=5min
[Install]
WantedBy=timers.target

View File

@@ -86,7 +86,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [

View File

@@ -1,4 +1,4 @@
#!/bin/sh
#!/usr/bin/env bash
#
# Wrapper for cmdelpoy to run it in activated virtualenv.
set -e

View File

@@ -1,80 +0,0 @@
## Introduction to custom Dovecot builds
Chatmail servers use a custom Debian build of the IMAP 'dovecot' server software because
a) Dovecot developers did not yet merge a [pull request](https://github.com/dovecot/core/pull/216)
which majorly speeds up message delivery by removing a hardcoded 0.5 second delay
on relaying incoming messages.
b) Even if merged, it would take years for it to reach Debian stable.
c) The modified dovecot has been successfully used since December 2023 without issues
and we see no noticeable downside (theoretically higher CPU usage but not measureable)
but a considerable upside as the delay-removal facilitates end-to-end message
delivery of 200 ms in real networks.
The modified forked dovecot code lives at
[https://github.com/chatmail/dovecot](https://github.com/chatmail/dovecot).
The remainder of this document describes the setup of the Debian repository
containing the patched dovecot version.
## Building Debian packages at build.opensuse.org
Delta Chat developers maintain an [account](https://build.opensuse.org/project/show/home:deltachat)
in the [Open Build Service (OBS)](https://openbuildservice.org/),
where the [resulting package](https://build.opensuse.org/package/show/home:deltachat/dovecot)
is now used in deploying chatmail servers.
The Open Build Service (OBS) is a platform for building and distributing software packages
across various operating systems and architectures.
It supports openSUSE, Fedora, Debian, Ubuntu and Arch.
It's [primary instance](https://build.opensuse.org/) is ran by the openSUSE project
and is part of the pipeline of the creation of SUSE Linux Enterprise.
The OBS provides a mercurial-like interface to create source repositories
that are then automatically built.
While in theory a package can be created entirely over the web interface,
the use of the cli-tool `osc` is more convenient and is described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
### How to build the dovecot debian package on the OBS via our script
In scripts/dovecot/ is a shell script that prepares the required files and pushes them to build.opensuse.org.
Before using the script, you should have osc set up as described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
The script assumes you are on Debian. It automatically installs any needed dependencies and creates the source package. To upload the resulting source package to the OBS you need to enter the username and password for deltachat on build.opensuse.org in the last step of the script.
Use `source build-obs.sh` to run it.
### Adding the resulting OBS repository to Debian 12
Our dovecot fork is automatically installed as part of the chatmail deployment. You can see it in cmdeploy/src/cmdeploy/__init__.py. If you want to add our fork manually to a system, you can do the following:
First add our signing key to your apt keyring:
```
sudo cp cmdeploy/src/cmdeploy/obs-home-deltachat.gpg /etc/apt/keyrings/obs-home-deltachat.gpg`
```
Now add our repository and key to /etc/apt/sources.list with a text editor of your choice:
```
deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./
```
You can now install dovecot like normal.
```
sudo apt update
sudo apt install dovecot-core
```
### Security concerns
The signing of the patched dovecot package is done in the OBS and
in theory SUSE could make changes to the package delivered.
It is probably reasonable to trust SUSE to not mess with the build
process because it would cause serious negative reputation damage for them
if they tried and someone finds out.
Our dovecot fork will receive the same security backports as the dovecot package in Debian Sid.

View File

@@ -1,54 +0,0 @@
#!/bin/sh
# Install dependencies
echo "Installing dependencies for this script:"
sudo apt install -y devscripts build-essential osc curl git debhelper-compat
# Define path of your local OBS repository
SCRIPT_DIR=$PWD
OBS_PATH=$SCRIPT_DIR/obs
REPO_PATH=$OBS_PATH/home:deltachat/dovecot/
# Download Debian Source Files
echo "Downloading precise files from Debian unstable repository..."
mkdir dovecot-build
cd dovecot-build
# taken May 6th 2024, from https://packages.debian.org/unstable/dovecot-core
curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1-3.debian.tar.xz -O
curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig.tar.gz -O
curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz -O
# Clone the Chatmail Dovecot Repo
echo "Cloning the Chatmail Dovecot fork..."
git clone https://github.com/chatmail/dovecot.git
# Build the source package
echo "Building the source package"
cd dovecot
dpkg-source -b .
# Setting up OSC
echo "Setting up OBS home repository"
mkdir $OBS_PATH
cd $OBS_PATH
rm -rf home:deltachat/dovecot
osc checkout home:deltachat/dovecot
# Copy Files to Your Local OBS Repository,
echo "Copying files to your local OBS repository..."
cd $SCRIPT_DIR/dovecot-build
cp -rf dovecot_2.3.21+dfsg1-3.debian.tar.xz $REPO_PATH
cp -rf dovecot_2.3.21+dfsg1.orig.tar.gz $REPO_PATH
cp -rf dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz $REPO_PATH
cp -rf dovecot_2.3.21+dfsg1-3.dsc $REPO_PATH
# Push Changes to OBS
echo "Pushing changes to OBS..."
cd $REPO_PATH
osc up
osc add dovecot_2.3.21+dfsg1-3.debian.tar.xz
osc add dovecot_2.3.21+dfsg1.orig.tar.gz
osc add dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz
osc add dovecot_2.3.21+dfsg1-3.dsc
osc commit

Submodule scripts/dovecot/dovecot-build/dovecot added at 4b7f802ca1

View File

@@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
set -e
python3 -m venv --upgrade-deps venv

View File

@@ -1,17 +1,11 @@
<img class="banner" src="collage-top.png"/>
## Dear [Delta Chat](https://get.delta.chat) users and newcomers ...
## Dear [Delta Chat](https://get.delta.chat) users and newcomers,
{% if config.mail_domain != "nine.testrun.org" %}
Welcome to instant, interoperable and [privacy-preserving](privacy.html) messaging :)
{% else %}
Welcome to the default onboarding server ({{ config.mail_domain }})
for Delta Chat users. For details how it avoids storing personal information
please see our [privacy policy](privacy.html).
{% endif %}
👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
<img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

View File

@@ -1,4 +1,6 @@
<img class="banner" src="collage-info.png"/>
## More information
{{ config.mail_domain }} provides a low-maintenance, resource efficient and
@@ -9,7 +11,7 @@ for the usage in chats, especially DeltaChat.
### Choosing a chatmail address instead of using a random one
In the Delta Chat account setup
you may tap `I already have a profile`
you may tap `LOG INTO YOUR E-MAIL ACCOUNT`
and fill the two fields like this:
- `Address`: invent a word with

View File

@@ -1,41 +1,21 @@
<img class="banner" src="collage-privacy.png"/>
# Privacy Policy for {{ config.mail_domain }}
{% if config.mail_domain == "nine.testrun.org" %}
Welcome to `{{config.mail_domain}}`, the default chatmail onboarding server for Delta Chat users.
It is operated on the side by a small sysops team employed by [merlinux](https://merlinux.eu),
an open-source R&D company also acting as the fiscal sponsor of Delta Chat app developments.
See [other chatmail servers](https://delta.chat/en/chatmail) for alternative server operators.
{% endif %}
## Summary: No personal data asked or collected
This chatmail server neither asks for nor retains personal information.
Chatmail servers exist to reliably transmit (store and deliver) end-to-end encrypted messages
between user's devices running the Delta Chat messenger app.
Technically, you may think of a Chatmail server as
an end-to-end encrypted "messaging router" at Internet-scale.
A chatmail server is very unlike classic e-mail servers (for example Google Mail servers)
that ask for personal data and permanently store messages.
A chatmail server behaves more like the Signal messaging server
but does not know about phone numbers and securely and automatically interoperates
with other chatmail and classic e-mail servers.
In particular, this chatmail server
- unconditionally removes messages after {{ config.delete_mails_after }} days,
- prohibits sending out un-encrypted messages,
- only has temporary log files used for debugging purposes.
Legally, authorities might still regard chatmail as a "classic e-mail" server
which collects and retains personal data.
We do not agree on this interpretation. Nevertheless, we provide more legal details below
to make life easier for data protection specialists and lawyers scrutinizing chatmail operations.
We want to show you in a fair and transparent way
what personal data is processed by us.
We follow a strict privacy-by-design approach
and try to avoid processing your data in the first place,
but as you may know,
the internet,
and in particular sending e-mail messages,
does not work without data.
Still,
it's only fair that you know at all times
what personal data is processed
when you use our service.
If you have any remaining questions about data protection, please contact us.
## 1. Name and contact information
@@ -77,7 +57,7 @@ we process the following data and details:
- Users can retrieve or delete all stored messages
without intervention from the operators using standard IMAP client tools.
### 2.1 Account setup
### 3.1 Account setup
Creating an account happens in one of two ways on our mail servers:
@@ -98,7 +78,7 @@ Art. 6 (1) lit. b GDPR,
as you have a usage contract with us
by using our services.
### 2.2 Processing of E-Mail-Messages
## 3.2 Processing of E-Mail-Messages
In addition,
we will process data
@@ -124,7 +104,7 @@ Therefore, limits are enforced:
- message size limits
- any other limit necessary for the whole server to function in a healthy way
- any other limit neccessary for the whole server to function in a healthy way
and to prevent abuse.
The processing and use of the above permissions
@@ -198,9 +178,8 @@ for the purpose of drawing conclusions about your person.
## 4. Transfer of Data
We do not retain any personal data but e-mail messages waiting to be delivered
may contain personal data.
Any such residual personal data will not be transferred to third parties
Your personal data
will not be transferred to third parties
for purposes other than those listed below:
a) you have given your express consent