Compare commits

..

2 Commits

Author SHA1 Message Date
missytake
76c53b667f cleanup(dovecot): remove redundant max_cache_file_size config 2026-04-14 00:38:03 +02:00
missytake
eb78fcf2e4 feat: remove dovecot.index.cache files if larger than 500k
fix #644
2026-04-14 00:37:55 +02:00
38 changed files with 327 additions and 1003 deletions

View File

@@ -1,33 +1,19 @@
name: Run unit-tests and container-based deploy+test verification
name: CI
on:
# Triggers when a PR is merged into main or a direct push occurs
push:
branches: [ "main" ]
# Triggers for any PR (and its subsequent commits) targeting the main branch
pull_request:
branches: [ "main" ]
permissions: {}
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
push:
jobs:
tox:
name: isolated chatmaild tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
# Checkout pull request HEAD commit instead of merge commit
# Otherwise `test_deployed_state` will be unhappy.
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- name: download filtermail
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
- name: run chatmaild tests
@@ -38,10 +24,7 @@ jobs:
name: deploy-chatmail tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- uses: actions/checkout@v4
- name: initenv
run: scripts/initenv.sh
@@ -55,23 +38,5 @@ jobs:
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
lxc-test:
name: LXC deploy and test
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
with:
cmlxc_commands: |
cmlxc init
# single cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo cm0
cmlxc -v test-mini cm0
cmlxc -v test-cmdeploy cm0
# cross cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
cmlxc -v test-cmdeploy cm0 cm1
# cross cmdeploy/madmail relay tests
cmlxc -v deploy-madmail mad0
cmlxc -v test-cmdeploy cm0 mad0
cmlxc -v test-mini cm0 mad0
cmlxc -v test-mini mad0 cm0
# all other cmdeploy commands require a staging server
# see https://github.com/deltachat/chatmail/issues/100

View File

@@ -1,37 +0,0 @@
# Notify the docker repo to build and test a new image after relay CI passes.
#
# Sends a repository_dispatch event to chatmail/docker with the relay ref
# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
# and run integration tests via cmlxc.
name: Trigger Docker build
on:
push:
branches: [main]
workflow_dispatch:
permissions: {}
jobs:
dispatch:
name: Dispatch build to chatmail/docker
runs-on: ubuntu-latest
if: github.repository == 'chatmail/relay'
steps:
- name: Compute short SHA
id: sha
run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
- name: Send repository_dispatch
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
with:
token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
repository: chatmail/docker
event-type: relay-updated
client-payload: >-
{
"relay_ref": "${{ github.ref_name }}",
"relay_sha": "${{ github.sha }}",
"relay_sha_short": "${{ steps.sha.outputs.short }}"
}

View File

@@ -7,8 +7,6 @@ on:
- 'scripts/build-docs.sh'
- '.github/workflows/docs-preview.yaml'
permissions: {}
jobs:
scripts:
name: build
@@ -18,8 +16,6 @@ jobs:
url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: initenv
run: scripts/initenv.sh
@@ -38,22 +34,18 @@ jobs:
- name: Get Pullrequest ID
id: prepare
run: |
export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
- run: |
echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
env:
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
- name: Upload preview
run: |
mkdir -p "$HOME/.ssh"
echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
chmod 600 "$HOME/.ssh/key"
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
env:
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
- name: check links
working-directory: doc

View File

@@ -10,8 +10,6 @@ on:
- 'scripts/build-docs.sh'
- '.github/workflows/docs.yaml'
permissions: {}
jobs:
scripts:
name: build
@@ -21,8 +19,6 @@ jobs:
url: https://chatmail.at/doc/relay/
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: initenv
run: scripts/initenv.sh

View File

@@ -0,0 +1,104 @@
name: deploy on staging-ipv4.testrun.org, and run tests
on:
push:
branches:
- main
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
deploy:
name: deploy on staging-ipv4.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency: staging-ipv4.testrun.org
steps:
- uses: actions/checkout@v4
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- name: setup dependencies
run: |
ssh root@staging-ipv4.testrun.org apt update
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
- name: initialize config
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
- name: set DNS entries
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
cat .github/workflows/staging-ipv4.testrun.org-default.zone
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
- name: cmdeploy dns
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"

97
.github/workflows/test-and-deploy.yaml vendored Normal file
View File

@@ -0,0 +1,97 @@
name: deploy on staging2.testrun.org, and run tests
on:
push:
branches:
- main
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
deploy:
name: deploy on staging2.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency: staging2.testrun.org
steps:
- uses: actions/checkout@v4
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging2.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: add hpk42 key to staging server
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- run: |
cmdeploy init staging2.testrun.org
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
- run: cmdeploy run --verbose --skip-dns-check
- name: set DNS entries
run: |
cmdeploy dns --zonefile staging-generated.zone --verbose
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
cat .github/workflows/staging.testrun.org-default.zone
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
- name: cmdeploy dns
run: cmdeploy dns -v

View File

@@ -1,26 +0,0 @@
name: GitHub Actions Security Analysis with zizmor
on:
push:
branches: ["main"]
pull_request:
branches: ["**"]
permissions: {}
jobs:
zizmor:
name: Run zizmor
runs-on: ubuntu-latest
permissions:
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Run zizmor
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3

7
.github/zizmor.yml vendored
View File

@@ -1,7 +0,0 @@
rules:
unpinned-uses:
config:
policies:
actions/*: ref-pin
dependabot/*: ref-pin
chatmail/*: ref-pin

View File

@@ -1,89 +1,5 @@
# Changelog for chatmail deployment
## 1.10.0 2026-04-30
* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
* stop installing cron package <https://github.com/chatmail/relay/pull/898>
* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
* dovecot update url <https://github.com/chatmail/relay/pull/880>
* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
* try to use "build machine" and "deployment server" consistently <https://github.com/chatmail/relay/pull/797>
* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
* streamline migration guide wording, provide titled steps <https://github.com/chatmail/relay/pull/789>
* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
## 1.9.0 2025-12-18
### Documentation

View File

@@ -21,8 +21,7 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
chatmail-expire = "chatmaild.expire:daily_expire_main"
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
chatmail-expire = "chatmaild.expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
lastlogin = "chatmaild.lastlogin:main"
turnserver = "chatmaild.turnserver:main"

View File

@@ -1,3 +1,4 @@
import os
from pathlib import Path
import iniconfig
@@ -37,15 +38,15 @@ class Config:
self.filtermail_smtp_port_incoming = int(
params.get("filtermail_smtp_port_incoming", "10081")
)
self.filtermail_http_port_incoming = int(
params.get("filtermail_http_port_incoming", "10082")
)
self.filtermail_http_port = int(params.get("filtermail_http_port", "10082"))
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
self.postfix_reinject_port_incoming = int(
params.get("postfix_reinject_port_incoming", "10026")
)
self.mtail_address = params.get("mtail_address")
self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
self.acme_email = params.get("acme_email", "")
self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -92,11 +93,6 @@ class Config:
# old unused option (except for first migration from sqlite to maildir store)
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
@property
def max_mailbox_size_mb(self):
"""Return max_mailbox_size as an integer in megabytes."""
return parse_size_mb(self.max_mailbox_size)
def _getbytefile(self):
return open(self._inipath, "rb")
@@ -110,16 +106,6 @@ class Config:
return User(maildir, addr, password_path, uid="vmail", gid="vmail")
def parse_size_mb(limit):
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
value = limit.strip().upper().removesuffix("B")
if value.endswith("G"):
return int(value[:-1]) * 1024
if value.endswith("M"):
return int(value[:-1])
return int(value)
def write_initial_config(inipath, mail_domain, overrides):
"""Write out default config file, using the specified config value overrides."""
content = get_default_config_content(mail_domain, **overrides)

View File

@@ -4,26 +4,17 @@ Expire old messages and addresses.
"""
import os
import re
import shutil
import sys
import time
from argparse import ArgumentParser
from collections import namedtuple
from datetime import datetime
from pathlib import Path
from stat import S_ISREG
from chatmaild.config import read_config
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
QUOTA_CLEANUP_FACTOR = 0.7
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
def iter_mailboxes(basedir, maxnum):
@@ -83,42 +74,6 @@ class MailboxStat:
self.extrafiles.sort(key=lambda x: -x.size)
def parse_dovecot_filename(relpath):
m = _dovecot_fn_rex.match(relpath)
if not m:
return None
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
def scan_mailbox_messages(mbox):
messages = []
for sub in ("cur", "new"):
for name in os_listdir_if_exists(mbox / sub):
if entry := parse_dovecot_filename(f"{sub}/{name}"):
messages.append(entry)
return messages
def expire_to_target(mbox, target_bytes):
messages = scan_mailbox_messages(mbox)
total_size = sum(m.quota_size for m in messages)
# Keep recent 24 hours of messages protected from expiry because
# likely something is wrong with interactions on that address
# and quota-full signal can help the address owner's device to notice it
undeletable_messages_cutoff = time.time() - (3600 * 24)
removed = 0
for entry in sorted(messages):
if total_size <= target_bytes:
break
if entry.mtime > undeletable_messages_cutoff:
break
(mbox / entry.path).unlink(missing_ok=True)
total_size -= entry.quota_size
removed += 1
return removed
def print_info(msg):
print(msg, file=sys.stderr)
@@ -188,21 +143,12 @@ class Expiry:
else:
continue
changed = True
target_bytes = (
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
)
removed = expire_to_target(Path(mbox.basedir), target_bytes)
if removed:
changed = True
self.del_files += removed
if self.verbose:
print_info(
f"quota-expire: removed {removed} message(s) from {mboxname}"
)
if changed:
self.remove_file(f"{mbox.basedir}/maildirsize")
for file in mbox.extrafiles:
if "dovecot.index.cache" in file.path.split("/")[-1]:
if file.size > 500 * 1024:
self.remove_file(file.path)
def get_summary(self):
return (
@@ -212,9 +158,9 @@ class Expiry:
)
def daily_expire_main(args=None):
def main(args=None):
"""Expire mailboxes and messages according to chatmail config"""
parser = ArgumentParser(description=daily_expire_main.__doc__)
parser = ArgumentParser(description=main.__doc__)
ini = "/usr/local/lib/chatmaild/chatmail.ini"
parser.add_argument(
"chatmail_ini",
@@ -260,33 +206,5 @@ def daily_expire_main(args=None):
print(exp.get_summary())
def quota_expire_main(args=None):
"""Remove mailbox messages to stay within a megabyte target.
This entry point is called by dovecot when a quota threshold is passed.
"""
parser = ArgumentParser(description=quota_expire_main.__doc__)
parser.add_argument(
"target_mb",
type=int,
help="target mailbox size in megabytes",
)
parser.add_argument(
"mailbox_path",
type=Path,
help="path to a user mailbox",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
removed_count = expire_to_target(args.mailbox_path, target_bytes)
if removed_count:
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
print(
f"quota-expire: removed {removed_count} message(s)"
f" from {args.mailbox_path.name}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
main(sys.argv[1:])

View File

@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
max_user_send_burst_size = 10
# maximum mailbox size of a chatmail address
# Oldest messages will be removed automatically, so mailboxes never run full.
max_mailbox_size = 500M
# maximum message size for an e-mail in bytes

View File

@@ -2,7 +2,6 @@
"""CGI script for creating new accounts."""
import ipaddress
import json
import secrets
import string
@@ -15,16 +14,6 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
def wrap_ip(host):
if host.startswith("[") and host.endswith("]"):
return host
try:
ipaddress.ip_address(host)
return f"[{host}]"
except ValueError:
return host
def create_newemail_dict(config: Config):
user = "".join(
secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -33,7 +22,7 @@ def create_newemail_dict(config: Config):
secrets.choice(ALPHANUMERIC_PUNCT)
for _ in range(config.password_min_length + 3)
)
return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
def create_dclogin_url(email, password):

View File

@@ -1,6 +1,6 @@
import pytest
from chatmaild.config import parse_size_mb, read_config
from chatmaild.config import read_config
def test_read_config_basic(example_config):
@@ -121,17 +121,3 @@ def test_config_tls_external_bad_format(make_config):
"tls_external_cert_and_key": "/only/one/path.pem",
},
)
def test_parse_size_mb():
assert parse_size_mb("500M") == 500
assert parse_size_mb("2G") == 2048
assert parse_size_mb(" 1g ") == 1024
assert parse_size_mb("100MB") == 100
assert parse_size_mb("256") == 256
def test_max_mailbox_size_mb(make_config):
config = make_config("chat.example.org")
assert config.max_mailbox_size == "500M"
assert config.max_mailbox_size_mb == 500

View File

@@ -1,7 +1,7 @@
import time
from chatmaild.doveauth import AuthDictProxy
from chatmaild.expire import daily_expire_main as main_expire
from chatmaild.expire import main as main_expire
def test_login_timestamps(example_config):

View File

@@ -1,7 +1,5 @@
import itertools
import os
import random
import time
from datetime import datetime
from fnmatch import fnmatch
from pathlib import Path
@@ -11,19 +9,13 @@ import pytest
from chatmaild.expire import (
FileEntry,
MailboxStat,
expire_to_target,
get_file_entry,
iter_mailboxes,
os_listdir_if_exists,
parse_dovecot_filename,
quota_expire_main,
scan_mailbox_messages,
)
from chatmaild.expire import daily_expire_main as expiry_main
from chatmaild.expire import main as expiry_main
from chatmaild.fsreport import main as report_main
MB = 1024 * 1024
def fill_mbox(folderdir):
password = folderdir.joinpath("password")
@@ -204,51 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
tmp_path.joinpath("x").write_text("hello")
assert len(os_listdir_if_exists(str(tmp_path))) == 1
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
# --- quota expire tests ---
_msg_counter = itertools.count(1)
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
seq = next(_msg_counter)
mtime = int(time.time() - days_old * 86400)
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
path = basedir / sub / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
os.utime(path, (mtime, mtime))
return path
def test_parse_dovecot_filename():
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
assert e.mtime == 1775324677
assert e.quota_size == 3235
assert parse_dovecot_filename("cur/msg_without_structure") is None
def test_expire_to_target(tmp_path):
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
_create_message(tmp_path, "new", MB, days_old=5)
_create_message(tmp_path, "cur", MB, days_old=0) # undeletable (<1 hour)
assert len(scan_mailbox_messages(tmp_path)) == 3
# removes oldest first, uses S= size not disk size
removed = expire_to_target(tmp_path, MB)
assert removed == 2
msgs = scan_mailbox_messages(tmp_path)
assert len(msgs) == 1
# the surviving message is the fresh undeletable one
assert msgs[0].mtime > time.time() - 3600
def test_quota_expire_main(tmp_path, capsys):
mbox = tmp_path / "user@example.org"
_create_message(mbox, "cur", 2 * MB, days_old=5)
(mbox / "maildirsize").write_text("x")
quota_expire_main([str(1), str(mbox)])
_, err = capsys.readouterr()
assert "quota-expire: removed 1 message(s) from user@example.org" in err
assert not (mbox / "maildirsize").exists()

View File

@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
assert ac1["password"] != ac2["password"]
def test_create_newemail_dict_ip(make_config):
config = make_config("1.2.3.4")
ac = create_newemail_dict(config)
assert ac["email"].endswith("@[1.2.3.4]")
def test_create_dclogin_url():
url = create_dclogin_url("user@example.org", "p@ss w+rd")
assert url.startswith("dclogin:")

View File

@@ -3,8 +3,6 @@ import io
import os
from contextlib import contextmanager
from pyinfra import host
from pyinfra.facts.server import Command
from pyinfra.operations import files, server, systemd
@@ -13,17 +11,6 @@ def has_systemd():
return os.path.isdir("/run/systemd/system")
def is_in_container() -> bool:
"""Return True if running inside a container (Docker, LXC, etc.)."""
return (
host.get_fact(
Command,
"systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
)
== "yes"
)
@contextmanager
def blocked_service_startup():
"""Prevent services from auto-starting during package installation.

View File

@@ -101,11 +101,16 @@ def run_cmd(args, out):
env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
if not args.dns_check_disabled:
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
if ssh_host == "localhost":
if ssh_host in ["localhost", "@docker"]:
if ssh_host == "@docker":
env["CHATMAIL_NOPORTCHECK"] = "True"
cmd = f"{pyinf} @local {deploy_path} -y"
if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -191,6 +196,12 @@ def status_cmd(args, out):
def test_cmd_options(parser):
parser.add_argument(
"--slow",
dest="slow",
action="store_true",
help="also run slow tests",
)
add_ssh_host_option(parser)
@@ -212,6 +223,8 @@ def test_cmd(args, out):
"-v",
"--durations=5",
]
if args.slow:
pytest_args.append("--slow")
ret = out.run_ret(pytest_args, env=env)
return ret
@@ -303,7 +316,7 @@ def add_ssh_host_option(parser):
parser.add_argument(
"--ssh-host",
dest="ssh_host",
help="Run commands on 'localhost' or on a specific SSH host "
help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
"instead of chatmail.ini's mail_domain.",
)
@@ -365,7 +378,9 @@ def get_parser():
def get_sshexec(ssh_host: str, verbose=True):
if ssh_host in ["localhost", "@local"]:
return LocalExec(verbose)
return LocalExec(verbose, docker=False)
elif ssh_host == "@docker":
return LocalExec(verbose, docker=True)
if verbose:
print(f"[ssh] login to {ssh_host}")
return SSHExec(ssh_host, verbose=verbose)

View File

@@ -2,6 +2,7 @@
Chat Mail pyinfra deploy.
"""
import os
import shutil
import subprocess
import sys
@@ -27,7 +28,6 @@ from .basedeploy import (
configure_remote_units,
get_resource,
has_systemd,
is_in_container,
)
from .dovecot.deployer import DovecotDeployer
from .external.deployer import ExternalTlsDeployer
@@ -150,6 +150,9 @@ class UnboundDeployer(Deployer):
self.need_restart = False
def install(self):
# Run local DNS resolver `unbound`. `resolvconf` takes care of
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
# On an IPv4-only system, if unbound is started but not configured,
# it causes subsequent steps to fail to resolve hosts.
with blocked_service_startup():
@@ -159,31 +162,6 @@ class UnboundDeployer(Deployer):
)
def configure(self):
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
apt.packages(
name="Purge resolvconf",
packages=["resolvconf"],
present=False,
extra_uninstall_args="--purge",
)
# systemd-resolved can't be purged due to dependencies; stop and mask.
server.shell(
name="Stop and mask systemd-resolved",
commands=[
"systemctl stop systemd-resolved.service || true",
"systemctl mask systemd-resolved.service",
],
)
# Configure unbound resolver with Quad9 fallback and a trailing newline
# (SolusVM bug).
files.put(
name="Write static resolv.conf",
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
dest="/etc/resolv.conf",
user="root",
group="root",
mode="644",
)
server.shell(
name="Generate root keys for validating DNSSEC",
commands=[
@@ -590,6 +568,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
Deployment().perform_stages([WebsiteDeployer(config)])
return
if host.get_fact(Port, port=53) != "unbound":
files.line(
name="Add 9.9.9.9 to resolv.conf",
path="/etc/resolv.conf",
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
line="\nnameserver 9.9.9.9",
)
# Check if mtail_address interface is available (if configured)
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
@@ -598,7 +584,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
exit(1)
if not is_in_container():
if not os.environ.get("CHATMAIL_NOPORTCHECK"):
port_services = [
(["master", "smtpd"], 25),
("unbound", 53),

View File

@@ -13,11 +13,9 @@ from cmdeploy.basedeploy import (
blocked_service_startup,
configure_remote_units,
get_resource,
is_in_container,
)
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
DOVECOT_VERSION = "2.3.21+dfsg1-3"
DOVECOT_SHA256 = {
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
@@ -42,14 +40,11 @@ class DovecotDeployer(Deployer):
with blocked_service_startup():
debs = []
for pkg in ("core", "imapd", "lmtpd"):
deb, changed = _download_dovecot_package(pkg, arch)
self.need_restart |= changed
deb = _download_dovecot_package(pkg, arch)
if deb:
debs.append(deb)
if debs:
deb_list = " ".join(debs)
# First dpkg may fail on missing dependencies (stderr suppressed);
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
server.shell(
name="Install dovecot packages",
commands=[
@@ -58,7 +53,6 @@ class DovecotDeployer(Deployer):
f"dpkg --force-confdef --force-confold -i {deb_list}",
],
)
self.need_restart = True
files.put(
name="Pin dovecot packages to block Debian dist-upgrades",
src=io.StringIO(
@@ -67,30 +61,15 @@ class DovecotDeployer(Deployer):
"Pin-Priority: -1\n"
),
dest="/etc/apt/preferences.d/pin-dovecot",
user="root",
group="root",
mode="644",
)
def configure(self):
configure_remote_units(self.config.mail_domain, self.units)
config_restart, self.daemon_reload = _configure_dovecot(self.config)
self.need_restart |= config_restart
self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
def activate(self):
activate_remote_units(self.units)
# Detect stale binary: package installed but service still runs old (deleted) binary.
if not self.disable_mail and not self.need_restart:
stale = host.get_fact(
Command,
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
" && echo STALE || true",
)
if stale == "STALE":
self.need_restart = True
restart = False if self.disable_mail else self.need_restart
systemd.service(
@@ -115,22 +94,22 @@ def _pick_url(primary, fallback):
return fallback
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
"""Download a dovecot .deb if needed, return (path, changed)."""
def _download_dovecot_package(package: str, arch: str):
"""Download a dovecot .deb if needed, return its path (or None)."""
arch = "amd64" if arch == "x86_64" else arch
arch = "arm64" if arch == "aarch64" else arch
pkg_name = f"dovecot-{package}"
sha256 = DOVECOT_SHA256.get((package, arch))
if sha256 is None:
op = apt.packages(packages=[pkg_name])
return None, bool(getattr(op, "changed", False))
apt.packages(packages=[pkg_name])
return None
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
if DOVECOT_PACKAGE_VERSION in installed_versions:
return None, False
if DOVECOT_VERSION in installed_versions:
return None
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
url_version = DOVECOT_VERSION.replace("+", "%2B")
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
@@ -145,7 +124,18 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
)
return deb_filename, True
return deb_filename
def _can_set_inotify_limits() -> bool:
is_container = (
host.get_fact(
Command,
"systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
)
== "yes"
)
return not is_container
def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
@@ -183,10 +173,10 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
# it is recommended to set the following inotify limits
can_modify = not is_in_container()
can_modify = _can_set_inotify_limits()
for name in ("max_user_instances", "max_user_watches"):
key = f"fs.inotify.{name}"
value = host.get_fact(Sysctl).get(key, 0)
value = host.get_fact(Sysctl)[key]
if value > 65534:
continue
if not can_modify:

View File

@@ -70,12 +70,6 @@ userdb {
# Mailboxes are stored in the "mail" directory of the vmail user home.
mail_location = maildir:{{ config.mailboxes_dir }}/%u
# index/cache files are not very useful for chatmail relay operations
# but it's not clear how to disable them completely.
# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
# if the cache file becomes larger than the specified size, it is truncated by dovecot
mail_cache_max_size = 500K
namespace inbox {
inbox = yes
@@ -133,11 +127,6 @@ protocol lmtp {
# mail_lua and push_notification_lua are needed for Lua push notification handler.
# <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
# Disable fsync for LMTP. May lose delivered message,
# but unlikely to cause problems with multiple relays.
# https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
mail_fsync = never
}
plugin {
@@ -149,26 +138,12 @@ plugin {
}
plugin {
# for now we define static quota-rules for all users
quota = maildir:User quota
quota_rule = *:storage={{ config.max_mailbox_size }}
quota_max_mail_size={{ config.max_message_size }}
quota_grace = 0
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
# The percentages are chosen to prevent current Delta Chat users
# from seeing "quota warnings" which trigger at 80% and 95%.
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
}
service quota-warning {
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
user = vmail
unix_listener quota-warning {
user = vmail
mode = 0600
}
# quota_over_flag_value = TRUE
}
# push_notification configuration
@@ -271,9 +246,6 @@ protocol imap {
# sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
rawlog_dir = %h
# Disable fsync for IMAP. May lose IMAP changes like setting flags.
mail_fsync = never
}
{% endif %}

View File

@@ -1,5 +1,3 @@
import os
from pyinfra import facts, host
from pyinfra.operations import files, systemd
@@ -15,16 +13,6 @@ class FiltermailDeployer(Deployer):
self.need_restart = False
def install(self):
local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
if local_bin:
self.need_restart |= files.put(
name="Upload locally built filtermail",
src=local_bin,
dest=self.bin_path,
mode="755",
).changed
return
arch = host.get_fact(facts.server.Arch)
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
sha256sum = {

View File

@@ -78,11 +78,3 @@ counter rejected_unencrypted_mail_count
/Rejected unencrypted mail/ {
rejected_unencrypted_mail_count++
}
counter quota_expire_runs
counter quota_expire_removed_files
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
quota_expire_runs++
quota_expire_removed_files += $count
}

View File

@@ -1,6 +1,5 @@
[Unit]
Description=mtail
After=multi-user.target
[Service]
Type=simple

View File

@@ -74,7 +74,7 @@ http {
access_log syslog:server=unix:/dev/log,facility=local7;
location /mxdeliv/ {
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port }};
}
location / {

View File

@@ -69,6 +69,15 @@ mynetworks = 127.0.0.0/8
{% else %}
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
{% endif %}
{% if config.addr_v4 %}
smtp_bind_address = {{ config.addr_v4 }}
{% endif %}
{% if config.addr_v6 %}
smtp_bind_address6 = {{ config.addr_v6 }}
{% endif %}
{% if config.addr_v4 or config.addr_v6 %}
smtp_bind_address_enforce = yes
{% endif %}
mailbox_size_limit = 0
message_size_limit = {{config.max_message_size}}
recipient_delimiter = +

View File

@@ -64,13 +64,11 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
)
def query_dns(typ, domain, shell_exec=None):
if shell_exec is None:
shell_exec = shell
def query_dns(typ, domain):
# Get autoritative nameserver from the SOA record.
soa_answers = [
x.split()
for x in shell_exec(
for x in shell(
f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
).split("\n")
]
@@ -80,27 +78,8 @@ def query_dns(typ, domain, shell_exec=None):
ns = soa[0][4]
# Query authoritative nameserver directly to bypass DNS cache.
return _dig_authoritative(ns, domain, typ, shell_exec=shell_exec)
def _parse_dig_result(output):
"""Return first non-comment, non-empty line from dig output, or empty string."""
lines = [line for line in output.split("\n") if not line.startswith(";")]
return next((line for line in lines if line.strip()), "")
def _dig_authoritative(ns, domain, typ, shell_exec=None):
"""Query authoritative NS, falling back to IPv4-only if default fails."""
if shell_exec is None:
shell_exec = shell
# limit timeout and tries to not hang on a broken default NS
cmd = f"dig @{ns} -r -q {domain} -t {typ} +short +timeout=10 +tries=2"
result = _parse_dig_result(shell_exec(cmd, print=log_progress))
if result:
return result
# Fallback: force IPv4 transport (handles broken IPv6 to NS)
return _parse_dig_result(shell_exec(cmd + " -4", print=log_progress))
res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
return next((line for line in res.split("\n") if not line.startswith(";")), "")
def check_zonefile(zonefile, verbose=True):

View File

@@ -87,8 +87,9 @@ class SSHExec:
class LocalExec:
FuncError = FuncError
def __init__(self, verbose=False):
def __init__(self, verbose=False, docker=False):
self.verbose = verbose
self.docker = docker
def __call__(self, call, kwargs=None, log_callback=None):
if kwargs is None:
@@ -100,6 +101,10 @@ class LocalExec:
if not title:
title = call.__name__
where = "locally"
if self.docker:
if call == remote.rdns.perform_initial_checks:
kwargs["pre_command"] = "docker exec chatmail "
where = "in docker"
if self.verbose:
print_stderr(f"Running {where}: {title}(**{kwargs})")
return self(call, kwargs, log_callback=print_stderr)

View File

@@ -71,44 +71,6 @@ class TestSSHExecutor:
assert (now - since_date).total_seconds() < 60 * 60 * 51
def test_dovecot_main_process_matches_installed_binary(sshdomain):
sshexec = get_sshexec(sshdomain)
main_pid = int(
sshexec(
call=remote.rshell.shell,
kwargs=dict(
command="timeout 10 systemctl show -p MainPID --value dovecot.service"
),
).strip()
)
assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
exe = sshexec(
call=remote.rshell.shell,
kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
).strip()
status_text = sshexec(
call=remote.rshell.shell,
kwargs=dict(
command="timeout 10 systemctl show -p StatusText --value dovecot.service"
),
).strip()
installed_version = sshexec(
call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
).strip()
assert not exe.endswith("(deleted)"), (
f"running dovecot binary was deleted (stale after upgrade): {exe}"
)
expected_status_text = f"v{installed_version}"
assert status_text == expected_status_text or status_text.startswith(
f"{expected_status_text} "
), (
f"dovecot status version mismatch: "
f"StatusText={status_text!r}, installed={installed_version!r}"
)
def test_timezone_env(remote):
for line in remote.iter_output("env"):
print(line)
@@ -221,6 +183,7 @@ def test_rewrite_subject(cmsetup, maildata):
assert "Subject: Unencrypted subject" not in rcvd_msg
@pytest.mark.slow
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
"""Test that the per-account send-mail limit is exceeded."""
user1, user2 = cmsetup.gen_users(2)
@@ -243,6 +206,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [

View File

@@ -1,5 +1,4 @@
import imaplib
import ipaddress
import itertools
import os
import random
@@ -15,12 +14,10 @@ from chatmaild.config import read_config
conftestdir = Path(__file__).parent
def _is_ip(domain):
try:
ipaddress.ip_address(domain)
return True
except ValueError:
return False
def pytest_addoption(parser):
parser.addoption(
"--slow", action="store_true", default=False, help="also run slow tests"
)
def pytest_configure(config):
@@ -30,6 +27,13 @@ def pytest_configure(config):
)
def pytest_runtest_setup(item):
markers = list(item.iter_markers(name="slow"))
if markers:
if not item.config.getoption("--slow"):
pytest.skip("skipping slow test, use --slow to run")
def _get_chatmail_config():
inipath = os.environ.get("CHATMAIL_INI")
if inipath:
@@ -278,7 +282,6 @@ def gencreds(chatmail_config):
def gen(domain=None):
domain = domain if domain else chatmail_config.mail_domain
addr_domain = f"[{domain}]" if _is_ip(domain) else domain
while 1:
num = next(count)
alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -292,7 +295,7 @@ def gencreds(chatmail_config):
password = "".join(
random.choices(alphanumeric, k=chatmail_config.password_min_length)
)
yield f"{user}@{addr_domain}", f"{password}"
yield f"{user}@{domain}", f"{password}"
return lambda domain=None: next(gen(domain))
@@ -341,22 +344,9 @@ class ChatmailACFactory:
accounts = []
for _ in range(num):
account = self.dc.add_account()
addr, password = self.gencreds(domain)
if _is_ip(domain):
# Use DCLOGIN scheme with explicit server hosts,
# matching how madmail presents its addresses to users.
qr = (
f"dclogin:{addr}"
f"?p={password}&v=1"
f"&ih={domain}&ip=993"
f"&sh={domain}&sp=465"
f"&ic=3&ss=default"
)
future = account.add_transport_from_qr.future(qr)
else:
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
futures.append(future)
# ensure messages stay in INBOX so that they can be

View File

@@ -214,59 +214,3 @@ class TestZonefileChecks:
assert not mockout.captured_red
assert "correct" in mockout.captured_green[0]
assert not mockout.captured_red
class TestDigAuthoritative:
@pytest.fixture
def dig_auth(self):
"""Helper that calls _dig_authoritative with a recording shell function."""
calls = []
def run(first_result, ipv4_result=None):
def shell(cmd, print=print):
calls.append(cmd)
if "-4" in cmd:
return ipv4_result or ""
return first_result
result = remote.rdns._dig_authoritative(
"ns1.example.", "example.com", "A", shell_exec=shell
)
return result, calls
return run
def test_ipv4_fallback_on_error(self, dig_auth):
"""Fallback to -4 when first query returns only error lines."""
result, calls = dig_auth(
first_result=(
";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
),
ipv4_result="1.2.3.4",
)
assert len(calls) == 2
assert "-4" not in calls[0]
assert "-4" in calls[1]
assert result == "1.2.3.4"
def test_no_fallback_on_success(self, dig_auth):
"""No -4 fallback when first query returns a valid answer."""
result, calls = dig_auth(first_result="1.2.3.4")
assert result == "1.2.3.4"
assert len(calls) == 1
def test_fallback_with_mixed_error_lines(self, dig_auth):
"""Fallback triggers when output has only error lines mixed with empty."""
result, calls = dig_auth(
first_result=(
";; communications error to 2a01:4f8::80#53: timed out\n"
"\n"
";; communications error to 2a01:4f8::80#53: timed out\n"
),
ipv4_result=";; some warning\n1.2.3.4",
)
assert len(calls) == 2
assert result == "1.2.3.4"

View File

@@ -1,238 +0,0 @@
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
from pyinfra.facts.deb import DebPackages
from cmdeploy.dovecot import deployer as dovecot_deployer
def make_host(*fact_pairs):
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
Args:
*fact_pairs: tuples of (fact_class, fact_value) to register
Returns:
SimpleNamespace with get_fact that raises a clear error if an
unexpected fact type is requested.
"""
facts = dict(fact_pairs)
def get_fact(cls):
if cls not in facts:
registered = ", ".join(c.__name__ for c in facts)
raise LookupError(
f"unexpected get_fact({cls.__name__}); "
f"only registered: {registered}"
)
return facts[cls]
return SimpleNamespace(get_fact=get_fact)
@pytest.fixture
def deployer():
return dovecot_deployer.DovecotDeployer(
SimpleNamespace(mail_domain="chat.example.org"),
disable_mail=False,
)
@pytest.fixture
def patch_blocked(monkeypatch):
monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
@pytest.fixture
def mock_files_put(monkeypatch):
monkeypatch.setattr(
dovecot_deployer.files,
"put",
lambda **kwargs: SimpleNamespace(changed=False),
)
@pytest.fixture
def track_shell(monkeypatch):
calls = []
monkeypatch.setattr(
dovecot_deployer.server,
"shell",
lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
)
return calls
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
downloads = []
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host((DebPackages, {"dovecot-core": [epoch_version]})),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
assert changed is False, "should not flag changed when version already installed"
assert downloads == [], "should not download when version already installed"
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
monkeypatch,
):
downloads = []
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host((DebPackages, {})),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
# Verify the returned path uses archive version, not package version (with epoch)
assert changed is True, "should flag changed when package not yet installed"
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
f"deb path should use archive version (no epoch), got {deb!r}"
)
assert len(downloads) == 1, "files.download should be called exactly once"
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(
dovecot_deployer.DebPackages,
{
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
},
),
(dovecot_deployer.Arch, "x86_64"),
),
)
downloads = []
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deployer.install()
assert downloads == [], "should not download when all packages epoch-matched"
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
assert deployer.need_restart is False, (
"need_restart should be False when nothing changed"
)
def test_install_unsupported_arch_falls_back_to_apt(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
# For unsupported architectures, all fact lookups return the arch string.
monkeypatch.setattr(
dovecot_deployer,
"host",
SimpleNamespace(get_fact=lambda cls: "riscv64"),
)
apt_calls = []
# Mirrors apt.packages() return value: OperationMeta with .changed property.
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
def fake_apt(**kwargs):
apt_calls.append(kwargs)
changed = "lmtpd" in kwargs["packages"][0]
return SimpleNamespace(changed=changed)
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
deployer.install()
actual_pkgs = [c["packages"] for c in apt_calls]
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
)
assert track_shell == [], "should not run dpkg for unsupported arch"
assert deployer.need_restart is True, (
"need_restart should be True when apt installed a package"
)
def test_install_runs_dpkg_when_packages_need_download(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.DebPackages, {}),
(dovecot_deployer.Arch, "x86_64"),
),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: SimpleNamespace(changed=True),
)
deployer.install()
assert len(track_shell) == 1, (
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
)
cmds = track_shell[0]["commands"]
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
assert "apt-get -y --fix-broken install" in cmds[1]
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
assert deployer.need_restart is True, (
"need_restart should be True after dpkg install"
)
def test_pick_url_falls_back_on_primary_error(monkeypatch):
def raise_error(req, timeout):
raise OSError("connection timeout")
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
assert result == "http://fallback", (
f"should fall back when primary fails, got {result!r}"
)

View File

@@ -4,14 +4,12 @@
You can use the `make` command and `make html` to build web pages.
You need a Python environment with `sphinx` and other
dependencies, you can create it by running `scripts/initenv.sh`
from the repository root.
You need a Python environment where the following install was excuted:
pip install furo sphinx-autobuild
To develop/change documentation, you can then do:
. venv/bin/activate
cd doc
make auto
A page will open at https://127.0.0.1:8000/ serving the docs and it will

View File

@@ -16,6 +16,5 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
proxy
migrate
overview
reverse_dns
related
faq

View File

@@ -102,12 +102,8 @@ short overview of ``chatmaild`` services:
Apple/Google/Huawei.
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
deletes old messages, large messages, and entire mailboxes
of users who have not logged in for longer than
``delete_inactive_users_after`` days.
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
deletes users if they have not logged in for a longer while.
The timeframe can be configured in ``chatmail.ini``.
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
is contacted by Dovecot when a user logs in and stores the date of
@@ -160,8 +156,6 @@ Chatmail relay dependency diagram
/home/vmail/.../user"];
dovecot --- |lastlogin.socket|lastlogin;
dovecot --- chatmail-metadata;
dovecot --- |quota-warning|chatmail-quota-expire;
chatmail-quota-expire --- maildir;
lastlogin --- maildir;
doveauth --- maildir;
chatmail-expire-daily --- maildir;

View File

@@ -1,64 +0,0 @@
Configuring reverse DNS
=======================
Some email servers reject the emails
if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
Passing the check requires that the IP address that email is sent from
should have a ``PTR`` record pointing to the domain name of the server,
and domain name record should have an ``A/AAAA`` record
pointing to the IP address.
Modern email relies on DKIM and SPF for authentication,
while iprev check exists for
`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
Chatmail relays don't resolve ``PTR`` records,
so you can ignore this section if configuring ``PTR`` records
is difficult and federation with legacy email servers that don't accept
valid DKIM signature for authentication is not important.
Multi-homed setups
------------------
If you have a server with multiple IP addresses,
also known as multi-homed setup,
and don't publish all IP addresses in DNS,
you need to make sure you are using
the published address when making outgoing connections.
For example, your server may have a static IP
address, and a so-called Floating IP or Virtual IP
that can be moved between servers in case of
migration or for failover.
By using Floating IP you can avoid downtime
and keep the IP address reputation
for destinatinons that rely on IP reputation and IP blocklists.
In this case you will only publish
the Floating IP to DNS and only use the static IP
to SSH into the server.
If you have such setup, make sure that
you not only set ``PTR`` records for the Floating IP,
but make outgoing connections using the Floating IP.
Otherwise reverse DNS check succeed,
but forward check making sure your domain name points
to the IP address will fail.
Such setup is indistinguishable from someone
setting IP address ``PTR`` with the domain they don't own
and as a result don't succeed.
On Linux you can configure source IP address with ``ip route`` command,
for example:
::
ip route change default via <default-gateway> dev eth0 src <source-address>
Make sure to persist the change after verifying it is working.
You can check what your outgoing IP address is
with ``curl icanhazip.com``.
Check both the IPv4 and IPv6 addresses.
For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
and similarly for IPv6 if you have it.