Compare commits

..

1 Commits

Author SHA1 Message Date
j4n
0808c9dc47 feat: add CI for docker
Docker and Compose support is provided through a standalone repo at
https://github.com/chatmail/docker, add reusable Docker build/test CI
jobs to staging workflows and a Docker docs stub pointing to the
chatmail/docker repository.

Requires CHATMAIL_DOCKER_DISPATCH_TOKEN secret in relay repo settings
(fine-grained PAT with contents:write on chatmail/docker).
2026-04-15 16:35:14 +02:00
25 changed files with 488 additions and 411 deletions

View File

@@ -1,26 +1,15 @@
name: Run unit-tests and container-based deploy+test verification
name: CI
on:
# Triggers when a PR is merged into main or a direct push occurs
push:
branches: [ "main" ]
# Triggers for any PR (and its subsequent commits) targeting the main branch
pull_request:
branches: [ "main" ]
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
push:
jobs:
tox:
name: isolated chatmaild tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
# Checkout pull request HEAD commit instead of merge commit
# Otherwise `test_deployed_state` will be unhappy.
with:
@@ -35,9 +24,7 @@ jobs:
name: deploy-chatmail tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/checkout@v4
- name: initenv
run: scripts/initenv.sh
@@ -51,23 +38,5 @@ jobs:
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
lxc-test:
name: LXC deploy and test
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
with:
cmlxc_commands: |
cmlxc init
# single cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo cm0
cmlxc -v test-mini cm0
cmlxc -v test-cmdeploy cm0
# cross cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
cmlxc -v test-cmdeploy cm0 cm1
# cross cmdeploy/madmail relay tests
cmlxc -v deploy-madmail mad0
cmlxc -v test-cmdeploy cm0 mad0
cmlxc -v test-mini cm0 mad0
cmlxc -v test-mini mad0 cm0
# all other cmdeploy commands require a staging server
# see https://github.com/deltachat/chatmail/issues/100

125
.github/workflows/docker-deploy.yaml vendored Normal file
View File

@@ -0,0 +1,125 @@
name: Docker deploy
on:
workflow_call:
inputs:
staging_host:
required: true
type: string
description: 'SSH hostname (e.g. staging2.testrun.org)'
mail_domain:
required: true
type: string
description: 'MAIL_DOMAIN for docker compose'
zone_file:
required: true
type: string
description: 'Default zone file basename (e.g. staging.testrun.org-default.zone)'
jobs:
deploy-docker:
name: Docker deploy on ${{ inputs.staging_host }}
runs-on: ubuntu-latest
timeout-minutes: 20
environment:
name: ${{ inputs.staging_host }}
url: https://${{ inputs.staging_host }}/
concurrency: ${{ inputs.staging_host }}
env:
VPS: root@${{ inputs.staging_host }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Setup SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan ${{ inputs.staging_host }} > ~/.ssh/known_hosts
# Reuse TCP connection for all subsequent ssh/scp calls
echo -e "Host ${{ inputs.staging_host }}\n ControlMaster auto\n ControlPath ~/.ssh/ctrl-%r@%h:%p\n ControlPersist 10m" >> ~/.ssh/config
- name: stop bare services, install Docker, prepare mounts
run: |
ssh $VPS bash -s <<'EOF'
systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
mkdir -p /srv/chatmail/certs /srv/chatmail/dkim
cp -a /var/lib/acme/. /srv/chatmail/certs/ || true
cp -a /etc/dkimkeys/. /srv/chatmail/dkim/ || true
cp /etc/chatmail/chatmail.ini /srv/chatmail/chatmail.ini
EOF
- name: deploy with Docker
run: |
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
GHCR_IMAGE="ghcr.io/chatmail/docker:sha-${SHORT_SHA}"
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ $VPS:/srv/chatmail/relay/
echo "${{ secrets.GITHUB_TOKEN }}" | ssh $VPS "docker login ghcr.io -u ${{ github.actor }} --password-stdin && \
docker pull ${GHCR_IMAGE} && \
cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=${{ inputs.mail_domain }} docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
- name: wait for container healthy
run: |
ssh $VPS 'docker exec chatmail journalctl -f --no-pager' &
LOG_PID=$!
trap "kill $LOG_PID 2>/dev/null || true" EXIT
for i in $(seq 1 60); do
status=$(ssh $VPS 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
echo " [$i/60] status=$status"
if [ "$status" = "healthy" ]; then
echo "Container is healthy."
exit 0
fi
if [ "$status" = "unhealthy" ]; then
echo "Container is unhealthy!"
break
fi
sleep 5
done
echo "Container did not become healthy."
kill $LOG_PID 2>/dev/null || true
ssh $VPS bash -s <<'EOF'
echo "--- failed units ---"
docker exec chatmail systemctl --failed --no-pager || true
echo "--- service logs ---"
docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50 || true
echo "--- listening ports ---"
docker exec chatmail ss -tlnp || true
echo "--- chatmail.ini ---"
docker exec chatmail cat /etc/chatmail/chatmail.ini || true
EOF
exit 1
- name: show container state
run: |
ssh $VPS bash -s <<'EOF'
echo "--- listening ports ---"
docker exec chatmail ss -tlnp
echo "--- chatmail.ini ---"
docker exec chatmail cat /etc/chatmail/chatmail.ini
EOF
- name: Docker integration tests
run: ssh $VPS 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
- name: Docker DNS
run: |
git checkout .github/workflows/${{ inputs.zone_file }}
ssh $VPS bash -s <<'EOF'
docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys
docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose
docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone
EOF
scp $VPS:/tmp/staging.zone staging-generated.zone
cat staging-generated.zone >> .github/workflows/${{ inputs.zone_file }}
cat .github/workflows/${{ inputs.zone_file }}
scp .github/workflows/${{ inputs.zone_file }} root@ns.testrun.org:/etc/nsd/${{ inputs.staging_host }}.zone
ssh root@ns.testrun.org "nsd-checkzone ${{ inputs.staging_host }} /etc/nsd/${{ inputs.staging_host }}.zone && systemctl reload nsd"
- name: Docker final DNS check
run: ssh $VPS 'docker exec chatmail cmdeploy dns -v --ssh-host @local'

View File

@@ -0,0 +1,121 @@
name: deploy on staging-ipv4.testrun.org, and run tests
on:
push:
branches:
- main
- j4n/docker-pr
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
trigger-docker-build:
if: github.event_name == 'push'
uses: ./.github/workflows/trigger-docker-build.yaml
secrets: inherit
deploy:
name: deploy on staging-ipv4.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency: staging-ipv4.testrun.org
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging-ipv4.testrun.org
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- name: setup dependencies
run: |
ssh root@staging-ipv4.testrun.org apt update
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
- name: initialize config
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
- name: set DNS entries
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
cat .github/workflows/staging-ipv4.testrun.org-default.zone
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
- name: cmdeploy dns
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
deploy-docker:
needs: [deploy, trigger-docker-build]
if: github.event_name == 'push'
uses: ./.github/workflows/docker-deploy.yaml
with:
staging_host: staging-ipv4.testrun.org
mail_domain: staging-ipv4.testrun.org
zone_file: staging-ipv4.testrun.org-default.zone
secrets: inherit

112
.github/workflows/test-and-deploy.yaml vendored Normal file
View File

@@ -0,0 +1,112 @@
name: deploy on staging2.testrun.org, and run tests
on:
push:
branches:
- main
- j4n/docker-pr
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
trigger-docker-build:
if: github.event_name == 'push'
uses: ./.github/workflows/trigger-docker-build.yaml
secrets: inherit
deploy:
name: deploy on staging2.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency: staging2.testrun.org
steps:
- uses: actions/checkout@v4
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging2.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: add hpk42 key to staging server
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- run: |
cmdeploy init staging2.testrun.org
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
- run: cmdeploy run --verbose --skip-dns-check
- name: set DNS entries
run: |
cmdeploy dns --zonefile staging-generated.zone --verbose
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
cat .github/workflows/staging.testrun.org-default.zone
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
- name: cmdeploy dns
run: cmdeploy dns -v
deploy-docker:
needs: [deploy, trigger-docker-build]
if: github.event_name == 'push'
uses: ./.github/workflows/docker-deploy.yaml
with:
staging_host: staging2.testrun.org
mail_domain: staging2.testrun.org
zone_file: staging.testrun.org-default.zone
secrets: inherit

View File

@@ -0,0 +1,24 @@
name: Trigger Docker image build
on:
workflow_call:
jobs:
trigger-docker-build:
name: Trigger Docker image build
runs-on: ubuntu-latest
steps:
- uses: actions/github-script@v7
with:
github-token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
script: |
await github.rest.repos.createDispatchEvent({
owner: 'chatmail',
repo: 'docker',
event_type: 'relay-updated',
client_payload: {
relay_ref: context.ref,
relay_sha: context.sha,
relay_sha_short: context.sha.slice(0, 7)
}
})

View File

@@ -1,24 +1,5 @@
# Changelog for chatmail deployment
## Unreleased
### Features
- Automated per-user quota-keeping.
Replace daily timer-based message expire script
with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`).
When a user reaches 90% of their mailbox quota
Dovecot calls the new script which removes the largest and oldest messages
until usage drops below 80%.
The daily `chatmail-expire` timer now only handles deletion
of inactive user mailboxes.
After upgrading, run the following once to clean up
mailboxes that are already over quota::
/usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \
400 /home/vmail/mail/YOURDOMAIN --sweep
## 1.9.0 2025-12-18
### Documentation

View File

@@ -21,8 +21,7 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
chatmail-expire = "chatmaild.expire_inactive_users:main"
chatmail-quota-expire = "chatmaild.quota_expire:main"
chatmail-expire = "chatmaild.expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
lastlogin = "chatmaild.lastlogin:main"
turnserver = "chatmaild.turnserver:main"

View File

@@ -25,6 +25,8 @@ class Config:
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
self.max_mailbox_size = params["max_mailbox_size"]
self.max_message_size = int(params.get("max_message_size", "31457280"))
self.delete_mails_after = params["delete_mails_after"]
self.delete_large_after = params["delete_large_after"]
self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
self.username_min_length = int(params["username_min_length"])
self.username_max_length = int(params["username_max_length"])
@@ -36,9 +38,7 @@ class Config:
self.filtermail_smtp_port_incoming = int(
params.get("filtermail_smtp_port_incoming", "10081")
)
self.filtermail_http_port_incoming = int(
params.get("filtermail_http_port_incoming", "10082")
)
self.filtermail_http_port = int(params.get("filtermail_http_port", "10082"))
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
self.postfix_reinject_port_incoming = int(
params.get("postfix_reinject_port_incoming", "10026")
@@ -93,11 +93,6 @@ class Config:
# old unused option (except for first migration from sqlite to maildir store)
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
@property
def max_mailbox_size_mb(self):
"""Return max_mailbox_size as an integer in megabytes."""
return parse_size_mb(self.max_mailbox_size)
def _getbytefile(self):
return open(self._inipath, "rb")
@@ -111,16 +106,6 @@ class Config:
return User(maildir, addr, password_path, uid="vmail", gid="vmail")
def parse_size_mb(limit):
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
value = limit.strip().upper().rstrip("B")
if value.endswith("G"):
return int(value[:-1]) * 1024
if value.endswith("M"):
return int(value[:-1])
return int(value)
def write_initial_config(inipath, mail_domain, overrides):
"""Write out default config file, using the specified config value overrides."""
content = get_default_config_content(mail_domain, **overrides)

View File

@@ -115,8 +115,11 @@ class Expiry:
cutoff_without_login = (
self.now - int(self.config.delete_inactive_users_after) * 86400
)
cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
self.all_mboxes += 1
changed = False
if mbox.last_login and mbox.last_login < cutoff_without_login:
self.remove_mailbox(mbox.basedir)
return
@@ -128,10 +131,25 @@ class Expiry:
print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
else:
print_info(f"checking mailbox (no last_login) {mboxname}")
self.all_files += len(mbox.messages)
for message in mbox.messages:
if message.mtime < cutoff_mails:
self.remove_file(message.path, mtime=message.mtime)
elif message.size > 200000 and message.mtime < cutoff_large_mails:
# we only remove noticed large files (not unnoticed ones in new/)
parts = message.path.split("/")
if len(parts) >= 2 and parts[-2] == "cur":
self.remove_file(message.path, mtime=message.mtime)
else:
continue
changed = True
if changed:
self.remove_file(f"{mbox.basedir}/maildirsize")
def get_summary(self):
return (
f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
f"in {time.time() - self.start:2.2f} seconds"
)

View File

@@ -23,6 +23,12 @@ max_mailbox_size = 500M
# maximum message size for an e-mail in bytes
max_message_size = 31457280
# days after which mails are unconditionally deleted
delete_mails_after = 20
# days after which large messages (>200k) are unconditionally deleted
delete_large_after = 7
# days after which users without a successful login are deleted (database and mails)
delete_inactive_users_after = 90

View File

@@ -2,7 +2,6 @@
"""CGI script for creating new accounts."""
import ipaddress
import json
import secrets
import string
@@ -15,16 +14,6 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
def wrap_ip(host):
if host.startswith("[") and host.endswith("]"):
return host
try:
ipaddress.ip_address(host)
return f"[{host}]"
except ValueError:
return host
def create_newemail_dict(config: Config):
user = "".join(
secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -33,7 +22,7 @@ def create_newemail_dict(config: Config):
secrets.choice(ALPHANUMERIC_PUNCT)
for _ in range(config.password_min_length + 3)
)
return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
def create_dclogin_url(email, password):

View File

@@ -1,152 +0,0 @@
"""
Remove messages from a mailbox to meet a size target.
Dovecot calls this script when a user's quota is near its limit.
Files are scored by ``size * age`` so that large, old messages
are removed first.
Usage::
quota_expire <target_mb> <mailbox_path>
"""
import os
import sys
import time
from argparse import ArgumentParser
from collections import namedtuple
from stat import S_ISREG
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
def _get_file_entry(path):
try:
st = os.stat(path)
except FileNotFoundError:
return None
if not S_ISREG(st.st_mode):
return None
return FileEntry(path, st.st_mtime, st.st_size)
def _listdir(path):
try:
return os.listdir(path)
except FileNotFoundError:
return []
def scan_mailbox_messages(mailbox_dir):
messages = []
for sub in ("cur", "new", "tmp"):
subdir = f"{mailbox_dir}/{sub}"
for name in _listdir(subdir):
entry = _get_file_entry(f"{subdir}/{name}")
if entry is not None:
messages.append(entry)
return messages
def _remove_stale_caches(mailbox_dir):
for name in ("maildirsize", "dovecot.index.cache"):
try:
os.unlink(f"{mailbox_dir}/{name}")
except FileNotFoundError:
pass
def expire_to_target(mailbox_dir, target_bytes, now=None):
"""Remove highest-scored files until total size <= *target_bytes*.
Returns the list of removed file paths.
"""
if now is None:
now = time.time()
messages = scan_mailbox_messages(mailbox_dir)
total_size = sum(m.size for m in messages)
if total_size <= target_bytes:
return []
# Score: large and old files get the highest score.
scored = sorted(
messages,
key=lambda m: m.size * (now - m.mtime),
reverse=True,
)
removed = []
for entry in scored:
if total_size <= target_bytes:
break
try:
os.unlink(entry.path)
except FileNotFoundError:
continue
total_size -= entry.size
removed.append(entry.path)
if removed:
_remove_stale_caches(mailbox_dir)
return removed
def main(args=None):
"""Remove mailbox messages to stay within a megabyte target."""
parser = ArgumentParser(description=main.__doc__)
parser.add_argument(
"target_mb",
type=int,
help="target mailbox size in megabytes",
)
parser.add_argument(
"mailbox_path",
help="path to a user mailbox, or with --sweep the mailboxes directory",
)
parser.add_argument(
"--sweep",
action="store_true",
help="sweep all mailboxes under mailbox_path",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
if args.sweep:
return _sweep(args.mailbox_path, target_bytes)
removed = expire_to_target(args.mailbox_path, target_bytes)
if removed:
print(
f"removed {len(removed)} file(s) from {args.mailbox_path}"
f" to reach {args.target_mb} MB target",
file=sys.stderr,
)
return 0
def _sweep(mailboxes_dir, target_bytes):
try:
names = os.listdir(mailboxes_dir)
except FileNotFoundError:
print(f"directory not found: {mailboxes_dir}", file=sys.stderr)
return 1
for name in sorted(names):
if "@" not in name:
continue
mbox = f"{mailboxes_dir}/{name}"
removed = expire_to_target(mbox, target_bytes)
if removed:
print(
f"removed {len(removed)} file(s) from {name}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -34,6 +34,8 @@ def test_read_config_testrun(make_config):
assert config.postfix_reinject_port == 10025
assert config.max_user_send_per_minute == 60
assert config.max_mailbox_size == "500M"
assert config.delete_mails_after == "20"
assert config.delete_large_after == "7"
assert config.username_min_length == 9
assert config.username_max_length == 9
assert config.password_min_length == 9

View File

@@ -1,6 +1,7 @@
import os
import random
from datetime import datetime
from fnmatch import fnmatch
from pathlib import Path
import pytest
@@ -153,6 +154,35 @@ def test_expiry_cli_basic(example_config, mbox1):
expiry_main(args)
def test_expiry_cli_old_files(capsys, example_config, mbox1):
relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
cutoff_days = int(example_config.delete_mails_after) + 1
create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
cutoff_days = int(example_config.delete_large_after) + 1
create_new_messages(
mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
)
create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
args = str(example_config._inipath), "--remove", "-v"
expiry_main(args)
out, err = capsys.readouterr()
allpaths = relpaths_old + relpaths_large + ["maildirsize"]
for path in allpaths:
for line in err.split("\n"):
if fnmatch(line, f"removing*{path}"):
break
else:
if path != "new/msg_old_large2":
pytest.fail(f"failed to remove {path}\n{err}")
assert "shouldstay" not in err
def test_get_file_entry(tmp_path):
assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
p = tmp_path.joinpath("x")

View File

@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
assert ac1["password"] != ac2["password"]
def test_create_newemail_dict_ip(make_config):
config = make_config("1.2.3.4")
ac = create_newemail_dict(config)
assert ac["email"].endswith("@[1.2.3.4]")
def test_create_dclogin_url():
url = create_dclogin_url("user@example.org", "p@ss w+rd")
assert url.startswith("dclogin:")

View File

@@ -1,91 +0,0 @@
import os
import time
from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages
MB = 1024 * 1024
def _create_message(basedir, relpath, size, days_old=0):
path = basedir / relpath
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * size)
mtime = time.time() - days_old * 86400
os.utime(path, (mtime, mtime))
return path
def test_scan_cur_new_tmp(tmp_path):
_create_message(tmp_path, "cur/msg1", 100)
_create_message(tmp_path, "new/msg2", 200)
_create_message(tmp_path, "tmp/msg3", 300)
messages = scan_mailbox_messages(str(tmp_path))
assert len(messages) == 3
sizes = sorted(m.size for m in messages)
assert sizes == [100, 200, 300]
def test_scan_ignores_subfolders(tmp_path):
_create_message(tmp_path, "cur/a", 10)
_create_message(tmp_path, ".DeltaChat/cur/b", 20)
assert len(scan_mailbox_messages(str(tmp_path))) == 1
def test_scan_empty(tmp_path):
assert scan_mailbox_messages(str(tmp_path)) == []
assert scan_mailbox_messages(str(tmp_path / "nope")) == []
def test_noop_under_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", MB)
assert expire_to_target(str(tmp_path), 2 * MB) == []
assert (tmp_path / "cur" / "msg1").exists()
def test_removes_to_target(tmp_path):
now = time.time()
for i in range(15):
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
assert len(removed) == 5
assert len(scan_mailbox_messages(str(tmp_path))) == 10
def test_scoring_prefers_large_old(tmp_path):
now = time.time()
_create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30)
_create_message(tmp_path, "cur/small_new", MB, days_old=1)
removed = expire_to_target(str(tmp_path), 2 * MB, now=now)
assert len(removed) == 1
assert "large_old" in removed[0]
def test_scoring_large_new_beats_small_old(tmp_path):
now = time.time()
_create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1)
_create_message(tmp_path, "cur/small_old", MB, days_old=5)
# big_new score: 10MB * 1d = 10 vs small_old score: 1MB * 5d = 5
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
assert len(removed) == 1
assert "big_new" in removed[0]
def test_exact_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", 5 * MB)
assert expire_to_target(str(tmp_path), 5 * MB) == []
def test_removes_stale_caches(tmp_path):
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
(tmp_path / "maildirsize").write_text("x")
(tmp_path / "dovecot.index.cache").write_text("x")
expire_to_target(str(tmp_path), MB)
assert not (tmp_path / "maildirsize").exists()
assert not (tmp_path / "dovecot.index.cache").exists()
def test_no_cache_removal_when_under_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", MB)
(tmp_path / "maildirsize").write_text("x")
expire_to_target(str(tmp_path), 2 * MB)
assert (tmp_path / "maildirsize").exists()

View File

@@ -158,7 +158,7 @@ class UnboundDeployer(Deployer):
with blocked_service_startup():
apt.packages(
name="Install unbound",
packages=["unbound", "unbound-anchor", "dnsutils", "resolvconf"],
packages=["unbound", "unbound-anchor", "dnsutils"],
)
def configure(self):

View File

@@ -4,7 +4,7 @@ import urllib.request
from chatmaild.config import Config
from pyinfra import host
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Arch, Command, Sysctl
from pyinfra.facts.server import Arch, Sysctl
from pyinfra.operations import apt, files, server, systemd
from cmdeploy.basedeploy import (
@@ -80,17 +80,6 @@ class DovecotDeployer(Deployer):
def activate(self):
activate_remote_units(self.units)
# Detect stale binary: package installed but service still runs old (deleted) binary.
if not self.disable_mail and not self.need_restart:
stale = host.get_fact(
Command,
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
" && echo STALE || true",
)
if stale == "STALE":
self.need_restart = True
restart = False if self.disable_mail else self.need_restart
systemd.service(

View File

@@ -133,11 +133,6 @@ protocol lmtp {
# mail_lua and push_notification_lua are needed for Lua push notification handler.
# <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
# Disable fsync for LMTP. May lose delivered message,
# but unlikely to cause problems with multiple relays.
# https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
mail_fsync = never
}
plugin {
@@ -149,22 +144,12 @@ plugin {
}
plugin {
# for now we define static quota-rules for all users
# for now we define static quota-rules for all users
quota = maildir:User quota
quota_rule = *:storage={{ config.max_mailbox_size }}
quota_max_mail_size={{ config.max_message_size }}
quota_grace = 0
# When a user reaches 90% quota, run chatmail-quota-expire
# to remove large/old messages until usage is below 80%.
quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
}
service quota-warning {
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
user = vmail
unix_listener quota-warning {
}
# quota_over_flag_value = TRUE
}
# push_notification configuration
@@ -267,9 +252,6 @@ protocol imap {
# sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
rawlog_dir = %h
# Disable fsync for IMAP. May lose IMAP changes like setting flags.
mail_fsync = never
}
{% endif %}

View File

@@ -74,7 +74,7 @@ http {
access_log syslog:server=unix:/dev/log,facility=local7;
location /mxdeliv/ {
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port }};
}
location / {

View File

@@ -244,6 +244,24 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
]
outdated_days = int(chatmail_config.delete_large_after) + 1
find_cmds.append(
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
)
for cmd in find_cmds:
for line in remote.iter_output(cmd):
assert not line
def test_deployed_state(remote):

View File

@@ -1,5 +1,4 @@
import imaplib
import ipaddress
import itertools
import os
import random
@@ -15,14 +14,6 @@ from chatmaild.config import read_config
conftestdir = Path(__file__).parent
def _is_ip(domain):
try:
ipaddress.ip_address(domain)
return True
except ValueError:
return False
def pytest_addoption(parser):
parser.addoption(
"--slow", action="store_true", default=False, help="also run slow tests"
@@ -291,7 +282,6 @@ def gencreds(chatmail_config):
def gen(domain=None):
domain = domain if domain else chatmail_config.mail_domain
addr_domain = f"[{domain}]" if _is_ip(domain) else domain
while 1:
num = next(count)
alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -305,7 +295,7 @@ def gencreds(chatmail_config):
password = "".join(
random.choices(alphanumeric, k=chatmail_config.password_min_length)
)
yield f"{user}@{addr_domain}", f"{password}"
yield f"{user}@{domain}", f"{password}"
return lambda domain=None: next(gen(domain))
@@ -354,22 +344,9 @@ class ChatmailACFactory:
accounts = []
for _ in range(num):
account = self.dc.add_account()
addr, password = self.gencreds(domain)
if _is_ip(domain):
# Use DCLOGIN scheme with explicit server hosts,
# matching how madmail presents its addresses to users.
qr = (
f"dclogin:{addr}"
f"?p={password}&v=1"
f"&ih={domain}&ip=993"
f"&sh={domain}&sp=465"
f"&ic=3&ss=default"
)
future = account.add_transport_from_qr.future(qr)
else:
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
futures.append(future)
# ensure messages stay in INBOX so that they can be

View File

@@ -2,9 +2,9 @@ from contextlib import nullcontext
from types import SimpleNamespace
import pytest
from pyinfra.facts.deb import DebPackages
from cmdeploy.dovecot import deployer as dovecot_deployer
from pyinfra.facts.deb import DebPackages
def make_host(*fact_pairs):

View File

@@ -98,6 +98,13 @@ steps. Please substitute it with your own domain.
configure at your DNS provider (it can take some time until they are
public).
Docker installation
-------------------
There is experimental support for running chatmail via Docker Compose.
See the `chatmail/docker README <https://github.com/chatmail/docker>`_
for full setup instructions.
Other helpful commands
----------------------

View File

@@ -102,14 +102,8 @@ short overview of ``chatmaild`` services:
Apple/Google/Huawei.
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
deletes entire mailboxes of users who have not logged in
for longer than ``delete_inactive_users_after`` days.
- `chatmail-quota-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/quota_expire.py>`_
is called by Dovecot's ``quota_warning`` mechanism when a
user reaches 90% of their mailbox quota.
It removes the largest and oldest messages
until usage drops below 80% of the quota.
deletes users if they have not logged in for a longer while.
The timeframe can be configured in ``chatmail.ini``.
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
is contacted by Dovecot when a user logs in and stores the date of
@@ -145,7 +139,7 @@ Chatmail relay dependency diagram
certs-nginx[("`TLS certs
/var/lib/acme`")] --> nginx-internal;
systemd-timer --- acmetool;
systemd-timer --- chatmail-expire-inactive;
systemd-timer --- chatmail-expire-daily;
systemd-timer --- chatmail-fsreport-daily;
acmetool --> certs[("`TLS certs
/var/lib/acme`")];
@@ -162,11 +156,9 @@ Chatmail relay dependency diagram
/home/vmail/.../user"];
dovecot --- |lastlogin.socket|lastlogin;
dovecot --- chatmail-metadata;
dovecot --- |quota-warning|chatmail-quota-expire;
chatmail-quota-expire --- maildir;
lastlogin --- maildir;
doveauth --- maildir;
chatmail-expire-inactive --- maildir;
chatmail-expire-daily --- maildir;
chatmail-fsreport-daily --- maildir;
chatmail-metadata --- iroh-relay;
chatmail-metadata --- |encrypted device token| notifications.delta.chat;