Compare commits

..

8 Commits

Author SHA1 Message Date
j4n d73a1baf51 fix(cmdeploy/dns): align zones, add multiline test records
And add requirements to lxc docs
2026-03-30 10:43:13 +02:00
holger krekel 386364ac70 refactor(cmdeploy/tests): don't use env vars but explicit pytest options to pass ssh info around.
Replace CHATMAIL_SSH env var with --ssh-host / --ssh-config pytest options.
Monkey-patch socket.getaddrinfo to resolve .localchat domains via
ssh-config mappings.  Add process cleanup to Remote, fix subprocess
stdin inheritance, and pass ssh_config through online tests.
2026-03-30 08:26:04 +02:00
holger krekel fc382b1062 refactor(cmdeploy): replace globals() subcommand scan with explicit SUBCOMMANDS list
The get_parser() loop that scanned globals() for *_cmd names was fragile
and forced # noqa: F401 on all lxc imports (ruff couldn't see they were
used dynamically).

Replace it with an explicit SUBCOMMANDS list of
(cmd_func, options_func, needs_config) tuples.  This makes the full set
of subcommands visible at a glance, their registration order defined,
and the imports unconditionally used (no more noqa suppressions).

Also add --ssh-config option to run/dns/status/test so all SSH
resolution can go through a config file, used by lxc-test for
completely local setups.
2026-03-30 08:25:50 +02:00
holger krekel bed80c119e feat(cmdeploy): add --ssh-config support to sshexec
SSHExec accepts ssh_config parameter, passed through to execnet.
New resolve_host_from_ssh_config() / resolve_key_from_ssh_config()
work around paramiko's silent failures with custom ssh configs.
2026-03-30 08:24:18 +02:00
holger krekel 9a03588e16 feat(lxc): add LXC container support for local chatmail development
Add cmdeploy "lxc-test" command to run cmdeploy against local containers,
with supplementary lxc-start, lxc-stop and lxc-status subcommands.
See doc/source/lxc.rst for full documentation including prerequisites,
DNS setup, TLS handling, DNS-free testing, and known limitations.
2026-03-30 08:24:11 +02:00
holger krekel 44b1cef7d2 fix(cmdeploy): deployer fixes for container compatibility
- UnboundDeployer: use blocked_service_startup() instead of manual policy-rc.d
- ChatmailDeployer: accept full Config object, ensure mailboxes_dir exists
- DovecotDeployer: replace CHATMAIL_NOSYSCTL env var with systemd-detect-virt -c
- SelfSignedTlsDeployer: add basicConstraints=critical,CA:FALSE
- WebsiteDeployer: use stable files.sync instead of experimental files.rsync
- GithashDeployer: use util.get_version_string()
- TurnDeployer: update to chatmail-turn v0.4
2026-03-30 08:23:47 +02:00
holger krekel dbd92a6b26 refactor(cmdeploy): unify zone-file handling to use actual BIND format
Remove chatmail.zone.j2 and build DNS records directly in dns.py
using standard BIND format (name TTL IN type rdata) as it is easy
enough to parse back.  Add parse_zone_records() for consuming the
new format, update rdns.py check_zonefile() and test data accordingly.
2026-03-30 08:23:22 +02:00
holger krekel 2ba13610bf refactor(cmdeploy): add Out class with --verbose, section timing, and coloured shell output
Move the Out output-printer class to cmdeploy/util.py so it is shared
across CLI modules.  All print/shell calls in lxc/cli.py, lxc/incus.py,
and dns.py now route through Out instead of bare print().

Key additions:
- Out.section() / Out.section_line(): coloured section headers scaled
  to the current terminal width (or $_CMDEPLOY_WIDTH for sub-processes).
- Out.shell(): merges stdout/stderr, prefixes each output line, and
  prints a red error line with the exit code on failure.
- Out.new_prefixed_out(): indented sub-printer that shares section_timings.
- 'cmdeploy -v / -vv' exposes the verbosity levels.
- shell(), collapse(), get_git_hash() and get_version_string() helpers.
- Tests for Out added to test_util.py.

Also update chatmaild MockOut fixture to match the new Out API.
2026-03-30 08:22:30 +02:00
129 changed files with 3701 additions and 12171 deletions
-15
View File
@@ -1,15 +0,0 @@
version: 2
updates:
- package-ecosystem: "cargo"
directory: "/filtermail"
schedule:
interval: "monthly"
cooldown:
default-days: 7
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
cooldown:
default-days: 7
-40
View File
@@ -1,40 +0,0 @@
name: No-DNS
on:
# Triggers when a PR is merged into main or a direct push occurs
push:
branches: [ "main" ]
# Triggers for any PR (and its subsequent commits) targeting the main branch
pull_request:
branches: [ "main" ]
permissions: {}
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
no-dns:
name: LXC deploy and test
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
with:
cmlxc_version: main
cmlxc_commands: |
cmlxc init
# single cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo --type ipv4 cm0
cmlxc -v test-cmdeploy cm0
# cross cmdeploy relay test (two ipv4 relays)
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only --type ipv4 cm1
cmlxc -v test-cmdeploy cm0 cm1
# cross cmdeploy/madmail relay tests
cmlxc -v deploy-madmail mad0
cmlxc -v test-cmdeploy cm0 mad0
cmlxc -v test-mini mad0 cm0
cmlxc -v test-mini cm0 mad0
+6 -43
View File
@@ -1,35 +1,21 @@
name: CI
on:
# Triggers when a PR is merged into main or a direct push occurs
push:
branches: [ "main" ]
# Triggers for any PR (and its subsequent commits) targeting the main branch
pull_request:
branches: [ "main" ]
permissions: {}
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
push:
jobs:
tox:
name: isolated chatmaild tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
# Checkout pull request HEAD commit instead of merge commit
# Otherwise `test_deployed_state` will be unhappy.
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- name: download filtermail
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.7.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
- name: run chatmaild tests
working-directory: chatmaild
run: pipx run tox
@@ -38,10 +24,7 @@ jobs:
name: deploy-chatmail tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- uses: actions/checkout@v4
- name: initenv
run: scripts/initenv.sh
@@ -55,25 +38,5 @@ jobs:
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
lxc-test:
name: LXC deploy and test
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
with:
cmlxc_version: main
cmlxc_commands: |
cmlxc init
# single cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo cm0
cmlxc -v test-mini cm0
cmlxc -v test-cmdeploy cm0
# cross cmdeploy relay test
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
cmlxc -v test-cmdeploy cm0 cm1
# cross cmdeploy/madmail relay tests
cmlxc -v deploy-madmail mad0
cmlxc -v test-cmdeploy cm0 mad0
cmlxc -v test-mini cm0 mad0
cmlxc -v test-mini mad0 cm0
# all other cmdeploy commands require a staging server
# see https://github.com/deltachat/chatmail/issues/100
-38
View File
@@ -1,38 +0,0 @@
# Notify the docker repo to build and test a new image after relay CI passes.
#
# Sends a repository_dispatch event to chatmail/docker with the relay ref
# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
# and run integration tests via cmlxc.
name: Trigger Docker build
on:
push:
branches: [main]
tags: ['[0-9]+.[0-9]+.[0-9]+']
workflow_dispatch:
permissions: {}
jobs:
dispatch:
name: Dispatch build to chatmail/docker
runs-on: ubuntu-latest
if: github.repository == 'chatmail/relay'
steps:
- name: Compute short SHA
id: sha
run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
- name: Send repository_dispatch
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
with:
token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
repository: chatmail/docker
event-type: relay-updated
client-payload: >-
{
"relay_ref": "${{ github.ref_name }}",
"relay_sha": "${{ github.sha }}",
"relay_sha_short": "${{ steps.sha.outputs.short }}"
}
+3 -11
View File
@@ -7,8 +7,6 @@ on:
- 'scripts/build-docs.sh'
- '.github/workflows/docs-preview.yaml'
permissions: {}
jobs:
scripts:
name: build
@@ -18,8 +16,6 @@ jobs:
url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: initenv
run: scripts/initenv.sh
@@ -38,22 +34,18 @@ jobs:
- name: Get Pullrequest ID
id: prepare
run: |
export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
- run: |
echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
env:
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
- name: Upload preview
run: |
mkdir -p "$HOME/.ssh"
echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
chmod 600 "$HOME/.ssh/key"
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
env:
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
- name: check links
working-directory: doc
-4
View File
@@ -10,8 +10,6 @@ on:
- 'scripts/build-docs.sh'
- '.github/workflows/docs.yaml'
permissions: {}
jobs:
scripts:
name: build
@@ -21,8 +19,6 @@ jobs:
url: https://chatmail.at/doc/relay/
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: initenv
run: scripts/initenv.sh
-107
View File
@@ -1,107 +0,0 @@
name: CI
permissions: {}
on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
fmt:
name: Check Formatting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master
with:
toolchain: nightly
components: rustfmt
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
- uses: taiki-e/install-action@2ca9b94c269419b7b0c711c09d0b21c4e1d51145 # v2.83.1
with:
tool: taplo-cli
- run: cd filtermail && taplo format --check && cargo fmt --check
cargo-machete:
name: Check Unused Dependencies
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: bnjbvr/cargo-machete@ac30a525c0a8d163a92d727b3ff079ee3f6ecb08 # v0.9.2
lint-clippy:
name: Check Clippy
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
toolchain: ["stable", "beta"]
continue-on-error: ${{ matrix.toolchain == 'beta' }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master
with:
toolchain: ${{ matrix.toolchain }}
components: clippy
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
- run: cd filtermail && cargo clippy
env:
RUSTUP_TOOLCHAIN: ${{ matrix.toolchain }}
rust-test:
name: Run rust tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master
with:
toolchain: stable
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
- run: cd filtermail && cargo test
relay-test:
name: Test the relay with cmlxc and custom filtermail
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
with:
cmlxc_commands: |
sudo apt-get update && sudo apt-get install -y musl-tools
rustup target add x86_64-unknown-linux-musl
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc
cd repo/filtermail && RUSTFLAGS="-Ctarget-feature=+crt-static -Clink-self-contained=yes" cargo build --release --target x86_64-unknown-linux-musl
cmlxc init
cmlxc -v deploy-cmdeploy --filtermail repo/filtermail/target/x86_64-unknown-linux-musl/release/filtermail cm0
cmlxc -v deploy-cmdeploy --filtermail repo/filtermail/target/x86_64-unknown-linux-musl/release/filtermail --type ipv4 cm1
cmlxc -v deploy-madmail mad0
# single-relay tests
cmlxc -v test-cmdeploy cm0
cmlxc -v test-cmdeploy cm1
cmlxc -v test-mini cm0
cmlxc -v test-mini cm1
# cross-relay tests (Standard <-> IP-only <-> Madmail)
cmlxc -v test-cmdeploy cm0 cm1
cmlxc -v test-cmdeploy cm1 cm0
cmlxc -v test-cmdeploy cm0 mad0
cmlxc -v test-cmdeploy cm1 mad0
cmlxc -v test-mini cm0 mad0
cmlxc -v test-mini mad0 cm0
cmlxc -v test-mini cm1 mad0
cmlxc -v test-mini mad0 cm1
-69
View File
@@ -1,69 +0,0 @@
on:
tags:
- 'filtermail-*'
jobs:
build:
permissions: {}
name: Build binaries.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master
with:
toolchain: stable
targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl
- uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1
- uses: taiki-e/install-action@2ca9b94c269419b7b0c711c09d0b21c4e1d51145 # v2.83.1
with:
tool: cargo-zigbuild
- name: Build
run: |
RUSTFLAGS="-Ctarget-feature=+crt-static -Clink-self-contained=yes" \
cd filtermail && cargo zigbuild \
--release \
--target x86_64-unknown-linux-musl \
--target aarch64-unknown-linux-musl
- name: Upload x86_64 binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: filtermail-x86_64
path: target/x86_64-unknown-linux-musl/release/filtermail
if-no-files-found: error
- name: Upload aarch64 binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: filtermail-aarch64
path: target/aarch64-unknown-linux-musl/release/filtermail
if-no-files-found: error
publish:
name: Upload binaries to the release
needs: ["build"]
runs-on: ubuntu-latest
steps:
- name: Download aarch64 binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: filtermail-x86_64
path: filtermail-x86_64
- name: Download x86_64 binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: filtermail-aarch64
path: filtermail-aarch64
- name: Upload binaries to the GitHub release
if: github.event_name == 'release'
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
REF_NAME: ${{ github.ref_name }}
run: |
mkdir dist
mv filtermail-x86_64/filtermail dist/filtermail-x86_64
mv filtermail-aarch64/filtermail dist/filtermail-aarch64
gh release upload "$REF_NAME" \
--repo ${{ github.repository }} \
dist/*
@@ -0,0 +1,104 @@
name: deploy on staging-ipv4.testrun.org, and run tests
on:
push:
branches:
- main
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
deploy:
name: deploy on staging-ipv4.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency: staging-ipv4.testrun.org
steps:
- uses: actions/checkout@v4
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- name: setup dependencies
run: |
ssh root@staging-ipv4.testrun.org apt update
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
- name: initialize config
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
- name: set DNS entries
run: |
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
cat .github/workflows/staging-ipv4.testrun.org-default.zone
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
- name: cmdeploy dns
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
+97
View File
@@ -0,0 +1,97 @@
name: deploy on staging2.testrun.org, and run tests
on:
push:
branches:
- main
pull_request:
paths-ignore:
- 'scripts/**'
- '**/README.md'
- 'CHANGELOG.md'
- 'LICENSE'
jobs:
deploy:
name: deploy on staging2.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency: staging2.testrun.org
steps:
- uses: actions/checkout@v4
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
# save previous acme & dkim state
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
# make sure CAA record isn't set
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: rebuild staging2.testrun.org to have a clean VPS
run: |
curl -X POST \
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-H "Content-Type: application/json" \
-d '{"image":"debian-12"}' \
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: upload TLS cert after rebuilding
run: |
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
rm ~/.ssh/known_hosts
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
# download acme & dkim state from ns.testrun.org
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
# restore acme & dkim state to staging2.testrun.org
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: add hpk42 key to staging server
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
- run: |
cmdeploy init staging2.testrun.org
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
- run: cmdeploy run --verbose --skip-dns-check
- name: set DNS entries
run: |
cmdeploy dns --zonefile staging-generated.zone --verbose
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
cat .github/workflows/staging.testrun.org-default.zone
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
- name: cmdeploy dns
run: cmdeploy dns -v
-26
View File
@@ -1,26 +0,0 @@
name: GitHub Actions Security Analysis with zizmor
on:
push:
branches: ["main"]
pull_request:
branches: ["**"]
permissions: {}
jobs:
zizmor:
name: Run zizmor
runs-on: ubuntu-latest
permissions:
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Run zizmor
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
-7
View File
@@ -1,7 +0,0 @@
rules:
unpinned-uses:
config:
policies:
actions/*: ref-pin
dependabot/*: ref-pin
chatmail/*: ref-pin
+1 -1
View File
@@ -5,6 +5,7 @@ __pycache__/
*.swp
*qr-*.png
chatmail*.ini
lxconfigs/
# C extensions
@@ -78,7 +79,6 @@ docs/_build/
# PyBuilder
.pybuilder/
target/
filtermail/target/
# Jupyter Notebook
.ipynb_checkpoints
+1 -127
View File
@@ -1,130 +1,4 @@
# Changelog for chatmail deployment
## [1.11.0] - 2026-05-15
### Breaking Changes
- [**breaking**] Drop passthrough_sender and passthrough_recipients chatmail.ini options to eliminate one more source of unencrypted messages
### Features
- Use filtermail for delivery to remote MTAs
- Expose metadata "maxsmtprecipients" value
- Support setup without domain, with only an IPv4 address (#963)
- *(doc/docker)* Introduce docker images in documentation
- DKIM-sign bounce messages (mainly "user does not exist")
- *(config)* Load default values from Config(), not chatmail.ini.f (#853)
- Make turn_socket_path configurable, and cleanup tests and turnserver code.
- Warn about any unused chatmail.ini parameter at the end of "cmdeploy run"
### Bug Fixes
- Make www tests work with editable instead of just plain installs
- Use path with no leading slash for mxdeliv
- Increase filtermail-transport concurrency limit
- Fix #972 by increasing file descriptors for filtermail
- *(mtail)* Correct boot ordering and deploy restart logic
- *(cmdeploy)* Stop and disable unbound-resolvconf
- *(nginx)* Properly redirect www to mail_domain
- *(dns)* Query correct NS if MNAME server is hidden (#954)
- Legacy token metadata storage used list type, but if no new setmetadata happened, the user would not be notified at all.
- *(logging)* Log all http requests to syslog
### Documentation
- Document how to upgrade to new version (#965)
### Other
- *(deps)* Upgrade to filtermail v0.6.4
### Refactor
- Introduce automated change-tracking across deployers
## 1.10.0 2026-04-30
* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
* stop installing cron package <https://github.com/chatmail/relay/pull/898>
* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
* dovecot update url <https://github.com/chatmail/relay/pull/880>
* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
* try to use "build machine" and "deployment server" consistently <https://github.com/chatmail/relay/pull/797>
* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
* streamline migration guide wording, provide titled steps <https://github.com/chatmail/relay/pull/789>
* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
# Changelog for chatmail deployment
## 1.9.0 2025-12-18
+1 -6
View File
@@ -8,12 +8,7 @@ Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed f
- **Instant/Realtime:** sub-second message delivery, realtime P2P
streaming, privacy-preserving Push Notifications for Apple, Google, and Huawei;
- **Security Enforcement**: Only connections with strict TLS are accepted;
all messages must be correctly signed with DKIM and OpenPGP-encrypted with minimized metadata.
There are experimental exceptions for no-DNS relays,
which are allowed use self-signed TLS certificates
and which do not need to DKIM-sign their messages.
Unencrypted messages are allowed in neither case.
- **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
- **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
depends on established IETF standards and protocols.
+3 -3
View File
@@ -9,7 +9,7 @@ dependencies = [
"iniconfig",
"filelock",
"requests",
"crypt-r >= 3.13.1 ; python_version >= '3.13'",
"crypt-r >= 3.13.1 ; python_version >= '3.11'",
]
[tool.setuptools]
@@ -21,10 +21,10 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
chatmail-expire = "chatmaild.expire:daily_expire_main"
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
chatmail-expire = "chatmaild.expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
lastlogin = "chatmaild.lastlogin:main"
turnserver = "chatmaild.turnserver:main"
[project.entry-points.pytest11]
"chatmaild.testplugin" = "chatmaild.tests.plugin"
+68 -84
View File
@@ -1,4 +1,4 @@
import ipaddress
import os
from pathlib import Path
import iniconfig
@@ -9,78 +9,63 @@ from chatmaild.user import User
def read_config(inipath):
assert Path(inipath).exists(), inipath
cfg = iniconfig.IniConfig(inipath)
return Config(inipath, params=cfg.sections["params"])
params = cfg.sections["params"]
default_config_content = get_default_config_content(params["mail_domain"])
df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
new_params = dict(df_params.items())
new_params.update(params)
return Config(inipath, params=new_params)
class Config:
def __init__(self, inipath, params):
self._inipath = inipath
params = dict(params)
raw_domain = params.pop("mail_domain")
self.mail_domain_bare = raw_domain
if is_valid_ipv4(raw_domain):
self.ipv4_relay = raw_domain
self.mail_domain = f"[{raw_domain}]"
self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
else:
self.ipv4_relay = None
self.mail_domain = raw_domain
self.postfix_myhostname = raw_domain
self.max_user_send_per_minute = int(params.pop("max_user_send_per_minute", 60))
self.max_user_send_burst_size = int(params.pop("max_user_send_burst_size", 10))
self.max_mailbox_size = params.pop("max_mailbox_size", "500M")
self.max_message_size = int(params.pop("max_message_size", 31457280))
self.delete_mails_after = params.pop("delete_mails_after", "20")
self.delete_large_after = params.pop("delete_large_after", "7")
self.delete_inactive_users_after = int(
params.pop("delete_inactive_users_after", 90)
)
self.username_min_length = int(params.pop("username_min_length", 9))
self.username_max_length = int(params.pop("username_max_length", 9))
self.password_min_length = int(params.pop("password_min_length", 9))
self.www_folder = params.pop("www_folder", "")
self.filtermail_smtp_port = int(params.pop("filtermail_smtp_port", "10080"))
self.mail_domain = params["mail_domain"]
self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
self.max_mailbox_size = params["max_mailbox_size"]
self.max_message_size = int(params.get("max_message_size", "31457280"))
self.delete_mails_after = params["delete_mails_after"]
self.delete_large_after = params["delete_large_after"]
self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
self.username_min_length = int(params["username_min_length"])
self.username_max_length = int(params["username_max_length"])
self.password_min_length = int(params["password_min_length"])
self.passthrough_senders = params["passthrough_senders"].split()
self.passthrough_recipients = params["passthrough_recipients"].split()
self.www_folder = params.get("www_folder", "")
self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
self.filtermail_smtp_port_incoming = int(
params.pop("filtermail_smtp_port_incoming", "10081")
params.get("filtermail_smtp_port_incoming", "10081")
)
self.filtermail_http_port_incoming = int(
params.pop("filtermail_http_port_incoming", "10082")
)
self.filtermail_lmtp_port_transport = int(
params.pop("filtermail_lmtp_port_transport", "10083")
)
self.postfix_reinject_port = int(params.pop("postfix_reinject_port", "10025"))
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
self.postfix_reinject_port_incoming = int(
params.pop("postfix_reinject_port_incoming", "10026")
params.get("postfix_reinject_port_incoming", "10026")
)
self.mtail_address = params.pop("mtail_address", None)
self.disable_ipv6 = params.pop("disable_ipv6", "false").lower() == "true"
self.acme_email = params.pop("acme_email", "")
self.imap_rawlog = params.pop("imap_rawlog", "false").lower() == "true"
self.imap_compress = params.pop("imap_compress", "false").lower() == "true"
self.turn_socket_path = params.pop(
"turn_socket_path", "/run/chatmail-turn/turn.socket"
)
iroh_relay = params.pop("iroh_relay", None)
if iroh_relay is None:
self.iroh_relay = "https://" + raw_domain
self.mtail_address = params.get("mtail_address")
self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
self.acme_email = params.get("acme_email", "")
self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
self.imap_compress = params.get("imap_compress", "false").lower() == "true"
if "iroh_relay" not in params:
self.iroh_relay = "https://" + params["mail_domain"]
self.enable_iroh_relay = True
else:
self.iroh_relay = iroh_relay.strip()
self.iroh_relay = params["iroh_relay"].strip()
self.enable_iroh_relay = False
self.privacy_postal = params.pop("privacy_postal", None)
self.privacy_mail = params.pop("privacy_mail", None)
self.privacy_pdo = params.pop("privacy_pdo", None)
self.privacy_supervisor = params.pop("privacy_supervisor", None)
self.privacy_postal = params.get("privacy_postal")
self.privacy_mail = params.get("privacy_mail")
self.privacy_pdo = params.get("privacy_pdo")
self.privacy_supervisor = params.get("privacy_supervisor")
# TLS certificate management.
# If tls_external_cert_and_key is set, use externally managed certs.
# Otherwise derived from the domain name:
# - Domains starting with "_" use self-signed certificates
# - All other domains use ACME.
external = params.pop("tls_external_cert_and_key", "").strip()
external = params.get("tls_external_cert_and_key", "").strip()
if external:
parts = external.split()
@@ -91,27 +76,21 @@ class Config:
)
self.tls_cert_mode = "external"
self.tls_cert_path, self.tls_key_path = parts
elif raw_domain.startswith("_") or self.ipv4_relay:
elif self.mail_domain.startswith("_"):
self.tls_cert_mode = "self"
self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
self.tls_key_path = "/etc/ssl/private/mailserver.key"
else:
self.tls_cert_mode = "acme"
self.tls_cert_path = f"/var/lib/acme/live/{raw_domain}/fullchain"
self.tls_key_path = f"/var/lib/acme/live/{raw_domain}/privkey"
self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
# deprecated option
mbdir = params.pop("mailboxes_dir", f"/home/vmail/mail/{raw_domain}")
mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
self.mailboxes_dir = Path(mbdir.strip())
# old unused option (except for first migration from sqlite to maildir store)
self.passdb_path = Path(params.pop("passdb_path", "/home/vmail/passdb.sqlite"))
self._unused_keys = list(params)
@property
def max_mailbox_size_mb(self):
"""Return max_mailbox_size as an integer in megabytes."""
return parse_size_mb(self.max_mailbox_size)
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
def _getbytefile(self):
return open(self._inipath, "rb")
@@ -126,16 +105,6 @@ class Config:
return User(maildir, addr, password_path, uid="vmail", gid="vmail")
def parse_size_mb(limit):
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
value = limit.strip().upper().removesuffix("B")
if value.endswith("G"):
return int(value[:-1]) * 1024
if value.endswith("M"):
return int(value[:-1])
return int(value)
def write_initial_config(inipath, mail_domain, overrides):
"""Write out default config file, using the specified config value overrides."""
content = get_default_config_content(mail_domain, **overrides)
@@ -163,13 +132,28 @@ def get_default_config_content(mail_domain, **overrides):
for name, value in extra.items():
new_line = f"{name} = {value}"
new_lines.append(new_line)
return "\n".join(new_lines)
content = "\n".join(new_lines)
def is_valid_ipv4(address: str) -> bool:
"""Check if a mail_domain is an IPv4 address."""
try:
ipaddress.IPv4Address(address)
return True
except ValueError:
return False
# apply testrun privacy overrides
if mail_domain.endswith(".testrun.org"):
override_inipath = inidir.joinpath("override-testrun.ini")
privacy = iniconfig.IniConfig(override_inipath)["privacy"]
lines = []
for line in content.split("\n"):
for key, value in privacy.items():
value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
if not line.startswith(f"{key} =") or not value_lines:
continue
if len(value_lines) == 1:
lines.append(f"{key} = {value}")
else:
lines.append(f"{key} =")
for vl in value_lines:
lines.append(f" {vl}")
break
else:
lines.append(line)
content = "\n".join(lines)
return content
+4 -100
View File
@@ -4,26 +4,17 @@ Expire old messages and addresses.
"""
import os
import re
import shutil
import sys
import time
from argparse import ArgumentParser
from collections import namedtuple
from datetime import datetime
from pathlib import Path
from stat import S_ISREG
from chatmaild.config import read_config
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
QUOTA_CLEANUP_FACTOR = 0.7
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
def iter_mailboxes(basedir, maxnum):
@@ -83,42 +74,6 @@ class MailboxStat:
self.extrafiles.sort(key=lambda x: -x.size)
def parse_dovecot_filename(relpath):
m = _dovecot_fn_rex.match(relpath)
if not m:
return None
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
def scan_mailbox_messages(mbox):
messages = []
for sub in ("cur", "new"):
for name in os_listdir_if_exists(mbox / sub):
if entry := parse_dovecot_filename(f"{sub}/{name}"):
messages.append(entry)
return messages
def expire_to_target(mbox, target_bytes):
messages = scan_mailbox_messages(mbox)
total_size = sum(m.quota_size for m in messages)
# Keep recent 24 hours of messages protected from expiry because
# likely something is wrong with interactions on that address
# and quota-full signal can help the address owner's device to notice it
undeletable_messages_cutoff = time.time() - (3600 * 24)
removed = 0
for entry in sorted(messages):
if total_size <= target_bytes:
break
if entry.mtime > undeletable_messages_cutoff:
break
(mbox / entry.path).unlink(missing_ok=True)
total_size -= entry.quota_size
removed += 1
return removed
def print_info(msg):
print(msg, file=sys.stderr)
@@ -168,16 +123,6 @@ class Expiry:
if mbox.last_login and mbox.last_login < cutoff_without_login:
self.remove_mailbox(mbox.basedir)
return
elif mbox.last_login is None:
try:
if not self.dry:
os.rmdir(mbox.basedir)
self.del_mboxes += 1
except OSError:
print_info(
f"Skipped deleting {mbox.basedir}, doesn't have last_login but isn't empty"
)
return
mboxname = os.path.basename(mbox.basedir)
if self.verbose:
@@ -198,19 +143,6 @@ class Expiry:
else:
continue
changed = True
target_bytes = (
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
)
removed = expire_to_target(Path(mbox.basedir), target_bytes)
if removed:
changed = True
self.del_files += removed
if self.verbose:
print_info(
f"quota-expire: removed {removed} message(s) from {mboxname}"
)
if changed:
self.remove_file(f"{mbox.basedir}/maildirsize")
@@ -222,9 +154,9 @@ class Expiry:
)
def daily_expire_main(args=None):
def main(args=None):
"""Expire mailboxes and messages according to chatmail config"""
parser = ArgumentParser(description=daily_expire_main.__doc__)
parser = ArgumentParser(description=main.__doc__)
ini = "/usr/local/lib/chatmaild/chatmail.ini"
parser.add_argument(
"chatmail_ini",
@@ -270,33 +202,5 @@ def daily_expire_main(args=None):
print(exp.get_summary())
def quota_expire_main(args=None):
"""Remove mailbox messages to stay within a megabyte target.
This entry point is called by dovecot when a quota threshold is passed.
"""
parser = ArgumentParser(description=quota_expire_main.__doc__)
parser.add_argument(
"target_mb",
type=int,
help="target mailbox size in megabytes",
)
parser.add_argument(
"mailbox_path",
type=Path,
help="path to a user mailbox",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
removed_count = expire_to_target(args.mailbox_path, target_bytes)
if removed_count:
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
print(
f"quota-expire: removed {removed_count} message(s)"
f" from {args.mailbox_path.name}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
main(sys.argv[1:])
+29 -15
View File
@@ -12,35 +12,41 @@ mail_domain = {mail_domain}
#
# email sending rate per user and minute
#max_user_send_per_minute = 60
max_user_send_per_minute = 60
# per-user max burst size for sending rate limiting (GCRA bucket capacity)
#max_user_send_burst_size = 10
max_user_send_burst_size = 10
# maximum mailbox size of a chatmail address
# (Oldest messages will be removed automatically, so mailboxes never run full)
#max_mailbox_size = 500M
max_mailbox_size = 500M
# maximum message size for an e-mail in bytes
#max_message_size = 31457280
max_message_size = 31457280
# days after which mails are unconditionally deleted
#delete_mails_after = 20
delete_mails_after = 20
# days after which large messages (>200k) are unconditionally deleted
#delete_large_after = 7
delete_large_after = 7
# days after which users without a successful login are deleted (database and mails)
#delete_inactive_users_after = 90
delete_inactive_users_after = 90
# minimum length a username must have
#username_min_length = 9
username_min_length = 9
# maximum length a username can have
#username_max_length = 9
username_max_length = 9
# minimum length a password must have
#password_min_length = 9
password_min_length = 9
# list of chatmail addresses which can send outbound un-encrypted mail
passthrough_senders =
# list of e-mail recipients for which to accept outbound un-encrypted mails
# (space-separated, item may start with "@" to whitelist whole recipient domains)
passthrough_recipients =
# Use externally managed TLS certificates instead of built-in acmetool.
# Paths refer to files on the deployment server (not the build machine).
@@ -56,11 +62,19 @@ mail_domain = {mail_domain}
# Deployment Details
#
# SMTP outgoing filtermail and reinjection
filtermail_smtp_port = 10080
postfix_reinject_port = 10025
# SMTP incoming filtermail and reinjection
filtermail_smtp_port_incoming = 10081
postfix_reinject_port_incoming = 10026
# if set to "True" IPv6 is disabled
#disable_ipv6 = False
disable_ipv6 = False
# Your email adress, which will be used in acmetool to manage Let's Encrypt SSL certificates
#acme_email =
acme_email =
# Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
# service.
@@ -93,13 +107,13 @@ mail_domain = {mail_domain}
# in per-maildir ".in/.out" files.
# Note that you need to manually cleanup these files
# so use this option with caution on production servers.
#imap_rawlog = false
imap_rawlog = false
# set to true if you want to enable the IMAP COMPRESS Extension,
# which allows IMAP connections to be efficiently compressed.
# WARNING: Enabling this makes it impossible to hibernate IMAP
# processes which will result in much higher memory/RAM usage.
#imap_compress = false
imap_compress = false
#
@@ -0,0 +1,16 @@
[privacy]
passthrough_recipients = privacy@testrun.org echo@{mail_domain}
privacy_postal =
Merlinux GmbH, Represented by the managing director H. Krekel,
Reichgrafen Str. 20, 79102 Freiburg, Germany
privacy_mail = privacy@testrun.org
privacy_pdo =
Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover.
You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO)
privacy_supervisor =
State Commissioner for Data Protection and Freedom of Information of
Baden-Württemberg in 70173 Stuttgart, Germany.
+29 -49
View File
@@ -1,5 +1,4 @@
import logging
import socket
import sys
import time
from contextlib import contextmanager
@@ -8,14 +7,7 @@ from .config import read_config
from .dictproxy import DictProxy
from .filedict import FileDict
from .notifier import Notifier
def turn_credentials(turn_socket_path):
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
client_socket.settimeout(5)
client_socket.connect(turn_socket_path)
with client_socket.makefile("rb") as file:
return file.readline().decode("utf-8").strip()
from .turnserver import turn_credentials
def _is_valid_token_timestamp(timestamp, now):
@@ -78,53 +70,44 @@ class Metadata:
# Some tokens have expired, remove them.
with self._modify_tokens(addr) as _tokens:
pass
elif isinstance(tokens, list):
with self._modify_tokens(addr) as tokens:
token_list = list(tokens.keys())
else:
token_list = []
return token_list
class MetadataDictProxy(DictProxy):
def __init__(
self,
notifier,
metadata,
iroh_relay=None,
turn_hostname=None,
turn_socket_path=None,
):
def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None):
super().__init__()
self.notifier = notifier
self.metadata = metadata
self.iroh_relay = iroh_relay
self.turn_hostname = turn_hostname
self.turn_socket_path = turn_socket_path
def handle_lookup(self, parts):
# Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
match parts[0].split("/", 2):
case ["priv", _, keyname] if keyname == self.metadata.DEVICETOKEN_KEY:
addr = parts[1]
keyparts = parts[0].split("/", 2)
if keyparts[0] == "priv":
keyname = keyparts[2]
addr = parts[1]
if keyname == self.metadata.DEVICETOKEN_KEY:
res = " ".join(self.metadata.get_tokens_for_addr(addr))
return f"O{res}\n"
case ["shared", _, keyname]:
prefix = "vendor/vendor.dovecot/pvt/server/vendor/deltachat/"
if keyname.startswith(prefix):
match keyname[len(prefix) :]:
case "irohrelay" if self.iroh_relay:
return f"O{self.iroh_relay}\n"
case "turn":
try:
res = turn_credentials(self.turn_socket_path)
except Exception:
logging.exception("failed to get TURN credentials")
return "N\n"
return f"O{self.turn_hostname}:3478:{res}\n"
case "maxsmtprecipients":
# postfix default (see "postconf smtpd_recipient_limit")
return "O1000\n"
elif keyparts[0] == "shared":
keyname = keyparts[2]
if (
keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
and self.iroh_relay
):
# Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
return f"O{self.iroh_relay}\n"
elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
try:
res = turn_credentials()
except Exception:
logging.exception("failed to get TURN credentials")
return "N\n"
port = 3478
return f"O{self.turn_hostname}:{port}:{res}\n"
logging.warning(f"lookup ignored: {parts!r}")
return "N\n"
@@ -134,13 +117,12 @@ class MetadataDictProxy(DictProxy):
# https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
keyname = parts[1].split("/")
value = parts[2] if len(parts) > 2 else ""
match keyname:
case ["priv", _, key] if key == self.metadata.DEVICETOKEN_KEY:
self.metadata.add_token_to_addr(addr, value)
return True
case ["priv", _, "messagenew"]:
self.notifier.new_message_for_addr(addr, self.metadata)
return True
if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
self.metadata.add_token_to_addr(addr, value)
return True
elif keyname[0] == "priv" and keyname[2] == "messagenew":
self.notifier.new_message_for_addr(addr, self.metadata)
return True
return False
@@ -151,7 +133,6 @@ def main():
config = read_config(config_path)
iroh_relay = config.iroh_relay
mail_domain = config.mail_domain
socket_path = config.turn_socket_path
vmail_dir = config.mailboxes_dir
if not vmail_dir.exists():
@@ -169,7 +150,6 @@ def main():
metadata=metadata,
iroh_relay=iroh_relay,
turn_hostname=mail_domain,
turn_socket_path=socket_path,
)
dictproxy.serve_forever_from_socket(socket)
+3 -11
View File
@@ -25,19 +25,13 @@ def create_newemail_dict(config: Config):
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
def create_dclogin_url(config, email, password):
def create_dclogin_url(email, password):
"""Build a dclogin: URL with credentials and self-signed cert acceptance.
Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
can connect to servers with self-signed TLS certificates.
"""
if config.ipv4_relay:
imap_host = "&ih=" + config.ipv4_relay
smtp_host = "&sh=" + config.ipv4_relay
else:
imap_host = ""
smtp_host = ""
return f"dclogin:{quote(email, safe='@[]')}?p={quote(password, safe='')}&v=1{imap_host}{smtp_host}&ic=3"
return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
def print_new_account():
@@ -46,9 +40,7 @@ def print_new_account():
result = dict(email=creds["email"], password=creds["password"])
if config.tls_cert_mode == "self":
result["dclogin_url"] = create_dclogin_url(
config, creds["email"], creds["password"]
)
result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
print("Content-Type: application/json")
print("")
+3 -8
View File
@@ -31,11 +31,6 @@ def example_config(make_config):
return make_config("chat.example.org")
@pytest.fixture
def ipv4_config(make_config):
return make_config("1.3.3.7")
@pytest.fixture
def maildomain(example_config):
return example_config.mail_domain
@@ -90,13 +85,13 @@ def mockout():
captured_green = []
captured_plain = []
def red(self, msg):
def red(self, msg, **kw):
self.captured_red.append(msg)
def green(self, msg):
def green(self, msg, **kw):
self.captured_green.append(msg)
def __call__(self, msg):
def print(self, msg="", **kw):
self.captured_plain.append(msg)
return MockOut()
+20 -58
View File
@@ -1,10 +1,6 @@
import pytest
from chatmaild.config import (
is_valid_ipv4,
parse_size_mb,
read_config,
)
from chatmaild.config import read_config
def test_read_config_basic(example_config):
@@ -13,21 +9,10 @@ def test_read_config_basic(example_config):
assert not example_config.privacy_pdo and not example_config.privacy_postal
inipath = example_config._inipath
inipath.write_text(
inipath.read_text().replace(
"#max_user_send_per_minute = 60",
"max_user_send_per_minute = 37",
)
)
inipath.write_text(inipath.read_text().replace("60", "37"))
example_config = read_config(inipath)
assert example_config.max_user_send_per_minute == 37
assert example_config.mail_domain == "chat.example.org"
assert example_config.ipv4_relay is None
def test_read_config_ipv4(ipv4_config):
assert ipv4_config.ipv4_relay == "1.3.3.7"
assert ipv4_config.mail_domain == "[1.3.3.7]"
def test_read_config_basic_using_defaults(tmp_path, maildomain):
@@ -36,21 +21,26 @@ def test_read_config_basic_using_defaults(tmp_path, maildomain):
example_config = read_config(inipath)
assert example_config.max_user_send_per_minute == 60
assert example_config.filtermail_smtp_port_incoming == 10081
assert example_config.filtermail_smtp_port == 10080
assert example_config.postfix_reinject_port == 10025
assert example_config.max_user_send_per_minute == 60
assert example_config.max_mailbox_size == "500M"
assert example_config.delete_mails_after == "20"
assert example_config.delete_large_after == "7"
assert example_config.username_min_length == 9
assert example_config.username_max_length == 9
assert example_config.password_min_length == 9
assert example_config._unused_keys == []
def test_config_unused_keys(make_config):
config = make_config("chat.example.org", {"passthrough_senders": "x@y.org"})
assert config._unused_keys == ["passthrough_senders"]
def test_read_config_testrun(make_config):
config = make_config("something.testrun.org")
assert config.mail_domain == "something.testrun.org"
assert len(config.privacy_postal.split("\n")) > 1
assert len(config.privacy_supervisor.split("\n")) > 1
assert len(config.privacy_pdo.split("\n")) > 1
assert config.privacy_mail == "privacy@testrun.org"
assert config.filtermail_smtp_port == 10080
assert config.postfix_reinject_port == 10025
assert config.max_user_send_per_minute == 60
assert config.max_mailbox_size == "500M"
assert config.delete_mails_after == "20"
assert config.delete_large_after == "7"
assert config.username_min_length == 9
assert config.username_max_length == 9
assert config.password_min_length == 9
assert "privacy@testrun.org" in config.passthrough_recipients
assert config.passthrough_senders == []
def test_config_userstate_paths(make_config, tmp_path):
@@ -131,31 +121,3 @@ def test_config_tls_external_bad_format(make_config):
"tls_external_cert_and_key": "/only/one/path.pem",
},
)
def test_parse_size_mb():
assert parse_size_mb("500M") == 500
assert parse_size_mb("2G") == 2048
assert parse_size_mb(" 1g ") == 1024
assert parse_size_mb("100MB") == 100
assert parse_size_mb("256") == 256
def test_max_mailbox_size_mb(make_config):
config = make_config("chat.example.org")
assert config.max_mailbox_size == "500M"
assert config.max_mailbox_size_mb == 500
@pytest.mark.parametrize(
["input", "result"],
[
("example.org", False),
("1.3.3.7", True),
("fe::1", False),
("ad.1e.dag.adf", False),
("12394142", False),
],
)
def test_is_valid_ipv4(input, result):
assert result == is_valid_ipv4(input)
@@ -1,7 +1,7 @@
import time
from chatmaild.doveauth import AuthDictProxy
from chatmaild.expire import daily_expire_main as main_expire
from chatmaild.expire import main as main_expire
def test_login_timestamps(example_config):
+1 -85
View File
@@ -1,8 +1,5 @@
import itertools
import os
import random
import shutil
import time
from datetime import datetime
from fnmatch import fnmatch
from pathlib import Path
@@ -10,22 +7,15 @@ from pathlib import Path
import pytest
from chatmaild.expire import (
Expiry,
FileEntry,
MailboxStat,
expire_to_target,
get_file_entry,
iter_mailboxes,
os_listdir_if_exists,
parse_dovecot_filename,
quota_expire_main,
scan_mailbox_messages,
)
from chatmaild.expire import daily_expire_main as expiry_main
from chatmaild.expire import main as expiry_main
from chatmaild.fsreport import main as report_main
MB = 1024 * 1024
def fill_mbox(folderdir):
password = folderdir.joinpath("password")
@@ -106,32 +96,6 @@ def test_stats_mailbox(mbox1):
assert mbox3.last_login is None
def test_mbox_without_password(mbox1, example_config, capsys):
password = Path(mbox1.basedir).joinpath("password")
os.remove(password)
mbox_rescan = MailboxStat(mbox1.basedir)
assert mbox_rescan.last_login is None
exp = Expiry(
example_config, dry=False, now=datetime.now().timestamp(), verbose=False
)
exp.process_mailbox_stat(mbox_rescan)
out, err = capsys.readouterr()
assert "doesn't have last_login but isn't empty" in err
assert os.path.isdir(mbox_rescan.basedir)
for entry in os.scandir(mbox_rescan.basedir):
if os.path.isdir(entry):
shutil.rmtree(entry)
else:
os.remove(entry)
exp.process_mailbox_stat(mbox_rescan)
out, err = capsys.readouterr()
assert "doesn't have last_login but isn't empty" not in err
assert not os.path.isdir(mbox_rescan.basedir)
def test_report_no_mailboxes(example_config):
args = (str(example_config._inipath),)
report_main(args)
@@ -232,51 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
tmp_path.joinpath("x").write_text("hello")
assert len(os_listdir_if_exists(str(tmp_path))) == 1
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
# --- quota expire tests ---
_msg_counter = itertools.count(1)
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
seq = next(_msg_counter)
mtime = int(time.time() - days_old * 86400)
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
path = basedir / sub / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
os.utime(path, (mtime, mtime))
return path
def test_parse_dovecot_filename():
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
assert e.mtime == 1775324677
assert e.quota_size == 3235
assert parse_dovecot_filename("cur/msg_without_structure") is None
def test_expire_to_target(tmp_path):
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
_create_message(tmp_path, "new", MB, days_old=5)
_create_message(tmp_path, "cur", MB, days_old=0) # undeletable (<1 hour)
assert len(scan_mailbox_messages(tmp_path)) == 3
# removes oldest first, uses S= size not disk size
removed = expire_to_target(tmp_path, MB)
assert removed == 2
msgs = scan_mailbox_messages(tmp_path)
assert len(msgs) == 1
# the surviving message is the fresh undeletable one
assert msgs[0].mtime > time.time() - 3600
def test_quota_expire_main(tmp_path, capsys):
mbox = tmp_path / "user@example.org"
_create_message(mbox, "cur", 2 * MB, days_old=5)
(mbox / "maildirsize").write_text("x")
quota_expire_main([str(1), str(mbox)])
_, err = capsys.readouterr()
assert "quota-expire: removed 1 message(s) from user@example.org" in err
assert not (mbox / "maildirsize").exists()
+11 -37
View File
@@ -324,7 +324,7 @@ def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
turn_hostname="turn.example.org",
)
def mock_turn_credentials(turn_socket_path):
def mock_turn_credentials():
raise ConnectionRefusedError("socket not available")
monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
@@ -348,9 +348,7 @@ def test_turn_credentials_success(notifier, metadata, monkeypatch):
turn_hostname="turn.example.org",
)
monkeypatch.setattr(
chatmaild.metadata, "turn_credentials", lambda path: "user:pass"
)
monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
transactions = {}
res = dictproxy.handle_dovecot_request(
@@ -362,39 +360,15 @@ def test_turn_credentials_success(notifier, metadata, monkeypatch):
def test_iroh_relay(dictproxy):
key = b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org"
rfile, wfile = io.BytesIO(b"H\n" + key), io.BytesIO()
rfile = io.BytesIO(
b"\n".join(
[
b"H",
b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
]
)
)
wfile = io.BytesIO()
dictproxy.iroh_relay = "https://example.org/"
dictproxy.loop_forever(rfile, wfile)
assert wfile.getvalue() == b"Ohttps://example.org/\n"
def test_legacy_token_migration(metadata, testaddr):
with metadata.get_metadata_dict(testaddr).modify() as data:
data[metadata.DEVICETOKEN_KEY] = ["oldtoken1", "oldtoken2"]
assert metadata.get_tokens_for_addr(testaddr) == ["oldtoken1", "oldtoken2"]
mdict = metadata.get_metadata_dict(testaddr).read()
tokens = mdict[metadata.DEVICETOKEN_KEY]
assert isinstance(tokens, dict)
assert "oldtoken1" in tokens and "oldtoken2" in tokens
@pytest.mark.parametrize(
"suffix, expected",
[
(b"vendor/deltachat/maxsmtprecipients", b"O1000\n"),
(b"wrong/prefix/key", b"N\n"),
(b"vendor/deltachat/unknown", b"N\n"),
],
ids=["maxsmtprecipients", "prefix_mismatch", "unknown_name"],
)
def test_shared_lookup(dictproxy, suffix, expected):
key = (
b"Lshared/0123/vendor/vendor.dovecot/pvt/server/"
+ suffix
+ b"\tuser@example.org"
)
rfile, wfile = io.BytesIO(b"H\n" + key), io.BytesIO()
dictproxy.loop_forever(rfile, wfile)
assert wfile.getvalue() == expected
@@ -48,8 +48,6 @@ def test_migration(tmp_path, example_config, caplog):
assert passdb_path.stat().st_size > 10000
example_config.passdb_path = passdb_path
# ensure logging.info records are captured regardless of global configuration
caplog.set_level("INFO")
assert not caplog.records
+3 -20
View File
@@ -19,35 +19,18 @@ def test_create_newemail_dict(example_config):
assert ac1["password"] != ac2["password"]
def test_create_newemail_dict_ip(ipv4_config):
ac = create_newemail_dict(ipv4_config)
assert ac["email"].endswith("@[1.3.3.7]")
def test_create_dclogin_url(example_config):
addr = "user@example.org"
password = "p@ss w+rd"
url = create_dclogin_url(example_config, addr, password)
def test_create_dclogin_url():
url = create_dclogin_url("user@example.org", "p@ss w+rd")
assert url.startswith("dclogin:")
assert "v=1" in url
assert "ic=3" in url
assert addr in url
assert "user@example.org" in url
# password special chars must be encoded
assert "p%40ss" in url
assert "w%2Brd" in url
def test_create_dclogin_url_ipv4(ipv4_config):
addr = "user@[1.3.3.7]"
password = "p@ss w+rd"
url = create_dclogin_url(ipv4_config, addr, password)
assert url.startswith("dclogin:")
assert "v=1" in url
assert "ic=3" in url
assert addr in url
def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
print_new_account()
@@ -1,46 +0,0 @@
import socket
import threading
import pytest
from chatmaild.metadata import turn_credentials
@pytest.fixture
def turn_socket(tmp_path):
sock_path = str(tmp_path / "turn.socket")
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(sock_path)
server.listen(1)
yield sock_path, server
server.close()
def test_turn_credentials_timeout(turn_socket):
sock_path, server = turn_socket
with pytest.raises(socket.timeout):
# Inside turn_credentials the kernel listen backlog (1)
# completes connect() without accept()
# so the client blocks on readline() until the 5s timeout fires.
turn_credentials(sock_path)
def test_turn_credentials_connection_refused_on_not_existing_socket(tmp_path):
missing = str(tmp_path / "nonexistent.socket")
with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
turn_credentials(missing)
def test_turn_credentials_socket_success(turn_socket):
sock_path, server = turn_socket
def respond():
conn, _ = server.accept()
conn.sendall(b"testuser:testpass\n")
conn.close()
t = threading.Thread(target=respond, daemon=True)
t.start()
result = turn_credentials(sock_path)
assert result == "testuser:testpass"
@@ -0,0 +1,73 @@
import socket
import threading
import time
from unittest.mock import patch
import pytest
from chatmaild.turnserver import turn_credentials
SOCKET_PATH = "/run/chatmail-turn/turn.socket"
@pytest.fixture
def turn_socket(tmp_path):
"""Create a real Unix socket server at a temp path."""
sock_path = str(tmp_path / "turn.socket")
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(sock_path)
server.listen(1)
yield sock_path, server
server.close()
def _call_turn_credentials(sock_path):
"""Call turn_credentials but connect to sock_path instead of hardcoded path."""
original_connect = socket.socket.connect
def patched_connect(self, address):
if address == SOCKET_PATH:
address = sock_path
return original_connect(self, address)
with patch.object(socket.socket, "connect", patched_connect):
return turn_credentials()
def test_turn_credentials_timeout(turn_socket):
"""Server accepts but never responds — must raise socket.timeout."""
sock_path, server = turn_socket
def accept_and_hang():
conn, _ = server.accept()
time.sleep(30)
conn.close()
t = threading.Thread(target=accept_and_hang, daemon=True)
t.start()
with pytest.raises(socket.timeout):
_call_turn_credentials(sock_path)
def test_turn_credentials_connection_refused(tmp_path):
"""Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
missing = str(tmp_path / "nonexistent.socket")
with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
_call_turn_credentials(missing)
def test_turn_credentials_success(turn_socket):
"""Server responds with credentials — must return stripped string."""
sock_path, server = turn_socket
def respond():
conn, _ = server.accept()
conn.sendall(b"testuser:testpass\n")
conn.close()
t = threading.Thread(target=respond, daemon=True)
t.start()
result = _call_turn_credentials(sock_path)
assert result == "testuser:testpass"
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python3
import socket
def turn_credentials() -> str:
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
client_socket.settimeout(5)
client_socket.connect("/run/chatmail-turn/turn.socket")
with client_socket.makefile("rb") as file:
return file.readline().decode("utf-8").strip()
+104 -19
View File
@@ -1,4 +1,6 @@
from pyinfra.operations import apt, server
import importlib.resources
from pyinfra.operations import apt, files, server, systemd
from ..basedeploy import Deployer
@@ -7,6 +9,9 @@ class AcmetoolDeployer(Deployer):
def __init__(self, email, domains):
self.domains = domains
self.email = email
self.need_restart_redirector = False
self.need_restart_reconcile_service = False
self.need_restart_reconcile_timer = False
def install(self):
apt.packages(
@@ -14,41 +19,121 @@ class AcmetoolDeployer(Deployer):
packages=["acmetool"],
)
self.remove_file("/etc/cron.d/acmetool")
files.file(
name="Remove old acmetool cronjob, it is replaced with systemd timer.",
path="/etc/cron.d/acmetool",
present=False,
)
self.put_executable("acmetool/acmetool.hook", "/etc/acme/hooks/nginx")
self.remove_file("/usr/lib/acme/hooks/nginx")
files.put(
name="Install acmetool hook.",
src=importlib.resources.files(__package__)
.joinpath("acmetool.hook")
.open("rb"),
dest="/etc/acme/hooks/nginx",
user="root",
group="root",
mode="755",
)
files.file(
name="Remove acmetool hook from the wrong location where it was previously installed.",
path="/usr/lib/acme/hooks/nginx",
present=False,
)
def configure(self):
self.put_template(
"acmetool/response-file.yaml.j2",
"/var/lib/acme/conf/responses",
files.template(
src=importlib.resources.files(__package__).joinpath(
"response-file.yaml.j2"
),
dest="/var/lib/acme/conf/responses",
user="root",
group="root",
mode="644",
email=self.email,
)
self.put_template(
"acmetool/target.yaml.j2",
"/var/lib/acme/conf/target",
files.template(
src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
dest="/var/lib/acme/conf/target",
user="root",
group="root",
mode="644",
)
server.shell(
name=f"Remove old acmetool desired files for {self.domains[0]}",
commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
)
self.put_template(
"acmetool/desired.yaml.j2",
f"/var/lib/acme/desired/{self.domains[0]}",
files.template(
src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
user="root",
group="root",
mode="644",
domains=self.domains,
)
self.ensure_systemd_unit("acmetool/acmetool-redirector.service")
self.ensure_systemd_unit("acmetool/acmetool-reconcile.service")
self.ensure_systemd_unit("acmetool/acmetool-reconcile.timer")
service_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-redirector.service"
),
dest="/etc/systemd/system/acmetool-redirector.service",
user="root",
group="root",
mode="644",
)
self.need_restart_redirector = service_file.changed
reconcile_service_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-reconcile.service"
),
dest="/etc/systemd/system/acmetool-reconcile.service",
user="root",
group="root",
mode="644",
)
self.need_restart_reconcile_service = reconcile_service_file.changed
reconcile_timer_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-reconcile.timer"
),
dest="/etc/systemd/system/acmetool-reconcile.timer",
user="root",
group="root",
mode="644",
)
self.need_restart_reconcile_timer = reconcile_timer_file.changed
def activate(self):
self.ensure_service("acmetool-redirector.service")
self.ensure_service("acmetool-reconcile.service", running=False, enabled=False)
self.ensure_service("acmetool-reconcile.timer")
systemd.service(
name="Setup acmetool-redirector service",
service="acmetool-redirector.service",
running=True,
enabled=True,
restarted=self.need_restart_redirector,
)
self.need_restart_redirector = False
systemd.service(
name="Setup acmetool-reconcile service",
service="acmetool-reconcile.service",
running=False,
enabled=False,
daemon_reload=self.need_restart_reconcile_service,
)
self.need_restart_reconcile_service = False
systemd.service(
name="Setup acmetool-reconcile timer",
service="acmetool-reconcile.timer",
running=True,
enabled=True,
daemon_reload=self.need_restart_reconcile_timer,
)
self.need_restart_reconcile_timer = False
server.shell(
name=f"Reconcile certificates for: {', '.join(self.domains)}",
@@ -1,2 +1,2 @@
"acme-enter-email": "{{ email }}"
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.8-July-06-2026.pdf": true
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true
+14 -131
View File
@@ -3,9 +3,6 @@ import io
import os
from contextlib import contextmanager
from pyinfra import host
from pyinfra.facts.files import Sha256File
from pyinfra.facts.server import Command
from pyinfra.operations import files, server, systemd
@@ -14,17 +11,6 @@ def has_systemd():
return os.path.isdir("/run/systemd/system")
def is_in_container() -> bool:
"""Return True if running inside a container (Docker, LXC, etc.)."""
return (
host.get_fact(
Command,
"systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
)
== "yes"
)
@contextmanager
def blocked_service_startup():
"""Prevent services from auto-starting during package installation.
@@ -51,10 +37,11 @@ def get_resource(arg, pkg=__package__):
return importlib.resources.files(pkg).joinpath(arg)
def configure_remote_units(deployer, mail_domain, units) -> None:
def configure_remote_units(mail_domain, units) -> None:
remote_base_dir = "/usr/local/lib/chatmaild"
remote_venv_dir = f"{remote_base_dir}/venv"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
# install systemd units
for fn in units:
@@ -70,13 +57,15 @@ def configure_remote_units(deployer, mail_domain, units) -> None:
source_path = get_resource(f"service/{basename}.f")
content = source_path.read_text().format(**params).encode()
deployer.put_file(
files.put(
name=f"Upload {basename}",
src=io.BytesIO(content),
dest=f"/etc/systemd/system/{basename}",
**root_owned,
)
def activate_remote_units(deployer, units) -> None:
def activate_remote_units(units) -> None:
# activate systemd units
for fn in units:
basename = fn if "." in fn else f"{fn}.service"
@@ -86,8 +75,14 @@ def activate_remote_units(deployer, units) -> None:
enabled = False
else:
enabled = True
deployer.ensure_service(basename, running=enabled, enabled=enabled)
systemd.service(
name=f"Setup {basename}",
service=basename,
running=enabled,
enabled=enabled,
restarted=enabled,
daemon_reload=True,
)
class Deployment:
@@ -133,7 +128,6 @@ class Deployment:
class Deployer:
need_restart = False
daemon_reload = False
def install(self):
pass
@@ -143,114 +137,3 @@ class Deployer:
def activate(self):
pass
def ensure_service(self, service, running=True, enabled=True):
if running:
verb = "Start and enable"
else:
verb = "Stop"
systemd.service(
name=f"{verb} {service}",
service=service,
running=running,
enabled=enabled,
restarted=self.need_restart if running else False,
daemon_reload=self.daemon_reload,
)
self.daemon_reload = False
def ensure_systemd_unit(self, src, **kwargs):
dest_name = src.split("/")[-1].replace(".j2", "")
dest = f"/etc/systemd/system/{dest_name}"
if src.endswith(".j2"):
return self.put_template(src, dest, **kwargs)
return self.put_file(src, dest)
def put_file(self, src, dest, mode="644", **kwargs):
if isinstance(src, str):
src = get_resource(src)
res = files.put(
name=f"Upload {dest}",
src=src,
dest=dest,
user="root",
group="root",
mode=mode,
**kwargs,
)
return self._update_restart_signals(dest, res)
def put_executable(self, src, dest):
return self.put_file(src, dest, mode="755")
def put_template(self, src, dest, owner="root", **kwargs):
if isinstance(src, str):
src = get_resource(src)
res = files.template(
name=f"Upload {dest}",
src=src,
dest=dest,
user=owner,
group=owner,
mode="644",
**kwargs,
)
return self._update_restart_signals(dest, res)
def remove_file(self, dest):
res = files.file(name=f"Remove {dest}", path=dest, present=False)
return self._update_restart_signals(dest, res)
def ensure_line(self, path, line, **kwargs):
name = kwargs.pop("name", f"Ensure line in {path}")
res = files.line(name=name, path=path, line=line, **kwargs)
return self._update_restart_signals(path, res)
def ensure_directory(self, path, owner="root", mode="755", **kwargs):
name = kwargs.pop("name", f"Ensure directory {path}")
res = files.directory(
name=name,
path=path,
user=owner,
group=owner,
mode=mode,
present=True,
**kwargs,
)
return self._update_restart_signals(path, res)
def remove_directory(self, path, **kwargs):
name = kwargs.pop("name", f"Remove directory {path}")
res = files.directory(name=name, path=path, present=False, **kwargs)
return self._update_restart_signals(path, res)
def download_executable(self, url, dest, sha256sum, extract=None):
existing = host.get_fact(Sha256File, dest)
if existing == sha256sum:
return
tmp = f"{dest}.new"
if extract:
dl_cmd = f"curl -fSL {url} | {extract} >{tmp}"
else:
dl_cmd = f"curl -fSL {url} -o {tmp}"
server.shell(
name=f"Download {dest}",
commands=[
f"({dl_cmd}"
f" && echo '{sha256sum} {tmp}' | sha256sum -c"
f" && mv {tmp} {dest})",
f"chmod 755 {dest}",
],
)
self.need_restart = True
def _update_restart_signals(self, path, res):
if res.changed:
self.need_restart = True
if str(path).startswith("/etc/systemd/system/"):
self.daemon_reload = True
return res
+136 -91
View File
@@ -15,10 +15,27 @@ from pathlib import Path
import pyinfra
from chatmaild.config import read_config, write_initial_config
from packaging import version
from termcolor import colored
from . import dns, remote
from .sshexec import LocalExec, SSHExec
from .lxc.cli import (
lxc_start_cmd,
lxc_start_cmd_options,
lxc_status_cmd,
lxc_status_cmd_options,
lxc_stop_cmd,
lxc_stop_cmd_options,
lxc_test_cmd,
lxc_test_cmd_options,
)
from .lxc.incus import DNSConfigurationError
from .sshexec import (
LocalExec,
SSHExec,
resolve_host_from_ssh_config,
resolve_key_from_ssh_config,
)
from .util import Out
from .www import main as webdev_main
#
# cmdeploy sub commands and options
@@ -82,29 +99,21 @@ def run_cmd_options(parser):
help="disable checks nslookup for dns",
)
add_ssh_host_option(parser)
def _warn_unused_settings(unused_keys, out):
if unused_keys:
names = ", ".join(unused_keys)
out.red(
f"WARNING: chatmail.ini contains settings that have no effect: {names}\n"
"Please remove them from chatmail.ini."
)
add_ssh_config_option(parser)
def run_cmd(args, out):
"""Deploy chatmail services on the remote server."""
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
sshexec = get_sshexec(ssh_host)
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
require_iroh = args.config.enable_iroh_relay
strict_tls = args.config.tls_cert_mode == "acme"
if args.config.ipv4_relay:
args.dns_check_disabled = True
if not args.dns_check_disabled:
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
if not dns.check_initial_remote_data(
remote_data, strict_tls=strict_tls, print=out.red
):
return 1
env = os.environ.copy()
@@ -112,11 +121,31 @@ def run_cmd(args, out):
env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
if not args.dns_check_disabled:
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
env["DEBIAN_FRONTEND"] = "noninteractive"
env["TERM"] = "linux"
deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
if ssh_host == "localhost":
ssh_config = args.ssh_config
if ssh_config:
ssh_config = str(Path(ssh_config).resolve())
# Use pyinfra's native SSH data keys to configure the connection directly
# rather than relying on paramiko config parsing (see also sshexec.py)
ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
key = resolve_key_from_ssh_config(ssh_host, ssh_config)
data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
if key:
data_args += f" --data ssh_key={key}"
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
if ssh_host in ["localhost", "@docker"]:
if ssh_host == "@docker":
env["CHATMAIL_NOPORTCHECK"] = "True"
env["CHATMAIL_NOSYSCTL"] = "True"
cmd = f"{pyinf} @local {deploy_path} -y"
if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -124,17 +153,21 @@ def run_cmd(args, out):
return 1
try:
out.check_call(cmd, env=env)
ret = out.shell(cmd, env=env)
if ret:
out.red("Deploy failed")
return 1
if args.website_only:
out.green("Website deployment completed.")
elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
elif (
not args.dns_check_disabled
and strict_tls
and not remote_data["acme_account_url"]
):
out.red("Deploy completed but letsencrypt not configured")
out.red("Run 'cmdeploy run' again")
elif args.config.ipv4_relay:
out.green("Deploy completed.")
else:
out.green("Deploy completed, call `cmdeploy dns` next.")
_warn_unused_settings(args.config._unused_keys, out)
return 0
except subprocess.CalledProcessError:
out.red("Deploy failed")
@@ -147,19 +180,16 @@ def dns_cmd_options(parser):
dest="zonefile",
type=pathlib.Path,
default=None,
help="write out a zonefile",
help="write DNS records in standard BIND format to the given file",
)
add_ssh_host_option(parser)
add_ssh_config_option(parser)
def dns_cmd(args, out):
"""Check DNS entries and optionally generate dns zone file."""
if args.config.ipv4_relay:
ipv4 = args.config.ipv4_relay
print(f"[WARNING] {ipv4} is not a domain, skipping DNS checks.")
return 0
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
tls_cert_mode = args.config.tls_cert_mode
strict_tls = tls_cert_mode == "acme"
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
@@ -190,13 +220,14 @@ def dns_cmd(args, out):
def status_cmd_options(parser):
add_ssh_host_option(parser)
add_ssh_config_option(parser)
def status_cmd(args, out):
"""Display status for online chatmail instance."""
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
out.green(f"chatmail domain: {args.config.mail_domain}")
if args.config.privacy_mail:
@@ -209,16 +240,21 @@ def status_cmd(args, out):
def test_cmd_options(parser):
parser.add_argument(
"--slow",
dest="slow",
action="store_true",
help="also run slow tests",
)
add_ssh_host_option(parser)
add_ssh_config_option(parser)
def test_cmd(args, out):
"""Run local and online tests for chatmail deployment."""
env = os.environ.copy()
env["CHATMAIL_INI"] = str(args.inipath.absolute())
if args.ssh_host:
env["CHATMAIL_SSH"] = args.ssh_host
env["CHATMAIL_INI"] = str(args.inipath.resolve())
pytest_path = shutil.which("pytest")
pytest_args = [
@@ -230,7 +266,13 @@ def test_cmd(args, out):
"-v",
"--durations=5",
]
ret = out.run_ret(pytest_args, env=env)
if args.slow:
pytest_args.append("--slow")
if args.ssh_host:
pytest_args.extend(["--ssh-host", args.ssh_host])
if args.ssh_config:
pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
ret = out.shell(" ".join(pytest_args), env=env)
return ret
@@ -267,8 +309,8 @@ def fmt_cmd(args, out):
format_args.extend(sources)
check_args.extend(sources)
out.check_call(" ".join(format_args), quiet=not args.verbose)
out.check_call(" ".join(check_args), quiet=not args.verbose)
out.shell(" ".join(format_args), quiet=not args.verbose)
out.shell(" ".join(check_args), quiet=not args.verbose)
def bench_cmd(args, out):
@@ -281,9 +323,7 @@ def bench_cmd(args, out):
def webdev_cmd(args, out):
"""Run local web development loop for static web pages."""
from .www import main
main()
webdev_main()
#
@@ -291,41 +331,25 @@ def webdev_cmd(args, out):
#
class Out:
"""Convenience output printer providing coloring."""
def red(self, msg, file=sys.stderr):
print(colored(msg, "red"), file=file)
def green(self, msg, file=sys.stderr):
print(colored(msg, "green"), file=file)
def __call__(self, msg, red=False, green=False, file=sys.stdout):
color = "red" if red else ("green" if green else None)
print(colored(msg, color), file=file)
def check_call(self, arg, env=None, quiet=False):
if not quiet:
self(f"[$ {arg}]", file=sys.stderr)
return subprocess.check_call(arg, shell=True, env=env)
def run_ret(self, args, env=None, quiet=False):
if not quiet:
cmdstring = " ".join(args)
self(f"[$ {cmdstring}]", file=sys.stderr)
proc = subprocess.run(args, env=env, check=False)
return proc.returncode
def add_ssh_host_option(parser):
parser.add_argument(
"--ssh-host",
dest="ssh_host",
help="Run commands on 'localhost' or on a specific SSH host "
help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
"instead of chatmail.ini's mail_domain.",
)
def add_ssh_config_option(parser):
parser.add_argument(
"--ssh-config",
dest="ssh_config",
type=Path,
default=None,
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
)
def add_config_option(parser):
parser.add_argument(
"--config",
@@ -335,25 +359,26 @@ def add_config_option(parser):
type=Path,
help="path to the chatmail.ini file",
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store_true",
default=False,
help="provide verbose logging",
)
def add_subcommand(subparsers, func):
def add_subcommand(subparsers, func, add_config=True):
name = func.__name__
assert name.endswith("_cmd")
name = name[:-4]
name = name[:-4].replace("_", "-")
doc = func.__doc__.strip()
help = doc.split("\n")[0].strip(".")
p = subparsers.add_parser(name, description=doc, help=help)
p.set_defaults(func=func)
add_config_option(p)
if add_config:
add_config_option(p)
p.add_argument(
"-v",
"--verbose",
dest="verbose",
action="count",
default=0,
help="increase verbosity (can be repeated: -v, -vv)",
)
return p
@@ -362,43 +387,60 @@ Setup your chatmail server configuration and
deploy it via SSH to your remote location.
"""
# Explicit subcommand registry: (cmd_func, options_func_or_None, needs_config).
# LXC commands don't need a chatmail.ini (no config); all others do.
SUBCOMMANDS = [
(init_cmd, init_cmd_options, True),
(run_cmd, run_cmd_options, True),
(dns_cmd, dns_cmd_options, True),
(status_cmd, status_cmd_options, True),
(test_cmd, test_cmd_options, True),
(fmt_cmd, fmt_cmd_options, True),
(bench_cmd, None, True),
(webdev_cmd, None, True),
(lxc_start_cmd, lxc_start_cmd_options, False),
(lxc_stop_cmd, lxc_stop_cmd_options, False),
(lxc_status_cmd, lxc_status_cmd_options, False),
(lxc_test_cmd, lxc_test_cmd_options, False),
]
def get_parser():
"""Return an ArgumentParser for the 'cmdeploy' CLI"""
parser = argparse.ArgumentParser(description=description.strip())
parser.set_defaults(func=None, inipath=None)
subparsers = parser.add_subparsers(title="subcommands")
# find all subcommands in the module namespace
glob = globals()
for name, func in glob.items():
if name.endswith("_cmd"):
subparser = add_subcommand(subparsers, func)
addopts = glob.get(name + "_options")
if addopts is not None:
addopts(subparser)
for func, addopts, needs_config in SUBCOMMANDS:
subparser = add_subcommand(subparsers, func, add_config=needs_config)
if addopts is not None:
addopts(subparser)
return parser
def get_sshexec(ssh_host: str, verbose=True):
def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
if ssh_host in ["localhost", "@local"]:
return LocalExec(verbose)
return LocalExec(verbose, docker=False)
elif ssh_host == "@docker":
return LocalExec(verbose, docker=True)
if verbose:
print(f"[ssh] login to {ssh_host}")
return SSHExec(ssh_host, verbose=verbose)
return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
def main(args=None):
"""Provide main entry point for 'cmdeploy' CLI invocation."""
parser = get_parser()
args = parser.parse_args(args=args)
if not hasattr(args, "func"):
if args.func is None:
return parser.parse_args(["-h"])
out = Out()
out = Out(verbosity=args.verbose)
kwargs = {}
if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
if not args.inipath.exists():
out.red(f"expecting {args.inipath} to exist, run init first?")
raise SystemExit(1)
@@ -413,6 +455,9 @@ def main(args=None):
if res is None:
res = 0
return res
except DNSConfigurationError as exc:
out.red(str(exc))
return 1
except KeyboardInterrupt:
out.red("KeyboardInterrupt")
sys.exit(130)
+206 -99
View File
@@ -2,6 +2,7 @@
Chat Mail pyinfra deploy.
"""
import os
import shutil
import subprocess
import sys
@@ -12,11 +13,10 @@ from chatmaild.config import read_config
from pyinfra import facts, host, logger
from pyinfra.api import FactBase
from pyinfra.facts import hardware
from pyinfra.facts.files import Sha256File
from pyinfra.facts.systemd import SystemdEnabled
from pyinfra.operations import apt, files, pip, server, systemd
from cmdeploy.cmdeploy import Out
from .acmetool import AcmetoolDeployer
from .basedeploy import (
Deployer,
@@ -24,8 +24,8 @@ from .basedeploy import (
activate_remote_units,
blocked_service_startup,
configure_remote_units,
get_resource,
has_systemd,
is_in_container,
)
from .dovecot.deployer import DovecotDeployer
from .external.deployer import ExternalTlsDeployer
@@ -35,6 +35,7 @@ from .nginx.deployer import NginxDeployer
from .opendkim.deployer import OpendkimDeployer
from .postfix.deployer import PostfixDeployer
from .selfsigned.deployer import SelfSignedTlsDeployer
from .util import Out, get_version_string
from .www import build_webpages, find_merge_conflict, get_paths
@@ -80,22 +81,25 @@ def remove_legacy_artifacts():
)
def _install_remote_venv_with_chatmaild(deployer) -> None:
def _install_remote_venv_with_chatmaild() -> None:
remove_legacy_artifacts()
dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
remote_base_dir = "/usr/local/lib/chatmaild"
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
remote_venv_dir = f"{remote_base_dir}/venv"
root_owned = dict(user="root", group="root", mode="644")
apt.packages(
name="apt install python3-virtualenv",
packages=["python3-virtualenv"],
)
deployer.ensure_directory(f"{remote_base_dir}/dist")
deployer.put_file(
files.put(
name="Upload chatmaild source package",
src=dist_file.open("rb"),
dest=remote_dist_file,
create_remote_dir=True,
**root_owned,
)
pip.virtualenv(
@@ -117,24 +121,37 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
)
def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
def _configure_remote_venv_with_chatmaild(config) -> None:
remote_base_dir = "/usr/local/lib/chatmaild"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
deployer.put_file(
files.put(
name=f"Upload {remote_chatmail_inipath}",
src=config._getbytefile(),
dest=remote_chatmail_inipath,
**root_owned,
)
deployer.remove_file("/etc/cron.d/chatmail-metrics")
deployer.remove_file("/var/www/html/metrics")
files.file(
path="/etc/cron.d/chatmail-metrics",
present=False,
)
files.file(
path="/var/www/html/metrics",
present=False,
)
class UnboundDeployer(Deployer):
def __init__(self, config):
self.config = config
self.need_restart = False
def install(self):
# Run local DNS resolver `unbound`. `resolvconf` takes care of
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
# On an IPv4-only system, if unbound is started but not configured,
# it causes subsequent steps to fail to resolve hosts.
with blocked_service_startup():
@@ -144,42 +161,33 @@ class UnboundDeployer(Deployer):
)
def configure(self):
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
apt.packages(
name="Purge resolvconf",
packages=["resolvconf"],
present=False,
extra_uninstall_args="--purge",
)
# systemd-resolved can't be purged due to dependencies; stop and mask.
server.shell(
name="Stop and mask systemd-resolved",
commands=[
"systemctl stop systemd-resolved.service || true",
"systemctl mask systemd-resolved.service",
],
)
# Configure unbound resolver with Quad9 fallback and a trailing newline
# (SolusVM bug).
self.put_file(
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
dest="/etc/resolv.conf",
force=True,
)
server.shell(
name="Generate root keys for validating DNSSEC",
commands=[
"unbound-anchor -a /var/lib/unbound/root.key || true",
],
)
self.ensure_directory(
path="/etc/unbound/unbound.conf.d",
)
self.put_template(
"unbound/unbound.conf.j2",
"/etc/unbound/unbound.conf.d/chatmail.conf",
disable_ipv6=self.config.disable_ipv6,
)
if self.config.disable_ipv6:
files.directory(
path="/etc/unbound/unbound.conf.d",
present=True,
user="root",
group="root",
mode="755",
)
conf = files.put(
src=get_resource("unbound/unbound.conf.j2"),
dest="/etc/unbound/unbound.conf.d/chatmail.conf",
user="root",
group="root",
mode="644",
)
else:
conf = files.file(
path="/etc/unbound/unbound.conf.d/chatmail.conf",
present=False,
)
self.need_restart |= conf.changed
def activate(self):
server.shell(
@@ -189,25 +197,27 @@ class UnboundDeployer(Deployer):
],
)
self.ensure_service("unbound.service")
self.ensure_service(
"unbound-resolvconf.service",
running=False,
enabled=False,
systemd.service(
name="Start and enable unbound",
service="unbound.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
class MtastsDeployer(Deployer):
def configure(self):
# Remove configuration.
self.remove_file("/etc/mta-sts-daemon.yml")
self.remove_directory("/usr/local/lib/postfix-mta-sts-resolver")
self.remove_file("/etc/systemd/system/mta-sts-daemon.service")
files.file("/etc/mta-sts-daemon.yml", present=False)
files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
def activate(self):
self.ensure_service(
"mta-sts-daemon.service",
systemd.service(
name="Stop MTA-STS daemon",
service="mta-sts-daemon.service",
daemon_reload=True,
running=False,
enabled=False,
)
@@ -218,7 +228,14 @@ class WebsiteDeployer(Deployer):
self.config = config
def install(self):
self.ensure_directory("/var/www")
files.directory(
name="Ensure /var/www exists",
path="/var/www",
user="root",
group="root",
mode="755",
present=True,
)
def configure(self):
www_path, src_dir, build_dir = get_paths(self.config)
@@ -237,8 +254,14 @@ class WebsiteDeployer(Deployer):
logger.warning("Web page build failed, skipping website deployment")
return
# if it is not a hugo page, upload it as is
files.rsync(
f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
# pyinfra files.rsync (experimental) causes problems with ssh-config configuration
# the stable files.sync should do
files.sync(
src=str(www_path),
dest="/var/www/html",
user="www-data",
group="www-data",
delete=True,
)
@@ -248,11 +271,15 @@ class LegacyRemoveDeployer(Deployer):
# remove historic expunge script
# which is now implemented through a systemd timer (chatmail-expire)
self.remove_file("/etc/cron.d/expunge")
files.file(
path="/etc/cron.d/expunge",
present=False,
)
# Remove OBS repository key that is no longer used.
self.remove_file("/etc/apt/keyrings/obs-home-deltachat.gpg")
self.ensure_line(
files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
files.line(
name="Remove DeltaChat OBS home repository from sources.list",
path="/etc/apt/sources.list",
line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
escape_regex_characters=True,
@@ -260,7 +287,11 @@ class LegacyRemoveDeployer(Deployer):
)
# prior relay versions used filelogging
self.remove_directory("/var/log/journal/")
files.directory(
name="Ensure old logs on disk are deleted",
path="/var/log/journal/",
present=False,
)
# remove echobot if it is still running
if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
systemd.service(
@@ -302,13 +333,22 @@ class TurnDeployer(Deployer):
"0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
),
}[host.get_fact(facts.server.Arch)]
self.download_executable(url, "/usr/local/bin/chatmail-turn", sha256sum)
existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
if existing_sha256sum != sha256sum:
server.shell(
name="Download chatmail-turn",
commands=[
f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
"chmod 755 /usr/local/bin/chatmail-turn",
],
)
def configure(self):
configure_remote_units(self, self.mail_domain, self.units)
configure_remote_units(self.mail_domain, self.units)
def activate(self):
activate_remote_units(self, self.units)
activate_remote_units(self.units)
class IrohDeployer(Deployer):
@@ -326,30 +366,72 @@ class IrohDeployer(Deployer):
"f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
),
}[host.get_fact(facts.server.Arch)]
self.download_executable(
url,
"/usr/local/bin/iroh-relay",
sha256sum,
extract="gunzip | tar -xf - ./iroh-relay -O",
)
existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
if existing_sha256sum != sha256sum:
server.shell(
name="Download iroh-relay",
commands=[
f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
"chmod 755 /usr/local/bin/iroh-relay",
],
)
self.need_restart = True
def configure(self):
self.ensure_systemd_unit("iroh-relay.service")
self.put_file("iroh-relay.toml", "/etc/iroh-relay.toml")
systemd_unit = files.put(
name="Upload iroh-relay systemd unit",
src=get_resource("iroh-relay.service"),
dest="/etc/systemd/system/iroh-relay.service",
user="root",
group="root",
mode="644",
)
self.need_restart |= systemd_unit.changed
iroh_config = files.put(
name="Upload iroh-relay config",
src=get_resource("iroh-relay.toml"),
dest="/etc/iroh-relay.toml",
user="root",
group="root",
mode="644",
)
self.need_restart |= iroh_config.changed
def activate(self):
self.ensure_service(
"iroh-relay.service",
systemd.service(
name="Start and enable iroh-relay",
service="iroh-relay.service",
running=True,
enabled=self.enable_iroh_relay,
restarted=self.need_restart,
)
self.need_restart = False
class JournaldDeployer(Deployer):
def configure(self):
self.put_file("journald.conf", "/etc/systemd/journald.conf")
journald_conf = files.put(
name="Configure journald",
src=get_resource("journald.conf"),
dest="/etc/systemd/journald.conf",
user="root",
group="root",
mode="644",
)
self.need_restart = journald_conf.changed
def activate(self):
self.ensure_service("systemd-journald.service")
systemd.service(
name="Start and enable journald",
service="systemd-journald.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
self.need_restart = False
class ChatmailVenvDeployer(Deployer):
@@ -365,14 +447,14 @@ class ChatmailVenvDeployer(Deployer):
)
def install(self):
_install_remote_venv_with_chatmaild(self)
_install_remote_venv_with_chatmaild()
def configure(self):
_configure_remote_venv_with_chatmaild(self, self.config)
configure_remote_units(self, self.config.mail_domain_bare, self.units)
_configure_remote_venv_with_chatmaild(self.config)
configure_remote_units(self.config.mail_domain, self.units)
def activate(self):
activate_remote_units(self, self.units)
activate_remote_units(self.units)
class ChatmailDeployer(Deployer):
@@ -386,9 +468,13 @@ class ChatmailDeployer(Deployer):
self.mail_domain = config.mail_domain
def install(self):
self.put_file(
files.put(
name="Disable installing recommended packages globally",
src=BytesIO(b'APT::Install-Recommends "false";\n'),
dest="/etc/apt/apt.conf.d/00InstallRecommends",
user="root",
group="root",
mode="644",
)
apt.update(name="apt update", cache_time=24 * 3600)
apt.upgrade(name="upgrade apt packages", auto_remove=True)
@@ -404,11 +490,15 @@ class ChatmailDeployer(Deployer):
)
def configure(self):
# metadata crashes if the mailboxes dir does not exist
self.ensure_directory(
str(self.config.mailboxes_dir),
owner="vmail",
# Ensure the per-domain mailbox directory exists before
# chatmail-metadata starts (it crashes without it).
files.directory(
name="Ensure vmail mailbox directory exists",
path=f"/home/vmail/mail/{self.mail_domain}",
user="vmail",
group="vmail",
mode="700",
present=True,
)
# This file is used by auth proxy.
@@ -420,6 +510,15 @@ class ChatmailDeployer(Deployer):
],
)
files.directory(
name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
path=str(self.config.mailboxes_dir),
user="vmail",
group="vmail",
mode="700",
present=True,
)
class FcgiwrapDeployer(Deployer):
def install(self):
@@ -429,20 +528,22 @@ class FcgiwrapDeployer(Deployer):
)
def activate(self):
self.ensure_service("fcgiwrap.service")
systemd.service(
name="Start and enable fcgiwrap",
service="fcgiwrap.service",
running=True,
enabled=True,
)
class GithashDeployer(Deployer):
def activate(self):
try:
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
except Exception:
git_hash = "unknown\n"
try:
git_diff = subprocess.check_output(["git", "diff"]).decode()
except Exception:
git_diff = ""
self.put_file(src=StringIO(git_hash + git_diff), dest="/etc/chatmail-version")
files.put(
name="Upload chatmail relay git commit hash",
src=StringIO(get_version_string()),
dest="/etc/chatmail-version",
mode="700",
)
def get_tls_deployer(config, mail_domain):
@@ -468,12 +569,20 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
"""
config = read_config(config_path)
check_config(config)
bare_host = config.mail_domain_bare
mail_domain = config.mail_domain
if website_only:
Deployment().perform_stages([WebsiteDeployer(config)])
return
if host.get_fact(Port, port=53) != "unbound":
files.line(
name="Add 9.9.9.9 to resolv.conf",
path="/etc/resolv.conf",
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
line="\nnameserver 9.9.9.9",
)
# Check if mtail_address interface is available (if configured)
if config.mtail_address and config.mtail_address not in (
"127.0.0.1",
@@ -488,7 +597,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
)
exit(1)
if not is_in_container():
if not os.environ.get("CHATMAIL_NOPORTCHECK"):
port_services = [
(["master", "smtpd"], 25),
("unbound", 53),
@@ -513,8 +622,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
(["master", "smtpd"], config.postfix_reinject_port_incoming),
("filtermail", config.filtermail_smtp_port),
("filtermail", config.filtermail_smtp_port_incoming),
("filtermail", config.filtermail_http_port_incoming),
("filtermail", config.filtermail_lmtp_port_transport),
]
for service, port in port_services:
print(f"Checking if port {port} is available for {service}...")
@@ -527,7 +634,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
)
exit(1)
tls_deployer = get_tls_deployer(config, bare_host)
tls_deployer = get_tls_deployer(config, mail_domain)
all_deployers = [
ChatmailDeployer(config),
@@ -535,13 +642,13 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
FiltermailDeployer(),
JournaldDeployer(),
UnboundDeployer(config),
TurnDeployer(bare_host),
TurnDeployer(mail_domain),
IrohDeployer(config.enable_iroh_relay),
tls_deployer,
WebsiteDeployer(config),
ChatmailVenvDeployer(config),
MtastsDeployer(),
*([] if config.ipv4_relay else [OpendkimDeployer(bare_host)]),
OpendkimDeployer(mail_domain),
# Dovecot should be started before Postfix
# because it creates authentication socket
# required by Postfix.
+34 -26
View File
@@ -4,7 +4,11 @@ from . import remote
def parse_zone_records(text):
"""Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text."""
"""Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
Skips comment lines (starting with ``;``) and blank lines.
Each record line must have the format ``name TTL IN type rdata``.
"""
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith(";"):
@@ -44,35 +48,39 @@ def get_filled_zone_file(remote_data):
d = remote_data["mail_domain"]
def append_record(name, rtype, rdata, ttl=3600):
lines.append(f"{name:<40} {ttl:<6} IN {rtype:<5} {rdata}")
def rec(name, rtype, rdata, ttl=3600):
return f"{name:<40} {ttl:<6} IN {rtype:<5} {rdata}"
lines = ["; Required DNS entries"]
if remote_data.get("A"):
append_record(f"{d}.", "A", remote_data["A"])
lines.append(rec(f"{d}.", "A", remote_data["A"]))
if remote_data.get("AAAA"):
append_record(f"{d}.", "AAAA", remote_data["AAAA"])
append_record(f"{d}.", "MX", f"10 {d}.")
lines.append(rec(f"{d}.", "AAAA", remote_data["AAAA"]))
lines.append(rec(f"{d}.", "MX", f"10 {d}."))
if remote_data.get("strict_tls"):
append_record(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
append_record(f"mta-sts.{d}.", "CNAME", f"{d}.")
append_record(f"www.{d}.", "CNAME", f"{d}.")
lines.append(
rec(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
)
lines.append(rec(f"mta-sts.{d}.", "CNAME", f"{d}."))
lines.append(rec(f"www.{d}.", "CNAME", f"{d}."))
lines.append(remote_data["dkim_entry"])
lines.append("")
lines.append("; Recommended DNS entries")
append_record(f"{d}.", "TXT", '"v=spf1 a ~all"')
append_record(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"')
lines.append(rec(f"{d}.", "TXT", '"v=spf1 a ~all"'))
lines.append(rec(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"'))
if remote_data.get("acme_account_url"):
append_record(
f"{d}.",
"CAA",
f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
lines.append(
rec(
f"{d}.",
"CAA",
f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
)
)
append_record(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"')
append_record(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}.")
append_record(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}.")
append_record(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}.")
append_record(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}.")
lines.append(rec(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"'))
lines.append(rec(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}."))
lines.append(rec(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}."))
lines.append(rec(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}."))
lines.append(rec(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}."))
lines.append("")
return "\n".join(lines)
@@ -90,19 +98,19 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
if required_diff:
out.red("Please set required DNS entries at your DNS provider:\n")
for line in required_diff:
out(line)
out("")
out.print(line)
out.print()
returncode = 1
if remote_data.get("dkim_entry") in required_diff:
out(
out.print(
"If the DKIM entry above does not work with your DNS provider,"
" you can try this one:\n"
)
out(remote_data.get("web_dkim_entry") + "\n")
out.print(remote_data.get("web_dkim_entry") + "\n")
if recommended_diff:
out("WARNING: these recommended DNS entries are not set:\n")
out.print("WARNING: these recommended DNS entries are not set:\n")
for line in recommended_diff:
out(line)
out.print(line)
if not (recommended_diff or required_diff):
out.green("Great! All your DNS entries are verified and correct.")
+66 -59
View File
@@ -1,22 +1,20 @@
import io
import urllib.request
from chatmaild.config import Config
from pyinfra import host
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Arch, Command, Sysctl
from pyinfra.operations import apt, files, server
from pyinfra.operations import apt, files, server, systemd
from cmdeploy.basedeploy import (
Deployer,
activate_remote_units,
blocked_service_startup,
configure_remote_units,
is_in_container,
get_resource,
)
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
DOVECOT_VERSION = "2.3.21+dfsg1-3"
DOVECOT_SHA256 = {
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
@@ -41,14 +39,11 @@ class DovecotDeployer(Deployer):
with blocked_service_startup():
debs = []
for pkg in ("core", "imapd", "lmtpd"):
deb, changed = _download_dovecot_package(pkg, arch)
self.need_restart |= changed
deb = _download_dovecot_package(pkg, arch)
if deb:
debs.append(deb)
if debs:
deb_list = " ".join(debs)
# First dpkg may fail on missing dependencies (stderr suppressed);
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
server.shell(
name="Install dovecot packages",
commands=[
@@ -57,40 +52,27 @@ class DovecotDeployer(Deployer):
f"dpkg --force-confdef --force-confold -i {deb_list}",
],
)
self.need_restart = True
self.put_file(
src=io.StringIO(
"Package: dovecot-*\n"
"Pin: version *\n"
"Pin-Priority: -1\n"
),
dest="/etc/apt/preferences.d/pin-dovecot",
)
def configure(self):
configure_remote_units(self, self.config.mail_domain_bare, self.units)
_configure_dovecot(self, self.config)
configure_remote_units(self.config.mail_domain, self.units)
self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
def activate(self):
activate_remote_units(self, self.units)
activate_remote_units(self.units)
# Detect stale binary: package installed but service still runs old (deleted) binary.
if not self.disable_mail and not self.need_restart:
stale = host.get_fact(
Command,
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
" && echo STALE || true",
)
if stale == "STALE":
self.need_restart = True
restart = False if self.disable_mail else self.need_restart
active = not self.disable_mail
self.ensure_service(
"dovecot.service",
running=active,
enabled=active,
systemd.service(
name="Disable dovecot for now"
if self.disable_mail
else "Start and enable Dovecot",
service="dovecot.service",
running=False if self.disable_mail else True,
enabled=False if self.disable_mail else True,
restarted=restart,
daemon_reload=self.daemon_reload,
)
self.need_restart = False
def _pick_url(primary, fallback):
@@ -102,22 +84,22 @@ def _pick_url(primary, fallback):
return fallback
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
"""Download a dovecot .deb if needed, return (path, changed)."""
def _download_dovecot_package(package: str, arch: str):
"""Download a dovecot .deb if needed, return its path (or None)."""
arch = "amd64" if arch == "x86_64" else arch
arch = "arm64" if arch == "aarch64" else arch
pkg_name = f"dovecot-{package}"
sha256 = DOVECOT_SHA256.get((package, arch))
if sha256 is None:
op = apt.packages(packages=[pkg_name])
return None, bool(getattr(op, "changed", False))
apt.packages(packages=[pkg_name])
return None
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
if DOVECOT_PACKAGE_VERSION in installed_versions:
return None, False
if DOVECOT_VERSION in installed_versions:
return None
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
url_version = DOVECOT_VERSION.replace("+", "%2B")
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
@@ -132,33 +114,53 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
)
return deb_filename, True
return deb_filename
def _configure_dovecot(deployer, config: Config, debug: bool = False):
def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
"""Configures Dovecot IMAP server."""
deployer.put_template(
"dovecot/dovecot.conf.j2",
"/etc/dovecot/dovecot.conf",
need_restart = False
daemon_reload = False
main_config = files.template(
src=get_resource("dovecot/dovecot.conf.j2"),
dest="/etc/dovecot/dovecot.conf",
user="root",
group="root",
mode="644",
config=config,
debug=debug,
disable_ipv6=config.disable_ipv6,
)
deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
deployer.put_file(
"dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
need_restart |= main_config.changed
auth_config = files.put(
src=get_resource("dovecot/auth.conf"),
dest="/etc/dovecot/auth.conf",
user="root",
group="root",
mode="644",
)
need_restart |= auth_config.changed
lua_push_notification_script = files.put(
src=get_resource("dovecot/push_notification.lua"),
dest="/etc/dovecot/push_notification.lua",
user="root",
group="root",
mode="644",
)
need_restart |= lua_push_notification_script.changed
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
# it is recommended to set the following inotify limits
can_modify = not is_in_container()
can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
for name in ("max_user_instances", "max_user_watches"):
key = f"fs.inotify.{name}"
value = host.get_fact(Sysctl).get(key, 0)
value = host.get_fact(Sysctl)[key]
if value > 65534:
continue
if not can_modify:
print(
"\n!!!! refusing to attempt sysctl setting in containers\n"
"\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
"!!!!"
)
@@ -170,20 +172,25 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
persist=True,
)
deployer.ensure_line(
timezone_env = files.line(
name="Set TZ environment variable",
path="/etc/environment",
line="TZ=:/etc/localtime",
)
need_restart |= timezone_env.changed
deployer.put_file(
"service/10_restart_on_failure.conf",
"/etc/systemd/system/dovecot.service.d/10_restart.conf",
restart_conf = files.put(
name="dovecot: restart automatically on failure",
src=get_resource("service/10_restart.conf"),
dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
)
daemon_reload |= restart_conf.changed
# Validate dovecot configuration before restart
if deployer.need_restart:
if need_restart:
server.shell(
name="Validate dovecot configuration",
commands=["doveconf -n >/dev/null"],
)
return need_restart, daemon_reload
+3 -26
View File
@@ -7,7 +7,6 @@ listen = 0.0.0.0
protocols = imap lmtp
auth_mechanisms = plain
auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@[]
{% if debug == true %}
auth_verbose = yes
@@ -134,11 +133,6 @@ protocol lmtp {
# mail_lua and push_notification_lua are needed for Lua push notification handler.
# <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
# Disable fsync for LMTP. May lose delivered message,
# but unlikely to cause problems with multiple relays.
# https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
mail_fsync = never
}
plugin {
@@ -150,26 +144,12 @@ plugin {
}
plugin {
# for now we define static quota-rules for all users
quota = maildir:User quota
quota_rule = *:storage={{ config.max_mailbox_size }}
quota_max_mail_size={{ config.max_message_size }}
quota_grace = 0
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
# The percentages are chosen to prevent current Delta Chat users
# from seeing "quota warnings" which trigger at 80% and 95%.
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
}
service quota-warning {
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
user = vmail
unix_listener quota-warning {
user = vmail
mode = 0600
}
# quota_over_flag_value = TRUE
}
# push_notification configuration
@@ -272,9 +252,6 @@ protocol imap {
# sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
rawlog_dir = %h
# Disable fsync for IMAP. May lose IMAP changes like setting flags.
mail_fsync = never
}
{% endif %}
+38 -12
View File
@@ -1,7 +1,10 @@
import io
from pyinfra import host
from pyinfra.facts.files import File
from pyinfra.operations import files, systemd
from ..basedeploy import Deployer
from cmdeploy.basedeploy import Deployer, get_resource
class ExternalTlsDeployer(Deployer):
@@ -20,22 +23,45 @@ class ExternalTlsDeployer(Deployer):
def configure(self):
# Verify cert and key exist on the remote host using pyinfra facts.
for path in (self.cert_path, self.key_path):
if host.get_fact(File, path=path) is None:
info = host.get_fact(File, path=path)
if info is None:
raise Exception(f"External TLS file not found on server: {path}")
self.ensure_systemd_unit(
"external/tls-cert-reload.path.j2",
cert_path=self.cert_path,
)
self.ensure_systemd_unit(
"external/tls-cert-reload.service",
# Deploy the .path unit (templated with the cert path).
# pkg=__package__ is required here because the resource files
# live in cmdeploy.external, not the default cmdeploy package.
source = get_resource("tls-cert-reload.path.f", pkg=__package__)
content = source.read_text().format(cert_path=self.cert_path).encode()
path_unit = files.put(
name="Upload tls-cert-reload.path",
src=io.BytesIO(content),
dest="/etc/systemd/system/tls-cert-reload.path",
user="root",
group="root",
mode="644",
)
service_unit = files.put(
name="Upload tls-cert-reload.service",
src=get_resource("tls-cert-reload.service", pkg=__package__),
dest="/etc/systemd/system/tls-cert-reload.service",
user="root",
group="root",
mode="644",
)
if path_unit.changed or service_unit.changed:
self.need_restart = True
def activate(self):
# No explicit reload needed here: dovecot/nginx read the cert
# on startup, and the .path watcher handles live changes.
self.ensure_service(
"tls-cert-reload.path",
systemd.service(
name="Enable tls-cert-reload path watcher",
service="tls-cert-reload.path",
running=True,
enabled=True,
restarted=self.need_restart,
daemon_reload=self.need_restart,
)
# No explicit reload needed here: dovecot/nginx read the cert
# on startup, and the .path watcher handles live changes.
@@ -9,7 +9,7 @@
Description=Watch TLS certificate for changes
[Path]
PathChanged={{ cert_path }}
PathChanged={cert_path}
[Install]
WantedBy=multi-user.target
+32 -20
View File
@@ -1,40 +1,52 @@
import os
from pyinfra import facts, host
from pyinfra.operations import files, systemd
from cmdeploy.basedeploy import Deployer
from cmdeploy.basedeploy import Deployer, get_resource
class FiltermailDeployer(Deployer):
services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
services = ["filtermail", "filtermail-incoming"]
bin_path = "/usr/local/bin/filtermail"
config_path = "/usr/local/lib/chatmaild/chatmail.ini"
def install(self):
local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
if local_bin:
self.put_executable(
src=local_bin,
dest=self.bin_path,
)
return
def __init__(self):
self.need_restart = False
def install(self):
arch = host.get_fact(facts.server.Arch)
url = f"https://github.com/chatmail/filtermail/releases/download/v0.7.4/filtermail-{arch}"
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-{arch}"
sha256sum = {
"x86_64": "484cb8dff083134aefba9fce4a6b7ef4784a0f0e28e5108ecf8bb9e58a44fd2c",
"aarch64": "66aa0ca2ca9add7a12d92883d76f8786384092adfde24a3d3a1d0b1f30d23a9e",
"x86_64": "3fd8b18282252c75a5bbfa603d8c1b65f6563e5e920bddf3e64e451b7cdb43ce",
"aarch64": "2bd191de205f7fd60158dd8e3516ab7e3efb14627696f3d7dc186bdcd9e10a43",
}[arch]
self.download_executable(url, self.bin_path, sha256sum)
self.need_restart |= files.download(
name="Download filtermail",
src=url,
sha256sum=sha256sum,
dest=self.bin_path,
mode="755",
).changed
def configure(self):
for service in self.services:
self.ensure_systemd_unit(
f"filtermail/{service}.service.j2",
self.need_restart |= files.template(
src=get_resource(f"filtermail/{service}.service.j2"),
dest=f"/etc/systemd/system/{service}.service",
user="root",
group="root",
mode="644",
bin_path=self.bin_path,
config_path=self.config_path,
)
).changed
def activate(self):
for service in self.services:
self.ensure_service(f"{service}.service")
systemd.service(
name=f"Start and enable {service}",
service=f"{service}.service",
running=True,
enabled=True,
restarted=self.need_restart,
daemon_reload=True,
)
self.need_restart = False
@@ -1,12 +0,0 @@
[Unit]
Description=Chatmail transport service
[Service]
ExecStart={{ bin_path }} {{ config_path }} transport
Restart=always
RestartSec=30
User=vmail
LimitNOFILE=524288
[Install]
WantedBy=multi-user.target
+475
View File
@@ -0,0 +1,475 @@
"""lxc-start/stop/status/test subcommands for testing with local containers."""
import os
import time
from ..util import get_git_hash, get_version_string, shell
from .incus import RELAY_IMAGE_ALIAS, Incus, RelayContainer
RELAY_NAMES = ("test0", "test1")
# -------------------------------------------------------------------
# lxc-start
# -------------------------------------------------------------------
def lxc_start_cmd_options(parser):
_add_name_args(
parser,
help_text="User relay name(s) to create (default: test0).",
)
parser.add_argument(
"--ipv4-only",
dest="ipv4_only",
action="store_true",
help="Create an IPv4-only container.",
)
parser.add_argument(
"--run",
action="store_true",
help="Run 'cmdeploy run' on each container after starting it.",
)
def lxc_start_cmd(args, out):
"""Create/Ensure and start LXC relay and DNS containers."""
with out.section("Preparing container setup"):
_lxc_start_cmd(args, out)
def _lxc_start_cmd(args, out):
ix = Incus(out)
sub = out.new_prefixed_out()
out.green("Ensuring base image ...")
ix.ensure_base_image()
out.green("Ensuring DNS container (ns-localchat) ...")
dns_ct = ix.get_dns_container()
dns_ct.ensure()
sub.print(f"DNS container IP: {dns_ct.ipv4}")
names = args.names if args.names else RELAY_NAMES
relays = list(ix.get_container(n) for n in names)
for ct in relays:
out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
ct.ensure()
ip = ct.ipv4
sub.print("Configuring container hostname ...")
ct.configure_hosts(ip)
sub.print(f"Writing {ct.ini.name} ...")
ct.write_ini(disable_ipv6=args.ipv4_only)
sub.print(f"Config: {ct.ini}")
if args.ipv4_only:
ct.disable_ipv6()
ipv6 = None
else:
output = ct.bash(
"ip -6 addr show scope global -deprecated"
" | grep -oP '(?<=inet6 )[^/]+'",
check=False,
)
ipv6 = output.strip() if output else None
sub.print(f"{_format_addrs(ip, ipv6)}")
sub.green(f"Container {ct.name!r} ready: {ct.domain} -> {ip}")
out.print()
# Reset DNS zones only for the containers we just started
started_cnames = {ct.name for ct in relays}
managed = ix.list_managed()
started = [c for c in managed if c["name"] in started_cnames]
if started:
out.print(
f"Resetting DNS zones for {len(started)} domain(s) (A + AAAA records) ..."
)
dns_ct.reset_dns_records(dns_ct.ipv4, started)
for ct in relays:
if ct.name in started_cnames:
sub.print(f"Configuring DNS in {ct.name} ...")
ct.configure_dns(dns_ct.ipv4)
# Generate the unified SSH config
out.green("Writing ssh-config ...")
ssh_cfg = ix.write_ssh_config()
sub.print(f"{ssh_cfg}")
# Verify SSH via the generated config
for ct in relays:
sub.print(f"Verifying SSH to {ct.name} via ssh-config ...")
if ct.verify_ssh(ssh_cfg):
sub.print(f"SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
else:
sub.red(f"WARNING: SSH verification failed for {ct.name}")
# Print integration suggestions
ssh_cfg = ix.ssh_config_path
if not ix.check_ssh_include():
sub.green(
"\n(Optional) To use containers from any SSH client, add to ~/.ssh/config:"
)
sub.green(f" Include {ssh_cfg}")
# Optionally run cmdeploy run + dns on each relay
if args.run:
for ct in relays:
with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
if ret:
out.red(f"Deploy to {ct.sname} failed (exit {ret})")
return ret
with out.section("loading DNS zones"):
for ct in relays:
ret = _run_cmdeploy(
"dns", ct, ix, out,
extra=["--zonefile", str(ct.zone)],
)
if ret:
out.red(f"DNS for {ct.sname} failed (exit {ret})")
return ret
if ct.zone.exists():
dns_ct.set_dns_records(ct.zone.read_text())
out.print(f"Restarting filtermail-incoming on {ct.name}")
ct.bash("systemctl restart filtermail-incoming")
# -------------------------------------------------------------------
# lxc-stop
# -------------------------------------------------------------------
def lxc_stop_cmd_options(parser):
parser.add_argument(
"--destroy",
action="store_true",
help="Delete containers and their config files after stopping.",
)
parser.add_argument(
"--destroy-all",
dest="destroy_all",
action="store_true",
help="Like --destroy, but also remove the ns-localchat DNS container.",
)
_add_name_args(
parser,
help_text="Container name(s) to stop (default: test0 + test1).",
)
def lxc_stop_cmd(args, out):
"""Stop (and optionally destroy) local LXC relay containers."""
ix = Incus(out)
names = args.names or RELAY_NAMES
destroy = args.destroy or args.destroy_all
for ct in map(ix.get_container, names):
if destroy:
out.green(f"Destroying container {ct.name!r} ...")
ct.destroy()
else:
out.green(f"Stopping container {ct.name!r} ...")
ct.stop(force=True)
if args.destroy_all:
dns_ct = ix.get_dns_container()
out.green(f"Destroying DNS container {dns_ct.name!r} ...")
dns_ct.destroy()
ix.delete_images()
if destroy:
ix.write_ssh_config()
out.green("LXC containers destroyed.")
else:
out.green("LXC containers stopped.")
# -------------------------------------------------------------------
# lxc-test
# -------------------------------------------------------------------
def lxc_test_cmd_options(parser):
parser.add_argument(
"--one",
action="store_true",
help="Only deploy and test against test0 (skip test1).",
)
def lxc_test_cmd(args, out):
"""Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
All commands run directly on the host using
``--ssh-config lxconfigs/ssh-config`` for SSH access.
"""
ix = Incus(out)
t_total = time.time()
relay_names = list(RELAY_NAMES)
if args.one:
relay_names = relay_names[:1]
local_hash = get_git_hash()
# Per-relay: start, deploy, then snapshot the first relay as a
# reusable image so the second relay launches pre-deployed.
ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
for ct in map(ix.get_container, relay_names):
name = ct.sname
ipv4_only = ipv4_only_flags.get(name, False)
v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
start_cmd = f"cmdeploy lxc-start{v_flag} {name}"
if ipv4_only:
start_cmd += " --ipv4-only"
with out.section(f"cmdeploy lxc-start: {name}"):
ret = out.shell(start_cmd, cwd=str(ix.project_root))
if ret:
return ret
status = _deploy_status(ct, local_hash, ix)
with out.section(f"cmdeploy run: {name}"):
if "IN-SYNC" in status:
out.print(f"{name} is {status}, skipping")
else:
ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
if ret:
out.red(f"Deploy to {name} failed (exit {ret})")
return ret
# Snapshot the first relay so subsequent ones launch pre-deployed
if not ix.find_image([RELAY_IMAGE_ALIAS]):
with out.section("lxc-test: caching relay image"):
ct.publish_as_relay_image()
for ct in map(ix.get_container, relay_names):
with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
if ret:
out.red(f"DNS for {ct.sname} failed (exit {ret})")
return ret
with out.section(f"lxc-test: loading DNS zones {' & '.join(relay_names)}"):
dns_ct = ix.get_dns_container()
for ct in map(ix.get_container, relay_names):
if ct.zone.exists():
zone_data = ct.zone.read_text()
out.print(f"Loading {ct.zone} into PowerDNS ...")
dns_ct.set_dns_records(zone_data)
# Restart filtermail so its in-process DNS cache
# does not hold stale negative DKIM responses
# from before the zones were loaded.
for ct in map(ix.get_container, relay_names):
out.print(f"Restarting filtermail-incoming on {ct.name} ...")
ct.bash("systemctl restart filtermail-incoming")
with out.section("cmdeploy test"):
first = ix.get_container(relay_names[0])
env = None
if len(relay_names) > 1:
env = os.environ.copy()
env["CHATMAIL_DOMAIN2"] = ix.get_container(relay_names[1]).domain
ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
if ret:
out.red(f"Tests failed (exit {ret})")
return ret
elapsed = time.time() - t_total
out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
if out.section_timings:
out.print("Section timings:")
for name, secs in out.section_timings:
out.print(f" {name:.<50s} {secs:5.1f}s")
out.print(f" {'total':.<50s} {elapsed:5.1f}s")
out.section_timings.clear()
return 0
# -------------------------------------------------------------------
# lxc-status
# -------------------------------------------------------------------
def lxc_status_cmd_options(parser):
pass
def lxc_status_cmd(args, out):
"""Show status of local LXC chatmail containers."""
ix = Incus(out)
containers = ix.list_managed()
if not containers:
out.red("No LXC containers found. Run 'cmdeploy lxc-start' first.")
return 1
local_hash = get_git_hash()
# Get storage pool path for display
storage_path = None
data = ix.run_json(["storage", "show", "default"], check=False)
if data:
storage_path = data.get("config", {}).get("source")
msg = "Container status"
if storage_path:
msg += f": {storage_path}"
out.section_line(msg)
dns_ip = None
for c in containers:
_print_container_status(out, c, ix, local_hash)
if c["name"] == ix.get_dns_container().name:
dns_ip = c["ip"]
out.section_line("Host ssh and DNS configuration")
_print_ssh_status(out, ix)
_print_dns_forwarding_status(out, dns_ip)
return 0
def _print_container_status(out, c, ix, local_hash):
"""Print name/status, domain/IPs, and RAM for one container."""
cname = c["name"]
is_running = c.get("status") == "Running"
ct = ix.get_container(cname)
# First line: name + running/STOPPED + deploy status
if not is_running:
tag = "STOPPED"
elif not isinstance(ct, RelayContainer):
tag = "running"
else:
tag = f"running {_deploy_status(ct, local_hash, ix)}"
out.print(f"{cname:20s} {tag}")
# Second line: domain, IPv4, IPv6
domain = c.get("domain", "")
ip = c.get("ip") or "?"
ipv6 = c.get("ipv6")
out.print(f"{domain:20s} {_format_addrs(ip, ipv6)}")
# Third line: RAM (RSS), config
detail_out = out.new_prefixed_out(" " * 21)
try:
used, total = ct.rss_mib()
except Exception:
ram_str = "RSS ?"
else:
ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
if isinstance(ct, RelayContainer):
detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
else:
detail = ram_str
detail_out.print(detail)
out.print()
def _print_ssh_status(out, ix):
"""Print SSH integration status."""
ssh_cfg = ix.ssh_config_path
if ix.check_ssh_include():
out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
else:
out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
sub = out.new_prefixed_out()
sub.print("Add to ~/.ssh/config:")
sub.print(f" Include {ssh_cfg}")
def _print_dns_forwarding_status(out, dns_ip):
"""Print host DNS forwarding status for .localchat."""
sub = out.new_prefixed_out()
if not dns_ip:
out.red("DNS: ns-localchat container not found")
return
try:
rv = shell("resolvectl status incusbr0")
dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
except Exception:
dns_ok = None
if dns_ok is True:
out.green(f"DNS: .localchat forwarding to {dns_ip}")
elif dns_ok is False:
out.red("DNS: .localchat forwarding NOT configured")
sub.print("Run:")
sub.print(f" sudo resolvectl dns incusbr0 {dns_ip}")
sub.print(" sudo resolvectl domain incusbr0 ~localchat")
else:
sub.print("DNS: .localchat forwarding status UNKNOWN")
# -------------------------------------------------------------------
# Internal helpers
# -------------------------------------------------------------------
def _format_addrs(ip, ipv6=None):
parts = [f"IPv4 {ip}"]
if ipv6:
parts.append(f"IPv6 {ipv6}")
return ", ".join(parts)
def _deploy_status(ct, local_hash, ix):
"""Return a human-readable deploy status string.
Compares the full deployed version (hash + diff) against
the local state built by :func:`~cmdeploy.util.get_version_string`.
"""
deployed = ct.deployed_version()
if deployed is None:
return "NOT DEPLOYED"
# A container launched from the relay image has the same
# git hash but a different domain — always redeploy.
deployed_domain = ct.deployed_domain()
if deployed_domain and deployed_domain != ct.domain:
return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
deployed_lines = deployed.splitlines()
deployed_hash = deployed_lines[0] if deployed_lines else ""
short = deployed_hash[:12]
if not local_hash:
return f"UNKNOWN (deployed: {short})"
local_short = local_hash[:12]
if deployed_hash != local_hash:
return f"STALE (deployed: {short}, local: {local_short})"
# Hash matches — check for uncommitted diffs
local_version = get_version_string()
if deployed != local_version:
return f"DIRTY ({local_short}, undeployed changes)"
return f"IN-SYNC ({short})"
def _add_name_args(parser, help_text):
parser.add_argument("names", nargs="*", metavar="NAME", help=help_text)
def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
"""Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
*ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
Returns the subprocess exit code.
"""
extra_str = " ".join(extra) if extra else ""
v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
cmd = f"""
cmdeploy {subcmd}{v_flag}
--config {ct.ini}
--ssh-config {ix.ssh_config_path}
--ssh-host {ct.domain}
{extra_str}
"""
if "cwd" not in kwargs:
kwargs["cwd"] = str(ix.project_root)
return out.shell(cmd, **kwargs)
+768
View File
@@ -0,0 +1,768 @@
"""Core Incus operations for local chatmail LXC containers."""
import json
import subprocess
import textwrap
import time
from pathlib import Path
from ..util import shell
LABEL_KEY = "user.localchat-managed"
SSH_KEY_NAME = "id_localchat"
DOMAIN_SUFFIX = ".localchat"
UPSTREAM_IMAGE = "images:debian/12"
BASE_IMAGE_ALIAS = "localchat-base"
BASE_SETUP_NAME = "localchat-base-setup"
RELAY_IMAGE_ALIAS = "localchat-relay"
DNS_CONTAINER_NAME = "ns-localchat"
DNS_DOMAIN = "ns.localchat"
class DNSConfigurationError(Exception):
"""Raised when the DNS container is not reachable or not answering."""
def _extract_ip(net_data, family="inet"):
"""Extract the first global-scope IP of *family* from network state data.
*net_data* is the ``state.network`` dict from ``incus list --format=json``.
*family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
Returns the address string, or None.
"""
for iface_name, iface in net_data.items():
if iface_name == "lo":
continue
for addr in iface.get("addresses", []):
if addr["family"] == family and addr["scope"] == "global":
return addr["address"]
return None
class Incus:
"""Gateway for all Incus container operations.
Instantiated once per CLI command and passed around so that
all modules share a single entry point for Incus interactions.
"""
def __init__(self, out):
self.out = out
self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
self.lxconfigs_dir = self.project_root / "lxconfigs"
self.lxconfigs_dir.mkdir(exist_ok=True)
self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
if not self.ssh_key_path.exists():
shell(
f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
check=True,
)
self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
def write_ssh_config(self):
"""Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
Each Host block maps the container name, the domain name, and the
short relay name (e.g. ``_test0``) to the container's IP, using the
shared localchat SSH key. Returns the path to the file.
"""
containers = self.list_managed()
key_path = self.ssh_key_path
lines = ["# Auto-generated by cmdeploy lxc-start — do not edit\n"]
for c in containers:
hosts = [c["name"]]
domain = c.get("domain", "")
if domain and domain != c["name"]:
hosts.append(domain)
short = domain.split(".")[0]
if short and short not in hosts:
hosts.append(short)
lines.append(f"\nHost {' '.join(hosts)}\n")
lines.append(f" Hostname {c['ip']}\n")
lines.append(" User root\n")
lines.append(f" IdentityFile {key_path}\n")
lines.append(" IdentitiesOnly yes\n")
lines.append(" StrictHostKeyChecking accept-new\n")
lines.append(" UserKnownHostsFile /dev/null\n")
lines.append(" LogLevel ERROR\n")
path = self.ssh_config_path
path.write_text("".join(lines))
return path
def check_ssh_include(self):
"""Check if the user's ~/.ssh/config already includes our ssh-config."""
user_ssh_config = Path.home() / ".ssh" / "config"
if not user_ssh_config.exists():
return False
lines = user_ssh_config.read_text().splitlines()
target = f"include {self.ssh_config_path}".lower()
return any(line.strip().lower() == target for line in lines)
def get_host_nameservers(self):
"""Return upstream nameservers found on the host."""
ns = []
for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
p = Path(path)
if p.exists():
for line in p.read_text().splitlines():
if line.strip().startswith("nameserver "):
addr = line.split()[1]
if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
if addr not in ns:
ns.append(addr)
if ns:
break
return ns
def run(self, args, check=True, capture=True, input=None):
"""Run an incus command.
When *capture* is True and *verbosity* >= 1, output is streamed
to the terminal line-by-line while also being captured for
later return via result.stdout.
"""
cmd = ["incus", "--quiet"] + list(args)
sub = self.out.new_prefixed_out(" ")
if not capture:
# Simple case: let subprocess handle streams (no capture)
if self.out.verbosity >= 1:
sub.print(f"$ {' '.join(cmd)}")
return subprocess.run(
cmd, text=True, input=input, check=check, stdout=None, stderr=None
)
# Capture case: we may need to stream while capturing
if sub.verbosity >= 1:
cmd_lines = " ".join(cmd).splitlines()
sub.print(f"$ {cmd_lines.pop(0)}")
if sub.verbosity >= 2:
for line in cmd_lines:
sub.print(f" {line}")
proc = subprocess.Popen(
cmd,
text=True,
stdin=subprocess.PIPE if input else subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_lines = []
if input:
proc.stdin.write(input)
proc.stdin.close()
for line in proc.stdout:
stdout_lines.append(line)
if sub.verbosity >= 2:
sub.print(f" > {line.rstrip()}")
stderr = proc.stderr.read()
ret = proc.wait()
stdout = "".join(stdout_lines)
if check and ret != 0:
full_output = stdout + stderr
for line in full_output.splitlines():
if sub.verbosity < 1: # and we haven't printed it yet
sub.red(line)
raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
def run_json(self, args, check=True):
"""Run an incus command with ``--format=json``.
Returns the parsed JSON on success.
When *check* is True raises ``subprocess.CalledProcessError``
on non-zero exit; when False returns *None* instead.
"""
result = self.run(
list(args) + ["--format=json"],
check=check,
)
if result.returncode != 0:
return None
return json.loads(result.stdout)
def run_output(self, args, check=True):
"""Run an incus command and return its stripped stdout.
When *check* is False, returns *None* on non-zero exit
instead of raising.
"""
result = self.run(args, check=check)
if result.returncode != 0:
return None
return result.stdout.strip()
def find_image(self, aliases):
"""Return the first alias from *aliases* that exists, else None."""
images = self.run_json(["image", "list"], check=False) or []
existing = {a.get("name") for img in images for a in img.get("aliases", [])}
for alias in aliases:
if alias in existing:
return alias
return None
def delete_images(self):
"""Delete the cached base and relay images."""
for alias in (RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
self.run(["image", "delete", alias], check=False) # ok if absent
def list_managed(self):
"""Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
containers = []
for ct in self.run_json(["list"]):
config = ct.get("config", {})
if config.get(LABEL_KEY) != "true":
continue
name = ct["name"]
state = ct.get("state", {})
net = state.get("network") or {}
containers.append(
{
"name": name,
"ip": _extract_ip(net, "inet"),
"ipv6": _extract_ip(net, "inet6"),
"domain": config.get(
"user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
),
"status": ct.get("status", "Unknown"),
"memory_usage": state.get("memory", {}).get("usage", 0),
}
)
return containers
def ensure_base_image(self):
"""Build and cache a base image with openssh and the SSH key.
The image is published as a local incus image with alias
'localchat-base'. Subsequent container launches use this
image instead of the upstream Debian 12, skipping the
slow apt-get install step.
Returns the image alias.
"""
if self.find_image([BASE_IMAGE_ALIAS]):
self.out.print(f" Base image '{BASE_IMAGE_ALIAS}' already cached.")
return BASE_IMAGE_ALIAS
self.out.print(" Building base image (one-time setup) ...")
self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
self.run(["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME])
ct = Container(self, BASE_SETUP_NAME)
ct.wait_ready()
key_path = self.ssh_key_path
pub_key = key_path.with_suffix(".pub").read_text().strip()
host_ns = self.get_host_nameservers()
ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
ct.bash(f"""
printf '{ns_lines}\n' > /etc/resolv.conf
apt-get -o DPkg::Lock::Timeout=60 update
DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
systemctl enable ssh
apt-get clean
mkdir -p /root/.ssh
chmod 700 /root/.ssh
echo '{pub_key}' > /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
""")
self.run(["stop", BASE_SETUP_NAME])
self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
self.run(["delete", BASE_SETUP_NAME, "--force"])
self.out.print(f" Base image '{BASE_IMAGE_ALIAS}' ready.")
return BASE_IMAGE_ALIAS
def get_container(self, name):
"""Return a container handle for the given name.
Accepts both short relay names (``test0``) and full Incus
container names (``test0-localchat``). Returns
``DNSContainer`` for the DNS container and
``RelayContainer`` for everything else.
"""
if name == DNS_CONTAINER_NAME:
return DNSContainer(self)
return RelayContainer(self, name.removesuffix("-localchat"))
def get_dns_container(self):
"""Return a DNSContainer handle."""
return DNSContainer(self)
class Container:
"""The base container handle wraps all interactions with incus."""
def __init__(self, incus, name, domain=None):
self.incus = incus
self.out = incus.out
self.name = name
self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
self.ipv4 = None
self.ipv6 = None
def bash(self, script, check=True):
"""Returns stdout from executing ``bash -ec <script>`` inside this container.
*script* is dedented and stripped so callers can use triple-quoted strings.
When *check* is False, returns *None* on non-zero exit instead of raising.
"""
script = textwrap.dedent(script).strip()
cmd = ["exec", self.name, "--", "bash", "-ec", script]
return self.incus.run_output(cmd, check=check)
def run_cmd(self, *args, check=True):
"""Return stdout from running a command directly in the container (no shell).
When *check* is False, returns *None* on non-zero exit instead of raising.
"""
return self.incus.run_output(
["exec", self.name, "--", *args],
check=check,
)
def start(self):
self.incus.run(["start", self.name])
def stop(self, force=False):
cmd = ["stop", self.name]
if force:
cmd.append("--force")
self.incus.run(cmd, check=False)
def launch(self):
"""Launch from the best available image, return the alias used."""
image = self.incus.find_image([RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
if not image:
raise RuntimeError(
f"No base image '{BASE_IMAGE_ALIAS}' found. "
"Call ensure_base_image() before launching containers."
)
self.out.print(f" Launching from '{image}' image ...")
cfg = []
cfg += ("-c", f"{LABEL_KEY}=true")
cfg += ("-c", f"user.localchat-domain={self.domain}")
self.incus.run(["launch", image, self.name, *cfg])
return image
def ensure(self):
"""Create/start this container from the cached base image.
On first call, builds the base image (~30s).
Subsequent containers launch in ~2s from the cached image.
Returns ``self`` for chaining.
"""
data = self.incus.run_json(["list", self.name], check=False) or []
existing = [c for c in data if c["name"] == self.name]
if existing:
if existing[0]["status"] != "Running":
self.start()
else:
self.launch()
self.wait_ready()
return self
def destroy(self):
"""Stop, delete, and clean up config files."""
self.stop(force=True)
self.incus.run(["delete", self.name, "--force"], check=False)
def push_file_content(self, dest_path, content):
"""Write *content* to *dest_path* inside the container.
*content* is dedented and stripped so callers can use
indented triple-quoted strings.
"""
content = textwrap.dedent(content).strip() + "\n"
self.incus.run(
["file", "push", "-", f"{self.name}{dest_path}"],
input=content,
)
self.bash(f"chmod 644 {dest_path}")
def wait_ready(self, timeout=60):
"""Wait until the container is running with an IPv4 address.
Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
or raises ``TimeoutError``.
"""
deadline = time.time() + timeout
while time.time() < deadline:
data = self.incus.run_json(
["list", self.name],
check=False,
)
if data and data[0].get("status") == "Running":
net = data[0].get("state", {}).get("network", {})
self.ipv4 = _extract_ip(net, "inet")
self.ipv6 = _extract_ip(net, "inet6")
if self.ipv4:
return
time.sleep(1)
raise TimeoutError(
f"Container {self.name!r} did not become ready within {timeout}s"
)
def rss_mib(self):
"""Return ``(used, total)`` memory from container (or None if unobtainable)."""
output = self.bash("free -m", check=False)
if output:
for line in output.splitlines():
if line.startswith("Mem:"):
parts = line.split()
return int(parts[2]), int(parts[1])
class RelayContainer(Container):
"""Container handle for a chatmail relay.
Accepts the short relay name (e.g. ``test0``) and derives
the Incus container name and mail domain automatically.
"""
def __init__(self, incus, name):
super().__init__(
incus,
f"{name}-localchat",
domain=f"_{name}{DOMAIN_SUFFIX}",
)
self.sname = name
self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
self.zone = incus.lxconfigs_dir / f"{name}.zone"
def launch(self):
"""Launch (from a potentially cached image) and clear inherited chatmail-version."""
image = super().launch()
self.bash("rm -f /etc/chatmail-version")
return image
def destroy(self):
"""Stop, delete, and clean up config files."""
super().destroy()
if self.ini.exists():
self.ini.unlink()
def disable_ipv6(self):
"""Disable IPv6 inside the container via sysctl."""
# incus provides net.* virtualization for LXC containers so that
# these sysctls only affect the container's network namespace.
self.bash("""
sysctl -w net.ipv6.conf.all.disable_ipv6=1
sysctl -w net.ipv6.conf.default.disable_ipv6=1
""")
self.push_file_content(
"/etc/sysctl.d/99-disable-ipv6.conf",
"""
net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
""",
)
def configure_hosts(self, ip):
"""Set hostname and /etc/hosts inside the container."""
self.bash(f"""
echo '{self.name}' > /etc/hostname
hostname {self.name}
sed -i '/ {self.domain}$/d' /etc/hosts
echo '{ip} {self.name} {self.domain}' >> /etc/hosts
""")
def publish_as_relay_image(self):
"""Publish this container as a reusable relay image.
Stops the container, 'publishes' it as 'localchat-relay', then restarts it.
"""
if self.incus.find_image([RELAY_IMAGE_ALIAS]):
return
self.out.print(
f" Locally caching {self.name!r} as '{RELAY_IMAGE_ALIAS}' image ..."
)
self.incus.run(
["publish", self.name, f"--alias={RELAY_IMAGE_ALIAS}", "--force"]
)
self.wait_ready()
self.out.print(f" Relay image '{RELAY_IMAGE_ALIAS}' ready.")
def deployed_version(self):
"""Read /etc/chatmail-version, or None if absent."""
return self.bash("cat /etc/chatmail-version", check=False)
def deployed_domain(self):
"""Read the domain deployed on the container (postfix myhostname)."""
return self.bash(
"postconf -h myhostname 2>/dev/null",
check=False,
)
def verify_ssh(self, ssh_config):
"""Verify SSH connectivity to this container."""
cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
return shell(cmd, timeout=60).returncode == 0
def configure_dns(self, dns_ip):
"""Point this container's resolver at *dns_ip* and verify DNS is reachable."""
self.bash(f"""
systemctl disable --now systemd-resolved 2>/dev/null || true
rm -f /etc/resolv.conf
printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
mkdir -p /etc/unbound/unbound.conf.d
""")
self.push_file_content(
"/etc/unbound/unbound.conf.d/localchat-forward.conf",
f"""
server:
domain-insecure: "localchat"
forward-zone:
name: "localchat"
forward-addr: {dns_ip}
""",
)
self.bash("systemctl restart unbound 2>/dev/null || true")
self._wait_dns_reachable(dns_ip)
def _wait_dns_reachable(self, dns_ip, timeout=10):
"""Poll until *dns_ip* answers a DNS query from this container."""
if self.bash("which dig", check=False) is None:
self.bash(
"DEBIAN_FRONTEND=noninteractive "
"apt-get install -y dnsutils 2>/dev/null || true"
)
deadline = time.time() + timeout
while time.time() < deadline:
result = self.bash(
f"dig @{dns_ip} . SOA +short +time=1 +tries=1",
check=False,
)
if result and result.strip():
return
time.sleep(0.5)
raise DNSConfigurationError(
f"DNS at {dns_ip} not reachable from {self.name} after {timeout}s"
)
def write_ini(self, disable_ipv6=False):
"""Generate a chatmail.ini config file in lxconfigs/."""
from chatmaild.config import write_initial_config
overrides = {
"max_user_send_per_minute": 600,
"max_user_send_burst_size": 100,
"mtail_address": "127.0.0.1",
}
if disable_ipv6:
overrides["disable_ipv6"] = "True"
write_initial_config(self.ini, self.domain, overrides)
return self.ini
class DNSContainer(Container):
"""Container handle for the PowerDNS name server.
Manages the authoritative and recursive DNS services required for
name resolution in the local testing environment.
"""
def __init__(self, incus):
super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
def pdnsutil(self, *args, check=True):
"""Run ``pdnsutil <args>`` inside the DNS container."""
return self.run_cmd("pdnsutil", *args, check=check)
def replace_rrset(self, zone, name, rtype, ttl, rdata):
"""Shortcut for ``pdnsutil replace-rrset``."""
self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
def restart_services(self):
"""Restart pdns and pdns-recursor, then wait until DNS is answering."""
self.bash("""
systemctl restart pdns
systemctl restart pdns-recursor || true
""")
self._wait_dns_ready()
def _wait_dns_ready(self, timeout=60):
"""Poll until the recursor answers a query on port 53."""
deadline = time.time() + timeout
while time.time() < deadline:
result = self.bash(
"dig @127.0.0.1 . SOA +short +time=1 +tries=1",
check=False,
)
if result and result.strip():
return
time.sleep(0.5)
raise DNSConfigurationError(f"DNS recursor not answering after {timeout}s")
def ensure(self):
"""Create the DNS container with PowerDNS if needed.
Calls ``super().ensure()`` to create/start the container
and set up SSH, then installs PowerDNS and configures
the Incus bridge to use this container as DNS.
"""
super().ensure()
self._install_powerdns()
self.incus.run(
["network", "set", "incusbr0", "dns.mode=none"],
check=False,
)
self.incus.run(
["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
check=False,
)
def destroy(self):
"""Stop, delete, and reset bridge DNS config."""
super().destroy()
self.incus.run(["network", "unset", "incusbr0", "dns.mode"], check=False)
self.incus.run(["network", "unset", "incusbr0", "raw.dnsmasq"], check=False)
def _install_powerdns(self):
"""Install and configure PowerDNS if not already present."""
if self.run_cmd("which", "pdns_server", check=False) is not None:
return
host_ns = self.incus.get_host_nameservers()
ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
self.bash(f"""
systemctl disable --now systemd-resolved 2>/dev/null || true
rm -f /etc/resolv.conf
printf '{ns_lines}\n' > /etc/resolv.conf
# Block automatic service startup during package installation
printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
apt-get -o DPkg::Lock::Timeout=60 update
DEBIAN_FRONTEND=noninteractive apt-get install -y \
pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
# Remove the startup block
rm /usr/sbin/policy-rc.d
systemctl stop pdns pdns-recursor || true
mkdir -p /var/lib/powerdns
sqlite3 /var/lib/powerdns/pdns.sqlite3 \
</usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
chown -R pdns:pdns /var/lib/powerdns
""")
self.push_file_content(
"/etc/powerdns/pdns.conf",
"""
launch=gsqlite3
gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
local-address=127.0.0.1
local-port=5353
""",
)
self.push_file_content(
"/etc/powerdns/recursor.conf",
"""
local-address=0.0.0.0
local-port=53
forward-zones=localchat=127.0.0.1:5353
allow-from=0.0.0.0/0
dont-query=
dnssec=off
""",
)
self.bash("""
systemctl start pdns
systemctl start pdns-recursor
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
""")
self._wait_dns_ready()
def reset_dns_records(self, dns_ip, domains):
"""Create DNS zones with initial A records via pdnsutil.
Only sets SOA, NS, and A records as the minimal set
needed for SSH connectivity. Full records (MX, TXT, SRV,
CNAME, DKIM) are added later by ``cmdeploy dns``.
Args:
dns_ip: IP of the DNS container
domains: list of dicts with 'name', 'domain', 'ip'
"""
for d in domains:
domain = d["domain"]
ip = d["ip"]
self.out.print(f" {domain} -> {ip}")
# Delete and recreate zone fresh (removes stale records)
self.pdnsutil("delete-zone", domain, check=False)
self.pdnsutil("create-zone", domain, f"ns.{domain}")
serial = str(int(time.time()))
soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
self.replace_rrset(domain, ".", "SOA", "3600", soa)
self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
self.replace_rrset(domain, ".", "A", "3600", ip)
self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
# AAAA (domain -> container IPv6, if available)
ipv6 = d.get("ipv6")
if ipv6:
self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
self.out.print(f" zone reset: SOA, NS, A, AAAA ({ip}, {ipv6})")
else:
# Remove any stale AAAA record
self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
self.out.print(f" zone reset: SOA, NS, A ({ip}, IPv4-only)")
self.restart_services()
def set_dns_records(self, text):
"""Add or overwrite DNS records from standard BIND format.
Uses ``cmdeploy.dns.parse_zone_records`` to parse.
Zones are created automatically from the record names.
"""
from ..dns import parse_zone_records
zones_seen = set()
for name, ttl, rtype, rdata in parse_zone_records(text):
# Derive zone from name: find top-level .localchat domain
name_parts = name.split(".")
zone = name # fallback
for i in range(len(name_parts) - 1):
if name_parts[i + 1 :] == ["localchat"]:
zone = ".".join(name_parts[i:])
break
# Create zone if first time seeing it
if zone not in zones_seen:
self.pdnsutil(
"create-zone",
zone,
f"ns.{zone}",
check=False,
)
zones_seen.add(zone)
# Figure out the record name relative to zone
if name == zone:
relative = "."
elif name.endswith(f".{zone}"):
relative = name[: -(len(zone) + 1)]
else:
relative = name
self.replace_rrset(zone, relative, rtype, ttl, rdata)
if zones_seen:
self.restart_services()
@@ -73,21 +73,8 @@ counter incoming_unencrypted_mail_count
filtered_incoming_mail_count++
}
counter incoming_mailer_daemon_mail_count
/Incoming: Filtering mailer-daemon message from/ {
incoming_mailer_daemon_mail_count++
filtered_incoming_mail_count++
}
counter rejected_unencrypted_mail_count
/Rejected unencrypted mail/ {
rejected_unencrypted_mail_count++
}
counter quota_expire_runs
counter quota_expire_removed_files
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
quota_expire_runs++
quota_expire_removed_files += $count
}
+38 -14
View File
@@ -1,7 +1,10 @@
from pyinfra import facts, host
from pyinfra.operations import apt
from pyinfra.operations import apt, files, server, systemd
from cmdeploy.basedeploy import Deployer
from cmdeploy.basedeploy import (
Deployer,
get_resource,
)
class MtailDeployer(Deployer):
@@ -15,30 +18,51 @@ class MtailDeployer(Deployer):
(url, sha256sum) = {
"x86_64": (
"https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
"d55cb601049c5e61eabab29998dbbcea95d480e5448544f9470337ba2eea882e",
"123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
),
"aarch64": (
"https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
"f748db8ad2a1e0b63684d4c8868cf6a373a20f7e6922e5ece601fff0ee00eb1a",
"aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
),
}[host.get_fact(facts.server.Arch)]
self.download_executable(
url,
"/usr/local/bin/mtail",
sha256sum,
extract="gunzip | tar -xf - mtail -O",
server.shell(
name="Download mtail",
commands=[
f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
"chmod 755 /usr/local/bin/mtail",
],
)
def configure(self):
# Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
# This allows to read from journalctl instead of log files.
self.ensure_systemd_unit(
"mtail/mtail.service.j2",
files.template(
src=get_resource("mtail/mtail.service.j2"),
dest="/etc/systemd/system/mtail.service",
user="root",
group="root",
mode="644",
address=self.mtail_address or "127.0.0.1",
port=3903,
)
self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
mtail_conf = files.put(
name="Mtail configuration",
src=get_resource("mtail/delivered_mail.mtail"),
dest="/etc/mtail/delivered_mail.mtail",
user="root",
group="root",
mode="644",
)
self.need_restart = mtail_conf.changed
def activate(self):
active = bool(self.mtail_address)
self.ensure_service("mtail.service", running=active, enabled=active)
systemd.service(
name="Start and enable mtail",
service="mtail.service",
running=bool(self.mtail_address),
enabled=bool(self.mtail_address),
restarted=self.need_restart,
)
self.need_restart = False
@@ -1,13 +1,10 @@
[Unit]
Description=mtail
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
Restart=on-failure
RestartSec=2s
[Install]
WantedBy=multi-user.target
+54 -17
View File
@@ -1,5 +1,5 @@
from chatmaild.config import Config
from pyinfra.operations import apt
from pyinfra.operations import apt, files, systemd
from cmdeploy.basedeploy import (
Deployer,
@@ -31,50 +31,87 @@ class NginxDeployer(Deployer):
# For documentation about policy-rc.d, see:
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
#
self.put_executable(src="policy-rc.d", dest="/usr/sbin/policy-rc.d")
files.put(
src=get_resource("policy-rc.d"),
dest="/usr/sbin/policy-rc.d",
user="root",
group="root",
mode="755",
)
apt.packages(
name="Install nginx",
packages=["nginx", "libnginx-mod-stream"],
)
self.remove_file("/usr/sbin/policy-rc.d")
files.file("/usr/sbin/policy-rc.d", present=False)
def configure(self):
_configure_nginx(self, self.config)
self.need_restart = _configure_nginx(self.config)
def activate(self):
self.ensure_service("nginx.service")
systemd.service(
name="Start and enable nginx",
service="nginx.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
self.need_restart = False
def _configure_nginx(deployer, config: Config, debug: bool = False):
def _configure_nginx(config: Config, debug: bool = False) -> bool:
"""Configures nginx HTTP server."""
need_restart = False
deployer.put_template(
"nginx/nginx.conf.j2",
"/etc/nginx/nginx.conf",
main_config = files.template(
src=get_resource("nginx/nginx.conf.j2"),
dest="/etc/nginx/nginx.conf",
user="root",
group="root",
mode="644",
config=config,
disable_ipv6=config.disable_ipv6,
)
need_restart |= main_config.changed
deployer.put_template(
"nginx/autoconfig.xml.j2",
"/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
autoconfig = files.template(
src=get_resource("nginx/autoconfig.xml.j2"),
dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
user="root",
group="root",
mode="644",
config=config,
)
need_restart |= autoconfig.changed
deployer.put_template(
"nginx/mta-sts.txt.j2",
"/var/www/html/.well-known/mta-sts.txt",
mta_sts_config = files.template(
src=get_resource("nginx/mta-sts.txt.j2"),
dest="/var/www/html/.well-known/mta-sts.txt",
user="root",
group="root",
mode="644",
config=config,
)
need_restart |= mta_sts_config.changed
# install CGI newemail script
#
cgi_dir = "/usr/lib/cgi-bin"
deployer.ensure_directory(cgi_dir)
files.directory(
name=f"Ensure {cgi_dir} exists",
path=cgi_dir,
user="root",
group="root",
)
deployer.put_executable(
files.put(
name="Upload cgi newemail.py script",
src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
dest=f"{cgi_dir}/newemail.py",
user="root",
group="root",
mode="755",
)
return need_restart
+3 -7
View File
@@ -42,9 +42,6 @@ stream {
}
http {
# access_log setting is inherited by all server sections
access_log syslog:server=unix:/dev/log,facility=local7;
{% if config.tls_cert_mode == "self" %}
limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
{% endif %}
@@ -72,11 +69,9 @@ http {
index index.html index.htm;
server_name {{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
location /mxdeliv {
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
}
access_log syslog:server=unix:/dev/log,facility=local7;
location / {
# First attempt to serve request as file, then
@@ -144,6 +139,7 @@ http {
listen 127.0.0.1:8443 ssl;
server_name www.{{ config.mail_domain }};
return 301 $scheme://{{ config.mail_domain }}$request_uri;
access_log syslog:server=unix:/dev/log,facility=local7;
}
server {
+63 -25
View File
@@ -4,9 +4,9 @@ Installs OpenDKIM
from pyinfra import host
from pyinfra.facts.files import File
from pyinfra.operations import apt, files, server
from pyinfra.operations import apt, files, server, systemd
from cmdeploy.basedeploy import Deployer
from cmdeploy.basedeploy import Deployer, get_resource
class OpendkimDeployer(Deployer):
@@ -25,39 +25,65 @@ class OpendkimDeployer(Deployer):
domain = self.mail_domain
dkim_selector = "opendkim"
"""Configures OpenDKIM"""
need_restart = False
self.put_template(
"opendkim/opendkim.conf",
"/etc/opendkim.conf",
main_config = files.template(
src=get_resource("opendkim/opendkim.conf"),
dest="/etc/opendkim.conf",
user="root",
group="root",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= main_config.changed
self.remove_file("/etc/opendkim/screen.lua")
self.remove_file("/etc/opendkim/final.lua")
screen_script = files.file(
path="/etc/opendkim/screen.lua",
present=False,
)
need_restart |= screen_script.changed
self.ensure_directory(
"/etc/opendkim",
owner="opendkim",
final_script = files.file(
path="/etc/opendkim/final.lua",
present=False,
)
need_restart |= final_script.changed
files.directory(
name="Add opendkim directory to /etc",
path="/etc/opendkim",
user="opendkim",
group="opendkim",
mode="750",
present=True,
)
self.put_template(
"opendkim/KeyTable",
"/etc/dkimkeys/KeyTable",
owner="opendkim",
keytable = files.template(
src=get_resource("opendkim/KeyTable"),
dest="/etc/dkimkeys/KeyTable",
user="opendkim",
group="opendkim",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= keytable.changed
self.put_template(
"opendkim/SigningTable",
"/etc/dkimkeys/SigningTable",
owner="opendkim",
signing_table = files.template(
src=get_resource("opendkim/SigningTable"),
dest="/etc/dkimkeys/SigningTable",
user="opendkim",
group="opendkim",
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
self.ensure_directory(
"/var/spool/postfix/opendkim",
owner="opendkim",
need_restart |= signing_table.changed
files.directory(
name="Add opendkim socket directory to /var/spool/postfix",
path="/var/spool/postfix/opendkim",
user="opendkim",
group="opendkim",
mode="750",
present=True,
)
if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
@@ -70,10 +96,12 @@ class OpendkimDeployer(Deployer):
_su_user="opendkim",
)
self.put_file(
"opendkim/systemd.conf",
"/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
service_file = files.put(
name="Configure opendkim to restart once a day",
src=get_resource("opendkim/systemd.conf"),
dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
)
need_restart |= service_file.changed
files.file(
name="chown opendkim: /etc/dkimkeys/opendkim.private",
@@ -82,5 +110,15 @@ class OpendkimDeployer(Deployer):
group="opendkim",
)
self.need_restart = need_restart
def activate(self):
self.ensure_service("opendkim.service")
systemd.service(
name="Start and enable OpenDKIM",
service="opendkim.service",
running=True,
enabled=True,
daemon_reload=self.need_restart,
restarted=self.need_restart,
)
self.need_restart = False
+71 -27
View File
@@ -1,10 +1,11 @@
from pyinfra.operations import apt, server
from pyinfra.operations import apt, files, server, systemd
from cmdeploy.basedeploy import Deployer
from cmdeploy.basedeploy import Deployer, get_resource
class PostfixDeployer(Deployer):
required_users = [("postfix", None, ["opendkim"])]
daemon_reload = False
def __init__(self, config, disable_mail):
self.config = config
@@ -18,46 +19,81 @@ class PostfixDeployer(Deployer):
def configure(self):
config = self.config
need_restart = False
self.put_template(
"postfix/main.cf.j2",
"/etc/postfix/main.cf",
main_config = files.template(
src=get_resource("postfix/main.cf.j2"),
dest="/etc/postfix/main.cf",
user="root",
group="root",
mode="644",
config=config,
disable_ipv6=config.disable_ipv6,
)
need_restart |= main_config.changed
self.put_template(
"postfix/master.cf.j2",
"/etc/postfix/master.cf",
master_config = files.template(
src=get_resource("postfix/master.cf.j2"),
dest="/etc/postfix/master.cf",
user="root",
group="root",
mode="644",
debug=False,
config=config,
)
need_restart |= master_config.changed
self.put_file(
"postfix/submission_header_cleanup",
"/etc/postfix/submission_header_cleanup",
header_cleanup = files.put(
src=get_resource("postfix/submission_header_cleanup"),
dest="/etc/postfix/submission_header_cleanup",
user="root",
group="root",
mode="644",
)
self.put_file("postfix/lmtp_header_cleanup", "/etc/postfix/lmtp_header_cleanup")
need_restart |= header_cleanup.changed
res = self.put_file(
"postfix/smtp_tls_policy_map", "/etc/postfix/smtp_tls_policy_map"
lmtp_header_cleanup = files.put(
src=get_resource("postfix/lmtp_header_cleanup"),
dest="/etc/postfix/lmtp_header_cleanup",
user="root",
group="root",
mode="644",
)
tls_policy_changed = res.changed
if tls_policy_changed:
need_restart |= lmtp_header_cleanup.changed
tls_policy_map = files.put(
name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
src=get_resource("postfix/smtp_tls_policy_map"),
dest="/etc/postfix/smtp_tls_policy_map",
user="root",
group="root",
mode="644",
)
need_restart |= tls_policy_map.changed
if tls_policy_map.changed:
server.shell(
commands=["postmap /etc/postfix/smtp_tls_policy_map"],
)
# Login map that 1:1 maps email address to login.
self.put_file("postfix/login_map", "/etc/postfix/login_map")
self.put_file(
"service/10_restart_on_failure.conf",
"/etc/systemd/system/postfix@.service.d/10_restart.conf",
login_map = files.put(
src=get_resource("postfix/login_map"),
dest="/etc/postfix/login_map",
user="root",
group="root",
mode="644",
)
need_restart |= login_map.changed
restart_conf = files.put(
name="postfix: restart automatically on failure",
src=get_resource("service/10_restart.conf"),
dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
)
self.daemon_reload = restart_conf.changed
# Validate postfix configuration before restart
if self.need_restart:
if need_restart:
server.shell(
name="Validate postfix configuration",
# Extract stderr and quit with error if non-zero
@@ -65,11 +101,19 @@ class PostfixDeployer(Deployer):
"""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
],
)
self.need_restart = need_restart
def activate(self):
active = not self.disable_mail
self.ensure_service(
"postfix.service",
running=active,
enabled=active,
restart = False if self.disable_mail else self.need_restart
systemd.service(
name="disable postfix for now"
if self.disable_mail
else "Start and enable Postfix",
service="postfix.service",
running=False if self.disable_mail else True,
enabled=False if self.disable_mail else True,
restarted=restart,
daemon_reload=self.daemon_reload,
)
self.need_restart = False
+33 -50
View File
@@ -53,19 +53,15 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
# See <https://www.postfix.org/FORWARD_SECRECY_README.html#server_fs>.
tls_preempt_cipherlist = yes
# Reject by default, override per smtpd in master.cf
smtpd_relay_restrictions = reject
myhostname = {{ config.postfix_myhostname }}
smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
myhostname = {{ config.mail_domain }}
alias_maps = hash:/etc/aliases
alias_database = hash:/etc/aliases
# When postfix receives mail for $mydestination,
# it hands it over to dovecot via $local_transport.
# Note: IP literals must be handled via local delivery / mydestination.
mydestination = {{ config.mail_domain }}
local_transport = lmtp:unix:private/dovecot-lmtp
# postfix doesn't check whether local users exist or not:
local_recipient_maps =
# Postfix does not deliver mail for any domain by itself.
# Primary domain is listed in `virtual_mailbox_domains` instead
# and handed over to Dovecot.
mydestination =
relayhost =
{% if disable_ipv6 %}
@@ -73,6 +69,15 @@ mynetworks = 127.0.0.0/8
{% else %}
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
{% endif %}
{% if config.addr_v4 %}
smtp_bind_address = {{ config.addr_v4 }}
{% endif %}
{% if config.addr_v6 %}
smtp_bind_address6 = {{ config.addr_v6 }}
{% endif %}
{% if config.addr_v4 or config.addr_v6 %}
smtp_bind_address_enforce = yes
{% endif %}
mailbox_size_limit = 0
message_size_limit = {{config.max_message_size}}
recipient_delimiter = +
@@ -83,15 +88,25 @@ inet_protocols = ipv4
inet_protocols = all
{% endif %}
lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
# Postfix does not try IPv4 and IPv6 connections
# concurrently as of version 3.7.11.
#
# When relay has both A (IPv4) and AAAA (IPv6) records,
# but broken IPv6 connectivity,
# every second message is delayed by the connection timeout
# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
# which defaults to 30 seconds. Reducing timeouts is not a solution
# as this will result in a failure to connect to slow servers.
#
# As a workaround we always prefer IPv4 when it is available.
#
# The setting is documented at
# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
smtp_address_preference=ipv4
# Do not apply header checks to MIME headers
# and other headers that are actually part of the message body.
# Reference:
# <https://www.postfix.org/postconf.5.html#disable_mime_input_processing>
# <https://www.postfix.org/header_checks.5.html>
# <https://stump.io/blog/2020/11/29/a-little-gotcha-with-postfixs-header_checks/>
disable_mime_input_processing = yes
virtual_transport = lmtp:unix:private/dovecot-lmtp
virtual_mailbox_domains = {{ config.mail_domain }}
lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
mua_client_restrictions = permit_sasl_authenticated, reject
mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
@@ -103,35 +118,3 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
# Do not lookup SMTP client hostnames to reduce delays
# and avoid unnecessary DNS requests.
smtpd_peername_lookup = no
# Use filtermail-transport to relay messages.
# We can't force postfix to split messages per destination,
# when specifying a custom next-hop,
# so instead this is handled in filtermail.
# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}
# All deliveries over lmtp-filtermail are treated
# as having the same destination [127.0.0.1],
# so it is not possible to limit per-destination concurrency here,
# it is a job for filtermail-transport.
# Total number of parallel deliveries is limited
# by "maxproc" column in /etc/postfix/master.cf for lmtp-filtermail.
# Settings below are to prevent Postfix queue manager
# from limiting the number of LMTP connections to filtermail-transport.
# Read <https://www.postfix.org/TUNING_README.html#rope> and
# <https://www.postfix.org/SCHEDULER_README.html> for the details
# of the Postfix algorithm that we effectively disable here.
lmtp-filtermail_initial_destination_concurrency=10000
lmtp-filtermail_destination_concurrency_limit=10000
# Do not try to deliver messages for more than 2 days.
maximal_queue_lifetime = 2d
{% if not config.ipv4_relay %}
# DKIM-sign locally generated mail (bounces, DSNs).
# These bypass smtpd, so they need explicit milter configuration.
non_smtpd_milters = unix:opendkim/opendkim.sock
internal_mail_filter_classes = bounce
milter_macro_daemon_name = ORIGINATING
{% endif %}
+2 -18
View File
@@ -17,7 +17,6 @@ smtp inet n - y - - smtpd
-o smtpd_tls_security_level=encrypt
-o smtpd_tls_mandatory_protocols=>=TLSv1.2
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
-o smtpd_relay_restrictions=reject_unauth_destination
submission inet n - y - 5000 smtpd
-o syslog_name=postfix/submission
-o smtpd_tls_security_level=encrypt
@@ -76,19 +75,17 @@ lmtp unix - - y - - lmtp
anvil unix - - y - 1 anvil
scache unix - - y - 1 scache
postlog unix-dgram n - n - 1 postlogd
filter unix - n n - - lmtp
# Local SMTP server for reinjecting outgoing filtered mail.
127.0.0.1:{{ config.postfix_reinject_port }} inet n - n - 100 smtpd
-o syslog_name=postfix/reinject
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_milters=unix:opendkim/opendkim.sock
-o cleanup_service_name=authclean
-o smtpd_relay_restrictions=permit_mynetworks,reject
{% if not config.ipv4_relay %} -o smtpd_milters=unix:opendkim/opendkim.sock
{% endif %}
# Local SMTP server for reinjecting incoming filtered mail
127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet n - n - 100 smtpd
-o syslog_name=postfix/reinject_incoming
-o smtpd_relay_restrictions=reject_unauth_destination
# Cleanup `Received` headers for authenticated mail
# to avoid leaking client IP.
@@ -103,16 +100,3 @@ postlog unix-dgram n - n - 1 postlogd
# cannot send unprotected Subject.
authclean unix n - - - 0 cleanup
-o header_checks=regexp:/etc/postfix/submission_header_cleanup
# Reducing `maxproc` here may result in a head of line blocking
# when there are many messages sent to unreachable destinations
# at the same time.
# LMTP clients here talk to filtermail-transport.
# LMTP has no pipelining,
# so while filtermail-transport tries to deliver the message,
# possibly waiting for a long connection timeout
# or talking to a slow server, LMTP client cannot be reused.
lmtp-filtermail unix - - y - 500 lmtp
-o syslog_name=postfix/lmtp-filtermail
-o lmtp_header_checks=
-o lmtp_tls_security_level=none
+13 -15
View File
@@ -64,25 +64,21 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
)
def get_authoritative_ns(domain):
ns_replies = [
def query_dns(typ, domain):
# Get autoritative nameserver from the SOA record.
soa_answers = [
x.split()
for x in shell(
f"dig -r -q {domain} -t NS +noall +authority +answer", print=log_progress
f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
).split("\n")
]
filtered_replies = [a for a in ns_replies if len(a) >= 5 and a[3] == "NS"]
if not filtered_replies:
soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
if not soa:
return
return filtered_replies[0][4]
def query_dns(typ, domain):
ns = get_authoritative_ns(domain)
ns = soa[0][4]
# Query authoritative nameserver directly to bypass DNS cache.
direct_ns = f"@{ns}" if ns else ""
res = shell(f"dig {direct_ns} -r -q {domain} -t {typ} +short", print=log_progress)
res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
return next((line for line in res.split("\n") if not line.startswith(";")), "")
@@ -99,9 +95,11 @@ def check_zonefile(zonefile, verbose=True):
if not zf_line.strip() or zf_line.startswith(";"):
continue
print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
zf_domain, _ttl, _in, zf_typ, zf_value = zf_line.split(None, 4)
zf_domain = zf_domain.rstrip(".")
zf_value = zf_value.strip()
parts = zf_line.split(None, 4)
zf_domain = parts[0].rstrip(".")
# parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
zf_typ = parts[3]
zf_value = parts[4].strip()
query_value = query_dns(zf_typ, zf_domain)
if zf_value != query_value:
assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
+30 -14
View File
@@ -1,8 +1,8 @@
import shlex
from pyinfra.operations import server
from pyinfra.operations import apt, server
from ..basedeploy import Deployer
from cmdeploy.basedeploy import Deployer
def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
@@ -12,15 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
``www.<domain>`` and ``mta-sts.<domain>``.
"""
return [
"openssl", "req", "-x509",
"-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
"-noenc", "-days", str(days),
"-keyout", str(key_path),
"-out", str(cert_path),
"-subj", f"/CN={domain}",
"openssl",
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-noenc",
"-days",
str(days),
"-keyout",
str(key_path),
"-out",
str(cert_path),
"-subj",
f"/CN={domain}",
# Mark as end-entity cert so it cannot be used as a CA to sign others.
"-addext", "basicConstraints=critical,CA:FALSE",
"-addext", "extendedKeyUsage=serverAuth,clientAuth",
"-addext",
"basicConstraints=critical,CA:FALSE",
"-addext",
"extendedKeyUsage=serverAuth,clientAuth",
"-addext",
f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
]
@@ -34,11 +46,17 @@ class SelfSignedTlsDeployer(Deployer):
self.cert_path = "/etc/ssl/certs/mailserver.pem"
self.key_path = "/etc/ssl/private/mailserver.key"
def install(self):
apt.packages(
name="Install openssl",
packages=["openssl"],
)
def configure(self):
args = openssl_selfsigned_args(
self.mail_domain, self.cert_path, self.key_path,
self.mail_domain,
self.cert_path,
self.key_path,
)
cmd = shlex.join(args)
server.shell(
@@ -48,5 +66,3 @@ class SelfSignedTlsDeployer(Deployer):
def activate(self):
pass
+56 -3
View File
@@ -49,8 +49,13 @@ class SSHExec:
RemoteError = execnet.RemoteError
FuncError = FuncError
def __init__(self, host, verbose=False, python="python3", timeout=60):
self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
def __init__(
self, host, verbose=False, python="python3", timeout=60, ssh_config=None
):
spec = f"ssh=root@{host}//python={python}"
if ssh_config:
spec += f"//ssh_config={ssh_config}"
self.gateway = execnet.makegateway(spec)
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
self.timeout = timeout
self.verbose = verbose
@@ -87,8 +92,9 @@ class SSHExec:
class LocalExec:
FuncError = FuncError
def __init__(self, verbose=False):
def __init__(self, verbose=False, docker=False):
self.verbose = verbose
self.docker = docker
def __call__(self, call, kwargs=None, log_callback=None):
if kwargs is None:
@@ -100,6 +106,10 @@ class LocalExec:
if not title:
title = call.__name__
where = "locally"
if self.docker:
if call == remote.rdns.perform_initial_checks:
kwargs["pre_command"] = "docker exec chatmail "
where = "in docker"
if self.verbose:
print_stderr(f"Running {where}: {title}(**{kwargs})")
return self(call, kwargs, log_callback=print_stderr)
@@ -108,3 +118,46 @@ class LocalExec:
res = self(call, kwargs, log_callback=remote.rshell.log_progress)
print_stderr()
return res
# pyinfra exposes a ``ssh_config_file`` data key that *should* let
# paramiko parse an SSH config file directly. In practice it silently
# fails to connect (zero hosts / zero operations), so we resolve the
# hostname and identity-file ourselves and pass them via
# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
def _get_from_ssh_config(host, ssh_config_path, key):
"""Internal helper to parse a value for a specific key from ssh-config."""
current_hosts = []
found_value = None
with open(ssh_config_path) as f:
for raw_line in f:
line = raw_line.strip()
if not line or line.startswith("#"):
continue
parts = line.split(None, 1)
if not parts:
continue
directive = parts[0].lower()
if directive == "host":
if host in current_hosts and found_value:
return found_value
current_hosts = parts[1].split()
found_value = None
elif directive == key.lower():
found_value = parts[1]
if host in current_hosts and found_value:
return found_value
return None
def resolve_host_from_ssh_config(host, ssh_config_path):
"""Resolve a host alias to its IP from an ssh-config file."""
return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
def resolve_key_from_ssh_config(host, ssh_config_path):
"""Resolve a host alias to its IdentityFile from an ssh-config file."""
return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")
@@ -12,7 +12,7 @@ def test_init(tmp_path, maildomain):
inipath = tmp_path.joinpath("chatmail.ini")
main(["init", "--config", str(inipath), maildomain])
config = read_config(inipath)
assert config.mail_domain_bare == maildomain
assert config.mail_domain == maildomain
def test_capabilities(imap):
@@ -89,11 +89,12 @@ def test_concurrent_logins_same_account(
assert login_results.get()
def test_no_vrfy(cmfactory, chatmail_config, maildomain):
def test_no_vrfy(cmfactory, chatmail_config):
ac = cmfactory.get_online_account()
addr = ac.get_config("addr")
domain = chatmail_config.mail_domain
s = smtplib.SMTP(maildomain)
s = smtplib.SMTP(domain)
s.starttls()
s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
def test_newemail_configure(maildomain, rpc, chatmail_config):
def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
"""Test configuring accounts by scanning a QR code works."""
url = f"DCACCOUNT:https://{maildomain}/new"
for i in range(3):
@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
# set_config_from_qr, so fetch credentials via requests instead
res = requests.post(f"https://{maildomain}/new", verify=False)
data = res.json()
rpc.add_or_update_transport(account_id, {
"addr": data["email"],
"password": data["password"],
"imapServer": maildomain,
"smtpServer": maildomain,
"certificateChecks": "acceptInvalidCertificates",
})
rpc.add_or_update_transport(
account_id,
{
"addr": data["email"],
"password": data["password"],
"imapServer": maildomain_ip,
"smtpServer": maildomain_ip,
"certificateChecks": "acceptInvalidCertificates",
},
)
else:
rpc.add_transport_from_qr(account_id, url)
@@ -5,7 +5,6 @@ import subprocess
import time
import pytest
from chatmaild.config import is_valid_ipv4
from cmdeploy import remote
from cmdeploy.cmdeploy import get_sshexec
@@ -13,8 +12,9 @@ from cmdeploy.cmdeploy import get_sshexec
class TestSSHExecutor:
@pytest.fixture(scope="class")
def sshexec(self, sshdomain):
return get_sshexec(sshdomain)
def sshexec(self, sshdomain, pytestconfig):
ssh_config = pytestconfig.getoption("ssh_config")
return get_sshexec(sshdomain, ssh_config=ssh_config)
def test_ls(self, sshexec):
out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -22,8 +22,6 @@ class TestSSHExecutor:
assert out == out2
def test_perform_initial(self, sshexec, maildomain):
if is_valid_ipv4(maildomain):
pytest.skip(f"{maildomain} is not a domain")
res = sshexec(
remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
)
@@ -64,10 +62,8 @@ class TestSSHExecutor:
else:
pytest.fail("didn't raise exception")
def test_opendkim_restarted(self, sshexec, maildomain):
def test_opendkim_restarted(self, sshexec):
"""check that opendkim is not running for longer than a day."""
if is_valid_ipv4(maildomain):
pytest.skip(f"{maildomain} is an IPv4 relay, opendkim is not installed")
cmd = "systemctl show opendkim --timestamp=utc --property=ActiveEnterTimestamp"
out = sshexec(call=remote.rshell.shell, kwargs=dict(command=cmd))
datestring = out.split("=")[1]
@@ -76,44 +72,6 @@ class TestSSHExecutor:
assert (now - since_date).total_seconds() < 60 * 60 * 51
def test_dovecot_main_process_matches_installed_binary(sshdomain):
sshexec = get_sshexec(sshdomain)
main_pid = int(
sshexec(
call=remote.rshell.shell,
kwargs=dict(
command="timeout 10 systemctl show -p MainPID --value dovecot.service"
),
).strip()
)
assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
exe = sshexec(
call=remote.rshell.shell,
kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
).strip()
status_text = sshexec(
call=remote.rshell.shell,
kwargs=dict(
command="timeout 10 systemctl show -p StatusText --value dovecot.service"
),
).strip()
installed_version = sshexec(
call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
).strip()
assert not exe.endswith("(deleted)"), (
f"running dovecot binary was deleted (stale after upgrade): {exe}"
)
expected_status_text = f"v{installed_version}"
assert status_text == expected_status_text or status_text.startswith(
f"{expected_status_text} "
), (
f"dovecot status version mismatch: "
f"StatusText={status_text!r}, installed={installed_version!r}"
)
def test_timezone_env(remote):
for line in remote.iter_output("env"):
print(line)
@@ -175,11 +133,10 @@ def test_authenticated_from(cmsetup, maildata):
@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
def test_reject_missing_dkim(cmsetup, maildata, from_addr):
domain = cmsetup.maildomain
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(10)
try:
sock.connect((domain, 25))
except socket.timeout:
sock = socket.create_connection((domain, 25), timeout=10)
sock.close()
except (socket.timeout, OSError):
pytest.skip(f"port 25 not reachable for {domain}")
recipient = cmsetup.gen_users(1)[0]
@@ -194,34 +151,6 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
def test_bounces_are_dkim_signed(cmsetup, cmsetup2, maildata, maildomain):
# we send a message to non-existant user and expect a bounce message
# which will only get through if the bounce message was DKIM-signed
if is_valid_ipv4(maildomain):
pytest.skip("DKIM is not configured on IPv4-only relays")
sender = cmsetup2.gen_users(1)[0]
nonexistent = f"nosuchuser_test42@{cmsetup.maildomain}"
msg = maildata(
"encrypted.eml",
from_addr=sender.addr,
to_addr=nonexistent,
).as_string()
sender.smtp.sendmail(sender.addr, [nonexistent], msg)
def bounce_in_inbox():
messages = sender.imap.fetch_all_messages()
for m in messages:
if "mail delivery" in m.lower() or "undelivered" in m.lower():
return m
raise ValueError("bounce not yet in inbox")
bounce = try_n_times(30, bounce_in_inbox)
assert "nosuchuser_test42" in bounce
def try_n_times(n, f):
for _ in range(n - 1):
try:
@@ -254,6 +183,7 @@ def test_rewrite_subject(cmsetup, maildata):
assert "Subject: Unencrypted subject" not in rcvd_msg
@pytest.mark.slow
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
"""Test that the per-account send-mail limit is exceeded."""
user1, user2 = cmsetup.gen_users(2)
@@ -276,6 +206,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [
@@ -314,15 +245,3 @@ def test_deployed_state(remote):
# assert len(git_status) == len(remote_version) # for some reason, we only get 11 lines from remote.iter_output()
for i in range(len(remote_version)):
assert git_status[i] == remote_version[i], "You have undeployed changes."
def test_nginx_access_log_only_defined_once(sshdomain):
sshexec = get_sshexec(sshdomain)
conf = sshexec(
call=remote.rshell.shell,
kwargs=dict(command="nginx -T 2>/dev/null"),
)
access_logs = [l for l in conf.splitlines() if l.strip().startswith("access_log")]
assert len(access_logs) == 1, (
f"expected 1 access_log, found {len(access_logs)}: {access_logs}"
)
@@ -15,7 +15,7 @@ def imap_mailbox(cmfactory, ssl_context):
(ac1,) = cmfactory.get_online_accounts(1)
user = ac1.get_config("addr")
password = ac1.get_config("mail_pw")
host = user.split("@")[1].strip("[").strip("]")
host = user.split("@")[1]
mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
mailbox.login(user, password)
mailbox.dc_ac = ac1
@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
assert msg2.get_snapshot().text == "message0"
def test_exceed_quota(
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
):
"""This is a very slow test as it needs to upload >100MB of mail data
before quota is exceeded, and thus depends on the speed of the upload.
@@ -92,7 +92,9 @@ class TestEndToEndDeltaChat:
lp.sec(f"filling remote inbox for {user}")
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
sshexec = get_sshexec(sshdomain)
sshexec = get_sshexec(
sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
)
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
assert res["percent"] >= 100
@@ -178,7 +180,7 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
chat.send_text("testing submission header cleanup")
user2.wait_for_incoming_msg()
addr = user2.get_config("addr")
host = addr.split("@")[1].strip("[").strip("]")
host = addr.split("@")[1]
pw = user2.get_config("mail_pw")
mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
mailbox.login(addr, pw)
@@ -3,12 +3,15 @@ import os
from cmdeploy.cmdeploy import main
def test_status_cmd(chatmail_config, capsys, request):
def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
os.chdir(request.config.invocation_params.dir)
command = ["status"]
if os.getenv("CHATMAIL_SSH"):
command.append("--ssh-host")
command.append(os.getenv("CHATMAIL_SSH"))
ssh_host = pytestconfig.getoption("ssh_host")
if ssh_host:
command.extend(["--ssh-host", ssh_host])
ssh_config = pytestconfig.getoption("ssh_config")
if ssh_config:
command.extend(["--ssh-config", ssh_config])
assert main(command) == 0
status_out = capsys.readouterr()
print(status_out.out)
+152 -69
View File
@@ -2,25 +2,96 @@ import imaplib
import itertools
import os
import random
import re
import smtplib
import socket
import ssl
import subprocess
import time
from pathlib import Path
import pytest
from chatmaild.config import is_valid_ipv4, read_config
def format_mail_domain(raw_domain: str) -> str:
if is_valid_ipv4(raw_domain):
return f"[{raw_domain}]"
return raw_domain
from chatmaild.config import read_config
conftestdir = Path(__file__).parent
def pytest_addoption(parser):
parser.addoption(
"--slow", action="store_true", default=False, help="also run slow tests"
)
parser.addoption(
"--ssh-host",
dest="ssh_host",
default=None,
help="SSH host (overrides mail_domain for SSH operations).",
)
parser.addoption(
"--ssh-config",
dest="ssh_config",
default=None,
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
)
def _parse_ssh_config_hosts(path):
"""Parse an OpenSSH config file and return a dict of hostname -> IP."""
mapping = {}
current_names = []
for ln in Path(path).read_text().splitlines():
line = ln.strip()
m = re.match(r"^Host\s+(.+)", line)
if m:
current_names = m.group(1).split()
continue
m = re.match(r"^Hostname\s+(\S+)", line)
if m and current_names:
ip = m.group(1)
for name in current_names:
mapping[name] = ip
current_names = []
return mapping
_original_getaddrinfo = socket.getaddrinfo
def _make_patched_getaddrinfo(host_map):
"""Return a getaddrinfo that resolves hosts in host_map to their IPs."""
def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
if host in host_map:
ip = host_map[host]
return _original_getaddrinfo(ip, port, family, type, proto, flags)
return _original_getaddrinfo(host, port, family, type, proto, flags)
return patched_getaddrinfo
@pytest.fixture(autouse=True, scope="session")
def _setup_localchat_dns(pytestconfig):
"""Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
ssh_config = pytestconfig.getoption("ssh_config")
if not ssh_config or not Path(ssh_config).exists():
yield {}
return
host_map = _parse_ssh_config_hosts(ssh_config)
if not host_map:
yield {}
return
socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
try:
yield host_map
finally:
socket.getaddrinfo = _original_getaddrinfo
@pytest.fixture(scope="session")
def ssh_config_host_map(_setup_localchat_dns):
"""Return the host-name → IP map parsed from ssh-config."""
return _setup_localchat_dns
def pytest_configure(config):
config._benchresults = {}
config.addinivalue_line(
@@ -28,12 +99,19 @@ def pytest_configure(config):
)
def _get_chatmail_config():
inipath = os.environ.get("CHATMAIL_INI")
if inipath:
path = Path(inipath).resolve()
return read_config(path), path
def pytest_runtest_setup(item):
markers = list(item.iter_markers(name="slow"))
if markers:
if not item.config.getoption("--slow"):
pytest.skip("skipping slow test, use --slow to run")
def _get_chatmail_config():
ini = os.environ.get("CHATMAIL_INI")
if ini:
path = Path(ini).resolve()
if path.exists():
return read_config(path), path
current = Path().resolve()
while 1:
path = current.joinpath("chatmail.ini").resolve()
@@ -56,12 +134,18 @@ def chatmail_config(pytestconfig):
@pytest.fixture(scope="session")
def maildomain(chatmail_config):
return chatmail_config.mail_domain_bare
return chatmail_config.mail_domain
@pytest.fixture(scope="session")
def sshdomain(maildomain):
return os.environ.get("CHATMAIL_SSH", maildomain)
def sshdomain(maildomain, pytestconfig):
return pytestconfig.getoption("ssh_host") or maildomain
@pytest.fixture(scope="session")
def maildomain_ip(maildomain, ssh_config_host_map):
"""Return the IP for maildomain from ssh-config, or maildomain itself."""
return ssh_config_host_map.get(maildomain, maildomain)
@pytest.fixture
@@ -305,26 +389,37 @@ from deltachat_rpc_client import DeltaChat, Rpc
class ChatmailACFactory:
"""RPC-based account factory for chatmail testing."""
def __init__(self, rpc, maildomain, gencreds, chatmail_config):
def __init__(
self,
rpc,
maildomain,
maildomain_ip,
gencreds,
chatmail_config,
ssh_config_host_map,
):
self.dc = DeltaChat(rpc)
self.rpc = rpc
self._maildomain = maildomain
self._maildomain_ip = maildomain_ip
self.gencreds = gencreds
self.chatmail_config = chatmail_config
self._ssh_config_host_map = ssh_config_host_map
def _make_transport(self, domain):
"""Build a transport config dict for the given domain."""
domain_deliverable = format_mail_domain(domain)
addr, password = self.gencreds(domain_deliverable)
addr, password = self.gencreds(domain)
transport = {
"addr": addr,
"password": password,
# Setting server explicitly skips requesting autoconfig XML,
# see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
"imapServer": domain,
"smtpServer": domain,
}
if domain.startswith("_") or is_valid_ipv4(domain):
# To support running against local relays without host DNS resolution
# we attempt resolving the domain via ssh-config
# because otherwise core fails to find the address
server = self._ssh_config_host_map.get(domain)
if server is not None:
transport.update({"imapServer": server, "smtpServer": server})
if self.chatmail_config.tls_cert_mode == "self":
transport["certificateChecks"] = "acceptInvalidCertificates"
return transport
@@ -339,28 +434,14 @@ class ChatmailACFactory:
accounts = []
for _ in range(num):
account = self.dc.add_account()
domain_deliverable = format_mail_domain(domain)
addr, password = self.gencreds(domain_deliverable)
if is_valid_ipv4(domain):
# Use DCLOGIN scheme with explicit server hosts,
# matching how madmail presents its addresses to users.
qr = (
f"dclogin:{addr}"
f"?p={password}&v=1"
f"&ih={domain}&ip=993&is=ssl"
f"&sh={domain}&sp=465&ss=ssl"
f"&ic=3"
)
future = account.add_transport_from_qr.future(qr)
else:
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
future = account.add_or_update_transport.future(
self._make_transport(domain)
)
futures.append(future)
# ensure messages stay in INBOX so that they can be
# concurrently fetched via extra IMAP connections during tests
account.set_config("bcc_self", "1")
account.set_config("delete_server_after", "10")
accounts.append(account)
for future in futures:
@@ -390,35 +471,46 @@ def rpc(tmp_path_factory):
@pytest.fixture
def cmfactory(rpc, gencreds, maildomain, chatmail_config):
def cmfactory(
rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
):
"""Return a ChatmailACFactory for creating online Delta Chat accounts."""
return ChatmailACFactory(
rpc=rpc,
maildomain=maildomain,
maildomain_ip=maildomain_ip,
gencreds=gencreds,
chatmail_config=chatmail_config,
ssh_config_host_map=ssh_config_host_map,
)
@pytest.fixture
def remote(sshdomain):
r = Remote(sshdomain)
def remote(sshdomain, pytestconfig):
r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
yield r
r.close()
class Remote:
def __init__(self, sshdomain):
def __init__(self, sshdomain, ssh_config=None):
self.sshdomain = sshdomain
self.ssh_config = ssh_config
self._procs = []
def iter_output(self, logcmd="", ready=None):
getjournal = "journalctl -f" if not logcmd else logcmd
print(self.sshdomain)
if self.sshdomain in ("@local", "localhost"):
command = []
else:
command = ["ssh", f"root@{self.sshdomain}"]
match self.sshdomain:
case "@local":
command = []
case "localhost":
command = []
case _:
command = ["ssh"]
if self.ssh_config:
command.extend(["-F", self.ssh_config])
command.append(f"root@{self.sshdomain}")
[command.append(arg) for arg in getjournal.split()]
popen = subprocess.Popen(
command,
@@ -427,19 +519,15 @@ class Remote:
stderr=subprocess.DEVNULL,
)
self._procs.append(popen)
try:
while 1:
line = popen.stdout.readline()
res = line.decode().strip().lower()
if not res:
break
if ready is not None:
ready()
ready = None
yield res
finally:
popen.terminate()
popen.wait()
while 1:
line = popen.stdout.readline()
res = line.decode().strip().lower()
if not res:
break
if ready is not None:
ready()
ready = None
yield res
def close(self):
while self._procs:
@@ -465,11 +553,6 @@ def cmsetup(maildomain, gencreds, ssl_context):
return CMSetup(maildomain, gencreds, ssl_context)
@pytest.fixture
def cmsetup2(maildomain2, gencreds, ssl_context):
return CMSetup(maildomain2, gencreds, ssl_context)
class CMSetup:
def __init__(self, maildomain, gencreds, ssl_context):
self.maildomain = maildomain
@@ -480,7 +563,7 @@ class CMSetup:
print(f"Creating {num} online users")
users = []
for i in range(num):
addr, password = self.gencreds(format_mail_domain(self.maildomain))
addr, password = self.gencreds()
user = CMUser(self.maildomain, addr, password, self.ssl_context)
assert user.smtp
users.append(user)
@@ -1,118 +0,0 @@
from unittest.mock import MagicMock, patch
from cmdeploy.basedeploy import Deployer
def test_put_file_restart_and_reload():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res):
deployer.put_file("foo.conf", "/etc/foo.conf")
assert deployer.need_restart is True
assert deployer.daemon_reload is False
deployer = Deployer()
deployer.put_file("test.service", "/etc/systemd/system/test.service")
assert deployer.need_restart is True
assert deployer.daemon_reload is True
def test_remove_file():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
with patch("cmdeploy.basedeploy.files.file", return_value=mock_res) as mock_file:
deployer.remove_file("/etc/foo.conf")
mock_file.assert_called_once_with(
name="Remove /etc/foo.conf", path="/etc/foo.conf", present=False
)
assert deployer.need_restart is True
def test_ensure_systemd_unit():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
# Plain service file
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
deployer.ensure_systemd_unit("iroh-relay.service")
assert (
mock_put.call_args.kwargs["dest"]
== "/etc/systemd/system/iroh-relay.service"
)
assert deployer.need_restart is True
assert deployer.daemon_reload is True
deployer = Deployer()
# Template (.j2) dispatches to put_template and strips .j2 suffix
with patch("cmdeploy.basedeploy.files.template", return_value=mock_res) as mock_tpl:
deployer.ensure_systemd_unit(
"filtermail/chatmaild.service.j2",
bin_path="/usr/local/bin/filtermail",
)
assert (
mock_tpl.call_args.kwargs["dest"] == "/etc/systemd/system/chatmaild.service"
)
deployer = Deployer()
# Explicit dest_name override
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
deployer.ensure_systemd_unit(
"acmetool/acmetool-reconcile.timer",
dest_name="acmetool-reconcile.timer",
)
assert (
mock_put.call_args.kwargs["dest"]
== "/etc/systemd/system/acmetool-reconcile.timer"
)
def test_ensure_service():
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service("nginx.service")
mock_svc.assert_called_once_with(
name="Start and enable nginx.service",
service="nginx.service",
running=True,
enabled=True,
restarted=True,
daemon_reload=True,
)
# daemon_reload is cleared to avoid multiple systemctl daemon-reload calls
# need_restart is kept to ensure all subsequent services also restart
assert deployer.need_restart is True
assert deployer.daemon_reload is False
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
# Stopping suppresses restarted even when need_restart is True
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service(
"mta-sts-daemon.service",
running=False,
enabled=False,
)
assert mock_svc.call_args.kwargs["restarted"] is False
assert deployer.need_restart is True
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
# Multiple calls: daemon_reload resets after first, need_restart persists
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service("chatmaild.service")
deployer.ensure_service("chatmaild-metadata.service")
second_call = mock_svc.call_args_list[1]
assert second_call.kwargs["restarted"] is True
assert second_call.kwargs["daemon_reload"] is False
+6 -15
View File
@@ -23,30 +23,21 @@ class TestCmdline:
run = parser.parse_args(["run"])
assert init and run
def test_init_not_overwrite(self, capsys, tmp_path, monkeypatch):
def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
monkeypatch.delenv("CHATMAIL_INI", raising=False)
inipath = tmp_path / "chatmail.ini"
args = ["init", "--config", str(inipath), "chat.example.org"]
assert main(args) == 0
monkeypatch.chdir(tmp_path)
assert main(["init", "chat.example.org"]) == 0
capsys.readouterr()
assert main(args) == 1
assert main(["init", "chat.example.org"]) == 1
out, err = capsys.readouterr()
assert "path exists" in out.lower()
args.insert(1, "--force")
assert main(args) == 0
assert main(["init", "chat.example.org", "--force"]) == 0
out, err = capsys.readouterr()
assert "deleting config file" in out.lower()
def test_dns_skip_on_ip(self, capsys, tmp_path, monkeypatch):
monkeypatch.delenv("CHATMAIL_INI", raising=False)
inipath = tmp_path / "chatmail.ini"
assert main(["init", "--config", str(inipath), "1.3.3.7"]) == 0
assert main(["dns", "--config", str(inipath)]) == 0
out, err = capsys.readouterr()
assert out == "[WARNING] 1.3.3.7 is not a domain, skipping DNS checks.\n"
def test_www_folder(example_config, tmp_path):
reporoot = importlib.resources.files(__package__).joinpath("../../../../").resolve()
+4 -19
View File
@@ -4,7 +4,6 @@ import pytest
from cmdeploy import remote
from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
from cmdeploy.remote.rdns import get_authoritative_ns
@pytest.fixture
@@ -15,15 +14,11 @@ def mockdns_base(monkeypatch):
if command.startswith("dig"):
if command == "dig":
return "."
if "with.public.soa" in command and "NS" in command:
return "domain.with.public.soa. 2419 IN NS ns1.first-ns.de."
if "with.hidden.soa" in command and "NS" in command:
if "SOA" in command:
return (
"domain.with.hidden.soa. 2137 IN NS ns1.desec.io.\n"
"domain.with.hidden.soa. 2137 IN NS ns2.desec.org."
"delta.chat. 21600 IN SOA ns1.first-ns.de. dns.hetzner.com."
" 2025102800 14400 1800 604800 3600"
)
if "NS" in command:
return "delta.chat. 21600 IN NS ns1.first-ns.de."
command_chunks = command.split()
domain, typ = command_chunks[4], command_chunks[6]
try:
@@ -130,17 +125,6 @@ class TestPerformInitialChecks:
assert not l
@pytest.mark.parametrize(
("domain", "ns"),
[
("domain.with.public.soa", "ns1.first-ns.de."),
("domain.with.hidden.soa", "ns1.desec.io."),
],
)
def test_get_authoritative_ns(domain, ns, mockdns):
assert get_authoritative_ns(domain) == ns
def test_parse_zone_records():
text = """
; This is a comment
@@ -181,6 +165,7 @@ def test_parse_zone_records_invalid_line():
def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
if only_required:
# Only take records before the "; Recommended" section
zonefile = zonefile.split("; Recommended")[0]
for name, ttl, rtype, rdata in parse_zone_records(zonefile):
mockdns_base.setdefault(rtype, {})[name] = rdata
@@ -1,237 +0,0 @@
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
from pyinfra.facts.deb import DebPackages
from cmdeploy.dovecot import deployer as dovecot_deployer
def make_host(*fact_pairs):
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
Args:
*fact_pairs: tuples of (fact_class, fact_value) to register
Returns:
SimpleNamespace with get_fact that raises a clear error if an
unexpected fact type is requested.
"""
facts = dict(fact_pairs)
def get_fact(cls):
if cls not in facts:
registered = ", ".join(c.__name__ for c in facts)
raise LookupError(
f"unexpected get_fact({cls.__name__}); only registered: {registered}"
)
return facts[cls]
return SimpleNamespace(get_fact=get_fact)
@pytest.fixture
def deployer():
return dovecot_deployer.DovecotDeployer(
SimpleNamespace(mail_domain="chat.example.org"),
disable_mail=False,
)
@pytest.fixture
def patch_blocked(monkeypatch):
monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
@pytest.fixture
def mock_files_put(monkeypatch):
monkeypatch.setattr(
dovecot_deployer.files,
"put",
lambda **kwargs: SimpleNamespace(changed=False),
)
@pytest.fixture
def track_shell(monkeypatch):
calls = []
monkeypatch.setattr(
dovecot_deployer.server,
"shell",
lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
)
return calls
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
downloads = []
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host((DebPackages, {"dovecot-core": [epoch_version]})),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
assert changed is False, "should not flag changed when version already installed"
assert downloads == [], "should not download when version already installed"
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
monkeypatch,
):
downloads = []
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host((DebPackages, {})),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
# Verify the returned path uses archive version, not package version (with epoch)
assert changed is True, "should flag changed when package not yet installed"
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
f"deb path should use archive version (no epoch), got {deb!r}"
)
assert len(downloads) == 1, "files.download should be called exactly once"
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(
dovecot_deployer.DebPackages,
{
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
},
),
(dovecot_deployer.Arch, "x86_64"),
),
)
downloads = []
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: downloads.append(kwargs),
)
deployer.install()
assert downloads == [], "should not download when all packages epoch-matched"
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
assert deployer.need_restart is False, (
"need_restart should be False when nothing changed"
)
def test_install_unsupported_arch_falls_back_to_apt(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
# For unsupported architectures, all fact lookups return the arch string.
monkeypatch.setattr(
dovecot_deployer,
"host",
SimpleNamespace(get_fact=lambda cls: "riscv64"),
)
apt_calls = []
# Mirrors apt.packages() return value: OperationMeta with .changed property.
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
def fake_apt(**kwargs):
apt_calls.append(kwargs)
changed = "lmtpd" in kwargs["packages"][0]
return SimpleNamespace(changed=changed)
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
deployer.install()
actual_pkgs = [c["packages"] for c in apt_calls]
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
)
assert track_shell == [], "should not run dpkg for unsupported arch"
assert deployer.need_restart is True, (
"need_restart should be True when apt installed a package"
)
def test_install_runs_dpkg_when_packages_need_download(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.DebPackages, {}),
(dovecot_deployer.Arch, "x86_64"),
),
)
monkeypatch.setattr(
dovecot_deployer,
"_pick_url",
lambda primary, fallback: primary,
)
monkeypatch.setattr(
dovecot_deployer.files,
"download",
lambda **kwargs: SimpleNamespace(changed=True),
)
deployer.install()
assert len(track_shell) == 1, (
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
)
cmds = track_shell[0]["commands"]
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
assert "apt-get -y --fix-broken install" in cmds[1]
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
assert deployer.need_restart is True, (
"need_restart should be True after dpkg install"
)
def test_pick_url_falls_back_on_primary_error(monkeypatch):
def raise_error(req, timeout):
raise OSError("connection timeout")
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
assert result == "http://fallback", (
f"should fall back when primary fails, got {result!r}"
)
+3 -2
View File
@@ -1,10 +1,11 @@
from pathlib import Path
import importlib.resources
from cmdeploy.www import build_webpages
def test_build_webpages(tmp_path, make_config):
src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
pkgroot = importlib.resources.files("cmdeploy")
src_dir = pkgroot.joinpath("../../../www/src").resolve()
assert src_dir.exists(), src_dir
config = make_config("chat.example.org")
build_dir = tmp_path.joinpath("build")
+174
View File
@@ -0,0 +1,174 @@
"""Tests for cmdeploy lxc-* subcommands."""
import shutil
import subprocess
import sys
import pytest
from cmdeploy.lxc import cli
from cmdeploy.lxc.incus import Incus
from cmdeploy.util import Out
pytestmark = pytest.mark.skipif(
not shutil.which("incus"),
reason="incus not installed",
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def ix():
out = Out()
return Incus(out)
@pytest.fixture(scope="session")
def lxc_setup():
out = Out()
ix = Incus(out)
ix.get_dns_container().ensure()
return ix.list_managed()
@pytest.fixture(scope="session")
def relay_container(lxc_setup):
test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
if not relays:
pytest.skip("no test relay containers running")
return relays[0]
@pytest.fixture
def cmdeploy():
def run(*args):
return subprocess.run(
[sys.executable, "-m", "cmdeploy.cmdeploy", *args],
capture_output=True,
text=True,
check=False,
)
return run
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"subcmd, expected, absent",
[
(None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
("lxc-test", ["--one"], ["--config"]),
("lxc-status", [], ["--config"]),
("run", ["--ssh-config"], ["--lxc"]),
("dns", ["--ssh-config"], []),
("test", ["--ssh-config"], []),
("status", ["--ssh-config"], []),
],
)
def test_help_options(cmdeploy, subcmd, expected, absent):
args = [subcmd, "--help"] if subcmd else ["--help"]
result = cmdeploy(*args)
output = result.stdout + result.stderr
assert result.returncode == 0
for flag in expected:
assert flag in output
for flag in absent:
assert flag not in output
class TestSSHConfig:
def test_lxconfigs(self, ix, lxc_setup):
d = ix.lxconfigs_dir
assert d.name == "lxconfigs"
assert d.exists()
path = ix.ssh_config_path
assert path.name == "ssh-config"
assert path.parent.name == "lxconfigs"
def test_write_ssh_config(self, ix, lxc_setup):
path = ix.write_ssh_config()
assert path.exists()
text = path.read_text()
for c in lxc_setup:
if c.get("ip"):
assert c["name"] in text
assert f"Hostname {c['ip']}" in text
assert "User root" in text
assert "IdentityFile" in text
assert "StrictHostKeyChecking accept-new" in text
def test_dns(ix, relay_container):
def dig(qname, qtype):
ct = ix.get_dns_container()
return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
domain = relay_container["domain"]
assert dig(domain, "A") == relay_container["ip"]
assert domain in dig(domain, "MX")
assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
class TestLxcStatus:
def test_cli_lxc_status_help(self, cmdeploy):
result = cmdeploy("lxc-status", "--help")
assert result.returncode == 0
assert "status" in result.stdout.lower()
def test_shows_containers(self, lxc_setup, capsys):
class QuietOut(Out):
def red(self, msg, **kw):
pass
def green(self, msg, **kw):
pass
ret = cli.lxc_status_cmd(None, QuietOut())
assert ret == 0
captured = capsys.readouterr().out
assert "ns-localchat" in captured
assert "running" in captured
def test_deploy_freshness(self, ix, monkeypatch):
ct = ix.get_container("x")
monkeypatch.setattr(
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
lambda _self: "abc123def456",
)
monkeypatch.setattr(
"cmdeploy.lxc.incus.RelayContainer.deployed_domain",
lambda _self: ct.domain,
)
monkeypatch.setattr(
"cmdeploy.lxc.cli.get_version_string",
lambda: "abc123def456",
)
assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
# Hash matches but local has uncommitted changes
monkeypatch.setattr(
"cmdeploy.lxc.cli.get_version_string",
lambda: "abc123def456\ndiff --git a/foo",
)
assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
monkeypatch.setattr(
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
lambda _self: None,
)
assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)
+120
View File
@@ -0,0 +1,120 @@
import sys
from cmdeploy.util import Out, collapse, get_git_hash, get_version_string, shell
class TestOut:
def test_prefix_default(self, capsys):
out = Out()
out.print("hello")
assert capsys.readouterr().out == "hello\n"
def test_prefix_custom(self, capsys):
out = Out(prefix=">> ")
out.print("hello")
assert capsys.readouterr().out == ">> hello\n"
def test_prefix_print_file(self):
import io
buf = io.StringIO()
out = Out(prefix=":: ")
out.print("msg", file=buf)
assert ":: msg" in buf.getvalue()
def test_new_prefixed_out(self, capsys):
parent = Out(prefix="A")
child = parent.new_prefixed_out("B")
child.print("x")
assert capsys.readouterr().out == "ABx\n"
# shares section_timings
assert child.section_timings is parent.section_timings
def test_section_no_auto_indent(self, capsys):
out = Out(prefix="")
with out.section("test"):
out.print("inside")
captured = capsys.readouterr().out
# "inside" should NOT be indented by section()
lines = captured.strip().splitlines()
inside_line = [l for l in lines if "inside" in l][0]
assert inside_line == "inside"
def test_section_records_timing(self):
out = Out()
with out.section("s1"):
pass
assert len(out.section_timings) == 1
assert out.section_timings[0][0] == "s1"
def test_shell_failure_shows_output(self):
"""When a shell command fails, its output and exit code are shown."""
import subprocess
result = subprocess.run(
[
sys.executable,
"-c",
"from cmdeploy.util import Out; Out(prefix='').shell("
"\"echo 'boom on stderr' >&2; exit 42\")",
],
capture_output=True,
text=True,
check=False,
)
# the command's stderr is merged into stdout by Popen
assert "boom on stderr" in result.stdout
# Out.red() prints the failure notice to stderr
assert "exit code 42" in result.stderr
def test_collapse():
text = """
line 1
line 2
"""
assert collapse(text) == "line 1 line 2"
assert collapse(" single line ") == "single line"
def test_git_helpers_no_git(tmp_path):
# Not a git repo
assert get_git_hash(root=tmp_path) is None
assert get_version_string(root=tmp_path) == "unknown"
def test_git_helpers_empty_repo(tmp_path):
shell("git init", cwd=tmp_path, check=True)
# No commits yet
assert get_git_hash(root=tmp_path) is None
assert get_version_string(root=tmp_path) == "unknown"
def test_git_helpers_with_commits_and_diffs(tmp_path):
shell("git init", cwd=tmp_path, check=True)
shell("git config user.email you@example.com", cwd=tmp_path, check=True)
shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
# First commit
path = tmp_path / "file.txt"
path.write_text("content")
shell("git add file.txt", cwd=tmp_path, check=True)
shell("git commit -m initial", cwd=tmp_path, check=True)
git_hash = get_git_hash(root=tmp_path)
assert len(git_hash) >= 7 # usually 40, but git is git
assert get_version_string(root=tmp_path) == git_hash
# Create a diff
path.write_text("new content")
v = get_version_string(root=tmp_path)
assert v.startswith(git_hash + "\n")
assert "new content" in v
assert not v.endswith("\n")
# Commit again -> no diff
shell("git add file.txt", cwd=tmp_path, check=True)
shell("git commit -m second", cwd=tmp_path, check=True)
new_hash = get_git_hash(root=tmp_path)
assert new_hash != git_hash
assert get_version_string(root=tmp_path) == new_hash
@@ -1,7 +1,4 @@
# Managed by cmdeploy
# Managed by cmdeploy: disable IPv6 in unbound.
server:
{% if disable_ipv6 %}
interface: 127.0.0.1
do-ip6: no
{% endif %}
cache-max-negative-ttl: 0
+169
View File
@@ -0,0 +1,169 @@
"""Shared utility functions for cmdeploy."""
import os
import shutil
import subprocess
import sys
import textwrap
import time
from contextlib import contextmanager
from pathlib import Path
from termcolor import colored
class Out:
"""Convenience output printer providing coloring and section formatting."""
def __init__(self, prefix="", verbosity=0):
self.section_timings = []
self.prefix = prefix
self.sepchar = "\u2501"
self.verbosity = verbosity
env_width = os.environ.get("_CMDEPLOY_WIDTH")
if env_width:
self.section_width = int(env_width)
else:
self.section_width = shutil.get_terminal_size((80, 24)).columns
def new_prefixed_out(self, newprefix=" "):
"""Return a new Out with an extended prefix,
sharing section_timings with the parent.
"""
out = Out(
prefix=self.prefix + newprefix,
verbosity=self.verbosity,
)
out.section_timings = self.section_timings
return out
def red(self, msg, file=sys.stderr):
print(colored(self.prefix + msg, "red"), file=file, flush=True)
def green(self, msg, file=sys.stderr):
print(colored(self.prefix + msg, "green"), file=file, flush=True)
def print(self, msg="", **kwargs):
"""Print to stdout with automatic flush."""
if msg:
msg = self.prefix + msg
print(msg, flush=True, **kwargs)
def _format_header(self, title):
"""Return a formatted section header string."""
width = self.section_width - len(self.prefix)
bar = self.sepchar * (width - len(title) - 5)
return f"{self.sepchar * 3} {title} {bar}"
@contextmanager
def section(self, title):
"""Context manager that prints a section header and records elapsed time."""
self.green(self._format_header(title))
t0 = time.time()
yield
elapsed = time.time() - t0
self.section_timings.append((title, elapsed))
def section_line(self, title):
"""Print a section header without timing."""
self.green(self._format_header(title))
def shell(self, cmd, quiet=False, **kwargs):
"""Print *cmd*, run it, and re-print its output with the current prefix.
*cmd* is passed through :func:`collapse`, so callers
can use triple-quoted f-strings freely.
Stdout and stderr are merged, read line-by-line,
and each line is printed with ``self.prefix`` prepended.
When the command exits non-zero, a red error line is printed.
"""
cmd = collapse(cmd)
if not quiet:
self.print(f"$ {cmd}")
indent = self.prefix + " "
env = kwargs.pop("env", None)
if env is None:
env = os.environ.copy()
env["_CMDEPLOY_WIDTH"] = str(self.section_width - len(indent))
proc = subprocess.Popen(
cmd,
shell=True,
text=True,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
**kwargs,
)
for line in proc.stdout:
sys.stdout.write(indent + line)
sys.stdout.flush()
ret = proc.wait()
if ret:
self.red(f"command failed with exit code {ret}: {cmd}")
return ret
def _project_root():
"""Return the project root directory."""
return Path(__file__).resolve().parent.parent.parent.parent
def collapse(text):
"""Dedent, join lines, and strip a (triple-quoted) string.
Handy for writing shell commands across multiple lines::
cmd = collapse(f\"""
cmdeploy run
--config {ct.ini}
--ssh-host {ct.domain}
\""")
"""
return textwrap.dedent(text).replace("\n", " ").strip()
def shell(cmd, check=False, **kwargs):
"""Run a shell command string with sensible defaults.
*cmd* is passed through :func:`collapse` first, so callers
can use triple-quoted f-strings freely.
Captures stdout/stderr by default; pass ``capture_output=False``
to stream output to the terminal instead.
"""
if "capture_output" not in kwargs and "stdout" not in kwargs:
kwargs["capture_output"] = True
kwargs.setdefault("stdin", subprocess.DEVNULL)
return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
def get_git_hash(root=None):
"""Return the local HEAD commit hash, or None."""
if root is None:
root = _project_root()
result = shell(
"git rev-parse HEAD",
cwd=str(root),
)
if result.returncode == 0:
return result.stdout.strip()
return None
def get_version_string(root=None):
"""Return ``git_hash\\ngit_diff`` for the local working tree.
Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
``/etc/chatmail-version`` and by ``lxc-status`` to compare
the deployed state against the local checkout.
"""
if root is None:
root = _project_root()
git_hash = get_git_hash(root=root) or "unknown"
try:
git_diff = shell("git diff", cwd=str(root)).stdout.strip()
except Exception:
git_diff = ""
if git_diff:
return f"{git_hash}\n{git_diff}"
return git_hash
+4 -2
View File
@@ -1,4 +1,5 @@
import hashlib
import importlib.resources
import re
import time
import traceback
@@ -36,7 +37,7 @@ def prepare_template(source):
def get_paths(config) -> (Path, Path, Path):
reporoot = (Path(__file__).resolve() / "../../../../").resolve()
reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
www_path = Path(config.www_folder)
# if www_folder was not set, use default directory
if config.www_folder == "":
@@ -132,7 +133,8 @@ def find_merge_conflict(src_dir) -> Path:
def main():
reporoot = (Path(__file__).resolve() / "../../../../").resolve()
path = importlib.resources.files(__package__)
reporoot = path.joinpath("../../../").resolve()
inipath = reporoot.joinpath("chatmail.ini")
config = read_config(inipath)
config.webdev = True
+3 -5
View File
@@ -4,14 +4,12 @@
You can use the `make` command and `make html` to build web pages.
You need a Python environment with `sphinx` and other
dependencies, you can create it by running `scripts/initenv.sh`
from the repository root.
You need a Python environment where the following install was excuted:
pip install furo sphinx-autobuild
To develop/change documentation, you can then do:
. venv/bin/activate
cd doc
make auto
A page will open at https://127.0.0.1:8000/ serving the docs and it will
+1 -1
View File
@@ -15,7 +15,7 @@ author = 'chatmail collective'
extensions = [
#'sphinx.ext.autodoc',
#'sphinx.ext.viewdoc',
#'sphinx.ext.viewcode',
'sphinxcontrib.mermaid',
]
-24
View File
@@ -15,7 +15,6 @@ goes beyond what classic email servers offer:
streaming, privacy-preserving Push Notifications for Apple, Google, and `Ubuntu Touch <https://docs.ubports.com/en/latest/appdev/guides/pushnotifications.html>`_;
- **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
(DKIM is not enforced on :ref:`IP-only relays <iponly>`)
- **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
depends on established IETF standards and protocols.
@@ -48,29 +47,6 @@ Dovecot, and are configured to run unattended without much maintenance
effort. Chatmail relays happily run on low-end hardware like a Raspberry
Pi.
.. _upgrade:
How can I upgrade my chatmail relay?
------------------------------------
To upgrade to the latest ``main`` branch,
``cd`` into your local checkout of `https://github.com/chatmail/relay/`_
and run the following commands:
::
git pull origin main --rebase --autostash
scripts/initenv.sh
scripts/cmdeploy run
If you don't want the latest development version,
but a specific tagged release like `1.10.0 <https://github.com/chatmail/relay/releases/tag/1.10.0>`_,
run ``git pull origin 1.10.0`` instead.
If you made local changes for your setup,
they will be reapplied as long as they don't conflict with the upgrade.
If a conflict arises, ``git status`` will tell you how to resolve it.
How trustable are chatmail relays?
----------------------------------
+2 -14
View File
@@ -14,6 +14,8 @@ Minimal requirements and prerequisites
You will need the following:
- Control over a domain through a DNS provider of your choice.
- A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
IPv6 is encouraged if available. Chatmail relay servers only require
1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
@@ -26,11 +28,6 @@ You will need the following:
(An ed25519 private key is required due to an `upstream bug in
paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
- Control over a domain through a DNS provider of your choice
(there is experimental support for :ref:`IP-only relays <iponly>`).
.. _setup:
Setup with ``scripts/cmdeploy``
-------------------------------------
@@ -101,15 +98,6 @@ steps. Please substitute it with your own domain.
configure at your DNS provider (it can take some time until they are
public).
Docker installation
-------------------
There is experimental support for running chatmail via Docker.
A monolithic image based on the above cmdeploy method is available `through a separate repository <https://github.com/chatmail/docker/pkgs/container/docker>`_.
See the `chatmail/docker README <https://github.com/chatmail/docker>`_ for full setup instructions.
Other helpful commands
----------------------
+1 -2
View File
@@ -16,7 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
proxy
migrate
overview
reverse_dns
lxc
related
faq
iponly
-40
View File
@@ -1,40 +0,0 @@
.. _iponly:
Hosting without DNS records
===========================
.. note::
This option is experimental and might change without notice.
In case you don't have a domain,
for example in a local network,
you can run a chatmail relay with only an IPv4 address as well.
To deploy a relay without a domain,
run ``cmdeploy init`` with only the IPv4 address
during the :ref:`installation steps <setup>`,
for example ``cmdeploy init 13.12.23.42``.
Drawbacks
---------
- your transport encryption will only use self-signed TLS certificates,
which are vulnerable against MITM attacks.
the chatmail core's end-to-end encryption should suffice in most scenarios though.
- your messages will not be DKIM-signed;
experimentally, most chatmail relays accept non-DKIM-signed messages from IP-only relays,
but some relays might not accept messages from yours.
Email addresses
---------------
When running without a domain,
your chatmail addresses will use the IPv4 address
in brackets as the domain part,
for example ``user@[13.12.23.42]``.
This is a valid email address format
according to :rfc:`5321`.
+275
View File
@@ -0,0 +1,275 @@
Local testing with LXC/Incus
============================
The ``cmdeploy`` tool includes support for running
chatmail relays inside local
`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
This is meant for development, testing, and CI
without requiring a remote server.
LXC system containers are lightweight virtual machines
that share the host's kernel but run their own init system,
package manager, and network stack,
so the cmdeploy deployment scripts work pretty much
as they would on a real Debian server or cloud VPS.
Prerequisites
-------------
- Around 4-5 GiB free disk space
- `systemd-networkd` for the automagic hostname resolution
- No other service occupying Port 53
Install `Incus <https://linuxcontainers.org/incus/>`_
(LXC container manager).
See the `official installation guide
<https://linuxcontainers.org/incus/docs/main/installing/>`_
for full details.
After installing incus, initialise and grant yourself access::
sudo incus admin init --minimal
sudo usermod -aG incus-admin $USER
.. caution::
Adding yourself to ``incus-admin`` grants effective root access
to the host: any member can mount host directories into a container
and manipulate them as root.
This is fine for local testing of your own relay branches,
but do **not** use it for production setups
or for testing untrusted relay branches from others.
.. warning::
You **must now log out and back in** (or run ``newgrp incus-admin``)
after adding yourself to the group.
Without this, all ``cmdeploy lxc-*`` commands
will fail with permission errors.
Verify the installation works by running ``incus list``,
which should print an empty table without errors.
Quick start
-----------
::
cd relay
scripts/initenv.sh # bootstrap venv
source venv/bin/activate # activate venv
cmdeploy lxc-test # create containers, deploy, test
The ``lxc-test`` command provides an automated way
to run the full deployment and test pipeline.
It executes several ``cmdeploy`` subcommands in sequential steps.
If a step fails, you can copy-paste the printed command
and run it manually to debug.
No host DNS delegation or ``~/.ssh/config`` changes are needed
because ``lxc-test`` passes the required SSH and DNS options directly.
CLI reference
--------------
``lxc-start [--ipv4-only] [--run] [NAME ...]``
Create and start containers.
Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
Pass one or more ``NAME`` arguments to create user relay containers instead
(e.g. ``cmdeploy lxc-start myrelay``).
Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
producing an IPv4-only relay.
Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
Generates ``lxconfigs/ssh-config``.
It reuses existing containers and resets DNS zones to minimal records.
``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
Stop relay containers.
Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
Pass ``NAME`` to stop specific containers.
Use ``--destroy`` to also delete the containers and their config files.
Use ``--destroy-all`` to additionally destroy
the ``ns-localchat`` DNS container **and** remove
the cached ``localchat-base`` and ``localchat-relay``
images, giving a fully clean slate for the next ``lxc-test``.
User containers are **never** destroyed unless named explicitly.
``lxc-test [--one]``
By default creates, deploys, and tests both ``test0`` and ``test1``
for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
Pass ``--one`` to only deploy and test against ``test0``
(skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
``lxc-status``
Show live status of all LXC containers (including the DNS container),
deploy freshness (comparing ``/etc/chatmail-version``
against local ``git rev-parse HEAD`` and ``git diff``),
SSH config inclusion, and host DNS forwarding for ``.localchat``.
Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
**STALE** (different commit), or **NOT DEPLOYED**.
Container types
-----------------
**Test relay containers** (``test0-localchat``, ``test1-localchat``)
Created automatically by ``lxc-test``.
**test0** has IPv4 and IPv6 configured,
**test1** is IPv4-only (``disable_ipv6 = True``).
**User relay containers** (``<name>-localchat``)
Created by ``cmdeploy lxc-start <name>``
where ``<name>`` does not start with ``test``.
These are personal development instances,
never touched by ``lxc-stop --destroy`` unless named explicitly.
**DNS container** (``ns-localchat``)
Singleton container running PowerDNS.
Created automatically when any relay is started.
.. _lxc-ssh-config:
SSH configuration
-----------------
``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
a standard OpenSSH config file mapping every container name,
its domain, and a short alias to the container's IP address::
Host test0-localchat _test0.localchat _test0
Hostname 10.204.0.42
User root
IdentityFile /path/to/relay/lxconfigs/id_localchat
IdentitiesOnly yes
StrictHostKeyChecking accept-new
UserKnownHostsFile /dev/null
LogLevel ERROR
All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
``lxc-test`` passes it automatically.
**Using containers from the host shell:**
To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
Include /absolute/path/to/relay/lxconfigs/ssh-config
.. _lxc-dns-setup:
.. _localchat-tld:
``.localchat`` DNS and name resolution
---------------------------------------
All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
(e.g. ``_test0.localchat``, ``_test1.localchat``),
a non-delegated suffix that exists only within the local PowerDNS infrastructure.
A dedicated DNS container (``ns-localchat``)
is created so that local test relays interact
with DNS similar to a regular public Internet setup.
On first start, ``cmdeploy lxc-start`` creates this container
running two `PowerDNS <https://www.powerdns.com/>`_ services:
* **pdns-server** (authoritative) serves ``.localchat``
zones from a local SQLite database.
* **pdns-recursor** (recursive) listens on the Incus
bridge so all containers can use it.
Forwards ``.localchat`` queries to the local
authoritative server and resolves everything else recursively.
After the DNS container is up, ``lxc-start`` configures the Incus bridge
to advertise its IP via DHCP and disables Incus's own DNS.
DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
1. **``lxc-start``** resets each relay zone to
**SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
If host DNS resolution is configured, users can
afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
--ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
LXC subcommands do not depend on host DNS resolution
and resolve addresses via ``lxconfigs/ssh-config``.
2. **``cmdeploy dns --zonefile``** generates a standard
BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
into PowerDNS.
This two-phase approach prevents premature configuration of mail records
before the relay is actually deployed and running.
Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
inside a relay container, Unbound has a configuration plugin snippet
that forwards all ``.localchat`` queries to the PowerDNS recursor,
and lets all other queries go through normal recursive resolution.
State outside the repository
-----------------------------
All generated configuration by lxc subcommands live in ``lxconfigs/``
(git-ignored), including the SSH key pair (``id_localchat``),
per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
The only state *outside* the repository is the Incus containers and images themselves
(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
The Incus image store retains the following snapshot images:
* ``localchat-base``: Debian 12 with openssh-server and Python (built on first run)
* ``localchat-relay``: fully deployed relay snapshot,
cached after the first successful ``cmdeploy run``.
Subsequent relay containers launch from this image
so the deploy step is mostly no-ops (roughly 3× faster than a fresh deploy).
.. _lxc-tls:
TLS handling and underscore domains
------------------------------------
Container domains start with ``_`` (e.g. ``_test0.localchat``).
As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
underscore domains automatically use self-signed TLS
and ``smtp_tls_security_level = encrypt``.
This permits cross-relay federation between LXC containers
without any external certificate authority.
Delta Chat clients connecting to these relays
must be configured with
``certificateChecks = acceptInvalidCertificates``
(the test fixtures handle this automatically).
`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
is meant to make this special setting unnecessary for chatmail clients
that are connecting to underscore domains.
Known limitations
------------------
The LXC environment differs from a production
deployment in several ways:
**No ACME / Let's Encrypt**:
Self-signed TLS only (see :ref:`lxc-tls`);
ACME code paths are never exercised locally.
**No inbound connections from the internet**:
Containers sit on a private Incus bridge and are not port-forwarded.
Only the host and other containers on the same bridge can reach them.
**Local federation only**:
Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
but these relays are invisible to any external mail server.
**DNS is local only**:
The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
(see :ref:`lxc-dns-setup`).
**IPv6 is ULA-only**:
Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
These are not globally routable, but are sufficient for testing IPv6 service binding
(Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.
+21 -53
View File
@@ -102,12 +102,8 @@ short overview of ``chatmaild`` services:
Apple/Google/Huawei.
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
deletes old messages, large messages, and entire mailboxes
of users who have not logged in for longer than
``delete_inactive_users_after`` days.
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
deletes users if they have not logged in for a longer while.
The timeframe can be configured in ``chatmail.ini``.
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
is contacted by Dovecot when a user logs in and stores the date of
@@ -153,17 +149,13 @@ Chatmail relay dependency diagram
autoconfig.xml --- dovecot;
postfix --- |10080|filtermail-outgoing;
postfix --- |10081|filtermail-incoming;
postfix --- |10083|filtermail-transport;
filtermail-outgoing --- |10025 reinject|postfix;
filtermail-incoming --- |10026 reinject|postfix;
postfix --- |milter opendkim.sock|OpenDKIM
dovecot --- |doveauth.socket|doveauth;
dovecot --- |message delivery|maildir["maildir
/home/vmail/.../user"];
dovecot --- |lastlogin.socket|lastlogin;
dovecot --- chatmail-metadata;
dovecot --- |quota-warning|chatmail-quota-expire;
chatmail-quota-expire --- maildir;
lastlogin --- maildir;
doveauth --- maildir;
chatmail-expire-daily --- maildir;
@@ -180,51 +172,26 @@ Chatmail relay dependency diagram
style nginx-right fill:#f66;
style postfix fill:#f66;
style dovecot fill:#f66;
style OpenDKIM fill:#f66;
style notification-proxy fill:#f66;
Accepting and delivering mail
-----------------------------
Message between users on the same relay
---------------------------------------
.. mermaid::
:caption: This diagram shows all the paths a message can take.
:caption: This diagram shows the path a non-federated message takes.
flowchart LR
subgraph chatmail relay
subgraph postfix
qmgr .-> lmtp-filtermail["lmtp/lmtp-filtermail (default_transport)"]
qmgr .-> lmtp["lmtp (local_transport)"]
lmtp --> cleanup["cleanup (lmtp_header_cleanup)"]
bounce
smtpd-submission["smtpd/submission"]
smtpd-smtps["smtpd/smtps"]
smtpd-reinject-outgoing["smtpd/reinject-outgoing"] --> authclean["cleanup/authclean (submission_header_cleanup)"]
authclean --> qmgr
smtpd-smtp["smtpd/smtp"]
smtpd-reinject-incoming["smtpd/reinject-incoming"] --> qmgr
end
lmtp-filtermail --LMTP inet:10083--> filtermail-transport
cleanup --LMTP unix:private/dovecot-lmtp --> dovecot
dovecot --> maildir
smtpd-submission --SMTP inet:10080--> filtermail-outgoing
smtpd-smtps --SMTP inet:10080--> filtermail-outgoing
filtermail-outgoing --SMTP inet:10025--> smtpd-reinject-outgoing
open-dkim["OpenDKIM (signing only)"] <--milter unix:opendkim/opendkim.sock--> smtpd-reinject-outgoing
bounce <--milter unix:opendkim/opendkim.sock--> open-dkim
bounce --> qmgr
nginx
smtpd-smtp -.SMTP inet:10081.-> filtermail-incoming
nginx -.HTTP inet:10082.-> filtermail-incoming
filtermail-incoming --SMTP inet:10026--> smtpd-reinject-incoming
end
filtermail-transport -.SMTP inet:25.-> mta1[Remote relay]
filtermail-transport -.HTTPS /mxdeliv.-> mta1
client[Client] -.SMTP inet:587.-> smtpd-submission
client -.SMTP inet:465.-> smtpd-smtps
client -.SMTP inet:443.-> nginx
nginx -.SMTP inet:465.-> smtpd-smtps
mta2[Remote relay] -.SMTP inet:25.-> smtpd-smtp
mta2 -.HTTPS /mxdeliv.-> nginx
graph LR;
sender --> |465|smtps/smtpd;
sender --> |587|submission/smtpd;
smtps/smtpd --> |10080|filtermail;
submission/smtpd --> |10080|filtermail;
filtermail --> |10025|smtpd_reinject;
smtpd_reinject --> cleanup;
cleanup --> qmgr;
qmgr --> smtpd_accepts_message;
qmgr --> |lmtp|dovecot;
dovecot --> recipient;
dovecot --> sender's_other_devices;
Operational details of a chatmail relay
----------------------------------------
@@ -291,8 +258,7 @@ from the chatmail relay server.
Email domain authentication (DKIM)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Chatmail relays enforce :rfc:`DKIM <6376>` to authenticate incoming emails
(except for :ref:`IP-only relays <iponly>`).
Chatmail relays enforce :rfc:`DKIM <6376>` to authenticate incoming emails.
Incoming emails must have a valid DKIM signature with
Signing Domain Identifier (SDID, ``d=`` parameter in the DKIM-Signature
header) equal to the ``From:`` header domain. This property is checked
@@ -323,7 +289,9 @@ ensured by ``filtermail`` proxy.
TLS requirements
~~~~~~~~~~~~~~~~
Filtermail (used for delivery) requires a valid TLS.
Postfix is configured to require valid TLS by setting
`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
to ``verify``.
You can test it by resolving ``MX`` records of your relay domain and
then connecting to MX relays (e.g ``mx.example.org``) with
-64
View File
@@ -1,64 +0,0 @@
Configuring reverse DNS
=======================
Some email servers reject the emails
if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
Passing the check requires that the IP address that email is sent from
should have a ``PTR`` record pointing to the domain name of the server,
and domain name record should have an ``A/AAAA`` record
pointing to the IP address.
Modern email relies on DKIM and SPF for authentication,
while iprev check exists for
`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
Chatmail relays don't resolve ``PTR`` records,
so you can ignore this section if configuring ``PTR`` records
is difficult and federation with legacy email servers that don't accept
valid DKIM signature for authentication is not important.
Multi-homed setups
------------------
If you have a server with multiple IP addresses,
also known as multi-homed setup,
and don't publish all IP addresses in DNS,
you need to make sure you are using
the published address when making outgoing connections.
For example, your server may have a static IP
address, and a so-called Floating IP or Virtual IP
that can be moved between servers in case of
migration or for failover.
By using Floating IP you can avoid downtime
and keep the IP address reputation
for destinatinons that rely on IP reputation and IP blocklists.
In this case you will only publish
the Floating IP to DNS and only use the static IP
to SSH into the server.
If you have such setup, make sure that
you not only set ``PTR`` records for the Floating IP,
but make outgoing connections using the Floating IP.
Otherwise reverse DNS check succeed,
but forward check making sure your domain name points
to the IP address will fail.
Such setup is indistinguishable from someone
setting IP address ``PTR`` with the domain they don't own
and as a result don't succeed.
On Linux you can configure source IP address with ``ip route`` command,
for example:
::
ip route change default via <default-gateway> dev eth0 src <source-address>
Make sure to persist the change after verifying it is working.
You can check what your outgoing IP address is
with ``curl icanhazip.com``.
Check both the IPv4 and IPv6 addresses.
For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
and similarly for IPv6 if you have it.
-291
View File
@@ -1,291 +0,0 @@
## 0.7.4 - 2026-07-01
### Features
- *(logs)* Log incoming mailer-daemon message sources (#191)
### Miscellaneous Tasks
- Update filtermail.mtail (#190)
### Testing
- Place #[tokio::test] after rstest case macros (#192)
## 0.7.3 - 2026-06-27
### Features
- *(transport)* Worker eviction (#185)
## 0.7.2 - 2026-06-26
### Documentation
- *(readme)* Disable colors in mermaid diagrams (#180)
- *(readme)* Update Transport mode doc
### Features
- *(transport)* Destination worker pool
### Refactor
- Implement Display for AddressDomain
### Testing
- Test filtermail-transport
## 0.7.1 - 2026-06-09
### Bug Fixes
- Switch ratelimiter to MonotonicClock (#167)
- *(smtp-server)* Correct error when EOF while reading DATA (#168)
- *(resolver)* Disable negative caching (#170)
- Ignore CNAME records when resolving TXT records (#177)
### Features
- Switch to aws-lc-rs cryptography provider (#178)
### Miscellaneous Tasks
- Add filtermail.mtail so filtermail failures can be monitored (#169)
## 0.7.0 - 2026-05-26
### Bug Fixes
- Do not crash if accepting new connection fails
### Documentation
- *(readme)* Remove docs for options removed in da9a116 (#162)
### Features
- [**breaking**] Remove passthrough options that allowed unencrypted mail to pass
## 0.6.6 - 2026-05-12
### Bug Fixes
- Return HTTP 200 because madmail expects it, and make sure https is immediately retried when SMTP fails (#153)
### Features
- Improved SMTP error responses (#147)
### Miscellaneous Tasks
- Remove mac and windows from matrix tests (#154)
- Run cmlxc tests in all classic/classic-ipv4/madmail combinations
## 0.6.5 - 2026-05-12
### Bug Fixes
- *(smtp-client)* Handle 421 on reused connection (#145)
- Advertise 8BITMIME to prevent conversion after DKIM signing (#149)
- Validate mail data (#150)
### Miscellaneous Tasks
- Build and deploy relays with filtermail binary and run interop tests against madmail (#131)
### Performance
- *(smtp-client)* Use pipelining if server advertises support (#146)
## 0.6.4 - 2026-05-01
### Bug Fixes
- Implement missing "implicit MX" rule
### Features
- Https transport channel (#122)
### Refactor
- *(transport)* Explicitly handle RFC7505 null MX
## 0.6.3 - 2026-04-20
### Features
- *(smtp-server)* Log malformed SMTP commands (#118)
### Performance
- *(smtp-client)* Cache connections (#117)
## 0.6.2 - 2026-04-16
### Documentation
- *(readme)* Reformat README.md (#115)
### Features
- Log disabled recipients (#113)
- *(transport)* Remote delivery over SMTP (#104)
### Miscellaneous Tasks
- *(dependabot)* Update configuration (#114)
## 0.6.1 - 2026-04-01
### Documentation
- *(readme)* Fix typo (#99)
### Features
- Configurable hosts for listen and reinject (#84)
- *(resolver)* Enable DNSSEC (#94)
- Add experimental option to disable mailboxes (#108)
### Refactor
- Derive Default for Envelope (#100)
## 0.6.0 - 2026-03-13
### Documentation
- *(readme)* Fix license link
- *(readme)* Clarify licensing of the binaries
- *(readme)* Improve README.md (#88)
### Features
- [**breaking**] Remove IP verification for domain-literals (#90)
### Miscellaneous Tasks
- *(ci)* Fix binary publish job (#79)
### Refactor
- Use enum for mode cli arg (#83)
## 0.5.2 - 2026-02-27
### Bug Fixes
- *(logs)* Log correct address for outbound messages (#70)
### Features
- Check incoming email return address (#72)
### Miscellaneous Tasks
- Bump cargo-dist (#73)
- *(ci)* Build and upload binaries in CI (#76)
- *(ci)* Add missing Zig dependency (#77)
### Refactor
- Check if email is encrypted before verifying DKIM (#71)
## 0.5.1 - 2026-02-24
### Bug Fixes
- *(dkim)* Accept TXT records with no `v=` tag (#62)
- *(smtp)* Properly handle bounce messages (#63)
- *(dkim)* Accept TXT records with escaped quotes (#61)
- *(logs)* Log `From` address instead of envelope `MAIL FROM`. (#66)
## 0.5.0 - 2026-02-20
### Bug Fixes
- *(dkim)* Make simple header canonicalization work properly (#53)
### Features
- Save rejected messages to `/tmp` (#55)
### Refactor
- Do not copy the mail in memory for DKIM verification (#54)
## 0.4.1 - 2026-02-17
### Miscellaneous Tasks
- *(tests)* Add a way to disable DKIM for tests (#50)
## 0.4.0 - 2026-02-17
### Features
- [**breaking**] DKIM verifier (#35)
- Support addresses using domain literals (#42)
## 0.3.0 - 2026-02-14
### Features
- Support legacy, pre-OpenPGP packet format (#44)
### Miscellaneous Tasks
- *(dist)* Switch to musl targets (#31)
### Refactor
- Remove unnecessary Arc (#36)
- Use a custom, minimal SMTP client instead of lettre (#33)
## 0.2.0 - 2026-01-28
### Features
- Configurable rate limiter max burst size (#28)
### Performance
- Disable Nagle's algorithm and do own buffering on server connections
### Refactor
- Remove Mutex around rate limiter
## 0.1.2 - 2026-01-22
### Bug Fixes
- Set logs required by grafana to INFO (#21)
- Make inbound/outbound log messages consistent (#23)
### Performance
- Use governor for rate limiting (#20)
## 0.1.1 - 2026-01-21
### Bug Fixes
- Improve address extraction from SMTP commands (#14)
- Correct a typo in SMTP answer (#11)
- *(config)* Set default values for internal SMTP ports and max message size (#12)
### Features
- Improve logging (#13)
### Miscellaneous Tasks
- *(dist)* Configure cargo-dist (#10)
- Configure git-cliff
### Refactor
- Get rid of indexing and slicing in check_armored_payload() (#15)
- Apply more lints (#17)
## 0.1.0 - 2026-01-19
### Documentation
- *(readme)* Add README.md
- *(license)* Add LICENSE
### Features
- Initial implementation
### Miscellaneous Tasks
- Init repository
- *(dependabot)* Setup dependabot
- *(ci)* Setup CI
- *(dependabot)* Add github-actions to dependabot
- *(cargo)* Add metadata
-3109
View File
File diff suppressed because it is too large Load Diff
-61
View File
@@ -1,61 +0,0 @@
[package]
name = "filtermail"
version = "0.7.4"
edition = "2024"
license = "MIT"
readme = "README.md"
authors = ["Chatmail Developers"]
description = "A mail filtering service used by chatmail relay"
repository = "https://github.com/chatmail/filtermail"
[dependencies]
async-trait = "0.1.89"
base64 = "0.22.1"
serde = { version = "1.0.228", features = ["derive"] }
serini = "0.2.2"
tokio = { version = "1.52.3", features = ["full"] }
thiserror = "2.0.18"
mailparse = "0.16.1"
memchr = "2.8.1"
log = "0.4.30"
env_logger = "0.11.10"
governor = { version = "0.10.4", default-features = false, features = [
"std",
"dashmap",
] }
viadkim = { version = "0.2.0" }
hickory-resolver = { version = "0.26.1", features = ["dnssec-aws-lc-rs"] }
lru = "0.18.0"
parking_lot = "0.12.5"
tokio-rustls = { version = "0.26.4", default-features = false, features = [
"aws-lc-rs",
"logging",
"tls12",
] }
webpki-roots = "1.0.7"
tokio-io-timeout = "1.2.1"
retainer = "0.4.0"
hyper = { version = "1.8.1", features = ["server"] }
hyper-util = { version = "0.1.20", features = [
"tokio",
"server",
"server-auto",
"client-legacy",
] }
http-body-util = "0.1.3"
hyper-rustls = { version = "0.27.9", default-features = false, features = [
"aws-lc-rs",
"http1",
"http2",
"logging",
"tls12",
] }
[dev-dependencies]
rstest = "0.26.1"
testresult = "0.4.1"
insta = "1.48.0"
serial_test = "3.5.0"
[profile.release]
lto = "thin"
-21
View File
@@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2026, chatmail and delta chat teams
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-674
View File
@@ -1,674 +0,0 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.
-203
View File
@@ -1,203 +0,0 @@
# filtermail
A postfix smtpd proxy filter used by [chatmail relay](https://github.com/chatmail/relay).
Filtermail is a fast, minimal and secure Rust-based SMTP before-queue filter.
By acting as a protocol-aware proxy for incoming and outgoing messages,
it enforces mandatory end-to-end encryption,
performs DKIM verification,
and handles per-sender rate limiting.
## Usage
```plain
filtermail <config> (incoming|outgoing|transport)
```
where `<config>` is a path to `chatmail.ini` configuration file.
Filtermail can be used in `incoming`, `outgoing` or `transport` mode.
### Incoming mode
```mermaid
flowchart LR
subgraph chatmail relay
subgraph postfix
smtpd1[smtpd]
smtpd2[smtpd] --> queue[...]
end
nginx[nginx]
smtpd1 -.SMTP :10081.-> filtermail[filtermail-incoming]
nginx -.HTTP :10082.-> filtermail
filtermail --SMTP :10026--> smtpd2
end
mta[Sender's relay] -.SMTP :25.-> smtpd1
mta -.HTTPS /mxdeliv.-> nginx
```
Filtermail in incoming mode acts as a proxy filter
for messages received from remote MTAs and performs following steps:
1. Rejects messages if `DATA` exceeds configured message size limit.
2. Rejects messages that do not meet at least one of the following criteria:
- PGP encrypted,
- securejoin message,
- mailer-daemon message,
- all recipients allow cleartext
(`enforceE2EEincoming` is not present in their mailbox directory).
3. If `MAIL FROM` doesn't match `From` header,
the address is removed from `MAIL FROM` on reinjection
(prevents bounces to possibly spoofed `MAIL FROM`).
4. Checks message origin,
depending on address type:
- **domain** - performs a strict DKIM verification and domain alignment check
(domain of address from `From` header must exactly match the DKIM signature domain),
rejecting messages that fail.
- **domain-literal (IP address)** - currently no-op.
5. In case of a DKIM failure,
the message is saved to `/tmp/filtermail-rejected/dkim-verify` directory for later inspection.
In contrast to outgoing mode, incoming mode starts with not only SMTP but also HTTP listener.
Built-in HTTP server doesn't handle TLS, and should be placed behind a TLS-terminating reverse proxy.
### Outgoing mode
```mermaid
flowchart LR
subgraph chatmail relay
subgraph postfix
smtpd1[smtpd]
smtpd2[smtpd] --> queue[...]
end
smtpd1 --SMTP :10080--> filtermail[filtermail-outgoing]
filtermail --SMTP :10025--> smtpd2
open-dkim[OpenDKIM] <--milter--> smtpd2
end
client[Client] --SMTP :587--> smtpd1
```
Filtermail in outgoing mode acts as a proxy filter
for messages received from clients and performs following steps:
1. Rejects messages at `MAIL FROM` stage if the address exceeded rate limit.
2. Rejects messages if `DATA` exceeds configured message size limit.
3. Rejects messages which `From` header address does not match one in `MAIL FROM`.
4. Rejects messages that do not meet at least one of the following criteria:
- PGP encrypted,
- securejoin message,
- self-sent Autocrypt Setup Message,
### Transport mode
```mermaid
flowchart LR
subgraph chatmail relay
subgraph postfix
qmgr[...] --> lmtp[default-transport lmtp]
end
lmtp --LMTP :10083--> filtermail[filtermail-transport]
end
filtermail -.SMTP :25.-> mta[Recipient's relay]
filtermail -.HTTPS /mxdeliv.-> mta
```
Filtermail in transport mode is used for final delivery to remote MTAs.
As opposed to incoming/outgoing, it accepts connections from postfix over LMTP instead of SMTP,
to allow returning per-recipient status back to postfix.
Received message is split per-domain and sent to recipients' MX servers over HTTP and SMTP,
enforcing TLS.
As opposed to postfix, IPv4 and IPv6 connections are tried in parallel and first successful connection is used.
HTTP delivery channel is preferred,
and SMTP is used only if HTTP delivery fails.
Filtermail spawns a separate worker for each destination
(distinguished by [domain][RFC5322_3_4_1] part of the recipient's [addr-spec][RFC5322_3_4_1],
NOT the actual MX server).
Only messages to the same destination are guaranteed to be sent in-order
(if not deferred and sent over the same LMTP connection);
messages to different destinations are NOT synchronized.
[RFC5322_3_4_1]: https://datatracker.ietf.org/doc/html/rfc5322#section-3.4.1
## Configuration
### chatmail.ini
Filtermail shares the same configuration file as chatmail relay,
but implements a custom parser that only requires a small subset of configuration options:
- `filtermail_smtp_port` - port to listen on in outgoing mode,
defaults to `10080`.
- `filtermail_smtp_port_incoming` - SMTP port to listen on in incoming mode,
defaults to `10081`.
- `filtermail_http_port_incoming` - HTTP port to listen on in incoming mode,
defaults to `10082`.
- `filtermail_lmtp_port_transport` - port to listen on in transport mode,
defaults to `10083`.
- `postfix_reinject_port` - port to reinject messages to postfix in outgoing mode,
defaults to `10025`.
- `postfix_reinject_port_incoming` - port to reinject messages to postfix in incoming mode,
defaults to `10026`.
- `max_message_size` - maximum allowed message size in bytes,
defaults to `31457280` (30 MiB).
- `max_user_send_per_minute` - email sending rate per user and minute,
defaults to `60`.
- `max_user_send_burst_size` - per-user max burst size for sending rate limiting (GCRA bucket capacity),
defaults to `10`.
- `mail_domain` - domain name used in email addresses.
- `mailboxes_dir` - path to mailboxes directory,
defaults to `/home/vmail/mail/<mail_domain>`.
The following options are Filtermail-specific,
they are not read by other chatmail relay components
and usually do not need to be set at all:
- `filtermail_host` - IP address to listen on,
defaults to `127.0.0.1`.
- `postfix_host` - hostname or IP address where postfix is set up,
a host is resolved only on Filtermail startup,
useful in case MTA runs somewhere outside of localhost,
defaults to `127.0.0.1`.
### Environment variables
Additional options that can be set using environment variables:
- `RUST_LOG` - set log level,
defaults to `info`.
- `FILTERMAIL_SKIP_DKIM` - completely skip DKIM verification;
only for testing purposes and not recommended for production use,
defaults to `0`.
## Usage outside of chatmail relay
**Filtermail development is focused on supporting it as a systemd service used by chatmail relay.**
Although unsupported, it may still work outside of this context or even without postfix,
with few considerations:
- Filtermail expects to receive messages from trusted clients,
and thus should not listen on ports exposed directly to the internet.
- Issues outside of chatmail relay context are not necessarily considered bugs;
PRs fixing them are not guaranteed to be accepted.
(Trivial changes may still be considered,
please open an issue to discuss any such changes before working on them).
## Releases
Filtermail is distributed as a statically linked linux binary,
available for `x86_64` and `aarch64` architectures.
Binaries are available on the [releases page](https://github.com/chatmail/filtermail/releases).
## License
Code licensed under [MIT](LICENSE).
Binary releases of `filtermail` link with `viadkim`
and are thus subject to the [GPL-3.0-or-later](LICENSE-GPL).
-94
View File
@@ -1,94 +0,0 @@
# git-cliff ~ configuration file
# https://git-cliff.org/docs/configuration
[changelog]
# A Tera template to be rendered for each release in the changelog.
# See https://keats.github.io/tera/docs/#introduction
body = """
{% if version %}\
## {{ version | trim_start_matches(pat="v") }} - {{ timestamp | date(format="%Y-%m-%d") }}
{% else %}\
## [unreleased]
{% endif %}\
{% for group, commits in commits | group_by(attribute="group") %}
### {{ group | striptags | trim | upper_first }}
{% for commit in commits %}
- {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
{% if commit.breaking %}[**breaking**] {% endif %}\
{{ commit.message | upper_first }}\
{% endfor %}
{% endfor %}
"""
# Remove leading and trailing whitespaces from the changelog's body.
trim = true
# Render body even when there are no releases to process.
render_always = true
# An array of regex based postprocessors to modify the changelog.
postprocessors = [
# Replace the placeholder <REPO> with a URL.
#{ pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" },
]
# render body even when there are no releases to process
# render_always = true
# output file path
# output = "test.md"
[git]
# Parse commits according to the conventional commits specification.
# See https://www.conventionalcommits.org
conventional_commits = true
# Exclude commits that do not match the conventional commits specification.
filter_unconventional = true
# Require all commits to be conventional.
# Takes precedence over filter_unconventional.
require_conventional = false
# Split commits on newlines, treating each line as an individual commit.
split_commits = false
# An array of regex based parsers to modify commit messages prior to further processing.
commit_preprocessors = [
# Replace issue numbers with link templates to be updated in `changelog.postprocessors`.
#{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
# Check spelling of the commit message using https://github.com/crate-ci/typos.
# If the spelling is incorrect, it will be fixed automatically.
#{ pattern = '.*', replace_command = 'typos --write-changes -' },
]
# Prevent commits that are breaking from being excluded by commit parsers.
protect_breaking_commits = false
# An array of regex based parsers for extracting data from the commit message.
# Assigns commits to groups.
# Optionally sets the commit's scope and can decide to exclude commits from further processing.
commit_parsers = [
{ message = "^feat", group = "Features" },
{ message = "^fix", group = "Bug Fixes" },
{ message = "^docs", group = "Documentation" },
{ message = "^perf", group = "Performance" },
{ message = "^refactor", group = "Refactor" },
{ message = "^style", group = "Styling" },
{ message = "^test", group = "Testing" },
{ message = "^chore\\(release\\): prepare for", skip = true },
{ message = "^chore\\(deps.*\\)", skip = true },
{ message = "^chore\\(pr\\)", skip = true },
{ message = "^chore\\(pull\\)", skip = true },
{ message = "^chore|^ci", group = "Miscellaneous Tasks" },
{ body = ".*security", group = "Security" },
{ message = "^revert", group = "Revert" },
{ message = ".*", group = "Other" },
]
# Exclude commits that are not matched by any commit parser.
filter_commits = false
# Fail on a commit that is not matched by any commit parser.
fail_on_unmatched_commit = false
# An array of link parsers for extracting external references, and turning them into URLs, using regex.
link_parsers = []
# Include only the tags that belong to the current branch.
use_branch_tags = false
# Order releases topologically instead of chronologically.
topo_order = false
# Order commits topologically instead of chronologically.
topo_order_commits = true
# Order of commits in each group/release within the changelog.
# Allowed values: newest, oldest
sort_commits = "oldest"
# Process submodules commits
recurse_submodules = false
-101
View File
@@ -1,101 +0,0 @@
# filtermail.mtail: filtermail process event counters
#
# Counters
#
# Connection errors in the transport path
# Error::Io / Error::ConnectionFailed / Error::Tls: the reason string filtermail strips
counter filtermail_transport_error_total by reason
# Unexpected transport errors: DNS failure, HTTP error, config error.
counter filtermail_transport_unexpected_total
# Silent drops: inbound
# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail.
counter filtermail_inbound_drop_total by reason
# Silent drops: outbound
# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail;
# "sender_disabled" and "all_recipients_disabled" are new.
counter filtermail_outbound_drop_total by reason
# Reinject failures
counter filtermail_reinject_error_total by direction
# SMTP-level connection errors (client dropped mid-session)
counter filtermail_smtp_error_total by reason
#
# filtermail::transport
#
/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Connection error relaying to mail server \S+: / {
/timed out/ {
filtermail_transport_error_total["connection_timeout"]++
} otherwise {
/Failed to connect to any of the following addresses/ {
filtermail_transport_error_total["connection_refused"]++
} otherwise {
/Connection refused/ {
filtermail_transport_error_total["connection_refused"]++
} otherwise {
/invalid certificate/ {
filtermail_transport_error_total["tls_cert_not_verified"]++
} otherwise {
/peer sent fatal alert|handshake/ {
filtermail_transport_error_total["tls_handshake"]++
} otherwise {
/close_notify|connection closed/ {
filtermail_transport_error_total["lost_connection"]++
} otherwise {
filtermail_transport_error_total["other"]++
}}}}}}}
/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Unexpected error while delivering/ {
filtermail_transport_unexpected_total++
}
#
# filtermail::smtp_server
#
/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Unexpected EoF while receiving DATA/ {
filtermail_smtp_error_total["unexpected_eof"]++
}
/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Malformed DATA line without CRLF/ {
filtermail_smtp_error_total["malformed_data_line"]++
}
#
# filtermail::outbound
#
/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Rejected unencrypted mail/ {
filtermail_outbound_drop_total["unencrypted"]++
}
/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; Sender .* is disabled/ {
filtermail_outbound_drop_total["sender_disabled"]++
}
/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; All recipients disabled/ {
filtermail_outbound_drop_total["all_recipients_disabled"]++
}
/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Failed to re.inject mail/ {
filtermail_reinject_error_total["outbound"]++
}
#
# filtermail::inbound
#
/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Rejected unencrypted mail/ {
filtermail_inbound_drop_total["unencrypted"]++
}
/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Failed to re.inject mail/ {
filtermail_reinject_error_total["inbound"]++
}
-143
View File
@@ -1,143 +0,0 @@
//! Configuration file handling for filtermail.
use serde::Deserialize;
use std::net::IpAddr;
use std::num::NonZeroU32;
use std::path::{Path, PathBuf};
/// Chatmail configuration subset used by filtermail.
#[derive(Debug, Clone, Deserialize)]
pub struct Config {
#[serde(default = "Config::default_filtermail_host")]
pub filtermail_host: IpAddr,
#[serde(default = "Config::default_filtermail_smtp_port")]
pub filtermail_smtp_port: u16,
#[serde(default = "Config::default_filtermail_smtp_port_incoming")]
pub filtermail_smtp_port_incoming: u16,
#[serde(default = "Config::default_filtermail_http_port_incoming")]
pub filtermail_http_port_incoming: u16,
#[serde(default = "Config::default_filtermail_lmtp_port_transport")]
pub filtermail_lmtp_port_transport: u16,
#[serde(default = "Config::default_postfix_host")]
pub postfix_host: String,
#[serde(default = "Config::default_postfix_reinject_port")]
pub postfix_reinject_port: u16,
#[serde(default = "Config::default_postfix_reinject_port_incoming")]
pub postfix_reinject_port_incoming: u16,
#[serde(default = "Config::default_max_message_size")]
pub max_message_size: usize,
#[serde(default = "Config::default_max_user_send_per_minute")]
pub max_user_send_per_minute: NonZeroU32,
#[serde(default = "Config::default_max_user_send_burst_size")]
pub max_user_send_burst_size: NonZeroU32,
pub mail_domain: String,
mailboxes_dir: Option<PathBuf>,
}
#[derive(Debug, Clone, Deserialize)]
struct ConfigWrapper {
// The whole actual config is under `params` section.
pub params: Config,
}
impl Config {
/// Load configuration from a file.
pub fn from_file(path: impl AsRef<Path>) -> Result<Self, crate::error::Error> {
let content = std::fs::read_to_string(path)?;
let wrapped_config: ConfigWrapper = serini::from_str(&content)?;
Ok(wrapped_config.params)
}
/// Get the mailboxes directory, defaulting to `/home/vmail/mail/<mail_domain>` if not set.
fn mailboxes_dir(&self) -> PathBuf {
match &self.mailboxes_dir {
Some(dir) => dir.clone(),
None => PathBuf::from(format!("/home/vmail/mail/{}", self.mail_domain)),
}
}
/// Check if a specific flag file exists for the given address.
///
/// Returns `default` if the address is invalid.
fn check_flag(&self, addr: &str, flag: &str, default: bool) -> bool {
if addr.is_empty() || !addr.contains('@') || addr.contains('/') {
return default;
}
let mut path = self.mailboxes_dir();
path.push(addr);
path.push(flag);
path.exists()
}
/// Check if not encrypted mail is allowed for the given address.
pub fn is_cleartext_ok(&self, addr: &str) -> bool {
!self.check_flag(addr, "enforceE2EEincoming", true)
}
/// Check if the given address is disabled.
pub fn is_disabled(&self, addr: &str) -> bool {
self.check_flag(addr, "DISABLED", false)
}
// Following are needed since serde does not support default literals.
const fn default_filtermail_host() -> IpAddr {
IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)
}
const fn default_filtermail_smtp_port() -> u16 {
10080
}
const fn default_filtermail_smtp_port_incoming() -> u16 {
10081
}
const fn default_filtermail_http_port_incoming() -> u16 {
10082
}
const fn default_filtermail_lmtp_port_transport() -> u16 {
10083
}
fn default_postfix_host() -> String {
"127.0.0.1".to_owned()
}
const fn default_postfix_reinject_port() -> u16 {
10025
}
const fn default_postfix_reinject_port_incoming() -> u16 {
10026
}
const fn default_max_message_size() -> usize {
31457280
}
const fn default_max_user_send_per_minute() -> NonZeroU32 {
NonZeroU32::new(60).expect("60 != 0")
}
const fn default_max_user_send_burst_size() -> NonZeroU32 {
NonZeroU32::new(10).expect("10 != 0")
}
}
#[cfg(test)]
impl Default for Config {
/// Creates a default configuration with example.org domain.
///
/// Used for tests.
fn default() -> Self {
Self {
filtermail_host: Self::default_filtermail_host(),
filtermail_smtp_port: Self::default_filtermail_smtp_port(),
filtermail_smtp_port_incoming: Self::default_filtermail_smtp_port_incoming(),
filtermail_http_port_incoming: Self::default_filtermail_http_port_incoming(),
filtermail_lmtp_port_transport: Self::default_filtermail_lmtp_port_transport(),
postfix_host: Self::default_postfix_host(),
postfix_reinject_port: Self::default_postfix_reinject_port(),
postfix_reinject_port_incoming: Self::default_postfix_reinject_port_incoming(),
max_message_size: Self::default_max_message_size(),
max_user_send_per_minute: Self::default_max_user_send_per_minute(),
max_user_send_burst_size: Self::default_max_user_send_burst_size(),
mail_domain: "example.org".to_string(),
mailboxes_dir: None,
}
}
}
-298
View File
@@ -1,298 +0,0 @@
use hickory_resolver::{TokioResolver, proto::rr::Name};
use lru::LruCache;
use std::io;
use std::num::NonZeroUsize;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use viadkim::VerificationStatus;
use viadkim::message_hash::BodyHasherStance;
use viadkim::verifier::LookupTxt;
// ~500kB when fully saturated (~420B per RDATA + selector).
// "top 1000 relays" is much more than enough, the limit is mostly to prevent DoS attacks.
const LRU_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(1000).expect("1000 != 0");
/// Normalizes a TXT record RDATA by removing irrelevant characters.
///
/// Some DKIM key records use e.g. LF + WSP line breaks.
/// This is technically not correct, and `viadkim` fails to parse such records,
/// but in practice this is accepted by many implementations.
///
/// Additionally, removes escaped quotes, as such records as:
/// `"...UL9" "\" \"7vGm..."` proved to still be accepted by e.g. dkimpy or OpenDKIM.
fn normalize_rdata(txt_data: &str) -> String {
txt_data.replace([' ', '\t', '\n', '\r', '"'], "")
}
/// DNS resolver for DKIM TXT records, that caches RDATA in memory.
#[derive(Clone)]
struct CachedResolver {
dns_resolver: Arc<TokioResolver>,
// Note: Arc is required despite we are holding the whole handler in an Arc,
// because viadkim will internally clone the resolver (LookupTxt + Clone + 'static)
// to parallelize lookups in case of multiple signatures...
cache: Arc<parking_lot::Mutex<LruCache<Name, Vec<Vec<u8>>>>>,
}
impl CachedResolver {
/// Creates a new [`CachedResolver`].
pub fn new(dns_resolver: Arc<TokioResolver>) -> Self {
let cache = Arc::new(parking_lot::Mutex::new(LruCache::new(LRU_CACHE_CAPACITY)));
Self {
dns_resolver,
cache,
}
}
/// Invalidates the cached RDATA for a given selector and domain.
///
/// Fails silently.
fn invalidate_cache(&self, selector: &str, domain: &str) {
let selector_domain_str = format!("{}._domainkey.{}.", selector, domain);
if let Ok(selector_domain) = Name::from_ascii(&selector_domain_str) {
let mut cache = self.cache.lock();
cache.pop(&selector_domain);
log::debug!("Cache invalidated for {}", selector_domain_str);
} else {
log::warn!(
"Failed to parse selector domain for cache invalidation: {}",
selector_domain_str
);
}
}
}
impl LookupTxt for CachedResolver {
type Answer = Box<dyn Iterator<Item = io::Result<Vec<u8>>>>;
type Query<'a> = Pin<Box<dyn Future<Output = io::Result<Self::Answer>> + Send + 'a>>;
fn lookup_txt(&self, domain: &str) -> Self::Query<'_> {
let name = Name::from_ascii(domain);
Box::pin(async move {
let name = name.map_err(|_| io::ErrorKind::InvalidInput)?;
{
let mut cache = self.cache.lock();
if let Some(txts) = cache.get(&name) {
let txts: Self::Answer = Box::new(txts.clone().into_iter().map(Ok));
log::debug!("Using cached TXT records for {}", name);
return Ok(txts);
}
}
log::debug!("Trying to resolve TXT records for {}", name);
let txts: Vec<Vec<u8>> = {
let lookup = self
.dns_resolver
.txt_lookup(name.clone())
.await
.map_err(io::Error::other)?;
// viadkim would filter out non-DKIM TXT records,
// but we filter it here anyway so that we know which one should be cached.
lookup
.answers()
.iter()
.filter(|record| {
// Select only TXT records.
// When resolving TXT query, CNAMEs are also returned as answers.
// We want to filter out CNAMEs first.
matches!(record.data, hickory_resolver::proto::rr::RData::TXT(_))
})
// We don't check all records, as this can be a DoS attack vector.
// In theory, selector domains should only have a single TXT record.
// In practice, we check at most 3, just in case of weird configuration.
.take(3)
.map(|txt| {
let rdata = txt.data.to_string();
log::trace!("TXT (raw rdata): {:?}", rdata);
let normalized = normalize_rdata(&rdata);
log::trace!("TXT (concatenated and normalized): {:?}", normalized);
normalized.into_bytes()
})
.collect()
};
{
let mut cache = self.cache.lock();
cache.put(name, txts.clone());
}
let txts: Self::Answer = Box::new(txts.into_iter().map(Ok));
Ok(txts)
})
}
}
/// Dummy resolver that always returns the same TXT record, for testing purposes.
#[derive(Clone)]
struct MockResolver(String);
impl LookupTxt for MockResolver {
type Answer = Box<dyn Iterator<Item = io::Result<Vec<u8>>>>;
type Query<'a> = Pin<Box<dyn Future<Output = io::Result<Self::Answer>> + Send + 'a>>;
fn lookup_txt(&self, _domain: &str) -> Self::Query<'_> {
Box::pin(async move {
let txts: Self::Answer =
Box::new(std::iter::once(Ok(normalize_rdata(&self.0).into_bytes())));
Ok(txts)
})
}
}
/// Either a real resolver or a mock.
#[derive(Clone)]
enum Resolver {
/// A [`CachedResolver`]
Real(CachedResolver),
/// A [`MockResolver`]
Mock(MockResolver),
}
impl LookupTxt for Resolver {
type Answer = Box<dyn Iterator<Item = io::Result<Vec<u8>>>>;
type Query<'a> = Pin<Box<dyn Future<Output = io::Result<Self::Answer>> + Send + 'a>>;
fn lookup_txt(&self, domain: &str) -> Self::Query<'_> {
match self {
Resolver::Real(resolver) => resolver.lookup_txt(domain),
Resolver::Mock(resolver) => resolver.lookup_txt(domain),
}
}
}
impl From<CachedResolver> for Resolver {
fn from(value: CachedResolver) -> Self {
Resolver::Real(value)
}
}
impl From<MockResolver> for Resolver {
fn from(value: MockResolver) -> Self {
Resolver::Mock(value)
}
}
/// DKIM verifier using a pre-configured [`viadkim`] verifier, a [`CachedResolver`] for DNS lookups,
/// and strict domain name alignment check.
pub struct DkimVerifier {
resolver: Resolver,
config: viadkim::Config,
}
impl DkimVerifier {
/// Creates a new [`DkimVerifier`] with the provided resolver.
pub fn new(dns_resolver: Arc<TokioResolver>) -> Self {
let resolver = CachedResolver::new(dns_resolver).into();
let config = viadkim::Config {
lookup_timeout: Duration::from_secs(60),
..Default::default()
};
Self { resolver, config }
}
/// Creates a new [`DkimVerifier`] with a mock resolver that always returns the provided TXT record.
#[cfg(test)]
fn mock(txt: String) -> Self {
let resolver = MockResolver(txt).into();
let config = viadkim::Config {
lookup_timeout: Duration::from_secs(60),
..Default::default()
};
Self { resolver, config }
}
/// Verifies the DKIM signature of a raw email message and its alignment with the provided
/// domain.
pub async fn verify(&self, raw_mail: &[u8], from_domain: &str) -> Result<(), String> {
let mail_data = str::from_utf8(raw_mail).or(Err("554 Non-UTF-8 message"))?;
let (header, body) = mail_data
.split_once("\r\n\r\n")
.ok_or("554 Malformed data")?;
let header = header.parse().map_err(|_| "554 Malformed header")?;
let Some(mut verifier) =
viadkim::Verifier::verify_header(&self.resolver, &header, &self.config).await
else {
return Err("554 5.7.1 No DKIM signature found".to_string());
};
'hasher: for chunk in body.as_bytes().chunks(8192) {
if verifier.process_body_chunk(chunk) == BodyHasherStance::Done {
break 'hasher;
}
}
for res in verifier.finish() {
log::debug!("Signature {}: {:?}", res.index, res.status);
let Some(signature) = &res.signature else {
log::debug!("Signature {}: No signature found, skipping", res.index);
continue;
};
if matches!(res.status, VerificationStatus::Failure(_)) {
log::debug!("Signature {}: Verification failed, skipping", res.index);
// We only invalidate cache on actual validation error, and not alignment error.
// TODO: ideally we should retry without cache and swap cached value only on success.
if let Resolver::Real(resolver) = &self.resolver {
resolver
.invalidate_cache(signature.selector.as_ref(), signature.domain.as_ref());
}
continue;
}
if !signature
.domain
.to_string()
.eq_ignore_ascii_case(from_domain)
{
log::debug!(
"Signature {}: Domain different than in From header, skipping",
res.index
);
continue;
}
return Ok(());
}
Err("554 5.7.1 No valid DKIM signature found".to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
#[rstest]
#[case::simple_simple_canonicalization(
r#"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5krC4Xi5Wkr6eMlla38LCFmV645E3FLAgsRl2YJ0SrZ4N2Vw1/yH0mefvtk7HYE7ytV7RQl/er2CkSsaHLJSYLmPCBw5CO6PSsBSXuh6DBqdylh/1t9vVQ9p38fTwn9gU1QvplcpRQL9eepRra1k24VMIaVy2ZZcu3LI9zkPsR7o7TyNaeMhsL8ouWInWc1NSid+p0SgliQuwHIejZhlTPE60JLbJE0OR9I4wmq3377H6z/QrO8XeabCgtmTuzE/hTRyIyNS40jql/99pjlhIcjM2U+P2B0FjwYt7BwLHsgANr74ctlnKY+SdH25rNwVpPmkotaULG5SJCByKBkfCwIDAQAB;s=email;t=s"#,
include_bytes!("../test_data/dkim-abjadiyah.eml"),
"abjadiyah.xyz"
)]
#[case::txt_escaped_quotes(
r#"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1giTh8KDkEchWhrAB6hGnb+V87kTezkt5I3SP7BGNg8wpv0yAuj/SUmnsttYmcEU+zmNAPqxePmCNvmjLYi/c3YyWEBwHcLyZE9OlS9W4enPdsoCuEN3DayzN4JCV3MsXMedCORvLFXmIARDXDLJUSJeqCeQoudXa9GmF1CrCmx70YyTtV0xOIxEzo7z0DkUL9" "7vGmNJCv6EMpi9wccMKKu8NSmOv+DBw1MLIJqChSZMCs8CYZ5i0KT/+Lijtn6B7wyOcAuQsVL+zr7DWYrFdrePe0wGuivfJ3SvUEfUo1SIykl0nvm0iLGhjNmNa1e/tUw4ULXhQ12Qw685+sq7wIDAQAB;s=email;t=s"#,
include_bytes!("../test_data/dkim-privitty.eml"),
"chat.privittytech.com"
)]
#[tokio::test]
async fn test_dkim_verifier(#[case] txt: &str, #[case] message: &[u8], #[case] domain: &str) {
let verifier = DkimVerifier::mock(txt.to_string());
verifier.verify(message, domain).await.unwrap();
}
#[rstest]
#[case::escaped_quotes(
r#"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1giTh8KDkEchWhrAB6hGnb+V87kTezkt5I3SP7BGNg8wpv0yAuj/SUmnsttYmcEU+zmNAPqxePmCNvmjLYi/c3YyWEBwHcLyZE9OlS9W4enPdsoCuEN3DayzN4JCV3MsXMedCORvLFXmIARDXDLJUSJeqCeQoudXa9GmF1CrCmx70YyTtV0xOIxEzo7z0DkUL9" "7vGmNJCv6EMpi9wccMKKu8NSmOv+DBw1MLIJqChSZMCs8CYZ5i0KT/+Lijtn6B7wyOcAuQsVL+zr7DWYrFdrePe0wGuivfJ3SvUEfUo1SIykl0nvm0iLGhjNmNa1e/tUw4ULXhQ12Qw685+sq7wIDAQAB;s=email;t=s"#,
r#"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1giTh8KDkEchWhrAB6hGnb+V87kTezkt5I3SP7BGNg8wpv0yAuj/SUmnsttYmcEU+zmNAPqxePmCNvmjLYi/c3YyWEBwHcLyZE9OlS9W4enPdsoCuEN3DayzN4JCV3MsXMedCORvLFXmIARDXDLJUSJeqCeQoudXa9GmF1CrCmx70YyTtV0xOIxEzo7z0DkUL97vGmNJCv6EMpi9wccMKKu8NSmOv+DBw1MLIJqChSZMCs8CYZ5i0KT/+Lijtn6B7wyOcAuQsVL+zr7DWYrFdrePe0wGuivfJ3SvUEfUo1SIykl0nvm0iLGhjNmNa1e/tUw4ULXhQ12Qw685+sq7wIDAQAB;s=email;t=s"#
)]
fn test_normalize_rdata(#[case] input: &str, #[case] expected: &str) {
assert_eq!(normalize_rdata(input), expected);
}
}
-83
View File
@@ -1,83 +0,0 @@
//! Error types.
use tokio_rustls::rustls;
/// Error type for filtermail.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
#[error("Chatmail config is invalid: {0}")]
Config(#[from] serini::Error),
#[error(transparent)]
Io(#[from] std::io::Error),
#[error(transparent)]
Resolve(#[from] hickory_resolver::net::NetError),
#[error("OpenPGP packet header is truncated - can't validate!")]
TruncatedHeader,
#[error("Unable to send email, Error during {context}, host {host} said: {raw_smtp_answer}")]
MailSend {
context: String,
raw_smtp_answer: String,
host: String,
},
#[error("Invalid email address: {0}")]
InvalidEmailAddress(String),
#[error("Failed to connect to any of the following addresses: {0:?}")]
ConnectionFailed(Vec<String>),
#[error(transparent)]
Tls(#[from] rustls::Error),
#[error(transparent)]
InvalidDnsName(#[from] rustls::pki_types::InvalidDnsNameError),
#[error(transparent)]
Hyper(#[from] hyper::Error),
#[error(transparent)]
HyperHttp(#[from] hyper::http::Error),
#[error(transparent)]
HyperClient(#[from] hyper_util::client::legacy::Error),
}
impl Error {
/// Formats [`Error`] as an SMTP response.
pub fn smtp_response(&self) -> String {
macro_rules! format_smtp {
($code:expr) => {
format!("{} {}", $code, self.to_string())
};
}
match self {
// Errors returned by server we connect to are forwarded.
// We add "(forwarded from ...)" to distinguish these from our local errors.
Error::MailSend {
raw_smtp_answer,
host,
..
} => format!("{raw_smtp_answer} (forwarded from {host})"),
// Permanent errors
Error::TruncatedHeader => format_smtp!("554"),
Error::InvalidEmailAddress(_) => format_smtp!("553"),
Error::InvalidDnsName(_) => format_smtp!("501"),
// Transient errors
Error::ConnectionFailed(_) => format_smtp!("450"),
// We don't want to leak chatmail.ini config and other local error details.
Error::Config(_) => "451 Filtermail misconfigured; contact admin".to_string(),
Error::Io(_) => "451 I/O error".to_string(),
Error::Tls(_) => "451 TLS error".to_string(),
Error::Resolve(_) => "451 Resolver error".to_string(),
Error::Hyper(_) | Error::HyperHttp(_) | Error::HyperClient(_) => {
"451 HTTP error".to_string()
}
}
}
/// Same as [`smtp_response`](Self::smtp_response) but formats the same response
/// for each recipient, as expected by LMTP.
pub fn lmtp_response(&self, recipient_count: usize) -> String {
let response = self.smtp_response();
std::iter::repeat_n(response, recipient_count)
.collect::<Vec<_>>()
.join("\r\n")
}
}
-169
View File
@@ -1,169 +0,0 @@
use crate::smtp_server::{SmtpHandler, Transaction};
use http_body_util::combinators::BoxBody;
use http_body_util::{BodyExt, Full};
use hyper::body::{Bytes, Incoming};
use hyper::service::Service;
use hyper::{Request, Response};
use hyper_util::rt::TokioIo;
use std::convert::Infallible;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::{TcpListener, TcpStream};
/// Runs the HTTP server on the specified address with the given handler and maximum message size.
pub async fn run_http_server<H>(
addr: &impl tokio::net::ToSocketAddrs,
handler: Arc<H>,
max_size: usize,
) -> Result<(), crate::error::Error>
where
H: SmtpHandler + 'static,
{
let listener = TcpListener::bind(addr).await?;
loop {
match listener.accept().await {
Ok((socket, _peer_addr)) => {
// Disable Nagle's algorithm.
socket.set_nodelay(true)?;
let handler = handler.clone();
tokio::spawn(async move {
if let Err(e) = handle_connection(socket, handler, max_size).await {
log::error!("Error handling connection: {e}");
}
});
}
Err(e) => {
log::error!("Error accepting connection: {e}");
// Sleep to avoid busy looping in case we ran into file descriptor limit.
tokio::time::sleep(Duration::from_secs(10)).await;
}
}
}
}
/// Handles a single HTTP connection.
async fn handle_connection<H>(
socket: TcpStream,
handler: Arc<H>,
max_size: usize,
) -> Result<(), String>
where
H: SmtpHandler + 'static,
{
let service = MxDelivService::new(handler, max_size);
hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new())
.serve_connection(TokioIo::new(socket), service)
.await
.map_err(|e| e.to_string())?;
Ok(())
}
struct MxDelivService<H: SmtpHandler> {
handler: Arc<H>,
max_size: usize,
}
impl<H: SmtpHandler> MxDelivService<H> {
/// Creates a new [`MxDelivService`].
fn new(handler: Arc<H>, max_size: usize) -> Self {
Self { handler, max_size }
}
}
impl<H: SmtpHandler + 'static> Service<Request<Incoming>> for MxDelivService<H> {
type Response = Response<BoxBody<Bytes, Infallible>>;
type Error = crate::error::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn call(&self, req: Request<Incoming>) -> Self::Future {
let handler = self.handler.clone();
let max_size = self.max_size;
let fut = async move {
if req.method() != hyper::Method::POST {
return Ok(Response::builder().status(405).body(
// This is client's implementation error if it happens,
// so we don't care about sending a proper smtp response.
Full::new(Bytes::from("Method Not Allowed")).boxed(),
)?);
}
let mut transaction = Transaction::default();
let mail_from = req
.headers()
.get(crate::transport::HEADER_MAIL_FROM)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
if let Err(e) = handler.handle_mail_from(&mail_from) {
return Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(e)).boxed())?);
}
transaction.envelope.mail_from = mail_from;
let rcpt_to: Vec<String> = req
.headers()
.get_all(crate::transport::HEADER_RCPT_TO)
.iter()
.filter_map(|v| v.to_str().ok())
.map(ToString::to_string)
.collect();
for r in &rcpt_to {
if let Err(e) = handler.handle_rcpt_to(r, &mut transaction) {
return Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(e)).boxed())?);
}
}
transaction.envelope.rcpt_to = rcpt_to;
if let Err(e) = handler.handle_data_start(&transaction) {
return Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(e)).boxed())?);
}
let body_limited = http_body_util::Limited::new(req.into_body(), max_size);
let body_bytes = match body_limited.collect().await {
Ok(body) => body.to_bytes(),
Err(_) => {
return Ok(Response::builder().status(413).body(
Full::new(Bytes::from("552 Message exceeds maximum size")).boxed(),
)?);
}
};
transaction.envelope.data = body_bytes.to_vec();
log::debug!("(HTTP) MAIL FROM:<{}>", transaction.envelope.mail_from);
for rcpt in &transaction.envelope.rcpt_to {
log::debug!("(HTTP) RCPT TO:<{}>", rcpt);
}
log::trace!(
"(HTTP) DATA:\n{:?}",
String::from_utf8_lossy(&transaction.envelope.data)
);
match handler.handle_data_dot(&mut transaction).await {
Ok(response) => Ok(Response::builder()
.status(200)
.body(Full::new(Bytes::from(response)).boxed())?),
Err(e) => Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(e)).boxed())?),
}
};
Box::pin(fut)
}
}
-222
View File
@@ -1,222 +0,0 @@
//! Module for handling incoming SMTP messages.
use crate::config::Config;
use crate::dkim_verifier::DkimVerifier;
use crate::message::{check_encrypted, is_securejoin};
use crate::smtp_client::SmtpConnectionPool;
use crate::smtp_responses::ENCRYPTION_NEEDED_523;
pub use crate::smtp_server::Envelope;
use crate::smtp_server::{SmtpHandler, Transaction};
use crate::tcp::{TcpConnect, TcpStreamTrait};
use crate::utils::{AddressDomain, build_resolver, extract_address, log_eml};
use async_trait::async_trait;
use hickory_resolver::TokioResolver;
use mailparse::{MailHeaderMap, parse_mail};
use std::str::FromStr;
use std::sync::Arc;
/// Handler for incoming SMTP messages.
pub struct IncomingBeforeQueueHandler<S: TcpConnect> {
config: Config,
dns_resolver: Arc<TokioResolver>,
dkim_verifier: DkimVerifier,
skip_dkim: bool,
smtp_connection_pool: Arc<SmtpConnectionPool<S>>,
}
impl<S> IncomingBeforeQueueHandler<S>
where
S: TcpStreamTrait + TcpConnect,
S::ConnectionContext: Default,
{
pub fn new(config: Config, skip_dkim: bool) -> Result<Self, crate::error::Error> {
let dns_resolver = Arc::new(build_resolver()?);
Ok(Self {
config,
dns_resolver: dns_resolver.clone(),
dkim_verifier: DkimVerifier::new(dns_resolver),
skip_dkim,
smtp_connection_pool: SmtpConnectionPool::new(Default::default()),
})
}
/// Verify the origin of the email by performing a DKIM verification on a regular domain.
///
/// Currently a no-op for valid domain-literals.
async fn verify_origin(&self, envelope: &Envelope, from_addr: &str) -> Result<(), String> {
let from_domain = AddressDomain::from_str(from_addr).map_err(|e| e.smtp_response())?;
match from_domain {
AddressDomain::Literal(_) => {
// Subject to change: we currently don't perform any additional authentication
// for domain-literals and rely purely on encryption.
}
AddressDomain::Name(domain) => {
if !self.skip_dkim
&& let Err(e) = self.dkim_verifier.verify(&envelope.data, &domain).await
{
let eml_path = log_eml("dkim-verify", &envelope.data)
.await
.map(|path| path.to_string_lossy().to_string())
.unwrap_or_else(|e| {
log::error!("Failed to save rejected message to file: {e}");
"ERR".to_string()
});
log::info!("Rejected message stored at: {eml_path}");
return Err(e);
}
}
}
Ok(())
}
}
#[async_trait]
impl<S> SmtpHandler for IncomingBeforeQueueHandler<S>
where
S: TcpStreamTrait + TcpConnect,
S::ConnectionContext: Default,
{
type State = ();
async fn check_data(&self, transaction: &mut Transaction<Self::State>) -> Result<(), String> {
let message = match parse_mail(&transaction.envelope.data) {
Ok(m) => m,
Err(e) => return Err(format!("500 Failed to parse message: {}", e)),
};
let from_header = message
.headers
.get_first_value("From")
.unwrap_or_default()
.trim()
.to_string();
let Some(from_addr) = extract_address(&from_header) else {
return Err(format!("500 Invalid FROM header: {from_header}"));
};
log::debug!("Processing DATA message from {from_addr}");
if !transaction
.envelope
.mail_from
.eq_ignore_ascii_case(&from_addr)
{
// If the MAIL FROM doesn't match the From header, we do not reject the mail,
// as this can be caused by e.g. SRS forwarding.
// Instead, we reset the envelope address, so it is reinjected as
// `MAIL FROM:<>` to prevent sending a bounce message.
// <https://github.com/chatmail/filtermail/issues/67>
transaction.envelope.mail_from = String::new();
}
transaction.envelope.rcpt_to = transaction
.envelope
.rcpt_to
.iter()
.filter(|s| {
let disabled = self.config.is_disabled(s);
if disabled {
log::warn!("Disabled recipient: {s}; removing from RCPT TO");
}
!disabled
})
.cloned()
.collect();
let mail_encrypted = check_encrypted(&message, false);
log::debug!("mail_encrypted: {mail_encrypted}");
log::debug!("is_securejoin: {}", is_securejoin(&message));
// Allow encrypted or securejoin messages
if mail_encrypted || is_securejoin(&message) {
log::info!("Incoming: Filtering encrypted mail.");
return self.verify_origin(&transaction.envelope, &from_addr).await;
}
// Allow cleartext mailer-daemon messages
if let Some(auto_submitted) = message.headers.get_first_value("Auto-Submitted")
&& !auto_submitted.is_empty()
&& from_addr.to_lowercase().starts_with("mailer-daemon@")
&& message.ctype.mimetype == "multipart/report"
{
log::info!("Incoming: Filtering mailer-daemon message from <{from_addr}>");
return self.verify_origin(&transaction.envelope, &from_addr).await;
} else {
log::info!("Incoming: Filtering unencrypted mail.");
}
for recipient in &transaction.envelope.rcpt_to {
if !self.config.is_cleartext_ok(recipient) {
log::warn!("Rejected unencrypted mail from: {from_addr}");
return Err(ENCRYPTION_NEEDED_523.to_string());
}
}
self.verify_origin(&transaction.envelope, &from_addr).await
}
async fn reinject_mail(&self, transaction: &Transaction<Self::State>) -> Result<(), String> {
log::debug!("Re-injecting the mail that passed checks");
let hostname = format!("[{}]", self.config.filtermail_host);
let client_config = crate::smtp_client::ClientConfig {
client_hostname: &hostname,
tls_config: None,
lmtp: false,
};
crate::smtp_client::send(
&self.config.postfix_host,
self.config.postfix_reinject_port_incoming,
&transaction.envelope,
client_config,
self.dns_resolver.clone(),
self.smtp_connection_pool.clone(),
)
.await
.map_err(|e| {
log::warn!("Failed to re-inject mail: {}", e);
e.smtp_response()
})?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::{fixture, rstest};
use testresult::TestResult;
use tokio::net::TcpStream;
#[fixture]
fn config() -> Config {
Config::default()
}
/// Test that domain-literals are not rejected by origin check.
#[rstest]
#[case::ipv4(include_bytes!("../test_data/encrypted-ipv4.eml"), "one@[192.0.2.0]")]
// Waiting for a release of mailparse with the fix https://github.com/staktrace/mailparse/pull/138
// for the issue https://github.com/staktrace/mailparse/issues/137 to be released.
//#[case::ipv6(include_bytes!("../test_data/encrypted-ipv6.eml"), "one@[IPv6:2001:db8::1]")]
#[tokio::test]
async fn test_domain_literals_allowed(
#[case] eml: &[u8],
#[case] address: &str,
config: Config,
) -> TestResult {
let handler = IncomingBeforeQueueHandler::<TcpStream>::new(config, false)?;
let mut transaction = Transaction {
envelope: Envelope {
mail_from: address.to_string(),
data: eml.to_vec(),
rcpt_to: vec!["does.not.matter@example.org".to_string()],
},
..Default::default()
};
Ok(handler.check_data(&mut transaction).await?)
}
}
-199
View File
@@ -1,199 +0,0 @@
#![doc = include_str!("../README.md")]
#![forbid(unsafe_code)]
#![warn(
unused,
clippy::correctness,
missing_debug_implementations,
missing_docs,
clippy::all,
clippy::wildcard_imports,
clippy::needless_borrow,
clippy::cast_lossless,
clippy::unused_async,
clippy::explicit_iter_loop,
clippy::explicit_into_iter_loop,
clippy::cloned_instead_of_copied
)]
#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
#![cfg_attr(not(test), forbid(clippy::string_slice))]
#![allow(
clippy::match_bool,
clippy::mixed_read_write_in_expression,
clippy::bool_assert_comparison,
clippy::manual_split_once,
clippy::format_push_string,
clippy::bool_to_int_with_if
)]
mod config;
mod dkim_verifier;
pub(crate) mod error;
mod http_server;
pub(crate) mod inbound;
pub(crate) mod message;
pub(crate) mod openpgp;
pub(crate) mod outbound;
pub(crate) mod smtp_client;
mod smtp_responses;
pub(crate) mod smtp_server;
mod tcp;
mod tls;
mod transport;
pub(crate) mod utils;
use crate::http_server::run_http_server;
use crate::transport::TransportHandler;
use config::Config;
use env_logger::Env;
use inbound::IncomingBeforeQueueHandler;
use outbound::OutgoingBeforeQueueHandler;
use smtp_server::run_smtp_server;
use std::env;
use std::process;
use std::str::FromStr;
use std::sync::Arc;
use tokio::net::TcpStream;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum Mode {
Outgoing,
Incoming,
Transport,
}
impl FromStr for Mode {
type Err = &'static str;
fn from_str(mode: &str) -> Result<Self, Self::Err> {
match mode {
"outgoing" => Ok(Mode::Outgoing),
"incoming" => Ok(Mode::Incoming),
"transport" => Ok(Mode::Transport),
_ => Err("Error: mode must be 'incoming', 'outgoing' or 'transport'"),
}
}
}
#[tokio::main]
async fn main() -> Result<(), error::Error> {
// default to info level
let env = Env::new().filter_or("RUST_LOG", "info");
env_logger::Builder::from_env(env)
// disable timestamps - automatically added by systemd
.format_timestamp(None)
.init();
tokio_rustls::rustls::crypto::aws_lc_rs::default_provider()
.install_default()
.expect("Failed to set up rustls crypto provider.");
let args: Vec<String> = env::args().collect();
if args.len() != 3 {
eprintln!(
"Usage: {} <config_file> <mode>",
args.first().unwrap_or(&"filtermail".to_string())
);
eprintln!(" mode: incoming, outgoing or transport");
process::exit(1);
}
let Some(config_path) = args.get(1) else {
unreachable!("args length checked above")
};
let Some(mode) = args.get(2) else {
unreachable!("args length checked above")
};
let mode = match Mode::from_str(mode) {
Ok(mode) => mode,
Err(e) => {
eprintln!("{e}");
process::exit(1);
}
};
let config = match Config::from_file(config_path) {
Ok(c) => c,
Err(e) => {
eprintln!("Failed to read config: {}", e);
process::exit(1);
}
};
match mode {
Mode::Outgoing => {
let addr = (config.filtermail_host, config.filtermail_smtp_port);
let handler = Arc::new(OutgoingBeforeQueueHandler::<TcpStream>::new(
config.clone(),
)?);
let max_size = config.max_message_size;
log::debug!("Outgoing SMTP server listening on {}:{}", addr.0, addr.1);
if let Err(e) = run_smtp_server(&addr, handler, max_size).await {
eprintln!("Server error: {}", e);
process::exit(1);
}
}
Mode::Incoming => {
// Skip DKIM verification (used for tests).
let skip_dkim = env::var("FILTERMAIL_SKIP_DKIM")
.map(|val| val == "1" || val.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if skip_dkim {
log::warn!("DKIM verification DISABLED! This should not be used in production.");
}
let handler = Arc::new(IncomingBeforeQueueHandler::<TcpStream>::new(
config.clone(),
skip_dkim,
)?);
let max_size = config.max_message_size;
let mut server_set = tokio::task::JoinSet::new();
let addr_smtp = (config.filtermail_host, config.filtermail_smtp_port_incoming);
let handler_smtp = handler.clone();
server_set
.spawn(async move { run_smtp_server(&addr_smtp, handler_smtp, max_size).await });
log::debug!(
"Incoming SMTP server listening on {}:{}",
addr_smtp.0,
addr_smtp.1
);
let addr_http = (config.filtermail_host, config.filtermail_http_port_incoming);
let handler_http = handler.clone();
server_set
.spawn(async move { run_http_server(&addr_http, handler_http, max_size).await });
log::debug!(
"Incoming HTTP server listening on {}:{}",
addr_http.0,
addr_http.1
);
while let Some(result) = server_set.join_next().await {
if let Err(e) = result {
eprintln!("Server error: {}", e);
process::exit(1);
}
}
}
Mode::Transport => {
let addr = (
config.filtermail_host,
config.filtermail_lmtp_port_transport,
);
let handler = Arc::new(TransportHandler::<TcpStream>::new(config.clone())?);
let max_size = config.max_message_size;
log::debug!("Transport SMTP server listening on {}:{}", addr.0, addr.1);
if let Err(e) = run_smtp_server(&addr, handler, max_size).await {
eprintln!("Server error: {}", e);
process::exit(1);
}
}
};
Ok(())
}
-174
View File
@@ -1,174 +0,0 @@
//! Message-related checks.
use crate::openpgp::check_armored_payload;
use mailparse::MailHeaderMap;
/// Check if message is a secure-join message (vc-request or vg-request)
pub fn is_securejoin(mail: &mailparse::ParsedMail) -> bool {
// Check for secure-join header
let secure_join = mail.headers.get_first_value("Secure-Join");
if let Some(ref val) = secure_join {
if val != "vc-request" && val != "vg-request" {
return false;
}
} else {
return false;
}
// Must be multipart
if mail.subparts.is_empty() {
return false;
}
// Must have only one part
if mail.subparts.len() != 1 {
return false;
}
let Some(part) = &mail.subparts.first() else {
return false;
};
// Part must not be multipart
if !part.subparts.is_empty() {
return false;
}
// Part must be text/plain
if part.ctype.mimetype != "text/plain" {
return false;
}
// Check payload content
let payload = match part.get_body() {
Ok(p) => p.trim().to_lowercase(),
Err(_) => return false,
};
payload == "secure-join: vc-request" || payload == "secure-join: vg-request"
}
/// Check that the message is an OpenPGP-encrypted message
///
/// MIME structure must correspond to RFC3156
pub fn check_encrypted(mail: &mailparse::ParsedMail, outgoing: bool) -> bool {
if mail.subparts.is_empty() {
log::debug!("check_encrypted: not multipart");
return false;
}
if !mail
.ctype
.mimetype
.eq_ignore_ascii_case("multipart/encrypted")
{
log::debug!("check_encrypted: not multipart/encrypted");
return false;
}
for (part_idx, part) in mail.subparts.iter().enumerate() {
// Each part must not be multipart
if !part.subparts.is_empty() {
log::debug!("check_encrypted: part of multipart/encrypted is itself multipart");
return false;
}
if part_idx == 0 {
// First part must be application/pgp-encrypted
if !part
.ctype
.mimetype
.eq_ignore_ascii_case("application/pgp-encrypted")
{
log::debug!(
"check_encrypted: first part not application/pgp-encrypted, got: {}",
part.ctype.mimetype
);
return false;
}
// Payload must be "Version: 1"
let payload = match part.get_body() {
Ok(p) => p,
Err(_) => {
log::debug!("check_encrypted: failed to get body of first part");
return false;
}
};
if payload.trim() != "Version: 1" {
log::debug!(
"check_encrypted: first part payload not 'Version: 1', got {}",
payload.trim()
);
return false;
}
} else if part_idx == 1 {
// Second part must be application/octet-stream
if part.ctype.mimetype != "application/octet-stream" {
log::debug!(
"check_encrypted: second part not application/octet-stream, got: {}",
part.ctype.mimetype
);
return false;
}
// Check the armored payload
let payload = match part.get_body() {
Ok(p) => p,
Err(_) => {
log::debug!("check_encrypted: failed to get body of second part");
return false;
}
};
if !check_armored_payload(&payload, outgoing) {
log::debug!("check_encrypted: armored payload check failed");
return false;
}
} else {
log::debug!("check_encrypted: more than two parts found");
return false;
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
use mailparse::parse_mail;
use rstest::*;
use testresult::TestResult;
#[rstest]
#[case::asm("test_data/asm.eml", false)]
#[case::encrypted("test_data/encrypted.eml", false)]
#[case::fake_encrypted("test_data/fake-encrypted.eml", false)]
#[case::literal("test_data/literal.eml", false)]
#[case::mailer_daemon("test_data/mailer-daemon.eml", false)]
#[case::mdn("test_data/mdn.eml", false)]
#[case::plain("test_data/plain.eml", false)]
#[case::securejoin_vc("test_data/securejoin-vc.eml", true)]
#[case::securejoin_vc_fake("test_data/securejoin-vc-fake.eml", false)]
fn test_is_securejoin(#[case] file: &str, #[case] expected: bool) -> TestResult {
let raw_email = std::fs::read_to_string(file)?;
let parsed = parse_mail(raw_email.as_bytes())?;
assert_eq!(is_securejoin(&parsed), expected);
Ok(())
}
#[rstest]
#[case::asm("test_data/asm.eml", false)]
#[case::encrypted("test_data/encrypted.eml", true)]
#[case::fake_encrypted("test_data/fake-encrypted.eml", false)]
#[case::literal("test_data/literal.eml", false)]
#[case::mailer_daemon("test_data/mailer-daemon.eml", false)]
#[case::mdn("test_data/mdn.eml", false)]
#[case::plain("test_data/plain.eml", false)]
#[case::securejoin_vc("test_data/securejoin-vc.eml", false)]
#[case::securejoin_vc_fake("test_data/securejoin-vc-fake.eml", false)]
fn test_check_encrypted(#[case] file: &str, #[case] expected: bool) -> TestResult {
let raw_email = std::fs::read_to_string(file)?;
let parsed = parse_mail(raw_email.as_bytes())?;
assert_eq!(check_encrypted(&parsed, false), expected);
Ok(())
}
}

Some files were not shown because too many files have changed in this diff Show More