mirror of
https://github.com/chatmail/relay.git
synced 2026-05-10 16:04:37 +00:00
Compare commits
1 Commits
link2xt/an
...
j4n/docker
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0808c9dc47 |
49
.github/workflows/ci.yaml
vendored
49
.github/workflows/ci.yaml
vendored
@@ -1,35 +1,21 @@
|
||||
name: Run unit-tests and container-based deploy+test verification
|
||||
name: CI
|
||||
|
||||
on:
|
||||
# Triggers when a PR is merged into main or a direct push occurs
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
|
||||
# Triggers for any PR (and its subsequent commits) targeting the main branch
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
permissions: {}
|
||||
|
||||
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
push:
|
||||
|
||||
jobs:
|
||||
tox:
|
||||
name: isolated chatmaild tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
# Checkout pull request HEAD commit instead of merge commit
|
||||
# Otherwise `test_deployed_state` will be unhappy.
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
persist-credentials: false
|
||||
- name: download filtermail
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
- name: run chatmaild tests
|
||||
working-directory: chatmaild
|
||||
run: pipx run tox
|
||||
@@ -38,10 +24,7 @@ jobs:
|
||||
name: deploy-chatmail tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
@@ -55,23 +38,5 @@ jobs:
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
lxc-test:
|
||||
name: LXC deploy and test
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
|
||||
with:
|
||||
cmlxc_commands: |
|
||||
cmlxc init
|
||||
# single cmdeploy relay test
|
||||
cmlxc -v deploy-cmdeploy --source ./repo cm0
|
||||
cmlxc -v test-mini cm0
|
||||
cmlxc -v test-cmdeploy cm0
|
||||
|
||||
# cross cmdeploy relay test
|
||||
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
|
||||
cmlxc -v test-cmdeploy cm0 cm1
|
||||
|
||||
# cross cmdeploy/madmail relay tests
|
||||
cmlxc -v deploy-madmail mad0
|
||||
cmlxc -v test-cmdeploy cm0 mad0
|
||||
cmlxc -v test-mini cm0 mad0
|
||||
cmlxc -v test-mini mad0 cm0
|
||||
# all other cmdeploy commands require a staging server
|
||||
# see https://github.com/deltachat/chatmail/issues/100
|
||||
|
||||
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
@@ -0,0 +1,125 @@
|
||||
name: Docker deploy
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
staging_host:
|
||||
required: true
|
||||
type: string
|
||||
description: 'SSH hostname (e.g. staging2.testrun.org)'
|
||||
mail_domain:
|
||||
required: true
|
||||
type: string
|
||||
description: 'MAIL_DOMAIN for docker compose'
|
||||
zone_file:
|
||||
required: true
|
||||
type: string
|
||||
description: 'Default zone file basename (e.g. staging.testrun.org-default.zone)'
|
||||
|
||||
jobs:
|
||||
deploy-docker:
|
||||
name: Docker deploy on ${{ inputs.staging_host }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
environment:
|
||||
name: ${{ inputs.staging_host }}
|
||||
url: https://${{ inputs.staging_host }}/
|
||||
concurrency: ${{ inputs.staging_host }}
|
||||
env:
|
||||
VPS: root@${{ inputs.staging_host }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- name: Setup SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan ${{ inputs.staging_host }} > ~/.ssh/known_hosts
|
||||
# Reuse TCP connection for all subsequent ssh/scp calls
|
||||
echo -e "Host ${{ inputs.staging_host }}\n ControlMaster auto\n ControlPath ~/.ssh/ctrl-%r@%h:%p\n ControlPersist 10m" >> ~/.ssh/config
|
||||
|
||||
- name: stop bare services, install Docker, prepare mounts
|
||||
run: |
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||
mkdir -p /srv/chatmail/certs /srv/chatmail/dkim
|
||||
cp -a /var/lib/acme/. /srv/chatmail/certs/ || true
|
||||
cp -a /etc/dkimkeys/. /srv/chatmail/dkim/ || true
|
||||
cp /etc/chatmail/chatmail.ini /srv/chatmail/chatmail.ini
|
||||
EOF
|
||||
|
||||
- name: deploy with Docker
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
GHCR_IMAGE="ghcr.io/chatmail/docker:sha-${SHORT_SHA}"
|
||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ $VPS:/srv/chatmail/relay/
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh $VPS "docker login ghcr.io -u ${{ github.actor }} --password-stdin && \
|
||||
docker pull ${GHCR_IMAGE} && \
|
||||
cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=${{ inputs.mail_domain }} docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
||||
|
||||
- name: wait for container healthy
|
||||
run: |
|
||||
ssh $VPS 'docker exec chatmail journalctl -f --no-pager' &
|
||||
LOG_PID=$!
|
||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
||||
for i in $(seq 1 60); do
|
||||
status=$(ssh $VPS 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
||||
echo " [$i/60] status=$status"
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "Container is healthy."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$status" = "unhealthy" ]; then
|
||||
echo "Container is unhealthy!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Container did not become healthy."
|
||||
kill $LOG_PID 2>/dev/null || true
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
echo "--- failed units ---"
|
||||
docker exec chatmail systemctl --failed --no-pager || true
|
||||
echo "--- service logs ---"
|
||||
docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50 || true
|
||||
echo "--- listening ports ---"
|
||||
docker exec chatmail ss -tlnp || true
|
||||
echo "--- chatmail.ini ---"
|
||||
docker exec chatmail cat /etc/chatmail/chatmail.ini || true
|
||||
EOF
|
||||
exit 1
|
||||
|
||||
- name: show container state
|
||||
run: |
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
echo "--- listening ports ---"
|
||||
docker exec chatmail ss -tlnp
|
||||
echo "--- chatmail.ini ---"
|
||||
docker exec chatmail cat /etc/chatmail/chatmail.ini
|
||||
EOF
|
||||
|
||||
- name: Docker integration tests
|
||||
run: ssh $VPS 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
||||
|
||||
- name: Docker DNS
|
||||
run: |
|
||||
git checkout .github/workflows/${{ inputs.zone_file }}
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys
|
||||
docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose
|
||||
docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone
|
||||
EOF
|
||||
scp $VPS:/tmp/staging.zone staging-generated.zone
|
||||
cat staging-generated.zone >> .github/workflows/${{ inputs.zone_file }}
|
||||
cat .github/workflows/${{ inputs.zone_file }}
|
||||
scp .github/workflows/${{ inputs.zone_file }} root@ns.testrun.org:/etc/nsd/${{ inputs.staging_host }}.zone
|
||||
ssh root@ns.testrun.org "nsd-checkzone ${{ inputs.staging_host }} /etc/nsd/${{ inputs.staging_host }}.zone && systemctl reload nsd"
|
||||
|
||||
- name: Docker final DNS check
|
||||
run: ssh $VPS 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
||||
37
.github/workflows/docker-dispatch.yaml
vendored
37
.github/workflows/docker-dispatch.yaml
vendored
@@ -1,37 +0,0 @@
|
||||
# Notify the docker repo to build and test a new image after relay CI passes.
|
||||
#
|
||||
# Sends a repository_dispatch event to chatmail/docker with the relay ref
|
||||
# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
|
||||
# and run integration tests via cmlxc.
|
||||
|
||||
name: Trigger Docker build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
name: Dispatch build to chatmail/docker
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'chatmail/relay'
|
||||
steps:
|
||||
- name: Compute short SHA
|
||||
id: sha
|
||||
run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Send repository_dispatch
|
||||
uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
|
||||
with:
|
||||
token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
|
||||
repository: chatmail/docker
|
||||
event-type: relay-updated
|
||||
client-payload: >-
|
||||
{
|
||||
"relay_ref": "${{ github.ref_name }}",
|
||||
"relay_sha": "${{ github.sha }}",
|
||||
"relay_sha_short": "${{ steps.sha.outputs.short }}"
|
||||
}
|
||||
14
.github/workflows/docs-preview.yaml
vendored
14
.github/workflows/docs-preview.yaml
vendored
@@ -7,8 +7,6 @@ on:
|
||||
- 'scripts/build-docs.sh'
|
||||
- '.github/workflows/docs-preview.yaml'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
scripts:
|
||||
name: build
|
||||
@@ -18,8 +16,6 @@ jobs:
|
||||
url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
@@ -38,22 +34,18 @@ jobs:
|
||||
- name: Get Pullrequest ID
|
||||
id: prepare
|
||||
run: |
|
||||
export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
|
||||
export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
|
||||
echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
|
||||
if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
|
||||
- run: |
|
||||
echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
|
||||
env:
|
||||
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
|
||||
echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
|
||||
|
||||
- name: Upload preview
|
||||
run: |
|
||||
mkdir -p "$HOME/.ssh"
|
||||
echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
|
||||
chmod 600 "$HOME/.ssh/key"
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
|
||||
env:
|
||||
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
|
||||
|
||||
- name: check links
|
||||
working-directory: doc
|
||||
|
||||
4
.github/workflows/docs.yaml
vendored
4
.github/workflows/docs.yaml
vendored
@@ -10,8 +10,6 @@ on:
|
||||
- 'scripts/build-docs.sh'
|
||||
- '.github/workflows/docs.yaml'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
scripts:
|
||||
name: build
|
||||
@@ -21,8 +19,6 @@ jobs:
|
||||
url: https://chatmail.at/doc/relay/
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
|
||||
121
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
Normal file
121
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
Normal file
@@ -0,0 +1,121 @@
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||
secrets: inherit
|
||||
|
||||
deploy:
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging-ipv4.testrun.org
|
||||
url: https://staging-ipv4.testrun.org/
|
||||
concurrency: staging-ipv4.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
|
||||
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
run: |
|
||||
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging-ipv4.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- name: setup dependencies
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org apt update
|
||||
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
|
||||
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
|
||||
|
||||
- name: initialize config
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
|
||||
|
||||
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
|
||||
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
||||
|
||||
deploy-docker:
|
||||
needs: [deploy, trigger-docker-build]
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/docker-deploy.yaml
|
||||
with:
|
||||
staging_host: staging-ipv4.testrun.org
|
||||
mail_domain: staging-ipv4.testrun.org
|
||||
zone_file: staging-ipv4.testrun.org-default.zone
|
||||
secrets: inherit
|
||||
112
.github/workflows/test-and-deploy.yaml
vendored
Normal file
112
.github/workflows/test-and-deploy.yaml
vendored
Normal file
@@ -0,0 +1,112 @@
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||
secrets: inherit
|
||||
|
||||
deploy:
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging2.testrun.org
|
||||
url: https://staging2.testrun.org/
|
||||
concurrency: staging2.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
|
||||
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging2.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
run: |
|
||||
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging2.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: add hpk42 key to staging server
|
||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- run: |
|
||||
cmdeploy init staging2.testrun.org
|
||||
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
|
||||
|
||||
- run: cmdeploy run --verbose --skip-dns-check
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
cmdeploy dns --zonefile staging-generated.zone --verbose
|
||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
||||
cat .github/workflows/staging.testrun.org-default.zone
|
||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: cmdeploy dns -v
|
||||
|
||||
deploy-docker:
|
||||
needs: [deploy, trigger-docker-build]
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/docker-deploy.yaml
|
||||
with:
|
||||
staging_host: staging2.testrun.org
|
||||
mail_domain: staging2.testrun.org
|
||||
zone_file: staging.testrun.org-default.zone
|
||||
secrets: inherit
|
||||
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
name: Trigger Docker image build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
name: Trigger Docker image build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
|
||||
script: |
|
||||
await github.rest.repos.createDispatchEvent({
|
||||
owner: 'chatmail',
|
||||
repo: 'docker',
|
||||
event_type: 'relay-updated',
|
||||
client_payload: {
|
||||
relay_ref: context.ref,
|
||||
relay_sha: context.sha,
|
||||
relay_sha_short: context.sha.slice(0, 7)
|
||||
}
|
||||
})
|
||||
26
.github/workflows/zizmor-scan.yml
vendored
26
.github/workflows/zizmor-scan.yml
vendored
@@ -1,26 +0,0 @@
|
||||
name: GitHub Actions Security Analysis with zizmor
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
zizmor:
|
||||
name: Run zizmor
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
|
||||
7
.github/zizmor.yml
vendored
7
.github/zizmor.yml
vendored
@@ -1,7 +0,0 @@
|
||||
rules:
|
||||
unpinned-uses:
|
||||
config:
|
||||
policies:
|
||||
actions/*: ref-pin
|
||||
dependabot/*: ref-pin
|
||||
chatmail/*: ref-pin
|
||||
84
CHANGELOG.md
84
CHANGELOG.md
@@ -1,89 +1,5 @@
|
||||
# Changelog for chatmail deployment
|
||||
|
||||
## 1.10.0 2026-04-30
|
||||
|
||||
* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
|
||||
* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
|
||||
* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
|
||||
* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
|
||||
* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
|
||||
* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
|
||||
* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
|
||||
* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
|
||||
* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
|
||||
* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
|
||||
* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
|
||||
* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
|
||||
* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
|
||||
* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
|
||||
* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
|
||||
* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
|
||||
* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
|
||||
* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
|
||||
* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
|
||||
* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
|
||||
* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
|
||||
* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
|
||||
* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
|
||||
* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
|
||||
* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
|
||||
* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
|
||||
* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
|
||||
* stop installing cron package <https://github.com/chatmail/relay/pull/898>
|
||||
* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
|
||||
* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
|
||||
* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
|
||||
* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
|
||||
* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
|
||||
* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
|
||||
* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
|
||||
* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
|
||||
* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
|
||||
* dovecot update url <https://github.com/chatmail/relay/pull/880>
|
||||
* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
|
||||
* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
|
||||
* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
|
||||
* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
|
||||
* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
|
||||
* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
|
||||
* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
|
||||
* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
|
||||
* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
|
||||
* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
|
||||
* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
|
||||
* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
|
||||
* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
|
||||
* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
|
||||
* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
|
||||
* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
|
||||
* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
|
||||
* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
|
||||
* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
|
||||
* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
|
||||
* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
|
||||
* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
|
||||
* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
|
||||
* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
|
||||
* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
|
||||
* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
|
||||
* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
|
||||
* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
|
||||
* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
|
||||
* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
|
||||
* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
|
||||
* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
|
||||
* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
|
||||
* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
|
||||
* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
|
||||
* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
|
||||
* try to use "build machine" and "deployment server" consistently <https://github.com/chatmail/relay/pull/797>
|
||||
* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
|
||||
* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
|
||||
* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
|
||||
* streamline migration guide wording, provide titled steps <https://github.com/chatmail/relay/pull/789>
|
||||
* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
|
||||
* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
|
||||
|
||||
## 1.9.0 2025-12-18
|
||||
|
||||
### Documentation
|
||||
|
||||
@@ -21,10 +21,8 @@ where = ['src']
|
||||
[project.scripts]
|
||||
doveauth = "chatmaild.doveauth:main"
|
||||
chatmail-metadata = "chatmaild.metadata:main"
|
||||
chatmail-expire = "chatmaild.expire:daily_expire_main"
|
||||
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
|
||||
chatmail-expire = "chatmaild.expire:main"
|
||||
chatmail-fsreport = "chatmaild.fsreport:main"
|
||||
chatmail-deferred = "chatmaild.deferred:main"
|
||||
lastlogin = "chatmaild.lastlogin:main"
|
||||
turnserver = "chatmaild.turnserver:main"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import iniconfig
|
||||
@@ -37,18 +38,15 @@ class Config:
|
||||
self.filtermail_smtp_port_incoming = int(
|
||||
params.get("filtermail_smtp_port_incoming", "10081")
|
||||
)
|
||||
self.filtermail_http_port_incoming = int(
|
||||
params.get("filtermail_http_port_incoming", "10082")
|
||||
)
|
||||
self.filtermail_lmtp_port_transport = int(
|
||||
params.get("filtermail_lmtp_port_transport", "10083")
|
||||
)
|
||||
self.filtermail_http_port = int(params.get("filtermail_http_port", "10082"))
|
||||
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
|
||||
self.postfix_reinject_port_incoming = int(
|
||||
params.get("postfix_reinject_port_incoming", "10026")
|
||||
)
|
||||
self.mtail_address = params.get("mtail_address")
|
||||
self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
|
||||
self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
|
||||
self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
|
||||
self.acme_email = params.get("acme_email", "")
|
||||
self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
|
||||
self.imap_compress = params.get("imap_compress", "false").lower() == "true"
|
||||
@@ -95,11 +93,6 @@ class Config:
|
||||
# old unused option (except for first migration from sqlite to maildir store)
|
||||
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
|
||||
|
||||
@property
|
||||
def max_mailbox_size_mb(self):
|
||||
"""Return max_mailbox_size as an integer in megabytes."""
|
||||
return parse_size_mb(self.max_mailbox_size)
|
||||
|
||||
def _getbytefile(self):
|
||||
return open(self._inipath, "rb")
|
||||
|
||||
@@ -113,16 +106,6 @@ class Config:
|
||||
return User(maildir, addr, password_path, uid="vmail", gid="vmail")
|
||||
|
||||
|
||||
def parse_size_mb(limit):
|
||||
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
|
||||
value = limit.strip().upper().removesuffix("B")
|
||||
if value.endswith("G"):
|
||||
return int(value[:-1]) * 1024
|
||||
if value.endswith("M"):
|
||||
return int(value[:-1])
|
||||
return int(value)
|
||||
|
||||
|
||||
def write_initial_config(inipath, mail_domain, overrides):
|
||||
"""Write out default config file, using the specified config value overrides."""
|
||||
content = get_default_config_content(mail_domain, **overrides)
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
"""
|
||||
Analyze deferred mails and print most common failing destinations.
|
||||
|
||||
Example:
|
||||
|
||||
python -m chatmaild.deferred
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
|
||||
def main():
|
||||
p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
|
||||
domain_reasons = defaultdict(Counter)
|
||||
domain_total = Counter()
|
||||
|
||||
for line in p.stdout:
|
||||
item = json.loads(line)
|
||||
if item["queue_name"] != "deferred":
|
||||
continue
|
||||
|
||||
for recipient in item["recipients"]:
|
||||
_, domain = recipient["address"].rsplit("@", 1)
|
||||
reason = recipient["delay_reason"]
|
||||
domain_total[domain] += 1
|
||||
domain_reasons[domain][reason] += 1
|
||||
|
||||
for domain, total in reversed(domain_total.most_common()):
|
||||
print(f"{domain} ({total} recipients)")
|
||||
for reason, count in domain_reasons[domain].most_common():
|
||||
print(f" {count}: {reason}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,26 +4,17 @@ Expire old messages and addresses.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from argparse import ArgumentParser
|
||||
from collections import namedtuple
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from stat import S_ISREG
|
||||
|
||||
from chatmaild.config import read_config
|
||||
|
||||
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
||||
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
|
||||
|
||||
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
|
||||
QUOTA_CLEANUP_FACTOR = 0.7
|
||||
|
||||
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
||||
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
|
||||
|
||||
|
||||
def iter_mailboxes(basedir, maxnum):
|
||||
@@ -83,42 +74,6 @@ class MailboxStat:
|
||||
self.extrafiles.sort(key=lambda x: -x.size)
|
||||
|
||||
|
||||
def parse_dovecot_filename(relpath):
|
||||
m = _dovecot_fn_rex.match(relpath)
|
||||
if not m:
|
||||
return None
|
||||
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
|
||||
|
||||
|
||||
def scan_mailbox_messages(mbox):
|
||||
messages = []
|
||||
for sub in ("cur", "new"):
|
||||
for name in os_listdir_if_exists(mbox / sub):
|
||||
if entry := parse_dovecot_filename(f"{sub}/{name}"):
|
||||
messages.append(entry)
|
||||
return messages
|
||||
|
||||
|
||||
def expire_to_target(mbox, target_bytes):
|
||||
messages = scan_mailbox_messages(mbox)
|
||||
total_size = sum(m.quota_size for m in messages)
|
||||
# Keep recent 24 hours of messages protected from expiry because
|
||||
# likely something is wrong with interactions on that address
|
||||
# and quota-full signal can help the address owner's device to notice it
|
||||
undeletable_messages_cutoff = time.time() - (3600 * 24)
|
||||
removed = 0
|
||||
for entry in sorted(messages):
|
||||
if total_size <= target_bytes:
|
||||
break
|
||||
if entry.mtime > undeletable_messages_cutoff:
|
||||
break
|
||||
(mbox / entry.path).unlink(missing_ok=True)
|
||||
total_size -= entry.quota_size
|
||||
removed += 1
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def print_info(msg):
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
@@ -188,19 +143,6 @@ class Expiry:
|
||||
else:
|
||||
continue
|
||||
changed = True
|
||||
|
||||
target_bytes = (
|
||||
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
|
||||
)
|
||||
removed = expire_to_target(Path(mbox.basedir), target_bytes)
|
||||
if removed:
|
||||
changed = True
|
||||
self.del_files += removed
|
||||
if self.verbose:
|
||||
print_info(
|
||||
f"quota-expire: removed {removed} message(s) from {mboxname}"
|
||||
)
|
||||
|
||||
if changed:
|
||||
self.remove_file(f"{mbox.basedir}/maildirsize")
|
||||
|
||||
@@ -212,9 +154,9 @@ class Expiry:
|
||||
)
|
||||
|
||||
|
||||
def daily_expire_main(args=None):
|
||||
def main(args=None):
|
||||
"""Expire mailboxes and messages according to chatmail config"""
|
||||
parser = ArgumentParser(description=daily_expire_main.__doc__)
|
||||
parser = ArgumentParser(description=main.__doc__)
|
||||
ini = "/usr/local/lib/chatmaild/chatmail.ini"
|
||||
parser.add_argument(
|
||||
"chatmail_ini",
|
||||
@@ -260,33 +202,5 @@ def daily_expire_main(args=None):
|
||||
print(exp.get_summary())
|
||||
|
||||
|
||||
def quota_expire_main(args=None):
|
||||
"""Remove mailbox messages to stay within a megabyte target.
|
||||
|
||||
This entry point is called by dovecot when a quota threshold is passed.
|
||||
"""
|
||||
|
||||
parser = ArgumentParser(description=quota_expire_main.__doc__)
|
||||
parser.add_argument(
|
||||
"target_mb",
|
||||
type=int,
|
||||
help="target mailbox size in megabytes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"mailbox_path",
|
||||
type=Path,
|
||||
help="path to a user mailbox",
|
||||
)
|
||||
args = parser.parse_args(args)
|
||||
|
||||
target_bytes = args.target_mb * 1024 * 1024
|
||||
|
||||
removed_count = expire_to_target(args.mailbox_path, target_bytes)
|
||||
if removed_count:
|
||||
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
|
||||
print(
|
||||
f"quota-expire: removed {removed_count} message(s)"
|
||||
f" from {args.mailbox_path.name}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
|
||||
@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
|
||||
max_user_send_burst_size = 10
|
||||
|
||||
# maximum mailbox size of a chatmail address
|
||||
# Oldest messages will be removed automatically, so mailboxes never run full.
|
||||
max_mailbox_size = 500M
|
||||
|
||||
# maximum message size for an e-mail in bytes
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
"""CGI script for creating new accounts."""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import secrets
|
||||
import string
|
||||
@@ -15,16 +14,6 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
|
||||
ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
|
||||
|
||||
|
||||
def wrap_ip(host):
|
||||
if host.startswith("[") and host.endswith("]"):
|
||||
return host
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
return f"[{host}]"
|
||||
except ValueError:
|
||||
return host
|
||||
|
||||
|
||||
def create_newemail_dict(config: Config):
|
||||
user = "".join(
|
||||
secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
|
||||
@@ -33,7 +22,7 @@ def create_newemail_dict(config: Config):
|
||||
secrets.choice(ALPHANUMERIC_PUNCT)
|
||||
for _ in range(config.password_min_length + 3)
|
||||
)
|
||||
return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
|
||||
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
|
||||
|
||||
|
||||
def create_dclogin_url(email, password):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from chatmaild.config import parse_size_mb, read_config
|
||||
from chatmaild.config import read_config
|
||||
|
||||
|
||||
def test_read_config_basic(example_config):
|
||||
@@ -121,17 +121,3 @@ def test_config_tls_external_bad_format(make_config):
|
||||
"tls_external_cert_and_key": "/only/one/path.pem",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_parse_size_mb():
|
||||
assert parse_size_mb("500M") == 500
|
||||
assert parse_size_mb("2G") == 2048
|
||||
assert parse_size_mb(" 1g ") == 1024
|
||||
assert parse_size_mb("100MB") == 100
|
||||
assert parse_size_mb("256") == 256
|
||||
|
||||
|
||||
def test_max_mailbox_size_mb(make_config):
|
||||
config = make_config("chat.example.org")
|
||||
assert config.max_mailbox_size == "500M"
|
||||
assert config.max_mailbox_size_mb == 500
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import time
|
||||
|
||||
from chatmaild.doveauth import AuthDictProxy
|
||||
from chatmaild.expire import daily_expire_main as main_expire
|
||||
from chatmaild.expire import main as main_expire
|
||||
|
||||
|
||||
def test_login_timestamps(example_config):
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from fnmatch import fnmatch
|
||||
from pathlib import Path
|
||||
@@ -11,19 +9,13 @@ import pytest
|
||||
from chatmaild.expire import (
|
||||
FileEntry,
|
||||
MailboxStat,
|
||||
expire_to_target,
|
||||
get_file_entry,
|
||||
iter_mailboxes,
|
||||
os_listdir_if_exists,
|
||||
parse_dovecot_filename,
|
||||
quota_expire_main,
|
||||
scan_mailbox_messages,
|
||||
)
|
||||
from chatmaild.expire import daily_expire_main as expiry_main
|
||||
from chatmaild.expire import main as expiry_main
|
||||
from chatmaild.fsreport import main as report_main
|
||||
|
||||
MB = 1024 * 1024
|
||||
|
||||
|
||||
def fill_mbox(folderdir):
|
||||
password = folderdir.joinpath("password")
|
||||
@@ -204,51 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
|
||||
tmp_path.joinpath("x").write_text("hello")
|
||||
assert len(os_listdir_if_exists(str(tmp_path))) == 1
|
||||
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
|
||||
|
||||
|
||||
# --- quota expire tests ---
|
||||
|
||||
_msg_counter = itertools.count(1)
|
||||
|
||||
|
||||
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
|
||||
seq = next(_msg_counter)
|
||||
mtime = int(time.time() - days_old * 86400)
|
||||
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
|
||||
path = basedir / sub / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def test_parse_dovecot_filename():
|
||||
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
|
||||
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
||||
assert e.mtime == 1775324677
|
||||
assert e.quota_size == 3235
|
||||
assert parse_dovecot_filename("cur/msg_without_structure") is None
|
||||
|
||||
|
||||
def test_expire_to_target(tmp_path):
|
||||
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
|
||||
_create_message(tmp_path, "new", MB, days_old=5)
|
||||
_create_message(tmp_path, "cur", MB, days_old=0) # undeletable (<1 hour)
|
||||
assert len(scan_mailbox_messages(tmp_path)) == 3
|
||||
# removes oldest first, uses S= size not disk size
|
||||
removed = expire_to_target(tmp_path, MB)
|
||||
assert removed == 2
|
||||
msgs = scan_mailbox_messages(tmp_path)
|
||||
assert len(msgs) == 1
|
||||
# the surviving message is the fresh undeletable one
|
||||
assert msgs[0].mtime > time.time() - 3600
|
||||
|
||||
|
||||
def test_quota_expire_main(tmp_path, capsys):
|
||||
mbox = tmp_path / "user@example.org"
|
||||
_create_message(mbox, "cur", 2 * MB, days_old=5)
|
||||
(mbox / "maildirsize").write_text("x")
|
||||
quota_expire_main([str(1), str(mbox)])
|
||||
_, err = capsys.readouterr()
|
||||
assert "quota-expire: removed 1 message(s) from user@example.org" in err
|
||||
assert not (mbox / "maildirsize").exists()
|
||||
|
||||
@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
|
||||
assert ac1["password"] != ac2["password"]
|
||||
|
||||
|
||||
def test_create_newemail_dict_ip(make_config):
|
||||
config = make_config("1.2.3.4")
|
||||
ac = create_newemail_dict(config)
|
||||
assert ac["email"].endswith("@[1.2.3.4]")
|
||||
|
||||
|
||||
def test_create_dclogin_url():
|
||||
url = create_dclogin_url("user@example.org", "p@ss w+rd")
|
||||
assert url.startswith("dclogin:")
|
||||
|
||||
@@ -101,6 +101,9 @@ def run_cmd(args, out):
|
||||
env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
|
||||
env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
|
||||
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
|
||||
if not args.dns_check_disabled:
|
||||
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
|
||||
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
|
||||
deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
|
||||
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
||||
|
||||
@@ -191,6 +194,12 @@ def status_cmd(args, out):
|
||||
|
||||
|
||||
def test_cmd_options(parser):
|
||||
parser.add_argument(
|
||||
"--slow",
|
||||
dest="slow",
|
||||
action="store_true",
|
||||
help="also run slow tests",
|
||||
)
|
||||
add_ssh_host_option(parser)
|
||||
|
||||
|
||||
@@ -212,6 +221,8 @@ def test_cmd(args, out):
|
||||
"-v",
|
||||
"--durations=5",
|
||||
]
|
||||
if args.slow:
|
||||
pytest_args.append("--slow")
|
||||
ret = out.run_ret(pytest_args, env=env)
|
||||
return ret
|
||||
|
||||
|
||||
@@ -150,6 +150,9 @@ class UnboundDeployer(Deployer):
|
||||
self.need_restart = False
|
||||
|
||||
def install(self):
|
||||
# Run local DNS resolver `unbound`. `resolvconf` takes care of
|
||||
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
|
||||
|
||||
# On an IPv4-only system, if unbound is started but not configured,
|
||||
# it causes subsequent steps to fail to resolve hosts.
|
||||
with blocked_service_startup():
|
||||
@@ -159,31 +162,6 @@ class UnboundDeployer(Deployer):
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
|
||||
apt.packages(
|
||||
name="Purge resolvconf",
|
||||
packages=["resolvconf"],
|
||||
present=False,
|
||||
extra_uninstall_args="--purge",
|
||||
)
|
||||
# systemd-resolved can't be purged due to dependencies; stop and mask.
|
||||
server.shell(
|
||||
name="Stop and mask systemd-resolved",
|
||||
commands=[
|
||||
"systemctl stop systemd-resolved.service || true",
|
||||
"systemctl mask systemd-resolved.service",
|
||||
],
|
||||
)
|
||||
# Configure unbound resolver with Quad9 fallback and a trailing newline
|
||||
# (SolusVM bug).
|
||||
files.put(
|
||||
name="Write static resolv.conf",
|
||||
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
|
||||
dest="/etc/resolv.conf",
|
||||
user="root",
|
||||
group="root",
|
||||
mode="644",
|
||||
)
|
||||
server.shell(
|
||||
name="Generate root keys for validating DNSSEC",
|
||||
commands=[
|
||||
@@ -590,6 +568,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
||||
Deployment().perform_stages([WebsiteDeployer(config)])
|
||||
return
|
||||
|
||||
if host.get_fact(Port, port=53) != "unbound":
|
||||
files.line(
|
||||
name="Add 9.9.9.9 to resolv.conf",
|
||||
path="/etc/resolv.conf",
|
||||
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
|
||||
line="\nnameserver 9.9.9.9",
|
||||
)
|
||||
|
||||
# Check if mtail_address interface is available (if configured)
|
||||
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
||||
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
||||
|
||||
@@ -4,7 +4,7 @@ import urllib.request
|
||||
from chatmaild.config import Config
|
||||
from pyinfra import host
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
||||
from pyinfra.facts.server import Arch, Sysctl
|
||||
from pyinfra.operations import apt, files, server, systemd
|
||||
|
||||
from cmdeploy.basedeploy import (
|
||||
@@ -80,17 +80,6 @@ class DovecotDeployer(Deployer):
|
||||
def activate(self):
|
||||
activate_remote_units(self.units)
|
||||
|
||||
# Detect stale binary: package installed but service still runs old (deleted) binary.
|
||||
if not self.disable_mail and not self.need_restart:
|
||||
stale = host.get_fact(
|
||||
Command,
|
||||
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
|
||||
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
|
||||
" && echo STALE || true",
|
||||
)
|
||||
if stale == "STALE":
|
||||
self.need_restart = True
|
||||
|
||||
restart = False if self.disable_mail else self.need_restart
|
||||
|
||||
systemd.service(
|
||||
@@ -186,7 +175,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
|
||||
can_modify = not is_in_container()
|
||||
for name in ("max_user_instances", "max_user_watches"):
|
||||
key = f"fs.inotify.{name}"
|
||||
value = host.get_fact(Sysctl).get(key, 0)
|
||||
value = host.get_fact(Sysctl)[key]
|
||||
if value > 65534:
|
||||
continue
|
||||
if not can_modify:
|
||||
|
||||
@@ -133,11 +133,6 @@ protocol lmtp {
|
||||
# mail_lua and push_notification_lua are needed for Lua push notification handler.
|
||||
# <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
|
||||
mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
|
||||
|
||||
# Disable fsync for LMTP. May lose delivered message,
|
||||
# but unlikely to cause problems with multiple relays.
|
||||
# https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
|
||||
mail_fsync = never
|
||||
}
|
||||
|
||||
plugin {
|
||||
@@ -149,26 +144,12 @@ plugin {
|
||||
}
|
||||
|
||||
plugin {
|
||||
# for now we define static quota-rules for all users
|
||||
quota = maildir:User quota
|
||||
quota_rule = *:storage={{ config.max_mailbox_size }}
|
||||
quota_max_mail_size={{ config.max_message_size }}
|
||||
quota_grace = 0
|
||||
|
||||
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
|
||||
|
||||
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
|
||||
# The percentages are chosen to prevent current Delta Chat users
|
||||
# from seeing "quota warnings" which trigger at 80% and 95%.
|
||||
|
||||
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
|
||||
}
|
||||
|
||||
service quota-warning {
|
||||
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
|
||||
user = vmail
|
||||
unix_listener quota-warning {
|
||||
user = vmail
|
||||
mode = 0600
|
||||
}
|
||||
# quota_over_flag_value = TRUE
|
||||
}
|
||||
|
||||
# push_notification configuration
|
||||
@@ -271,9 +252,6 @@ protocol imap {
|
||||
# sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
|
||||
|
||||
rawlog_dir = %h
|
||||
|
||||
# Disable fsync for IMAP. May lose IMAP changes like setting flags.
|
||||
mail_fsync = never
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import os
|
||||
|
||||
from pyinfra import facts, host
|
||||
from pyinfra.operations import files, systemd
|
||||
|
||||
@@ -7,7 +5,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
|
||||
|
||||
|
||||
class FiltermailDeployer(Deployer):
|
||||
services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
|
||||
services = ["filtermail", "filtermail-incoming"]
|
||||
bin_path = "/usr/local/bin/filtermail"
|
||||
config_path = "/usr/local/lib/chatmaild/chatmail.ini"
|
||||
|
||||
@@ -15,21 +13,11 @@ class FiltermailDeployer(Deployer):
|
||||
self.need_restart = False
|
||||
|
||||
def install(self):
|
||||
local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
|
||||
if local_bin:
|
||||
self.need_restart |= files.put(
|
||||
name="Upload locally built filtermail",
|
||||
src=local_bin,
|
||||
dest=self.bin_path,
|
||||
mode="755",
|
||||
).changed
|
||||
return
|
||||
|
||||
arch = host.get_fact(facts.server.Arch)
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
|
||||
sha256sum = {
|
||||
"x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
|
||||
"aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
|
||||
"x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
|
||||
"aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
|
||||
}[arch]
|
||||
self.need_restart |= files.download(
|
||||
name="Download filtermail",
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
[Unit]
|
||||
Description=Chatmail transport service
|
||||
|
||||
[Service]
|
||||
ExecStart={{ bin_path }} {{ config_path }} transport
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
User=vmail
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -78,11 +78,3 @@ counter rejected_unencrypted_mail_count
|
||||
/Rejected unencrypted mail/ {
|
||||
rejected_unencrypted_mail_count++
|
||||
}
|
||||
|
||||
counter quota_expire_runs
|
||||
counter quota_expire_removed_files
|
||||
|
||||
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
|
||||
quota_expire_runs++
|
||||
quota_expire_removed_files += $count
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
[Unit]
|
||||
Description=mtail
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
@@ -73,8 +73,8 @@ http {
|
||||
|
||||
access_log syslog:server=unix:/dev/log,facility=local7;
|
||||
|
||||
location /mxdeliv {
|
||||
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
|
||||
location /mxdeliv/ {
|
||||
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port }};
|
||||
}
|
||||
|
||||
location / {
|
||||
|
||||
@@ -69,6 +69,15 @@ mynetworks = 127.0.0.0/8
|
||||
{% else %}
|
||||
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
|
||||
{% endif %}
|
||||
{% if config.addr_v4 %}
|
||||
smtp_bind_address = {{ config.addr_v4 }}
|
||||
{% endif %}
|
||||
{% if config.addr_v6 %}
|
||||
smtp_bind_address6 = {{ config.addr_v6 }}
|
||||
{% endif %}
|
||||
{% if config.addr_v4 or config.addr_v6 %}
|
||||
smtp_bind_address_enforce = yes
|
||||
{% endif %}
|
||||
mailbox_size_limit = 0
|
||||
message_size_limit = {{config.max_message_size}}
|
||||
recipient_delimiter = +
|
||||
@@ -79,6 +88,22 @@ inet_protocols = ipv4
|
||||
inet_protocols = all
|
||||
{% endif %}
|
||||
|
||||
# Postfix does not try IPv4 and IPv6 connections
|
||||
# concurrently as of version 3.7.11.
|
||||
#
|
||||
# When relay has both A (IPv4) and AAAA (IPv6) records,
|
||||
# but broken IPv6 connectivity,
|
||||
# every second message is delayed by the connection timeout
|
||||
# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
|
||||
# which defaults to 30 seconds. Reducing timeouts is not a solution
|
||||
# as this will result in a failure to connect to slow servers.
|
||||
#
|
||||
# As a workaround we always prefer IPv4 when it is available.
|
||||
#
|
||||
# The setting is documented at
|
||||
# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
|
||||
smtp_address_preference=ipv4
|
||||
|
||||
virtual_transport = lmtp:unix:private/dovecot-lmtp
|
||||
virtual_mailbox_domains = {{ config.mail_domain }}
|
||||
lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
|
||||
@@ -93,10 +118,3 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
|
||||
# Do not lookup SMTP client hostnames to reduce delays
|
||||
# and avoid unnecessary DNS requests.
|
||||
smtpd_peername_lookup = no
|
||||
|
||||
# Use filtermail-transport to relay messages.
|
||||
# We can't force postfix to split messages per destination,
|
||||
# when specifying a custom next-hop,
|
||||
# so instead this is handled in filtermail.
|
||||
# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
|
||||
default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}
|
||||
|
||||
@@ -100,8 +100,3 @@ filter unix - n n - - lmtp
|
||||
# cannot send unprotected Subject.
|
||||
authclean unix n - - - 0 cleanup
|
||||
-o header_checks=regexp:/etc/postfix/submission_header_cleanup
|
||||
|
||||
lmtp-filtermail unix - - y - - lmtp
|
||||
-o syslog_name=postfix/lmtp-filtermail
|
||||
-o lmtp_header_checks=
|
||||
-o lmtp_tls_security_level=none
|
||||
|
||||
@@ -221,6 +221,7 @@ def test_rewrite_subject(cmsetup, maildata):
|
||||
assert "Subject: Unencrypted subject" not in rcvd_msg
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||
"""Test that the per-account send-mail limit is exceeded."""
|
||||
user1, user2 = cmsetup.gen_users(2)
|
||||
@@ -243,6 +244,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||
pytest.fail("Rate limit was not exceeded")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_expunged(remote, chatmail_config):
|
||||
outdated_days = int(chatmail_config.delete_mails_after) + 1
|
||||
find_cmds = [
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import imaplib
|
||||
import ipaddress
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
@@ -15,12 +14,10 @@ from chatmaild.config import read_config
|
||||
conftestdir = Path(__file__).parent
|
||||
|
||||
|
||||
def _is_ip(domain):
|
||||
try:
|
||||
ipaddress.ip_address(domain)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--slow", action="store_true", default=False, help="also run slow tests"
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
@@ -30,6 +27,13 @@ def pytest_configure(config):
|
||||
)
|
||||
|
||||
|
||||
def pytest_runtest_setup(item):
|
||||
markers = list(item.iter_markers(name="slow"))
|
||||
if markers:
|
||||
if not item.config.getoption("--slow"):
|
||||
pytest.skip("skipping slow test, use --slow to run")
|
||||
|
||||
|
||||
def _get_chatmail_config():
|
||||
inipath = os.environ.get("CHATMAIL_INI")
|
||||
if inipath:
|
||||
@@ -278,7 +282,6 @@ def gencreds(chatmail_config):
|
||||
|
||||
def gen(domain=None):
|
||||
domain = domain if domain else chatmail_config.mail_domain
|
||||
addr_domain = f"[{domain}]" if _is_ip(domain) else domain
|
||||
while 1:
|
||||
num = next(count)
|
||||
alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
|
||||
@@ -292,7 +295,7 @@ def gencreds(chatmail_config):
|
||||
password = "".join(
|
||||
random.choices(alphanumeric, k=chatmail_config.password_min_length)
|
||||
)
|
||||
yield f"{user}@{addr_domain}", f"{password}"
|
||||
yield f"{user}@{domain}", f"{password}"
|
||||
|
||||
return lambda domain=None: next(gen(domain))
|
||||
|
||||
@@ -341,22 +344,9 @@ class ChatmailACFactory:
|
||||
accounts = []
|
||||
for _ in range(num):
|
||||
account = self.dc.add_account()
|
||||
addr, password = self.gencreds(domain)
|
||||
if _is_ip(domain):
|
||||
# Use DCLOGIN scheme with explicit server hosts,
|
||||
# matching how madmail presents its addresses to users.
|
||||
qr = (
|
||||
f"dclogin:{addr}"
|
||||
f"?p={password}&v=1"
|
||||
f"&ih={domain}&ip=993"
|
||||
f"&sh={domain}&sp=465"
|
||||
f"&ic=3&ss=default"
|
||||
)
|
||||
future = account.add_transport_from_qr.future(qr)
|
||||
else:
|
||||
future = account.add_or_update_transport.future(
|
||||
self._make_transport(domain)
|
||||
)
|
||||
future = account.add_or_update_transport.future(
|
||||
self._make_transport(domain)
|
||||
)
|
||||
futures.append(future)
|
||||
|
||||
# ensure messages stay in INBOX so that they can be
|
||||
|
||||
@@ -2,9 +2,9 @@ from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
|
||||
from cmdeploy.dovecot import deployer as dovecot_deployer
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
|
||||
|
||||
def make_host(*fact_pairs):
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
from pathlib import Path
|
||||
import importlib.resources
|
||||
|
||||
from cmdeploy.www import build_webpages
|
||||
|
||||
|
||||
def test_build_webpages(tmp_path, make_config):
|
||||
src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
|
||||
pkgroot = importlib.resources.files("cmdeploy")
|
||||
src_dir = pkgroot.joinpath("../../../www/src").resolve()
|
||||
assert src_dir.exists(), src_dir
|
||||
config = make_config("chat.example.org")
|
||||
build_dir = tmp_path.joinpath("build")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import hashlib
|
||||
import importlib.resources
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
@@ -36,7 +37,7 @@ def prepare_template(source):
|
||||
|
||||
|
||||
def get_paths(config) -> (Path, Path, Path):
|
||||
reporoot = (Path(__file__).resolve() / "../../../../").resolve()
|
||||
reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
|
||||
www_path = Path(config.www_folder)
|
||||
# if www_folder was not set, use default directory
|
||||
if config.www_folder == "":
|
||||
@@ -132,7 +133,8 @@ def find_merge_conflict(src_dir) -> Path:
|
||||
|
||||
|
||||
def main():
|
||||
reporoot = (Path(__file__).resolve() / "../../../../").resolve()
|
||||
path = importlib.resources.files(__package__)
|
||||
reporoot = path.joinpath("../../../").resolve()
|
||||
inipath = reporoot.joinpath("chatmail.ini")
|
||||
config = read_config(inipath)
|
||||
config.webdev = True
|
||||
|
||||
@@ -4,14 +4,12 @@
|
||||
|
||||
You can use the `make` command and `make html` to build web pages.
|
||||
|
||||
You need a Python environment with `sphinx` and other
|
||||
dependencies, you can create it by running `scripts/initenv.sh`
|
||||
from the repository root.
|
||||
You need a Python environment where the following install was excuted:
|
||||
|
||||
pip install furo sphinx-autobuild
|
||||
|
||||
To develop/change documentation, you can then do:
|
||||
|
||||
. venv/bin/activate
|
||||
cd doc
|
||||
make auto
|
||||
|
||||
A page will open at https://127.0.0.1:8000/ serving the docs and it will
|
||||
|
||||
@@ -98,6 +98,13 @@ steps. Please substitute it with your own domain.
|
||||
configure at your DNS provider (it can take some time until they are
|
||||
public).
|
||||
|
||||
Docker installation
|
||||
-------------------
|
||||
|
||||
There is experimental support for running chatmail via Docker Compose.
|
||||
See the `chatmail/docker README <https://github.com/chatmail/docker>`_
|
||||
for full setup instructions.
|
||||
|
||||
Other helpful commands
|
||||
----------------------
|
||||
|
||||
|
||||
@@ -16,6 +16,5 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
|
||||
proxy
|
||||
migrate
|
||||
overview
|
||||
reverse_dns
|
||||
related
|
||||
faq
|
||||
|
||||
@@ -102,12 +102,8 @@ short overview of ``chatmaild`` services:
|
||||
Apple/Google/Huawei.
|
||||
|
||||
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
|
||||
deletes old messages, large messages, and entire mailboxes
|
||||
of users who have not logged in for longer than
|
||||
``delete_inactive_users_after`` days.
|
||||
|
||||
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
|
||||
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
|
||||
deletes users if they have not logged in for a longer while.
|
||||
The timeframe can be configured in ``chatmail.ini``.
|
||||
|
||||
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
||||
is contacted by Dovecot when a user logs in and stores the date of
|
||||
@@ -153,7 +149,6 @@ Chatmail relay dependency diagram
|
||||
autoconfig.xml --- dovecot;
|
||||
postfix --- |10080|filtermail-outgoing;
|
||||
postfix --- |10081|filtermail-incoming;
|
||||
postfix --- |10083|filtermail-transport;
|
||||
filtermail-outgoing --- |10025 reinject|postfix;
|
||||
filtermail-incoming --- |10026 reinject|postfix;
|
||||
dovecot --- |doveauth.socket|doveauth;
|
||||
@@ -161,8 +156,6 @@ Chatmail relay dependency diagram
|
||||
/home/vmail/.../user"];
|
||||
dovecot --- |lastlogin.socket|lastlogin;
|
||||
dovecot --- chatmail-metadata;
|
||||
dovecot --- |quota-warning|chatmail-quota-expire;
|
||||
chatmail-quota-expire --- maildir;
|
||||
lastlogin --- maildir;
|
||||
doveauth --- maildir;
|
||||
chatmail-expire-daily --- maildir;
|
||||
@@ -296,7 +289,9 @@ ensured by ``filtermail`` proxy.
|
||||
TLS requirements
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Filtermail (used for delivery) requires a valid TLS.
|
||||
Postfix is configured to require valid TLS by setting
|
||||
`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
|
||||
to ``verify``.
|
||||
|
||||
You can test it by resolving ``MX`` records of your relay domain and
|
||||
then connecting to MX relays (e.g ``mx.example.org``) with
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
Configuring reverse DNS
|
||||
=======================
|
||||
|
||||
Some email servers reject the emails
|
||||
if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
|
||||
|
||||
.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
|
||||
.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
|
||||
|
||||
Passing the check requires that the IP address that email is sent from
|
||||
should have a ``PTR`` record pointing to the domain name of the server,
|
||||
and domain name record should have an ``A/AAAA`` record
|
||||
pointing to the IP address.
|
||||
|
||||
Modern email relies on DKIM and SPF for authentication,
|
||||
while iprev check exists for
|
||||
`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
|
||||
Chatmail relays don't resolve ``PTR`` records,
|
||||
so you can ignore this section if configuring ``PTR`` records
|
||||
is difficult and federation with legacy email servers that don't accept
|
||||
valid DKIM signature for authentication is not important.
|
||||
|
||||
Multi-homed setups
|
||||
------------------
|
||||
|
||||
If you have a server with multiple IP addresses,
|
||||
also known as multi-homed setup,
|
||||
and don't publish all IP addresses in DNS,
|
||||
you need to make sure you are using
|
||||
the published address when making outgoing connections.
|
||||
|
||||
For example, your server may have a static IP
|
||||
address, and a so-called Floating IP or Virtual IP
|
||||
that can be moved between servers in case of
|
||||
migration or for failover.
|
||||
By using Floating IP you can avoid downtime
|
||||
and keep the IP address reputation
|
||||
for destinatinons that rely on IP reputation and IP blocklists.
|
||||
In this case you will only publish
|
||||
the Floating IP to DNS and only use the static IP
|
||||
to SSH into the server.
|
||||
|
||||
If you have such setup, make sure that
|
||||
you not only set ``PTR`` records for the Floating IP,
|
||||
but make outgoing connections using the Floating IP.
|
||||
Otherwise reverse DNS check succeed,
|
||||
but forward check making sure your domain name points
|
||||
to the IP address will fail.
|
||||
Such setup is indistinguishable from someone
|
||||
setting IP address ``PTR`` with the domain they don't own
|
||||
and as a result don't succeed.
|
||||
|
||||
On Linux you can configure source IP address with ``ip route`` command,
|
||||
for example:
|
||||
::
|
||||
|
||||
ip route change default via <default-gateway> dev eth0 src <source-address>
|
||||
|
||||
Make sure to persist the change after verifying it is working.
|
||||
You can check what your outgoing IP address is
|
||||
with ``curl icanhazip.com``.
|
||||
Check both the IPv4 and IPv6 addresses.
|
||||
For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
|
||||
and similarly for IPv6 if you have it.
|
||||
Reference in New Issue
Block a user