mirror of
https://github.com/chatmail/relay.git
synced 2026-05-11 08:24:37 +00:00
Compare commits
1 Commits
three-tier
...
j4n/docker
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0808c9dc47 |
43
.github/workflows/ci.yaml
vendored
43
.github/workflows/ci.yaml
vendored
@@ -1,26 +1,15 @@
|
||||
name: Run unit-tests and container-based deploy+test verification
|
||||
name: CI
|
||||
|
||||
on:
|
||||
# Triggers when a PR is merged into main or a direct push occurs
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
|
||||
# Triggers for any PR (and its subsequent commits) targeting the main branch
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
push:
|
||||
|
||||
jobs:
|
||||
tox:
|
||||
name: isolated chatmaild tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v4
|
||||
# Checkout pull request HEAD commit instead of merge commit
|
||||
# Otherwise `test_deployed_state` will be unhappy.
|
||||
with:
|
||||
@@ -35,9 +24,7 @@ jobs:
|
||||
name: deploy-chatmail tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
@@ -51,23 +38,5 @@ jobs:
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
lxc-test:
|
||||
name: LXC deploy and test
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
|
||||
with:
|
||||
cmlxc_commands: |
|
||||
cmlxc init
|
||||
# single cmdeploy relay test
|
||||
cmlxc -v deploy-cmdeploy --source ./repo cm0
|
||||
cmlxc -v test-mini cm0
|
||||
cmlxc -v test-cmdeploy cm0
|
||||
|
||||
# cross cmdeploy relay test
|
||||
cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
|
||||
cmlxc -v test-cmdeploy cm0 cm1
|
||||
|
||||
# cross cmdeploy/madmail relay tests
|
||||
cmlxc -v deploy-madmail mad0
|
||||
cmlxc -v test-cmdeploy cm0 mad0
|
||||
cmlxc -v test-mini cm0 mad0
|
||||
cmlxc -v test-mini mad0 cm0
|
||||
# all other cmdeploy commands require a staging server
|
||||
# see https://github.com/deltachat/chatmail/issues/100
|
||||
|
||||
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
@@ -0,0 +1,125 @@
|
||||
name: Docker deploy
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
staging_host:
|
||||
required: true
|
||||
type: string
|
||||
description: 'SSH hostname (e.g. staging2.testrun.org)'
|
||||
mail_domain:
|
||||
required: true
|
||||
type: string
|
||||
description: 'MAIL_DOMAIN for docker compose'
|
||||
zone_file:
|
||||
required: true
|
||||
type: string
|
||||
description: 'Default zone file basename (e.g. staging.testrun.org-default.zone)'
|
||||
|
||||
jobs:
|
||||
deploy-docker:
|
||||
name: Docker deploy on ${{ inputs.staging_host }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
environment:
|
||||
name: ${{ inputs.staging_host }}
|
||||
url: https://${{ inputs.staging_host }}/
|
||||
concurrency: ${{ inputs.staging_host }}
|
||||
env:
|
||||
VPS: root@${{ inputs.staging_host }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- name: Setup SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan ${{ inputs.staging_host }} > ~/.ssh/known_hosts
|
||||
# Reuse TCP connection for all subsequent ssh/scp calls
|
||||
echo -e "Host ${{ inputs.staging_host }}\n ControlMaster auto\n ControlPath ~/.ssh/ctrl-%r@%h:%p\n ControlPersist 10m" >> ~/.ssh/config
|
||||
|
||||
- name: stop bare services, install Docker, prepare mounts
|
||||
run: |
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list
|
||||
apt-get update
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||
mkdir -p /srv/chatmail/certs /srv/chatmail/dkim
|
||||
cp -a /var/lib/acme/. /srv/chatmail/certs/ || true
|
||||
cp -a /etc/dkimkeys/. /srv/chatmail/dkim/ || true
|
||||
cp /etc/chatmail/chatmail.ini /srv/chatmail/chatmail.ini
|
||||
EOF
|
||||
|
||||
- name: deploy with Docker
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
GHCR_IMAGE="ghcr.io/chatmail/docker:sha-${SHORT_SHA}"
|
||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ $VPS:/srv/chatmail/relay/
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh $VPS "docker login ghcr.io -u ${{ github.actor }} --password-stdin && \
|
||||
docker pull ${GHCR_IMAGE} && \
|
||||
cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=${{ inputs.mail_domain }} docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
||||
|
||||
- name: wait for container healthy
|
||||
run: |
|
||||
ssh $VPS 'docker exec chatmail journalctl -f --no-pager' &
|
||||
LOG_PID=$!
|
||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
||||
for i in $(seq 1 60); do
|
||||
status=$(ssh $VPS 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
||||
echo " [$i/60] status=$status"
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "Container is healthy."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$status" = "unhealthy" ]; then
|
||||
echo "Container is unhealthy!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Container did not become healthy."
|
||||
kill $LOG_PID 2>/dev/null || true
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
echo "--- failed units ---"
|
||||
docker exec chatmail systemctl --failed --no-pager || true
|
||||
echo "--- service logs ---"
|
||||
docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50 || true
|
||||
echo "--- listening ports ---"
|
||||
docker exec chatmail ss -tlnp || true
|
||||
echo "--- chatmail.ini ---"
|
||||
docker exec chatmail cat /etc/chatmail/chatmail.ini || true
|
||||
EOF
|
||||
exit 1
|
||||
|
||||
- name: show container state
|
||||
run: |
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
echo "--- listening ports ---"
|
||||
docker exec chatmail ss -tlnp
|
||||
echo "--- chatmail.ini ---"
|
||||
docker exec chatmail cat /etc/chatmail/chatmail.ini
|
||||
EOF
|
||||
|
||||
- name: Docker integration tests
|
||||
run: ssh $VPS 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
||||
|
||||
- name: Docker DNS
|
||||
run: |
|
||||
git checkout .github/workflows/${{ inputs.zone_file }}
|
||||
ssh $VPS bash -s <<'EOF'
|
||||
docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys
|
||||
docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose
|
||||
docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone
|
||||
EOF
|
||||
scp $VPS:/tmp/staging.zone staging-generated.zone
|
||||
cat staging-generated.zone >> .github/workflows/${{ inputs.zone_file }}
|
||||
cat .github/workflows/${{ inputs.zone_file }}
|
||||
scp .github/workflows/${{ inputs.zone_file }} root@ns.testrun.org:/etc/nsd/${{ inputs.staging_host }}.zone
|
||||
ssh root@ns.testrun.org "nsd-checkzone ${{ inputs.staging_host }} /etc/nsd/${{ inputs.staging_host }}.zone && systemctl reload nsd"
|
||||
|
||||
- name: Docker final DNS check
|
||||
run: ssh $VPS 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
||||
121
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
Normal file
121
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
Normal file
@@ -0,0 +1,121 @@
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||
secrets: inherit
|
||||
|
||||
deploy:
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging-ipv4.testrun.org
|
||||
url: https://staging-ipv4.testrun.org/
|
||||
concurrency: staging-ipv4.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
|
||||
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
run: |
|
||||
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging-ipv4.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- name: setup dependencies
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org apt update
|
||||
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
|
||||
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
|
||||
|
||||
- name: initialize config
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
|
||||
|
||||
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
|
||||
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
||||
|
||||
deploy-docker:
|
||||
needs: [deploy, trigger-docker-build]
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/docker-deploy.yaml
|
||||
with:
|
||||
staging_host: staging-ipv4.testrun.org
|
||||
mail_domain: staging-ipv4.testrun.org
|
||||
zone_file: staging-ipv4.testrun.org-default.zone
|
||||
secrets: inherit
|
||||
112
.github/workflows/test-and-deploy.yaml
vendored
Normal file
112
.github/workflows/test-and-deploy.yaml
vendored
Normal file
@@ -0,0 +1,112 @@
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||
secrets: inherit
|
||||
|
||||
deploy:
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging2.testrun.org
|
||||
url: https://staging2.testrun.org/
|
||||
concurrency: staging2.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
|
||||
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging2.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
run: |
|
||||
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging2.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: add hpk42 key to staging server
|
||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- run: |
|
||||
cmdeploy init staging2.testrun.org
|
||||
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
|
||||
|
||||
- run: cmdeploy run --verbose --skip-dns-check
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
cmdeploy dns --zonefile staging-generated.zone --verbose
|
||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
||||
cat .github/workflows/staging.testrun.org-default.zone
|
||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: cmdeploy dns -v
|
||||
|
||||
deploy-docker:
|
||||
needs: [deploy, trigger-docker-build]
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/docker-deploy.yaml
|
||||
with:
|
||||
staging_host: staging2.testrun.org
|
||||
mail_domain: staging2.testrun.org
|
||||
zone_file: staging.testrun.org-default.zone
|
||||
secrets: inherit
|
||||
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
name: Trigger Docker image build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
trigger-docker-build:
|
||||
name: Trigger Docker image build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
|
||||
script: |
|
||||
await github.rest.repos.createDispatchEvent({
|
||||
owner: 'chatmail',
|
||||
repo: 'docker',
|
||||
event_type: 'relay-updated',
|
||||
client_payload: {
|
||||
relay_ref: context.ref,
|
||||
relay_sha: context.sha,
|
||||
relay_sha_short: context.sha.slice(0, 7)
|
||||
}
|
||||
})
|
||||
@@ -21,8 +21,7 @@ where = ['src']
|
||||
[project.scripts]
|
||||
doveauth = "chatmaild.doveauth:main"
|
||||
chatmail-metadata = "chatmaild.metadata:main"
|
||||
chatmail-expire = "chatmaild.expire:daily_expire_main"
|
||||
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
|
||||
chatmail-expire = "chatmaild.expire:main"
|
||||
chatmail-fsreport = "chatmaild.fsreport:main"
|
||||
lastlogin = "chatmaild.lastlogin:main"
|
||||
turnserver = "chatmaild.turnserver:main"
|
||||
|
||||
@@ -38,9 +38,7 @@ class Config:
|
||||
self.filtermail_smtp_port_incoming = int(
|
||||
params.get("filtermail_smtp_port_incoming", "10081")
|
||||
)
|
||||
self.filtermail_http_port_incoming = int(
|
||||
params.get("filtermail_http_port_incoming", "10082")
|
||||
)
|
||||
self.filtermail_http_port = int(params.get("filtermail_http_port", "10082"))
|
||||
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
|
||||
self.postfix_reinject_port_incoming = int(
|
||||
params.get("postfix_reinject_port_incoming", "10026")
|
||||
@@ -95,11 +93,6 @@ class Config:
|
||||
# old unused option (except for first migration from sqlite to maildir store)
|
||||
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
|
||||
|
||||
@property
|
||||
def max_mailbox_size_mb(self):
|
||||
"""Return max_mailbox_size as an integer in megabytes."""
|
||||
return parse_size_mb(self.max_mailbox_size)
|
||||
|
||||
def _getbytefile(self):
|
||||
return open(self._inipath, "rb")
|
||||
|
||||
@@ -113,16 +106,6 @@ class Config:
|
||||
return User(maildir, addr, password_path, uid="vmail", gid="vmail")
|
||||
|
||||
|
||||
def parse_size_mb(limit):
|
||||
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
|
||||
value = limit.strip().upper().removesuffix("B")
|
||||
if value.endswith("G"):
|
||||
return int(value[:-1]) * 1024
|
||||
if value.endswith("M"):
|
||||
return int(value[:-1])
|
||||
return int(value)
|
||||
|
||||
|
||||
def write_initial_config(inipath, mail_domain, overrides):
|
||||
"""Write out default config file, using the specified config value overrides."""
|
||||
content = get_default_config_content(mail_domain, **overrides)
|
||||
|
||||
@@ -4,26 +4,17 @@ Expire old messages and addresses.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from argparse import ArgumentParser
|
||||
from collections import namedtuple
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from stat import S_ISREG
|
||||
|
||||
from chatmaild.config import read_config
|
||||
|
||||
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
||||
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
|
||||
|
||||
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
|
||||
QUOTA_CLEANUP_FACTOR = 0.7
|
||||
|
||||
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
||||
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
|
||||
|
||||
|
||||
def iter_mailboxes(basedir, maxnum):
|
||||
@@ -83,54 +74,6 @@ class MailboxStat:
|
||||
self.extrafiles.sort(key=lambda x: -x.size)
|
||||
|
||||
|
||||
def parse_dovecot_filename(relpath):
|
||||
m = _dovecot_fn_rex.match(relpath)
|
||||
if not m:
|
||||
return None
|
||||
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
|
||||
|
||||
|
||||
def scan_mailbox_messages(mbox):
|
||||
messages = []
|
||||
for sub in ("cur", "new"):
|
||||
for name in os_listdir_if_exists(mbox / sub):
|
||||
if entry := parse_dovecot_filename(f"{sub}/{name}"):
|
||||
messages.append(entry)
|
||||
return messages
|
||||
|
||||
|
||||
# Within this window, large messages are deleted before small ones.
|
||||
DELETE_LARGE_FIRST_DAYS = 7
|
||||
|
||||
|
||||
def expire_to_target(mbox, target_bytes):
|
||||
cutoff = time.time() - DELETE_LARGE_FIRST_DAYS * 86400
|
||||
messages = scan_mailbox_messages(mbox)
|
||||
|
||||
def sort_key(msg):
|
||||
# prio 0: Older than cutoff -> remove oldest first
|
||||
if msg.mtime < cutoff:
|
||||
return (0, msg.mtime)
|
||||
|
||||
# prio 1: more recent than cutoff, large -> remove largest first
|
||||
if msg.quota_size > 200000:
|
||||
return (1, -msg.quota_size, msg.mtime)
|
||||
|
||||
# prio 2: more recent than cutoff, small -> remove oldest first
|
||||
return (2, msg.mtime)
|
||||
|
||||
total_size = sum(m.quota_size for m in messages)
|
||||
removed = 0
|
||||
for entry in sorted(messages, key=sort_key):
|
||||
if total_size <= target_bytes:
|
||||
break
|
||||
(mbox / entry.path).unlink(missing_ok=True)
|
||||
total_size -= entry.quota_size
|
||||
removed += 1
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def print_info(msg):
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
@@ -200,19 +143,6 @@ class Expiry:
|
||||
else:
|
||||
continue
|
||||
changed = True
|
||||
|
||||
target_bytes = (
|
||||
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
|
||||
)
|
||||
removed = expire_to_target(Path(mbox.basedir), target_bytes)
|
||||
if removed:
|
||||
changed = True
|
||||
self.del_files += removed
|
||||
if self.verbose:
|
||||
print_info(
|
||||
f"quota-expire: removed {removed} message(s) from {mboxname}"
|
||||
)
|
||||
|
||||
if changed:
|
||||
self.remove_file(f"{mbox.basedir}/maildirsize")
|
||||
|
||||
@@ -224,9 +154,9 @@ class Expiry:
|
||||
)
|
||||
|
||||
|
||||
def daily_expire_main(args=None):
|
||||
def main(args=None):
|
||||
"""Expire mailboxes and messages according to chatmail config"""
|
||||
parser = ArgumentParser(description=daily_expire_main.__doc__)
|
||||
parser = ArgumentParser(description=main.__doc__)
|
||||
ini = "/usr/local/lib/chatmaild/chatmail.ini"
|
||||
parser.add_argument(
|
||||
"chatmail_ini",
|
||||
@@ -272,33 +202,5 @@ def daily_expire_main(args=None):
|
||||
print(exp.get_summary())
|
||||
|
||||
|
||||
def quota_expire_main(args=None):
|
||||
"""Remove mailbox messages to stay within a megabyte target.
|
||||
|
||||
This entry point is called by dovecot when a quota threshold is passed.
|
||||
"""
|
||||
|
||||
parser = ArgumentParser(description=quota_expire_main.__doc__)
|
||||
parser.add_argument(
|
||||
"target_mb",
|
||||
type=int,
|
||||
help="target mailbox size in megabytes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"mailbox_path",
|
||||
type=Path,
|
||||
help="path to a user mailbox",
|
||||
)
|
||||
args = parser.parse_args(args)
|
||||
|
||||
target_bytes = args.target_mb * 1024 * 1024
|
||||
|
||||
removed_count = expire_to_target(args.mailbox_path, target_bytes)
|
||||
if removed_count:
|
||||
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
|
||||
print(
|
||||
f"quota-expire: removed {removed_count} message(s)"
|
||||
f" from {args.mailbox_path.name}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
|
||||
@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
|
||||
max_user_send_burst_size = 10
|
||||
|
||||
# maximum mailbox size of a chatmail address
|
||||
# Oldest messages will be removed automatically, so mailboxes never run full.
|
||||
max_mailbox_size = 500M
|
||||
|
||||
# maximum message size for an e-mail in bytes
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
"""CGI script for creating new accounts."""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import secrets
|
||||
import string
|
||||
@@ -15,16 +14,6 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
|
||||
ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
|
||||
|
||||
|
||||
def wrap_ip(host):
|
||||
if host.startswith("[") and host.endswith("]"):
|
||||
return host
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
return f"[{host}]"
|
||||
except ValueError:
|
||||
return host
|
||||
|
||||
|
||||
def create_newemail_dict(config: Config):
|
||||
user = "".join(
|
||||
secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
|
||||
@@ -33,7 +22,7 @@ def create_newemail_dict(config: Config):
|
||||
secrets.choice(ALPHANUMERIC_PUNCT)
|
||||
for _ in range(config.password_min_length + 3)
|
||||
)
|
||||
return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
|
||||
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
|
||||
|
||||
|
||||
def create_dclogin_url(email, password):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from chatmaild.config import parse_size_mb, read_config
|
||||
from chatmaild.config import read_config
|
||||
|
||||
|
||||
def test_read_config_basic(example_config):
|
||||
@@ -121,17 +121,3 @@ def test_config_tls_external_bad_format(make_config):
|
||||
"tls_external_cert_and_key": "/only/one/path.pem",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_parse_size_mb():
|
||||
assert parse_size_mb("500M") == 500
|
||||
assert parse_size_mb("2G") == 2048
|
||||
assert parse_size_mb(" 1g ") == 1024
|
||||
assert parse_size_mb("100MB") == 100
|
||||
assert parse_size_mb("256") == 256
|
||||
|
||||
|
||||
def test_max_mailbox_size_mb(make_config):
|
||||
config = make_config("chat.example.org")
|
||||
assert config.max_mailbox_size == "500M"
|
||||
assert config.max_mailbox_size_mb == 500
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import time
|
||||
|
||||
from chatmaild.doveauth import AuthDictProxy
|
||||
from chatmaild.expire import daily_expire_main as main_expire
|
||||
from chatmaild.expire import main as main_expire
|
||||
|
||||
|
||||
def test_login_timestamps(example_config):
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from fnmatch import fnmatch
|
||||
from pathlib import Path
|
||||
@@ -12,19 +9,13 @@ import pytest
|
||||
from chatmaild.expire import (
|
||||
FileEntry,
|
||||
MailboxStat,
|
||||
expire_to_target,
|
||||
get_file_entry,
|
||||
iter_mailboxes,
|
||||
os_listdir_if_exists,
|
||||
parse_dovecot_filename,
|
||||
quota_expire_main,
|
||||
scan_mailbox_messages,
|
||||
)
|
||||
from chatmaild.expire import daily_expire_main as expiry_main
|
||||
from chatmaild.expire import main as expiry_main
|
||||
from chatmaild.fsreport import main as report_main
|
||||
|
||||
MB = 1024 * 1024
|
||||
|
||||
|
||||
def fill_mbox(folderdir):
|
||||
password = folderdir.joinpath("password")
|
||||
@@ -205,74 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
|
||||
tmp_path.joinpath("x").write_text("hello")
|
||||
assert len(os_listdir_if_exists(str(tmp_path))) == 1
|
||||
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
|
||||
|
||||
|
||||
# --- quota expire tests ---
|
||||
|
||||
_msg_counter = itertools.count(1)
|
||||
|
||||
|
||||
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
|
||||
seq = next(_msg_counter)
|
||||
mtime = int(time.time() - days_old * 86400)
|
||||
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
|
||||
path = basedir / sub / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def test_parse_dovecot_filename():
|
||||
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
|
||||
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
||||
assert e.mtime == 1775324677
|
||||
assert e.quota_size == 3235
|
||||
assert parse_dovecot_filename("cur/msg_without_structure") is None
|
||||
|
||||
|
||||
def test_expire_to_target(tmp_path):
|
||||
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
|
||||
_create_message(tmp_path, "new", MB, days_old=5)
|
||||
assert len(scan_mailbox_messages(tmp_path)) == 2
|
||||
# removes oldest first, uses S= size not disk size
|
||||
removed = expire_to_target(tmp_path, MB)
|
||||
assert removed == 1
|
||||
assert len(scan_mailbox_messages(tmp_path)) == 1
|
||||
|
||||
|
||||
def test_expire_to_target_prioritization(tmp_path):
|
||||
def create_messages():
|
||||
for sub in ("cur", "new"):
|
||||
if (tmp_path / sub).exists():
|
||||
shutil.rmtree(tmp_path / sub)
|
||||
# prio 0: older than 7 days
|
||||
_create_message(tmp_path, "cur", 5 * MB, days_old=10)
|
||||
# prio 1: last 7 days, large (>200KB)
|
||||
_create_message(tmp_path, "cur", 5 * MB, days_old=1)
|
||||
# prio 2: last 7 days, small
|
||||
_create_message(tmp_path, "cur", 1000, days_old=2)
|
||||
|
||||
# Shrink to 6MB: only the old message (prio 0) is removed.
|
||||
create_messages()
|
||||
assert expire_to_target(tmp_path, 6 * MB) == 1
|
||||
msgs = scan_mailbox_messages(tmp_path)
|
||||
assert len(msgs) == 2
|
||||
assert all(m.mtime > time.time() - 7 * 86400 for m in msgs)
|
||||
|
||||
# Shrink to 1KB: old and recent-large removed, small survives.
|
||||
create_messages()
|
||||
assert expire_to_target(tmp_path, 1024) == 2
|
||||
msgs = scan_mailbox_messages(tmp_path)
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0].quota_size == 1000
|
||||
|
||||
|
||||
def test_quota_expire_main(tmp_path, capsys):
|
||||
mbox = tmp_path / "user@example.org"
|
||||
_create_message(mbox, "cur", 2 * MB, days_old=5)
|
||||
(mbox / "maildirsize").write_text("x")
|
||||
quota_expire_main([str(1), str(mbox)])
|
||||
_, err = capsys.readouterr()
|
||||
assert "quota-expire: removed 1 message(s) from user@example.org" in err
|
||||
assert not (mbox / "maildirsize").exists()
|
||||
|
||||
@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
|
||||
assert ac1["password"] != ac2["password"]
|
||||
|
||||
|
||||
def test_create_newemail_dict_ip(make_config):
|
||||
config = make_config("1.2.3.4")
|
||||
ac = create_newemail_dict(config)
|
||||
assert ac["email"].endswith("@[1.2.3.4]")
|
||||
|
||||
|
||||
def test_create_dclogin_url():
|
||||
url = create_dclogin_url("user@example.org", "p@ss w+rd")
|
||||
assert url.startswith("dclogin:")
|
||||
|
||||
@@ -194,6 +194,12 @@ def status_cmd(args, out):
|
||||
|
||||
|
||||
def test_cmd_options(parser):
|
||||
parser.add_argument(
|
||||
"--slow",
|
||||
dest="slow",
|
||||
action="store_true",
|
||||
help="also run slow tests",
|
||||
)
|
||||
add_ssh_host_option(parser)
|
||||
|
||||
|
||||
@@ -215,6 +221,8 @@ def test_cmd(args, out):
|
||||
"-v",
|
||||
"--durations=5",
|
||||
]
|
||||
if args.slow:
|
||||
pytest_args.append("--slow")
|
||||
ret = out.run_ret(pytest_args, env=env)
|
||||
return ret
|
||||
|
||||
|
||||
@@ -150,6 +150,9 @@ class UnboundDeployer(Deployer):
|
||||
self.need_restart = False
|
||||
|
||||
def install(self):
|
||||
# Run local DNS resolver `unbound`. `resolvconf` takes care of
|
||||
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
|
||||
|
||||
# On an IPv4-only system, if unbound is started but not configured,
|
||||
# it causes subsequent steps to fail to resolve hosts.
|
||||
with blocked_service_startup():
|
||||
@@ -159,31 +162,6 @@ class UnboundDeployer(Deployer):
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
|
||||
apt.packages(
|
||||
name="Purge resolvconf",
|
||||
packages=["resolvconf"],
|
||||
present=False,
|
||||
extra_uninstall_args="--purge",
|
||||
)
|
||||
# systemd-resolved can't be purged due to dependencies; stop and mask.
|
||||
server.shell(
|
||||
name="Stop and mask systemd-resolved",
|
||||
commands=[
|
||||
"systemctl stop systemd-resolved.service || true",
|
||||
"systemctl mask systemd-resolved.service",
|
||||
],
|
||||
)
|
||||
# Configure unbound resolver with Quad9 fallback and a trailing newline
|
||||
# (SolusVM bug).
|
||||
files.put(
|
||||
name="Write static resolv.conf",
|
||||
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
|
||||
dest="/etc/resolv.conf",
|
||||
user="root",
|
||||
group="root",
|
||||
mode="644",
|
||||
)
|
||||
server.shell(
|
||||
name="Generate root keys for validating DNSSEC",
|
||||
commands=[
|
||||
@@ -590,6 +568,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
||||
Deployment().perform_stages([WebsiteDeployer(config)])
|
||||
return
|
||||
|
||||
if host.get_fact(Port, port=53) != "unbound":
|
||||
files.line(
|
||||
name="Add 9.9.9.9 to resolv.conf",
|
||||
path="/etc/resolv.conf",
|
||||
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
|
||||
line="\nnameserver 9.9.9.9",
|
||||
)
|
||||
|
||||
# Check if mtail_address interface is available (if configured)
|
||||
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
||||
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
||||
|
||||
@@ -4,7 +4,7 @@ import urllib.request
|
||||
from chatmaild.config import Config
|
||||
from pyinfra import host
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
||||
from pyinfra.facts.server import Arch, Sysctl
|
||||
from pyinfra.operations import apt, files, server, systemd
|
||||
|
||||
from cmdeploy.basedeploy import (
|
||||
@@ -80,17 +80,6 @@ class DovecotDeployer(Deployer):
|
||||
def activate(self):
|
||||
activate_remote_units(self.units)
|
||||
|
||||
# Detect stale binary: package installed but service still runs old (deleted) binary.
|
||||
if not self.disable_mail and not self.need_restart:
|
||||
stale = host.get_fact(
|
||||
Command,
|
||||
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
|
||||
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
|
||||
" && echo STALE || true",
|
||||
)
|
||||
if stale == "STALE":
|
||||
self.need_restart = True
|
||||
|
||||
restart = False if self.disable_mail else self.need_restart
|
||||
|
||||
systemd.service(
|
||||
@@ -186,7 +175,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
|
||||
can_modify = not is_in_container()
|
||||
for name in ("max_user_instances", "max_user_watches"):
|
||||
key = f"fs.inotify.{name}"
|
||||
value = host.get_fact(Sysctl).get(key, 0)
|
||||
value = host.get_fact(Sysctl)[key]
|
||||
if value > 65534:
|
||||
continue
|
||||
if not can_modify:
|
||||
|
||||
@@ -133,11 +133,6 @@ protocol lmtp {
|
||||
# mail_lua and push_notification_lua are needed for Lua push notification handler.
|
||||
# <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
|
||||
mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
|
||||
|
||||
# Disable fsync for LMTP. May lose delivered message,
|
||||
# but unlikely to cause problems with multiple relays.
|
||||
# https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
|
||||
mail_fsync = never
|
||||
}
|
||||
|
||||
plugin {
|
||||
@@ -149,26 +144,12 @@ plugin {
|
||||
}
|
||||
|
||||
plugin {
|
||||
# for now we define static quota-rules for all users
|
||||
quota = maildir:User quota
|
||||
quota_rule = *:storage={{ config.max_mailbox_size }}
|
||||
quota_max_mail_size={{ config.max_message_size }}
|
||||
quota_grace = 0
|
||||
|
||||
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
|
||||
|
||||
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
|
||||
# The percentages are chosen to prevent current Delta Chat users
|
||||
# from seeing "quota warnings" which trigger at 80% and 95%.
|
||||
|
||||
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
|
||||
}
|
||||
|
||||
service quota-warning {
|
||||
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
|
||||
user = vmail
|
||||
unix_listener quota-warning {
|
||||
user = vmail
|
||||
mode = 0600
|
||||
}
|
||||
# quota_over_flag_value = TRUE
|
||||
}
|
||||
|
||||
# push_notification configuration
|
||||
@@ -271,9 +252,6 @@ protocol imap {
|
||||
# sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
|
||||
|
||||
rawlog_dir = %h
|
||||
|
||||
# Disable fsync for IMAP. May lose IMAP changes like setting flags.
|
||||
mail_fsync = never
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -78,11 +78,3 @@ counter rejected_unencrypted_mail_count
|
||||
/Rejected unencrypted mail/ {
|
||||
rejected_unencrypted_mail_count++
|
||||
}
|
||||
|
||||
counter quota_expire_runs
|
||||
counter quota_expire_removed_files
|
||||
|
||||
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
|
||||
quota_expire_runs++
|
||||
quota_expire_removed_files += $count
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ http {
|
||||
access_log syslog:server=unix:/dev/log,facility=local7;
|
||||
|
||||
location /mxdeliv/ {
|
||||
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
|
||||
proxy_pass http://127.0.0.1:{{ config.filtermail_http_port }};
|
||||
}
|
||||
|
||||
location / {
|
||||
|
||||
@@ -221,6 +221,7 @@ def test_rewrite_subject(cmsetup, maildata):
|
||||
assert "Subject: Unencrypted subject" not in rcvd_msg
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||
"""Test that the per-account send-mail limit is exceeded."""
|
||||
user1, user2 = cmsetup.gen_users(2)
|
||||
@@ -243,6 +244,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||
pytest.fail("Rate limit was not exceeded")
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_expunged(remote, chatmail_config):
|
||||
outdated_days = int(chatmail_config.delete_mails_after) + 1
|
||||
find_cmds = [
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import imaplib
|
||||
import ipaddress
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
@@ -15,12 +14,10 @@ from chatmaild.config import read_config
|
||||
conftestdir = Path(__file__).parent
|
||||
|
||||
|
||||
def _is_ip(domain):
|
||||
try:
|
||||
ipaddress.ip_address(domain)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--slow", action="store_true", default=False, help="also run slow tests"
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
@@ -30,6 +27,13 @@ def pytest_configure(config):
|
||||
)
|
||||
|
||||
|
||||
def pytest_runtest_setup(item):
|
||||
markers = list(item.iter_markers(name="slow"))
|
||||
if markers:
|
||||
if not item.config.getoption("--slow"):
|
||||
pytest.skip("skipping slow test, use --slow to run")
|
||||
|
||||
|
||||
def _get_chatmail_config():
|
||||
inipath = os.environ.get("CHATMAIL_INI")
|
||||
if inipath:
|
||||
@@ -278,7 +282,6 @@ def gencreds(chatmail_config):
|
||||
|
||||
def gen(domain=None):
|
||||
domain = domain if domain else chatmail_config.mail_domain
|
||||
addr_domain = f"[{domain}]" if _is_ip(domain) else domain
|
||||
while 1:
|
||||
num = next(count)
|
||||
alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
|
||||
@@ -292,7 +295,7 @@ def gencreds(chatmail_config):
|
||||
password = "".join(
|
||||
random.choices(alphanumeric, k=chatmail_config.password_min_length)
|
||||
)
|
||||
yield f"{user}@{addr_domain}", f"{password}"
|
||||
yield f"{user}@{domain}", f"{password}"
|
||||
|
||||
return lambda domain=None: next(gen(domain))
|
||||
|
||||
@@ -341,22 +344,9 @@ class ChatmailACFactory:
|
||||
accounts = []
|
||||
for _ in range(num):
|
||||
account = self.dc.add_account()
|
||||
addr, password = self.gencreds(domain)
|
||||
if _is_ip(domain):
|
||||
# Use DCLOGIN scheme with explicit server hosts,
|
||||
# matching how madmail presents its addresses to users.
|
||||
qr = (
|
||||
f"dclogin:{addr}"
|
||||
f"?p={password}&v=1"
|
||||
f"&ih={domain}&ip=993"
|
||||
f"&sh={domain}&sp=465"
|
||||
f"&ic=3&ss=default"
|
||||
)
|
||||
future = account.add_transport_from_qr.future(qr)
|
||||
else:
|
||||
future = account.add_or_update_transport.future(
|
||||
self._make_transport(domain)
|
||||
)
|
||||
future = account.add_or_update_transport.future(
|
||||
self._make_transport(domain)
|
||||
)
|
||||
futures.append(future)
|
||||
|
||||
# ensure messages stay in INBOX so that they can be
|
||||
|
||||
@@ -2,9 +2,9 @@ from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
|
||||
from cmdeploy.dovecot import deployer as dovecot_deployer
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
|
||||
|
||||
def make_host(*fact_pairs):
|
||||
|
||||
@@ -98,6 +98,13 @@ steps. Please substitute it with your own domain.
|
||||
configure at your DNS provider (it can take some time until they are
|
||||
public).
|
||||
|
||||
Docker installation
|
||||
-------------------
|
||||
|
||||
There is experimental support for running chatmail via Docker Compose.
|
||||
See the `chatmail/docker README <https://github.com/chatmail/docker>`_
|
||||
for full setup instructions.
|
||||
|
||||
Other helpful commands
|
||||
----------------------
|
||||
|
||||
|
||||
@@ -102,12 +102,8 @@ short overview of ``chatmaild`` services:
|
||||
Apple/Google/Huawei.
|
||||
|
||||
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
|
||||
deletes old messages, large messages, and entire mailboxes
|
||||
of users who have not logged in for longer than
|
||||
``delete_inactive_users_after`` days.
|
||||
|
||||
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
|
||||
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
|
||||
deletes users if they have not logged in for a longer while.
|
||||
The timeframe can be configured in ``chatmail.ini``.
|
||||
|
||||
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
||||
is contacted by Dovecot when a user logs in and stores the date of
|
||||
@@ -160,8 +156,6 @@ Chatmail relay dependency diagram
|
||||
/home/vmail/.../user"];
|
||||
dovecot --- |lastlogin.socket|lastlogin;
|
||||
dovecot --- chatmail-metadata;
|
||||
dovecot --- |quota-warning|chatmail-quota-expire;
|
||||
chatmail-quota-expire --- maildir;
|
||||
lastlogin --- maildir;
|
||||
doveauth --- maildir;
|
||||
chatmail-expire-daily --- maildir;
|
||||
|
||||
Reference in New Issue
Block a user