mirror of
https://github.com/chatmail/relay.git
synced 2026-05-14 18:04:38 +00:00
Compare commits
6 Commits
expire-ind
...
j4n/docker
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0808c9dc47 | ||
|
|
2bdfecff72 | ||
|
|
cef739e3b3 | ||
|
|
3d128d3c64 | ||
|
|
79f68342f4 | ||
|
|
54863453c2 |
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
125
.github/workflows/docker-deploy.yaml
vendored
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
name: Docker deploy
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
staging_host:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'SSH hostname (e.g. staging2.testrun.org)'
|
||||||
|
mail_domain:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'MAIL_DOMAIN for docker compose'
|
||||||
|
zone_file:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
description: 'Default zone file basename (e.g. staging.testrun.org-default.zone)'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy-docker:
|
||||||
|
name: Docker deploy on ${{ inputs.staging_host }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 20
|
||||||
|
environment:
|
||||||
|
name: ${{ inputs.staging_host }}
|
||||||
|
url: https://${{ inputs.staging_host }}/
|
||||||
|
concurrency: ${{ inputs.staging_host }}
|
||||||
|
env:
|
||||||
|
VPS: root@${{ inputs.staging_host }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: true
|
||||||
|
- name: Setup SSH
|
||||||
|
run: |
|
||||||
|
mkdir ~/.ssh
|
||||||
|
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||||
|
chmod 600 ~/.ssh/id_ed25519
|
||||||
|
ssh-keyscan ${{ inputs.staging_host }} > ~/.ssh/known_hosts
|
||||||
|
# Reuse TCP connection for all subsequent ssh/scp calls
|
||||||
|
echo -e "Host ${{ inputs.staging_host }}\n ControlMaster auto\n ControlPath ~/.ssh/ctrl-%r@%h:%p\n ControlPersist 10m" >> ~/.ssh/config
|
||||||
|
|
||||||
|
- name: stop bare services, install Docker, prepare mounts
|
||||||
|
run: |
|
||||||
|
ssh $VPS bash -s <<'EOF'
|
||||||
|
systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true
|
||||||
|
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc
|
||||||
|
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||||
|
mkdir -p /srv/chatmail/certs /srv/chatmail/dkim
|
||||||
|
cp -a /var/lib/acme/. /srv/chatmail/certs/ || true
|
||||||
|
cp -a /etc/dkimkeys/. /srv/chatmail/dkim/ || true
|
||||||
|
cp /etc/chatmail/chatmail.ini /srv/chatmail/chatmail.ini
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: deploy with Docker
|
||||||
|
run: |
|
||||||
|
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||||
|
GHCR_IMAGE="ghcr.io/chatmail/docker:sha-${SHORT_SHA}"
|
||||||
|
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ $VPS:/srv/chatmail/relay/
|
||||||
|
echo "${{ secrets.GITHUB_TOKEN }}" | ssh $VPS "docker login ghcr.io -u ${{ github.actor }} --password-stdin && \
|
||||||
|
docker pull ${GHCR_IMAGE} && \
|
||||||
|
cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=${{ inputs.mail_domain }} docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
||||||
|
|
||||||
|
- name: wait for container healthy
|
||||||
|
run: |
|
||||||
|
ssh $VPS 'docker exec chatmail journalctl -f --no-pager' &
|
||||||
|
LOG_PID=$!
|
||||||
|
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
status=$(ssh $VPS 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
||||||
|
echo " [$i/60] status=$status"
|
||||||
|
if [ "$status" = "healthy" ]; then
|
||||||
|
echo "Container is healthy."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "$status" = "unhealthy" ]; then
|
||||||
|
echo "Container is unhealthy!"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
echo "Container did not become healthy."
|
||||||
|
kill $LOG_PID 2>/dev/null || true
|
||||||
|
ssh $VPS bash -s <<'EOF'
|
||||||
|
echo "--- failed units ---"
|
||||||
|
docker exec chatmail systemctl --failed --no-pager || true
|
||||||
|
echo "--- service logs ---"
|
||||||
|
docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50 || true
|
||||||
|
echo "--- listening ports ---"
|
||||||
|
docker exec chatmail ss -tlnp || true
|
||||||
|
echo "--- chatmail.ini ---"
|
||||||
|
docker exec chatmail cat /etc/chatmail/chatmail.ini || true
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
- name: show container state
|
||||||
|
run: |
|
||||||
|
ssh $VPS bash -s <<'EOF'
|
||||||
|
echo "--- listening ports ---"
|
||||||
|
docker exec chatmail ss -tlnp
|
||||||
|
echo "--- chatmail.ini ---"
|
||||||
|
docker exec chatmail cat /etc/chatmail/chatmail.ini
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Docker integration tests
|
||||||
|
run: ssh $VPS 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
||||||
|
|
||||||
|
- name: Docker DNS
|
||||||
|
run: |
|
||||||
|
git checkout .github/workflows/${{ inputs.zone_file }}
|
||||||
|
ssh $VPS bash -s <<'EOF'
|
||||||
|
docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys
|
||||||
|
docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose
|
||||||
|
docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone
|
||||||
|
EOF
|
||||||
|
scp $VPS:/tmp/staging.zone staging-generated.zone
|
||||||
|
cat staging-generated.zone >> .github/workflows/${{ inputs.zone_file }}
|
||||||
|
cat .github/workflows/${{ inputs.zone_file }}
|
||||||
|
scp .github/workflows/${{ inputs.zone_file }} root@ns.testrun.org:/etc/nsd/${{ inputs.staging_host }}.zone
|
||||||
|
ssh root@ns.testrun.org "nsd-checkzone ${{ inputs.staging_host }} /etc/nsd/${{ inputs.staging_host }}.zone && systemctl reload nsd"
|
||||||
|
|
||||||
|
- name: Docker final DNS check
|
||||||
|
run: ssh $VPS 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
||||||
23
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
23
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- j4n/docker-pr
|
||||||
pull_request:
|
pull_request:
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'scripts/**'
|
- 'scripts/**'
|
||||||
@@ -12,6 +13,11 @@ on:
|
|||||||
- 'LICENSE'
|
- 'LICENSE'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
trigger-docker-build:
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -22,6 +28,8 @@ jobs:
|
|||||||
concurrency: staging-ipv4.testrun.org
|
concurrency: staging-ipv4.testrun.org
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: true
|
||||||
|
|
||||||
- name: prepare SSH
|
- name: prepare SSH
|
||||||
run: |
|
run: |
|
||||||
@@ -63,13 +71,13 @@ jobs:
|
|||||||
# download acme & dkim state from ns.testrun.org
|
# download acme & dkim state from ns.testrun.org
|
||||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
||||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
||||||
# restore acme & dkim state to staging2.testrun.org
|
# restore acme & dkim state to staging-ipv4.testrun.org
|
||||||
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
||||||
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
||||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
||||||
|
|
||||||
- name: run deploy-chatmail offline tests
|
- name: run deploy-chatmail offline tests
|
||||||
run: pytest --pyargs cmdeploy
|
run: pytest --pyargs cmdeploy
|
||||||
|
|
||||||
- name: setup dependencies
|
- name: setup dependencies
|
||||||
run: |
|
run: |
|
||||||
@@ -102,3 +110,12 @@ jobs:
|
|||||||
- name: cmdeploy dns
|
- name: cmdeploy dns
|
||||||
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
||||||
|
|
||||||
|
deploy-docker:
|
||||||
|
needs: [deploy, trigger-docker-build]
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
uses: ./.github/workflows/docker-deploy.yaml
|
||||||
|
with:
|
||||||
|
staging_host: staging-ipv4.testrun.org
|
||||||
|
mail_domain: staging-ipv4.testrun.org
|
||||||
|
zone_file: staging-ipv4.testrun.org-default.zone
|
||||||
|
secrets: inherit
|
||||||
|
|||||||
15
.github/workflows/test-and-deploy.yaml
vendored
15
.github/workflows/test-and-deploy.yaml
vendored
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- j4n/docker-pr
|
||||||
pull_request:
|
pull_request:
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'scripts/**'
|
- 'scripts/**'
|
||||||
@@ -12,6 +13,11 @@ on:
|
|||||||
- 'LICENSE'
|
- 'LICENSE'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
trigger-docker-build:
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
uses: ./.github/workflows/trigger-docker-build.yaml
|
||||||
|
secrets: inherit
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
name: deploy on staging2.testrun.org, and run tests
|
name: deploy on staging2.testrun.org, and run tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -95,3 +101,12 @@ jobs:
|
|||||||
- name: cmdeploy dns
|
- name: cmdeploy dns
|
||||||
run: cmdeploy dns -v
|
run: cmdeploy dns -v
|
||||||
|
|
||||||
|
deploy-docker:
|
||||||
|
needs: [deploy, trigger-docker-build]
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
uses: ./.github/workflows/docker-deploy.yaml
|
||||||
|
with:
|
||||||
|
staging_host: staging2.testrun.org
|
||||||
|
mail_domain: staging2.testrun.org
|
||||||
|
zone_file: staging.testrun.org-default.zone
|
||||||
|
secrets: inherit
|
||||||
|
|||||||
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
24
.github/workflows/trigger-docker-build.yaml
vendored
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
name: Trigger Docker image build
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
trigger-docker-build:
|
||||||
|
name: Trigger Docker image build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
|
||||||
|
script: |
|
||||||
|
await github.rest.repos.createDispatchEvent({
|
||||||
|
owner: 'chatmail',
|
||||||
|
repo: 'docker',
|
||||||
|
event_type: 'relay-updated',
|
||||||
|
client_payload: {
|
||||||
|
relay_ref: context.ref,
|
||||||
|
relay_sha: context.sha,
|
||||||
|
relay_sha_short: context.sha.slice(0, 7)
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -145,10 +145,6 @@ class Expiry:
|
|||||||
changed = True
|
changed = True
|
||||||
if changed:
|
if changed:
|
||||||
self.remove_file(f"{mbox.basedir}/maildirsize")
|
self.remove_file(f"{mbox.basedir}/maildirsize")
|
||||||
for file in mbox.extrafiles:
|
|
||||||
if "dovecot.index.cache" in file.path.split("/")[-1]:
|
|
||||||
if file.size > 500 * 1024:
|
|
||||||
self.remove_file(file.path)
|
|
||||||
|
|
||||||
def get_summary(self):
|
def get_summary(self):
|
||||||
return (
|
return (
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ import io
|
|||||||
import os
|
import os
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
from pyinfra import host
|
||||||
|
from pyinfra.facts.server import Command
|
||||||
from pyinfra.operations import files, server, systemd
|
from pyinfra.operations import files, server, systemd
|
||||||
|
|
||||||
|
|
||||||
@@ -11,6 +13,17 @@ def has_systemd():
|
|||||||
return os.path.isdir("/run/systemd/system")
|
return os.path.isdir("/run/systemd/system")
|
||||||
|
|
||||||
|
|
||||||
|
def is_in_container() -> bool:
|
||||||
|
"""Return True if running inside a container (Docker, LXC, etc.)."""
|
||||||
|
return (
|
||||||
|
host.get_fact(
|
||||||
|
Command,
|
||||||
|
"systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
|
||||||
|
)
|
||||||
|
== "yes"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def blocked_service_startup():
|
def blocked_service_startup():
|
||||||
"""Prevent services from auto-starting during package installation.
|
"""Prevent services from auto-starting during package installation.
|
||||||
|
|||||||
@@ -108,9 +108,7 @@ def run_cmd(args, out):
|
|||||||
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
||||||
|
|
||||||
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
|
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
|
||||||
if ssh_host in ["localhost", "@docker"]:
|
if ssh_host == "localhost":
|
||||||
if ssh_host == "@docker":
|
|
||||||
env["CHATMAIL_NOPORTCHECK"] = "True"
|
|
||||||
cmd = f"{pyinf} @local {deploy_path} -y"
|
cmd = f"{pyinf} @local {deploy_path} -y"
|
||||||
|
|
||||||
if version.parse(pyinfra.__version__) < version.parse("3"):
|
if version.parse(pyinfra.__version__) < version.parse("3"):
|
||||||
@@ -316,7 +314,7 @@ def add_ssh_host_option(parser):
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--ssh-host",
|
"--ssh-host",
|
||||||
dest="ssh_host",
|
dest="ssh_host",
|
||||||
help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
|
help="Run commands on 'localhost' or on a specific SSH host "
|
||||||
"instead of chatmail.ini's mail_domain.",
|
"instead of chatmail.ini's mail_domain.",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -378,9 +376,7 @@ def get_parser():
|
|||||||
|
|
||||||
def get_sshexec(ssh_host: str, verbose=True):
|
def get_sshexec(ssh_host: str, verbose=True):
|
||||||
if ssh_host in ["localhost", "@local"]:
|
if ssh_host in ["localhost", "@local"]:
|
||||||
return LocalExec(verbose, docker=False)
|
return LocalExec(verbose)
|
||||||
elif ssh_host == "@docker":
|
|
||||||
return LocalExec(verbose, docker=True)
|
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"[ssh] login to {ssh_host}")
|
print(f"[ssh] login to {ssh_host}")
|
||||||
return SSHExec(ssh_host, verbose=verbose)
|
return SSHExec(ssh_host, verbose=verbose)
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
Chat Mail pyinfra deploy.
|
Chat Mail pyinfra deploy.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -28,6 +27,7 @@ from .basedeploy import (
|
|||||||
configure_remote_units,
|
configure_remote_units,
|
||||||
get_resource,
|
get_resource,
|
||||||
has_systemd,
|
has_systemd,
|
||||||
|
is_in_container,
|
||||||
)
|
)
|
||||||
from .dovecot.deployer import DovecotDeployer
|
from .dovecot.deployer import DovecotDeployer
|
||||||
from .external.deployer import ExternalTlsDeployer
|
from .external.deployer import ExternalTlsDeployer
|
||||||
@@ -584,7 +584,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
|||||||
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
|
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
if not os.environ.get("CHATMAIL_NOPORTCHECK"):
|
if not is_in_container():
|
||||||
port_services = [
|
port_services = [
|
||||||
(["master", "smtpd"], 25),
|
(["master", "smtpd"], 25),
|
||||||
("unbound", 53),
|
("unbound", 53),
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import urllib.request
|
|||||||
from chatmaild.config import Config
|
from chatmaild.config import Config
|
||||||
from pyinfra import host
|
from pyinfra import host
|
||||||
from pyinfra.facts.deb import DebPackages
|
from pyinfra.facts.deb import DebPackages
|
||||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
from pyinfra.facts.server import Arch, Sysctl
|
||||||
from pyinfra.operations import apt, files, server, systemd
|
from pyinfra.operations import apt, files, server, systemd
|
||||||
|
|
||||||
from cmdeploy.basedeploy import (
|
from cmdeploy.basedeploy import (
|
||||||
@@ -13,9 +13,11 @@ from cmdeploy.basedeploy import (
|
|||||||
blocked_service_startup,
|
blocked_service_startup,
|
||||||
configure_remote_units,
|
configure_remote_units,
|
||||||
get_resource,
|
get_resource,
|
||||||
|
is_in_container,
|
||||||
)
|
)
|
||||||
|
|
||||||
DOVECOT_VERSION = "2.3.21+dfsg1-3"
|
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
|
||||||
|
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
|
||||||
|
|
||||||
DOVECOT_SHA256 = {
|
DOVECOT_SHA256 = {
|
||||||
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
|
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
|
||||||
@@ -40,11 +42,14 @@ class DovecotDeployer(Deployer):
|
|||||||
with blocked_service_startup():
|
with blocked_service_startup():
|
||||||
debs = []
|
debs = []
|
||||||
for pkg in ("core", "imapd", "lmtpd"):
|
for pkg in ("core", "imapd", "lmtpd"):
|
||||||
deb = _download_dovecot_package(pkg, arch)
|
deb, changed = _download_dovecot_package(pkg, arch)
|
||||||
|
self.need_restart |= changed
|
||||||
if deb:
|
if deb:
|
||||||
debs.append(deb)
|
debs.append(deb)
|
||||||
if debs:
|
if debs:
|
||||||
deb_list = " ".join(debs)
|
deb_list = " ".join(debs)
|
||||||
|
# First dpkg may fail on missing dependencies (stderr suppressed);
|
||||||
|
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
|
||||||
server.shell(
|
server.shell(
|
||||||
name="Install dovecot packages",
|
name="Install dovecot packages",
|
||||||
commands=[
|
commands=[
|
||||||
@@ -53,6 +58,7 @@ class DovecotDeployer(Deployer):
|
|||||||
f"dpkg --force-confdef --force-confold -i {deb_list}",
|
f"dpkg --force-confdef --force-confold -i {deb_list}",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
self.need_restart = True
|
||||||
files.put(
|
files.put(
|
||||||
name="Pin dovecot packages to block Debian dist-upgrades",
|
name="Pin dovecot packages to block Debian dist-upgrades",
|
||||||
src=io.StringIO(
|
src=io.StringIO(
|
||||||
@@ -61,11 +67,15 @@ class DovecotDeployer(Deployer):
|
|||||||
"Pin-Priority: -1\n"
|
"Pin-Priority: -1\n"
|
||||||
),
|
),
|
||||||
dest="/etc/apt/preferences.d/pin-dovecot",
|
dest="/etc/apt/preferences.d/pin-dovecot",
|
||||||
|
user="root",
|
||||||
|
group="root",
|
||||||
|
mode="644",
|
||||||
)
|
)
|
||||||
|
|
||||||
def configure(self):
|
def configure(self):
|
||||||
configure_remote_units(self.config.mail_domain, self.units)
|
configure_remote_units(self.config.mail_domain, self.units)
|
||||||
self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
|
config_restart, self.daemon_reload = _configure_dovecot(self.config)
|
||||||
|
self.need_restart |= config_restart
|
||||||
|
|
||||||
def activate(self):
|
def activate(self):
|
||||||
activate_remote_units(self.units)
|
activate_remote_units(self.units)
|
||||||
@@ -94,22 +104,22 @@ def _pick_url(primary, fallback):
|
|||||||
return fallback
|
return fallback
|
||||||
|
|
||||||
|
|
||||||
def _download_dovecot_package(package: str, arch: str):
|
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
|
||||||
"""Download a dovecot .deb if needed, return its path (or None)."""
|
"""Download a dovecot .deb if needed, return (path, changed)."""
|
||||||
arch = "amd64" if arch == "x86_64" else arch
|
arch = "amd64" if arch == "x86_64" else arch
|
||||||
arch = "arm64" if arch == "aarch64" else arch
|
arch = "arm64" if arch == "aarch64" else arch
|
||||||
|
|
||||||
pkg_name = f"dovecot-{package}"
|
pkg_name = f"dovecot-{package}"
|
||||||
sha256 = DOVECOT_SHA256.get((package, arch))
|
sha256 = DOVECOT_SHA256.get((package, arch))
|
||||||
if sha256 is None:
|
if sha256 is None:
|
||||||
apt.packages(packages=[pkg_name])
|
op = apt.packages(packages=[pkg_name])
|
||||||
return None
|
return None, bool(getattr(op, "changed", False))
|
||||||
|
|
||||||
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
|
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
|
||||||
if DOVECOT_VERSION in installed_versions:
|
if DOVECOT_PACKAGE_VERSION in installed_versions:
|
||||||
return None
|
return None, False
|
||||||
|
|
||||||
url_version = DOVECOT_VERSION.replace("+", "%2B")
|
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
|
||||||
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
|
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
|
||||||
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
|
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
|
||||||
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
|
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
|
||||||
@@ -124,18 +134,7 @@ def _download_dovecot_package(package: str, arch: str):
|
|||||||
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
|
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
|
||||||
)
|
)
|
||||||
|
|
||||||
return deb_filename
|
return deb_filename, True
|
||||||
|
|
||||||
|
|
||||||
def _can_set_inotify_limits() -> bool:
|
|
||||||
is_container = (
|
|
||||||
host.get_fact(
|
|
||||||
Command,
|
|
||||||
"systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
|
|
||||||
)
|
|
||||||
== "yes"
|
|
||||||
)
|
|
||||||
return not is_container
|
|
||||||
|
|
||||||
|
|
||||||
def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
|
def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
|
||||||
@@ -173,7 +172,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
|
|||||||
|
|
||||||
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
|
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
|
||||||
# it is recommended to set the following inotify limits
|
# it is recommended to set the following inotify limits
|
||||||
can_modify = _can_set_inotify_limits()
|
can_modify = not is_in_container()
|
||||||
for name in ("max_user_instances", "max_user_watches"):
|
for name in ("max_user_instances", "max_user_watches"):
|
||||||
key = f"fs.inotify.{name}"
|
key = f"fs.inotify.{name}"
|
||||||
value = host.get_fact(Sysctl)[key]
|
value = host.get_fact(Sysctl)[key]
|
||||||
|
|||||||
@@ -70,6 +70,12 @@ userdb {
|
|||||||
# Mailboxes are stored in the "mail" directory of the vmail user home.
|
# Mailboxes are stored in the "mail" directory of the vmail user home.
|
||||||
mail_location = maildir:{{ config.mailboxes_dir }}/%u
|
mail_location = maildir:{{ config.mailboxes_dir }}/%u
|
||||||
|
|
||||||
|
# index/cache files are not very useful for chatmail relay operations
|
||||||
|
# but it's not clear how to disable them completely.
|
||||||
|
# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
|
||||||
|
# if the cache file becomes larger than the specified size, it is truncated by dovecot
|
||||||
|
mail_cache_max_size = 500K
|
||||||
|
|
||||||
namespace inbox {
|
namespace inbox {
|
||||||
inbox = yes
|
inbox = yes
|
||||||
|
|
||||||
|
|||||||
@@ -87,9 +87,8 @@ class SSHExec:
|
|||||||
class LocalExec:
|
class LocalExec:
|
||||||
FuncError = FuncError
|
FuncError = FuncError
|
||||||
|
|
||||||
def __init__(self, verbose=False, docker=False):
|
def __init__(self, verbose=False):
|
||||||
self.verbose = verbose
|
self.verbose = verbose
|
||||||
self.docker = docker
|
|
||||||
|
|
||||||
def __call__(self, call, kwargs=None, log_callback=None):
|
def __call__(self, call, kwargs=None, log_callback=None):
|
||||||
if kwargs is None:
|
if kwargs is None:
|
||||||
@@ -101,10 +100,6 @@ class LocalExec:
|
|||||||
if not title:
|
if not title:
|
||||||
title = call.__name__
|
title = call.__name__
|
||||||
where = "locally"
|
where = "locally"
|
||||||
if self.docker:
|
|
||||||
if call == remote.rdns.perform_initial_checks:
|
|
||||||
kwargs["pre_command"] = "docker exec chatmail "
|
|
||||||
where = "in docker"
|
|
||||||
if self.verbose:
|
if self.verbose:
|
||||||
print_stderr(f"Running {where}: {title}(**{kwargs})")
|
print_stderr(f"Running {where}: {title}(**{kwargs})")
|
||||||
return self(call, kwargs, log_callback=print_stderr)
|
return self(call, kwargs, log_callback=print_stderr)
|
||||||
|
|||||||
@@ -71,6 +71,44 @@ class TestSSHExecutor:
|
|||||||
assert (now - since_date).total_seconds() < 60 * 60 * 51
|
assert (now - since_date).total_seconds() < 60 * 60 * 51
|
||||||
|
|
||||||
|
|
||||||
|
def test_dovecot_main_process_matches_installed_binary(sshdomain):
|
||||||
|
sshexec = get_sshexec(sshdomain)
|
||||||
|
main_pid = int(
|
||||||
|
sshexec(
|
||||||
|
call=remote.rshell.shell,
|
||||||
|
kwargs=dict(
|
||||||
|
command="timeout 10 systemctl show -p MainPID --value dovecot.service"
|
||||||
|
),
|
||||||
|
).strip()
|
||||||
|
)
|
||||||
|
assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
|
||||||
|
|
||||||
|
exe = sshexec(
|
||||||
|
call=remote.rshell.shell,
|
||||||
|
kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
|
||||||
|
).strip()
|
||||||
|
status_text = sshexec(
|
||||||
|
call=remote.rshell.shell,
|
||||||
|
kwargs=dict(
|
||||||
|
command="timeout 10 systemctl show -p StatusText --value dovecot.service"
|
||||||
|
),
|
||||||
|
).strip()
|
||||||
|
installed_version = sshexec(
|
||||||
|
call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
assert not exe.endswith("(deleted)"), (
|
||||||
|
f"running dovecot binary was deleted (stale after upgrade): {exe}"
|
||||||
|
)
|
||||||
|
expected_status_text = f"v{installed_version}"
|
||||||
|
assert status_text == expected_status_text or status_text.startswith(
|
||||||
|
f"{expected_status_text} "
|
||||||
|
), (
|
||||||
|
f"dovecot status version mismatch: "
|
||||||
|
f"StatusText={status_text!r}, installed={installed_version!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_timezone_env(remote):
|
def test_timezone_env(remote):
|
||||||
for line in remote.iter_output("env"):
|
for line in remote.iter_output("env"):
|
||||||
print(line)
|
print(line)
|
||||||
|
|||||||
238
cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py
Normal file
238
cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
from contextlib import nullcontext
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from cmdeploy.dovecot import deployer as dovecot_deployer
|
||||||
|
from pyinfra.facts.deb import DebPackages
|
||||||
|
|
||||||
|
|
||||||
|
def make_host(*fact_pairs):
|
||||||
|
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
*fact_pairs: tuples of (fact_class, fact_value) to register
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
SimpleNamespace with get_fact that raises a clear error if an
|
||||||
|
unexpected fact type is requested.
|
||||||
|
"""
|
||||||
|
facts = dict(fact_pairs)
|
||||||
|
|
||||||
|
def get_fact(cls):
|
||||||
|
if cls not in facts:
|
||||||
|
registered = ", ".join(c.__name__ for c in facts)
|
||||||
|
raise LookupError(
|
||||||
|
f"unexpected get_fact({cls.__name__}); "
|
||||||
|
f"only registered: {registered}"
|
||||||
|
)
|
||||||
|
return facts[cls]
|
||||||
|
|
||||||
|
return SimpleNamespace(get_fact=get_fact)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def deployer():
|
||||||
|
return dovecot_deployer.DovecotDeployer(
|
||||||
|
SimpleNamespace(mail_domain="chat.example.org"),
|
||||||
|
disable_mail=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def patch_blocked(monkeypatch):
|
||||||
|
monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_files_put(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.files,
|
||||||
|
"put",
|
||||||
|
lambda **kwargs: SimpleNamespace(changed=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def track_shell(monkeypatch):
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.server,
|
||||||
|
"shell",
|
||||||
|
lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
|
||||||
|
)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
|
||||||
|
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
|
||||||
|
downloads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"host",
|
||||||
|
make_host((DebPackages, {"dovecot-core": [epoch_version]})),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"_pick_url",
|
||||||
|
lambda primary, fallback: primary,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.files,
|
||||||
|
"download",
|
||||||
|
lambda **kwargs: downloads.append(kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
|
||||||
|
|
||||||
|
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
|
||||||
|
assert changed is False, "should not flag changed when version already installed"
|
||||||
|
assert downloads == [], "should not download when version already installed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
downloads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"host",
|
||||||
|
make_host((DebPackages, {})),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"_pick_url",
|
||||||
|
lambda primary, fallback: primary,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.files,
|
||||||
|
"download",
|
||||||
|
lambda **kwargs: downloads.append(kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
|
||||||
|
|
||||||
|
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
|
||||||
|
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
|
||||||
|
|
||||||
|
# Verify the returned path uses archive version, not package version (with epoch)
|
||||||
|
assert changed is True, "should flag changed when package not yet installed"
|
||||||
|
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
|
||||||
|
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
|
||||||
|
f"deb path should use archive version (no epoch), got {deb!r}"
|
||||||
|
)
|
||||||
|
assert len(downloads) == 1, "files.download should be called exactly once"
|
||||||
|
|
||||||
|
|
||||||
|
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
|
||||||
|
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"host",
|
||||||
|
make_host(
|
||||||
|
(
|
||||||
|
dovecot_deployer.DebPackages,
|
||||||
|
{
|
||||||
|
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||||
|
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||||
|
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(dovecot_deployer.Arch, "x86_64"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
downloads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.files,
|
||||||
|
"download",
|
||||||
|
lambda **kwargs: downloads.append(kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
deployer.install()
|
||||||
|
|
||||||
|
assert downloads == [], "should not download when all packages epoch-matched"
|
||||||
|
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
|
||||||
|
assert deployer.need_restart is False, (
|
||||||
|
"need_restart should be False when nothing changed"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_install_unsupported_arch_falls_back_to_apt(
|
||||||
|
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
|
||||||
|
):
|
||||||
|
# For unsupported architectures, all fact lookups return the arch string.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"host",
|
||||||
|
SimpleNamespace(get_fact=lambda cls: "riscv64"),
|
||||||
|
)
|
||||||
|
apt_calls = []
|
||||||
|
|
||||||
|
# Mirrors apt.packages() return value: OperationMeta with .changed property.
|
||||||
|
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
|
||||||
|
def fake_apt(**kwargs):
|
||||||
|
apt_calls.append(kwargs)
|
||||||
|
changed = "lmtpd" in kwargs["packages"][0]
|
||||||
|
return SimpleNamespace(changed=changed)
|
||||||
|
|
||||||
|
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
|
||||||
|
|
||||||
|
deployer.install()
|
||||||
|
|
||||||
|
actual_pkgs = [c["packages"] for c in apt_calls]
|
||||||
|
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
|
||||||
|
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
|
||||||
|
)
|
||||||
|
assert track_shell == [], "should not run dpkg for unsupported arch"
|
||||||
|
assert deployer.need_restart is True, (
|
||||||
|
"need_restart should be True when apt installed a package"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_install_runs_dpkg_when_packages_need_download(
|
||||||
|
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"host",
|
||||||
|
make_host(
|
||||||
|
(dovecot_deployer.DebPackages, {}),
|
||||||
|
(dovecot_deployer.Arch, "x86_64"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer,
|
||||||
|
"_pick_url",
|
||||||
|
lambda primary, fallback: primary,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
dovecot_deployer.files,
|
||||||
|
"download",
|
||||||
|
lambda **kwargs: SimpleNamespace(changed=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
deployer.install()
|
||||||
|
|
||||||
|
assert len(track_shell) == 1, (
|
||||||
|
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
|
||||||
|
)
|
||||||
|
cmds = track_shell[0]["commands"]
|
||||||
|
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
|
||||||
|
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
|
||||||
|
assert "apt-get -y --fix-broken install" in cmds[1]
|
||||||
|
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
|
||||||
|
assert deployer.need_restart is True, (
|
||||||
|
"need_restart should be True after dpkg install"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pick_url_falls_back_on_primary_error(monkeypatch):
|
||||||
|
def raise_error(req, timeout):
|
||||||
|
raise OSError("connection timeout")
|
||||||
|
|
||||||
|
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
|
||||||
|
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
|
||||||
|
assert result == "http://fallback", (
|
||||||
|
f"should fall back when primary fails, got {result!r}"
|
||||||
|
)
|
||||||
@@ -98,6 +98,13 @@ steps. Please substitute it with your own domain.
|
|||||||
configure at your DNS provider (it can take some time until they are
|
configure at your DNS provider (it can take some time until they are
|
||||||
public).
|
public).
|
||||||
|
|
||||||
|
Docker installation
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
There is experimental support for running chatmail via Docker Compose.
|
||||||
|
See the `chatmail/docker README <https://github.com/chatmail/docker>`_
|
||||||
|
for full setup instructions.
|
||||||
|
|
||||||
Other helpful commands
|
Other helpful commands
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user