mirror of
https://github.com/chatmail/relay.git
synced 2026-05-10 16:04:37 +00:00
Compare commits
32 Commits
j4n/docker
...
hpk/lxc-ci
| Author | SHA1 | Date | |
|---|---|---|---|
|
ddc36902c8 | ||
|
|
8a1c1d9288 | ||
|
|
c3fd52e5dd | ||
|
|
233d442bca | ||
|
|
8aabb9a955 | ||
|
|
9fc05646e0 | ||
|
|
155c1221b8 | ||
|
|
3393d071e5 | ||
|
|
52c73658f9 | ||
|
|
b022a61955 | ||
|
|
2e383a8e94 | ||
|
|
9908a4c88c | ||
|
|
c04f4a4b44 | ||
|
|
6d0ce061bc | ||
|
|
a48c525455 | ||
|
|
2786b60658 | ||
|
|
fe46b573a6 | ||
|
|
674e496a53 | ||
|
|
86e5708709 | ||
|
|
04ac2cf700 | ||
|
|
e2ec0cf2c5 | ||
|
|
ee9d54f7d6 | ||
|
|
1604321d5b | ||
|
|
4ab04fa6c4 | ||
|
|
8b6829b906 | ||
|
|
cf2cb57cca | ||
|
|
693c3f8555 | ||
|
|
371efdfafb | ||
|
|
23765a5ed8 | ||
|
|
0adeefbdd7 | ||
|
|
624838eedd | ||
|
|
1abdc407af |
@@ -1,18 +0,0 @@
|
||||
data/
|
||||
venv/
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.orig
|
||||
*.ini
|
||||
.pytest_cache
|
||||
.env
|
||||
|
||||
# Slim build context — .git/ alone can be 100s of MB
|
||||
.git
|
||||
.github/
|
||||
docs/
|
||||
tests/
|
||||
|
||||
# Exclude markdown files but keep www/src/*.md (used by WebsiteDeployer)
|
||||
*.md
|
||||
!www/**/*.md
|
||||
90
.github/workflows/ci.yaml
vendored
90
.github/workflows/ci.yaml
vendored
@@ -15,28 +15,98 @@ jobs:
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
- name: download filtermail
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
- name: run chatmaild tests
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
- name: run chatmaild tests
|
||||
working-directory: chatmaild
|
||||
run: pipx run tox
|
||||
|
||||
scripts:
|
||||
name: deploy-chatmail tests
|
||||
name: deploy-chatmail tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: initenv
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: run formatting checks
|
||||
run: cmdeploy fmt -v
|
||||
- name: run formatting checks
|
||||
run: cmdeploy fmt -v
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
# all other cmdeploy commands require a staging server
|
||||
# see https://github.com/deltachat/chatmail/issues/100
|
||||
lxc-test:
|
||||
name: LXC deploy and test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: install incus
|
||||
run: sudo apt-get update && sudo apt-get install -y incus
|
||||
|
||||
- name: initialise incus
|
||||
run: |
|
||||
sudo systemctl stop docker.socket docker || true
|
||||
sudo iptables -P FORWARD ACCEPT
|
||||
sudo sysctl -w fs.inotify.max_user_instances=65535
|
||||
sudo sysctl -w fs.inotify.max_user_watches=65535
|
||||
sudo incus admin init --minimal --quiet
|
||||
sudo usermod -aG incus-admin "$USER"
|
||||
|
||||
- name: initenv
|
||||
run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: restore cached images
|
||||
id: cache-images
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
/tmp/localchat-base.tar.gz
|
||||
/tmp/localchat-ns.tar.gz
|
||||
/tmp/localchat-test0.tar.gz
|
||||
/tmp/localchat-test1.tar.gz
|
||||
lxconfigs/id_localchat*
|
||||
key: incus-contain-v2-${{ runner.os }}-${{ github.ref_name }}
|
||||
restore-keys: |
|
||||
incus-contain-v2-${{ runner.os }}-${{ github.ref_name }}-
|
||||
incus-contain-v2-${{ runner.os }}-main-
|
||||
incus-contain-v2-${{ runner.os }}-
|
||||
|
||||
- name: import cached images
|
||||
run: |
|
||||
for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
|
||||
if [ -f /tmp/$alias.tar.gz ]; then
|
||||
sg incus-admin -c "incus --quiet image import /tmp/$alias.tar.gz --alias $alias" || true
|
||||
fi
|
||||
done
|
||||
|
||||
- name: cmdeploy lxc-test
|
||||
run: sg incus-admin -c 'cmdeploy lxc-test -vv'
|
||||
|
||||
- name: show container logs on failure
|
||||
if: failure() || cancelled()
|
||||
run: |
|
||||
for c in test0-localchat test1-localchat; do
|
||||
echo "::group::$c journal (warnings+errors)"
|
||||
sg incus-admin -c "incus exec $c -- journalctl -p warning --no-pager -n200" 2>/dev/null || echo "no log"
|
||||
echo "::endgroup::"
|
||||
echo "::group::$c failed services"
|
||||
sg incus-admin -c "incus exec $c -- systemctl --no-pager --failed" || true
|
||||
echo "::endgroup::"
|
||||
done
|
||||
|
||||
- name: export images for cache
|
||||
if: always()
|
||||
run: |
|
||||
for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
|
||||
sg incus-admin -c "incus --quiet image export $alias /tmp/$alias" || true
|
||||
done
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
;; Zone file for staging-ipv4.testrun.org
|
||||
|
||||
$ORIGIN staging-ipv4.testrun.org.
|
||||
$TTL 300
|
||||
|
||||
@ IN SOA ns.testrun.org. root.nine.testrun.org (
|
||||
2023010101 ; Serial
|
||||
7200 ; Refresh
|
||||
3600 ; Retry
|
||||
1209600 ; Expire
|
||||
3600 ; Negative response caching TTL
|
||||
)
|
||||
|
||||
;; Nameservers.
|
||||
@ IN NS ns.testrun.org.
|
||||
|
||||
;; DNS records.
|
||||
@ IN A 37.27.95.249
|
||||
mta-sts.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org.
|
||||
www.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org.
|
||||
@@ -1,21 +0,0 @@
|
||||
;; Zone file for staging2.testrun.org
|
||||
|
||||
$ORIGIN staging2.testrun.org.
|
||||
$TTL 300
|
||||
|
||||
@ IN SOA ns.testrun.org. root.nine.testrun.org (
|
||||
2023010101 ; Serial
|
||||
7200 ; Refresh
|
||||
3600 ; Retry
|
||||
1209600 ; Expire
|
||||
3600 ; Negative response caching TTL
|
||||
)
|
||||
|
||||
;; Nameservers.
|
||||
@ IN NS ns.testrun.org.
|
||||
|
||||
;; DNS records.
|
||||
@ IN A 37.27.24.139
|
||||
mta-sts.staging2.testrun.org. CNAME staging2.testrun.org.
|
||||
www.staging2.testrun.org. CNAME staging2.testrun.org.
|
||||
|
||||
296
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
296
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
@@ -1,296 +0,0 @@
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
name: Build Docker image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
outputs:
|
||||
image: ${{ steps.image-ref.outputs.image }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
# Tagged releases: v1.2.3 -> :1.2.3, :1.2, :latest
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
# Branch pushes: foo/docker-pr -> :foo-docker-pr
|
||||
type=ref,event=branch
|
||||
# Always: :sha-<hash>
|
||||
type=sha
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: docker/chatmail_relay.dockerfile
|
||||
push: ${{ github.event_name == 'push' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
GIT_HASH=${{ github.sha }}
|
||||
|
||||
- name: Output image reference
|
||||
id: image-ref
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
||||
echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
deploy:
|
||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging-ipv4.testrun.org
|
||||
url: https://staging-ipv4.testrun.org/
|
||||
concurrency: staging-ipv4.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
|
||||
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
id: wait-for-vps
|
||||
run: |
|
||||
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging2.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- name: setup dependencies
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org apt update
|
||||
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
|
||||
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
|
||||
|
||||
- name: initialize config
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
|
||||
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
|
||||
|
||||
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
|
||||
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
||||
|
||||
# --- Docker deploy (push only, runs even if bare failed) ---
|
||||
|
||||
- name: stop bare services
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org 'systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true'
|
||||
|
||||
- name: install Docker on VPS
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org 'apt-get update && apt-get install -y ca-certificates curl'
|
||||
ssh root@staging-ipv4.testrun.org 'install -m 0755 -d /etc/apt/keyrings'
|
||||
ssh root@staging-ipv4.testrun.org 'curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc'
|
||||
ssh root@staging-ipv4.testrun.org 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list'
|
||||
ssh root@staging-ipv4.testrun.org 'apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin'
|
||||
|
||||
- name: prepare Docker bind mounts
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org 'mkdir -p /srv/chatmail/certs /srv/chatmail/dkim'
|
||||
ssh root@staging-ipv4.testrun.org 'cp -a /var/lib/acme/. /srv/chatmail/certs/ && cp -a /etc/dkimkeys/. /srv/chatmail/dkim/' || true
|
||||
|
||||
- name: upload chatmail.ini for Docker
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Reuse chatmail.ini already created by the bare-metal deploy steps
|
||||
ssh root@staging-ipv4.testrun.org "cp relay/chatmail.ini /srv/chatmail/chatmail.ini"
|
||||
|
||||
- name: deploy with Docker
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
GHCR_IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ root@staging-ipv4.testrun.org:/srv/chatmail/relay/
|
||||
# Login to GHCR on VPS and pull pre-built image
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh root@staging-ipv4.testrun.org 'docker login ghcr.io -u ${{ github.actor }} --password-stdin'
|
||||
ssh root@staging-ipv4.testrun.org "docker pull ${GHCR_IMAGE}"
|
||||
ssh root@staging-ipv4.testrun.org "cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=staging-ipv4.testrun.org docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
||||
|
||||
- name: wait for container healthy
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Stream journald inside the container
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail journalctl -f --no-pager' &
|
||||
LOG_PID=$!
|
||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
||||
for i in $(seq 1 60); do
|
||||
status=$(ssh root@staging-ipv4.testrun.org 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
||||
echo " [$i/60] status=$status"
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "Container is healthy."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$status" = "unhealthy" ]; then
|
||||
echo "Container is unhealthy!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Container did not become healthy."
|
||||
kill $LOG_PID 2>/dev/null || true
|
||||
echo "--- failed units ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail systemctl --failed --no-pager' || true
|
||||
echo "--- service logs ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50' || true
|
||||
echo "--- listening ports ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail ss -tlnp' || true
|
||||
echo "--- chatmail.ini ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini' || true
|
||||
exit 1
|
||||
|
||||
- name: show container state
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
echo "--- listening ports ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail ss -tlnp'
|
||||
echo "--- chatmail.ini ---"
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini'
|
||||
|
||||
- name: Docker integration tests
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
||||
|
||||
- name: Docker DNS
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Reset zone file in case bare DNS already appended to it
|
||||
git checkout .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys'
|
||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose'
|
||||
ssh root@staging-ipv4.testrun.org 'docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone'
|
||||
scp root@staging-ipv4.testrun.org:/tmp/staging.zone staging-generated.zone
|
||||
cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: Docker final DNS check
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
||||
|
||||
# --- Cleanup ---
|
||||
|
||||
- name: add SSH keys
|
||||
if: >-
|
||||
!cancelled()
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: ssh root@staging-ipv4.testrun.org 'curl -s https://github.com/hpk42.keys https://github.com/j4n.keys >> .ssh/authorized_keys'
|
||||
289
.github/workflows/test-and-deploy.yaml
vendored
289
.github/workflows/test-and-deploy.yaml
vendored
@@ -1,289 +0,0 @@
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- j4n/docker-pr
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'scripts/**'
|
||||
- '**/README.md'
|
||||
- 'CHANGELOG.md'
|
||||
- 'LICENSE'
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
name: Build Docker image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
outputs:
|
||||
image: ${{ steps.image-ref.outputs.image }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
# Tagged releases: v1.2.3 -> :1.2.3, :1.2, :latest
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
# Branch pushes: foo/docker-pr -> :foo-docker-pr
|
||||
type=ref,event=branch
|
||||
# Always: :sha-<hash>
|
||||
type=sha
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: docker/chatmail_relay.dockerfile
|
||||
push: ${{ github.event_name == 'push' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
GIT_HASH=${{ github.sha }}
|
||||
|
||||
- name: Output image reference
|
||||
id: image-ref
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
||||
echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
deploy:
|
||||
name: deploy on staging2.testrun.org, and run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment:
|
||||
name: staging2.testrun.org
|
||||
url: https://staging2.testrun.org/
|
||||
concurrency: staging2.testrun.org
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: prepare SSH
|
||||
run: |
|
||||
mkdir ~/.ssh
|
||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
|
||||
# save previous acme & dkim state
|
||||
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
|
||||
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
|
||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
||||
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
|
||||
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
|
||||
# make sure CAA record isn't set
|
||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: rebuild staging2.testrun.org to have a clean VPS
|
||||
run: |
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"image":"debian-12"}' \
|
||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
|
||||
|
||||
- run: scripts/initenv.sh
|
||||
|
||||
- name: append venv/bin to PATH
|
||||
run: echo venv/bin >>$GITHUB_PATH
|
||||
|
||||
- name: upload TLS cert after rebuilding
|
||||
id: wait-for-vps
|
||||
run: |
|
||||
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
|
||||
rm ~/.ssh/known_hosts
|
||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
|
||||
# download acme & dkim state from ns.testrun.org
|
||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
|
||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
|
||||
# restore acme & dkim state to staging2.testrun.org
|
||||
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
|
||||
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
|
||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
|
||||
|
||||
- name: add hpk42 key to staging server
|
||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
|
||||
|
||||
- name: run deploy-chatmail offline tests
|
||||
run: pytest --pyargs cmdeploy
|
||||
|
||||
- run: |
|
||||
cmdeploy init staging2.testrun.org
|
||||
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
|
||||
|
||||
- run: cmdeploy run --verbose --skip-dns-check
|
||||
|
||||
- name: set DNS entries
|
||||
run: |
|
||||
cmdeploy dns --zonefile staging-generated.zone --verbose
|
||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
||||
cat .github/workflows/staging.testrun.org-default.zone
|
||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: cmdeploy test
|
||||
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
|
||||
|
||||
- name: cmdeploy dns
|
||||
run: cmdeploy dns -v
|
||||
|
||||
# --- Docker deploy (push only, runs even if bare failed) ---
|
||||
|
||||
- name: stop bare services
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging2.testrun.org 'systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true'
|
||||
|
||||
- name: install Docker on VPS
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging2.testrun.org 'apt-get update && apt-get install -y ca-certificates curl'
|
||||
ssh root@staging2.testrun.org 'install -m 0755 -d /etc/apt/keyrings'
|
||||
ssh root@staging2.testrun.org 'curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc'
|
||||
ssh root@staging2.testrun.org 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list'
|
||||
ssh root@staging2.testrun.org 'apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin'
|
||||
|
||||
- name: prepare Docker bind mounts
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging2.testrun.org 'mkdir -p /srv/chatmail/certs /srv/chatmail/dkim'
|
||||
ssh root@staging2.testrun.org 'cp -a /var/lib/acme/. /srv/chatmail/certs/ && cp -a /etc/dkimkeys/. /srv/chatmail/dkim/' || true
|
||||
|
||||
- name: upload chatmail.ini for Docker
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Reuse chatmail.ini already created by the bare-metal deploy steps
|
||||
scp chatmail.ini root@staging2.testrun.org:/srv/chatmail/chatmail.ini
|
||||
|
||||
- name: deploy with Docker
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
GHCR_IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ root@staging2.testrun.org:/srv/chatmail/relay/
|
||||
# Login to GHCR on VPS and pull pre-built image
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh root@staging2.testrun.org 'docker login ghcr.io -u ${{ github.actor }} --password-stdin'
|
||||
ssh root@staging2.testrun.org "docker pull ${GHCR_IMAGE}"
|
||||
ssh root@staging2.testrun.org "cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=staging2.testrun.org docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
||||
|
||||
- name: wait for container healthy
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Stream journald inside the container
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail journalctl -f --no-pager' &
|
||||
LOG_PID=$!
|
||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
||||
for i in $(seq 1 60); do
|
||||
status=$(ssh root@staging2.testrun.org 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
||||
echo " [$i/60] status=$status"
|
||||
if [ "$status" = "healthy" ]; then
|
||||
echo "Container is healthy."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$status" = "unhealthy" ]; then
|
||||
echo "Container is unhealthy!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "Container did not become healthy."
|
||||
kill $LOG_PID 2>/dev/null || true
|
||||
echo "--- failed units ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail systemctl --failed --no-pager' || true
|
||||
echo "--- service logs ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50' || true
|
||||
echo "--- listening ports ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail ss -tlnp' || true
|
||||
echo "--- chatmail.ini ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini' || true
|
||||
exit 1
|
||||
|
||||
- name: show container state
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
echo "--- listening ports ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail ss -tlnp'
|
||||
echo "--- chatmail.ini ---"
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini'
|
||||
|
||||
- name: Docker integration tests
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
||||
|
||||
- name: Docker DNS
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: |
|
||||
# Reset zone file in case bare DNS already appended to it
|
||||
git checkout .github/workflows/staging.testrun.org-default.zone
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys'
|
||||
ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose'
|
||||
ssh root@staging2.testrun.org 'docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone'
|
||||
scp root@staging2.testrun.org:/tmp/staging.zone staging-generated.zone
|
||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
||||
cat .github/workflows/staging.testrun.org-default.zone
|
||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
||||
ssh root@ns.testrun.org systemctl reload nsd
|
||||
|
||||
- name: Docker final DNS check
|
||||
if: >-
|
||||
!cancelled() && github.event_name == 'push'
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
||||
|
||||
# --- Cleanup ---
|
||||
|
||||
- name: add SSH keys
|
||||
if: >-
|
||||
!cancelled()
|
||||
&& steps.wait-for-vps.outcome == 'success'
|
||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys https://github.com/j4n.keys >> .ssh/authorized_keys'
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -5,6 +5,7 @@ __pycache__/
|
||||
*.swp
|
||||
*qr-*.png
|
||||
chatmail*.ini
|
||||
lxconfigs/
|
||||
|
||||
|
||||
# C extensions
|
||||
@@ -164,9 +165,3 @@ cython_debug/
|
||||
#.idea/
|
||||
|
||||
chatmail.zone
|
||||
|
||||
# docker
|
||||
/data/
|
||||
/custom/
|
||||
docker/docker-compose.override.yaml
|
||||
docker/.env
|
||||
|
||||
@@ -85,13 +85,13 @@ def mockout():
|
||||
captured_green = []
|
||||
captured_plain = []
|
||||
|
||||
def red(self, msg):
|
||||
def red(self, msg, **kw):
|
||||
self.captured_red.append(msg)
|
||||
|
||||
def green(self, msg):
|
||||
def green(self, msg, **kw):
|
||||
self.captured_green.append(msg)
|
||||
|
||||
def __call__(self, msg):
|
||||
def print(self, msg="", **kw):
|
||||
self.captured_plain.append(msg)
|
||||
|
||||
return MockOut()
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
;
|
||||
; Required DNS entries for chatmail servers
|
||||
;
|
||||
{% if A %}
|
||||
{{ mail_domain }}. A {{ A }}
|
||||
{% endif %}
|
||||
{% if AAAA %}
|
||||
{{ mail_domain }}. AAAA {{ AAAA }}
|
||||
{% endif %}
|
||||
{{ mail_domain }}. MX 10 {{ mail_domain }}.
|
||||
{% if strict_tls %}
|
||||
_mta-sts.{{ mail_domain }}. TXT "v=STSv1; id={{ sts_id }}"
|
||||
mta-sts.{{ mail_domain }}. CNAME {{ mail_domain }}.
|
||||
{% endif %}
|
||||
www.{{ mail_domain }}. CNAME {{ mail_domain }}.
|
||||
{{ dkim_entry }}
|
||||
|
||||
;
|
||||
; Recommended DNS entries for interoperability and security-hardening
|
||||
;
|
||||
{{ mail_domain }}. TXT "v=spf1 a ~all"
|
||||
_dmarc.{{ mail_domain }}. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
||||
|
||||
{% if acme_account_url %}
|
||||
{{ mail_domain }}. CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
|
||||
{% endif %}
|
||||
_adsp._domainkey.{{ mail_domain }}. TXT "dkim=discardable"
|
||||
|
||||
_submission._tcp.{{ mail_domain }}. SRV 0 1 587 {{ mail_domain }}.
|
||||
_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
|
||||
_imap._tcp.{{ mail_domain }}. SRV 0 1 143 {{ mail_domain }}.
|
||||
_imaps._tcp.{{ mail_domain }}. SRV 0 1 993 {{ mail_domain }}.
|
||||
@@ -15,10 +15,27 @@ from pathlib import Path
|
||||
import pyinfra
|
||||
from chatmaild.config import read_config, write_initial_config
|
||||
from packaging import version
|
||||
from termcolor import colored
|
||||
|
||||
from . import dns, remote
|
||||
from .sshexec import LocalExec, SSHExec
|
||||
from .lxc.cli import (
|
||||
lxc_start_cmd,
|
||||
lxc_start_cmd_options,
|
||||
lxc_status_cmd,
|
||||
lxc_status_cmd_options,
|
||||
lxc_stop_cmd,
|
||||
lxc_stop_cmd_options,
|
||||
lxc_test_cmd,
|
||||
lxc_test_cmd_options,
|
||||
)
|
||||
from .lxc.incus import DNSConfigurationError
|
||||
from .sshexec import (
|
||||
LocalExec,
|
||||
SSHExec,
|
||||
resolve_host_from_ssh_config,
|
||||
resolve_key_from_ssh_config,
|
||||
)
|
||||
from .util import Out
|
||||
from .www import main as webdev_main
|
||||
|
||||
#
|
||||
# cmdeploy sub commands and options
|
||||
@@ -82,18 +99,21 @@ def run_cmd_options(parser):
|
||||
help="disable checks nslookup for dns",
|
||||
)
|
||||
add_ssh_host_option(parser)
|
||||
add_ssh_config_option(parser)
|
||||
|
||||
|
||||
def run_cmd(args, out):
|
||||
"""Deploy chatmail services on the remote server."""
|
||||
|
||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||
sshexec = get_sshexec(ssh_host)
|
||||
sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
|
||||
require_iroh = args.config.enable_iroh_relay
|
||||
strict_tls = args.config.tls_cert_mode == "acme"
|
||||
if not args.dns_check_disabled:
|
||||
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
||||
if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
|
||||
if not dns.check_initial_remote_data(
|
||||
remote_data, strict_tls=strict_tls, print=out.red
|
||||
):
|
||||
return 1
|
||||
|
||||
env = os.environ.copy()
|
||||
@@ -104,11 +124,28 @@ def run_cmd(args, out):
|
||||
if not args.dns_check_disabled:
|
||||
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
|
||||
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
|
||||
env["DEBIAN_FRONTEND"] = "noninteractive"
|
||||
env["TERM"] = "linux"
|
||||
deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
|
||||
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
||||
|
||||
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
|
||||
if ssh_host == "localhost":
|
||||
ssh_config = args.ssh_config
|
||||
if ssh_config:
|
||||
ssh_config = str(Path(ssh_config).resolve())
|
||||
|
||||
# Use pyinfra's native SSH data keys to configure the connection directly
|
||||
# rather than relying on paramiko config parsing (see also sshexec.py)
|
||||
ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
|
||||
key = resolve_key_from_ssh_config(ssh_host, ssh_config)
|
||||
data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
|
||||
if key:
|
||||
data_args += f" --data ssh_key={key}"
|
||||
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
|
||||
if ssh_host in ["localhost", "@docker"]:
|
||||
if ssh_host == "@docker":
|
||||
env["CHATMAIL_NOPORTCHECK"] = "True"
|
||||
env["CHATMAIL_NOSYSCTL"] = "True"
|
||||
cmd = f"{pyinf} @local {deploy_path} -y"
|
||||
|
||||
if version.parse(pyinfra.__version__) < version.parse("3"):
|
||||
@@ -116,10 +153,17 @@ def run_cmd(args, out):
|
||||
return 1
|
||||
|
||||
try:
|
||||
out.check_call(cmd, env=env)
|
||||
ret = out.shell(cmd, env=env)
|
||||
if ret:
|
||||
out.red("Deploy failed")
|
||||
return 1
|
||||
if args.website_only:
|
||||
out.green("Website deployment completed.")
|
||||
elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
|
||||
elif (
|
||||
not args.dns_check_disabled
|
||||
and strict_tls
|
||||
and not remote_data["acme_account_url"]
|
||||
):
|
||||
out.red("Deploy completed but letsencrypt not configured")
|
||||
out.red("Run 'cmdeploy run' again")
|
||||
else:
|
||||
@@ -136,15 +180,16 @@ def dns_cmd_options(parser):
|
||||
dest="zonefile",
|
||||
type=pathlib.Path,
|
||||
default=None,
|
||||
help="write out a zonefile",
|
||||
help="write DNS records in standard BIND format to the given file",
|
||||
)
|
||||
add_ssh_host_option(parser)
|
||||
add_ssh_config_option(parser)
|
||||
|
||||
|
||||
def dns_cmd(args, out):
|
||||
"""Check DNS entries and optionally generate dns zone file."""
|
||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
|
||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
|
||||
tls_cert_mode = args.config.tls_cert_mode
|
||||
strict_tls = tls_cert_mode == "acme"
|
||||
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
||||
@@ -175,13 +220,14 @@ def dns_cmd(args, out):
|
||||
|
||||
def status_cmd_options(parser):
|
||||
add_ssh_host_option(parser)
|
||||
add_ssh_config_option(parser)
|
||||
|
||||
|
||||
def status_cmd(args, out):
|
||||
"""Display status for online chatmail instance."""
|
||||
|
||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
|
||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
|
||||
|
||||
out.green(f"chatmail domain: {args.config.mail_domain}")
|
||||
if args.config.privacy_mail:
|
||||
@@ -201,28 +247,32 @@ def test_cmd_options(parser):
|
||||
help="also run slow tests",
|
||||
)
|
||||
add_ssh_host_option(parser)
|
||||
add_ssh_config_option(parser)
|
||||
|
||||
|
||||
def test_cmd(args, out):
|
||||
"""Run local and online tests for chatmail deployment."""
|
||||
|
||||
env = os.environ.copy()
|
||||
if args.ssh_host:
|
||||
env["CHATMAIL_SSH"] = args.ssh_host
|
||||
env["CHATMAIL_INI"] = str(args.inipath.resolve())
|
||||
|
||||
pytest_path = shutil.which("pytest")
|
||||
pytest_args = [
|
||||
pytest_path,
|
||||
"cmdeploy/src/",
|
||||
"-n4",
|
||||
"-rs",
|
||||
"-x",
|
||||
"-v",
|
||||
"-vv" if args.verbose > 1 else "-v",
|
||||
"-s",
|
||||
"--durations=5",
|
||||
]
|
||||
if args.slow:
|
||||
pytest_args.append("--slow")
|
||||
ret = out.run_ret(pytest_args, env=env)
|
||||
if args.ssh_host:
|
||||
pytest_args.extend(["--ssh-host", args.ssh_host])
|
||||
if args.ssh_config:
|
||||
pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
|
||||
ret = out.shell(" ".join(pytest_args), env=env)
|
||||
return ret
|
||||
|
||||
|
||||
@@ -259,8 +309,8 @@ def fmt_cmd(args, out):
|
||||
format_args.extend(sources)
|
||||
check_args.extend(sources)
|
||||
|
||||
out.check_call(" ".join(format_args), quiet=not args.verbose)
|
||||
out.check_call(" ".join(check_args), quiet=not args.verbose)
|
||||
out.shell(" ".join(format_args), quiet=not args.verbose)
|
||||
out.shell(" ".join(check_args), quiet=not args.verbose)
|
||||
|
||||
|
||||
def bench_cmd(args, out):
|
||||
@@ -273,9 +323,7 @@ def bench_cmd(args, out):
|
||||
|
||||
def webdev_cmd(args, out):
|
||||
"""Run local web development loop for static web pages."""
|
||||
from .www import main
|
||||
|
||||
main()
|
||||
webdev_main()
|
||||
|
||||
|
||||
#
|
||||
@@ -283,41 +331,25 @@ def webdev_cmd(args, out):
|
||||
#
|
||||
|
||||
|
||||
class Out:
|
||||
"""Convenience output printer providing coloring."""
|
||||
|
||||
def red(self, msg, file=sys.stderr):
|
||||
print(colored(msg, "red"), file=file)
|
||||
|
||||
def green(self, msg, file=sys.stderr):
|
||||
print(colored(msg, "green"), file=file)
|
||||
|
||||
def __call__(self, msg, red=False, green=False, file=sys.stdout):
|
||||
color = "red" if red else ("green" if green else None)
|
||||
print(colored(msg, color), file=file)
|
||||
|
||||
def check_call(self, arg, env=None, quiet=False):
|
||||
if not quiet:
|
||||
self(f"[$ {arg}]", file=sys.stderr)
|
||||
return subprocess.check_call(arg, shell=True, env=env)
|
||||
|
||||
def run_ret(self, args, env=None, quiet=False):
|
||||
if not quiet:
|
||||
cmdstring = " ".join(args)
|
||||
self(f"[$ {cmdstring}]", file=sys.stderr)
|
||||
proc = subprocess.run(args, env=env, check=False)
|
||||
return proc.returncode
|
||||
|
||||
|
||||
def add_ssh_host_option(parser):
|
||||
parser.add_argument(
|
||||
"--ssh-host",
|
||||
dest="ssh_host",
|
||||
help="Run commands on 'localhost' or on a specific SSH host "
|
||||
help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
|
||||
"instead of chatmail.ini's mail_domain.",
|
||||
)
|
||||
|
||||
|
||||
def add_ssh_config_option(parser):
|
||||
parser.add_argument(
|
||||
"--ssh-config",
|
||||
dest="ssh_config",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
|
||||
)
|
||||
|
||||
|
||||
def add_config_option(parser):
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
@@ -327,25 +359,26 @@ def add_config_option(parser):
|
||||
type=Path,
|
||||
help="path to the chatmail.ini file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
"-v",
|
||||
dest="verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="provide verbose logging",
|
||||
)
|
||||
|
||||
|
||||
def add_subcommand(subparsers, func):
|
||||
def add_subcommand(subparsers, func, add_config=True):
|
||||
name = func.__name__
|
||||
assert name.endswith("_cmd")
|
||||
name = name[:-4]
|
||||
name = name[:-4].replace("_", "-")
|
||||
doc = func.__doc__.strip()
|
||||
help = doc.split("\n")[0].strip(".")
|
||||
p = subparsers.add_parser(name, description=doc, help=help)
|
||||
p.set_defaults(func=func)
|
||||
add_config_option(p)
|
||||
if add_config:
|
||||
add_config_option(p)
|
||||
p.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
dest="verbose",
|
||||
action="count",
|
||||
default=0,
|
||||
help="increase verbosity (can be repeated: -v, -vv)",
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
@@ -354,43 +387,60 @@ Setup your chatmail server configuration and
|
||||
deploy it via SSH to your remote location.
|
||||
"""
|
||||
|
||||
# Explicit subcommand registry: (cmd_func, options_func_or_None, needs_config).
|
||||
# LXC commands don't need a chatmail.ini (no config); all others do.
|
||||
SUBCOMMANDS = [
|
||||
(init_cmd, init_cmd_options, True),
|
||||
(run_cmd, run_cmd_options, True),
|
||||
(dns_cmd, dns_cmd_options, True),
|
||||
(status_cmd, status_cmd_options, True),
|
||||
(test_cmd, test_cmd_options, True),
|
||||
(fmt_cmd, fmt_cmd_options, True),
|
||||
(bench_cmd, None, True),
|
||||
(webdev_cmd, None, True),
|
||||
(lxc_start_cmd, lxc_start_cmd_options, False),
|
||||
(lxc_stop_cmd, lxc_stop_cmd_options, False),
|
||||
(lxc_status_cmd, lxc_status_cmd_options, False),
|
||||
(lxc_test_cmd, lxc_test_cmd_options, False),
|
||||
]
|
||||
|
||||
|
||||
def get_parser():
|
||||
"""Return an ArgumentParser for the 'cmdeploy' CLI"""
|
||||
|
||||
parser = argparse.ArgumentParser(description=description.strip())
|
||||
parser.set_defaults(func=None, inipath=None)
|
||||
subparsers = parser.add_subparsers(title="subcommands")
|
||||
|
||||
# find all subcommands in the module namespace
|
||||
glob = globals()
|
||||
for name, func in glob.items():
|
||||
if name.endswith("_cmd"):
|
||||
subparser = add_subcommand(subparsers, func)
|
||||
addopts = glob.get(name + "_options")
|
||||
if addopts is not None:
|
||||
addopts(subparser)
|
||||
for func, addopts, needs_config in SUBCOMMANDS:
|
||||
subparser = add_subcommand(subparsers, func, add_config=needs_config)
|
||||
if addopts is not None:
|
||||
addopts(subparser)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def get_sshexec(ssh_host: str, verbose=True):
|
||||
def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
|
||||
if ssh_host in ["localhost", "@local"]:
|
||||
return LocalExec(verbose)
|
||||
return LocalExec(verbose, docker=False)
|
||||
elif ssh_host == "@docker":
|
||||
return LocalExec(verbose, docker=True)
|
||||
if verbose:
|
||||
print(f"[ssh] login to {ssh_host}")
|
||||
return SSHExec(ssh_host, verbose=verbose)
|
||||
return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
|
||||
|
||||
|
||||
def main(args=None):
|
||||
"""Provide main entry point for 'cmdeploy' CLI invocation."""
|
||||
parser = get_parser()
|
||||
args = parser.parse_args(args=args)
|
||||
if not hasattr(args, "func"):
|
||||
if args.func is None:
|
||||
return parser.parse_args(["-h"])
|
||||
|
||||
out = Out()
|
||||
out = Out(verbosity=args.verbose)
|
||||
kwargs = {}
|
||||
if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
|
||||
|
||||
if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
|
||||
if not args.inipath.exists():
|
||||
out.red(f"expecting {args.inipath} to exist, run init first?")
|
||||
raise SystemExit(1)
|
||||
@@ -405,6 +455,9 @@ def main(args=None):
|
||||
if res is None:
|
||||
res = 0
|
||||
return res
|
||||
except DNSConfigurationError as exc:
|
||||
out.red(str(exc))
|
||||
return 1
|
||||
except KeyboardInterrupt:
|
||||
out.red("KeyboardInterrupt")
|
||||
sys.exit(130)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
Chat Mail pyinfra deploy.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -13,17 +14,15 @@ from pyinfra import facts, host, logger
|
||||
from pyinfra.api import FactBase
|
||||
from pyinfra.facts import hardware
|
||||
from pyinfra.facts.files import Sha256File
|
||||
from pyinfra.facts.server import Command
|
||||
from pyinfra.facts.systemd import SystemdEnabled
|
||||
from pyinfra.operations import apt, files, pip, server, systemd
|
||||
|
||||
from cmdeploy.cmdeploy import Out
|
||||
|
||||
from .acmetool import AcmetoolDeployer
|
||||
from .basedeploy import (
|
||||
Deployer,
|
||||
Deployment,
|
||||
activate_remote_units,
|
||||
blocked_service_startup,
|
||||
configure_remote_units,
|
||||
get_resource,
|
||||
has_systemd,
|
||||
@@ -36,6 +35,7 @@ from .nginx.deployer import NginxDeployer
|
||||
from .opendkim.deployer import OpendkimDeployer
|
||||
from .postfix.deployer import PostfixDeployer
|
||||
from .selfsigned.deployer import SelfSignedTlsDeployer
|
||||
from .util import Out, get_version_string
|
||||
from .www import build_webpages, find_merge_conflict, get_paths
|
||||
|
||||
|
||||
@@ -149,33 +149,16 @@ class UnboundDeployer(Deployer):
|
||||
self.need_restart = False
|
||||
|
||||
def install(self):
|
||||
# Run local DNS resolver `unbound`.
|
||||
# `resolvconf` takes care of setting up /etc/resolv.conf
|
||||
# to use 127.0.0.1 as the resolver.
|
||||
# Run local DNS resolver `unbound`. `resolvconf` takes care of
|
||||
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
|
||||
|
||||
#
|
||||
# On an IPv4-only system, if unbound is started but not
|
||||
# configured, it causes subsequent steps to fail to resolve hosts.
|
||||
# Here, we use policy-rc.d to prevent unbound from starting up
|
||||
# on initial install. Later, we will configure it and start it.
|
||||
#
|
||||
# For documentation about policy-rc.d, see:
|
||||
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
|
||||
#
|
||||
files.put(
|
||||
src=get_resource("policy-rc.d"),
|
||||
dest="/usr/sbin/policy-rc.d",
|
||||
user="root",
|
||||
group="root",
|
||||
mode="755",
|
||||
)
|
||||
|
||||
apt.packages(
|
||||
name="Install unbound",
|
||||
packages=["unbound", "unbound-anchor", "dnsutils"],
|
||||
)
|
||||
|
||||
files.file("/usr/sbin/policy-rc.d", present=False)
|
||||
# On an IPv4-only system, if unbound is started but not configured,
|
||||
# it causes subsequent steps to fail to resolve hosts.
|
||||
with blocked_service_startup():
|
||||
apt.packages(
|
||||
name="Install unbound",
|
||||
packages=["unbound", "unbound-anchor", "dnsutils"],
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
server.shell(
|
||||
@@ -271,8 +254,14 @@ class WebsiteDeployer(Deployer):
|
||||
logger.warning("Web page build failed, skipping website deployment")
|
||||
return
|
||||
# if it is not a hugo page, upload it as is
|
||||
files.rsync(
|
||||
f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
|
||||
# pyinfra files.rsync (experimental) causes problems with ssh-config configuration
|
||||
# the stable files.sync should do
|
||||
files.sync(
|
||||
src=str(www_path),
|
||||
dest="/var/www/html",
|
||||
user="www-data",
|
||||
group="www-data",
|
||||
delete=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -336,12 +325,12 @@ class TurnDeployer(Deployer):
|
||||
def install(self):
|
||||
(url, sha256sum) = {
|
||||
"x86_64": (
|
||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
|
||||
"841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
|
||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
|
||||
"1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
|
||||
),
|
||||
"aarch64": (
|
||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
|
||||
"a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
|
||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
|
||||
"0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
|
||||
),
|
||||
}[host.get_fact(facts.server.Arch)]
|
||||
|
||||
@@ -474,8 +463,9 @@ class ChatmailDeployer(Deployer):
|
||||
("iroh", None, None),
|
||||
]
|
||||
|
||||
def __init__(self, mail_domain):
|
||||
self.mail_domain = mail_domain
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.mail_domain = config.mail_domain
|
||||
|
||||
def install(self):
|
||||
files.put(
|
||||
@@ -504,6 +494,17 @@ class ChatmailDeployer(Deployer):
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
# Ensure the per-domain mailbox directory exists before
|
||||
# chatmail-metadata starts (it crashes without it).
|
||||
files.directory(
|
||||
name="Ensure vmail mailbox directory exists",
|
||||
path=f"/home/vmail/mail/{self.mail_domain}",
|
||||
user="vmail",
|
||||
group="vmail",
|
||||
mode="700",
|
||||
present=True,
|
||||
)
|
||||
|
||||
# This file is used by auth proxy.
|
||||
# https://wiki.debian.org/EtcMailName
|
||||
server.shell(
|
||||
@@ -513,6 +514,15 @@ class ChatmailDeployer(Deployer):
|
||||
],
|
||||
)
|
||||
|
||||
files.directory(
|
||||
name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
|
||||
path=str(self.config.mailboxes_dir),
|
||||
user="vmail",
|
||||
group="vmail",
|
||||
mode="700",
|
||||
present=True,
|
||||
)
|
||||
|
||||
|
||||
class FcgiwrapDeployer(Deployer):
|
||||
def install(self):
|
||||
@@ -532,17 +542,9 @@ class FcgiwrapDeployer(Deployer):
|
||||
|
||||
class GithashDeployer(Deployer):
|
||||
def activate(self):
|
||||
try:
|
||||
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
|
||||
except Exception:
|
||||
git_hash = "unknown\n"
|
||||
try:
|
||||
git_diff = subprocess.check_output(["git", "diff"]).decode()
|
||||
except Exception:
|
||||
git_diff = ""
|
||||
files.put(
|
||||
name="Upload chatmail relay git commit hash",
|
||||
src=StringIO(git_hash + git_diff),
|
||||
src=StringIO(get_version_string()),
|
||||
dest="/etc/chatmail-version",
|
||||
mode="700",
|
||||
)
|
||||
@@ -586,14 +588,20 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
||||
)
|
||||
|
||||
# Check if mtail_address interface is available (if configured)
|
||||
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
||||
if config.mtail_address and config.mtail_address not in (
|
||||
"127.0.0.1",
|
||||
"::1",
|
||||
"localhost",
|
||||
):
|
||||
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
||||
all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
|
||||
if config.mtail_address not in all_addresses:
|
||||
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
|
||||
Out().red(
|
||||
f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
|
||||
)
|
||||
exit(1)
|
||||
|
||||
if host.get_fact(Command, "systemd-detect-virt -c || true") == "none":
|
||||
if not os.environ.get("CHATMAIL_NOPORTCHECK"):
|
||||
port_services = [
|
||||
(["master", "smtpd"], 25),
|
||||
("unbound", 53),
|
||||
@@ -633,7 +641,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
||||
tls_deployer = get_tls_deployer(config, mail_domain)
|
||||
|
||||
all_deployers = [
|
||||
ChatmailDeployer(mail_domain),
|
||||
ChatmailDeployer(config),
|
||||
LegacyRemoveDeployer(),
|
||||
FiltermailDeployer(),
|
||||
JournaldDeployer(),
|
||||
|
||||
@@ -1,11 +1,26 @@
|
||||
import datetime
|
||||
import importlib
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
from . import remote
|
||||
|
||||
|
||||
def parse_zone_records(text):
|
||||
"""Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
|
||||
|
||||
Skips comment lines (starting with ``;``) and blank lines.
|
||||
Each record line must have the format ``name TTL IN type rdata``.
|
||||
"""
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith(";"):
|
||||
continue
|
||||
try:
|
||||
name, ttl, _in, rtype, rdata = line.split(None, 4)
|
||||
except ValueError:
|
||||
raise ValueError(f"Bad zone record line: {line!r}") from None
|
||||
name = name.rstrip(".")
|
||||
yield name, ttl, rtype.upper(), rdata
|
||||
|
||||
|
||||
def get_initial_remote_data(sshexec, mail_domain):
|
||||
return sshexec.logged(
|
||||
call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
|
||||
@@ -31,13 +46,36 @@ def get_filled_zone_file(remote_data):
|
||||
if not sts_id:
|
||||
remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
|
||||
|
||||
template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
|
||||
content = template.read_text()
|
||||
zonefile = Template(content).render(**remote_data)
|
||||
lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
|
||||
d = remote_data["mail_domain"]
|
||||
lines = ["; Required DNS entries"]
|
||||
if remote_data.get("A"):
|
||||
lines.append(f"{d}. 3600 IN A {remote_data['A']}")
|
||||
if remote_data.get("AAAA"):
|
||||
lines.append(f"{d}. 3600 IN AAAA {remote_data['AAAA']}")
|
||||
lines.append(f"{d}. 3600 IN MX 10 {d}.")
|
||||
if remote_data.get("strict_tls"):
|
||||
lines.append(
|
||||
f'_mta-sts.{d}. 3600 IN TXT "v=STSv1; id={remote_data["sts_id"]}"'
|
||||
)
|
||||
lines.append(f"mta-sts.{d}. 3600 IN CNAME {d}.")
|
||||
lines.append(f"www.{d}. 3600 IN CNAME {d}.")
|
||||
lines.append(remote_data["dkim_entry"])
|
||||
lines.append("")
|
||||
zonefile = "\n".join(lines)
|
||||
return zonefile
|
||||
lines.append("; Recommended DNS entries")
|
||||
lines.append(f'{d}. 3600 IN TXT "v=spf1 a ~all"')
|
||||
lines.append(f'_dmarc.{d}. 3600 IN TXT "v=DMARC1;p=reject;adkim=s;aspf=s"')
|
||||
if remote_data.get("acme_account_url"):
|
||||
lines.append(
|
||||
f"{d}. 3600 IN CAA 0 issue"
|
||||
f' "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"'
|
||||
)
|
||||
lines.append(f'_adsp._domainkey.{d}. 3600 IN TXT "dkim=discardable"')
|
||||
lines.append(f"_submission._tcp.{d}. 3600 IN SRV 0 1 587 {d}.")
|
||||
lines.append(f"_submissions._tcp.{d}. 3600 IN SRV 0 1 465 {d}.")
|
||||
lines.append(f"_imap._tcp.{d}. 3600 IN SRV 0 1 143 {d}.")
|
||||
lines.append(f"_imaps._tcp.{d}. 3600 IN SRV 0 1 993 {d}.")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
|
||||
@@ -53,18 +91,19 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
|
||||
if required_diff:
|
||||
out.red("Please set required DNS entries at your DNS provider:\n")
|
||||
for line in required_diff:
|
||||
out(line)
|
||||
out("")
|
||||
out.print(line)
|
||||
out.print()
|
||||
returncode = 1
|
||||
if remote_data.get("dkim_entry") in required_diff:
|
||||
out(
|
||||
"If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
|
||||
out.print(
|
||||
"If the DKIM entry above does not work with your DNS provider,"
|
||||
" you can try this one:\n"
|
||||
)
|
||||
out(remote_data.get("web_dkim_entry") + "\n")
|
||||
out.print(remote_data.get("web_dkim_entry") + "\n")
|
||||
if recommended_diff:
|
||||
out("WARNING: these recommended DNS entries are not set:\n")
|
||||
out.print("WARNING: these recommended DNS entries are not set:\n")
|
||||
for line in recommended_diff:
|
||||
out(line)
|
||||
out.print(line)
|
||||
|
||||
if not (recommended_diff or required_diff):
|
||||
out.green("Great! All your DNS entries are verified and correct.")
|
||||
|
||||
@@ -2,8 +2,8 @@ import urllib.request
|
||||
|
||||
from chatmaild.config import Config
|
||||
from pyinfra import host
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
||||
from pyinfra.facts.systemd import SystemdEnabled
|
||||
from pyinfra.operations import apt, files, server, systemd
|
||||
|
||||
from cmdeploy.basedeploy import (
|
||||
@@ -12,19 +12,9 @@ from cmdeploy.basedeploy import (
|
||||
blocked_service_startup,
|
||||
configure_remote_units,
|
||||
get_resource,
|
||||
has_systemd,
|
||||
)
|
||||
|
||||
DOVECOT_VERSION = "2.3.21+dfsg1-3"
|
||||
|
||||
DOVECOT_SHA256 = {
|
||||
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
|
||||
("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
|
||||
("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
|
||||
("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
|
||||
("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
|
||||
("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
|
||||
}
|
||||
|
||||
|
||||
class DovecotDeployer(Deployer):
|
||||
daemon_reload = False
|
||||
@@ -36,6 +26,9 @@ class DovecotDeployer(Deployer):
|
||||
|
||||
def install(self):
|
||||
arch = host.get_fact(Arch)
|
||||
if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
|
||||
return # already installed and running
|
||||
|
||||
with blocked_service_startup():
|
||||
_install_dovecot_package("core", arch)
|
||||
_install_dovecot_package("imapd", arch)
|
||||
@@ -75,33 +68,37 @@ def _pick_url(primary, fallback):
|
||||
def _install_dovecot_package(package: str, arch: str):
|
||||
arch = "amd64" if arch == "x86_64" else arch
|
||||
arch = "arm64" if arch == "aarch64" else arch
|
||||
|
||||
pkg_name = f"dovecot-{package}"
|
||||
sha256 = DOVECOT_SHA256.get((package, arch))
|
||||
if sha256 is None:
|
||||
apt.packages(packages=[pkg_name])
|
||||
return
|
||||
|
||||
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
|
||||
if DOVECOT_VERSION in installed_versions:
|
||||
return
|
||||
|
||||
url_version = DOVECOT_VERSION.replace("+", "%2B")
|
||||
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
|
||||
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
|
||||
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
|
||||
primary_url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
|
||||
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F2.3.21%2Bdfsg1/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
|
||||
url = _pick_url(primary_url, fallback_url)
|
||||
deb_filename = f"/root/{deb_base}"
|
||||
deb_filename = "/root/" + url.split("/")[-1]
|
||||
|
||||
match (package, arch):
|
||||
case ("core", "amd64"):
|
||||
sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
|
||||
case ("core", "arm64"):
|
||||
sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
|
||||
case ("imapd", "amd64"):
|
||||
sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
|
||||
case ("imapd", "arm64"):
|
||||
sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
|
||||
case ("lmtpd", "amd64"):
|
||||
sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
|
||||
case ("lmtpd", "arm64"):
|
||||
sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
|
||||
case _:
|
||||
apt.packages(packages=[f"dovecot-{package}"])
|
||||
return
|
||||
|
||||
files.download(
|
||||
name=f"Download {pkg_name}",
|
||||
name=f"Download dovecot-{package}",
|
||||
src=url,
|
||||
dest=deb_filename,
|
||||
sha256sum=sha256,
|
||||
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
|
||||
)
|
||||
|
||||
apt.deb(name=f"Install {pkg_name}", src=deb_filename)
|
||||
apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
|
||||
|
||||
|
||||
def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
|
||||
@@ -148,7 +145,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
|
||||
if not can_modify:
|
||||
print(
|
||||
"\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
|
||||
f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
|
||||
f"!!!! dovecot: sysctl {key!r}={value}, should be >65535 for production setups\n"
|
||||
"!!!!"
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
|
||||
|
||||
def install(self):
|
||||
arch = host.get_fact(facts.server.Arch)
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-{arch}"
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-{arch}"
|
||||
sha256sum = {
|
||||
"x86_64": "3fd8b18282252c75a5bbfa603d8c1b65f6563e5e920bddf3e64e451b7cdb43ce",
|
||||
"aarch64": "2bd191de205f7fd60158dd8e3516ab7e3efb14627696f3d7dc186bdcd9e10a43",
|
||||
"x86_64": "ce24ca0075aa445510291d775fb3aea8f4411818c7b885ae51a0fe18c5f789ce",
|
||||
"aarch64": "c5d783eefa5332db3d97a0e6a23917d72849e3eb45da3d16ce908a9b4e5a797d",
|
||||
}[arch]
|
||||
self.need_restart |= files.download(
|
||||
name="Download filtermail",
|
||||
|
||||
456
cmdeploy/src/cmdeploy/lxc/cli.py
Normal file
456
cmdeploy/src/cmdeploy/lxc/cli.py
Normal file
@@ -0,0 +1,456 @@
|
||||
"""lxc-start/stop/status/test subcommands for testing with local containers."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from ..util import get_git_hash, get_version_string, shell
|
||||
from .incus import Incus, RelayContainer
|
||||
|
||||
RELAY_NAMES = ("test0", "test1")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# lxc-start
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def lxc_start_cmd_options(parser):
|
||||
_add_name_args(
|
||||
parser,
|
||||
help_text="User relay name(s) to create (default: test0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ipv4-only",
|
||||
dest="ipv4_only",
|
||||
action="store_true",
|
||||
help="Create an IPv4-only container.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--run",
|
||||
action="store_true",
|
||||
help="Run 'cmdeploy run' on each container after starting it.",
|
||||
)
|
||||
|
||||
|
||||
def lxc_start_cmd(args, out):
|
||||
"""Create/Ensure and start LXC relay and DNS containers."""
|
||||
|
||||
with out.section("Preparing container setup"):
|
||||
_lxc_start_cmd(args, out)
|
||||
|
||||
|
||||
def _lxc_start_cmd(args, out):
|
||||
ix = Incus(out)
|
||||
sub = out.new_prefixed_out()
|
||||
out.green("Ensuring base image ...")
|
||||
ix.ensure_base_image()
|
||||
out.green("Ensuring DNS container (ns-localchat) ...")
|
||||
dns_ct = ix.get_dns_container()
|
||||
dns_ct.ensure()
|
||||
dns_ct.ensure_cached_as_image()
|
||||
sub.print(f"DNS container IP: {dns_ct.ipv4}")
|
||||
|
||||
names = args.names if args.names else RELAY_NAMES
|
||||
relays = list(ix.get_container(n) for n in names)
|
||||
for ct in relays:
|
||||
out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
|
||||
ct.ensure()
|
||||
ip = ct.ipv4
|
||||
|
||||
sub.print("Configuring container hostname ...")
|
||||
ct.configure_hosts(ip)
|
||||
|
||||
sub.print(f"Writing {ct.ini.name} ...")
|
||||
ct.write_ini(disable_ipv6=args.ipv4_only)
|
||||
sub.print(f"Config: {ct.ini}")
|
||||
if args.ipv4_only:
|
||||
ct.disable_ipv6()
|
||||
ipv6 = None
|
||||
else:
|
||||
output = ct.bash(
|
||||
"ip -6 addr show scope global -deprecated"
|
||||
" | grep -oP '(?<=inet6 )[^/]+'",
|
||||
check=False,
|
||||
)
|
||||
ipv6 = output.strip() if output else None
|
||||
sub.print(f"{_format_addrs(ip, ipv6)}")
|
||||
|
||||
sub.green(f"Container {ct.name!r} ready: {ct.domain} -> {ip}")
|
||||
out.print()
|
||||
|
||||
# Reset DNS zones only for the containers we just started
|
||||
started_cnames = {ct.name for ct in relays}
|
||||
managed = ix.list_managed()
|
||||
started = [c for c in managed if c["name"] in started_cnames]
|
||||
|
||||
if started:
|
||||
out.print(
|
||||
f"Resetting DNS zones for {len(started)} domain(s) (A + AAAA records) ..."
|
||||
)
|
||||
dns_ct.reset_dns_records(dns_ct.ipv4, started)
|
||||
|
||||
for ct in relays:
|
||||
if ct.name in started_cnames:
|
||||
sub.print(f"Configuring DNS in {ct.name} ...")
|
||||
ct.configure_dns(dns_ct.ipv4)
|
||||
|
||||
# Generate the unified SSH config
|
||||
out.green("Writing ssh-config ...")
|
||||
ssh_cfg = ix.write_ssh_config()
|
||||
sub.print(f"{ssh_cfg}")
|
||||
|
||||
# Verify SSH via the generated config
|
||||
for ct in relays:
|
||||
sub.print(f"Verifying SSH to {ct.name} via ssh-config ...")
|
||||
if ct.verify_ssh(ssh_cfg):
|
||||
sub.print(f"SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
|
||||
else:
|
||||
sub.red(f"WARNING: SSH verification failed for {ct.name}")
|
||||
|
||||
# Print integration suggestions
|
||||
ssh_cfg = ix.ssh_config_path
|
||||
if not ix.check_ssh_include():
|
||||
sub.green(
|
||||
"\n(Optional) To use containers from any SSH client, add to ~/.ssh/config:"
|
||||
)
|
||||
sub.green(f" Include {ssh_cfg}")
|
||||
|
||||
# Optionally run cmdeploy run + dns on each relay
|
||||
if args.run:
|
||||
local_hash = get_git_hash()
|
||||
for ct in relays:
|
||||
status = _deploy_status(ct, local_hash, ix)
|
||||
with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
|
||||
if "IN-SYNC" in status:
|
||||
out.print(f"{ct.sname} is {status}, skipping")
|
||||
else:
|
||||
ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
|
||||
if ret:
|
||||
out.red(f"Deploy to {ct.sname} failed (exit {ret})")
|
||||
return ret
|
||||
# Cache a per-relay image after each successful deploy
|
||||
# so the next run can launch directly from the deployed state.
|
||||
with out.section(f"lxc-test: caching {ct.sname} image"):
|
||||
ct.ensure_cached_as_image()
|
||||
|
||||
# Restart mail services to flush stale DNS state.
|
||||
# Cached container images boot with a resolv.conf
|
||||
# pointing to the previous run's DNS IP;
|
||||
# configure_dns() already restarted unbound,
|
||||
# but postfix/dovecot may hold stale results
|
||||
# from the window between boot and DNS fix.
|
||||
for ct in relays:
|
||||
out.print(f"Restarting mail services on {ct.name} ...")
|
||||
ct.bash("systemctl restart postfix dovecot opendkim")
|
||||
|
||||
for ct in relays:
|
||||
with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
|
||||
ret = _run_cmdeploy(
|
||||
"dns",
|
||||
ct,
|
||||
ix,
|
||||
out,
|
||||
extra=["--zonefile", str(ct.zone)],
|
||||
)
|
||||
if ret:
|
||||
out.red(f"DNS for {ct.sname} failed (exit {ret})")
|
||||
return ret
|
||||
if ct.zone.exists():
|
||||
dns_ct.set_dns_records(ct.zone.read_text())
|
||||
# Restart filtermail so its in-process DNS cache
|
||||
# does not hold stale negative DKIM responses
|
||||
# from before the zones were loaded.
|
||||
out.print(f"Restarting filtermail-incoming on {ct.name} ...")
|
||||
ct.bash("systemctl restart filtermail-incoming")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# lxc-stop
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def lxc_stop_cmd_options(parser):
|
||||
parser.add_argument(
|
||||
"--destroy",
|
||||
action="store_true",
|
||||
help="Delete containers and their config files after stopping.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--destroy-all",
|
||||
dest="destroy_all",
|
||||
action="store_true",
|
||||
help="Like --destroy, but also remove the ns-localchat DNS container.",
|
||||
)
|
||||
_add_name_args(
|
||||
parser,
|
||||
help_text="Container name(s) to stop (default: test0 + test1).",
|
||||
)
|
||||
|
||||
|
||||
def lxc_stop_cmd(args, out):
|
||||
"""Stop (and optionally destroy) local LXC relay containers."""
|
||||
ix = Incus(out)
|
||||
names = args.names or RELAY_NAMES
|
||||
destroy = args.destroy or args.destroy_all
|
||||
|
||||
for ct in map(ix.get_container, names):
|
||||
if destroy:
|
||||
out.green(f"Destroying container {ct.name!r} ...")
|
||||
ct.destroy()
|
||||
else:
|
||||
out.green(f"Stopping container {ct.name!r} ...")
|
||||
ct.stop(force=True)
|
||||
|
||||
if args.destroy_all:
|
||||
dns_ct = ix.get_dns_container()
|
||||
out.green(f"Destroying DNS container {dns_ct.name!r} ...")
|
||||
dns_ct.destroy()
|
||||
ix.delete_images()
|
||||
|
||||
if destroy:
|
||||
ix.write_ssh_config()
|
||||
out.green("LXC containers destroyed.")
|
||||
else:
|
||||
out.green("LXC containers stopped.")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# lxc-test
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def lxc_test_cmd_options(parser):
|
||||
parser.add_argument(
|
||||
"--one",
|
||||
action="store_true",
|
||||
help="Only deploy and test against test0 (skip test1).",
|
||||
)
|
||||
|
||||
|
||||
def lxc_test_cmd(args, out):
|
||||
"""Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
|
||||
|
||||
All commands run directly on the host using
|
||||
``--ssh-config lxconfigs/ssh-config`` for SSH access.
|
||||
"""
|
||||
ix = Incus(out)
|
||||
t_total = time.time()
|
||||
v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
|
||||
|
||||
ret = out.shell(f"cmdeploy lxc-start{v_flag} --run test0", cwd=str(ix.project_root))
|
||||
if ret:
|
||||
return ret
|
||||
|
||||
if not args.one:
|
||||
ret = out.shell(
|
||||
f"cmdeploy lxc-start{v_flag} --run test1 --ipv4-only",
|
||||
cwd=str(ix.project_root),
|
||||
)
|
||||
if ret:
|
||||
return ret
|
||||
|
||||
with out.section("cmdeploy test"):
|
||||
first = ix.get_container("test0")
|
||||
env = None
|
||||
if not args.one:
|
||||
env = os.environ.copy()
|
||||
env["CHATMAIL_DOMAIN2"] = ix.get_container("test1").domain
|
||||
ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
|
||||
if ret:
|
||||
out.red(f"Tests failed (exit {ret})")
|
||||
return ret
|
||||
|
||||
elapsed = time.time() - t_total
|
||||
out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
|
||||
if out.section_timings:
|
||||
out.print("Section timings:")
|
||||
for name, secs in out.section_timings:
|
||||
out.print(f" {name:.<50s} {secs:5.1f}s")
|
||||
out.print(f" {'total':.<50s} {elapsed:5.1f}s")
|
||||
out.section_timings.clear()
|
||||
return 0
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# lxc-status
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def lxc_status_cmd_options(parser):
|
||||
pass
|
||||
|
||||
|
||||
def lxc_status_cmd(args, out):
|
||||
"""Show status of local LXC chatmail containers."""
|
||||
ix = Incus(out)
|
||||
containers = ix.list_managed()
|
||||
if not containers:
|
||||
out.red("No LXC containers found. Run 'cmdeploy lxc-start' first.")
|
||||
return 1
|
||||
|
||||
local_hash = get_git_hash()
|
||||
|
||||
# Get storage pool path for display
|
||||
storage_path = None
|
||||
data = ix.run_json(["storage", "show", "default"], check=False)
|
||||
if data:
|
||||
storage_path = data.get("config", {}).get("source")
|
||||
msg = "Container status"
|
||||
if storage_path:
|
||||
msg += f": {storage_path}"
|
||||
out.section_line(msg)
|
||||
|
||||
dns_ip = None
|
||||
for c in containers:
|
||||
_print_container_status(out, c, ix, local_hash)
|
||||
if c["name"] == ix.get_dns_container().name:
|
||||
dns_ip = c["ip"]
|
||||
|
||||
out.section_line("Host ssh and DNS configuration")
|
||||
_print_ssh_status(out, ix)
|
||||
_print_dns_forwarding_status(out, dns_ip)
|
||||
return 0
|
||||
|
||||
|
||||
def _print_container_status(out, c, ix, local_hash):
|
||||
"""Print name/status, domain/IPs, and RAM for one container."""
|
||||
cname = c["name"]
|
||||
is_running = c.get("status") == "Running"
|
||||
ct = ix.get_container(cname)
|
||||
|
||||
# First line: name + running/STOPPED + deploy status
|
||||
if not is_running:
|
||||
tag = "STOPPED"
|
||||
elif not isinstance(ct, RelayContainer):
|
||||
tag = "running"
|
||||
else:
|
||||
tag = f"running {_deploy_status(ct, local_hash, ix)}"
|
||||
out.print(f"{cname:20s} {tag}")
|
||||
|
||||
# Second line: domain, IPv4, IPv6
|
||||
domain = c.get("domain", "")
|
||||
ip = c.get("ip") or "?"
|
||||
ipv6 = c.get("ipv6")
|
||||
out.print(f"{domain:20s} {_format_addrs(ip, ipv6)}")
|
||||
|
||||
# Third line: RAM (RSS), config
|
||||
detail_out = out.new_prefixed_out(" " * 21)
|
||||
try:
|
||||
used, total = ct.rss_mib()
|
||||
except Exception:
|
||||
ram_str = "RSS ?"
|
||||
else:
|
||||
ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
|
||||
|
||||
if isinstance(ct, RelayContainer):
|
||||
detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
|
||||
else:
|
||||
detail = ram_str
|
||||
|
||||
detail_out.print(detail)
|
||||
out.print()
|
||||
|
||||
|
||||
def _print_ssh_status(out, ix):
|
||||
"""Print SSH integration status."""
|
||||
ssh_cfg = ix.ssh_config_path
|
||||
if ix.check_ssh_include():
|
||||
out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
|
||||
else:
|
||||
out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
|
||||
sub = out.new_prefixed_out()
|
||||
sub.print("Add to ~/.ssh/config:")
|
||||
sub.print(f" Include {ssh_cfg}")
|
||||
|
||||
|
||||
def _print_dns_forwarding_status(out, dns_ip):
|
||||
"""Print host DNS forwarding status for .localchat."""
|
||||
sub = out.new_prefixed_out()
|
||||
if not dns_ip:
|
||||
out.red("DNS: ns-localchat container not found")
|
||||
return
|
||||
try:
|
||||
rv = shell("resolvectl status incusbr0")
|
||||
dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
|
||||
except Exception:
|
||||
dns_ok = None
|
||||
if dns_ok is True:
|
||||
out.green(f"DNS: .localchat forwarding to {dns_ip} ✓")
|
||||
elif dns_ok is False:
|
||||
out.red("DNS: .localchat forwarding NOT configured")
|
||||
sub.print("Run:")
|
||||
sub.print(f" sudo resolvectl dns incusbr0 {dns_ip}")
|
||||
sub.print(" sudo resolvectl domain incusbr0 ~localchat")
|
||||
else:
|
||||
sub.print("DNS: .localchat forwarding status UNKNOWN")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
|
||||
def _format_addrs(ip, ipv6=None):
|
||||
parts = [f"IPv4 {ip}"]
|
||||
if ipv6:
|
||||
parts.append(f"IPv6 {ipv6}")
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def _deploy_status(ct, local_hash, ix):
|
||||
"""Return a human-readable deploy status string.
|
||||
|
||||
Compares the full deployed version (hash + diff) against
|
||||
the local state built by :func:`~cmdeploy.util.get_version_string`.
|
||||
"""
|
||||
deployed = ct.deployed_version()
|
||||
if deployed is None:
|
||||
return "NOT DEPLOYED"
|
||||
|
||||
# A container launched from the relay image has the same
|
||||
# git hash but a different domain — always redeploy.
|
||||
deployed_domain = ct.deployed_domain()
|
||||
if deployed_domain and deployed_domain != ct.domain:
|
||||
return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
|
||||
|
||||
deployed_lines = deployed.splitlines()
|
||||
deployed_hash = deployed_lines[0] if deployed_lines else ""
|
||||
short = deployed_hash[:12]
|
||||
|
||||
if not local_hash:
|
||||
return f"UNKNOWN (deployed: {short})"
|
||||
|
||||
local_short = local_hash[:12]
|
||||
if deployed_hash != local_hash:
|
||||
return f"STALE (deployed: {short}, local: {local_short})"
|
||||
|
||||
# Hash matches — check for uncommitted diffs
|
||||
local_version = get_version_string()
|
||||
if deployed != local_version:
|
||||
return f"DIRTY ({local_short}, undeployed changes)"
|
||||
|
||||
return f"IN-SYNC ({short})"
|
||||
|
||||
|
||||
def _add_name_args(parser, help_text):
|
||||
parser.add_argument("names", nargs="*", metavar="NAME", help=help_text)
|
||||
|
||||
|
||||
def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
|
||||
"""Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
|
||||
|
||||
*ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
|
||||
Returns the subprocess exit code.
|
||||
"""
|
||||
extra_str = " ".join(extra) if extra else ""
|
||||
v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
|
||||
cmd = f"""
|
||||
cmdeploy {subcmd}{v_flag}
|
||||
--config {ct.ini}
|
||||
--ssh-config {ix.ssh_config_path}
|
||||
--ssh-host {ct.domain}
|
||||
{extra_str}
|
||||
"""
|
||||
if "cwd" not in kwargs:
|
||||
kwargs["cwd"] = str(ix.project_root)
|
||||
return out.shell(cmd, **kwargs)
|
||||
803
cmdeploy/src/cmdeploy/lxc/incus.py
Normal file
803
cmdeploy/src/cmdeploy/lxc/incus.py
Normal file
@@ -0,0 +1,803 @@
|
||||
"""Core Incus operations for local chatmail LXC containers."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import textwrap
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from ..util import shell
|
||||
|
||||
LABEL_KEY = "user.localchat-managed"
|
||||
SSH_KEY_NAME = "id_localchat"
|
||||
DOMAIN_SUFFIX = ".localchat"
|
||||
UPSTREAM_IMAGE = "images:debian/12"
|
||||
BASE_IMAGE_ALIAS = "localchat-base"
|
||||
BASE_SETUP_NAME = "localchat-base-setup"
|
||||
|
||||
DNS_CONTAINER_NAME = "ns-localchat"
|
||||
DNS_IMAGE_ALIAS = "localchat-ns"
|
||||
DNS_DOMAIN = "ns.localchat"
|
||||
|
||||
|
||||
class DNSConfigurationError(Exception):
|
||||
"""Raised when the DNS container is not reachable or not answering."""
|
||||
|
||||
|
||||
def _extract_ip(net_data, family="inet"):
|
||||
"""Extract the first global-scope IP of *family* from network state data.
|
||||
|
||||
*net_data* is the ``state.network`` dict from ``incus list --format=json``.
|
||||
*family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
|
||||
Returns the address string, or None.
|
||||
"""
|
||||
for iface_name, iface in net_data.items():
|
||||
if iface_name == "lo":
|
||||
continue
|
||||
for addr in iface.get("addresses", []):
|
||||
if addr["family"] == family and addr["scope"] == "global":
|
||||
return addr["address"]
|
||||
return None
|
||||
|
||||
|
||||
class Incus:
|
||||
"""Gateway for all Incus container operations.
|
||||
|
||||
Instantiated once per CLI command and passed around so that
|
||||
all modules share a single entry point for Incus interactions.
|
||||
"""
|
||||
|
||||
def __init__(self, out):
|
||||
self.out = out
|
||||
self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
|
||||
self.lxconfigs_dir = self.project_root / "lxconfigs"
|
||||
self.lxconfigs_dir.mkdir(exist_ok=True)
|
||||
self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
|
||||
if not self.ssh_key_path.exists():
|
||||
shell(
|
||||
f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
|
||||
check=True,
|
||||
)
|
||||
self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
|
||||
|
||||
def write_ssh_config(self):
|
||||
"""Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
|
||||
|
||||
Each Host block maps the container name, the domain name, and the
|
||||
short relay name (e.g. ``_test0``) to the container's IP, using the
|
||||
shared localchat SSH key. Returns the path to the file.
|
||||
"""
|
||||
containers = self.list_managed()
|
||||
key_path = self.ssh_key_path
|
||||
lines = ["# Auto-generated by cmdeploy lxc-start — do not edit\n"]
|
||||
for c in containers:
|
||||
hosts = [c["name"]]
|
||||
domain = c.get("domain", "")
|
||||
if domain and domain != c["name"]:
|
||||
hosts.append(domain)
|
||||
short = domain.split(".")[0]
|
||||
if short and short not in hosts:
|
||||
hosts.append(short)
|
||||
lines.append(f"\nHost {' '.join(hosts)}\n")
|
||||
lines.append(f" Hostname {c['ip']}\n")
|
||||
lines.append(" User root\n")
|
||||
lines.append(f" IdentityFile {key_path}\n")
|
||||
lines.append(" IdentitiesOnly yes\n")
|
||||
lines.append(" StrictHostKeyChecking accept-new\n")
|
||||
lines.append(" UserKnownHostsFile /dev/null\n")
|
||||
lines.append(" LogLevel ERROR\n")
|
||||
path = self.ssh_config_path
|
||||
path.write_text("".join(lines))
|
||||
return path
|
||||
|
||||
def check_ssh_include(self):
|
||||
"""Check if the user's ~/.ssh/config already includes our ssh-config."""
|
||||
user_ssh_config = Path.home() / ".ssh" / "config"
|
||||
if not user_ssh_config.exists():
|
||||
return False
|
||||
lines = user_ssh_config.read_text().splitlines()
|
||||
target = f"include {self.ssh_config_path}".lower()
|
||||
return any(line.strip().lower() == target for line in lines)
|
||||
|
||||
def get_host_nameservers(self):
|
||||
"""Return upstream nameservers found on the host."""
|
||||
ns = []
|
||||
for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
|
||||
p = Path(path)
|
||||
if p.exists():
|
||||
for line in p.read_text().splitlines():
|
||||
if line.strip().startswith("nameserver "):
|
||||
addr = line.split()[1]
|
||||
if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
|
||||
if addr not in ns:
|
||||
ns.append(addr)
|
||||
if ns:
|
||||
break
|
||||
return ns
|
||||
|
||||
def run(self, args, check=True, capture=True, input=None):
|
||||
"""Run an incus command.
|
||||
|
||||
When *capture* is True and *verbosity* >= 1, output is streamed
|
||||
to the terminal line-by-line while also being captured for
|
||||
later return via result.stdout.
|
||||
"""
|
||||
cmd = ["incus", "--quiet"] + list(args)
|
||||
sub = self.out.new_prefixed_out(" ")
|
||||
|
||||
if not capture:
|
||||
# Simple case: let subprocess handle streams (no capture)
|
||||
if self.out.verbosity >= 1:
|
||||
sub.print(f"$ {' '.join(cmd)}")
|
||||
return subprocess.run(
|
||||
cmd, text=True, input=input, check=check, stdout=None, stderr=None
|
||||
)
|
||||
|
||||
# Capture case: we may need to stream while capturing
|
||||
if sub.verbosity >= 1:
|
||||
cmd_lines = " ".join(cmd).splitlines()
|
||||
sub.print(f"$ {cmd_lines.pop(0)}")
|
||||
if sub.verbosity >= 2:
|
||||
for line in cmd_lines:
|
||||
sub.print(f" {line}")
|
||||
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
text=True,
|
||||
stdin=subprocess.PIPE if input else subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
stdout_lines = []
|
||||
if input:
|
||||
proc.stdin.write(input)
|
||||
proc.stdin.close()
|
||||
|
||||
for line in proc.stdout:
|
||||
stdout_lines.append(line)
|
||||
if sub.verbosity >= 2:
|
||||
sub.print(f" > {line.rstrip()}")
|
||||
|
||||
stderr = proc.stderr.read()
|
||||
ret = proc.wait()
|
||||
stdout = "".join(stdout_lines)
|
||||
if check and ret != 0:
|
||||
full_output = stdout + stderr
|
||||
for line in full_output.splitlines():
|
||||
if sub.verbosity < 1: # and we haven't printed it yet
|
||||
sub.red(line)
|
||||
raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
|
||||
|
||||
return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
|
||||
|
||||
def run_json(self, args, check=True):
|
||||
"""Run an incus command with ``--format=json``.
|
||||
|
||||
Returns the parsed JSON on success.
|
||||
When *check* is True raises ``subprocess.CalledProcessError``
|
||||
on non-zero exit; when False returns *None* instead.
|
||||
"""
|
||||
result = self.run(
|
||||
list(args) + ["--format=json"],
|
||||
check=check,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
msg = f"Incus JSON processing failed for {args!r}: {e!s}"
|
||||
self.out.red(msg)
|
||||
self.out.red(f"Captured stdout: {result.stdout!r}")
|
||||
self.out.red(f"Captured stderr: {result.stderr!r}")
|
||||
if check:
|
||||
raise
|
||||
return None
|
||||
|
||||
def run_output(self, args, check=True):
|
||||
"""Run an incus command and return its stripped stdout.
|
||||
|
||||
When *check* is False, returns *None* on non-zero exit
|
||||
instead of raising.
|
||||
"""
|
||||
result = self.run(args, check=check)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
return result.stdout.strip()
|
||||
|
||||
def find_image(self, aliases):
|
||||
"""Return the first alias from *aliases* that exists, else None."""
|
||||
images = self.run_json(["image", "list"], check=False) or []
|
||||
existing = {a.get("name") for img in images for a in img.get("aliases", [])}
|
||||
for alias in aliases:
|
||||
if alias in existing:
|
||||
return alias
|
||||
return None
|
||||
|
||||
def delete_images(self):
|
||||
"""Delete localchat-base and per-container images."""
|
||||
for alias in [
|
||||
BASE_IMAGE_ALIAS,
|
||||
DNS_IMAGE_ALIAS,
|
||||
"localchat-test0",
|
||||
"localchat-test1",
|
||||
]:
|
||||
self.run(["image", "delete", alias], check=False) # ok if absent
|
||||
|
||||
def list_managed(self):
|
||||
"""Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
|
||||
containers = []
|
||||
for ct in self.run_json(["list"]):
|
||||
config = ct.get("config", {})
|
||||
if config.get(LABEL_KEY) != "true":
|
||||
continue
|
||||
name = ct["name"]
|
||||
state = ct.get("state", {})
|
||||
net = state.get("network") or {}
|
||||
containers.append(
|
||||
{
|
||||
"name": name,
|
||||
"ip": _extract_ip(net, "inet"),
|
||||
"ipv6": _extract_ip(net, "inet6"),
|
||||
"domain": config.get(
|
||||
"user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
|
||||
),
|
||||
"status": ct.get("status", "Unknown"),
|
||||
"memory_usage": state.get("memory", {}).get("usage", 0),
|
||||
}
|
||||
)
|
||||
return containers
|
||||
|
||||
def ensure_base_image(self):
|
||||
"""Build and cache a base image with openssh and the SSH key.
|
||||
|
||||
The image is cached as a local incus image with alias
|
||||
'localchat-base'. Subsequent container launches use this
|
||||
image instead of the upstream Debian 12, skipping the
|
||||
slow apt-get install step.
|
||||
Returns the image alias.
|
||||
"""
|
||||
if self.find_image([BASE_IMAGE_ALIAS]):
|
||||
self.out.print(f" Base image '{BASE_IMAGE_ALIAS}' already cached.")
|
||||
return BASE_IMAGE_ALIAS
|
||||
|
||||
self.out.print(" Building base image (one-time setup) ...")
|
||||
|
||||
self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
|
||||
self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
|
||||
self.run(["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME])
|
||||
|
||||
ct = Container(self, BASE_SETUP_NAME)
|
||||
ct.wait_ready()
|
||||
|
||||
key_path = self.ssh_key_path
|
||||
pub_key = key_path.with_suffix(".pub").read_text().strip()
|
||||
host_ns = self.get_host_nameservers()
|
||||
ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
|
||||
ct.bash(f"""
|
||||
printf '{ns_lines}\n' > /etc/resolv.conf
|
||||
apt-get -o DPkg::Lock::Timeout=60 update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
|
||||
systemctl enable ssh
|
||||
apt-get clean
|
||||
mkdir -p /root/.ssh
|
||||
chmod 700 /root/.ssh
|
||||
echo '{pub_key}' > /root/.ssh/authorized_keys
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
""")
|
||||
|
||||
self.run(["stop", BASE_SETUP_NAME])
|
||||
self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
|
||||
self.run(["delete", BASE_SETUP_NAME, "--force"])
|
||||
self.out.print(f" Base image '{BASE_IMAGE_ALIAS}' ready.")
|
||||
return BASE_IMAGE_ALIAS
|
||||
|
||||
def get_container(self, name):
|
||||
"""Return a container handle for the given name.
|
||||
|
||||
Accepts both short relay names (``test0``) and full Incus
|
||||
container names (``test0-localchat``). Returns
|
||||
``DNSContainer`` for the DNS container and
|
||||
``RelayContainer`` for everything else.
|
||||
"""
|
||||
if name == DNS_CONTAINER_NAME:
|
||||
return DNSContainer(self)
|
||||
return RelayContainer(self, name.removesuffix("-localchat"))
|
||||
|
||||
def get_dns_container(self):
|
||||
"""Return a DNSContainer handle."""
|
||||
return DNSContainer(self)
|
||||
|
||||
|
||||
class Container:
|
||||
"""The base container handle wraps all interactions with incus."""
|
||||
|
||||
def __init__(self, incus, name, domain=None):
|
||||
self.incus = incus
|
||||
self.out = incus.out
|
||||
self.name = name
|
||||
self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
|
||||
self.ipv4 = None
|
||||
self.ipv6 = None
|
||||
|
||||
def bash(self, script, check=True):
|
||||
"""Returns stdout from executing ``bash -ec <script>`` inside this container.
|
||||
|
||||
*script* is dedented and stripped so callers can use triple-quoted strings.
|
||||
When *check* is False, returns *None* on non-zero exit instead of raising.
|
||||
"""
|
||||
script = textwrap.dedent(script).strip()
|
||||
cmd = ["exec", self.name, "--", "bash", "-ec", script]
|
||||
return self.incus.run_output(cmd, check=check)
|
||||
|
||||
def run_cmd(self, *args, check=True):
|
||||
"""Return stdout from running a command directly in the container (no shell).
|
||||
|
||||
When *check* is False, returns *None* on non-zero exit instead of raising.
|
||||
"""
|
||||
return self.incus.run_output(
|
||||
["exec", self.name, "--", *args],
|
||||
check=check,
|
||||
)
|
||||
|
||||
def start(self):
|
||||
self.incus.run(["start", self.name])
|
||||
|
||||
def stop(self, force=False):
|
||||
cmd = ["stop", self.name]
|
||||
if force:
|
||||
cmd.append("--force")
|
||||
self.incus.run(cmd, check=False)
|
||||
|
||||
def launch(self):
|
||||
"""Launch from the best available image, return the alias used."""
|
||||
image = self.incus.find_image([BASE_IMAGE_ALIAS])
|
||||
if not image:
|
||||
raise RuntimeError(
|
||||
f"No base image '{BASE_IMAGE_ALIAS}' found. "
|
||||
"Call ensure_base_image() before launching containers."
|
||||
)
|
||||
self.out.print(f" Launching from '{image}' image ...")
|
||||
cfg = []
|
||||
cfg += ("-c", f"{LABEL_KEY}=true")
|
||||
cfg += ("-c", f"user.localchat-domain={self.domain}")
|
||||
self.incus.run(["launch", image, self.name, *cfg])
|
||||
return image
|
||||
|
||||
def ensure(self):
|
||||
"""Create/start this container from the cached base image.
|
||||
|
||||
On first call, builds the base image (~30s).
|
||||
Subsequent containers launch in ~2s from the cached image.
|
||||
Returns ``self`` for chaining.
|
||||
"""
|
||||
data = self.incus.run_json(["list", self.name], check=False) or []
|
||||
|
||||
existing = [c for c in data if c["name"] == self.name]
|
||||
if existing:
|
||||
if existing[0]["status"] != "Running":
|
||||
self.start()
|
||||
else:
|
||||
self.launch()
|
||||
self.wait_ready()
|
||||
return self
|
||||
|
||||
def destroy(self):
|
||||
"""Stop, delete, and clean up config files."""
|
||||
self.stop(force=True)
|
||||
self.incus.run(["delete", self.name, "--force"], check=False)
|
||||
|
||||
def push_file_content(self, dest_path, content):
|
||||
"""Write *content* to *dest_path* inside the container.
|
||||
|
||||
*content* is dedented and stripped so callers can use
|
||||
indented triple-quoted strings.
|
||||
"""
|
||||
content = textwrap.dedent(content).strip() + "\n"
|
||||
self.incus.run(
|
||||
["file", "push", "-", f"{self.name}{dest_path}"],
|
||||
input=content,
|
||||
)
|
||||
self.bash(f"chmod 644 {dest_path}")
|
||||
|
||||
def wait_ready(self, timeout=60):
|
||||
"""Wait until the container is running with an IPv4 address.
|
||||
|
||||
Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
|
||||
or raises ``TimeoutError``.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
data = self.incus.run_json(
|
||||
["list", self.name],
|
||||
check=False,
|
||||
)
|
||||
if data and data[0].get("status") == "Running":
|
||||
net = data[0].get("state", {}).get("network", {})
|
||||
self.ipv4 = _extract_ip(net, "inet")
|
||||
self.ipv6 = _extract_ip(net, "inet6")
|
||||
if self.ipv4:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(
|
||||
f"Container {self.name!r} did not become ready within {timeout}s"
|
||||
)
|
||||
|
||||
def rss_mib(self):
|
||||
"""Return ``(used, total)`` memory from container (or None if unobtainable)."""
|
||||
output = self.bash("free -m", check=False)
|
||||
if output:
|
||||
for line in output.splitlines():
|
||||
if line.startswith("Mem:"):
|
||||
parts = line.split()
|
||||
return int(parts[2]), int(parts[1])
|
||||
|
||||
def ensure_cached_as_image(self):
|
||||
"""Cache this container as a respective image."""
|
||||
alias = self.image_alias
|
||||
if self.incus.find_image([alias]):
|
||||
return
|
||||
self.out.print(" Cleaning apt cache before caching image ...")
|
||||
self.bash("apt-get clean")
|
||||
self.out.print(f" Caching {self.name!r} as '{alias}' ...")
|
||||
self.incus.run(["publish", self.name, f"--alias={alias}", "--force"])
|
||||
self.out.print(f" Image '{alias}' cached.")
|
||||
self.wait_ready()
|
||||
|
||||
|
||||
class RelayContainer(Container):
|
||||
"""Container handle for a chatmail relay.
|
||||
|
||||
Accepts the short relay name (e.g. ``test0``) and derives
|
||||
the Incus container name and mail domain automatically.
|
||||
"""
|
||||
|
||||
def __init__(self, incus, name):
|
||||
super().__init__(
|
||||
incus,
|
||||
f"{name}-localchat",
|
||||
domain=f"_{name}{DOMAIN_SUFFIX}",
|
||||
)
|
||||
self.sname = name
|
||||
self.image_alias = f"localchat-{name}"
|
||||
self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
|
||||
self.zone = incus.lxconfigs_dir / f"{name}.zone"
|
||||
|
||||
def launch(self):
|
||||
"""Launch from localchat-{sname} if cached, else localchat-base."""
|
||||
|
||||
candidates = [self.image_alias]
|
||||
candidates.append(BASE_IMAGE_ALIAS)
|
||||
image = self.incus.find_image(candidates)
|
||||
assert image, f"No deployment base, candidates: {','.join(candidates)}"
|
||||
self.out.print(f" Launching from '{image}' image ...")
|
||||
cfg = []
|
||||
cfg += ("-c", f"{LABEL_KEY}=true")
|
||||
cfg += ("-c", f"user.localchat-domain={self.domain}")
|
||||
self.incus.run(["launch", image, self.name, *cfg])
|
||||
self.bash("rm -f /etc/chatmail-version")
|
||||
return image
|
||||
|
||||
def destroy(self):
|
||||
"""Stop, delete, and clean up config files."""
|
||||
super().destroy()
|
||||
if self.ini.exists():
|
||||
self.ini.unlink()
|
||||
|
||||
def disable_ipv6(self):
|
||||
"""Disable IPv6 inside the container via sysctl."""
|
||||
# incus provides net.* virtualization for LXC containers so that
|
||||
# these sysctls only affect the container's network namespace.
|
||||
self.bash("""
|
||||
sysctl -w net.ipv6.conf.all.disable_ipv6=1
|
||||
sysctl -w net.ipv6.conf.default.disable_ipv6=1
|
||||
""")
|
||||
self.push_file_content(
|
||||
"/etc/sysctl.d/99-disable-ipv6.conf",
|
||||
"""
|
||||
net.ipv6.conf.all.disable_ipv6=1
|
||||
net.ipv6.conf.default.disable_ipv6=1
|
||||
""",
|
||||
)
|
||||
|
||||
def configure_hosts(self, ip):
|
||||
"""Set hostname and /etc/hosts inside the container."""
|
||||
self.bash(f"""
|
||||
echo '{self.name}' > /etc/hostname
|
||||
hostname {self.name}
|
||||
sed -i '/ {self.domain}$/d' /etc/hosts
|
||||
echo '{ip} {self.name} {self.domain}' >> /etc/hosts
|
||||
""")
|
||||
|
||||
def deployed_version(self):
|
||||
"""Read /etc/chatmail-version, or None if absent."""
|
||||
return self.bash("cat /etc/chatmail-version", check=False)
|
||||
|
||||
def deployed_domain(self):
|
||||
"""Read the domain deployed on the container (postfix myhostname)."""
|
||||
return self.bash(
|
||||
"postconf -h myhostname 2>/dev/null",
|
||||
check=False,
|
||||
)
|
||||
|
||||
def verify_ssh(self, ssh_config):
|
||||
"""Verify SSH connectivity to this container."""
|
||||
cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
|
||||
return shell(cmd, timeout=60).returncode == 0
|
||||
|
||||
def configure_dns(self, dns_ip):
|
||||
"""Point this container's resolver at *dns_ip* and verify DNS is reachable."""
|
||||
self.bash(f"""
|
||||
systemctl disable --now systemd-resolved 2>/dev/null || true
|
||||
rm -f /etc/resolv.conf
|
||||
printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
|
||||
mkdir -p /etc/unbound/unbound.conf.d
|
||||
""")
|
||||
self.push_file_content(
|
||||
"/etc/unbound/unbound.conf.d/localchat-forward.conf",
|
||||
f"""
|
||||
server:
|
||||
domain-insecure: "localchat"
|
||||
|
||||
forward-zone:
|
||||
name: "localchat"
|
||||
forward-addr: {dns_ip}
|
||||
""",
|
||||
)
|
||||
self.bash("systemctl restart unbound 2>/dev/null || true")
|
||||
self._wait_dns_reachable(dns_ip)
|
||||
|
||||
def _wait_dns_reachable(self, dns_ip, timeout=10):
|
||||
"""Poll until *dns_ip* answers a DNS query from this container."""
|
||||
if self.bash("which dig", check=False) is None:
|
||||
self.bash(
|
||||
"DEBIAN_FRONTEND=noninteractive "
|
||||
"apt-get install -y dnsutils 2>/dev/null || true"
|
||||
)
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
result = self.bash(
|
||||
f"dig @{dns_ip} . SOA +short +time=1 +tries=1",
|
||||
check=False,
|
||||
)
|
||||
if result and result.strip():
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise DNSConfigurationError(
|
||||
f"DNS at {dns_ip} not reachable from {self.name} after {timeout}s"
|
||||
)
|
||||
|
||||
def write_ini(self, disable_ipv6=False):
|
||||
"""Generate a chatmail.ini config file in lxconfigs/."""
|
||||
from chatmaild.config import write_initial_config
|
||||
|
||||
overrides = {
|
||||
"max_user_send_per_minute": 600,
|
||||
"max_user_send_burst_size": 100,
|
||||
"mtail_address": "127.0.0.1",
|
||||
}
|
||||
if disable_ipv6:
|
||||
overrides["disable_ipv6"] = "True"
|
||||
write_initial_config(self.ini, self.domain, overrides)
|
||||
return self.ini
|
||||
|
||||
|
||||
class DNSContainer(Container):
|
||||
"""Container handle for the PowerDNS name server.
|
||||
|
||||
Manages the authoritative and recursive DNS services required for
|
||||
name resolution in the local testing environment.
|
||||
"""
|
||||
|
||||
def __init__(self, incus):
|
||||
super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
|
||||
self.image_alias = DNS_IMAGE_ALIAS
|
||||
|
||||
def launch(self):
|
||||
"""Launch from localchat-ns if cached, else localchat-base."""
|
||||
image = self.incus.find_image([DNS_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
|
||||
if not image:
|
||||
raise RuntimeError(
|
||||
f"No base image '{BASE_IMAGE_ALIAS}' found. "
|
||||
"Call ensure_base_image() before launching containers."
|
||||
)
|
||||
self.out.print(f" Launching from '{image}' image ...")
|
||||
cfg = []
|
||||
cfg += ("-c", f"{LABEL_KEY}=true")
|
||||
cfg += ("-c", f"user.localchat-domain={self.domain}")
|
||||
self.incus.run(["launch", image, self.name, *cfg])
|
||||
|
||||
def pdnsutil(self, *args, check=True):
|
||||
"""Run ``pdnsutil <args>`` inside the DNS container."""
|
||||
return self.run_cmd("pdnsutil", *args, check=check)
|
||||
|
||||
def replace_rrset(self, zone, name, rtype, ttl, rdata):
|
||||
"""Shortcut for ``pdnsutil replace-rrset``."""
|
||||
self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
|
||||
|
||||
def restart_services(self):
|
||||
"""Restart pdns and pdns-recursor, then wait until DNS is answering."""
|
||||
self.bash("""
|
||||
systemctl restart pdns
|
||||
systemctl restart pdns-recursor || true
|
||||
""")
|
||||
self._wait_dns_ready()
|
||||
|
||||
def _wait_dns_ready(self, timeout=60):
|
||||
"""Poll until the recursor answers a query on port 53."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
result = self.bash(
|
||||
"dig @127.0.0.1 . SOA +short +time=1 +tries=1",
|
||||
check=False,
|
||||
)
|
||||
if result and result.strip():
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise DNSConfigurationError(f"DNS recursor not answering after {timeout}s")
|
||||
|
||||
def ensure(self):
|
||||
"""Create the DNS container with PowerDNS if needed.
|
||||
|
||||
Calls ``super().ensure()`` to create/start the container
|
||||
and set up SSH, then installs PowerDNS and configures
|
||||
the Incus bridge to use this container as DNS.
|
||||
"""
|
||||
super().ensure()
|
||||
self._install_powerdns()
|
||||
self.incus.run(
|
||||
["network", "set", "incusbr0", "dns.mode=none"],
|
||||
check=False,
|
||||
)
|
||||
self.incus.run(
|
||||
["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
"""Stop, delete, and reset bridge DNS config."""
|
||||
super().destroy()
|
||||
self.incus.run(["network", "unset", "incusbr0", "dns.mode"], check=False)
|
||||
self.incus.run(["network", "unset", "incusbr0", "raw.dnsmasq"], check=False)
|
||||
|
||||
def _install_powerdns(self):
|
||||
"""Install and configure PowerDNS if not already present."""
|
||||
if self.run_cmd("which", "pdns_server", check=False) is not None:
|
||||
return
|
||||
|
||||
host_ns = self.incus.get_host_nameservers()
|
||||
ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
|
||||
|
||||
self.bash(f"""
|
||||
systemctl disable --now systemd-resolved 2>/dev/null || true
|
||||
rm -f /etc/resolv.conf
|
||||
printf '{ns_lines}\n' > /etc/resolv.conf
|
||||
|
||||
# Block automatic service startup during package installation
|
||||
printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
|
||||
chmod +x /usr/sbin/policy-rc.d
|
||||
|
||||
apt-get -o DPkg::Lock::Timeout=60 update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
|
||||
|
||||
# Remove the startup block
|
||||
rm /usr/sbin/policy-rc.d
|
||||
|
||||
systemctl stop pdns pdns-recursor || true
|
||||
mkdir -p /var/lib/powerdns
|
||||
sqlite3 /var/lib/powerdns/pdns.sqlite3 \
|
||||
</usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
|
||||
chown -R pdns:pdns /var/lib/powerdns
|
||||
""")
|
||||
|
||||
self.push_file_content(
|
||||
"/etc/powerdns/pdns.conf",
|
||||
"""
|
||||
launch=gsqlite3
|
||||
gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
|
||||
local-address=127.0.0.1
|
||||
local-port=5353
|
||||
""",
|
||||
)
|
||||
|
||||
self.push_file_content(
|
||||
"/etc/powerdns/recursor.conf",
|
||||
"""
|
||||
local-address=0.0.0.0
|
||||
local-port=53
|
||||
forward-zones=localchat=127.0.0.1:5353
|
||||
allow-from=0.0.0.0/0
|
||||
dont-query=
|
||||
dnssec=off
|
||||
""",
|
||||
)
|
||||
|
||||
self.bash("""
|
||||
systemctl start pdns
|
||||
systemctl start pdns-recursor
|
||||
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
|
||||
""")
|
||||
self._wait_dns_ready()
|
||||
|
||||
def reset_dns_records(self, dns_ip, domains):
|
||||
"""Create DNS zones with initial A records via pdnsutil.
|
||||
|
||||
Only sets SOA, NS, and A records as the minimal set
|
||||
needed for SSH connectivity. Full records (MX, TXT, SRV,
|
||||
CNAME, DKIM) are added later by ``cmdeploy dns``.
|
||||
|
||||
Args:
|
||||
dns_ip: IP of the DNS container
|
||||
domains: list of dicts with 'name', 'domain', 'ip'
|
||||
"""
|
||||
for d in domains:
|
||||
domain = d["domain"]
|
||||
ip = d["ip"]
|
||||
self.out.print(f" {domain} -> {ip}")
|
||||
|
||||
# Delete and recreate zone fresh (removes stale records)
|
||||
self.pdnsutil("delete-zone", domain, check=False)
|
||||
self.pdnsutil("create-zone", domain, f"ns.{domain}")
|
||||
|
||||
serial = str(int(time.time()))
|
||||
soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
|
||||
self.replace_rrset(domain, ".", "SOA", "3600", soa)
|
||||
self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
|
||||
self.replace_rrset(domain, ".", "A", "3600", ip)
|
||||
self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
|
||||
|
||||
# AAAA (domain -> container IPv6, if available)
|
||||
ipv6 = d.get("ipv6")
|
||||
if ipv6:
|
||||
self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
|
||||
self.out.print(f" zone reset: SOA, NS, A, AAAA ({ip}, {ipv6})")
|
||||
else:
|
||||
# Remove any stale AAAA record
|
||||
self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
|
||||
self.out.print(f" zone reset: SOA, NS, A ({ip}, IPv4-only)")
|
||||
|
||||
self.restart_services()
|
||||
|
||||
def set_dns_records(self, text):
|
||||
"""Add or overwrite DNS records from standard BIND format.
|
||||
|
||||
Uses ``cmdeploy.dns.parse_zone_records`` to parse.
|
||||
Zones are created automatically from the record names.
|
||||
"""
|
||||
from ..dns import parse_zone_records
|
||||
|
||||
zones_seen = set()
|
||||
|
||||
for name, ttl, rtype, rdata in parse_zone_records(text):
|
||||
# Derive zone from name: find top-level .localchat domain
|
||||
name_parts = name.split(".")
|
||||
zone = name # fallback
|
||||
for i in range(len(name_parts) - 1):
|
||||
if name_parts[i + 1 :] == ["localchat"]:
|
||||
zone = ".".join(name_parts[i:])
|
||||
break
|
||||
|
||||
# Create zone if first time seeing it
|
||||
if zone not in zones_seen:
|
||||
self.pdnsutil(
|
||||
"create-zone",
|
||||
zone,
|
||||
f"ns.{zone}",
|
||||
check=False,
|
||||
)
|
||||
zones_seen.add(zone)
|
||||
|
||||
# Figure out the record name relative to zone
|
||||
if name == zone:
|
||||
relative = "."
|
||||
elif name.endswith(f".{zone}"):
|
||||
relative = name[: -(len(zone) + 1)]
|
||||
else:
|
||||
relative = name
|
||||
|
||||
self.replace_rrset(zone, relative, rtype, ttl, rdata)
|
||||
|
||||
if zones_seen:
|
||||
self.restart_services()
|
||||
@@ -60,6 +60,10 @@ PidFile /run/opendkim/opendkim.pid
|
||||
# by the package dns-root-data.
|
||||
TrustAnchorFile /usr/share/dns/root.key
|
||||
|
||||
# Use the local unbound resolver rather than querying root servers directly.
|
||||
# on IPv4-only hosts opendkim may otherwise try ipv6 requests and time out.
|
||||
Nameservers 127.0.0.1
|
||||
|
||||
# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
|
||||
MTA ORIGINATING
|
||||
|
||||
|
||||
@@ -58,8 +58,8 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
|
||||
dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
|
||||
web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
|
||||
return (
|
||||
f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
|
||||
f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
|
||||
f'{dkim_selector}._domainkey.{mail_domain}. 3600 IN TXT "{dkim_value}"',
|
||||
f'{dkim_selector}._domainkey.{mail_domain}. 3600 IN TXT "{web_dkim_value}"',
|
||||
)
|
||||
|
||||
|
||||
@@ -94,9 +94,11 @@ def check_zonefile(zonefile, verbose=True):
|
||||
if not zf_line.strip() or zf_line.startswith(";"):
|
||||
continue
|
||||
print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
|
||||
zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
|
||||
zf_domain = zf_domain.rstrip(".")
|
||||
zf_value = zf_value.strip()
|
||||
parts = zf_line.split(None, 4)
|
||||
zf_domain = parts[0].rstrip(".")
|
||||
# parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
|
||||
zf_typ = parts[3]
|
||||
zf_value = parts[4].strip()
|
||||
query_value = query_dns(zf_typ, zf_domain)
|
||||
if zf_value != query_value:
|
||||
assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
|
||||
|
||||
@@ -12,13 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
|
||||
``www.<domain>`` and ``mta-sts.<domain>``.
|
||||
"""
|
||||
return [
|
||||
"openssl", "req", "-x509",
|
||||
"-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
|
||||
"-noenc", "-days", str(days),
|
||||
"-keyout", str(key_path),
|
||||
"-out", str(cert_path),
|
||||
"-subj", f"/CN={domain}",
|
||||
"-addext", "extendedKeyUsage=serverAuth,clientAuth",
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-noenc",
|
||||
"-days",
|
||||
str(days),
|
||||
"-keyout",
|
||||
str(key_path),
|
||||
"-out",
|
||||
str(cert_path),
|
||||
"-subj",
|
||||
f"/CN={domain}",
|
||||
# Mark as end-entity cert so it cannot be used as a CA to sign others.
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:FALSE",
|
||||
"-addext",
|
||||
"extendedKeyUsage=serverAuth,clientAuth",
|
||||
"-addext",
|
||||
f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
|
||||
]
|
||||
@@ -40,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
|
||||
|
||||
def configure(self):
|
||||
args = openssl_selfsigned_args(
|
||||
self.mail_domain, self.cert_path, self.key_path,
|
||||
self.mail_domain,
|
||||
self.cert_path,
|
||||
self.key_path,
|
||||
)
|
||||
cmd = shlex.join(args)
|
||||
server.shell(
|
||||
|
||||
@@ -49,8 +49,13 @@ class SSHExec:
|
||||
RemoteError = execnet.RemoteError
|
||||
FuncError = FuncError
|
||||
|
||||
def __init__(self, host, verbose=False, python="python3", timeout=60):
|
||||
self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
|
||||
def __init__(
|
||||
self, host, verbose=False, python="python3", timeout=60, ssh_config=None
|
||||
):
|
||||
spec = f"ssh=root@{host}//python={python}"
|
||||
if ssh_config:
|
||||
spec += f"//ssh_config={ssh_config}"
|
||||
self.gateway = execnet.makegateway(spec)
|
||||
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
|
||||
self.timeout = timeout
|
||||
self.verbose = verbose
|
||||
@@ -87,8 +92,9 @@ class SSHExec:
|
||||
class LocalExec:
|
||||
FuncError = FuncError
|
||||
|
||||
def __init__(self, verbose=False):
|
||||
def __init__(self, verbose=False, docker=False):
|
||||
self.verbose = verbose
|
||||
self.docker = docker
|
||||
|
||||
def __call__(self, call, kwargs=None, log_callback=None):
|
||||
if kwargs is None:
|
||||
@@ -100,6 +106,10 @@ class LocalExec:
|
||||
if not title:
|
||||
title = call.__name__
|
||||
where = "locally"
|
||||
if self.docker:
|
||||
if call == remote.rdns.perform_initial_checks:
|
||||
kwargs["pre_command"] = "docker exec chatmail "
|
||||
where = "in docker"
|
||||
if self.verbose:
|
||||
print_stderr(f"Running {where}: {title}(**{kwargs})")
|
||||
return self(call, kwargs, log_callback=print_stderr)
|
||||
@@ -108,3 +118,46 @@ class LocalExec:
|
||||
res = self(call, kwargs, log_callback=remote.rshell.log_progress)
|
||||
print_stderr()
|
||||
return res
|
||||
|
||||
|
||||
# pyinfra exposes a ``ssh_config_file`` data key that *should* let
|
||||
# paramiko parse an SSH config file directly. In practice it silently
|
||||
# fails to connect (zero hosts / zero operations), so we resolve the
|
||||
# hostname and identity-file ourselves and pass them via
|
||||
# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
|
||||
# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
|
||||
|
||||
|
||||
def _get_from_ssh_config(host, ssh_config_path, key):
|
||||
"""Internal helper to parse a value for a specific key from ssh-config."""
|
||||
current_hosts = []
|
||||
found_value = None
|
||||
with open(ssh_config_path) as f:
|
||||
for raw_line in f:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split(None, 1)
|
||||
if not parts:
|
||||
continue
|
||||
directive = parts[0].lower()
|
||||
if directive == "host":
|
||||
if host in current_hosts and found_value:
|
||||
return found_value
|
||||
current_hosts = parts[1].split()
|
||||
found_value = None
|
||||
elif directive == key.lower():
|
||||
found_value = parts[1]
|
||||
if host in current_hosts and found_value:
|
||||
return found_value
|
||||
return None
|
||||
|
||||
|
||||
def resolve_host_from_ssh_config(host, ssh_config_path):
|
||||
"""Resolve a host alias to its IP from an ssh-config file."""
|
||||
return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
|
||||
|
||||
|
||||
def resolve_key_from_ssh_config(host, ssh_config_path):
|
||||
"""Resolve a host alias to its IdentityFile from an ssh-config file."""
|
||||
return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
; Required DNS entries for chatmail servers
|
||||
zftest.testrun.org. A 135.181.204.127
|
||||
zftest.testrun.org. AAAA 2a01:4f9:c012:52f4::1
|
||||
zftest.testrun.org. MX 10 zftest.testrun.org.
|
||||
_mta-sts.zftest.testrun.org. TXT "v=STSv1; id=202403211706"
|
||||
mta-sts.zftest.testrun.org. CNAME zftest.testrun.org.
|
||||
www.zftest.testrun.org. CNAME zftest.testrun.org.
|
||||
opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
|
||||
; Required DNS entries
|
||||
zftest.testrun.org. 3600 IN A 135.181.204.127
|
||||
zftest.testrun.org. 3600 IN AAAA 2a01:4f9:c012:52f4::1
|
||||
zftest.testrun.org. 3600 IN MX 10 zftest.testrun.org.
|
||||
_mta-sts.zftest.testrun.org. 3600 IN TXT "v=STSv1; id=202403211706"
|
||||
mta-sts.zftest.testrun.org. 3600 IN CNAME zftest.testrun.org.
|
||||
www.zftest.testrun.org. 3600 IN CNAME zftest.testrun.org.
|
||||
opendkim._domainkey.zftest.testrun.org. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
|
||||
|
||||
; Recommended DNS entries
|
||||
_submission._tcp.zftest.testrun.org. SRV 0 1 587 zftest.testrun.org.
|
||||
_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
|
||||
_imap._tcp.zftest.testrun.org. SRV 0 1 143 zftest.testrun.org.
|
||||
_imaps._tcp.zftest.testrun.org. SRV 0 1 993 zftest.testrun.org.
|
||||
zftest.testrun.org. CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
|
||||
zftest.testrun.org. TXT "v=spf1 a:zftest.testrun.org ~all"
|
||||
_dmarc.zftest.testrun.org. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
||||
_adsp._domainkey.zftest.testrun.org. TXT "dkim=discardable"
|
||||
zftest.testrun.org. 3600 IN TXT "v=spf1 a ~all"
|
||||
_dmarc.zftest.testrun.org. 3600 IN TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
||||
zftest.testrun.org. 3600 IN CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
|
||||
_adsp._domainkey.zftest.testrun.org. 3600 IN TXT "dkim=discardable"
|
||||
_submission._tcp.zftest.testrun.org. 3600 IN SRV 0 1 587 zftest.testrun.org.
|
||||
_submissions._tcp.zftest.testrun.org. 3600 IN SRV 0 1 465 zftest.testrun.org.
|
||||
_imap._tcp.zftest.testrun.org. 3600 IN SRV 0 1 143 zftest.testrun.org.
|
||||
_imaps._tcp.zftest.testrun.org. 3600 IN SRV 0 1 993 zftest.testrun.org.
|
||||
|
||||
@@ -89,9 +89,7 @@ def test_concurrent_logins_same_account(
|
||||
assert login_results.get()
|
||||
|
||||
|
||||
def test_no_vrfy(cmfactory, chatmail_config):
|
||||
ac = cmfactory.get_online_account()
|
||||
addr = ac.get_config("addr")
|
||||
def test_no_vrfy(chatmail_config):
|
||||
domain = chatmail_config.mail_domain
|
||||
|
||||
s = smtplib.SMTP(domain)
|
||||
@@ -100,7 +98,7 @@ def test_no_vrfy(cmfactory, chatmail_config):
|
||||
s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
|
||||
result = s.getreply()
|
||||
print(result)
|
||||
s.putcmd("vrfy", addr)
|
||||
s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
|
||||
result2 = s.getreply()
|
||||
print(result2)
|
||||
assert result[0] == result2[0] == 252
|
||||
|
||||
@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
|
||||
def test_newemail_configure(maildomain, rpc, chatmail_config):
|
||||
def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
|
||||
"""Test configuring accounts by scanning a QR code works."""
|
||||
url = f"DCACCOUNT:https://{maildomain}/new"
|
||||
for i in range(3):
|
||||
@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
|
||||
# set_config_from_qr, so fetch credentials via requests instead
|
||||
res = requests.post(f"https://{maildomain}/new", verify=False)
|
||||
data = res.json()
|
||||
rpc.add_or_update_transport(account_id, {
|
||||
"addr": data["email"],
|
||||
"password": data["password"],
|
||||
"imapServer": maildomain,
|
||||
"smtpServer": maildomain,
|
||||
"certificateChecks": "acceptInvalidCertificates",
|
||||
})
|
||||
rpc.add_or_update_transport(
|
||||
account_id,
|
||||
{
|
||||
"addr": data["email"],
|
||||
"password": data["password"],
|
||||
"imapServer": maildomain_ip,
|
||||
"smtpServer": maildomain_ip,
|
||||
"certificateChecks": "acceptInvalidCertificates",
|
||||
},
|
||||
)
|
||||
else:
|
||||
rpc.add_transport_from_qr(account_id, url)
|
||||
|
||||
@@ -12,8 +12,9 @@ from cmdeploy.cmdeploy import get_sshexec
|
||||
|
||||
class TestSSHExecutor:
|
||||
@pytest.fixture(scope="class")
|
||||
def sshexec(self, sshdomain):
|
||||
return get_sshexec(sshdomain)
|
||||
def sshexec(self, sshdomain, pytestconfig):
|
||||
ssh_config = pytestconfig.getoption("ssh_config")
|
||||
return get_sshexec(sshdomain, ssh_config=ssh_config)
|
||||
|
||||
def test_ls(self, sshexec):
|
||||
out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
|
||||
@@ -132,11 +133,10 @@ def test_authenticated_from(cmsetup, maildata):
|
||||
@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
|
||||
def test_reject_missing_dkim(cmsetup, maildata, from_addr):
|
||||
domain = cmsetup.maildomain
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(10)
|
||||
try:
|
||||
sock.connect((domain, 25))
|
||||
except socket.timeout:
|
||||
sock = socket.create_connection((domain, 25), timeout=10)
|
||||
sock.close()
|
||||
except (socket.timeout, OSError):
|
||||
pytest.skip(f"port 25 not reachable for {domain}")
|
||||
|
||||
recipient = cmsetup.gen_users(1)[0]
|
||||
|
||||
@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
|
||||
assert msg2.get_snapshot().text == "message0"
|
||||
|
||||
def test_exceed_quota(
|
||||
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
|
||||
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
|
||||
):
|
||||
"""This is a very slow test as it needs to upload >100MB of mail data
|
||||
before quota is exceeded, and thus depends on the speed of the upload.
|
||||
@@ -92,7 +92,9 @@ class TestEndToEndDeltaChat:
|
||||
lp.sec(f"filling remote inbox for {user}")
|
||||
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
|
||||
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
||||
sshexec = get_sshexec(sshdomain)
|
||||
sshexec = get_sshexec(
|
||||
sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
|
||||
)
|
||||
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
||||
res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
|
||||
assert res["percent"] >= 100
|
||||
|
||||
@@ -3,12 +3,15 @@ import os
|
||||
from cmdeploy.cmdeploy import main
|
||||
|
||||
|
||||
def test_status_cmd(chatmail_config, capsys, request):
|
||||
def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
|
||||
os.chdir(request.config.invocation_params.dir)
|
||||
command = ["status"]
|
||||
if os.getenv("CHATMAIL_SSH"):
|
||||
command.append("--ssh-host")
|
||||
command.append(os.getenv("CHATMAIL_SSH"))
|
||||
ssh_host = pytestconfig.getoption("ssh_host")
|
||||
if ssh_host:
|
||||
command.extend(["--ssh-host", ssh_host])
|
||||
ssh_config = pytestconfig.getoption("ssh_config")
|
||||
if ssh_config:
|
||||
command.extend(["--ssh-config", ssh_config])
|
||||
assert main(command) == 0
|
||||
status_out = capsys.readouterr()
|
||||
print(status_out.out)
|
||||
|
||||
@@ -2,7 +2,9 @@ import imaplib
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import smtplib
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
@@ -18,6 +20,76 @@ def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--slow", action="store_true", default=False, help="also run slow tests"
|
||||
)
|
||||
parser.addoption(
|
||||
"--ssh-host",
|
||||
dest="ssh_host",
|
||||
default=None,
|
||||
help="SSH host (overrides mail_domain for SSH operations).",
|
||||
)
|
||||
parser.addoption(
|
||||
"--ssh-config",
|
||||
dest="ssh_config",
|
||||
default=None,
|
||||
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
|
||||
)
|
||||
|
||||
|
||||
def _parse_ssh_config_hosts(path):
|
||||
"""Parse an OpenSSH config file and return a dict of hostname -> IP."""
|
||||
mapping = {}
|
||||
current_names = []
|
||||
for ln in Path(path).read_text().splitlines():
|
||||
line = ln.strip()
|
||||
m = re.match(r"^Host\s+(.+)", line)
|
||||
if m:
|
||||
current_names = m.group(1).split()
|
||||
continue
|
||||
m = re.match(r"^Hostname\s+(\S+)", line)
|
||||
if m and current_names:
|
||||
ip = m.group(1)
|
||||
for name in current_names:
|
||||
mapping[name] = ip
|
||||
current_names = []
|
||||
return mapping
|
||||
|
||||
|
||||
_original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
|
||||
def _make_patched_getaddrinfo(host_map):
|
||||
"""Return a getaddrinfo that resolves hosts in host_map to their IPs."""
|
||||
|
||||
def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
|
||||
if host in host_map:
|
||||
ip = host_map[host]
|
||||
return _original_getaddrinfo(ip, port, family, type, proto, flags)
|
||||
return _original_getaddrinfo(host, port, family, type, proto, flags)
|
||||
|
||||
return patched_getaddrinfo
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="session")
|
||||
def _setup_localchat_dns(pytestconfig):
|
||||
"""Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
|
||||
ssh_config = pytestconfig.getoption("ssh_config")
|
||||
if not ssh_config or not Path(ssh_config).exists():
|
||||
yield {}
|
||||
return
|
||||
host_map = _parse_ssh_config_hosts(ssh_config)
|
||||
if not host_map:
|
||||
yield {}
|
||||
return
|
||||
socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
|
||||
try:
|
||||
yield host_map
|
||||
finally:
|
||||
socket.getaddrinfo = _original_getaddrinfo
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ssh_config_host_map(_setup_localchat_dns):
|
||||
"""Return the host-name → IP map parsed from ssh-config."""
|
||||
return _setup_localchat_dns
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
@@ -35,6 +107,11 @@ def pytest_runtest_setup(item):
|
||||
|
||||
|
||||
def _get_chatmail_config():
|
||||
ini = os.environ.get("CHATMAIL_INI")
|
||||
if ini:
|
||||
path = Path(ini).resolve()
|
||||
if path.exists():
|
||||
return read_config(path), path
|
||||
current = Path().resolve()
|
||||
while 1:
|
||||
path = current.joinpath("chatmail.ini").resolve()
|
||||
@@ -61,8 +138,14 @@ def maildomain(chatmail_config):
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def sshdomain(maildomain):
|
||||
return os.environ.get("CHATMAIL_SSH", maildomain)
|
||||
def sshdomain(maildomain, pytestconfig):
|
||||
return pytestconfig.getoption("ssh_host") or maildomain
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def maildomain_ip(maildomain, ssh_config_host_map):
|
||||
"""Return the IP for maildomain from ssh-config, or maildomain itself."""
|
||||
return ssh_config_host_map.get(maildomain, maildomain)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -306,12 +389,22 @@ from deltachat_rpc_client import DeltaChat, Rpc
|
||||
class ChatmailACFactory:
|
||||
"""RPC-based account factory for chatmail testing."""
|
||||
|
||||
def __init__(self, rpc, maildomain, gencreds, chatmail_config):
|
||||
def __init__(
|
||||
self,
|
||||
rpc,
|
||||
maildomain,
|
||||
maildomain_ip,
|
||||
gencreds,
|
||||
chatmail_config,
|
||||
ssh_config_host_map,
|
||||
):
|
||||
self.dc = DeltaChat(rpc)
|
||||
self.rpc = rpc
|
||||
self._maildomain = maildomain
|
||||
self._maildomain_ip = maildomain_ip
|
||||
self.gencreds = gencreds
|
||||
self.chatmail_config = chatmail_config
|
||||
self._ssh_config_host_map = ssh_config_host_map
|
||||
|
||||
def _make_transport(self, domain):
|
||||
"""Build a transport config dict for the given domain."""
|
||||
@@ -319,11 +412,13 @@ class ChatmailACFactory:
|
||||
transport = {
|
||||
"addr": addr,
|
||||
"password": password,
|
||||
# Setting server explicitly skips requesting autoconfig XML,
|
||||
# see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
|
||||
"imapServer": domain,
|
||||
"smtpServer": domain,
|
||||
}
|
||||
# To support running against local relays without host DNS resolution
|
||||
# we attempt resolving the domain via ssh-config
|
||||
# because otherwise core fails to find the address
|
||||
server = self._ssh_config_host_map.get(domain)
|
||||
if server is not None:
|
||||
transport.update({"imapServer": server, "smtpServer": server})
|
||||
if self.chatmail_config.tls_cert_mode == "self":
|
||||
transport["certificateChecks"] = "acceptInvalidCertificates"
|
||||
return transport
|
||||
@@ -376,39 +471,56 @@ def rpc(tmp_path_factory):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cmfactory(rpc, gencreds, maildomain, chatmail_config):
|
||||
def cmfactory(
|
||||
rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
|
||||
):
|
||||
"""Return a ChatmailACFactory for creating online Delta Chat accounts."""
|
||||
return ChatmailACFactory(
|
||||
rpc=rpc,
|
||||
maildomain=maildomain,
|
||||
maildomain_ip=maildomain_ip,
|
||||
gencreds=gencreds,
|
||||
chatmail_config=chatmail_config,
|
||||
ssh_config_host_map=ssh_config_host_map,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote(sshdomain):
|
||||
return Remote(sshdomain)
|
||||
def remote(sshdomain, pytestconfig):
|
||||
r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
|
||||
yield r
|
||||
r.close()
|
||||
|
||||
|
||||
class Remote:
|
||||
def __init__(self, sshdomain):
|
||||
def __init__(self, sshdomain, ssh_config=None):
|
||||
self.sshdomain = sshdomain
|
||||
self.ssh_config = ssh_config
|
||||
self._procs = []
|
||||
|
||||
def iter_output(self, logcmd="", ready=None):
|
||||
getjournal = "journalctl -f" if not logcmd else logcmd
|
||||
print(self.sshdomain)
|
||||
match self.sshdomain:
|
||||
case "@local": command = []
|
||||
case "localhost": command = []
|
||||
case _: command = ["ssh", f"root@{self.sshdomain}"]
|
||||
case "@local":
|
||||
command = []
|
||||
case "localhost":
|
||||
command = []
|
||||
case _:
|
||||
command = ["ssh"]
|
||||
if self.ssh_config:
|
||||
command.extend(["-F", self.ssh_config])
|
||||
command.append(f"root@{self.sshdomain}")
|
||||
[command.append(arg) for arg in getjournal.split()]
|
||||
self.popen = subprocess.Popen(
|
||||
popen = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
self._procs.append(popen)
|
||||
while 1:
|
||||
line = self.popen.stdout.readline()
|
||||
line = popen.stdout.readline()
|
||||
res = line.decode().strip().lower()
|
||||
if not res:
|
||||
break
|
||||
@@ -417,6 +529,12 @@ class Remote:
|
||||
ready = None
|
||||
yield res
|
||||
|
||||
def close(self):
|
||||
while self._procs:
|
||||
proc = self._procs.pop()
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lp(request):
|
||||
|
||||
@@ -23,7 +23,10 @@ class TestCmdline:
|
||||
run = parser.parse_args(["run"])
|
||||
assert init and run
|
||||
|
||||
def test_init_not_overwrite(self, capsys):
|
||||
def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
|
||||
monkeypatch.delenv("CHATMAIL_INI", raising=False)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
assert main(["init", "chat.example.org"]) == 0
|
||||
capsys.readouterr()
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from copy import deepcopy
|
||||
import pytest
|
||||
|
||||
from cmdeploy import remote
|
||||
from cmdeploy.dns import check_full_zone, check_initial_remote_data
|
||||
from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -125,18 +125,33 @@ class TestPerformInitialChecks:
|
||||
assert not l
|
||||
|
||||
|
||||
def test_parse_zone_records():
|
||||
text = """
|
||||
; This is a comment
|
||||
some.domain. 3600 IN A 1.1.1.1
|
||||
|
||||
; Another comment
|
||||
www.some.domain. 3600 IN CNAME some.domain.
|
||||
"""
|
||||
records = list(parse_zone_records(text))
|
||||
assert records == [
|
||||
("some.domain", "3600", "A", "1.1.1.1"),
|
||||
("www.some.domain", "3600", "CNAME", "some.domain."),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_zone_records_invalid_line():
|
||||
text = "invalid line"
|
||||
with pytest.raises(ValueError, match="Bad zone record line"):
|
||||
list(parse_zone_records(text))
|
||||
|
||||
|
||||
def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
|
||||
for zf_line in zonefile.split("\n"):
|
||||
if zf_line.startswith("#"):
|
||||
if "Recommended" in zf_line and only_required:
|
||||
return
|
||||
continue
|
||||
if not zf_line.strip():
|
||||
continue
|
||||
zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
|
||||
zf_domain = zf_domain.rstrip(".")
|
||||
zf_value = zf_value.strip()
|
||||
mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
|
||||
if only_required:
|
||||
# Only take records before the "; Recommended" section
|
||||
zonefile = zonefile.split("; Recommended")[0]
|
||||
for name, ttl, rtype, rdata in parse_zone_records(zonefile):
|
||||
mockdns_base.setdefault(rtype, {})[name] = rdata
|
||||
|
||||
|
||||
class MockSSHExec:
|
||||
|
||||
174
cmdeploy/src/cmdeploy/tests/test_lxc.py
Normal file
174
cmdeploy/src/cmdeploy/tests/test_lxc.py
Normal file
@@ -0,0 +1,174 @@
|
||||
"""Tests for cmdeploy lxc-* subcommands."""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from cmdeploy.lxc import cli
|
||||
from cmdeploy.lxc.incus import Incus
|
||||
from cmdeploy.util import Out
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not shutil.which("incus"),
|
||||
reason="incus not installed",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ix():
|
||||
out = Out()
|
||||
return Incus(out)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def lxc_setup():
|
||||
out = Out()
|
||||
ix = Incus(out)
|
||||
ix.get_dns_container().ensure()
|
||||
return ix.list_managed()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def relay_container(lxc_setup):
|
||||
test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
|
||||
relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
|
||||
if not relays:
|
||||
pytest.skip("no test relay containers running")
|
||||
return relays[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cmdeploy():
|
||||
def run(*args):
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "cmdeploy.cmdeploy", *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
return run
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"subcmd, expected, absent",
|
||||
[
|
||||
(None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
|
||||
("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
|
||||
("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
|
||||
("lxc-test", ["--one"], ["--config"]),
|
||||
("lxc-status", [], ["--config"]),
|
||||
("run", ["--ssh-config"], ["--lxc"]),
|
||||
("dns", ["--ssh-config"], []),
|
||||
("test", ["--ssh-config"], []),
|
||||
("status", ["--ssh-config"], []),
|
||||
],
|
||||
)
|
||||
def test_help_options(cmdeploy, subcmd, expected, absent):
|
||||
args = [subcmd, "--help"] if subcmd else ["--help"]
|
||||
result = cmdeploy(*args)
|
||||
output = result.stdout + result.stderr
|
||||
assert result.returncode == 0
|
||||
for flag in expected:
|
||||
assert flag in output
|
||||
for flag in absent:
|
||||
assert flag not in output
|
||||
|
||||
|
||||
class TestSSHConfig:
|
||||
def test_lxconfigs(self, ix, lxc_setup):
|
||||
d = ix.lxconfigs_dir
|
||||
assert d.name == "lxconfigs"
|
||||
assert d.exists()
|
||||
path = ix.ssh_config_path
|
||||
assert path.name == "ssh-config"
|
||||
assert path.parent.name == "lxconfigs"
|
||||
|
||||
def test_write_ssh_config(self, ix, lxc_setup):
|
||||
path = ix.write_ssh_config()
|
||||
assert path.exists()
|
||||
text = path.read_text()
|
||||
|
||||
for c in lxc_setup:
|
||||
if c.get("ip"):
|
||||
assert c["name"] in text
|
||||
assert f"Hostname {c['ip']}" in text
|
||||
|
||||
assert "User root" in text
|
||||
assert "IdentityFile" in text
|
||||
assert "StrictHostKeyChecking accept-new" in text
|
||||
|
||||
|
||||
def test_dns(ix, relay_container):
|
||||
def dig(qname, qtype):
|
||||
ct = ix.get_dns_container()
|
||||
return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
|
||||
|
||||
domain = relay_container["domain"]
|
||||
assert dig(domain, "A") == relay_container["ip"]
|
||||
assert domain in dig(domain, "MX")
|
||||
assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
|
||||
|
||||
|
||||
class TestLxcStatus:
|
||||
def test_cli_lxc_status_help(self, cmdeploy):
|
||||
result = cmdeploy("lxc-status", "--help")
|
||||
assert result.returncode == 0
|
||||
assert "status" in result.stdout.lower()
|
||||
|
||||
def test_shows_containers(self, lxc_setup, capsys):
|
||||
class QuietOut(Out):
|
||||
def red(self, msg, **kw):
|
||||
pass
|
||||
|
||||
def green(self, msg, **kw):
|
||||
pass
|
||||
|
||||
ret = cli.lxc_status_cmd(None, QuietOut())
|
||||
assert ret == 0
|
||||
captured = capsys.readouterr().out
|
||||
assert "ns-localchat" in captured
|
||||
assert "running" in captured
|
||||
|
||||
def test_deploy_freshness(self, ix, monkeypatch):
|
||||
ct = ix.get_container("x")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
|
||||
lambda _self: "abc123def456",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"cmdeploy.lxc.incus.RelayContainer.deployed_domain",
|
||||
lambda _self: ct.domain,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"cmdeploy.lxc.cli.get_version_string",
|
||||
lambda: "abc123def456",
|
||||
)
|
||||
assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
|
||||
assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
|
||||
|
||||
# Hash matches but local has uncommitted changes
|
||||
monkeypatch.setattr(
|
||||
"cmdeploy.lxc.cli.get_version_string",
|
||||
lambda: "abc123def456\ndiff --git a/foo",
|
||||
)
|
||||
assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
|
||||
lambda _self: None,
|
||||
)
|
||||
assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)
|
||||
120
cmdeploy/src/cmdeploy/tests/test_util.py
Normal file
120
cmdeploy/src/cmdeploy/tests/test_util.py
Normal file
@@ -0,0 +1,120 @@
|
||||
import sys
|
||||
|
||||
from cmdeploy.util import Out, collapse, get_git_hash, get_version_string, shell
|
||||
|
||||
|
||||
class TestOut:
|
||||
def test_prefix_default(self, capsys):
|
||||
out = Out()
|
||||
out.print("hello")
|
||||
assert capsys.readouterr().out == "hello\n"
|
||||
|
||||
def test_prefix_custom(self, capsys):
|
||||
out = Out(prefix=">> ")
|
||||
out.print("hello")
|
||||
assert capsys.readouterr().out == ">> hello\n"
|
||||
|
||||
def test_prefix_print_file(self):
|
||||
import io
|
||||
|
||||
buf = io.StringIO()
|
||||
out = Out(prefix=":: ")
|
||||
out.print("msg", file=buf)
|
||||
assert ":: msg" in buf.getvalue()
|
||||
|
||||
def test_new_prefixed_out(self, capsys):
|
||||
parent = Out(prefix="A")
|
||||
child = parent.new_prefixed_out("B")
|
||||
child.print("x")
|
||||
assert capsys.readouterr().out == "ABx\n"
|
||||
# shares section_timings
|
||||
assert child.section_timings is parent.section_timings
|
||||
|
||||
def test_section_no_auto_indent(self, capsys):
|
||||
out = Out(prefix="")
|
||||
with out.section("test"):
|
||||
out.print("inside")
|
||||
captured = capsys.readouterr().out
|
||||
# "inside" should NOT be indented by section()
|
||||
lines = captured.strip().splitlines()
|
||||
inside_line = [l for l in lines if "inside" in l][0]
|
||||
assert inside_line == "inside"
|
||||
|
||||
def test_section_records_timing(self):
|
||||
out = Out()
|
||||
with out.section("s1"):
|
||||
pass
|
||||
assert len(out.section_timings) == 1
|
||||
assert out.section_timings[0][0] == "s1"
|
||||
|
||||
def test_shell_failure_shows_output(self):
|
||||
"""When a shell command fails, its output and exit code are shown."""
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from cmdeploy.util import Out; Out(prefix='').shell("
|
||||
"\"echo 'boom on stderr' >&2; exit 42\")",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
# the command's stderr is merged into stdout by Popen
|
||||
assert "boom on stderr" in result.stdout
|
||||
# Out.red() prints the failure notice to stderr
|
||||
assert "exit code 42" in result.stderr
|
||||
|
||||
|
||||
def test_collapse():
|
||||
text = """
|
||||
line 1
|
||||
line 2
|
||||
"""
|
||||
assert collapse(text) == "line 1 line 2"
|
||||
assert collapse(" single line ") == "single line"
|
||||
|
||||
|
||||
def test_git_helpers_no_git(tmp_path):
|
||||
# Not a git repo
|
||||
assert get_git_hash(root=tmp_path) is None
|
||||
assert get_version_string(root=tmp_path) == "unknown"
|
||||
|
||||
|
||||
def test_git_helpers_empty_repo(tmp_path):
|
||||
shell("git init", cwd=tmp_path, check=True)
|
||||
# No commits yet
|
||||
assert get_git_hash(root=tmp_path) is None
|
||||
assert get_version_string(root=tmp_path) == "unknown"
|
||||
|
||||
|
||||
def test_git_helpers_with_commits_and_diffs(tmp_path):
|
||||
shell("git init", cwd=tmp_path, check=True)
|
||||
shell("git config user.email you@example.com", cwd=tmp_path, check=True)
|
||||
shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
|
||||
|
||||
# First commit
|
||||
path = tmp_path / "file.txt"
|
||||
path.write_text("content")
|
||||
shell("git add file.txt", cwd=tmp_path, check=True)
|
||||
shell("git commit -m initial", cwd=tmp_path, check=True)
|
||||
|
||||
git_hash = get_git_hash(root=tmp_path)
|
||||
assert len(git_hash) >= 7 # usually 40, but git is git
|
||||
assert get_version_string(root=tmp_path) == git_hash
|
||||
|
||||
# Create a diff
|
||||
path.write_text("new content")
|
||||
v = get_version_string(root=tmp_path)
|
||||
assert v.startswith(git_hash + "\n")
|
||||
assert "new content" in v
|
||||
assert not v.endswith("\n")
|
||||
|
||||
# Commit again -> no diff
|
||||
shell("git add file.txt", cwd=tmp_path, check=True)
|
||||
shell("git commit -m second", cwd=tmp_path, check=True)
|
||||
new_hash = get_git_hash(root=tmp_path)
|
||||
assert new_hash != git_hash
|
||||
assert get_version_string(root=tmp_path) == new_hash
|
||||
169
cmdeploy/src/cmdeploy/util.py
Normal file
169
cmdeploy/src/cmdeploy/util.py
Normal file
@@ -0,0 +1,169 @@
|
||||
"""Shared utility functions for cmdeploy."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from termcolor import colored
|
||||
|
||||
|
||||
class Out:
|
||||
"""Convenience output printer providing coloring and section formatting."""
|
||||
|
||||
def __init__(self, prefix="", verbosity=0):
|
||||
self.section_timings = []
|
||||
self.prefix = prefix
|
||||
self.sepchar = "\u2501"
|
||||
self.verbosity = verbosity
|
||||
env_width = os.environ.get("_CMDEPLOY_WIDTH")
|
||||
if env_width:
|
||||
self.section_width = int(env_width)
|
||||
else:
|
||||
self.section_width = shutil.get_terminal_size((80, 24)).columns
|
||||
|
||||
def new_prefixed_out(self, newprefix=" "):
|
||||
"""Return a new Out with an extended prefix,
|
||||
sharing section_timings with the parent.
|
||||
"""
|
||||
out = Out(
|
||||
prefix=self.prefix + newprefix,
|
||||
verbosity=self.verbosity,
|
||||
)
|
||||
out.section_timings = self.section_timings
|
||||
return out
|
||||
|
||||
def red(self, msg, file=sys.stderr):
|
||||
print(colored(self.prefix + msg, "red"), file=file, flush=True)
|
||||
|
||||
def green(self, msg, file=sys.stderr):
|
||||
print(colored(self.prefix + msg, "green"), file=file, flush=True)
|
||||
|
||||
def print(self, msg="", **kwargs):
|
||||
"""Print to stdout with automatic flush."""
|
||||
if msg:
|
||||
msg = self.prefix + msg
|
||||
print(msg, flush=True, **kwargs)
|
||||
|
||||
def _format_header(self, title):
|
||||
"""Return a formatted section header string."""
|
||||
width = self.section_width - len(self.prefix)
|
||||
bar = self.sepchar * (width - len(title) - 5)
|
||||
return f"{self.sepchar * 3} {title} {bar}"
|
||||
|
||||
@contextmanager
|
||||
def section(self, title):
|
||||
"""Context manager that prints a section header and records elapsed time."""
|
||||
self.green(self._format_header(title))
|
||||
t0 = time.time()
|
||||
yield
|
||||
elapsed = time.time() - t0
|
||||
self.section_timings.append((title, elapsed))
|
||||
|
||||
def section_line(self, title):
|
||||
"""Print a section header without timing."""
|
||||
self.green(self._format_header(title))
|
||||
|
||||
def shell(self, cmd, quiet=False, **kwargs):
|
||||
"""Print *cmd*, run it, and re-print its output with the current prefix.
|
||||
|
||||
*cmd* is passed through :func:`collapse`, so callers
|
||||
can use triple-quoted f-strings freely.
|
||||
Stdout and stderr are merged, read line-by-line,
|
||||
and each line is printed with ``self.prefix`` prepended.
|
||||
When the command exits non-zero, a red error line is printed.
|
||||
"""
|
||||
cmd = collapse(cmd)
|
||||
if not quiet:
|
||||
self.print(f"$ {cmd}")
|
||||
indent = self.prefix + " "
|
||||
env = kwargs.pop("env", None)
|
||||
if env is None:
|
||||
env = os.environ.copy()
|
||||
env["_CMDEPLOY_WIDTH"] = str(self.section_width - len(indent))
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
shell=True,
|
||||
text=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
**kwargs,
|
||||
)
|
||||
for line in proc.stdout:
|
||||
sys.stdout.write(indent + line)
|
||||
sys.stdout.flush()
|
||||
ret = proc.wait()
|
||||
if ret:
|
||||
self.red(f"command failed with exit code {ret}: {cmd}")
|
||||
return ret
|
||||
|
||||
|
||||
def _project_root():
|
||||
"""Return the project root directory."""
|
||||
return Path(__file__).resolve().parent.parent.parent.parent
|
||||
|
||||
|
||||
def collapse(text):
|
||||
"""Dedent, join lines, and strip a (triple-quoted) string.
|
||||
|
||||
Handy for writing shell commands across multiple lines::
|
||||
|
||||
cmd = collapse(f\"""
|
||||
cmdeploy run
|
||||
--config {ct.ini}
|
||||
--ssh-host {ct.domain}
|
||||
\""")
|
||||
"""
|
||||
return textwrap.dedent(text).replace("\n", " ").strip()
|
||||
|
||||
|
||||
def shell(cmd, check=False, **kwargs):
|
||||
"""Run a shell command string with sensible defaults.
|
||||
|
||||
*cmd* is passed through :func:`collapse` first, so callers
|
||||
can use triple-quoted f-strings freely.
|
||||
Captures stdout/stderr by default; pass ``capture_output=False``
|
||||
to stream output to the terminal instead.
|
||||
"""
|
||||
if "capture_output" not in kwargs and "stdout" not in kwargs:
|
||||
kwargs["capture_output"] = True
|
||||
kwargs.setdefault("stdin", subprocess.DEVNULL)
|
||||
return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
|
||||
|
||||
|
||||
def get_git_hash(root=None):
|
||||
"""Return the local HEAD commit hash, or None."""
|
||||
if root is None:
|
||||
root = _project_root()
|
||||
result = shell(
|
||||
"git rev-parse HEAD",
|
||||
cwd=str(root),
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
return None
|
||||
|
||||
|
||||
def get_version_string(root=None):
|
||||
"""Return ``git_hash\\ngit_diff`` for the local working tree.
|
||||
|
||||
Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
|
||||
``/etc/chatmail-version`` and by ``lxc-status`` to compare
|
||||
the deployed state against the local checkout.
|
||||
"""
|
||||
if root is None:
|
||||
root = _project_root()
|
||||
git_hash = get_git_hash(root=root) or "unknown"
|
||||
try:
|
||||
git_diff = shell("git diff", cwd=str(root)).stdout.strip()
|
||||
except Exception:
|
||||
git_diff = ""
|
||||
if git_diff:
|
||||
return f"{git_hash}\n{git_diff}"
|
||||
return git_hash
|
||||
@@ -15,7 +15,7 @@ author = 'chatmail collective'
|
||||
|
||||
extensions = [
|
||||
#'sphinx.ext.autodoc',
|
||||
#'sphinx.ext.viewdoc',
|
||||
#'sphinx.ext.viewcode',
|
||||
'sphinxcontrib.mermaid',
|
||||
]
|
||||
|
||||
|
||||
@@ -1,264 +0,0 @@
|
||||
Docker installation
|
||||
===================
|
||||
|
||||
This section provides instructions for installing a chatmail relay
|
||||
using Docker Compose.
|
||||
|
||||
.. note::
|
||||
|
||||
- Docker support is experimental, CI builds and tests the image automatically, but please report bugs.
|
||||
- The image wraps the cmdeploy process detailed in the :doc:`getting_started` instructions in a Debian-systemd image with r/w access to `/sys/fs`
|
||||
- Currently amd64-only (arm64 should work but is untested).
|
||||
|
||||
|
||||
Setup Preparation
|
||||
-----------------
|
||||
|
||||
We use ``chat.example.org`` as the chatmail domain in the following
|
||||
steps. Please substitute it with your own domain.
|
||||
|
||||
1. Install docker and docker compose v2 (check with `docker compose version`), install, e.g., on
|
||||
- Debian 12 through the `official install instructions <https://docs.docker.com/engine/install/debian/#install-using-the-repository>`_
|
||||
- Debian 13+ with `apt install docker docker-compose`
|
||||
|
||||
2. Setup the initial DNS records.
|
||||
The following is an example in the familiar BIND zone file format with
|
||||
a TTL of 1 hour (3600 seconds).
|
||||
Please substitute your domain and IP addresses.
|
||||
|
||||
::
|
||||
|
||||
chat.example.org. 3600 IN A 198.51.100.5
|
||||
chat.example.org. 3600 IN AAAA 2001:db8::5
|
||||
www.chat.example.org. 3600 IN CNAME chat.example.org.
|
||||
mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
|
||||
|
||||
3. Configure kernel parameters on the host, as these can not be set from the container::
|
||||
|
||||
echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
|
||||
echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
|
||||
sudo sysctl --system
|
||||
|
||||
|
||||
Docker Compose Setup
|
||||
--------------------
|
||||
|
||||
Pre-built images are available from GitHub Container Registry. The
|
||||
``main`` branch and tagged releases are pushed automatically by CI::
|
||||
|
||||
docker pull ghcr.io/chatmail/relay:main # latest main branch
|
||||
docker pull ghcr.io/chatmail/relay:1.2.3 # tagged release
|
||||
|
||||
|
||||
Create service directory
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Either:
|
||||
|
||||
- Create a service directory and download the compose files::
|
||||
|
||||
mkdir -p /srv/chatmail-relay && cd /srv/chatmail-relay
|
||||
wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker/docker-compose.yaml
|
||||
wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker/docker-compose.override.yaml.example -O docker-compose.override.yaml
|
||||
|
||||
- or clone the chatmail repo and enter the docker directory::
|
||||
|
||||
git clone https://github.com/chatmail/relay
|
||||
cd relay/docker
|
||||
|
||||
|
||||
Customize and start
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
1. Set the fully qualified domain name of the relay::
|
||||
|
||||
echo 'MAIL_DOMAIN=chat.example.org' > .env
|
||||
|
||||
The container generates a ``chatmail.ini`` with defaults from
|
||||
``MAIL_DOMAIN`` on first start. To customize chatmail settings, mount
|
||||
your own ``chatmail.ini`` instead (see `Custom chatmail.ini`_ below).
|
||||
|
||||
2. All local customizations (data paths, extra volumes, config mounts) go in
|
||||
``docker-compose.override.yaml``, which Compose merges automatically with
|
||||
the base file. By default, all data is stored in docker volumes, you will
|
||||
likely want to at least create and configure the mail storage location, but
|
||||
you might also want to configure external TLS certificates there.
|
||||
|
||||
3. Start the container::
|
||||
|
||||
docker compose up -d
|
||||
docker compose logs -f chatmail # view logs, Ctrl+C to exit
|
||||
|
||||
4. After installation is complete, open ``https://chat.example.org`` in
|
||||
your browser.
|
||||
|
||||
Finish install and test
|
||||
-----------------------
|
||||
|
||||
You can test the installation with::
|
||||
|
||||
pip install cmping chat.example.org # or
|
||||
uvx cmping chat.example.org # if you use https://docs.astral.sh/uv/
|
||||
|
||||
You should check and extend your DNS records for better interoperability::
|
||||
|
||||
# Show required DNS records
|
||||
docker exec chatmail cmdeploy dns --ssh-host @local
|
||||
|
||||
You can check server status with::
|
||||
|
||||
docker exec chatmail cmdeploy status --ssh-host @local
|
||||
|
||||
You can run some benchmarks (can also run from any machine with cmdeploy installed)::
|
||||
|
||||
docker exec chatmail cmdeploy bench
|
||||
|
||||
You can run the test suite with::
|
||||
|
||||
docker exec chatmail cmdeploy test --ssh-host localhost
|
||||
|
||||
You can look at logs::
|
||||
|
||||
docker exec chatmail journalctl -fu postfix@-
|
||||
|
||||
|
||||
Customization
|
||||
-------------
|
||||
|
||||
Website
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
You can customize the chatmail landing page by mounting a directory with
|
||||
your own website source files.
|
||||
|
||||
1. Create a directory with your custom website source::
|
||||
|
||||
mkdir -p ./custom/www/src
|
||||
nano ./custom/www/src/index.md
|
||||
|
||||
2. Add the volume mount in ``docker-compose.override.yaml``::
|
||||
|
||||
services:
|
||||
chatmail:
|
||||
volumes:
|
||||
- ./custom/www:/opt/chatmail-www
|
||||
|
||||
3. Restart the service::
|
||||
|
||||
docker compose down
|
||||
docker compose up -d
|
||||
|
||||
|
||||
Custom chatmail.ini
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If you want to go beyond simply setting the ``MAIL_DOMAIN`` in ``.env``, you
|
||||
can use a regular `chatmail.ini` to give you full control.
|
||||
|
||||
1. Extract the generated config from a running container::
|
||||
|
||||
docker cp chatmail:/etc/chatmail/chatmail.ini ./chatmail.ini
|
||||
|
||||
2. Edit ``chatmail.ini`` as needed.
|
||||
|
||||
3. Add the volume mount in ``docker-compose.override.yaml`` ::
|
||||
|
||||
services:
|
||||
chatmail:
|
||||
volumes:
|
||||
- ./chatmail.ini:/etc/chatmail/chatmail.ini
|
||||
|
||||
4. Restart the container, the container skips generating a new one: ::
|
||||
|
||||
docker compose down && docker compose up -d
|
||||
|
||||
|
||||
External TLS certificates
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If TLS certificates are managed outside the container (e.g. by certbot,
|
||||
acmetool, or Traefik on the host), mount them into the container and set
|
||||
``TLS_EXTERNAL_CERT_AND_KEY`` in ``docker-compose.override.yaml``.
|
||||
Changed certificates are picked up automatically via inotify.
|
||||
See the examples in the example override and :ref:`external-tls` in the getting started guide for details.
|
||||
|
||||
|
||||
Migrating from a bare-metal install
|
||||
------------------------------------
|
||||
|
||||
If you have an existing bare-metal chatmail installation and want to
|
||||
switch to Docker:
|
||||
|
||||
1. Stop all existing services::
|
||||
|
||||
systemctl stop postfix dovecot doveauth nginx opendkim unbound \
|
||||
acmetool-redirector filtermail filtermail-incoming chatmail-turn \
|
||||
iroh-relay chatmail-metadata lastlogin mtail
|
||||
systemctl disable postfix dovecot doveauth nginx opendkim unbound \
|
||||
acmetool-redirector filtermail filtermail-incoming chatmail-turn \
|
||||
iroh-relay chatmail-metadata lastlogin mtail
|
||||
|
||||
2. Copy your existing ``chatmail.ini`` and mount it into the container
|
||||
(see `Custom chatmail.ini`_ above)::
|
||||
|
||||
cp /usr/local/lib/chatmaild/chatmail.ini ./chatmail.ini
|
||||
|
||||
3. Copy persistent data into the ``./data/`` subdirectories (for example, as configured in `Customize and start`_) ::
|
||||
|
||||
mkdir -p data/dkim data/certs data/mail
|
||||
|
||||
# DKIM keys
|
||||
cp -a /etc/dkimkeys/* data/dkim/
|
||||
|
||||
# TLS certificates
|
||||
rsync -a /var/lib/acme/ data/certs/
|
||||
|
||||
Note that ownership of dkim and acme is adjusted on container start.
|
||||
|
||||
For the mail directory::
|
||||
|
||||
rsync -a /home/vmail/ data/mail/
|
||||
|
||||
Alternatively, mount ``/home/vmail`` directly by changing the volume
|
||||
in ``docker-compose-override.yaml``::
|
||||
|
||||
- /home/vmail:/home/vmail
|
||||
|
||||
The three ``./data/`` subdirectories cover all persistent state.
|
||||
Everything else is regenerated by the ``configure`` and ``activate``
|
||||
stages on container start.
|
||||
|
||||
Building the image
|
||||
------------------
|
||||
|
||||
Clone the repository and build the Docker image::
|
||||
|
||||
git clone https://github.com/chatmail/relay
|
||||
cd relay
|
||||
docker/build.sh
|
||||
|
||||
The build bakes all binaries, Python packages, and the install stage
|
||||
into the image. After building, only the ``docker/`` directory and a ``.env``
|
||||
with ``MAIL_DOMAIN`` are needed to run the container. The `build.sh` passes the
|
||||
git hash onto the docker build so it can be determined if there has been a
|
||||
change that warrants a redeploy.
|
||||
|
||||
You can transfer a locally built image to your server directly (pigz is parallel `gzip` which can be used instead as well) ::
|
||||
|
||||
docker save chatmail-relay:latest | pigz | ssh chat.example.org 'pigz -d | docker load'
|
||||
|
||||
|
||||
Forcing a full reinstall
|
||||
------------------------
|
||||
|
||||
On container start, only the ``configure`` and ``activate`` stages run by default.
|
||||
|
||||
To force a full reinstall (e.g. after updating the source), either
|
||||
rebuild the image::
|
||||
|
||||
docker compose build chatmail
|
||||
docker compose up -d
|
||||
|
||||
Or override the stages at runtime without rebuilding::
|
||||
|
||||
CMDEPLOY_STAGES="install,configure,activate" docker compose up -d
|
||||
@@ -98,12 +98,6 @@ steps. Please substitute it with your own domain.
|
||||
configure at your DNS provider (it can take some time until they are
|
||||
public).
|
||||
|
||||
Docker installation
|
||||
-------------------
|
||||
|
||||
There is experimental support for running chatmail via Docker Compose.
|
||||
See :doc:`docker` for full setup instructions.
|
||||
|
||||
Other helpful commands
|
||||
----------------------
|
||||
|
||||
|
||||
@@ -13,9 +13,9 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
|
||||
:maxdepth: 5
|
||||
|
||||
getting_started
|
||||
docker
|
||||
proxy
|
||||
migrate
|
||||
overview
|
||||
lxc
|
||||
related
|
||||
faq
|
||||
|
||||
271
doc/source/lxc.rst
Normal file
271
doc/source/lxc.rst
Normal file
@@ -0,0 +1,271 @@
|
||||
Local testing with LXC/Incus
|
||||
============================
|
||||
|
||||
The ``cmdeploy`` tool includes support for running
|
||||
chatmail relays inside local
|
||||
`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
|
||||
This is meant for development, testing, and CI
|
||||
without requiring a remote server.
|
||||
LXC system containers are lightweight virtual machines
|
||||
that share the host's kernel but run their own init system,
|
||||
package manager, and network stack,
|
||||
so the cmdeploy deployment scripts work pretty much
|
||||
as they would on a real Debian server or cloud VPS.
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
Install `Incus <https://linuxcontainers.org/incus/>`_
|
||||
(LXC container manager).
|
||||
See the `official installation guide
|
||||
<https://linuxcontainers.org/incus/docs/main/installing/>`_
|
||||
for full details.
|
||||
|
||||
After installing incus, initialise and grant yourself access::
|
||||
|
||||
sudo incus admin init --minimal
|
||||
sudo usermod -aG incus-admin $USER
|
||||
|
||||
|
||||
.. caution::
|
||||
|
||||
Adding yourself to ``incus-admin`` grants effective root access
|
||||
to the host: any member can mount host directories into a container
|
||||
and manipulate them as root.
|
||||
This is fine for local testing of your own relay branches,
|
||||
but do **not** use it for production setups
|
||||
or for testing untrusted relay branches from others.
|
||||
|
||||
.. warning::
|
||||
|
||||
You **must now log out and back in** (or run ``newgrp incus-admin``)
|
||||
after adding yourself to the group.
|
||||
Without this, all ``cmdeploy lxc-*`` commands
|
||||
will fail with permission errors.
|
||||
|
||||
Verify the installation works by running ``incus list``,
|
||||
which should print an empty table without errors.
|
||||
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
|
||||
::
|
||||
|
||||
cd relay
|
||||
scripts/initenv.sh # bootstrap venv
|
||||
source venv/bin/activate # activate venv
|
||||
cmdeploy lxc-test # create containers, deploy, test
|
||||
|
||||
The ``lxc-test`` command provides an automated way
|
||||
to run the full deployment and test pipeline.
|
||||
It executes several ``cmdeploy`` subcommands in sequential steps.
|
||||
If a step fails, you can copy-paste the printed command
|
||||
and run it manually to debug.
|
||||
No host DNS delegation or ``~/.ssh/config`` changes are needed
|
||||
because ``lxc-test`` passes the required SSH and DNS options directly.
|
||||
|
||||
|
||||
CLI reference
|
||||
--------------
|
||||
|
||||
``lxc-start [--ipv4-only] [--run] [NAME ...]``
|
||||
Create and start containers.
|
||||
Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
|
||||
Pass one or more ``NAME`` arguments to create user relay containers instead
|
||||
(e.g. ``cmdeploy lxc-start myrelay``).
|
||||
Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
|
||||
producing an IPv4-only relay.
|
||||
Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
|
||||
Generates ``lxconfigs/ssh-config``.
|
||||
It reuses existing containers and resets DNS zones to minimal records.
|
||||
|
||||
``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
|
||||
Stop relay containers.
|
||||
Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
|
||||
Pass ``NAME`` to stop specific containers.
|
||||
Use ``--destroy`` to also delete the containers and their config files.
|
||||
Use ``--destroy-all`` to additionally destroy
|
||||
the ``ns-localchat`` DNS container **and** remove
|
||||
the cached ``localchat-base`` and ``localchat-relay``
|
||||
images, giving a fully clean slate for the next ``lxc-test``.
|
||||
User containers are **never** destroyed unless named explicitly.
|
||||
|
||||
``lxc-test [--one]``
|
||||
By default creates, deploys, and tests both ``test0`` and ``test1``
|
||||
for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
|
||||
test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
|
||||
Pass ``--one`` to only deploy and test against ``test0``
|
||||
(skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
|
||||
|
||||
``lxc-status``
|
||||
Show live status of all LXC containers (including the DNS container),
|
||||
deploy freshness (comparing ``/etc/chatmail-version``
|
||||
against local ``git rev-parse HEAD`` and ``git diff``),
|
||||
SSH config inclusion, and host DNS forwarding for ``.localchat``.
|
||||
Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
|
||||
**STALE** (different commit), or **NOT DEPLOYED**.
|
||||
|
||||
|
||||
Container types
|
||||
-----------------
|
||||
|
||||
**Test relay containers** (``test0-localchat``, ``test1-localchat``)
|
||||
Created automatically by ``lxc-test``.
|
||||
**test0** has IPv4 and IPv6 configured,
|
||||
**test1** is IPv4-only (``disable_ipv6 = True``).
|
||||
|
||||
**User relay containers** (``<name>-localchat``)
|
||||
Created by ``cmdeploy lxc-start <name>``
|
||||
where ``<name>`` does not start with ``test``.
|
||||
These are personal development instances,
|
||||
never touched by ``lxc-stop --destroy`` unless named explicitly.
|
||||
|
||||
**DNS container** (``ns-localchat``)
|
||||
Singleton container running PowerDNS.
|
||||
Created automatically when any relay is started.
|
||||
|
||||
|
||||
.. _lxc-ssh-config:
|
||||
|
||||
SSH configuration
|
||||
-----------------
|
||||
|
||||
``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
|
||||
a standard OpenSSH config file mapping every container name,
|
||||
its domain, and a short alias to the container's IP address::
|
||||
|
||||
Host test0-localchat _test0.localchat _test0
|
||||
Hostname 10.204.0.42
|
||||
User root
|
||||
IdentityFile /path/to/relay/lxconfigs/id_localchat
|
||||
IdentitiesOnly yes
|
||||
StrictHostKeyChecking accept-new
|
||||
UserKnownHostsFile /dev/null
|
||||
LogLevel ERROR
|
||||
|
||||
All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
|
||||
accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
|
||||
``lxc-test`` passes it automatically.
|
||||
|
||||
**Using containers from the host shell:**
|
||||
|
||||
To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
|
||||
|
||||
Include /absolute/path/to/relay/lxconfigs/ssh-config
|
||||
|
||||
|
||||
.. _lxc-dns-setup:
|
||||
.. _localchat-tld:
|
||||
|
||||
``.localchat`` DNS and name resolution
|
||||
---------------------------------------
|
||||
|
||||
All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
|
||||
(e.g. ``_test0.localchat``, ``_test1.localchat``),
|
||||
a non-delegated suffix that exists only within the local PowerDNS infrastructure.
|
||||
A dedicated DNS container (``ns-localchat``)
|
||||
is created so that local test relays interact
|
||||
with DNS similar to a regular public Internet setup.
|
||||
On first start, ``cmdeploy lxc-start`` creates this container
|
||||
running two `PowerDNS <https://www.powerdns.com/>`_ services:
|
||||
|
||||
* **pdns-server** (authoritative) serves ``.localchat``
|
||||
zones from a local SQLite database.
|
||||
|
||||
* **pdns-recursor** (recursive) listens on the Incus
|
||||
bridge so all containers can use it.
|
||||
Forwards ``.localchat`` queries to the local
|
||||
authoritative server and resolves everything else recursively.
|
||||
|
||||
After the DNS container is up, ``lxc-start`` configures the Incus bridge
|
||||
to advertise its IP via DHCP and disables Incus's own DNS.
|
||||
DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
|
||||
|
||||
1. **``lxc-start``** resets each relay zone to
|
||||
**SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
|
||||
If host DNS resolution is configured, users can
|
||||
afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
|
||||
--ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
|
||||
LXC subcommands do not depend on host DNS resolution
|
||||
and resolve addresses via ``lxconfigs/ssh-config``.
|
||||
|
||||
2. **``cmdeploy dns --zonefile``** generates a standard
|
||||
BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
|
||||
TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
|
||||
into PowerDNS.
|
||||
|
||||
This two-phase approach prevents premature configuration of mail records
|
||||
before the relay is actually deployed and running.
|
||||
Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
|
||||
inside a relay container, Unbound has a configuration plugin snippet
|
||||
that forwards all ``.localchat`` queries to the PowerDNS recursor,
|
||||
and lets all other queries go through normal recursive resolution.
|
||||
|
||||
|
||||
State outside the repository
|
||||
-----------------------------
|
||||
|
||||
All generated configuration by lxc subcommands live in ``lxconfigs/``
|
||||
(git-ignored), including the SSH key pair (``id_localchat``),
|
||||
per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
|
||||
|
||||
The only state *outside* the repository is the Incus containers and images themselves
|
||||
(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
|
||||
The Incus image store retains the following snapshot images:
|
||||
|
||||
* ``localchat-base``: Debian 12 with openssh-server and Python (built on first run)
|
||||
|
||||
* ``localchat-relay``: fully deployed relay snapshot,
|
||||
cached after the first successful ``cmdeploy run``.
|
||||
Subsequent relay containers launch from this image
|
||||
so the deploy step is mostly no-ops (roughly 3× faster than a fresh deploy).
|
||||
|
||||
|
||||
.. _lxc-tls:
|
||||
|
||||
TLS handling and underscore domains
|
||||
------------------------------------
|
||||
|
||||
Container domains start with ``_`` (e.g. ``_test0.localchat``).
|
||||
As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
|
||||
underscore domains automatically use self-signed TLS
|
||||
and ``smtp_tls_security_level = encrypt``.
|
||||
This permits cross-relay federation between LXC containers
|
||||
without any external certificate authority.
|
||||
Delta Chat clients connecting to these relays
|
||||
must be configured with
|
||||
``certificateChecks = acceptInvalidCertificates``
|
||||
(the test fixtures handle this automatically).
|
||||
`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
|
||||
is meant to make this special setting unnecessary for chatmail clients
|
||||
that are connecting to underscore domains.
|
||||
|
||||
|
||||
Known limitations
|
||||
------------------
|
||||
|
||||
The LXC environment differs from a production
|
||||
deployment in several ways:
|
||||
|
||||
**No ACME / Let's Encrypt**:
|
||||
Self-signed TLS only (see :ref:`lxc-tls`);
|
||||
ACME code paths are never exercised locally.
|
||||
|
||||
**No inbound connections from the internet**:
|
||||
Containers sit on a private Incus bridge and are not port-forwarded.
|
||||
Only the host and other containers on the same bridge can reach them.
|
||||
|
||||
**Local federation only**:
|
||||
Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
|
||||
but these relays are invisible to any external mail server.
|
||||
|
||||
**DNS is local only**:
|
||||
The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
|
||||
(see :ref:`lxc-dns-setup`).
|
||||
|
||||
**IPv6 is ULA-only**:
|
||||
Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
|
||||
These are not globally routable, but are sufficient for testing IPv6 service binding
|
||||
(Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
|
||||
test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Build the chatmail Docker image with the current git hash baked in.
|
||||
# Usage: ./docker/build.sh [extra docker-compose build args...]
|
||||
#
|
||||
# .git/ is excluded from the build context (.dockerignore) so the hash
|
||||
# must be passed as a build arg from the host.
|
||||
|
||||
export GIT_HASH=$(git rev-parse HEAD)
|
||||
exec docker compose -f docker/docker-compose.yaml build "$@"
|
||||
@@ -1,14 +0,0 @@
|
||||
[Unit]
|
||||
Description=Run container setup commands
|
||||
After=multi-user.target
|
||||
ConditionPathExists=/chatmail-init.sh
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/bash /chatmail-init.sh
|
||||
RemainAfterExit=true
|
||||
WorkingDirectory=/opt/chatmail
|
||||
PassEnvironment=<envs_list>
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
export CHATMAIL_INI="${CHATMAIL_INI:-/etc/chatmail/chatmail.ini}"
|
||||
|
||||
CMDEPLOY=/opt/cmdeploy/bin/cmdeploy
|
||||
|
||||
if [ -z "$MAIL_DOMAIN" ]; then
|
||||
echo "ERROR: Environment variable 'MAIL_DOMAIN' must be set!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate DKIM keys if not mounted
|
||||
if [ ! -f /etc/dkimkeys/opendkim.private ]; then
|
||||
/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d "$MAIL_DOMAIN" -s opendkim
|
||||
fi
|
||||
# Fix ownership for bind-mounted keys (host opendkim UID may differ from container)
|
||||
chown -R opendkim:opendkim /etc/dkimkeys
|
||||
|
||||
# Create chatmail.ini, skip if mounted
|
||||
mkdir -p "$(dirname "$CHATMAIL_INI")"
|
||||
if [ ! -f "$CHATMAIL_INI" ]; then
|
||||
$CMDEPLOY init --config "$CHATMAIL_INI" "$MAIL_DOMAIN"
|
||||
fi
|
||||
|
||||
# Auto-detect IPv6: if the host has no IPv6 connectivity, set disable_ipv6
|
||||
# in the ini so dovecot/postfix/nginx bind to IPv4 only.
|
||||
# Uses network_mode:host so /proc/net/if_inet6 reflects the host's stack.
|
||||
if [ ! -e /proc/net/if_inet6 ]; then
|
||||
if grep -q '^disable_ipv6 = False' "$CHATMAIL_INI"; then
|
||||
sed -i 's/^disable_ipv6 = False/disable_ipv6 = True/' "$CHATMAIL_INI"
|
||||
echo "[INFO] IPv6 not available, set disable_ipv6 = True"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Inject external TLS paths from env var unless defined in chatmail.ini
|
||||
if [ -n "${TLS_EXTERNAL_CERT_AND_KEY:-}" ]; then
|
||||
if ! grep -q '^tls_external_cert_and_key' "$CHATMAIL_INI"; then
|
||||
echo "tls_external_cert_and_key = $TLS_EXTERNAL_CERT_AND_KEY" >> "$CHATMAIL_INI"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ensure mailboxes directory exists (chatmail-metadata needs it at startup,
|
||||
# but Dovecot only creates it on first mail delivery)
|
||||
mkdir -p "/home/vmail/mail/${MAIL_DOMAIN}"
|
||||
chown vmail:vmail "/home/vmail/mail/${MAIL_DOMAIN}"
|
||||
|
||||
# --- Deploy fingerprint: skip cmdeploy run if nothing changed ---
|
||||
# On restart with identical image+config, systemd already brings up all
|
||||
# enabled services only configure+activate are needed here.
|
||||
IMAGE_VERSION_FILE="/etc/chatmail-image-version"
|
||||
FINGERPRINT_FILE="/etc/chatmail/.deploy-fingerprint"
|
||||
image_ver="none"
|
||||
[ -f "$IMAGE_VERSION_FILE" ] && image_ver=$(cat "$IMAGE_VERSION_FILE")
|
||||
config_hash=$(sha256sum "$CHATMAIL_INI" | cut -c1-16)
|
||||
current_fp="${image_ver}:${config_hash}"
|
||||
|
||||
# CMDEPLOY_STAGES non-empty in env = operator override -> always run.
|
||||
# Otherwise, if fingerprint matches the last successful deploy, skip.
|
||||
if [ -z "${CMDEPLOY_STAGES:-}" ] \
|
||||
&& [ -f "$FINGERPRINT_FILE" ] \
|
||||
&& [ "$(cat "$FINGERPRINT_FILE")" = "$current_fp" ]; then
|
||||
echo "[INFO] No changes detected ($current_fp), skipping deploy."
|
||||
else
|
||||
export CMDEPLOY_STAGES="${CMDEPLOY_STAGES:-configure,activate}"
|
||||
|
||||
# Skip DNS check when MAIL_DOMAIN is a bare IP address
|
||||
SKIP_DNS=""
|
||||
if [[ "$MAIL_DOMAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$MAIL_DOMAIN" =~ : ]]; then
|
||||
SKIP_DNS="--skip-dns-check"
|
||||
fi
|
||||
$CMDEPLOY run --config "$CHATMAIL_INI" --ssh-host @local $SKIP_DNS
|
||||
|
||||
# Restore the build-time hash
|
||||
cp /etc/chatmail-image-version /etc/chatmail-version
|
||||
echo "$current_fp" > "$FINGERPRINT_FILE"
|
||||
fi
|
||||
|
||||
# Signal success to Docker healthcheck
|
||||
touch /run/chatmail-init.done
|
||||
|
||||
# Forward journald to console so `docker compose logs` works
|
||||
grep -q '^ForwardToConsole=yes' /etc/systemd/journald.conf \
|
||||
|| echo "ForwardToConsole=yes" >> /etc/systemd/journald.conf
|
||||
systemctl restart systemd-journald
|
||||
@@ -1,108 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM jrei/systemd-debian:12 AS base
|
||||
|
||||
ENV LANG=en_US.UTF-8
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
|
||||
echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend && \
|
||||
echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend && \
|
||||
apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive TZ=UTC \
|
||||
apt-get install -y \
|
||||
ca-certificates \
|
||||
gcc \
|
||||
git \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-venv \
|
||||
tzdata \
|
||||
locales && \
|
||||
sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
|
||||
dpkg-reconfigure --frontend=noninteractive locales && \
|
||||
update-locale LANG=$LANG
|
||||
|
||||
# --- Build-time: install cmdeploy venv and run install stage ---
|
||||
# Editable install so importlib.resources reads directly from the source tree.
|
||||
# On container start only "configure,activate" stages run.
|
||||
|
||||
# Copy dependency metadata first so pip install layer is cached
|
||||
COPY cmdeploy/pyproject.toml /opt/chatmail/cmdeploy/pyproject.toml
|
||||
COPY chatmaild/pyproject.toml /opt/chatmail/chatmaild/pyproject.toml
|
||||
|
||||
# Dummy scaffolding so editable install can discover packages
|
||||
RUN mkdir -p /opt/chatmail/cmdeploy/src/cmdeploy \
|
||||
/opt/chatmail/chatmaild/src/chatmaild && \
|
||||
touch /opt/chatmail/cmdeploy/src/cmdeploy/__init__.py \
|
||||
/opt/chatmail/chatmaild/src/chatmaild/__init__.py
|
||||
|
||||
# Dummy git repo: .git/ is excluded from the build context (.dockerignore)
|
||||
# but setuptools calls `git ls-files` when building the sdist.
|
||||
WORKDIR /opt/chatmail
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
git init -q && \
|
||||
python3 -m venv /opt/cmdeploy && \
|
||||
/opt/cmdeploy/bin/pip install -e chatmaild/ -e cmdeploy/
|
||||
|
||||
# Full source copy (editable install's .egg-link still points here)
|
||||
COPY . /opt/chatmail/
|
||||
|
||||
# Minimal chatmail.ini
|
||||
RUN printf '[params]\nmail_domain = build.local\n' > /tmp/chatmail.ini
|
||||
|
||||
RUN CMDEPLOY_STAGES=install \
|
||||
CHATMAIL_INI=/tmp/chatmail.ini \
|
||||
/opt/cmdeploy/bin/pyinfra @local \
|
||||
/opt/chatmail/cmdeploy/src/cmdeploy/run.py -y
|
||||
|
||||
RUN cp -a www/ /opt/chatmail-www/
|
||||
|
||||
# Remove build-only packages — not needed at runtime.
|
||||
# Keep git: test_deployed_state needs `git rev-parse HEAD` to verify the
|
||||
# deployed version hash matches /etc/chatmail-version.
|
||||
RUN apt-get purge -y gcc python3-dev && \
|
||||
apt-get autoremove -y && \
|
||||
rm -f /tmp/chatmail.ini
|
||||
|
||||
# Record image version (used in deploy fingerprint at runtime).
|
||||
# GIT_HASH is passed as a build arg (from docker-compose or CI) so that
|
||||
# .git/ can be excluded from the build context via .dockerignore.
|
||||
# Two files: chatmail-image-version is the immutable build hash (survives
|
||||
# deploys); chatmail-version is overwritten by cmdeploy run and restored
|
||||
# from the image version after each deploy in chatmail-init.sh.
|
||||
ARG GIT_HASH=unknown
|
||||
RUN echo "$GIT_HASH" > /etc/chatmail-image-version && \
|
||||
echo "$GIT_HASH" > /etc/chatmail-version
|
||||
|
||||
# Mock git HEAD so `git rev-parse HEAD` returns the source repo's commit hash.
|
||||
# The .git/ dir was created by `git init` earlier (for setuptools); we just
|
||||
# write the build hash into whatever branch HEAD points to.
|
||||
RUN head_ref=$(sed 's/^ref: //' /opt/chatmail/.git/HEAD) && \
|
||||
mkdir -p "/opt/chatmail/.git/$(dirname "$head_ref")" && \
|
||||
echo "$GIT_HASH" > "/opt/chatmail/.git/$head_ref"
|
||||
# --- End build-time install ---
|
||||
|
||||
ENV TZ=:/etc/localtime
|
||||
ENV PATH="/opt/cmdeploy/bin:${PATH}"
|
||||
RUN ln -s /etc/chatmail/chatmail.ini /opt/chatmail/chatmail.ini
|
||||
|
||||
ARG CHATMAIL_INIT_SERVICE_PATH=/lib/systemd/system/chatmail-init.service
|
||||
COPY ./docker/chatmail-init.service "$CHATMAIL_INIT_SERVICE_PATH"
|
||||
RUN ln -sf "$CHATMAIL_INIT_SERVICE_PATH" "/etc/systemd/system/multi-user.target.wants/chatmail-init.service"
|
||||
|
||||
# Remove default nginx site config at build time (not in entrypoint)
|
||||
RUN rm -f /etc/nginx/sites-enabled/default
|
||||
|
||||
COPY --chmod=555 ./docker/chatmail-init.sh /chatmail-init.sh
|
||||
COPY --chmod=555 ./docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --chmod=555 ./docker/healthcheck.sh /healthcheck.sh
|
||||
|
||||
HEALTHCHECK --interval=10s --start-period=180s --timeout=10s --retries=3 \
|
||||
CMD /healthcheck.sh
|
||||
|
||||
STOPSIGNAL SIGRTMIN+3
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
CMD [ "--default-standard-output=journal+console", \
|
||||
"--default-standard-error=journal+console" ]
|
||||
@@ -1,70 +0,0 @@
|
||||
# Traefik reverse-proxy example — use as a compose override:
|
||||
#
|
||||
# docker compose -f docker-compose.yaml -f docker-compose-traefik.yaml up -d
|
||||
#
|
||||
# Traefik handles HTTP→HTTPS redirect and ACME certificate issuance.
|
||||
# traefik-certs-dumper extracts the certificates to the filesystem so
|
||||
# chatmail's Postfix/Dovecot/nginx can use them via TLS_EXTERNAL_CERT_AND_KEY.
|
||||
#
|
||||
# Prerequisites:
|
||||
# mkdir -p traefik/data traefik/dynamic-configs
|
||||
# touch traefik/data/acme.json && chmod 600 traefik/data/acme.json
|
||||
# cp traefik/config.yaml.example traefik/config.yaml # see below
|
||||
#
|
||||
# Required .env variables (in addition to MAIL_DOMAIN):
|
||||
# ACME_EMAIL=admin@example.org
|
||||
|
||||
services:
|
||||
chatmail:
|
||||
environment:
|
||||
# Point chatmail at the certs dumped by traefik-certs-dumper.
|
||||
# The container's tls-cert-reload.path watches for changes.
|
||||
TLS_EXTERNAL_CERT_AND_KEY: >-
|
||||
/traefik-certs/${MAIL_DOMAIN}/certificate.crt
|
||||
/traefik-certs/${MAIL_DOMAIN}/privatekey.key
|
||||
volumes:
|
||||
- traefik-certs:/traefik-certs:ro
|
||||
depends_on:
|
||||
- traefik-certs-dumper
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.http.services.chatmail.loadbalancer.server.scheme=https
|
||||
- traefik.http.services.chatmail.loadbalancer.server.port=443
|
||||
- traefik.http.routers.chatmail.rule=Host(`${MAIL_DOMAIN}`) || Host(`mta-sts.${MAIL_DOMAIN}`) || Host(`www.${MAIL_DOMAIN}`)
|
||||
- traefik.http.routers.chatmail.tls=true
|
||||
- traefik.http.routers.chatmail.tls.certresolver=letsEncrypt
|
||||
|
||||
traefik:
|
||||
image: traefik:v3.3
|
||||
container_name: traefik
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
command:
|
||||
- "--configFile=/config.yaml"
|
||||
- "--certificatesresolvers.letsEncrypt.acme.email=${ACME_EMAIL}"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- ./traefik/config.yaml:/config.yaml:ro
|
||||
- ./traefik/data/acme.json:/acme.json
|
||||
- ./traefik/dynamic-configs:/dynamic/conf:ro
|
||||
|
||||
traefik-certs-dumper:
|
||||
image: ldez/traefik-certs-dumper:v2.10.0
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- traefik
|
||||
entrypoint: sh -c '
|
||||
apk add openssl
|
||||
&& while ! [ -e /data/acme.json ]
|
||||
|| ! [ $$(jq ".[] | .Certificates | length" /data/acme.json | jq -s "add") != 0 ]; do
|
||||
sleep 1;
|
||||
done
|
||||
&& traefik-certs-dumper file
|
||||
--version v3 --watch --domain-subdir=true
|
||||
--source /data/acme.json --dest /certs'
|
||||
volumes:
|
||||
- ./traefik/data/acme.json:/data/acme.json:ro
|
||||
- traefik-certs:/certs
|
||||
|
||||
volumes:
|
||||
traefik-certs:
|
||||
@@ -1,11 +0,0 @@
|
||||
# Used by .github/workflows/docker-ci.yaml
|
||||
# The GHCR image is set via CHATMAIL_IMAGE env var at deploy time.
|
||||
services:
|
||||
chatmail:
|
||||
image: ${CHATMAIL_IMAGE:-chatmail-relay:latest}
|
||||
volumes:
|
||||
- /srv/chatmail/chatmail.ini:/etc/chatmail/chatmail.ini
|
||||
- /srv/chatmail/dkim:/etc/dkimkeys
|
||||
- /srv/chatmail/certs:/var/lib/acme
|
||||
environment:
|
||||
TLS_EXTERNAL_CERT_AND_KEY: /var/lib/acme/live/${MAIL_DOMAIN}/fullchain /var/lib/acme/live/${MAIL_DOMAIN}/privkey
|
||||
@@ -1,44 +0,0 @@
|
||||
# Local overrides: copy to docker-compose.override.yaml in this directory.
|
||||
# Compose automatically merges this with docker-compose.yaml.
|
||||
#
|
||||
# cp docker-compose.override.yaml.example docker-compose.override.yaml
|
||||
#
|
||||
# Volumes are APPENDED to the base file's volumes list, environment and other scalar keys are MERGED by key.
|
||||
services:
|
||||
chatmail:
|
||||
volumes:
|
||||
## Data paths — bind-mount to host directories for easy access/backup.
|
||||
|
||||
# - ./data/dkim:/etc/dkimkeys
|
||||
# - ./data/certs:/var/lib/acme
|
||||
|
||||
# - ./data/mail:/home/vmail
|
||||
## Or mount from an existing bare-metal install.
|
||||
# - /home/vmail:/home/vmail
|
||||
|
||||
## Mount your own chatmail.ini (skips auto-generation):
|
||||
# - ./chatmail.ini:/etc/chatmail/chatmail.ini
|
||||
|
||||
## Custom website:
|
||||
# - ./custom/www:/opt/chatmail-www
|
||||
|
||||
## Debug — mount scripts for live editing:
|
||||
# - ./chatmail-init.sh:/chatmail-init.sh
|
||||
# - ./entrypoint.sh:/entrypoint.sh
|
||||
|
||||
# environment:
|
||||
## Mount certs (above) and set TLS_EXTERNAL_CERT_AND_KEY to in-container paths.
|
||||
## A tls-cert-reload.path watcher inside the container reloads services
|
||||
## when the cert file changes. However, inotify does not cross bind-mount
|
||||
## boundaries, so host-side renewals (certbot, acmetool, etc.) must
|
||||
## notify the container explicitly. Add this to your renewal hook:
|
||||
##
|
||||
## docker exec chatmail systemctl start tls-cert-reload.service
|
||||
##
|
||||
## Host acmetool (bare-metal migration): create mount above, and
|
||||
## rsync -a /var/lib/acme/live data/certs
|
||||
# TLS_EXTERNAL_CERT_AND_KEY: "/var/lib/acme/live/${MAIL_DOMAIN}/fullchain /var/lib/acme/live/${MAIL_DOMAIN}/privkey"
|
||||
##
|
||||
## (Untested) Traefik certs-dumper (see docker/docker-compose-traefik.yaml) - also add volume:
|
||||
## - traefik-certs:/certs:ro
|
||||
# TLS_EXTERNAL_CERT_AND_KEY: "/certs/${MAIL_DOMAIN}/certificate.crt /certs/${MAIL_DOMAIN}/privatekey.key"
|
||||
@@ -1,48 +0,0 @@
|
||||
# Base compose file — do not edit. Put customizations (data paths, extra
|
||||
# volumes, env overrides) in docker-compose.override.yaml instead.
|
||||
# See docker-compose.override.yaml.example in this directory for a starting point.
|
||||
#
|
||||
# Security notes: this container uses
|
||||
# - network_mode:host chatmail needs many ports (25, 53, 80, 143, 443, 465,
|
||||
# 587, 993, 3340, 8443) and needs to operate from the real IP, which bridging
|
||||
# would make tricky
|
||||
# - cgroup:host (required for systemd).
|
||||
# Together these give the container near-host-level access. This is acceptable
|
||||
# for a dedicated mail server, but be aware that the container can bind any
|
||||
# port and see all host network traffic.
|
||||
|
||||
services:
|
||||
chatmail:
|
||||
build:
|
||||
context: ../
|
||||
dockerfile: docker/chatmail_relay.dockerfile
|
||||
args:
|
||||
GIT_HASH: ${GIT_HASH:-unknown}
|
||||
image: chatmail-relay:latest
|
||||
restart: unless-stopped
|
||||
container_name: chatmail
|
||||
# Required for systemd — use only one of the following:
|
||||
cgroup: host # compose v2
|
||||
# privileged: true # compose v1 (less restricted)
|
||||
tty: true # required for logs
|
||||
tmpfs: # required for systemd
|
||||
- /tmp
|
||||
- /run
|
||||
- /run/lock
|
||||
logging:
|
||||
driver: none
|
||||
environment:
|
||||
MAIL_DOMAIN: $MAIL_DOMAIN
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
## system (required)
|
||||
- /sys/fs/cgroup:/sys/fs/cgroup:rw
|
||||
## data (defaults — override in docker-compose.override.yaml)
|
||||
- mail:/home/vmail
|
||||
- dkim:/etc/dkimkeys
|
||||
- certs:/var/lib/acme
|
||||
|
||||
volumes:
|
||||
mail:
|
||||
dkim:
|
||||
certs:
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
CHATMAIL_INIT_SERVICE_PATH="${CHATMAIL_INIT_SERVICE_PATH:-/lib/systemd/system/chatmail-init.service}"
|
||||
|
||||
env_vars="MAIL_DOMAIN CMDEPLOY_STAGES CHATMAIL_INI TLS_EXTERNAL_CERT_AND_KEY PATH"
|
||||
sed -i "s|<envs_list>|$env_vars|g" "$CHATMAIL_INIT_SERVICE_PATH"
|
||||
|
||||
exec /lib/systemd/systemd "$@"
|
||||
@@ -1 +0,0 @@
|
||||
MAIL_DOMAIN=chat.example.com
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
# returns 0 when chatmail-init succeeded and all expected services are running.
|
||||
|
||||
set -e
|
||||
|
||||
test -f /run/chatmail-init.done
|
||||
|
||||
# Core services
|
||||
services="chatmail-metadata doveauth dovecot filtermail filtermail-incoming nginx postfix unbound"
|
||||
|
||||
# Optional services
|
||||
for svc in iroh-relay turnserver; do
|
||||
systemctl is-enabled "$svc" 2>/dev/null && services="$services $svc"
|
||||
done
|
||||
|
||||
exec systemctl is-active $services
|
||||
Reference in New Issue
Block a user