mirror of
https://github.com/chatmail/relay.git
synced 2026-05-16 03:24:48 +00:00
Compare commits
13 Commits
j4n/docker
...
hpk/test-c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6573ccc05f | ||
|
|
25285005c3 | ||
|
|
482194437d | ||
|
|
735e9d3e7f | ||
|
|
4b79606d49 | ||
|
|
6e52bfe8c4 | ||
|
|
95c76aa2b0 | ||
|
|
4f109e8c31 | ||
|
|
8c30714279 | ||
|
|
23f21d36b1 | ||
|
|
4ed3f5dd91 | ||
|
|
972b46be74 | ||
|
|
7edb4e860a |
@@ -1,18 +0,0 @@
|
|||||||
data/
|
|
||||||
venv/
|
|
||||||
__pycache__
|
|
||||||
*.pyc
|
|
||||||
*.orig
|
|
||||||
*.ini
|
|
||||||
.pytest_cache
|
|
||||||
.env
|
|
||||||
|
|
||||||
# Slim build context — .git/ alone can be 100s of MB
|
|
||||||
.git
|
|
||||||
.github/
|
|
||||||
docs/
|
|
||||||
tests/
|
|
||||||
|
|
||||||
# Exclude markdown files but keep www/src/*.md (used by WebsiteDeployer)
|
|
||||||
*.md
|
|
||||||
!www/**/*.md
|
|
||||||
94
.github/workflows/ci.yaml
vendored
94
.github/workflows/ci.yaml
vendored
@@ -15,28 +15,102 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
ref: ${{ github.event.pull_request.head.sha }}
|
ref: ${{ github.event.pull_request.head.sha }}
|
||||||
- name: download filtermail
|
- name: download filtermail
|
||||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||||
- name: run chatmaild tests
|
- name: run chatmaild tests
|
||||||
working-directory: chatmaild
|
working-directory: chatmaild
|
||||||
run: pipx run tox
|
run: pipx run tox
|
||||||
|
|
||||||
scripts:
|
scripts:
|
||||||
name: deploy-chatmail tests
|
name: deploy-chatmail tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: initenv
|
- name: initenv
|
||||||
run: scripts/initenv.sh
|
run: scripts/initenv.sh
|
||||||
|
|
||||||
- name: append venv/bin to PATH
|
- name: append venv/bin to PATH
|
||||||
run: echo venv/bin >>$GITHUB_PATH
|
run: echo venv/bin >>$GITHUB_PATH
|
||||||
|
|
||||||
- name: run formatting checks
|
- name: run formatting checks
|
||||||
run: cmdeploy fmt -v
|
run: cmdeploy fmt -v
|
||||||
|
|
||||||
- name: run deploy-chatmail offline tests
|
- name: run deploy-chatmail offline tests
|
||||||
run: pytest --pyargs cmdeploy
|
run: pytest --pyargs cmdeploy
|
||||||
|
|
||||||
# all other cmdeploy commands require a staging server
|
lxc-test:
|
||||||
# see https://github.com/deltachat/chatmail/issues/100
|
name: LXC deploy and test
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
timeout-minutes: 30
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ github.event.pull_request.head.sha }}
|
||||||
|
|
||||||
|
- name: install incus
|
||||||
|
run: |
|
||||||
|
# zabbly is the official incus community packages source
|
||||||
|
curl -fsSL https://pkgs.zabbly.com/key.asc \
|
||||||
|
| sudo gpg --dearmor -o /etc/apt/keyrings/zabbly.gpg
|
||||||
|
sudo sh -c 'cat <<EOF > /etc/apt/sources.list.d/zabbly-incus-stable.sources
|
||||||
|
Enabled: yes
|
||||||
|
Types: deb
|
||||||
|
URIs: https://pkgs.zabbly.com/incus/stable
|
||||||
|
Suites: $(. /etc/os-release && echo ${VERSION_CODENAME})
|
||||||
|
Components: main
|
||||||
|
Architectures: $(dpkg --print-architecture)
|
||||||
|
Signed-By: /etc/apt/keyrings/zabbly.gpg
|
||||||
|
EOF'
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y incus
|
||||||
|
|
||||||
|
- name: initialise incus
|
||||||
|
run: |
|
||||||
|
sudo systemctl stop docker.socket docker || true
|
||||||
|
sudo iptables -P FORWARD ACCEPT
|
||||||
|
sudo sysctl -w fs.inotify.max_user_instances=65535
|
||||||
|
sudo sysctl -w fs.inotify.max_user_watches=65535
|
||||||
|
sudo incus admin init --minimal
|
||||||
|
sudo usermod -aG incus-admin "$USER"
|
||||||
|
|
||||||
|
- name: initenv
|
||||||
|
run: scripts/initenv.sh
|
||||||
|
|
||||||
|
- name: append venv/bin to PATH
|
||||||
|
run: echo venv/bin >>$GITHUB_PATH
|
||||||
|
|
||||||
|
- name: restore cached images
|
||||||
|
id: cache-images
|
||||||
|
uses: actions/cache@v4
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
/tmp/localchat-base.tar.gz
|
||||||
|
/tmp/localchat-ns.tar.gz
|
||||||
|
/tmp/localchat-test0.tar.gz
|
||||||
|
/tmp/localchat-test1.tar.gz
|
||||||
|
lxconfigs/id_localchat*
|
||||||
|
key: incus-images-${{ runner.os }}-${{ github.ref_name }}
|
||||||
|
restore-keys: |
|
||||||
|
incus-images-${{ runner.os }}-${{ github.ref_name }}-
|
||||||
|
incus-images-${{ runner.os }}-main-
|
||||||
|
incus-images-${{ runner.os }}-
|
||||||
|
|
||||||
|
- name: import cached images
|
||||||
|
run: |
|
||||||
|
for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
|
||||||
|
if [ -f /tmp/$alias.tar.gz ]; then
|
||||||
|
sg incus-admin -c "incus image import /tmp/$alias.tar.gz --alias $alias" || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
- name: lxc-test
|
||||||
|
run: sg incus-admin -c 'cmdeploy lxc-test'
|
||||||
|
|
||||||
|
- name: export images for cache
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
|
||||||
|
if ! [ -f /tmp/$alias.tar.gz ]; then
|
||||||
|
sg incus-admin -c "incus image export $alias /tmp/$alias" || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|||||||
@@ -1,20 +0,0 @@
|
|||||||
;; Zone file for staging-ipv4.testrun.org
|
|
||||||
|
|
||||||
$ORIGIN staging-ipv4.testrun.org.
|
|
||||||
$TTL 300
|
|
||||||
|
|
||||||
@ IN SOA ns.testrun.org. root.nine.testrun.org (
|
|
||||||
2023010101 ; Serial
|
|
||||||
7200 ; Refresh
|
|
||||||
3600 ; Retry
|
|
||||||
1209600 ; Expire
|
|
||||||
3600 ; Negative response caching TTL
|
|
||||||
)
|
|
||||||
|
|
||||||
;; Nameservers.
|
|
||||||
@ IN NS ns.testrun.org.
|
|
||||||
|
|
||||||
;; DNS records.
|
|
||||||
@ IN A 37.27.95.249
|
|
||||||
mta-sts.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org.
|
|
||||||
www.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org.
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
;; Zone file for staging2.testrun.org
|
|
||||||
|
|
||||||
$ORIGIN staging2.testrun.org.
|
|
||||||
$TTL 300
|
|
||||||
|
|
||||||
@ IN SOA ns.testrun.org. root.nine.testrun.org (
|
|
||||||
2023010101 ; Serial
|
|
||||||
7200 ; Refresh
|
|
||||||
3600 ; Retry
|
|
||||||
1209600 ; Expire
|
|
||||||
3600 ; Negative response caching TTL
|
|
||||||
)
|
|
||||||
|
|
||||||
;; Nameservers.
|
|
||||||
@ IN NS ns.testrun.org.
|
|
||||||
|
|
||||||
;; DNS records.
|
|
||||||
@ IN A 37.27.24.139
|
|
||||||
mta-sts.staging2.testrun.org. CNAME staging2.testrun.org.
|
|
||||||
www.staging2.testrun.org. CNAME staging2.testrun.org.
|
|
||||||
|
|
||||||
296
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
296
.github/workflows/test-and-deploy-ipv4only.yaml
vendored
@@ -1,296 +0,0 @@
|
|||||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
- j4n/docker-pr
|
|
||||||
pull_request:
|
|
||||||
paths-ignore:
|
|
||||||
- 'scripts/**'
|
|
||||||
- '**/README.md'
|
|
||||||
- 'CHANGELOG.md'
|
|
||||||
- 'LICENSE'
|
|
||||||
|
|
||||||
env:
|
|
||||||
REGISTRY: ghcr.io
|
|
||||||
IMAGE_NAME: ${{ github.repository }}
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-docker:
|
|
||||||
name: Build Docker image
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
outputs:
|
|
||||||
image: ${{ steps.image-ref.outputs.image }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Login to GHCR
|
|
||||||
if: github.event_name == 'push'
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ${{ env.REGISTRY }}
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Extract metadata (tags, labels)
|
|
||||||
id: meta
|
|
||||||
uses: docker/metadata-action@v5
|
|
||||||
with:
|
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
||||||
tags: |
|
|
||||||
# Tagged releases: v1.2.3 -> :1.2.3, :1.2, :latest
|
|
||||||
type=semver,pattern={{version}}
|
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
|
||||||
# Branch pushes: foo/docker-pr -> :foo-docker-pr
|
|
||||||
type=ref,event=branch
|
|
||||||
# Always: :sha-<hash>
|
|
||||||
type=sha
|
|
||||||
|
|
||||||
- name: Build and push
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: docker/chatmail_relay.dockerfile
|
|
||||||
push: ${{ github.event_name == 'push' }}
|
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
|
||||||
cache-from: type=gha
|
|
||||||
cache-to: type=gha,mode=max
|
|
||||||
build-args: |
|
|
||||||
GIT_HASH=${{ github.sha }}
|
|
||||||
|
|
||||||
- name: Output image reference
|
|
||||||
id: image-ref
|
|
||||||
run: |
|
|
||||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
|
||||||
IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
|
||||||
echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
deploy:
|
|
||||||
name: deploy on staging-ipv4.testrun.org, and run tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 30
|
|
||||||
environment:
|
|
||||||
name: staging-ipv4.testrun.org
|
|
||||||
url: https://staging-ipv4.testrun.org/
|
|
||||||
concurrency: staging-ipv4.testrun.org
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: prepare SSH
|
|
||||||
run: |
|
|
||||||
mkdir ~/.ssh
|
|
||||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
|
||||||
chmod 600 ~/.ssh/id_ed25519
|
|
||||||
ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
|
|
||||||
# save previous acme & dkim state
|
|
||||||
rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
|
|
||||||
rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
|
|
||||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
|
||||||
if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
|
||||||
if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
|
|
||||||
# make sure CAA record isn't set
|
|
||||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: rebuild staging-ipv4.testrun.org to have a clean VPS
|
|
||||||
run: |
|
|
||||||
curl -X POST \
|
|
||||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"image":"debian-12"}' \
|
|
||||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
|
|
||||||
|
|
||||||
- run: scripts/initenv.sh
|
|
||||||
|
|
||||||
- name: append venv/bin to PATH
|
|
||||||
run: echo venv/bin >>$GITHUB_PATH
|
|
||||||
|
|
||||||
- name: upload TLS cert after rebuilding
|
|
||||||
id: wait-for-vps
|
|
||||||
run: |
|
|
||||||
echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
|
|
||||||
rm ~/.ssh/known_hosts
|
|
||||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
|
|
||||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
|
|
||||||
# download acme & dkim state from ns.testrun.org
|
|
||||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
|
|
||||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
|
|
||||||
# restore acme & dkim state to staging2.testrun.org
|
|
||||||
rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
|
|
||||||
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
|
|
||||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
|
|
||||||
|
|
||||||
- name: run deploy-chatmail offline tests
|
|
||||||
run: pytest --pyargs cmdeploy
|
|
||||||
|
|
||||||
- name: setup dependencies
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org apt update
|
|
||||||
ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
|
|
||||||
ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
|
|
||||||
ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
|
|
||||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
|
|
||||||
|
|
||||||
- name: initialize config
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
|
|
||||||
ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
|
|
||||||
ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
|
|
||||||
|
|
||||||
- run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
|
|
||||||
|
|
||||||
- name: set DNS entries
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
|
|
||||||
ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
|
||||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
|
||||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: cmdeploy test
|
|
||||||
run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
|
|
||||||
|
|
||||||
- name: cmdeploy dns
|
|
||||||
run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
|
|
||||||
|
|
||||||
# --- Docker deploy (push only, runs even if bare failed) ---
|
|
||||||
|
|
||||||
- name: stop bare services
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org 'systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true'
|
|
||||||
|
|
||||||
- name: install Docker on VPS
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org 'apt-get update && apt-get install -y ca-certificates curl'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'install -m 0755 -d /etc/apt/keyrings'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin'
|
|
||||||
|
|
||||||
- name: prepare Docker bind mounts
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org 'mkdir -p /srv/chatmail/certs /srv/chatmail/dkim'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'cp -a /var/lib/acme/. /srv/chatmail/certs/ && cp -a /etc/dkimkeys/. /srv/chatmail/dkim/' || true
|
|
||||||
|
|
||||||
- name: upload chatmail.ini for Docker
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Reuse chatmail.ini already created by the bare-metal deploy steps
|
|
||||||
ssh root@staging-ipv4.testrun.org "cp relay/chatmail.ini /srv/chatmail/chatmail.ini"
|
|
||||||
|
|
||||||
- name: deploy with Docker
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
|
||||||
GHCR_IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
|
||||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ root@staging-ipv4.testrun.org:/srv/chatmail/relay/
|
|
||||||
# Login to GHCR on VPS and pull pre-built image
|
|
||||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh root@staging-ipv4.testrun.org 'docker login ghcr.io -u ${{ github.actor }} --password-stdin'
|
|
||||||
ssh root@staging-ipv4.testrun.org "docker pull ${GHCR_IMAGE}"
|
|
||||||
ssh root@staging-ipv4.testrun.org "cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=staging-ipv4.testrun.org docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
|
||||||
|
|
||||||
- name: wait for container healthy
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Stream journald inside the container
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail journalctl -f --no-pager' &
|
|
||||||
LOG_PID=$!
|
|
||||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
|
||||||
for i in $(seq 1 60); do
|
|
||||||
status=$(ssh root@staging-ipv4.testrun.org 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
|
||||||
echo " [$i/60] status=$status"
|
|
||||||
if [ "$status" = "healthy" ]; then
|
|
||||||
echo "Container is healthy."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
if [ "$status" = "unhealthy" ]; then
|
|
||||||
echo "Container is unhealthy!"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
echo "Container did not become healthy."
|
|
||||||
kill $LOG_PID 2>/dev/null || true
|
|
||||||
echo "--- failed units ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail systemctl --failed --no-pager' || true
|
|
||||||
echo "--- service logs ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50' || true
|
|
||||||
echo "--- listening ports ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail ss -tlnp' || true
|
|
||||||
echo "--- chatmail.ini ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini' || true
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
- name: show container state
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
echo "--- listening ports ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail ss -tlnp'
|
|
||||||
echo "--- chatmail.ini ---"
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini'
|
|
||||||
|
|
||||||
- name: Docker integration tests
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
|
||||||
|
|
||||||
- name: Docker DNS
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Reset zone file in case bare DNS already appended to it
|
|
||||||
git checkout .github/workflows/staging-ipv4.testrun.org-default.zone
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose'
|
|
||||||
ssh root@staging-ipv4.testrun.org 'docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone'
|
|
||||||
scp root@staging-ipv4.testrun.org:/tmp/staging.zone staging-generated.zone
|
|
||||||
cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
|
|
||||||
cat .github/workflows/staging-ipv4.testrun.org-default.zone
|
|
||||||
scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: Docker final DNS check
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: ssh root@staging-ipv4.testrun.org 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
|
||||||
|
|
||||||
# --- Cleanup ---
|
|
||||||
|
|
||||||
- name: add SSH keys
|
|
||||||
if: >-
|
|
||||||
!cancelled()
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: ssh root@staging-ipv4.testrun.org 'curl -s https://github.com/hpk42.keys https://github.com/j4n.keys >> .ssh/authorized_keys'
|
|
||||||
289
.github/workflows/test-and-deploy.yaml
vendored
289
.github/workflows/test-and-deploy.yaml
vendored
@@ -1,289 +0,0 @@
|
|||||||
name: deploy on staging2.testrun.org, and run tests
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
- j4n/docker-pr
|
|
||||||
pull_request:
|
|
||||||
paths-ignore:
|
|
||||||
- 'scripts/**'
|
|
||||||
- '**/README.md'
|
|
||||||
- 'CHANGELOG.md'
|
|
||||||
- 'LICENSE'
|
|
||||||
|
|
||||||
env:
|
|
||||||
REGISTRY: ghcr.io
|
|
||||||
IMAGE_NAME: ${{ github.repository }}
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-docker:
|
|
||||||
name: Build Docker image
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
outputs:
|
|
||||||
image: ${{ steps.image-ref.outputs.image }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Login to GHCR
|
|
||||||
if: github.event_name == 'push'
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ${{ env.REGISTRY }}
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Extract metadata (tags, labels)
|
|
||||||
id: meta
|
|
||||||
uses: docker/metadata-action@v5
|
|
||||||
with:
|
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
||||||
tags: |
|
|
||||||
# Tagged releases: v1.2.3 -> :1.2.3, :1.2, :latest
|
|
||||||
type=semver,pattern={{version}}
|
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
|
||||||
# Branch pushes: foo/docker-pr -> :foo-docker-pr
|
|
||||||
type=ref,event=branch
|
|
||||||
# Always: :sha-<hash>
|
|
||||||
type=sha
|
|
||||||
|
|
||||||
- name: Build and push
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: docker/chatmail_relay.dockerfile
|
|
||||||
push: ${{ github.event_name == 'push' }}
|
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
|
||||||
cache-from: type=gha
|
|
||||||
cache-to: type=gha,mode=max
|
|
||||||
build-args: |
|
|
||||||
GIT_HASH=${{ github.sha }}
|
|
||||||
|
|
||||||
- name: Output image reference
|
|
||||||
id: image-ref
|
|
||||||
run: |
|
|
||||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
|
||||||
IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
|
||||||
echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
deploy:
|
|
||||||
name: deploy on staging2.testrun.org, and run tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 30
|
|
||||||
environment:
|
|
||||||
name: staging2.testrun.org
|
|
||||||
url: https://staging2.testrun.org/
|
|
||||||
concurrency: staging2.testrun.org
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: prepare SSH
|
|
||||||
run: |
|
|
||||||
mkdir ~/.ssh
|
|
||||||
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
|
|
||||||
chmod 600 ~/.ssh/id_ed25519
|
|
||||||
ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
|
|
||||||
# save previous acme & dkim state
|
|
||||||
rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
|
|
||||||
rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
|
|
||||||
# store previous acme & dkim state on ns.testrun.org, if it contains useful certs
|
|
||||||
if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
|
|
||||||
if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
|
|
||||||
# make sure CAA record isn't set
|
|
||||||
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: rebuild staging2.testrun.org to have a clean VPS
|
|
||||||
run: |
|
|
||||||
curl -X POST \
|
|
||||||
-H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"image":"debian-12"}' \
|
|
||||||
"https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
|
|
||||||
|
|
||||||
- run: scripts/initenv.sh
|
|
||||||
|
|
||||||
- name: append venv/bin to PATH
|
|
||||||
run: echo venv/bin >>$GITHUB_PATH
|
|
||||||
|
|
||||||
- name: upload TLS cert after rebuilding
|
|
||||||
id: wait-for-vps
|
|
||||||
run: |
|
|
||||||
echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
|
|
||||||
rm ~/.ssh/known_hosts
|
|
||||||
while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
|
|
||||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
|
|
||||||
# download acme & dkim state from ns.testrun.org
|
|
||||||
rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
|
|
||||||
rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
|
|
||||||
# restore acme & dkim state to staging2.testrun.org
|
|
||||||
rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
|
|
||||||
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
|
|
||||||
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
|
|
||||||
|
|
||||||
- name: add hpk42 key to staging server
|
|
||||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
|
|
||||||
|
|
||||||
- name: run deploy-chatmail offline tests
|
|
||||||
run: pytest --pyargs cmdeploy
|
|
||||||
|
|
||||||
- run: |
|
|
||||||
cmdeploy init staging2.testrun.org
|
|
||||||
sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
|
|
||||||
|
|
||||||
- run: cmdeploy run --verbose --skip-dns-check
|
|
||||||
|
|
||||||
- name: set DNS entries
|
|
||||||
run: |
|
|
||||||
cmdeploy dns --zonefile staging-generated.zone --verbose
|
|
||||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
|
||||||
cat .github/workflows/staging.testrun.org-default.zone
|
|
||||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: cmdeploy test
|
|
||||||
run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
|
|
||||||
|
|
||||||
- name: cmdeploy dns
|
|
||||||
run: cmdeploy dns -v
|
|
||||||
|
|
||||||
# --- Docker deploy (push only, runs even if bare failed) ---
|
|
||||||
|
|
||||||
- name: stop bare services
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging2.testrun.org 'systemctl stop postfix dovecot nginx opendkim unbound filtermail doveauth chatmail-metadata iroh-relay mtail fcgiwrap acmetool 2>/dev/null || true'
|
|
||||||
|
|
||||||
- name: install Docker on VPS
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging2.testrun.org 'apt-get update && apt-get install -y ca-certificates curl'
|
|
||||||
ssh root@staging2.testrun.org 'install -m 0755 -d /etc/apt/keyrings'
|
|
||||||
ssh root@staging2.testrun.org 'curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && chmod a+r /etc/apt/keyrings/docker.asc'
|
|
||||||
ssh root@staging2.testrun.org 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" > /etc/apt/sources.list.d/docker.list'
|
|
||||||
ssh root@staging2.testrun.org 'apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin'
|
|
||||||
|
|
||||||
- name: prepare Docker bind mounts
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging2.testrun.org 'mkdir -p /srv/chatmail/certs /srv/chatmail/dkim'
|
|
||||||
ssh root@staging2.testrun.org 'cp -a /var/lib/acme/. /srv/chatmail/certs/ && cp -a /etc/dkimkeys/. /srv/chatmail/dkim/' || true
|
|
||||||
|
|
||||||
- name: upload chatmail.ini for Docker
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Reuse chatmail.ini already created by the bare-metal deploy steps
|
|
||||||
scp chatmail.ini root@staging2.testrun.org:/srv/chatmail/chatmail.ini
|
|
||||||
|
|
||||||
- name: deploy with Docker
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
|
||||||
GHCR_IMAGE="${{ env.REGISTRY }}/$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]'):sha-${SHORT_SHA}"
|
|
||||||
rsync -avz --exclude='.git' --exclude='venv' --exclude='__pycache__' ./ root@staging2.testrun.org:/srv/chatmail/relay/
|
|
||||||
# Login to GHCR on VPS and pull pre-built image
|
|
||||||
echo "${{ secrets.GITHUB_TOKEN }}" | ssh root@staging2.testrun.org 'docker login ghcr.io -u ${{ github.actor }} --password-stdin'
|
|
||||||
ssh root@staging2.testrun.org "docker pull ${GHCR_IMAGE}"
|
|
||||||
ssh root@staging2.testrun.org "cd /srv/chatmail/relay && CHATMAIL_IMAGE=${GHCR_IMAGE} MAIL_DOMAIN=staging2.testrun.org docker compose -f docker/docker-compose.yaml -f docker/docker-compose.ci.yaml up -d"
|
|
||||||
|
|
||||||
- name: wait for container healthy
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Stream journald inside the container
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail journalctl -f --no-pager' &
|
|
||||||
LOG_PID=$!
|
|
||||||
trap "kill $LOG_PID 2>/dev/null || true" EXIT
|
|
||||||
for i in $(seq 1 60); do
|
|
||||||
status=$(ssh root@staging2.testrun.org 'docker inspect --format={{.State.Health.Status}} chatmail 2>/dev/null' || echo "missing")
|
|
||||||
echo " [$i/60] status=$status"
|
|
||||||
if [ "$status" = "healthy" ]; then
|
|
||||||
echo "Container is healthy."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
if [ "$status" = "unhealthy" ]; then
|
|
||||||
echo "Container is unhealthy!"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
echo "Container did not become healthy."
|
|
||||||
kill $LOG_PID 2>/dev/null || true
|
|
||||||
echo "--- failed units ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail systemctl --failed --no-pager' || true
|
|
||||||
echo "--- service logs ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail journalctl -u dovecot -u postfix -u nginx -u unbound --no-pager -n 50' || true
|
|
||||||
echo "--- listening ports ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail ss -tlnp' || true
|
|
||||||
echo "--- chatmail.ini ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini' || true
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
- name: show container state
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
echo "--- listening ports ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail ss -tlnp'
|
|
||||||
echo "--- chatmail.ini ---"
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail cat /etc/chatmail/chatmail.ini'
|
|
||||||
|
|
||||||
- name: Docker integration tests
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy test --slow --ssh-host @local'
|
|
||||||
|
|
||||||
- name: Docker DNS
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: |
|
|
||||||
# Reset zone file in case bare DNS already appended to it
|
|
||||||
git checkout .github/workflows/staging.testrun.org-default.zone
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail chown opendkim:opendkim -R /etc/dkimkeys'
|
|
||||||
ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy dns --ssh-host @local --zonefile /opt/chatmail/staging.zone --verbose'
|
|
||||||
ssh root@staging2.testrun.org 'docker cp chatmail:/opt/chatmail/staging.zone /tmp/staging.zone'
|
|
||||||
scp root@staging2.testrun.org:/tmp/staging.zone staging-generated.zone
|
|
||||||
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
|
|
||||||
cat .github/workflows/staging.testrun.org-default.zone
|
|
||||||
scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
|
|
||||||
ssh root@ns.testrun.org systemctl reload nsd
|
|
||||||
|
|
||||||
- name: Docker final DNS check
|
|
||||||
if: >-
|
|
||||||
!cancelled() && github.event_name == 'push'
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: ssh root@staging2.testrun.org 'docker exec chatmail cmdeploy dns -v --ssh-host @local'
|
|
||||||
|
|
||||||
# --- Cleanup ---
|
|
||||||
|
|
||||||
- name: add SSH keys
|
|
||||||
if: >-
|
|
||||||
!cancelled()
|
|
||||||
&& steps.wait-for-vps.outcome == 'success'
|
|
||||||
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys https://github.com/j4n.keys >> .ssh/authorized_keys'
|
|
||||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -5,6 +5,7 @@ __pycache__/
|
|||||||
*.swp
|
*.swp
|
||||||
*qr-*.png
|
*qr-*.png
|
||||||
chatmail*.ini
|
chatmail*.ini
|
||||||
|
lxconfigs/
|
||||||
|
|
||||||
|
|
||||||
# C extensions
|
# C extensions
|
||||||
@@ -164,9 +165,3 @@ cython_debug/
|
|||||||
#.idea/
|
#.idea/
|
||||||
|
|
||||||
chatmail.zone
|
chatmail.zone
|
||||||
|
|
||||||
# docker
|
|
||||||
/data/
|
|
||||||
/custom/
|
|
||||||
docker/docker-compose.override.yaml
|
|
||||||
docker/.env
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import importlib.resources
|
import importlib.resources
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
from contextlib import contextmanager
|
|
||||||
|
|
||||||
from pyinfra.operations import files, server, systemd
|
from pyinfra.operations import files, server, systemd
|
||||||
|
|
||||||
@@ -11,28 +10,6 @@ def has_systemd():
|
|||||||
return os.path.isdir("/run/systemd/system")
|
return os.path.isdir("/run/systemd/system")
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def blocked_service_startup():
|
|
||||||
"""Prevent services from auto-starting during package installation.
|
|
||||||
|
|
||||||
Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
|
|
||||||
service from being started by the package manager. This avoids bind
|
|
||||||
conflicts and CPU/RAM spikes during initial setup. The file is removed
|
|
||||||
when the context exits.
|
|
||||||
"""
|
|
||||||
# For documentation about policy-rc.d, see:
|
|
||||||
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
|
|
||||||
files.put(
|
|
||||||
src=get_resource("policy-rc.d"),
|
|
||||||
dest="/usr/sbin/policy-rc.d",
|
|
||||||
user="root",
|
|
||||||
group="root",
|
|
||||||
mode="755",
|
|
||||||
)
|
|
||||||
yield
|
|
||||||
files.file("/usr/sbin/policy-rc.d", present=False)
|
|
||||||
|
|
||||||
|
|
||||||
def get_resource(arg, pkg=__package__):
|
def get_resource(arg, pkg=__package__):
|
||||||
return importlib.resources.files(pkg).joinpath(arg)
|
return importlib.resources.files(pkg).joinpath(arg)
|
||||||
|
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
;
|
|
||||||
; Required DNS entries for chatmail servers
|
|
||||||
;
|
|
||||||
{% if A %}
|
|
||||||
{{ mail_domain }}. A {{ A }}
|
|
||||||
{% endif %}
|
|
||||||
{% if AAAA %}
|
|
||||||
{{ mail_domain }}. AAAA {{ AAAA }}
|
|
||||||
{% endif %}
|
|
||||||
{{ mail_domain }}. MX 10 {{ mail_domain }}.
|
|
||||||
{% if strict_tls %}
|
|
||||||
_mta-sts.{{ mail_domain }}. TXT "v=STSv1; id={{ sts_id }}"
|
|
||||||
mta-sts.{{ mail_domain }}. CNAME {{ mail_domain }}.
|
|
||||||
{% endif %}
|
|
||||||
www.{{ mail_domain }}. CNAME {{ mail_domain }}.
|
|
||||||
{{ dkim_entry }}
|
|
||||||
|
|
||||||
;
|
|
||||||
; Recommended DNS entries for interoperability and security-hardening
|
|
||||||
;
|
|
||||||
{{ mail_domain }}. TXT "v=spf1 a ~all"
|
|
||||||
_dmarc.{{ mail_domain }}. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
|
||||||
|
|
||||||
{% if acme_account_url %}
|
|
||||||
{{ mail_domain }}. CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
|
|
||||||
{% endif %}
|
|
||||||
_adsp._domainkey.{{ mail_domain }}. TXT "dkim=discardable"
|
|
||||||
|
|
||||||
_submission._tcp.{{ mail_domain }}. SRV 0 1 587 {{ mail_domain }}.
|
|
||||||
_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
|
|
||||||
_imap._tcp.{{ mail_domain }}. SRV 0 1 143 {{ mail_domain }}.
|
|
||||||
_imaps._tcp.{{ mail_domain }}. SRV 0 1 993 {{ mail_domain }}.
|
|
||||||
@@ -10,6 +10,8 @@ import pathlib
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
from contextlib import contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pyinfra
|
import pyinfra
|
||||||
@@ -18,7 +20,24 @@ from packaging import version
|
|||||||
from termcolor import colored
|
from termcolor import colored
|
||||||
|
|
||||||
from . import dns, remote
|
from . import dns, remote
|
||||||
from .sshexec import LocalExec, SSHExec
|
from .lxc.cli import ( # noqa: F401
|
||||||
|
lxc_start_cmd,
|
||||||
|
lxc_start_cmd_options,
|
||||||
|
lxc_status_cmd,
|
||||||
|
lxc_status_cmd_options,
|
||||||
|
lxc_stop_cmd,
|
||||||
|
lxc_stop_cmd_options,
|
||||||
|
lxc_test_cmd,
|
||||||
|
lxc_test_cmd_options,
|
||||||
|
)
|
||||||
|
from .sshexec import (
|
||||||
|
LocalExec,
|
||||||
|
SSHExec,
|
||||||
|
resolve_host_from_ssh_config,
|
||||||
|
resolve_key_from_ssh_config,
|
||||||
|
)
|
||||||
|
from .util import build_chatmaild_sdist
|
||||||
|
from .www import main as webdev_main
|
||||||
|
|
||||||
#
|
#
|
||||||
# cmdeploy sub commands and options
|
# cmdeploy sub commands and options
|
||||||
@@ -82,18 +101,21 @@ def run_cmd_options(parser):
|
|||||||
help="disable checks nslookup for dns",
|
help="disable checks nslookup for dns",
|
||||||
)
|
)
|
||||||
add_ssh_host_option(parser)
|
add_ssh_host_option(parser)
|
||||||
|
add_ssh_config_option(parser)
|
||||||
|
|
||||||
|
|
||||||
def run_cmd(args, out):
|
def run_cmd(args, out):
|
||||||
"""Deploy chatmail services on the remote server."""
|
"""Deploy chatmail services on the remote server."""
|
||||||
|
|
||||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||||
sshexec = get_sshexec(ssh_host)
|
sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
|
||||||
require_iroh = args.config.enable_iroh_relay
|
require_iroh = args.config.enable_iroh_relay
|
||||||
strict_tls = args.config.tls_cert_mode == "acme"
|
strict_tls = args.config.tls_cert_mode == "acme"
|
||||||
if not args.dns_check_disabled:
|
if not args.dns_check_disabled:
|
||||||
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
||||||
if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
|
if not dns.check_initial_remote_data(
|
||||||
|
remote_data, strict_tls=strict_tls, print=out.red
|
||||||
|
):
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
@@ -101,6 +123,9 @@ def run_cmd(args, out):
|
|||||||
env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
|
env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
|
||||||
env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
|
env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
|
||||||
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
|
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
|
||||||
|
|
||||||
|
if not args.website_only:
|
||||||
|
build_chatmaild_sdist()
|
||||||
if not args.dns_check_disabled:
|
if not args.dns_check_disabled:
|
||||||
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
|
env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
|
||||||
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
|
env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
|
||||||
@@ -108,7 +133,22 @@ def run_cmd(args, out):
|
|||||||
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
|
||||||
|
|
||||||
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
|
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
|
||||||
if ssh_host == "localhost":
|
ssh_config = args.ssh_config
|
||||||
|
if ssh_config:
|
||||||
|
ssh_config = str(Path(ssh_config).resolve())
|
||||||
|
|
||||||
|
# Use pyinfra's native SSH data keys to configure the connection directly
|
||||||
|
# rather than relying on paramiko config parsing (see also sshexec.py)
|
||||||
|
ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
|
||||||
|
key = resolve_key_from_ssh_config(ssh_host, ssh_config)
|
||||||
|
data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
|
||||||
|
if key:
|
||||||
|
data_args += f" --data ssh_key={key}"
|
||||||
|
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
|
||||||
|
if ssh_host in ["localhost", "@docker"]:
|
||||||
|
if ssh_host == "@docker":
|
||||||
|
env["CHATMAIL_NOPORTCHECK"] = "True"
|
||||||
|
env["CHATMAIL_NOSYSCTL"] = "True"
|
||||||
cmd = f"{pyinf} @local {deploy_path} -y"
|
cmd = f"{pyinf} @local {deploy_path} -y"
|
||||||
|
|
||||||
if version.parse(pyinfra.__version__) < version.parse("3"):
|
if version.parse(pyinfra.__version__) < version.parse("3"):
|
||||||
@@ -119,7 +159,11 @@ def run_cmd(args, out):
|
|||||||
out.check_call(cmd, env=env)
|
out.check_call(cmd, env=env)
|
||||||
if args.website_only:
|
if args.website_only:
|
||||||
out.green("Website deployment completed.")
|
out.green("Website deployment completed.")
|
||||||
elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
|
elif (
|
||||||
|
not args.dns_check_disabled
|
||||||
|
and strict_tls
|
||||||
|
and not remote_data["acme_account_url"]
|
||||||
|
):
|
||||||
out.red("Deploy completed but letsencrypt not configured")
|
out.red("Deploy completed but letsencrypt not configured")
|
||||||
out.red("Run 'cmdeploy run' again")
|
out.red("Run 'cmdeploy run' again")
|
||||||
else:
|
else:
|
||||||
@@ -136,15 +180,16 @@ def dns_cmd_options(parser):
|
|||||||
dest="zonefile",
|
dest="zonefile",
|
||||||
type=pathlib.Path,
|
type=pathlib.Path,
|
||||||
default=None,
|
default=None,
|
||||||
help="write out a zonefile",
|
help="write DNS records in standard BIND format to the given file",
|
||||||
)
|
)
|
||||||
add_ssh_host_option(parser)
|
add_ssh_host_option(parser)
|
||||||
|
add_ssh_config_option(parser)
|
||||||
|
|
||||||
|
|
||||||
def dns_cmd(args, out):
|
def dns_cmd(args, out):
|
||||||
"""Check DNS entries and optionally generate dns zone file."""
|
"""Check DNS entries and optionally generate dns zone file."""
|
||||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
|
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
|
||||||
tls_cert_mode = args.config.tls_cert_mode
|
tls_cert_mode = args.config.tls_cert_mode
|
||||||
strict_tls = tls_cert_mode == "acme"
|
strict_tls = tls_cert_mode == "acme"
|
||||||
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
|
||||||
@@ -175,13 +220,14 @@ def dns_cmd(args, out):
|
|||||||
|
|
||||||
def status_cmd_options(parser):
|
def status_cmd_options(parser):
|
||||||
add_ssh_host_option(parser)
|
add_ssh_host_option(parser)
|
||||||
|
add_ssh_config_option(parser)
|
||||||
|
|
||||||
|
|
||||||
def status_cmd(args, out):
|
def status_cmd(args, out):
|
||||||
"""Display status for online chatmail instance."""
|
"""Display status for online chatmail instance."""
|
||||||
|
|
||||||
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
|
||||||
sshexec = get_sshexec(ssh_host, verbose=args.verbose)
|
sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
|
||||||
|
|
||||||
out.green(f"chatmail domain: {args.config.mail_domain}")
|
out.green(f"chatmail domain: {args.config.mail_domain}")
|
||||||
if args.config.privacy_mail:
|
if args.config.privacy_mail:
|
||||||
@@ -201,14 +247,14 @@ def test_cmd_options(parser):
|
|||||||
help="also run slow tests",
|
help="also run slow tests",
|
||||||
)
|
)
|
||||||
add_ssh_host_option(parser)
|
add_ssh_host_option(parser)
|
||||||
|
add_ssh_config_option(parser)
|
||||||
|
|
||||||
|
|
||||||
def test_cmd(args, out):
|
def test_cmd(args, out):
|
||||||
"""Run local and online tests for chatmail deployment."""
|
"""Run local and online tests for chatmail deployment."""
|
||||||
|
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
if args.ssh_host:
|
env["CHATMAIL_INI"] = str(args.inipath.resolve())
|
||||||
env["CHATMAIL_SSH"] = args.ssh_host
|
|
||||||
|
|
||||||
pytest_path = shutil.which("pytest")
|
pytest_path = shutil.which("pytest")
|
||||||
pytest_args = [
|
pytest_args = [
|
||||||
@@ -222,6 +268,10 @@ def test_cmd(args, out):
|
|||||||
]
|
]
|
||||||
if args.slow:
|
if args.slow:
|
||||||
pytest_args.append("--slow")
|
pytest_args.append("--slow")
|
||||||
|
if args.ssh_host:
|
||||||
|
pytest_args.extend(["--ssh-host", args.ssh_host])
|
||||||
|
if args.ssh_config:
|
||||||
|
pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
|
||||||
ret = out.run_ret(pytest_args, env=env)
|
ret = out.run_ret(pytest_args, env=env)
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
@@ -273,9 +323,7 @@ def bench_cmd(args, out):
|
|||||||
|
|
||||||
def webdev_cmd(args, out):
|
def webdev_cmd(args, out):
|
||||||
"""Run local web development loop for static web pages."""
|
"""Run local web development loop for static web pages."""
|
||||||
from .www import main
|
webdev_main()
|
||||||
|
|
||||||
main()
|
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -284,17 +332,44 @@ def webdev_cmd(args, out):
|
|||||||
|
|
||||||
|
|
||||||
class Out:
|
class Out:
|
||||||
"""Convenience output printer providing coloring."""
|
"""Convenience output printer providing coloring and section formatting."""
|
||||||
|
|
||||||
|
SECTION_WIDTH = 72
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.section_timings = []
|
||||||
|
|
||||||
def red(self, msg, file=sys.stderr):
|
def red(self, msg, file=sys.stderr):
|
||||||
print(colored(msg, "red"), file=file)
|
print(colored(msg, "red"), file=file, flush=True)
|
||||||
|
|
||||||
def green(self, msg, file=sys.stderr):
|
def green(self, msg, file=sys.stderr):
|
||||||
print(colored(msg, "green"), file=file)
|
print(colored(msg, "green"), file=file, flush=True)
|
||||||
|
|
||||||
|
def print(self, msg="", **kwargs):
|
||||||
|
"""Print to stdout with automatic flush."""
|
||||||
|
print(msg, flush=True, **kwargs)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def section(self, title):
|
||||||
|
"""Context manager that prints a section header and records elapsed time."""
|
||||||
|
bar = "\u2501" * (self.SECTION_WIDTH - len(title) - 5)
|
||||||
|
self.green(f"\u2501\u2501\u2501 {title} {bar}")
|
||||||
|
t0 = time.time()
|
||||||
|
yield
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
self.section_timings.append((title, elapsed))
|
||||||
|
self.print(f"{'':>{self.SECTION_WIDTH - 10}}({elapsed:.1f}s)")
|
||||||
|
self.print()
|
||||||
|
|
||||||
|
def section_line(self, title):
|
||||||
|
"""Print a section header without timing."""
|
||||||
|
bar = "\u2501" * (self.SECTION_WIDTH - len(title) - 5)
|
||||||
|
self.green(f"\u2501\u2501\u2501 {title} {bar}")
|
||||||
|
self.print()
|
||||||
|
|
||||||
def __call__(self, msg, red=False, green=False, file=sys.stdout):
|
def __call__(self, msg, red=False, green=False, file=sys.stdout):
|
||||||
color = "red" if red else ("green" if green else None)
|
color = "red" if red else ("green" if green else None)
|
||||||
print(colored(msg, color), file=file)
|
print(colored(msg, color), file=file, flush=True)
|
||||||
|
|
||||||
def check_call(self, arg, env=None, quiet=False):
|
def check_call(self, arg, env=None, quiet=False):
|
||||||
if not quiet:
|
if not quiet:
|
||||||
@@ -313,11 +388,21 @@ def add_ssh_host_option(parser):
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--ssh-host",
|
"--ssh-host",
|
||||||
dest="ssh_host",
|
dest="ssh_host",
|
||||||
help="Run commands on 'localhost' or on a specific SSH host "
|
help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
|
||||||
"instead of chatmail.ini's mail_domain.",
|
"instead of chatmail.ini's mail_domain.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def add_ssh_config_option(parser):
|
||||||
|
parser.add_argument(
|
||||||
|
"--ssh-config",
|
||||||
|
dest="ssh_config",
|
||||||
|
type=Path,
|
||||||
|
default=None,
|
||||||
|
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_config_option(parser):
|
def add_config_option(parser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--config",
|
"--config",
|
||||||
@@ -327,6 +412,7 @@ def add_config_option(parser):
|
|||||||
type=Path,
|
type=Path,
|
||||||
help="path to the chatmail.ini file",
|
help="path to the chatmail.ini file",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--verbose",
|
"--verbose",
|
||||||
"-v",
|
"-v",
|
||||||
@@ -337,15 +423,16 @@ def add_config_option(parser):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def add_subcommand(subparsers, func):
|
def add_subcommand(subparsers, func, add_config=True):
|
||||||
name = func.__name__
|
name = func.__name__
|
||||||
assert name.endswith("_cmd")
|
assert name.endswith("_cmd")
|
||||||
name = name[:-4]
|
name = name[:-4].replace("_", "-")
|
||||||
doc = func.__doc__.strip()
|
doc = func.__doc__.strip()
|
||||||
help = doc.split("\n")[0].strip(".")
|
help = doc.split("\n")[0].strip(".")
|
||||||
p = subparsers.add_parser(name, description=doc, help=help)
|
p = subparsers.add_parser(name, description=doc, help=help)
|
||||||
p.set_defaults(func=func)
|
p.set_defaults(func=func)
|
||||||
add_config_option(p)
|
if add_config:
|
||||||
|
add_config_option(p)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
@@ -359,13 +446,15 @@ def get_parser():
|
|||||||
"""Return an ArgumentParser for the 'cmdeploy' CLI"""
|
"""Return an ArgumentParser for the 'cmdeploy' CLI"""
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description=description.strip())
|
parser = argparse.ArgumentParser(description=description.strip())
|
||||||
|
parser.set_defaults(func=None, inipath=None)
|
||||||
subparsers = parser.add_subparsers(title="subcommands")
|
subparsers = parser.add_subparsers(title="subcommands")
|
||||||
|
|
||||||
# find all subcommands in the module namespace
|
# find all subcommands in the module namespace
|
||||||
glob = globals()
|
glob = globals()
|
||||||
for name, func in glob.items():
|
for name, func in glob.items():
|
||||||
if name.endswith("_cmd"):
|
if name.endswith("_cmd"):
|
||||||
subparser = add_subcommand(subparsers, func)
|
needs_config = not name.startswith("lxc_")
|
||||||
|
subparser = add_subcommand(subparsers, func, add_config=needs_config)
|
||||||
addopts = glob.get(name + "_options")
|
addopts = glob.get(name + "_options")
|
||||||
if addopts is not None:
|
if addopts is not None:
|
||||||
addopts(subparser)
|
addopts(subparser)
|
||||||
@@ -373,24 +462,27 @@ def get_parser():
|
|||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
def get_sshexec(ssh_host: str, verbose=True):
|
def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
|
||||||
if ssh_host in ["localhost", "@local"]:
|
if ssh_host in ["localhost", "@local"]:
|
||||||
return LocalExec(verbose)
|
return LocalExec(verbose, docker=False)
|
||||||
|
elif ssh_host == "@docker":
|
||||||
|
return LocalExec(verbose, docker=True)
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"[ssh] login to {ssh_host}")
|
print(f"[ssh] login to {ssh_host}")
|
||||||
return SSHExec(ssh_host, verbose=verbose)
|
return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
|
||||||
|
|
||||||
|
|
||||||
def main(args=None):
|
def main(args=None):
|
||||||
"""Provide main entry point for 'cmdeploy' CLI invocation."""
|
"""Provide main entry point for 'cmdeploy' CLI invocation."""
|
||||||
parser = get_parser()
|
parser = get_parser()
|
||||||
args = parser.parse_args(args=args)
|
args = parser.parse_args(args=args)
|
||||||
if not hasattr(args, "func"):
|
if args.func is None:
|
||||||
return parser.parse_args(["-h"])
|
return parser.parse_args(["-h"])
|
||||||
|
|
||||||
out = Out()
|
out = Out()
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
|
|
||||||
|
if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
|
||||||
if not args.inipath.exists():
|
if not args.inipath.exists():
|
||||||
out.red(f"expecting {args.inipath} to exist, run init first?")
|
out.red(f"expecting {args.inipath} to exist, run init first?")
|
||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
|
|||||||
@@ -2,9 +2,7 @@
|
|||||||
Chat Mail pyinfra deploy.
|
Chat Mail pyinfra deploy.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import shutil
|
import os
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from io import BytesIO, StringIO
|
from io import BytesIO, StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -13,11 +11,11 @@ from pyinfra import facts, host, logger
|
|||||||
from pyinfra.api import FactBase
|
from pyinfra.api import FactBase
|
||||||
from pyinfra.facts import hardware
|
from pyinfra.facts import hardware
|
||||||
from pyinfra.facts.files import Sha256File
|
from pyinfra.facts.files import Sha256File
|
||||||
from pyinfra.facts.server import Command
|
|
||||||
from pyinfra.facts.systemd import SystemdEnabled
|
from pyinfra.facts.systemd import SystemdEnabled
|
||||||
from pyinfra.operations import apt, files, pip, server, systemd
|
from pyinfra.operations import apt, files, pip, server, systemd
|
||||||
|
|
||||||
from cmdeploy.cmdeploy import Out
|
from cmdeploy.cmdeploy import Out
|
||||||
|
from cmdeploy.util import get_chatmaild_sdist, get_version_string
|
||||||
|
|
||||||
from .acmetool import AcmetoolDeployer
|
from .acmetool import AcmetoolDeployer
|
||||||
from .basedeploy import (
|
from .basedeploy import (
|
||||||
@@ -54,20 +52,6 @@ class Port(FactBase):
|
|||||||
return output[0]
|
return output[0]
|
||||||
|
|
||||||
|
|
||||||
def _build_chatmaild(dist_dir) -> None:
|
|
||||||
dist_dir = Path(dist_dir).resolve()
|
|
||||||
if dist_dir.exists():
|
|
||||||
shutil.rmtree(dist_dir)
|
|
||||||
dist_dir.mkdir()
|
|
||||||
subprocess.check_output(
|
|
||||||
[sys.executable, "-m", "build", "-n"]
|
|
||||||
+ ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
|
|
||||||
)
|
|
||||||
entries = list(dist_dir.iterdir())
|
|
||||||
assert len(entries) == 1
|
|
||||||
return entries[0]
|
|
||||||
|
|
||||||
|
|
||||||
def remove_legacy_artifacts():
|
def remove_legacy_artifacts():
|
||||||
if not has_systemd():
|
if not has_systemd():
|
||||||
return
|
return
|
||||||
@@ -83,7 +67,7 @@ def remove_legacy_artifacts():
|
|||||||
|
|
||||||
def _install_remote_venv_with_chatmaild() -> None:
|
def _install_remote_venv_with_chatmaild() -> None:
|
||||||
remove_legacy_artifacts()
|
remove_legacy_artifacts()
|
||||||
dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
|
dist_file = get_chatmaild_sdist()
|
||||||
remote_base_dir = "/usr/local/lib/chatmaild"
|
remote_base_dir = "/usr/local/lib/chatmaild"
|
||||||
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
|
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
|
||||||
remote_venv_dir = f"{remote_base_dir}/venv"
|
remote_venv_dir = f"{remote_base_dir}/venv"
|
||||||
@@ -271,8 +255,14 @@ class WebsiteDeployer(Deployer):
|
|||||||
logger.warning("Web page build failed, skipping website deployment")
|
logger.warning("Web page build failed, skipping website deployment")
|
||||||
return
|
return
|
||||||
# if it is not a hugo page, upload it as is
|
# if it is not a hugo page, upload it as is
|
||||||
files.rsync(
|
# pyinfra files.rsync (experimental) causes problems with ssh-config configuration
|
||||||
f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
|
# the stable files.sync should do
|
||||||
|
files.sync(
|
||||||
|
src=str(www_path),
|
||||||
|
dest="/var/www/html",
|
||||||
|
user="www-data",
|
||||||
|
group="www-data",
|
||||||
|
delete=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -336,12 +326,12 @@ class TurnDeployer(Deployer):
|
|||||||
def install(self):
|
def install(self):
|
||||||
(url, sha256sum) = {
|
(url, sha256sum) = {
|
||||||
"x86_64": (
|
"x86_64": (
|
||||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
|
"https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
|
||||||
"841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
|
"1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
|
||||||
),
|
),
|
||||||
"aarch64": (
|
"aarch64": (
|
||||||
"https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
|
"https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
|
||||||
"a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
|
"0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
|
||||||
),
|
),
|
||||||
}[host.get_fact(facts.server.Arch)]
|
}[host.get_fact(facts.server.Arch)]
|
||||||
|
|
||||||
@@ -480,8 +470,8 @@ class ChatmailDeployer(Deployer):
|
|||||||
def install(self):
|
def install(self):
|
||||||
files.put(
|
files.put(
|
||||||
name="Disable installing recommended packages globally",
|
name="Disable installing recommended packages globally",
|
||||||
src=BytesIO(b'APT::Install-Recommends "false";\n'),
|
src=BytesIO(b'APT::Install-Recommends "0";\n'),
|
||||||
dest="/etc/apt/apt.conf.d/00InstallRecommends",
|
dest="/etc/apt/apt.conf.d/99no-recommends",
|
||||||
user="root",
|
user="root",
|
||||||
group="root",
|
group="root",
|
||||||
mode="644",
|
mode="644",
|
||||||
@@ -532,17 +522,9 @@ class FcgiwrapDeployer(Deployer):
|
|||||||
|
|
||||||
class GithashDeployer(Deployer):
|
class GithashDeployer(Deployer):
|
||||||
def activate(self):
|
def activate(self):
|
||||||
try:
|
|
||||||
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
|
|
||||||
except Exception:
|
|
||||||
git_hash = "unknown\n"
|
|
||||||
try:
|
|
||||||
git_diff = subprocess.check_output(["git", "diff"]).decode()
|
|
||||||
except Exception:
|
|
||||||
git_diff = ""
|
|
||||||
files.put(
|
files.put(
|
||||||
name="Upload chatmail relay git commit hash",
|
name="Upload chatmail relay git commit hash",
|
||||||
src=StringIO(git_hash + git_diff),
|
src=StringIO(get_version_string()),
|
||||||
dest="/etc/chatmail-version",
|
dest="/etc/chatmail-version",
|
||||||
mode="700",
|
mode="700",
|
||||||
)
|
)
|
||||||
@@ -586,14 +568,20 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Check if mtail_address interface is available (if configured)
|
# Check if mtail_address interface is available (if configured)
|
||||||
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
if config.mtail_address and config.mtail_address not in (
|
||||||
|
"127.0.0.1",
|
||||||
|
"::1",
|
||||||
|
"localhost",
|
||||||
|
):
|
||||||
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
||||||
all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
|
all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
|
||||||
if config.mtail_address not in all_addresses:
|
if config.mtail_address not in all_addresses:
|
||||||
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
|
Out().red(
|
||||||
|
f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
|
||||||
|
)
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
if host.get_fact(Command, "systemd-detect-virt -c || true") == "none":
|
if not os.environ.get("CHATMAIL_NOPORTCHECK"):
|
||||||
port_services = [
|
port_services = [
|
||||||
(["master", "smtpd"], 25),
|
(["master", "smtpd"], 25),
|
||||||
("unbound", 53),
|
("unbound", 53),
|
||||||
|
|||||||
@@ -1,11 +1,26 @@
|
|||||||
import datetime
|
import datetime
|
||||||
import importlib
|
|
||||||
|
|
||||||
from jinja2 import Template
|
|
||||||
|
|
||||||
from . import remote
|
from . import remote
|
||||||
|
|
||||||
|
|
||||||
|
def parse_zone_records(text):
|
||||||
|
"""Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
|
||||||
|
|
||||||
|
Skips comment lines (starting with ``;``) and blank lines.
|
||||||
|
Each record line must have the format ``name TTL IN type rdata``.
|
||||||
|
"""
|
||||||
|
for raw_line in text.splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith(";"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
name, ttl, _in, rtype, rdata = line.split(None, 4)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"Bad zone record line: {line!r}") from None
|
||||||
|
name = name.rstrip(".")
|
||||||
|
yield name, ttl, rtype.upper(), rdata
|
||||||
|
|
||||||
|
|
||||||
def get_initial_remote_data(sshexec, mail_domain):
|
def get_initial_remote_data(sshexec, mail_domain):
|
||||||
return sshexec.logged(
|
return sshexec.logged(
|
||||||
call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
|
call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
|
||||||
@@ -31,13 +46,36 @@ def get_filled_zone_file(remote_data):
|
|||||||
if not sts_id:
|
if not sts_id:
|
||||||
remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
|
remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
|
||||||
|
|
||||||
template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
|
d = remote_data["mail_domain"]
|
||||||
content = template.read_text()
|
lines = ["; Required DNS entries"]
|
||||||
zonefile = Template(content).render(**remote_data)
|
if remote_data.get("A"):
|
||||||
lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
|
lines.append(f"{d}. 3600 IN A {remote_data['A']}")
|
||||||
|
if remote_data.get("AAAA"):
|
||||||
|
lines.append(f"{d}. 3600 IN AAAA {remote_data['AAAA']}")
|
||||||
|
lines.append(f"{d}. 3600 IN MX 10 {d}.")
|
||||||
|
if remote_data.get("strict_tls"):
|
||||||
|
lines.append(
|
||||||
|
f'_mta-sts.{d}. 3600 IN TXT "v=STSv1; id={remote_data["sts_id"]}"'
|
||||||
|
)
|
||||||
|
lines.append(f"mta-sts.{d}. 3600 IN CNAME {d}.")
|
||||||
|
lines.append(f"www.{d}. 3600 IN CNAME {d}.")
|
||||||
|
lines.append(remote_data["dkim_entry"])
|
||||||
lines.append("")
|
lines.append("")
|
||||||
zonefile = "\n".join(lines)
|
lines.append("; Recommended DNS entries")
|
||||||
return zonefile
|
lines.append(f'{d}. 3600 IN TXT "v=spf1 a ~all"')
|
||||||
|
lines.append(f'_dmarc.{d}. 3600 IN TXT "v=DMARC1;p=reject;adkim=s;aspf=s"')
|
||||||
|
if remote_data.get("acme_account_url"):
|
||||||
|
lines.append(
|
||||||
|
f"{d}. 3600 IN CAA 0 issue"
|
||||||
|
f' "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"'
|
||||||
|
)
|
||||||
|
lines.append(f'_adsp._domainkey.{d}. 3600 IN TXT "dkim=discardable"')
|
||||||
|
lines.append(f"_submission._tcp.{d}. 3600 IN SRV 0 1 587 {d}.")
|
||||||
|
lines.append(f"_submissions._tcp.{d}. 3600 IN SRV 0 1 465 {d}.")
|
||||||
|
lines.append(f"_imap._tcp.{d}. 3600 IN SRV 0 1 143 {d}.")
|
||||||
|
lines.append(f"_imaps._tcp.{d}. 3600 IN SRV 0 1 993 {d}.")
|
||||||
|
lines.append("")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
|
def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
|
||||||
|
|||||||
@@ -1,30 +1,20 @@
|
|||||||
|
import os
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
from chatmaild.config import Config
|
from chatmaild.config import Config
|
||||||
from pyinfra import host
|
from pyinfra import host
|
||||||
from pyinfra.facts.deb import DebPackages
|
from pyinfra.facts.server import Arch, Sysctl
|
||||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
from pyinfra.facts.systemd import SystemdEnabled
|
||||||
from pyinfra.operations import apt, files, server, systemd
|
from pyinfra.operations import apt, files, server, systemd
|
||||||
|
|
||||||
from cmdeploy.basedeploy import (
|
from cmdeploy.basedeploy import (
|
||||||
Deployer,
|
Deployer,
|
||||||
activate_remote_units,
|
activate_remote_units,
|
||||||
blocked_service_startup,
|
|
||||||
configure_remote_units,
|
configure_remote_units,
|
||||||
get_resource,
|
get_resource,
|
||||||
|
has_systemd,
|
||||||
)
|
)
|
||||||
|
|
||||||
DOVECOT_VERSION = "2.3.21+dfsg1-3"
|
|
||||||
|
|
||||||
DOVECOT_SHA256 = {
|
|
||||||
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
|
|
||||||
("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
|
|
||||||
("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
|
|
||||||
("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
|
|
||||||
("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
|
|
||||||
("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class DovecotDeployer(Deployer):
|
class DovecotDeployer(Deployer):
|
||||||
daemon_reload = False
|
daemon_reload = False
|
||||||
@@ -36,10 +26,11 @@ class DovecotDeployer(Deployer):
|
|||||||
|
|
||||||
def install(self):
|
def install(self):
|
||||||
arch = host.get_fact(Arch)
|
arch = host.get_fact(Arch)
|
||||||
with blocked_service_startup():
|
if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
|
||||||
_install_dovecot_package("core", arch)
|
return # already installed and running
|
||||||
_install_dovecot_package("imapd", arch)
|
_install_dovecot_package("core", arch)
|
||||||
_install_dovecot_package("lmtpd", arch)
|
_install_dovecot_package("imapd", arch)
|
||||||
|
_install_dovecot_package("lmtpd", arch)
|
||||||
|
|
||||||
def configure(self):
|
def configure(self):
|
||||||
configure_remote_units(self.config.mail_domain, self.units)
|
configure_remote_units(self.config.mail_domain, self.units)
|
||||||
@@ -75,33 +66,37 @@ def _pick_url(primary, fallback):
|
|||||||
def _install_dovecot_package(package: str, arch: str):
|
def _install_dovecot_package(package: str, arch: str):
|
||||||
arch = "amd64" if arch == "x86_64" else arch
|
arch = "amd64" if arch == "x86_64" else arch
|
||||||
arch = "arm64" if arch == "aarch64" else arch
|
arch = "arm64" if arch == "aarch64" else arch
|
||||||
|
primary_url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
|
||||||
pkg_name = f"dovecot-{package}"
|
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F2.3.21%2Bdfsg1/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
|
||||||
sha256 = DOVECOT_SHA256.get((package, arch))
|
|
||||||
if sha256 is None:
|
|
||||||
apt.packages(packages=[pkg_name])
|
|
||||||
return
|
|
||||||
|
|
||||||
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
|
|
||||||
if DOVECOT_VERSION in installed_versions:
|
|
||||||
return
|
|
||||||
|
|
||||||
url_version = DOVECOT_VERSION.replace("+", "%2B")
|
|
||||||
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
|
|
||||||
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
|
|
||||||
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
|
|
||||||
url = _pick_url(primary_url, fallback_url)
|
url = _pick_url(primary_url, fallback_url)
|
||||||
deb_filename = f"/root/{deb_base}"
|
deb_filename = "/root/" + url.split("/")[-1]
|
||||||
|
|
||||||
|
match (package, arch):
|
||||||
|
case ("core", "amd64"):
|
||||||
|
sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
|
||||||
|
case ("core", "arm64"):
|
||||||
|
sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
|
||||||
|
case ("imapd", "amd64"):
|
||||||
|
sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
|
||||||
|
case ("imapd", "arm64"):
|
||||||
|
sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
|
||||||
|
case ("lmtpd", "amd64"):
|
||||||
|
sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
|
||||||
|
case ("lmtpd", "arm64"):
|
||||||
|
sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
|
||||||
|
case _:
|
||||||
|
apt.packages(packages=[f"dovecot-{package}"])
|
||||||
|
return
|
||||||
|
|
||||||
files.download(
|
files.download(
|
||||||
name=f"Download {pkg_name}",
|
name=f"Download dovecot-{package}",
|
||||||
src=url,
|
src=url,
|
||||||
dest=deb_filename,
|
dest=deb_filename,
|
||||||
sha256sum=sha256,
|
sha256sum=sha256,
|
||||||
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
|
cache_time=60 * 60 * 24 * 365 * 10, # never redownload the package
|
||||||
)
|
)
|
||||||
|
|
||||||
apt.deb(name=f"Install {pkg_name}", src=deb_filename)
|
apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
|
||||||
|
|
||||||
|
|
||||||
def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
|
def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
|
||||||
@@ -139,25 +134,19 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
|
|||||||
|
|
||||||
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
|
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
|
||||||
# it is recommended to set the following inotify limits
|
# it is recommended to set the following inotify limits
|
||||||
can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
|
if not os.environ.get("CHATMAIL_NOSYSCTL"):
|
||||||
for name in ("max_user_instances", "max_user_watches"):
|
for name in ("max_user_instances", "max_user_watches"):
|
||||||
key = f"fs.inotify.{name}"
|
key = f"fs.inotify.{name}"
|
||||||
value = host.get_fact(Sysctl)[key]
|
if host.get_fact(Sysctl)[key] > 65535:
|
||||||
if value > 65534:
|
# Skip updating limits if already sufficient
|
||||||
continue
|
# (enables running in incus containers where sysctl readonly)
|
||||||
if not can_modify:
|
continue
|
||||||
print(
|
server.sysctl(
|
||||||
"\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
|
name=f"Change {key}",
|
||||||
f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
|
key=key,
|
||||||
"!!!!"
|
value=65535,
|
||||||
|
persist=True,
|
||||||
)
|
)
|
||||||
continue
|
|
||||||
server.sysctl(
|
|
||||||
name=f"Change {key}",
|
|
||||||
key=key,
|
|
||||||
value=65535,
|
|
||||||
persist=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
timezone_env = files.line(
|
timezone_env = files.line(
|
||||||
name="Set TZ environment variable",
|
name="Set TZ environment variable",
|
||||||
|
|||||||
@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
|
|||||||
|
|
||||||
def install(self):
|
def install(self):
|
||||||
arch = host.get_fact(facts.server.Arch)
|
arch = host.get_fact(facts.server.Arch)
|
||||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-{arch}"
|
url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-{arch}"
|
||||||
sha256sum = {
|
sha256sum = {
|
||||||
"x86_64": "3fd8b18282252c75a5bbfa603d8c1b65f6563e5e920bddf3e64e451b7cdb43ce",
|
"x86_64": "ce24ca0075aa445510291d775fb3aea8f4411818c7b885ae51a0fe18c5f789ce",
|
||||||
"aarch64": "2bd191de205f7fd60158dd8e3516ab7e3efb14627696f3d7dc186bdcd9e10a43",
|
"aarch64": "c5d783eefa5332db3d97a0e6a23917d72849e3eb45da3d16ce908a9b4e5a797d",
|
||||||
}[arch]
|
}[arch]
|
||||||
self.need_restart |= files.download(
|
self.need_restart |= files.download(
|
||||||
name="Download filtermail",
|
name="Download filtermail",
|
||||||
|
|||||||
558
cmdeploy/src/cmdeploy/lxc/cli.py
Normal file
558
cmdeploy/src/cmdeploy/lxc/cli.py
Normal file
@@ -0,0 +1,558 @@
|
|||||||
|
"""lxc-start/stop/status/test subcommands for testing with local containers."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from ..util import (
|
||||||
|
collapse,
|
||||||
|
get_git_hash,
|
||||||
|
get_version_string,
|
||||||
|
shell,
|
||||||
|
)
|
||||||
|
from .incus import Incus, RelayContainer
|
||||||
|
|
||||||
|
RELAY_NAMES = ("test0", "test1")
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# lxc-start
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_start_cmd_options(parser):
|
||||||
|
_add_name_args(
|
||||||
|
parser,
|
||||||
|
help_text="User relay name(s) to create (default: test0).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ipv4-only",
|
||||||
|
dest="ipv4_only",
|
||||||
|
action="store_true",
|
||||||
|
help="Create an IPv4-only container.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--run",
|
||||||
|
action="store_true",
|
||||||
|
help="Run 'cmdeploy run' on each container after starting it.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_start_cmd(args, out):
|
||||||
|
"""Create/Ensure and start LXC relay and DNS containers."""
|
||||||
|
ix = Incus()
|
||||||
|
out.green("Ensuring DNS container (ns-localchat) ...")
|
||||||
|
dns_ct = ix.get_dns_container()
|
||||||
|
dns_ct.ensure()
|
||||||
|
if not ix.find_dns_image():
|
||||||
|
with out.section("LXC: publishing DNS image"):
|
||||||
|
dns_ct.publish_as_dns_image()
|
||||||
|
out.print(f" DNS container IP: {dns_ct.ipv4}")
|
||||||
|
|
||||||
|
names = args.names if args.names else RELAY_NAMES
|
||||||
|
relays = list(ix.get_container(n) for n in names)
|
||||||
|
for ct in relays:
|
||||||
|
out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
|
||||||
|
ct.ensure()
|
||||||
|
ip = ct.ipv4
|
||||||
|
|
||||||
|
out.print(" Configuring container hostname ...")
|
||||||
|
ct.configure_hosts(ip)
|
||||||
|
|
||||||
|
out.print(f" Writing {ct.ini.name} ...")
|
||||||
|
ct.write_ini(disable_ipv6=args.ipv4_only)
|
||||||
|
out.print(f" Config: {ct.ini}")
|
||||||
|
if args.ipv4_only:
|
||||||
|
ct.disable_ipv6()
|
||||||
|
ipv6 = None
|
||||||
|
else:
|
||||||
|
output = ct.bash(
|
||||||
|
"ip -6 addr show scope global -deprecated"
|
||||||
|
" | grep -oP '(?<=inet6 )[^/]+'",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
ipv6 = output.strip() if output else None
|
||||||
|
out.print(f" {_format_addrs(ip, ipv6)}")
|
||||||
|
|
||||||
|
out.green(f" Container {ct.name!r} ready: {ct.domain} -> {ip}")
|
||||||
|
out.print()
|
||||||
|
|
||||||
|
# Reset DNS zones only for the containers we just started
|
||||||
|
started_cnames = {ct.name for ct in relays}
|
||||||
|
managed = ix.list_managed()
|
||||||
|
started = [c for c in managed if c["name"] in started_cnames]
|
||||||
|
|
||||||
|
if started:
|
||||||
|
out.print(
|
||||||
|
f"Resetting DNS zones for {len(started)}"
|
||||||
|
" domain(s) (A + AAAA records) ..."
|
||||||
|
)
|
||||||
|
dns_ct.reset_dns_records(dns_ct.ipv4, started)
|
||||||
|
|
||||||
|
for ct in relays:
|
||||||
|
if ct.name in started_cnames:
|
||||||
|
out.print(f" Configuring and testing DNS in {ct.name} ...")
|
||||||
|
ct.configure_dns(dns_ct.ipv4)
|
||||||
|
if not ct.check_dns():
|
||||||
|
out.red(
|
||||||
|
f" DNS check failed for {ct.name}"
|
||||||
|
": cannot resolve external hosts"
|
||||||
|
)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
# Generate the unified SSH config
|
||||||
|
out.green("Writing ssh-config ...")
|
||||||
|
ssh_cfg = ix.write_ssh_config()
|
||||||
|
out.print(f" {ssh_cfg}")
|
||||||
|
|
||||||
|
# Verify SSH via the generated config
|
||||||
|
for ct in relays:
|
||||||
|
out.print(f" Verifying SSH to {ct.name} via ssh-config ...")
|
||||||
|
if ct.verify_ssh(ssh_cfg):
|
||||||
|
out.print(f" SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
|
||||||
|
else:
|
||||||
|
out.red(f" WARNING: SSH verification failed for {ct.name}")
|
||||||
|
|
||||||
|
# Print integration suggestions
|
||||||
|
ssh_cfg = ix.ssh_config_path
|
||||||
|
if not ix.check_ssh_include():
|
||||||
|
out.green(
|
||||||
|
"\n (Optional) To use containers from any SSH client, add to ~/.ssh/config:"
|
||||||
|
)
|
||||||
|
out.green(f" Include {ssh_cfg}")
|
||||||
|
|
||||||
|
# Optionally run cmdeploy run on each relay
|
||||||
|
if args.run:
|
||||||
|
for ct in relays:
|
||||||
|
with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
|
||||||
|
ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
|
||||||
|
if ret:
|
||||||
|
out.red(f"Deploy to {ct.sname} failed (exit {ret})")
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# lxc-stop
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_stop_cmd_options(parser):
|
||||||
|
parser.add_argument(
|
||||||
|
"--destroy",
|
||||||
|
action="store_true",
|
||||||
|
help="Delete containers and their config files after stopping.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--destroy-all",
|
||||||
|
dest="destroy_all",
|
||||||
|
action="store_true",
|
||||||
|
help="Like --destroy, but also remove the ns-localchat DNS container.",
|
||||||
|
)
|
||||||
|
_add_name_args(
|
||||||
|
parser,
|
||||||
|
help_text="Container name(s) to stop (default: test0 + test1).",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_stop_cmd(args, out):
|
||||||
|
"""Stop (and optionally destroy) local LXC relay containers."""
|
||||||
|
ix = Incus()
|
||||||
|
names = args.names or RELAY_NAMES
|
||||||
|
destroy = args.destroy or args.destroy_all
|
||||||
|
|
||||||
|
for ct in map(ix.get_container, names):
|
||||||
|
if destroy:
|
||||||
|
out.green(f"Destroying container {ct.name!r} ...")
|
||||||
|
ct.destroy()
|
||||||
|
if hasattr(ct, "image_alias"):
|
||||||
|
out.green(f" Deleting cached image {ct.image_alias!r} ...")
|
||||||
|
ix.run(["image", "delete", ct.image_alias], check=False)
|
||||||
|
else:
|
||||||
|
out.green(f"Stopping container {ct.name!r} ...")
|
||||||
|
ct.stop(force=True)
|
||||||
|
|
||||||
|
if args.destroy_all:
|
||||||
|
dns_ct = ix.get_dns_container()
|
||||||
|
out.green(f"Destroying DNS container {dns_ct.name!r} ...")
|
||||||
|
dns_ct.destroy()
|
||||||
|
ix.delete_images()
|
||||||
|
|
||||||
|
if destroy:
|
||||||
|
ix.write_ssh_config()
|
||||||
|
out.green("LXC containers destroyed.")
|
||||||
|
else:
|
||||||
|
out.green("LXC containers stopped.")
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# lxc-test
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_test_cmd_options(parser):
|
||||||
|
parser.add_argument(
|
||||||
|
"--one",
|
||||||
|
action="store_true",
|
||||||
|
help="Only deploy and test against test0 (skip test1).",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_test_cmd(args, out):
|
||||||
|
"""Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
|
||||||
|
|
||||||
|
All commands run directly on the host using
|
||||||
|
``--ssh-config lxconfigs/ssh-config`` for SSH access.
|
||||||
|
"""
|
||||||
|
ix = Incus()
|
||||||
|
t_total = time.time()
|
||||||
|
relay_names = list(RELAY_NAMES)
|
||||||
|
if args.one:
|
||||||
|
relay_names = relay_names[:1]
|
||||||
|
|
||||||
|
local_hash = get_git_hash()
|
||||||
|
|
||||||
|
# Per-relay: start containers, then deploy in parallel.
|
||||||
|
ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
|
||||||
|
|
||||||
|
# Phase 1: start all containers (sequential, fast)
|
||||||
|
for ct in map(ix.get_container, relay_names):
|
||||||
|
name = ct.sname
|
||||||
|
ipv4_only = ipv4_only_flags.get(name, False)
|
||||||
|
label = "IPv4-only" if ipv4_only else "dual-stack"
|
||||||
|
|
||||||
|
with out.section(f"LXC: lxc-start {name} ({label})"):
|
||||||
|
args.names = [name]
|
||||||
|
args.ipv4_only = ipv4_only
|
||||||
|
args.run = False
|
||||||
|
ret = lxc_start_cmd(args, out)
|
||||||
|
if ret:
|
||||||
|
return ret
|
||||||
|
|
||||||
|
# Phase 2: deploy all relays in parallel
|
||||||
|
to_deploy = []
|
||||||
|
for ct in map(ix.get_container, relay_names):
|
||||||
|
status = _deploy_status(ct, local_hash, ix)
|
||||||
|
if "IN-SYNC" in status:
|
||||||
|
out.section_line(f"cmdeploy run: {ct.sname}: {status}, skipping")
|
||||||
|
else:
|
||||||
|
to_deploy.append(ct)
|
||||||
|
|
||||||
|
if to_deploy:
|
||||||
|
with out.section("cmdeploy run (parallel)"):
|
||||||
|
ret = _run_cmdeploy_parallel(
|
||||||
|
"run", to_deploy, ix, out, extra=["--skip-dns-check"]
|
||||||
|
)
|
||||||
|
if ret:
|
||||||
|
return ret
|
||||||
|
|
||||||
|
# Phase 3: publish images (sequential, fast)
|
||||||
|
for ct in map(ix.get_container, relay_names):
|
||||||
|
if ct.publish_image():
|
||||||
|
out.section_line(f"LXC: published {ct.sname} image")
|
||||||
|
else:
|
||||||
|
out.section_line(
|
||||||
|
f"LXC: publish {ct.sname} image: skipped, cached",
|
||||||
|
)
|
||||||
|
|
||||||
|
for ct in map(ix.get_container, relay_names):
|
||||||
|
with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
|
||||||
|
ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
|
||||||
|
if ret:
|
||||||
|
out.red(f"DNS for {ct.sname} failed (exit {ret})")
|
||||||
|
return ret
|
||||||
|
|
||||||
|
with out.section("LXC: PowerDNS zone update"):
|
||||||
|
dns_ct = ix.get_dns_container()
|
||||||
|
for ct in map(ix.get_container, relay_names):
|
||||||
|
if ct.zone.exists():
|
||||||
|
zone_data = ct.zone.read_text()
|
||||||
|
out.print(f" Loading {ct.zone} into PowerDNS ...")
|
||||||
|
dns_ct.set_dns_records(zone_data)
|
||||||
|
|
||||||
|
# Run tests in both directions when two relays are available.
|
||||||
|
test_pairs = [(0, 1), (1, 0)] if len(relay_names) > 1 else [(0,)]
|
||||||
|
for pair in test_pairs:
|
||||||
|
first = ix.get_container(relay_names[pair[0]])
|
||||||
|
label = first.sname
|
||||||
|
env = None
|
||||||
|
if len(pair) > 1:
|
||||||
|
second = ix.get_container(relay_names[pair[1]])
|
||||||
|
label = f"{first.sname} \u2194 {second.sname}"
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["CHATMAIL_DOMAIN2"] = second.domain
|
||||||
|
|
||||||
|
with out.section(f"cmdeploy test: {label}"):
|
||||||
|
ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
|
||||||
|
if ret:
|
||||||
|
out.red(f"Tests failed (exit {ret})")
|
||||||
|
return ret
|
||||||
|
|
||||||
|
elapsed = time.time() - t_total
|
||||||
|
out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
|
||||||
|
if out.section_timings:
|
||||||
|
out.print("Section timings:")
|
||||||
|
for name, secs in out.section_timings:
|
||||||
|
out.print(f" {name:.<50s} {secs:5.1f}s")
|
||||||
|
out.print(f" {'total':.<50s} {elapsed:5.1f}s")
|
||||||
|
out.section_timings.clear()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# lxc-status
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_status_cmd_options(parser):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def lxc_status_cmd(args, out):
|
||||||
|
"""Show status of local LXC chatmail containers."""
|
||||||
|
ix = Incus()
|
||||||
|
containers = ix.list_managed()
|
||||||
|
if not containers:
|
||||||
|
out.red("No LXC containers found. Run 'cmdeploy lxc-start' first.")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
local_hash = get_git_hash()
|
||||||
|
|
||||||
|
# Get storage pool path for display
|
||||||
|
storage_path = None
|
||||||
|
data = ix.run_json(["storage", "show", "default"], check=False)
|
||||||
|
if data:
|
||||||
|
storage_path = data.get("config", {}).get("source")
|
||||||
|
if storage_path:
|
||||||
|
out.green(f"Containers: ({storage_path})")
|
||||||
|
else:
|
||||||
|
out.green("Containers:")
|
||||||
|
|
||||||
|
dns_ip = None
|
||||||
|
for c in containers:
|
||||||
|
_print_container_status(out, c, ix, local_hash)
|
||||||
|
if c["name"] == ix.get_dns_container().name:
|
||||||
|
dns_ip = c["ip"]
|
||||||
|
|
||||||
|
_print_ssh_status(out, ix)
|
||||||
|
_print_dns_forwarding_status(out, dns_ip)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _print_container_status(out, c, ix, local_hash):
|
||||||
|
"""Print name/status, domain/IPs, and RAM for one container."""
|
||||||
|
cname = c["name"]
|
||||||
|
is_running = c.get("status") == "Running"
|
||||||
|
ct = ix.get_container(cname)
|
||||||
|
|
||||||
|
# First line: name + running/STOPPED + deploy status
|
||||||
|
if not is_running:
|
||||||
|
tag = "STOPPED"
|
||||||
|
elif not isinstance(ct, RelayContainer):
|
||||||
|
tag = "running"
|
||||||
|
else:
|
||||||
|
tag = f"running {_deploy_status(ct, local_hash, ix)}"
|
||||||
|
out.print(f" {cname:20s} {tag}")
|
||||||
|
|
||||||
|
# Second line: domain, IPv4, IPv6
|
||||||
|
domain = c.get("domain", "")
|
||||||
|
ip = c.get("ip") or "?"
|
||||||
|
ipv6 = c.get("ipv6")
|
||||||
|
out.print(f" {domain:20s} {_format_addrs(ip, ipv6)}")
|
||||||
|
|
||||||
|
# Third line: RAM (RSS), config
|
||||||
|
indent = " " * 21
|
||||||
|
try:
|
||||||
|
used, total = ct.rss_mib()
|
||||||
|
except Exception:
|
||||||
|
ram_str = "RSS ?"
|
||||||
|
else:
|
||||||
|
ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
|
||||||
|
|
||||||
|
if isinstance(ct, RelayContainer):
|
||||||
|
detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
|
||||||
|
else:
|
||||||
|
detail = ram_str
|
||||||
|
|
||||||
|
out.print(f" {indent}{detail}")
|
||||||
|
out.print()
|
||||||
|
|
||||||
|
|
||||||
|
def _print_ssh_status(out, ix):
|
||||||
|
"""Print SSH integration status."""
|
||||||
|
out.print()
|
||||||
|
ssh_cfg = ix.ssh_config_path
|
||||||
|
if ix.check_ssh_include():
|
||||||
|
out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
|
||||||
|
else:
|
||||||
|
out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
|
||||||
|
out.print(" Add to ~/.ssh/config:")
|
||||||
|
out.print(f" Include {ssh_cfg}")
|
||||||
|
|
||||||
|
|
||||||
|
def _print_dns_forwarding_status(out, dns_ip):
|
||||||
|
"""Print host DNS forwarding status for .localchat."""
|
||||||
|
if not dns_ip:
|
||||||
|
out.red("DNS: ns-localchat container not found")
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
rv = shell("resolvectl status incusbr0", timeout=5)
|
||||||
|
dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
|
||||||
|
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
|
||||||
|
dns_ok = None
|
||||||
|
if dns_ok is True:
|
||||||
|
out.green(f"DNS: .localchat forwarding to {dns_ip} ✓")
|
||||||
|
elif dns_ok is False:
|
||||||
|
out.red("DNS: .localchat forwarding NOT configured")
|
||||||
|
out.print(" Run:")
|
||||||
|
out.print(f" sudo resolvectl dns incusbr0 {dns_ip}")
|
||||||
|
out.print(" sudo resolvectl domain incusbr0 ~localchat")
|
||||||
|
else:
|
||||||
|
out.print(" DNS: .localchat forwarding status UNKNOWN")
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
# Internal helpers
|
||||||
|
# -------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _format_addrs(ip, ipv6=None):
|
||||||
|
parts = [f"IPv4 {ip}"]
|
||||||
|
if ipv6:
|
||||||
|
parts.append(f"IPv6 {ipv6}")
|
||||||
|
return ", ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _deploy_status(ct, local_hash, ix):
|
||||||
|
"""Return a human-readable deploy status string.
|
||||||
|
|
||||||
|
Compares the full deployed version (hash + diff) against
|
||||||
|
the local state built by :func:`~cmdeploy.util.get_version_string`.
|
||||||
|
"""
|
||||||
|
deployed = ct.deployed_version()
|
||||||
|
if deployed is None:
|
||||||
|
return "NOT DEPLOYED"
|
||||||
|
|
||||||
|
# A container launched from the relay image has the same
|
||||||
|
# git hash but a different domain - always redeploy.
|
||||||
|
deployed_domain = ct.deployed_domain()
|
||||||
|
if deployed_domain and deployed_domain != ct.domain:
|
||||||
|
return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
|
||||||
|
|
||||||
|
deployed_lines = deployed.splitlines()
|
||||||
|
deployed_hash = deployed_lines[0] if deployed_lines else ""
|
||||||
|
short = deployed_hash[:12]
|
||||||
|
|
||||||
|
if not local_hash:
|
||||||
|
return f"UNKNOWN (deployed: {short})"
|
||||||
|
|
||||||
|
local_short = local_hash[:12]
|
||||||
|
if deployed_hash != local_hash:
|
||||||
|
return f"STALE (deployed: {short}, local: {local_short})"
|
||||||
|
|
||||||
|
# Hash matches - check for uncommitted diffs
|
||||||
|
local_version = get_version_string()
|
||||||
|
if deployed != local_version:
|
||||||
|
return f"DIRTY ({local_short}, undeployed changes)"
|
||||||
|
|
||||||
|
return f"IN-SYNC ({short})"
|
||||||
|
|
||||||
|
|
||||||
|
def _add_name_args(parser, help_text=None):
|
||||||
|
"""Add optional positional NAME arguments."""
|
||||||
|
parser.add_argument(
|
||||||
|
"names",
|
||||||
|
nargs="*",
|
||||||
|
metavar="NAME",
|
||||||
|
help=help_text or "Relay name(s) to operate on.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cmdeploy_cmd(subcmd, ct, ix, extra=None):
|
||||||
|
"""Build the ``cmdeploy <subcmd>`` command string."""
|
||||||
|
extra_str = " ".join(extra) if extra else ""
|
||||||
|
return collapse(f"""\
|
||||||
|
cmdeploy {subcmd}
|
||||||
|
--config {ct.ini}
|
||||||
|
--ssh-config {ix.ssh_config_path}
|
||||||
|
--ssh-host {ct.domain}
|
||||||
|
{extra_str}
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
|
||||||
|
"""Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
|
||||||
|
|
||||||
|
*ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
|
||||||
|
Returns the subprocess exit code.
|
||||||
|
"""
|
||||||
|
cmd = _build_cmdeploy_cmd(subcmd, ct, ix, extra=extra)
|
||||||
|
if "cwd" not in kwargs:
|
||||||
|
kwargs["cwd"] = str(ix.project_root)
|
||||||
|
out.print(f" [$ {cmd}]")
|
||||||
|
return shell(cmd, capture_output=False, **kwargs).returncode
|
||||||
|
|
||||||
|
|
||||||
|
# Number of tail lines to print on failure.
|
||||||
|
_FAIL_CONTEXT_LINES = 40
|
||||||
|
|
||||||
|
|
||||||
|
def _run_cmdeploy_parallel(subcmd, containers, ix, out, extra=None):
|
||||||
|
"""Run ``cmdeploy <subcmd>`` for every container in parallel.
|
||||||
|
|
||||||
|
Output is captured and filtered: only lines containing
|
||||||
|
``"Start operation"`` are printed (prefixed with the relay
|
||||||
|
short-name). On failure the last *_FAIL_CONTEXT_LINES*
|
||||||
|
lines of that process's output are shown.
|
||||||
|
"""
|
||||||
|
procs = [] # list of (container, Popen, collected_lines)
|
||||||
|
cwd = str(ix.project_root)
|
||||||
|
|
||||||
|
for ct in containers:
|
||||||
|
cmd = _build_cmdeploy_cmd(subcmd, ct, ix, extra=extra)
|
||||||
|
out.print(f" [{ct.sname}] $ {cmd}")
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
shell=True,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
cwd=cwd,
|
||||||
|
)
|
||||||
|
procs.append((ct, proc, []))
|
||||||
|
|
||||||
|
def _reader(ct, proc, lines):
|
||||||
|
prefix = f" [{ct.sname}]"
|
||||||
|
for raw in proc.stdout:
|
||||||
|
line = raw.rstrip("\n")
|
||||||
|
lines.append(line)
|
||||||
|
if "Starting operation" in line:
|
||||||
|
out.print(f"{prefix} {line}")
|
||||||
|
|
||||||
|
threads = []
|
||||||
|
for ct, proc, lines in procs:
|
||||||
|
t = threading.Thread(
|
||||||
|
target=_reader,
|
||||||
|
args=(ct, proc, lines),
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
t.start()
|
||||||
|
threads.append(t)
|
||||||
|
|
||||||
|
for t in threads:
|
||||||
|
t.join()
|
||||||
|
for _, proc, _ in procs:
|
||||||
|
proc.wait()
|
||||||
|
|
||||||
|
# Check results
|
||||||
|
first_failure = 0
|
||||||
|
for ct, proc, lines in procs:
|
||||||
|
if proc.returncode:
|
||||||
|
out.red(f"Deploy to {ct.sname} failed " f"(exit {proc.returncode})")
|
||||||
|
tail = lines[-_FAIL_CONTEXT_LINES:]
|
||||||
|
for tl in tail:
|
||||||
|
out.print(f" [{ct.sname}] {tl}")
|
||||||
|
if not first_failure:
|
||||||
|
first_failure = proc.returncode
|
||||||
|
|
||||||
|
return first_failure
|
||||||
754
cmdeploy/src/cmdeploy/lxc/incus.py
Normal file
754
cmdeploy/src/cmdeploy/lxc/incus.py
Normal file
@@ -0,0 +1,754 @@
|
|||||||
|
"""Core Incus operations for local chatmail LXC containers."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import textwrap
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..util import shell
|
||||||
|
|
||||||
|
LABEL_KEY = "user.localchat-managed"
|
||||||
|
SSH_KEY_NAME = "id_localchat"
|
||||||
|
DOMAIN_SUFFIX = ".localchat"
|
||||||
|
UPSTREAM_IMAGE = "images:debian/12"
|
||||||
|
BASE_IMAGE_ALIAS = "localchat-base"
|
||||||
|
BASE_SETUP_NAME = "localchat-base-setup"
|
||||||
|
DNS_IMAGE_ALIAS = "localchat-ns"
|
||||||
|
|
||||||
|
DNS_CONTAINER_NAME = "ns-localchat"
|
||||||
|
DNS_DOMAIN = "ns.localchat"
|
||||||
|
|
||||||
|
BRIDGE_IPV4 = "10.200.200.1/24"
|
||||||
|
DNS_IP = "10.200.200.2"
|
||||||
|
RELAY_IPS = {
|
||||||
|
"test0": "10.200.200.10",
|
||||||
|
"test1": "10.200.200.11",
|
||||||
|
"test2": "10.200.200.12",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_ip(net_data, family="inet"):
|
||||||
|
"""Extract the first global-scope IP of *family* from network state data.
|
||||||
|
|
||||||
|
*net_data* is the ``state.network`` dict from ``incus list --format=json``.
|
||||||
|
*family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
|
||||||
|
Returns the address string, or None.
|
||||||
|
"""
|
||||||
|
for iface_name, iface in net_data.items():
|
||||||
|
if iface_name == "lo":
|
||||||
|
continue
|
||||||
|
for addr in iface.get("addresses", []):
|
||||||
|
if addr["family"] == family and addr["scope"] == "global":
|
||||||
|
return addr["address"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class Incus:
|
||||||
|
"""Gateway for all Incus container operations.
|
||||||
|
|
||||||
|
Instantiated once per CLI command and passed around so that
|
||||||
|
all modules share a single entry point for Incus interactions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
|
||||||
|
self.lxconfigs_dir = self.project_root / "lxconfigs"
|
||||||
|
self.lxconfigs_dir.mkdir(exist_ok=True)
|
||||||
|
self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
|
||||||
|
if not self.ssh_key_path.exists():
|
||||||
|
shell(
|
||||||
|
f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
|
||||||
|
|
||||||
|
def write_ssh_config(self):
|
||||||
|
"""Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
|
||||||
|
|
||||||
|
Each Host block maps the container name, the domain name, and the
|
||||||
|
short relay name (e.g. ``_test0``) to the container's IP, using the
|
||||||
|
shared localchat SSH key. Returns the path to the file.
|
||||||
|
"""
|
||||||
|
containers = self.list_managed()
|
||||||
|
key_path = self.ssh_key_path
|
||||||
|
lines = ["# Auto-generated by cmdeploy lxc-start - do not edit\n"]
|
||||||
|
for c in containers:
|
||||||
|
hosts = [c["name"]]
|
||||||
|
domain = c.get("domain", "")
|
||||||
|
if domain and domain != c["name"]:
|
||||||
|
hosts.append(domain)
|
||||||
|
short = domain.split(".")[0]
|
||||||
|
if short and short not in hosts:
|
||||||
|
hosts.append(short)
|
||||||
|
lines.append(f"\nHost {' '.join(hosts)}\n")
|
||||||
|
lines.append(f" Hostname {c['ip']}\n")
|
||||||
|
lines.append(" User root\n")
|
||||||
|
lines.append(f" IdentityFile {key_path}\n")
|
||||||
|
lines.append(" IdentitiesOnly yes\n")
|
||||||
|
lines.append(" StrictHostKeyChecking accept-new\n")
|
||||||
|
lines.append(" UserKnownHostsFile /dev/null\n")
|
||||||
|
lines.append(" LogLevel ERROR\n")
|
||||||
|
path = self.ssh_config_path
|
||||||
|
path.write_text("".join(lines))
|
||||||
|
return path
|
||||||
|
|
||||||
|
def check_ssh_include(self):
|
||||||
|
"""Check if the user's ~/.ssh/config already includes our ssh-config."""
|
||||||
|
user_ssh_config = Path.home() / ".ssh" / "config"
|
||||||
|
if not user_ssh_config.exists():
|
||||||
|
return False
|
||||||
|
lines = filter(None, map(str.strip, user_ssh_config.open("r")))
|
||||||
|
return f"Include {self.ssh_config_path}" in lines
|
||||||
|
|
||||||
|
def run(self, args, check=True, capture=True, input=None):
|
||||||
|
"""Run an incus command."""
|
||||||
|
cmd = ["incus"] + list(args)
|
||||||
|
kwargs = dict(check=check, text=True, input=input)
|
||||||
|
if capture:
|
||||||
|
kwargs["capture_output"] = True
|
||||||
|
else:
|
||||||
|
kwargs["stdout"] = None
|
||||||
|
kwargs["stderr"] = None
|
||||||
|
return subprocess.run(cmd, **kwargs) # noqa: PLW1510
|
||||||
|
|
||||||
|
def run_json(self, args, check=True):
|
||||||
|
"""Run an incus command with ``--format=json``.
|
||||||
|
|
||||||
|
Returns the parsed JSON on success.
|
||||||
|
When *check* is True raises ``subprocess.CalledProcessError``
|
||||||
|
on non-zero exit; when False returns *None* instead.
|
||||||
|
"""
|
||||||
|
result = self.run(
|
||||||
|
list(args) + ["--format=json"],
|
||||||
|
check=check,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return None
|
||||||
|
return json.loads(result.stdout)
|
||||||
|
|
||||||
|
def run_output(self, args, check=True):
|
||||||
|
"""Run an incus command and return its stripped stdout.
|
||||||
|
|
||||||
|
When *check* is False, returns *None* on non-zero exit
|
||||||
|
instead of raising.
|
||||||
|
"""
|
||||||
|
result = self.run(args, check=check)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return None
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
def _find_image(self, alias):
|
||||||
|
"""Return *alias* if an image with that alias exists, else None."""
|
||||||
|
images = self.run_json(["image", "list"], check=False) or []
|
||||||
|
for img in images:
|
||||||
|
for a in img.get("aliases", []):
|
||||||
|
if a.get("name") == alias:
|
||||||
|
return alias
|
||||||
|
return None
|
||||||
|
|
||||||
|
def find_dns_image(self):
|
||||||
|
"""Return the DNS image alias if it exists, else None."""
|
||||||
|
return self._find_image(DNS_IMAGE_ALIAS)
|
||||||
|
|
||||||
|
def delete_images(self):
|
||||||
|
"""Delete all cached localchat images."""
|
||||||
|
for alias in (DNS_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
|
||||||
|
self.run(["image", "delete", alias], check=False)
|
||||||
|
for name in RELAY_IPS:
|
||||||
|
self.run(["image", "delete", f"localchat-{name}"], check=False)
|
||||||
|
|
||||||
|
def list_managed(self):
|
||||||
|
"""Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
|
||||||
|
containers = []
|
||||||
|
for ct in self.run_json(["list"]):
|
||||||
|
config = ct.get("config", {})
|
||||||
|
if config.get(LABEL_KEY) != "true":
|
||||||
|
continue
|
||||||
|
name = ct["name"]
|
||||||
|
state = ct.get("state", {})
|
||||||
|
net = state.get("network") or {}
|
||||||
|
containers.append(
|
||||||
|
{
|
||||||
|
"name": name,
|
||||||
|
"ip": _extract_ip(net, "inet"),
|
||||||
|
"ipv6": _extract_ip(net, "inet6"),
|
||||||
|
"domain": config.get(
|
||||||
|
"user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
|
||||||
|
),
|
||||||
|
"status": ct.get("status", "Unknown"),
|
||||||
|
"memory_usage": state.get("memory", {}).get("usage", 0),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return containers
|
||||||
|
|
||||||
|
def ensure_base_image(self):
|
||||||
|
"""Build and cache a base image with openssh and the SSH key.
|
||||||
|
|
||||||
|
The image is published as a local incus image with alias
|
||||||
|
'localchat-base'. Subsequent container launches use this
|
||||||
|
image instead of the upstream Debian 12, skipping the
|
||||||
|
slow apt-get install step.
|
||||||
|
Returns the image alias.
|
||||||
|
"""
|
||||||
|
if self._find_image(BASE_IMAGE_ALIAS):
|
||||||
|
return BASE_IMAGE_ALIAS
|
||||||
|
|
||||||
|
print(" Building base image (one-time setup) ...")
|
||||||
|
|
||||||
|
self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
|
||||||
|
self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
|
||||||
|
self.run(
|
||||||
|
["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME, "-c", "limits.memory=512MiB"]
|
||||||
|
)
|
||||||
|
|
||||||
|
ct = Container(self, BASE_SETUP_NAME, memory="512MiB")
|
||||||
|
ct.wait_ready()
|
||||||
|
|
||||||
|
key_path = self.ssh_key_path
|
||||||
|
pub_key = key_path.with_suffix(".pub").read_text().strip()
|
||||||
|
print(" ── apt-get install (base image) ──")
|
||||||
|
ct.bash(
|
||||||
|
f"""\
|
||||||
|
systemctl disable --now systemd-resolved 2>/dev/null || true
|
||||||
|
rm -f /etc/resolv.conf
|
||||||
|
echo 'nameserver 9.9.9.9' > /etc/resolv.conf
|
||||||
|
while fuser /var/lib/apt/lists/lock >/dev/null 2>&1 ; do
|
||||||
|
echo "Waiting for other apt-get instance to finish..."
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
apt-get -o DPkg::Lock::Timeout=60 update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
|
||||||
|
systemctl enable ssh
|
||||||
|
apt-get clean
|
||||||
|
mkdir -p /root/.ssh
|
||||||
|
chmod 700 /root/.ssh
|
||||||
|
echo '{pub_key}' > /root/.ssh/authorized_keys
|
||||||
|
chmod 600 /root/.ssh/authorized_keys
|
||||||
|
""",
|
||||||
|
capture=False,
|
||||||
|
)
|
||||||
|
print(" ── base image install done ──")
|
||||||
|
|
||||||
|
self.run(["stop", BASE_SETUP_NAME])
|
||||||
|
self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
|
||||||
|
self.run(["delete", BASE_SETUP_NAME, "--force"])
|
||||||
|
print(f" Base image '{BASE_IMAGE_ALIAS}' ready.")
|
||||||
|
return BASE_IMAGE_ALIAS
|
||||||
|
|
||||||
|
def ensure_bridge(self):
|
||||||
|
"""Ensure incusbr0 exists and uses our fixed IPv4 subnet."""
|
||||||
|
bridge = self.run_json(["network", "show", "incusbr0"], check=False)
|
||||||
|
if bridge and bridge.get("config", {}).get("ipv4.address") == BRIDGE_IPV4:
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f" Configuring incusbr0 with static subnet {BRIDGE_IPV4} ...")
|
||||||
|
if not bridge:
|
||||||
|
self.run(["network", "create", "incusbr0"], check=False)
|
||||||
|
|
||||||
|
self.run(
|
||||||
|
[
|
||||||
|
"network",
|
||||||
|
"set",
|
||||||
|
"incusbr0",
|
||||||
|
f"ipv4.address={BRIDGE_IPV4}",
|
||||||
|
"ipv4.nat=true",
|
||||||
|
"ipv6.address=none",
|
||||||
|
"dns.mode=none",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_container(self, name):
|
||||||
|
"""Return a container handle for the given name.
|
||||||
|
|
||||||
|
Accepts both short relay names (``test0``) and full Incus
|
||||||
|
container names (``test0-localchat``). Returns
|
||||||
|
``DNSContainer`` for the DNS container and
|
||||||
|
``RelayContainer`` for everything else.
|
||||||
|
"""
|
||||||
|
if name == DNS_CONTAINER_NAME:
|
||||||
|
return DNSContainer(self)
|
||||||
|
return RelayContainer(self, name.removesuffix("-localchat"))
|
||||||
|
|
||||||
|
def get_dns_container(self):
|
||||||
|
"""Return a DNSContainer handle."""
|
||||||
|
return DNSContainer(self)
|
||||||
|
|
||||||
|
|
||||||
|
class Container:
|
||||||
|
"""Lightweight handle for an Incus container.
|
||||||
|
|
||||||
|
Carries the container *name* and provides convenience methods
|
||||||
|
for running commands, managing lifecycle, and extracting state
|
||||||
|
so callers don't repeat the name everywhere.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, incus, name, domain=None, memory="200MiB", ipv4=None):
|
||||||
|
self.incus = incus
|
||||||
|
self.name = name
|
||||||
|
self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
|
||||||
|
self.memory = memory
|
||||||
|
self.ipv4 = ipv4
|
||||||
|
self.ipv6 = None
|
||||||
|
|
||||||
|
def bash(self, script, check=True, capture=True):
|
||||||
|
"""Returns stdout from executing ``bash -ec <script>`` inside this container.
|
||||||
|
|
||||||
|
*script* is dedented and stripped so callers can use triple-quoted strings.
|
||||||
|
When *check* is False, returns *None* on non-zero exit instead of raising.
|
||||||
|
When *capture* is False, output streams to the terminal and None is returned.
|
||||||
|
"""
|
||||||
|
cmd = ["exec", self.name, "--", "bash", "-ec", textwrap.dedent(script).strip()]
|
||||||
|
if not capture:
|
||||||
|
self.incus.run(cmd, check=check, capture=False)
|
||||||
|
return None
|
||||||
|
return self.incus.run_output(cmd, check=check)
|
||||||
|
|
||||||
|
def run_cmd(self, *args, check=True):
|
||||||
|
"""Return stdout from running a command directly in the container (no shell).
|
||||||
|
|
||||||
|
When *check* is False, returns *None* on non-zero exit instead of raising.
|
||||||
|
"""
|
||||||
|
return self.incus.run_output(
|
||||||
|
["exec", self.name, "--", *args],
|
||||||
|
check=check,
|
||||||
|
)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self.incus.run(["start", self.name])
|
||||||
|
|
||||||
|
def stop(self, force=False):
|
||||||
|
cmd = ["stop", self.name]
|
||||||
|
if force:
|
||||||
|
cmd.append("--force")
|
||||||
|
self.incus.run(cmd, check=False)
|
||||||
|
|
||||||
|
def launch(self, image=None):
|
||||||
|
"""Launch from the specified image, or the base image if None."""
|
||||||
|
self.incus.ensure_bridge()
|
||||||
|
if image is None:
|
||||||
|
image = self.incus.ensure_base_image()
|
||||||
|
cfg = []
|
||||||
|
cfg += ("-c", f"{LABEL_KEY}=true")
|
||||||
|
cfg += ("-c", f"user.localchat-domain={self.domain}")
|
||||||
|
cfg += ("-c", f"limits.memory={self.memory}")
|
||||||
|
self.incus.run(["init", image, self.name, *cfg])
|
||||||
|
if self.ipv4:
|
||||||
|
self.incus.run(
|
||||||
|
[
|
||||||
|
"config",
|
||||||
|
"device",
|
||||||
|
"override",
|
||||||
|
self.name,
|
||||||
|
"eth0",
|
||||||
|
f"ipv4.address={self.ipv4}",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.incus.run(["start", self.name])
|
||||||
|
return image
|
||||||
|
|
||||||
|
def ensure(self):
|
||||||
|
"""Create/start this container from the cached base image.
|
||||||
|
|
||||||
|
On first call, builds the base image (~30s).
|
||||||
|
Subsequent containers launch in ~2s from the cached image.
|
||||||
|
Returns ``self`` for chaining.
|
||||||
|
"""
|
||||||
|
data = self.incus.run_json(["list", self.name], check=False) or []
|
||||||
|
|
||||||
|
existing = [c for c in data if c["name"] == self.name]
|
||||||
|
image = None
|
||||||
|
if existing:
|
||||||
|
status = existing[0]["status"]
|
||||||
|
if status != "Running":
|
||||||
|
print(f" Starting stopped {self.name} container ...")
|
||||||
|
self.start()
|
||||||
|
else:
|
||||||
|
print(f" {self.name} already running")
|
||||||
|
else:
|
||||||
|
image = self.launch()
|
||||||
|
self.wait_ready()
|
||||||
|
if image:
|
||||||
|
print(f" Ensured {self.name} (launched from {image!r} image)")
|
||||||
|
return self
|
||||||
|
|
||||||
|
def destroy(self):
|
||||||
|
"""Stop, delete, and clean up config files."""
|
||||||
|
self.stop(force=True)
|
||||||
|
self.incus.run(["delete", self.name, "--force"], check=False)
|
||||||
|
|
||||||
|
def push_file_content(self, dest_path, content):
|
||||||
|
"""Write *content* to *dest_path* inside the container.
|
||||||
|
|
||||||
|
*content* is dedented and stripped so callers can use
|
||||||
|
indented triple-quoted strings.
|
||||||
|
"""
|
||||||
|
content = textwrap.dedent(content).strip() + "\n"
|
||||||
|
self.incus.run(
|
||||||
|
["file", "push", "-", f"{self.name}{dest_path}"],
|
||||||
|
input=content,
|
||||||
|
)
|
||||||
|
self.bash(f"chmod 644 {dest_path}")
|
||||||
|
|
||||||
|
def wait_ready(self, timeout=60):
|
||||||
|
"""Wait until the container is running with an IPv4 address.
|
||||||
|
|
||||||
|
Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
|
||||||
|
or raises ``TimeoutError``.
|
||||||
|
"""
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
data = self.incus.run_json(
|
||||||
|
["list", self.name],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if data and data[0].get("status") == "Running":
|
||||||
|
net = data[0].get("state", {}).get("network", {})
|
||||||
|
self.ipv4 = _extract_ip(net, "inet")
|
||||||
|
self.ipv6 = _extract_ip(net, "inet6")
|
||||||
|
if self.ipv4:
|
||||||
|
return
|
||||||
|
time.sleep(1)
|
||||||
|
raise TimeoutError(
|
||||||
|
f"Container {self.name!r} did not become ready within {timeout}s"
|
||||||
|
)
|
||||||
|
|
||||||
|
def rss_mib(self):
|
||||||
|
"""Return ``(used, total)`` memory from container (or None if unobtainable)."""
|
||||||
|
output = self.bash("free -m", check=False)
|
||||||
|
if output:
|
||||||
|
for line in output.splitlines():
|
||||||
|
if line.startswith("Mem:"):
|
||||||
|
parts = line.split()
|
||||||
|
return int(parts[2]), int(parts[1])
|
||||||
|
|
||||||
|
|
||||||
|
class RelayContainer(Container):
|
||||||
|
"""Container handle for a chatmail relay.
|
||||||
|
|
||||||
|
Accepts the short relay name (e.g. ``test0``) and derives
|
||||||
|
the Incus container name and mail domain automatically.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, incus, name):
|
||||||
|
super().__init__(
|
||||||
|
incus,
|
||||||
|
f"{name}-localchat",
|
||||||
|
domain=f"_{name}{DOMAIN_SUFFIX}",
|
||||||
|
memory="500MiB",
|
||||||
|
ipv4=RELAY_IPS.get(name),
|
||||||
|
)
|
||||||
|
self.sname = name
|
||||||
|
self.image_alias = f"localchat-{name}"
|
||||||
|
self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
|
||||||
|
self.zone = incus.lxconfigs_dir / f"{name}.zone"
|
||||||
|
|
||||||
|
def launch(self):
|
||||||
|
"""Launch from a cached per-relay image if available, else from base."""
|
||||||
|
cached = self.incus._find_image(self.image_alias)
|
||||||
|
if cached:
|
||||||
|
print(f" Using cached image {cached!r}")
|
||||||
|
else:
|
||||||
|
print(" No cached image, building from base")
|
||||||
|
image = super().launch(image=cached)
|
||||||
|
self.bash("rm -f /etc/chatmail-version")
|
||||||
|
return image
|
||||||
|
|
||||||
|
def destroy(self):
|
||||||
|
"""Stop, delete, and clean up config files."""
|
||||||
|
super().destroy()
|
||||||
|
if self.ini.exists():
|
||||||
|
self.ini.unlink()
|
||||||
|
|
||||||
|
def disable_ipv6(self):
|
||||||
|
"""Disable IPv6 inside the container via sysctl."""
|
||||||
|
self.bash("""\
|
||||||
|
sysctl -w net.ipv6.conf.all.disable_ipv6=1
|
||||||
|
sysctl -w net.ipv6.conf.default.disable_ipv6=1
|
||||||
|
mkdir -p /etc/sysctl.d
|
||||||
|
printf 'net.ipv6.conf.all.disable_ipv6=1\\n
|
||||||
|
net.ipv6.conf.default.disable_ipv6=1\\n'
|
||||||
|
> /etc/sysctl.d/99-disable-ipv6.conf
|
||||||
|
""")
|
||||||
|
|
||||||
|
def configure_hosts(self, ip):
|
||||||
|
"""Set hostname and /etc/hosts inside the container."""
|
||||||
|
self.bash(f"""
|
||||||
|
echo '{self.name}' > /etc/hostname
|
||||||
|
hostname {self.name}
|
||||||
|
sed -i '/ {self.domain}$/d' /etc/hosts
|
||||||
|
echo '{ip} {self.name} {self.domain}' >> /etc/hosts
|
||||||
|
""")
|
||||||
|
|
||||||
|
def publish_image(self):
|
||||||
|
"""Publish this container as a reusable per-relay image.
|
||||||
|
|
||||||
|
Returns True if an image was published,
|
||||||
|
False if a cached image already existed.
|
||||||
|
"""
|
||||||
|
if self.incus._find_image(self.image_alias):
|
||||||
|
return False
|
||||||
|
self.bash("apt-get clean && rm -rf /var/lib/apt/lists/*")
|
||||||
|
print(f" Publishing {self.name!r} as {self.image_alias!r} image ...")
|
||||||
|
self.incus.run(
|
||||||
|
["publish", self.name, f"--alias={self.image_alias}", "--force"],
|
||||||
|
capture=False,
|
||||||
|
)
|
||||||
|
self.wait_ready()
|
||||||
|
print(f" Image {self.image_alias!r} ready.")
|
||||||
|
return True
|
||||||
|
|
||||||
|
def deployed_version(self):
|
||||||
|
"""Read /etc/chatmail-version, or None if absent."""
|
||||||
|
return self.bash("cat /etc/chatmail-version", check=False)
|
||||||
|
|
||||||
|
def deployed_domain(self):
|
||||||
|
"""Read the domain deployed on the container (postfix myhostname)."""
|
||||||
|
return self.bash(
|
||||||
|
"postconf -h myhostname 2>/dev/null",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def verify_ssh(self, ssh_config):
|
||||||
|
"""Verify SSH connectivity to this container."""
|
||||||
|
cmd = f"ssh -F {ssh_config} -o ConnectTimeout=10 root@{self.domain} hostname"
|
||||||
|
return shell(cmd, timeout=15).returncode == 0
|
||||||
|
|
||||||
|
def configure_dns(self, dns_ip):
|
||||||
|
"""Point this container's resolver at *dns_ip*.
|
||||||
|
|
||||||
|
Disables systemd-resolved to free port 53 and writes
|
||||||
|
a static /etc/resolv.conf. Also configures unbound
|
||||||
|
(if present) to forward .localchat queries.
|
||||||
|
"""
|
||||||
|
self.bash(f"""\
|
||||||
|
systemctl disable --now systemd-resolved 2>/dev/null || true
|
||||||
|
rm -f /etc/resolv.conf
|
||||||
|
echo 'nameserver {dns_ip}' > /etc/resolv.conf
|
||||||
|
mkdir -p /etc/unbound/unbound.conf.d
|
||||||
|
printf 'server:\\n domain-insecure: "localchat"\\n\\n
|
||||||
|
forward-zone:\\n name: "localchat"\\n
|
||||||
|
forward-addr: {dns_ip}\\n'
|
||||||
|
> /etc/unbound/unbound.conf.d/localchat-forward.conf
|
||||||
|
systemctl restart unbound 2>/dev/null || true
|
||||||
|
""")
|
||||||
|
|
||||||
|
def check_dns(self, retries=5, delay=2):
|
||||||
|
"""Verify that external DNS resolution works inside the container."""
|
||||||
|
for i in range(retries):
|
||||||
|
result = self.bash(
|
||||||
|
"getent hosts pypi.org",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result:
|
||||||
|
return True
|
||||||
|
if i < retries - 1:
|
||||||
|
time.sleep(delay)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def write_ini(self, disable_ipv6=False):
|
||||||
|
"""Generate a chatmail.ini config file in lxconfigs/."""
|
||||||
|
from chatmaild.config import write_initial_config
|
||||||
|
|
||||||
|
overrides = {
|
||||||
|
"max_user_send_per_minute": 600,
|
||||||
|
"max_user_send_burst_size": 100,
|
||||||
|
"mtail_address": "127.0.0.1",
|
||||||
|
}
|
||||||
|
if disable_ipv6:
|
||||||
|
overrides["disable_ipv6"] = "True"
|
||||||
|
write_initial_config(self.ini, self.domain, overrides)
|
||||||
|
return self.ini
|
||||||
|
|
||||||
|
|
||||||
|
class DNSContainer(Container):
|
||||||
|
"""Specialised container handle for the PowerDNS name server."""
|
||||||
|
|
||||||
|
def __init__(self, incus):
|
||||||
|
super().__init__(
|
||||||
|
incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN, memory="256MiB", ipv4=DNS_IP
|
||||||
|
)
|
||||||
|
|
||||||
|
def launch(self):
|
||||||
|
"""Launch from cached DNS image if available, else from base image."""
|
||||||
|
cached = self.incus._find_image(DNS_IMAGE_ALIAS)
|
||||||
|
if cached:
|
||||||
|
print(f" Using cached image {cached!r}")
|
||||||
|
else:
|
||||||
|
print(" No cached image, building from base")
|
||||||
|
return super().launch(image=cached)
|
||||||
|
|
||||||
|
def publish_as_dns_image(self):
|
||||||
|
"""Publish this container as a reusable DNS image."""
|
||||||
|
if self.incus._find_image(DNS_IMAGE_ALIAS):
|
||||||
|
return
|
||||||
|
self.bash("apt-get clean && rm -rf /var/lib/apt/lists/*")
|
||||||
|
print(f" Publishing {self.name!r} as {DNS_IMAGE_ALIAS!r} image ...")
|
||||||
|
self.incus.run(
|
||||||
|
["publish", self.name, f"--alias={DNS_IMAGE_ALIAS}", "--force"],
|
||||||
|
capture=False,
|
||||||
|
)
|
||||||
|
self.wait_ready()
|
||||||
|
print(f" DNS image {DNS_IMAGE_ALIAS!r} ready.")
|
||||||
|
|
||||||
|
def pdnsutil(self, *args, check=True):
|
||||||
|
"""Run ``pdnsutil <args>`` inside the DNS container."""
|
||||||
|
return self.run_cmd("pdnsutil", *args, check=check)
|
||||||
|
|
||||||
|
def replace_rrset(self, zone, name, rtype, ttl, rdata):
|
||||||
|
"""Shortcut for ``pdnsutil replace-rrset``."""
|
||||||
|
self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
|
||||||
|
|
||||||
|
def restart_services(self):
|
||||||
|
"""Restart pdns and pdns-recursor."""
|
||||||
|
self.bash("""\
|
||||||
|
systemctl restart pdns
|
||||||
|
systemctl restart pdns-recursor || true
|
||||||
|
""")
|
||||||
|
|
||||||
|
def ensure(self):
|
||||||
|
"""Create the DNS container with PowerDNS if needed.
|
||||||
|
|
||||||
|
Calls ``super().ensure()`` to create/start the container
|
||||||
|
and set up SSH, then installs PowerDNS and configures
|
||||||
|
the Incus bridge to use this container as DNS.
|
||||||
|
"""
|
||||||
|
super().ensure()
|
||||||
|
self._install_powerdns()
|
||||||
|
self.incus.run(
|
||||||
|
["network", "set", "incusbr0", "dns.mode=none"],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.incus.run(
|
||||||
|
["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _install_powerdns(self):
|
||||||
|
"""Install and configure PowerDNS if not already present."""
|
||||||
|
if self.run_cmd("which", "pdns_server", check=False) is not None:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.bash("""\
|
||||||
|
systemctl disable --now systemd-resolved 2>/dev/null || true
|
||||||
|
rm -f /etc/resolv.conf
|
||||||
|
echo 'nameserver 9.9.9.9' > /etc/resolv.conf
|
||||||
|
apt-get -o DPkg::Lock::Timeout=60 update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||||
|
pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
|
||||||
|
systemctl stop pdns pdns-recursor || true
|
||||||
|
mkdir -p /var/lib/powerdns
|
||||||
|
sqlite3 /var/lib/powerdns/pdns.sqlite3 \
|
||||||
|
</usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
|
||||||
|
chown -R pdns:pdns /var/lib/powerdns
|
||||||
|
""")
|
||||||
|
|
||||||
|
self.push_file_content(
|
||||||
|
"/etc/powerdns/pdns.conf",
|
||||||
|
"""\
|
||||||
|
launch=gsqlite3
|
||||||
|
gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
|
||||||
|
local-address=127.0.0.1
|
||||||
|
local-port=5353
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.push_file_content(
|
||||||
|
"/etc/powerdns/recursor.conf",
|
||||||
|
"""\
|
||||||
|
local-address=0.0.0.0
|
||||||
|
local-port=53
|
||||||
|
forward-zones=localchat=127.0.0.1:5353
|
||||||
|
forward-zones-recurse=.=9.9.9.9;149.112.112.112
|
||||||
|
allow-from=0.0.0.0/0
|
||||||
|
dont-query=
|
||||||
|
dnssec=off
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.bash("""\
|
||||||
|
systemctl start pdns
|
||||||
|
systemctl start pdns-recursor
|
||||||
|
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
|
||||||
|
""")
|
||||||
|
|
||||||
|
def reset_dns_records(self, dns_ip, domains):
|
||||||
|
"""Create DNS zones with initial A records via pdnsutil.
|
||||||
|
|
||||||
|
Only sets SOA, NS, and A records as the minimal set
|
||||||
|
needed for SSH connectivity. Full records (MX, TXT, SRV,
|
||||||
|
CNAME, DKIM) are added later by ``cmdeploy dns``.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
dns_ip: IP of the DNS container
|
||||||
|
domains: list of dicts with 'name', 'domain', 'ip'
|
||||||
|
"""
|
||||||
|
for d in domains:
|
||||||
|
domain = d["domain"]
|
||||||
|
ip = d["ip"]
|
||||||
|
print(f" {domain} -> {ip}")
|
||||||
|
|
||||||
|
# Delete and recreate zone fresh (removes stale records)
|
||||||
|
self.pdnsutil("delete-zone", domain, check=False)
|
||||||
|
self.pdnsutil("create-zone", domain, f"ns.{domain}")
|
||||||
|
|
||||||
|
serial = str(int(time.time()))
|
||||||
|
soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
|
||||||
|
self.replace_rrset(domain, ".", "SOA", "3600", soa)
|
||||||
|
self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
|
||||||
|
self.replace_rrset(domain, ".", "A", "3600", ip)
|
||||||
|
self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
|
||||||
|
|
||||||
|
# AAAA (domain -> container IPv6, if available)
|
||||||
|
ipv6 = d.get("ipv6")
|
||||||
|
if ipv6:
|
||||||
|
self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
|
||||||
|
print(f" zone reset: SOA, NS, A, AAAA ({ip}, {ipv6})")
|
||||||
|
else:
|
||||||
|
# Remove any stale AAAA record
|
||||||
|
self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
|
||||||
|
print(f" zone reset: SOA, NS, A ({ip}, IPv4-only)")
|
||||||
|
|
||||||
|
self.restart_services()
|
||||||
|
|
||||||
|
def set_dns_records(self, text):
|
||||||
|
"""Add or overwrite DNS records from standard BIND format.
|
||||||
|
|
||||||
|
Uses ``cmdeploy.dns.parse_zone_records`` to parse.
|
||||||
|
Zones are created automatically from the record names.
|
||||||
|
"""
|
||||||
|
from ..dns import parse_zone_records
|
||||||
|
|
||||||
|
zones_seen = set()
|
||||||
|
|
||||||
|
for name, ttl, rtype, rdata in parse_zone_records(text):
|
||||||
|
# Derive zone from name: find top-level .localchat domain
|
||||||
|
name_parts = name.split(".")
|
||||||
|
zone = name # fallback
|
||||||
|
for i in range(len(name_parts) - 1):
|
||||||
|
if name_parts[i + 1 :] == ["localchat"]:
|
||||||
|
zone = ".".join(name_parts[i:])
|
||||||
|
break
|
||||||
|
|
||||||
|
# Create zone if first time seeing it
|
||||||
|
if zone not in zones_seen:
|
||||||
|
self.pdnsutil(
|
||||||
|
"create-zone",
|
||||||
|
zone,
|
||||||
|
f"ns.{zone}",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
zones_seen.add(zone)
|
||||||
|
|
||||||
|
# Figure out the record name relative to zone
|
||||||
|
if name == zone:
|
||||||
|
relative = "."
|
||||||
|
elif name.endswith(f".{zone}"):
|
||||||
|
relative = name[: -(len(zone) + 1)]
|
||||||
|
else:
|
||||||
|
relative = name
|
||||||
|
|
||||||
|
self.replace_rrset(zone, relative, rtype, ttl, rdata)
|
||||||
|
|
||||||
|
if zones_seen:
|
||||||
|
self.restart_services()
|
||||||
@@ -58,8 +58,8 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
|
|||||||
dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
|
dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
|
||||||
web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
|
web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
|
||||||
return (
|
return (
|
||||||
f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
|
f'{dkim_selector}._domainkey.{mail_domain}. 3600 IN TXT "{dkim_value}"',
|
||||||
f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
|
f'{dkim_selector}._domainkey.{mail_domain}. 3600 IN TXT "{web_dkim_value}"',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -94,9 +94,11 @@ def check_zonefile(zonefile, verbose=True):
|
|||||||
if not zf_line.strip() or zf_line.startswith(";"):
|
if not zf_line.strip() or zf_line.startswith(";"):
|
||||||
continue
|
continue
|
||||||
print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
|
print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
|
||||||
zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
|
parts = zf_line.split(None, 4)
|
||||||
zf_domain = zf_domain.rstrip(".")
|
zf_domain = parts[0].rstrip(".")
|
||||||
zf_value = zf_value.strip()
|
# parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
|
||||||
|
zf_typ = parts[3]
|
||||||
|
zf_value = parts[4].strip()
|
||||||
query_value = query_dns(zf_typ, zf_domain)
|
query_value = query_dns(zf_typ, zf_domain)
|
||||||
if zf_value != query_value:
|
if zf_value != query_value:
|
||||||
assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
|
assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
|
||||||
|
|||||||
@@ -12,13 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
|
|||||||
``www.<domain>`` and ``mta-sts.<domain>``.
|
``www.<domain>`` and ``mta-sts.<domain>``.
|
||||||
"""
|
"""
|
||||||
return [
|
return [
|
||||||
"openssl", "req", "-x509",
|
"openssl",
|
||||||
"-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
|
"req",
|
||||||
"-noenc", "-days", str(days),
|
"-x509",
|
||||||
"-keyout", str(key_path),
|
"-newkey",
|
||||||
"-out", str(cert_path),
|
"ec",
|
||||||
"-subj", f"/CN={domain}",
|
"-pkeyopt",
|
||||||
"-addext", "extendedKeyUsage=serverAuth,clientAuth",
|
"ec_paramgen_curve:P-256",
|
||||||
|
"-noenc",
|
||||||
|
"-days",
|
||||||
|
str(days),
|
||||||
|
"-keyout",
|
||||||
|
str(key_path),
|
||||||
|
"-out",
|
||||||
|
str(cert_path),
|
||||||
|
"-subj",
|
||||||
|
f"/CN={domain}",
|
||||||
|
# Mark as end-entity cert so it cannot be used as a CA to sign others.
|
||||||
|
"-addext",
|
||||||
|
"basicConstraints=critical,CA:FALSE",
|
||||||
|
"-addext",
|
||||||
|
"extendedKeyUsage=serverAuth,clientAuth",
|
||||||
"-addext",
|
"-addext",
|
||||||
f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
|
f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
|
||||||
]
|
]
|
||||||
@@ -40,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
|
|||||||
|
|
||||||
def configure(self):
|
def configure(self):
|
||||||
args = openssl_selfsigned_args(
|
args = openssl_selfsigned_args(
|
||||||
self.mail_domain, self.cert_path, self.key_path,
|
self.mail_domain,
|
||||||
|
self.cert_path,
|
||||||
|
self.key_path,
|
||||||
)
|
)
|
||||||
cmd = shlex.join(args)
|
cmd = shlex.join(args)
|
||||||
server.shell(
|
server.shell(
|
||||||
|
|||||||
@@ -49,8 +49,13 @@ class SSHExec:
|
|||||||
RemoteError = execnet.RemoteError
|
RemoteError = execnet.RemoteError
|
||||||
FuncError = FuncError
|
FuncError = FuncError
|
||||||
|
|
||||||
def __init__(self, host, verbose=False, python="python3", timeout=60):
|
def __init__(
|
||||||
self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
|
self, host, verbose=False, python="python3", timeout=60, ssh_config=None
|
||||||
|
):
|
||||||
|
spec = f"ssh=root@{host}//python={python}"
|
||||||
|
if ssh_config:
|
||||||
|
spec += f"//ssh_config={ssh_config}"
|
||||||
|
self.gateway = execnet.makegateway(spec)
|
||||||
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
|
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.verbose = verbose
|
self.verbose = verbose
|
||||||
@@ -87,8 +92,9 @@ class SSHExec:
|
|||||||
class LocalExec:
|
class LocalExec:
|
||||||
FuncError = FuncError
|
FuncError = FuncError
|
||||||
|
|
||||||
def __init__(self, verbose=False):
|
def __init__(self, verbose=False, docker=False):
|
||||||
self.verbose = verbose
|
self.verbose = verbose
|
||||||
|
self.docker = docker
|
||||||
|
|
||||||
def __call__(self, call, kwargs=None, log_callback=None):
|
def __call__(self, call, kwargs=None, log_callback=None):
|
||||||
if kwargs is None:
|
if kwargs is None:
|
||||||
@@ -100,6 +106,10 @@ class LocalExec:
|
|||||||
if not title:
|
if not title:
|
||||||
title = call.__name__
|
title = call.__name__
|
||||||
where = "locally"
|
where = "locally"
|
||||||
|
if self.docker:
|
||||||
|
if call == remote.rdns.perform_initial_checks:
|
||||||
|
kwargs["pre_command"] = "docker exec chatmail "
|
||||||
|
where = "in docker"
|
||||||
if self.verbose:
|
if self.verbose:
|
||||||
print_stderr(f"Running {where}: {title}(**{kwargs})")
|
print_stderr(f"Running {where}: {title}(**{kwargs})")
|
||||||
return self(call, kwargs, log_callback=print_stderr)
|
return self(call, kwargs, log_callback=print_stderr)
|
||||||
@@ -108,3 +118,46 @@ class LocalExec:
|
|||||||
res = self(call, kwargs, log_callback=remote.rshell.log_progress)
|
res = self(call, kwargs, log_callback=remote.rshell.log_progress)
|
||||||
print_stderr()
|
print_stderr()
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
# pyinfra exposes a ``ssh_config_file`` data key that *should* let
|
||||||
|
# paramiko parse an SSH config file directly. In practice it silently
|
||||||
|
# fails to connect (zero hosts / zero operations), so we resolve the
|
||||||
|
# hostname and identity-file ourselves and pass them via
|
||||||
|
# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
|
||||||
|
# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
|
||||||
|
|
||||||
|
|
||||||
|
def _get_from_ssh_config(host, ssh_config_path, key):
|
||||||
|
"""Internal helper to parse a value for a specific key from ssh-config."""
|
||||||
|
current_hosts = []
|
||||||
|
found_value = None
|
||||||
|
with open(ssh_config_path) as f:
|
||||||
|
for raw_line in f:
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
parts = line.split(None, 1)
|
||||||
|
if not parts:
|
||||||
|
continue
|
||||||
|
directive = parts[0].lower()
|
||||||
|
if directive == "host":
|
||||||
|
if host in current_hosts and found_value:
|
||||||
|
return found_value
|
||||||
|
current_hosts = parts[1].split()
|
||||||
|
found_value = None
|
||||||
|
elif directive == key.lower():
|
||||||
|
found_value = parts[1]
|
||||||
|
if host in current_hosts and found_value:
|
||||||
|
return found_value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_host_from_ssh_config(host, ssh_config_path):
|
||||||
|
"""Resolve a host alias to its IP from an ssh-config file."""
|
||||||
|
return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_key_from_ssh_config(host, ssh_config_path):
|
||||||
|
"""Resolve a host alias to its IdentityFile from an ssh-config file."""
|
||||||
|
return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
; Required DNS entries for chatmail servers
|
; Required DNS entries
|
||||||
zftest.testrun.org. A 135.181.204.127
|
zftest.testrun.org. 3600 IN A 135.181.204.127
|
||||||
zftest.testrun.org. AAAA 2a01:4f9:c012:52f4::1
|
zftest.testrun.org. 3600 IN AAAA 2a01:4f9:c012:52f4::1
|
||||||
zftest.testrun.org. MX 10 zftest.testrun.org.
|
zftest.testrun.org. 3600 IN MX 10 zftest.testrun.org.
|
||||||
_mta-sts.zftest.testrun.org. TXT "v=STSv1; id=202403211706"
|
_mta-sts.zftest.testrun.org. 3600 IN TXT "v=STSv1; id=202403211706"
|
||||||
mta-sts.zftest.testrun.org. CNAME zftest.testrun.org.
|
mta-sts.zftest.testrun.org. 3600 IN CNAME zftest.testrun.org.
|
||||||
www.zftest.testrun.org. CNAME zftest.testrun.org.
|
www.zftest.testrun.org. 3600 IN CNAME zftest.testrun.org.
|
||||||
opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
|
opendkim._domainkey.zftest.testrun.org. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
|
||||||
|
|
||||||
; Recommended DNS entries
|
; Recommended DNS entries
|
||||||
_submission._tcp.zftest.testrun.org. SRV 0 1 587 zftest.testrun.org.
|
zftest.testrun.org. 3600 IN TXT "v=spf1 a ~all"
|
||||||
_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
|
_dmarc.zftest.testrun.org. 3600 IN TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
||||||
_imap._tcp.zftest.testrun.org. SRV 0 1 143 zftest.testrun.org.
|
zftest.testrun.org. 3600 IN CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
|
||||||
_imaps._tcp.zftest.testrun.org. SRV 0 1 993 zftest.testrun.org.
|
_adsp._domainkey.zftest.testrun.org. 3600 IN TXT "dkim=discardable"
|
||||||
zftest.testrun.org. CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
|
_submission._tcp.zftest.testrun.org. 3600 IN SRV 0 1 587 zftest.testrun.org.
|
||||||
zftest.testrun.org. TXT "v=spf1 a:zftest.testrun.org ~all"
|
_submissions._tcp.zftest.testrun.org. 3600 IN SRV 0 1 465 zftest.testrun.org.
|
||||||
_dmarc.zftest.testrun.org. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
|
_imap._tcp.zftest.testrun.org. 3600 IN SRV 0 1 143 zftest.testrun.org.
|
||||||
_adsp._domainkey.zftest.testrun.org. TXT "dkim=discardable"
|
_imaps._tcp.zftest.testrun.org. 3600 IN SRV 0 1 993 zftest.testrun.org.
|
||||||
|
|||||||
@@ -89,9 +89,7 @@ def test_concurrent_logins_same_account(
|
|||||||
assert login_results.get()
|
assert login_results.get()
|
||||||
|
|
||||||
|
|
||||||
def test_no_vrfy(cmfactory, chatmail_config):
|
def test_no_vrfy(chatmail_config):
|
||||||
ac = cmfactory.get_online_account()
|
|
||||||
addr = ac.get_config("addr")
|
|
||||||
domain = chatmail_config.mail_domain
|
domain = chatmail_config.mail_domain
|
||||||
|
|
||||||
s = smtplib.SMTP(domain)
|
s = smtplib.SMTP(domain)
|
||||||
@@ -100,7 +98,7 @@ def test_no_vrfy(cmfactory, chatmail_config):
|
|||||||
s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
|
s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
|
||||||
result = s.getreply()
|
result = s.getreply()
|
||||||
print(result)
|
print(result)
|
||||||
s.putcmd("vrfy", addr)
|
s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
|
||||||
result2 = s.getreply()
|
result2 = s.getreply()
|
||||||
print(result2)
|
print(result2)
|
||||||
assert result[0] == result2[0] == 252
|
assert result[0] == result2[0] == 252
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
|
@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
|
||||||
def test_newemail_configure(maildomain, rpc, chatmail_config):
|
def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
|
||||||
"""Test configuring accounts by scanning a QR code works."""
|
"""Test configuring accounts by scanning a QR code works."""
|
||||||
url = f"DCACCOUNT:https://{maildomain}/new"
|
url = f"DCACCOUNT:https://{maildomain}/new"
|
||||||
for i in range(3):
|
for i in range(3):
|
||||||
@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
|
|||||||
# set_config_from_qr, so fetch credentials via requests instead
|
# set_config_from_qr, so fetch credentials via requests instead
|
||||||
res = requests.post(f"https://{maildomain}/new", verify=False)
|
res = requests.post(f"https://{maildomain}/new", verify=False)
|
||||||
data = res.json()
|
data = res.json()
|
||||||
rpc.add_or_update_transport(account_id, {
|
rpc.add_or_update_transport(
|
||||||
"addr": data["email"],
|
account_id,
|
||||||
"password": data["password"],
|
{
|
||||||
"imapServer": maildomain,
|
"addr": data["email"],
|
||||||
"smtpServer": maildomain,
|
"password": data["password"],
|
||||||
"certificateChecks": "acceptInvalidCertificates",
|
"imapServer": maildomain_ip,
|
||||||
})
|
"smtpServer": maildomain_ip,
|
||||||
|
"certificateChecks": "acceptInvalidCertificates",
|
||||||
|
},
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
rpc.add_transport_from_qr(account_id, url)
|
rpc.add_transport_from_qr(account_id, url)
|
||||||
|
|||||||
@@ -12,8 +12,9 @@ from cmdeploy.cmdeploy import get_sshexec
|
|||||||
|
|
||||||
class TestSSHExecutor:
|
class TestSSHExecutor:
|
||||||
@pytest.fixture(scope="class")
|
@pytest.fixture(scope="class")
|
||||||
def sshexec(self, sshdomain):
|
def sshexec(self, sshdomain, pytestconfig):
|
||||||
return get_sshexec(sshdomain)
|
ssh_config = pytestconfig.getoption("ssh_config")
|
||||||
|
return get_sshexec(sshdomain, ssh_config=ssh_config)
|
||||||
|
|
||||||
def test_ls(self, sshexec):
|
def test_ls(self, sshexec):
|
||||||
out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
|
out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
|
||||||
@@ -132,11 +133,10 @@ def test_authenticated_from(cmsetup, maildata):
|
|||||||
@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
|
@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
|
||||||
def test_reject_missing_dkim(cmsetup, maildata, from_addr):
|
def test_reject_missing_dkim(cmsetup, maildata, from_addr):
|
||||||
domain = cmsetup.maildomain
|
domain = cmsetup.maildomain
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
sock.settimeout(10)
|
|
||||||
try:
|
try:
|
||||||
sock.connect((domain, 25))
|
sock = socket.create_connection((domain, 25), timeout=10)
|
||||||
except socket.timeout:
|
sock.close()
|
||||||
|
except (socket.timeout, OSError):
|
||||||
pytest.skip(f"port 25 not reachable for {domain}")
|
pytest.skip(f"port 25 not reachable for {domain}")
|
||||||
|
|
||||||
recipient = cmsetup.gen_users(1)[0]
|
recipient = cmsetup.gen_users(1)[0]
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
|
|||||||
assert msg2.get_snapshot().text == "message0"
|
assert msg2.get_snapshot().text == "message0"
|
||||||
|
|
||||||
def test_exceed_quota(
|
def test_exceed_quota(
|
||||||
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
|
self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
|
||||||
):
|
):
|
||||||
"""This is a very slow test as it needs to upload >100MB of mail data
|
"""This is a very slow test as it needs to upload >100MB of mail data
|
||||||
before quota is exceeded, and thus depends on the speed of the upload.
|
before quota is exceeded, and thus depends on the speed of the upload.
|
||||||
@@ -92,7 +92,9 @@ class TestEndToEndDeltaChat:
|
|||||||
lp.sec(f"filling remote inbox for {user}")
|
lp.sec(f"filling remote inbox for {user}")
|
||||||
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
|
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
|
||||||
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
||||||
sshexec = get_sshexec(sshdomain)
|
sshexec = get_sshexec(
|
||||||
|
sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
|
||||||
|
)
|
||||||
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
||||||
res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
|
res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
|
||||||
assert res["percent"] >= 100
|
assert res["percent"] >= 100
|
||||||
|
|||||||
@@ -3,12 +3,15 @@ import os
|
|||||||
from cmdeploy.cmdeploy import main
|
from cmdeploy.cmdeploy import main
|
||||||
|
|
||||||
|
|
||||||
def test_status_cmd(chatmail_config, capsys, request):
|
def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
|
||||||
os.chdir(request.config.invocation_params.dir)
|
os.chdir(request.config.invocation_params.dir)
|
||||||
command = ["status"]
|
command = ["status"]
|
||||||
if os.getenv("CHATMAIL_SSH"):
|
ssh_host = pytestconfig.getoption("ssh_host")
|
||||||
command.append("--ssh-host")
|
if ssh_host:
|
||||||
command.append(os.getenv("CHATMAIL_SSH"))
|
command.extend(["--ssh-host", ssh_host])
|
||||||
|
ssh_config = pytestconfig.getoption("ssh_config")
|
||||||
|
if ssh_config:
|
||||||
|
command.extend(["--ssh-config", ssh_config])
|
||||||
assert main(command) == 0
|
assert main(command) == 0
|
||||||
status_out = capsys.readouterr()
|
status_out = capsys.readouterr()
|
||||||
print(status_out.out)
|
print(status_out.out)
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ import imaplib
|
|||||||
import itertools
|
import itertools
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
|
import re
|
||||||
import smtplib
|
import smtplib
|
||||||
|
import socket
|
||||||
import ssl
|
import ssl
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
@@ -18,6 +20,76 @@ def pytest_addoption(parser):
|
|||||||
parser.addoption(
|
parser.addoption(
|
||||||
"--slow", action="store_true", default=False, help="also run slow tests"
|
"--slow", action="store_true", default=False, help="also run slow tests"
|
||||||
)
|
)
|
||||||
|
parser.addoption(
|
||||||
|
"--ssh-host",
|
||||||
|
dest="ssh_host",
|
||||||
|
default=None,
|
||||||
|
help="SSH host (overrides mail_domain for SSH operations).",
|
||||||
|
)
|
||||||
|
parser.addoption(
|
||||||
|
"--ssh-config",
|
||||||
|
dest="ssh_config",
|
||||||
|
default=None,
|
||||||
|
help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ssh_config_hosts(path):
|
||||||
|
"""Parse an OpenSSH config file and return a dict of hostname -> IP."""
|
||||||
|
mapping = {}
|
||||||
|
current_names = []
|
||||||
|
for ln in Path(path).read_text().splitlines():
|
||||||
|
line = ln.strip()
|
||||||
|
m = re.match(r"^Host\s+(.+)", line)
|
||||||
|
if m:
|
||||||
|
current_names = m.group(1).split()
|
||||||
|
continue
|
||||||
|
m = re.match(r"^Hostname\s+(\S+)", line)
|
||||||
|
if m and current_names:
|
||||||
|
ip = m.group(1)
|
||||||
|
for name in current_names:
|
||||||
|
mapping[name] = ip
|
||||||
|
current_names = []
|
||||||
|
return mapping
|
||||||
|
|
||||||
|
|
||||||
|
_original_getaddrinfo = socket.getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
def _make_patched_getaddrinfo(host_map):
|
||||||
|
"""Return a getaddrinfo that resolves hosts in host_map to their IPs."""
|
||||||
|
|
||||||
|
def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
|
||||||
|
if host in host_map:
|
||||||
|
ip = host_map[host]
|
||||||
|
return _original_getaddrinfo(ip, port, family, type, proto, flags)
|
||||||
|
return _original_getaddrinfo(host, port, family, type, proto, flags)
|
||||||
|
|
||||||
|
return patched_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope="session")
|
||||||
|
def _setup_localchat_dns(pytestconfig):
|
||||||
|
"""Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
|
||||||
|
ssh_config = pytestconfig.getoption("ssh_config")
|
||||||
|
if not ssh_config or not Path(ssh_config).exists():
|
||||||
|
yield {}
|
||||||
|
return
|
||||||
|
host_map = _parse_ssh_config_hosts(ssh_config)
|
||||||
|
if not host_map:
|
||||||
|
yield {}
|
||||||
|
return
|
||||||
|
socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
|
||||||
|
try:
|
||||||
|
yield host_map
|
||||||
|
finally:
|
||||||
|
socket.getaddrinfo = _original_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def ssh_config_host_map(_setup_localchat_dns):
|
||||||
|
"""Return the host-name → IP map parsed from ssh-config."""
|
||||||
|
return _setup_localchat_dns
|
||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
@@ -35,6 +107,11 @@ def pytest_runtest_setup(item):
|
|||||||
|
|
||||||
|
|
||||||
def _get_chatmail_config():
|
def _get_chatmail_config():
|
||||||
|
ini = os.environ.get("CHATMAIL_INI")
|
||||||
|
if ini:
|
||||||
|
path = Path(ini).resolve()
|
||||||
|
if path.exists():
|
||||||
|
return read_config(path), path
|
||||||
current = Path().resolve()
|
current = Path().resolve()
|
||||||
while 1:
|
while 1:
|
||||||
path = current.joinpath("chatmail.ini").resolve()
|
path = current.joinpath("chatmail.ini").resolve()
|
||||||
@@ -61,8 +138,14 @@ def maildomain(chatmail_config):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
@pytest.fixture(scope="session")
|
||||||
def sshdomain(maildomain):
|
def sshdomain(maildomain, pytestconfig):
|
||||||
return os.environ.get("CHATMAIL_SSH", maildomain)
|
return pytestconfig.getoption("ssh_host") or maildomain
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def maildomain_ip(maildomain, ssh_config_host_map):
|
||||||
|
"""Return the IP for maildomain from ssh-config, or maildomain itself."""
|
||||||
|
return ssh_config_host_map.get(maildomain, maildomain)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -306,23 +389,32 @@ from deltachat_rpc_client import DeltaChat, Rpc
|
|||||||
class ChatmailACFactory:
|
class ChatmailACFactory:
|
||||||
"""RPC-based account factory for chatmail testing."""
|
"""RPC-based account factory for chatmail testing."""
|
||||||
|
|
||||||
def __init__(self, rpc, maildomain, gencreds, chatmail_config):
|
def __init__(
|
||||||
|
self,
|
||||||
|
rpc,
|
||||||
|
maildomain,
|
||||||
|
maildomain_ip,
|
||||||
|
gencreds,
|
||||||
|
chatmail_config,
|
||||||
|
ssh_config_host_map,
|
||||||
|
):
|
||||||
self.dc = DeltaChat(rpc)
|
self.dc = DeltaChat(rpc)
|
||||||
self.rpc = rpc
|
self.rpc = rpc
|
||||||
self._maildomain = maildomain
|
self._maildomain = maildomain
|
||||||
|
self._maildomain_ip = maildomain_ip
|
||||||
self.gencreds = gencreds
|
self.gencreds = gencreds
|
||||||
self.chatmail_config = chatmail_config
|
self.chatmail_config = chatmail_config
|
||||||
|
self._ssh_config_host_map = ssh_config_host_map
|
||||||
|
|
||||||
def _make_transport(self, domain):
|
def _make_transport(self, domain):
|
||||||
"""Build a transport config dict for the given domain."""
|
"""Build a transport config dict for the given domain."""
|
||||||
addr, password = self.gencreds(domain)
|
addr, password = self.gencreds(domain)
|
||||||
|
server = self._ssh_config_host_map.get(domain, domain)
|
||||||
transport = {
|
transport = {
|
||||||
"addr": addr,
|
"addr": addr,
|
||||||
"password": password,
|
"password": password,
|
||||||
# Setting server explicitly skips requesting autoconfig XML,
|
"imapServer": server,
|
||||||
# see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
|
"smtpServer": server,
|
||||||
"imapServer": domain,
|
|
||||||
"smtpServer": domain,
|
|
||||||
}
|
}
|
||||||
if self.chatmail_config.tls_cert_mode == "self":
|
if self.chatmail_config.tls_cert_mode == "self":
|
||||||
transport["certificateChecks"] = "acceptInvalidCertificates"
|
transport["certificateChecks"] = "acceptInvalidCertificates"
|
||||||
@@ -376,32 +468,43 @@ def rpc(tmp_path_factory):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def cmfactory(rpc, gencreds, maildomain, chatmail_config):
|
def cmfactory(
|
||||||
|
rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
|
||||||
|
):
|
||||||
"""Return a ChatmailACFactory for creating online Delta Chat accounts."""
|
"""Return a ChatmailACFactory for creating online Delta Chat accounts."""
|
||||||
return ChatmailACFactory(
|
return ChatmailACFactory(
|
||||||
rpc=rpc,
|
rpc=rpc,
|
||||||
maildomain=maildomain,
|
maildomain=maildomain,
|
||||||
|
maildomain_ip=maildomain_ip,
|
||||||
gencreds=gencreds,
|
gencreds=gencreds,
|
||||||
chatmail_config=chatmail_config,
|
chatmail_config=chatmail_config,
|
||||||
|
ssh_config_host_map=ssh_config_host_map,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def remote(sshdomain):
|
def remote(sshdomain, pytestconfig):
|
||||||
return Remote(sshdomain)
|
return Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
|
||||||
|
|
||||||
|
|
||||||
class Remote:
|
class Remote:
|
||||||
def __init__(self, sshdomain):
|
def __init__(self, sshdomain, ssh_config=None):
|
||||||
self.sshdomain = sshdomain
|
self.sshdomain = sshdomain
|
||||||
|
self.ssh_config = ssh_config
|
||||||
|
|
||||||
def iter_output(self, logcmd="", ready=None):
|
def iter_output(self, logcmd="", ready=None):
|
||||||
getjournal = "journalctl -f" if not logcmd else logcmd
|
getjournal = "journalctl -f" if not logcmd else logcmd
|
||||||
print(self.sshdomain)
|
print(self.sshdomain)
|
||||||
match self.sshdomain:
|
match self.sshdomain:
|
||||||
case "@local": command = []
|
case "@local":
|
||||||
case "localhost": command = []
|
command = []
|
||||||
case _: command = ["ssh", f"root@{self.sshdomain}"]
|
case "localhost":
|
||||||
|
command = []
|
||||||
|
case _:
|
||||||
|
command = ["ssh"]
|
||||||
|
if self.ssh_config:
|
||||||
|
command.extend(["-F", self.ssh_config])
|
||||||
|
command.append(f"root@{self.sshdomain}")
|
||||||
[command.append(arg) for arg in getjournal.split()]
|
[command.append(arg) for arg in getjournal.split()]
|
||||||
self.popen = subprocess.Popen(
|
self.popen = subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
|
|||||||
@@ -23,7 +23,10 @@ class TestCmdline:
|
|||||||
run = parser.parse_args(["run"])
|
run = parser.parse_args(["run"])
|
||||||
assert init and run
|
assert init and run
|
||||||
|
|
||||||
def test_init_not_overwrite(self, capsys):
|
def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
|
||||||
|
monkeypatch.delenv("CHATMAIL_INI", raising=False)
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
|
||||||
assert main(["init", "chat.example.org"]) == 0
|
assert main(["init", "chat.example.org"]) == 0
|
||||||
capsys.readouterr()
|
capsys.readouterr()
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from copy import deepcopy
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from cmdeploy import remote
|
from cmdeploy import remote
|
||||||
from cmdeploy.dns import check_full_zone, check_initial_remote_data
|
from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -125,18 +125,33 @@ class TestPerformInitialChecks:
|
|||||||
assert not l
|
assert not l
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_zone_records():
|
||||||
|
text = """
|
||||||
|
; This is a comment
|
||||||
|
some.domain. 3600 IN A 1.1.1.1
|
||||||
|
|
||||||
|
; Another comment
|
||||||
|
www.some.domain. 3600 IN CNAME some.domain.
|
||||||
|
"""
|
||||||
|
records = list(parse_zone_records(text))
|
||||||
|
assert records == [
|
||||||
|
("some.domain", "3600", "A", "1.1.1.1"),
|
||||||
|
("www.some.domain", "3600", "CNAME", "some.domain."),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_zone_records_invalid_line():
|
||||||
|
text = "invalid line"
|
||||||
|
with pytest.raises(ValueError, match="Bad zone record line"):
|
||||||
|
list(parse_zone_records(text))
|
||||||
|
|
||||||
|
|
||||||
def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
|
def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
|
||||||
for zf_line in zonefile.split("\n"):
|
if only_required:
|
||||||
if zf_line.startswith("#"):
|
# Only take records before the "; Recommended" section
|
||||||
if "Recommended" in zf_line and only_required:
|
zonefile = zonefile.split("; Recommended")[0]
|
||||||
return
|
for name, ttl, rtype, rdata in parse_zone_records(zonefile):
|
||||||
continue
|
mockdns_base.setdefault(rtype, {})[name] = rdata
|
||||||
if not zf_line.strip():
|
|
||||||
continue
|
|
||||||
zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
|
|
||||||
zf_domain = zf_domain.rstrip(".")
|
|
||||||
zf_value = zf_value.strip()
|
|
||||||
mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
|
|
||||||
|
|
||||||
|
|
||||||
class MockSSHExec:
|
class MockSSHExec:
|
||||||
|
|||||||
173
cmdeploy/src/cmdeploy/tests/test_lxc.py
Normal file
173
cmdeploy/src/cmdeploy/tests/test_lxc.py
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
"""Tests for cmdeploy lxc-* subcommands."""
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from cmdeploy.lxc import cli
|
||||||
|
from cmdeploy.lxc.incus import Incus
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skipif(
|
||||||
|
not shutil.which("incus") or not shutil.which("lxc"),
|
||||||
|
reason="incus/lxc not installed",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def ix():
|
||||||
|
return Incus()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def lxc_setup():
|
||||||
|
ix = Incus()
|
||||||
|
ix.get_dns_container().ensure()
|
||||||
|
return ix.list_managed()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def relay_container(lxc_setup):
|
||||||
|
test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
|
||||||
|
relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
|
||||||
|
if not relays:
|
||||||
|
pytest.skip("no test relay containers running")
|
||||||
|
return relays[0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def cmdeploy():
|
||||||
|
def run(*args):
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, "-m", "cmdeploy.cmdeploy", *args],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"subcmd, expected, absent",
|
||||||
|
[
|
||||||
|
(None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
|
||||||
|
("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
|
||||||
|
("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
|
||||||
|
("lxc-test", ["--one"], ["--config"]),
|
||||||
|
("lxc-status", [], ["--config"]),
|
||||||
|
("run", ["--ssh-config"], ["--lxc"]),
|
||||||
|
("dns", ["--ssh-config"], []),
|
||||||
|
("test", ["--ssh-config"], []),
|
||||||
|
("status", ["--ssh-config"], []),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_help_options(cmdeploy, subcmd, expected, absent):
|
||||||
|
args = [subcmd, "--help"] if subcmd else ["--help"]
|
||||||
|
result = cmdeploy(*args)
|
||||||
|
output = result.stdout + result.stderr
|
||||||
|
assert result.returncode == 0
|
||||||
|
for flag in expected:
|
||||||
|
assert flag in output
|
||||||
|
for flag in absent:
|
||||||
|
assert flag not in output
|
||||||
|
|
||||||
|
|
||||||
|
class TestSSHConfig:
|
||||||
|
def test_lxconfigs(self, ix, lxc_setup):
|
||||||
|
d = ix.lxconfigs_dir
|
||||||
|
assert d.name == "lxconfigs"
|
||||||
|
assert d.exists()
|
||||||
|
path = ix.ssh_config_path
|
||||||
|
assert path.name == "ssh-config"
|
||||||
|
assert path.parent.name == "lxconfigs"
|
||||||
|
|
||||||
|
def test_write_ssh_config(self, ix, lxc_setup):
|
||||||
|
path = ix.write_ssh_config()
|
||||||
|
assert path.exists()
|
||||||
|
text = path.read_text()
|
||||||
|
|
||||||
|
for c in lxc_setup:
|
||||||
|
if c.get("ip"):
|
||||||
|
assert c["name"] in text
|
||||||
|
assert f"Hostname {c['ip']}" in text
|
||||||
|
|
||||||
|
assert "User root" in text
|
||||||
|
assert "IdentityFile" in text
|
||||||
|
assert "StrictHostKeyChecking accept-new" in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_dns(ix, relay_container):
|
||||||
|
def dig(qname, qtype):
|
||||||
|
ct = ix.get_dns_container()
|
||||||
|
return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
|
||||||
|
|
||||||
|
domain = relay_container["domain"]
|
||||||
|
assert dig(domain, "A") == relay_container["ip"]
|
||||||
|
assert domain in dig(domain, "MX")
|
||||||
|
assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
|
||||||
|
|
||||||
|
|
||||||
|
class TestLxcStatus:
|
||||||
|
def test_cli_lxc_status_help(self, cmdeploy):
|
||||||
|
result = cmdeploy("lxc-status", "--help")
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert "status" in result.stdout.lower()
|
||||||
|
|
||||||
|
def test_shows_containers(self, lxc_setup, capsys):
|
||||||
|
from cmdeploy.cmdeploy import Out
|
||||||
|
|
||||||
|
class QuietOut(Out):
|
||||||
|
def red(self, msg, **kw):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def green(self, msg, **kw):
|
||||||
|
pass
|
||||||
|
|
||||||
|
ret = cli.lxc_status_cmd(None, QuietOut())
|
||||||
|
assert ret == 0
|
||||||
|
captured = capsys.readouterr().out
|
||||||
|
assert "ns-localchat" in captured
|
||||||
|
assert "running" in captured
|
||||||
|
|
||||||
|
def test_deploy_freshness(self, ix, monkeypatch):
|
||||||
|
ct = ix.get_container("x")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
|
||||||
|
lambda _self: "abc123def456",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdeploy.lxc.incus.RelayContainer.deployed_domain",
|
||||||
|
lambda _self: ct.domain,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdeploy.lxc.cli.get_version_string",
|
||||||
|
lambda: "abc123def456",
|
||||||
|
)
|
||||||
|
assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
|
||||||
|
assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
|
||||||
|
|
||||||
|
# Hash matches but local has uncommitted changes
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdeploy.lxc.cli.get_version_string",
|
||||||
|
lambda: "abc123def456\ndiff --git a/foo",
|
||||||
|
)
|
||||||
|
assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdeploy.lxc.incus.RelayContainer.deployed_version",
|
||||||
|
lambda _self: None,
|
||||||
|
)
|
||||||
|
assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)
|
||||||
97
cmdeploy/src/cmdeploy/tests/test_util.py
Normal file
97
cmdeploy/src/cmdeploy/tests/test_util.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from cmdeploy.util import (
|
||||||
|
build_chatmaild_sdist,
|
||||||
|
collapse,
|
||||||
|
get_chatmaild_sdist,
|
||||||
|
get_git_hash,
|
||||||
|
get_version_string,
|
||||||
|
shell,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collapse():
|
||||||
|
text = """
|
||||||
|
line 1
|
||||||
|
line 2
|
||||||
|
"""
|
||||||
|
assert collapse(text) == "line 1 line 2"
|
||||||
|
assert collapse(" single line ") == "single line"
|
||||||
|
|
||||||
|
|
||||||
|
def test_git_helpers_no_git(tmp_path):
|
||||||
|
# Not a git repo
|
||||||
|
assert get_git_hash(root=tmp_path) is None
|
||||||
|
assert get_version_string(root=tmp_path) == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_git_helpers_empty_repo(tmp_path):
|
||||||
|
shell("git init", cwd=tmp_path, check=True)
|
||||||
|
# No commits yet
|
||||||
|
assert get_git_hash(root=tmp_path) is None
|
||||||
|
assert get_version_string(root=tmp_path) == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_git_helpers_with_commits_and_diffs(tmp_path):
|
||||||
|
shell("git init", cwd=tmp_path, check=True)
|
||||||
|
shell("git config user.email you@example.com", cwd=tmp_path, check=True)
|
||||||
|
shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
|
||||||
|
|
||||||
|
# First commit
|
||||||
|
path = tmp_path / "file.txt"
|
||||||
|
path.write_text("content")
|
||||||
|
shell("git add file.txt", cwd=tmp_path, check=True)
|
||||||
|
shell("git commit -m initial", cwd=tmp_path, check=True)
|
||||||
|
|
||||||
|
git_hash = get_git_hash(root=tmp_path)
|
||||||
|
assert len(git_hash) >= 7 # usually 40, but git is git
|
||||||
|
assert get_version_string(root=tmp_path) == git_hash
|
||||||
|
|
||||||
|
# Create a diff
|
||||||
|
path.write_text("new content")
|
||||||
|
v = get_version_string(root=tmp_path)
|
||||||
|
assert v.startswith(git_hash + "\n")
|
||||||
|
assert "new content" in v
|
||||||
|
assert not v.endswith("\n")
|
||||||
|
|
||||||
|
# Commit again -> no diff
|
||||||
|
shell("git add file.txt", cwd=tmp_path, check=True)
|
||||||
|
shell("git commit -m second", cwd=tmp_path, check=True)
|
||||||
|
new_hash = get_git_hash(root=tmp_path)
|
||||||
|
assert new_hash != git_hash
|
||||||
|
assert get_version_string(root=tmp_path) == new_hash
|
||||||
|
|
||||||
|
# Diffs inside excluded test dirs are invisible to the version string
|
||||||
|
test_dir = tmp_path / "cmdeploy" / "src" / "cmdeploy" / "tests"
|
||||||
|
test_dir.mkdir(parents=True)
|
||||||
|
test_file = test_dir / "test_foo.py"
|
||||||
|
test_file.write_text("pass")
|
||||||
|
shell("git add .", cwd=tmp_path, check=True)
|
||||||
|
shell("git commit -m 'add test file'", cwd=tmp_path, check=True)
|
||||||
|
test_file.write_text("assert True")
|
||||||
|
assert get_version_string(root=tmp_path) == get_git_hash(root=tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_chatmaild_sdist(tmp_path):
|
||||||
|
dist_dir = tmp_path / "dist"
|
||||||
|
|
||||||
|
# First call builds the sdist
|
||||||
|
result = build_chatmaild_sdist(dist_dir)
|
||||||
|
assert result.name.endswith(".tar.gz")
|
||||||
|
assert result.stat().st_size > 0
|
||||||
|
|
||||||
|
# Second call is idempotent - returns the same file, no rebuild
|
||||||
|
mtime = result.stat().st_mtime
|
||||||
|
result2 = build_chatmaild_sdist(dist_dir)
|
||||||
|
assert result2 == result
|
||||||
|
assert result2.stat().st_mtime == mtime
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_chatmaild_sdist_errors(tmp_path):
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
get_chatmaild_sdist(tmp_path / "nonexistent")
|
||||||
|
|
||||||
|
empty = tmp_path / "empty"
|
||||||
|
empty.mkdir()
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
get_chatmaild_sdist(empty)
|
||||||
126
cmdeploy/src/cmdeploy/util.py
Normal file
126
cmdeploy/src/cmdeploy/util.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
"""Shared utility functions for cmdeploy."""
|
||||||
|
|
||||||
|
import fcntl
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _project_root():
|
||||||
|
"""Return the project root directory."""
|
||||||
|
return Path(__file__).resolve().parent.parent.parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
def collapse(text):
|
||||||
|
"""Dedent, join lines, and strip a (triple-quoted) string.
|
||||||
|
|
||||||
|
Handy for writing shell commands across multiple lines::
|
||||||
|
|
||||||
|
cmd = collapse(f\"""
|
||||||
|
cmdeploy run
|
||||||
|
--config {ct.ini}
|
||||||
|
--ssh-host {ct.domain}
|
||||||
|
\""")
|
||||||
|
"""
|
||||||
|
return textwrap.dedent(text).replace("\n", " ").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def shell(cmd, check=False, **kwargs):
|
||||||
|
"""Run a shell command string with sensible defaults.
|
||||||
|
|
||||||
|
*cmd* is passed through :func:`collapse` first, so callers
|
||||||
|
can use triple-quoted f-strings freely.
|
||||||
|
Captures stdout/stderr by default; pass ``capture_output=False``
|
||||||
|
to stream output to the terminal instead.
|
||||||
|
"""
|
||||||
|
if "capture_output" not in kwargs and "stdout" not in kwargs:
|
||||||
|
kwargs["capture_output"] = True
|
||||||
|
return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_git_hash(root=None):
|
||||||
|
"""Return the local HEAD commit hash, or None."""
|
||||||
|
if root is None:
|
||||||
|
root = _project_root()
|
||||||
|
result = shell(
|
||||||
|
"git rev-parse HEAD",
|
||||||
|
cwd=str(root),
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return result.stdout.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
DIFF_EXCLUDES = (
|
||||||
|
":(exclude)cmdeploy/src/cmdeploy/tests",
|
||||||
|
":(exclude)chatmaild/src/chatmaild/tests",
|
||||||
|
)
|
||||||
|
"""Git pathspecs appended to ``git diff`` so that changes
|
||||||
|
limited to test files do not affect the deployed version string."""
|
||||||
|
|
||||||
|
|
||||||
|
def get_version_string(root=None):
|
||||||
|
"""Return ``git_hash\\ngit_diff`` for the local working tree.
|
||||||
|
|
||||||
|
Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
|
||||||
|
``/etc/chatmail-version`` and by ``lxc-status`` to compare
|
||||||
|
the deployed state against the local checkout.
|
||||||
|
|
||||||
|
Changes inside directories listed in :data:`DIFF_EXCLUDES`
|
||||||
|
are ignored so that test-only edits do not trigger
|
||||||
|
a redeployment.
|
||||||
|
"""
|
||||||
|
if root is None:
|
||||||
|
root = _project_root()
|
||||||
|
git_hash = get_git_hash(root=root) or "unknown"
|
||||||
|
excludes = " ".join(f"'{e}'" for e in DIFF_EXCLUDES)
|
||||||
|
try:
|
||||||
|
git_diff = shell(
|
||||||
|
f"git diff -- . {excludes}",
|
||||||
|
cwd=str(root),
|
||||||
|
).stdout.strip()
|
||||||
|
except Exception:
|
||||||
|
git_diff = ""
|
||||||
|
if git_diff:
|
||||||
|
return f"{git_hash}\n{git_diff}"
|
||||||
|
return git_hash
|
||||||
|
|
||||||
|
|
||||||
|
def _chatmaild_default_dist_dir():
|
||||||
|
return _project_root() / "chatmaild" / "dist"
|
||||||
|
|
||||||
|
|
||||||
|
def build_chatmaild_sdist(dist_dir=None):
|
||||||
|
"""Build the chatmaild sdist if not already present (idempotent, process-safe)."""
|
||||||
|
|
||||||
|
if dist_dir is None:
|
||||||
|
dist_dir = _chatmaild_default_dist_dir()
|
||||||
|
dist_dir = Path(dist_dir).resolve()
|
||||||
|
dist_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
lockfile = dist_dir.parent / ".dist.lock"
|
||||||
|
with open(lockfile, "w") as fh:
|
||||||
|
fcntl.flock(fh, fcntl.LOCK_EX)
|
||||||
|
existing = [p for p in dist_dir.iterdir() if p.suffix == ".gz"]
|
||||||
|
if existing:
|
||||||
|
return existing[0]
|
||||||
|
subprocess.check_output(
|
||||||
|
[sys.executable, "-m", "build", "-n"]
|
||||||
|
+ ["--sdist", "chatmaild", "--outdir", str(dist_dir)],
|
||||||
|
cwd=str(_project_root()),
|
||||||
|
)
|
||||||
|
return get_chatmaild_sdist(dist_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def get_chatmaild_sdist(dist_dir=None):
|
||||||
|
"""Return the path to the pre-built chatmaild sdist."""
|
||||||
|
if dist_dir is None:
|
||||||
|
dist_dir = _chatmaild_default_dist_dir()
|
||||||
|
|
||||||
|
entries = list(Path(dist_dir).iterdir())
|
||||||
|
if len(entries) == 0:
|
||||||
|
raise FileNotFoundError(f"dist directory is empty: {dist_dir}")
|
||||||
|
if len(entries) > 1:
|
||||||
|
raise ValueError(f"expected one file in {dist_dir}, found {len(entries)}")
|
||||||
|
return entries[0]
|
||||||
@@ -15,7 +15,7 @@ author = 'chatmail collective'
|
|||||||
|
|
||||||
extensions = [
|
extensions = [
|
||||||
#'sphinx.ext.autodoc',
|
#'sphinx.ext.autodoc',
|
||||||
#'sphinx.ext.viewdoc',
|
#'sphinx.ext.viewcode',
|
||||||
'sphinxcontrib.mermaid',
|
'sphinxcontrib.mermaid',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1,264 +0,0 @@
|
|||||||
Docker installation
|
|
||||||
===================
|
|
||||||
|
|
||||||
This section provides instructions for installing a chatmail relay
|
|
||||||
using Docker Compose.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
- Docker support is experimental, CI builds and tests the image automatically, but please report bugs.
|
|
||||||
- The image wraps the cmdeploy process detailed in the :doc:`getting_started` instructions in a Debian-systemd image with r/w access to `/sys/fs`
|
|
||||||
- Currently amd64-only (arm64 should work but is untested).
|
|
||||||
|
|
||||||
|
|
||||||
Setup Preparation
|
|
||||||
-----------------
|
|
||||||
|
|
||||||
We use ``chat.example.org`` as the chatmail domain in the following
|
|
||||||
steps. Please substitute it with your own domain.
|
|
||||||
|
|
||||||
1. Install docker and docker compose v2 (check with `docker compose version`), install, e.g., on
|
|
||||||
- Debian 12 through the `official install instructions <https://docs.docker.com/engine/install/debian/#install-using-the-repository>`_
|
|
||||||
- Debian 13+ with `apt install docker docker-compose`
|
|
||||||
|
|
||||||
2. Setup the initial DNS records.
|
|
||||||
The following is an example in the familiar BIND zone file format with
|
|
||||||
a TTL of 1 hour (3600 seconds).
|
|
||||||
Please substitute your domain and IP addresses.
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
chat.example.org. 3600 IN A 198.51.100.5
|
|
||||||
chat.example.org. 3600 IN AAAA 2001:db8::5
|
|
||||||
www.chat.example.org. 3600 IN CNAME chat.example.org.
|
|
||||||
mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
|
|
||||||
|
|
||||||
3. Configure kernel parameters on the host, as these can not be set from the container::
|
|
||||||
|
|
||||||
echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
|
|
||||||
echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
|
|
||||||
sudo sysctl --system
|
|
||||||
|
|
||||||
|
|
||||||
Docker Compose Setup
|
|
||||||
--------------------
|
|
||||||
|
|
||||||
Pre-built images are available from GitHub Container Registry. The
|
|
||||||
``main`` branch and tagged releases are pushed automatically by CI::
|
|
||||||
|
|
||||||
docker pull ghcr.io/chatmail/relay:main # latest main branch
|
|
||||||
docker pull ghcr.io/chatmail/relay:1.2.3 # tagged release
|
|
||||||
|
|
||||||
|
|
||||||
Create service directory
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
Either:
|
|
||||||
|
|
||||||
- Create a service directory and download the compose files::
|
|
||||||
|
|
||||||
mkdir -p /srv/chatmail-relay && cd /srv/chatmail-relay
|
|
||||||
wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker/docker-compose.yaml
|
|
||||||
wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker/docker-compose.override.yaml.example -O docker-compose.override.yaml
|
|
||||||
|
|
||||||
- or clone the chatmail repo and enter the docker directory::
|
|
||||||
|
|
||||||
git clone https://github.com/chatmail/relay
|
|
||||||
cd relay/docker
|
|
||||||
|
|
||||||
|
|
||||||
Customize and start
|
|
||||||
^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
1. Set the fully qualified domain name of the relay::
|
|
||||||
|
|
||||||
echo 'MAIL_DOMAIN=chat.example.org' > .env
|
|
||||||
|
|
||||||
The container generates a ``chatmail.ini`` with defaults from
|
|
||||||
``MAIL_DOMAIN`` on first start. To customize chatmail settings, mount
|
|
||||||
your own ``chatmail.ini`` instead (see `Custom chatmail.ini`_ below).
|
|
||||||
|
|
||||||
2. All local customizations (data paths, extra volumes, config mounts) go in
|
|
||||||
``docker-compose.override.yaml``, which Compose merges automatically with
|
|
||||||
the base file. By default, all data is stored in docker volumes, you will
|
|
||||||
likely want to at least create and configure the mail storage location, but
|
|
||||||
you might also want to configure external TLS certificates there.
|
|
||||||
|
|
||||||
3. Start the container::
|
|
||||||
|
|
||||||
docker compose up -d
|
|
||||||
docker compose logs -f chatmail # view logs, Ctrl+C to exit
|
|
||||||
|
|
||||||
4. After installation is complete, open ``https://chat.example.org`` in
|
|
||||||
your browser.
|
|
||||||
|
|
||||||
Finish install and test
|
|
||||||
-----------------------
|
|
||||||
|
|
||||||
You can test the installation with::
|
|
||||||
|
|
||||||
pip install cmping chat.example.org # or
|
|
||||||
uvx cmping chat.example.org # if you use https://docs.astral.sh/uv/
|
|
||||||
|
|
||||||
You should check and extend your DNS records for better interoperability::
|
|
||||||
|
|
||||||
# Show required DNS records
|
|
||||||
docker exec chatmail cmdeploy dns --ssh-host @local
|
|
||||||
|
|
||||||
You can check server status with::
|
|
||||||
|
|
||||||
docker exec chatmail cmdeploy status --ssh-host @local
|
|
||||||
|
|
||||||
You can run some benchmarks (can also run from any machine with cmdeploy installed)::
|
|
||||||
|
|
||||||
docker exec chatmail cmdeploy bench
|
|
||||||
|
|
||||||
You can run the test suite with::
|
|
||||||
|
|
||||||
docker exec chatmail cmdeploy test --ssh-host localhost
|
|
||||||
|
|
||||||
You can look at logs::
|
|
||||||
|
|
||||||
docker exec chatmail journalctl -fu postfix@-
|
|
||||||
|
|
||||||
|
|
||||||
Customization
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Website
|
|
||||||
^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
You can customize the chatmail landing page by mounting a directory with
|
|
||||||
your own website source files.
|
|
||||||
|
|
||||||
1. Create a directory with your custom website source::
|
|
||||||
|
|
||||||
mkdir -p ./custom/www/src
|
|
||||||
nano ./custom/www/src/index.md
|
|
||||||
|
|
||||||
2. Add the volume mount in ``docker-compose.override.yaml``::
|
|
||||||
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
volumes:
|
|
||||||
- ./custom/www:/opt/chatmail-www
|
|
||||||
|
|
||||||
3. Restart the service::
|
|
||||||
|
|
||||||
docker compose down
|
|
||||||
docker compose up -d
|
|
||||||
|
|
||||||
|
|
||||||
Custom chatmail.ini
|
|
||||||
^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
If you want to go beyond simply setting the ``MAIL_DOMAIN`` in ``.env``, you
|
|
||||||
can use a regular `chatmail.ini` to give you full control.
|
|
||||||
|
|
||||||
1. Extract the generated config from a running container::
|
|
||||||
|
|
||||||
docker cp chatmail:/etc/chatmail/chatmail.ini ./chatmail.ini
|
|
||||||
|
|
||||||
2. Edit ``chatmail.ini`` as needed.
|
|
||||||
|
|
||||||
3. Add the volume mount in ``docker-compose.override.yaml`` ::
|
|
||||||
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
volumes:
|
|
||||||
- ./chatmail.ini:/etc/chatmail/chatmail.ini
|
|
||||||
|
|
||||||
4. Restart the container, the container skips generating a new one: ::
|
|
||||||
|
|
||||||
docker compose down && docker compose up -d
|
|
||||||
|
|
||||||
|
|
||||||
External TLS certificates
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
If TLS certificates are managed outside the container (e.g. by certbot,
|
|
||||||
acmetool, or Traefik on the host), mount them into the container and set
|
|
||||||
``TLS_EXTERNAL_CERT_AND_KEY`` in ``docker-compose.override.yaml``.
|
|
||||||
Changed certificates are picked up automatically via inotify.
|
|
||||||
See the examples in the example override and :ref:`external-tls` in the getting started guide for details.
|
|
||||||
|
|
||||||
|
|
||||||
Migrating from a bare-metal install
|
|
||||||
------------------------------------
|
|
||||||
|
|
||||||
If you have an existing bare-metal chatmail installation and want to
|
|
||||||
switch to Docker:
|
|
||||||
|
|
||||||
1. Stop all existing services::
|
|
||||||
|
|
||||||
systemctl stop postfix dovecot doveauth nginx opendkim unbound \
|
|
||||||
acmetool-redirector filtermail filtermail-incoming chatmail-turn \
|
|
||||||
iroh-relay chatmail-metadata lastlogin mtail
|
|
||||||
systemctl disable postfix dovecot doveauth nginx opendkim unbound \
|
|
||||||
acmetool-redirector filtermail filtermail-incoming chatmail-turn \
|
|
||||||
iroh-relay chatmail-metadata lastlogin mtail
|
|
||||||
|
|
||||||
2. Copy your existing ``chatmail.ini`` and mount it into the container
|
|
||||||
(see `Custom chatmail.ini`_ above)::
|
|
||||||
|
|
||||||
cp /usr/local/lib/chatmaild/chatmail.ini ./chatmail.ini
|
|
||||||
|
|
||||||
3. Copy persistent data into the ``./data/`` subdirectories (for example, as configured in `Customize and start`_) ::
|
|
||||||
|
|
||||||
mkdir -p data/dkim data/certs data/mail
|
|
||||||
|
|
||||||
# DKIM keys
|
|
||||||
cp -a /etc/dkimkeys/* data/dkim/
|
|
||||||
|
|
||||||
# TLS certificates
|
|
||||||
rsync -a /var/lib/acme/ data/certs/
|
|
||||||
|
|
||||||
Note that ownership of dkim and acme is adjusted on container start.
|
|
||||||
|
|
||||||
For the mail directory::
|
|
||||||
|
|
||||||
rsync -a /home/vmail/ data/mail/
|
|
||||||
|
|
||||||
Alternatively, mount ``/home/vmail`` directly by changing the volume
|
|
||||||
in ``docker-compose-override.yaml``::
|
|
||||||
|
|
||||||
- /home/vmail:/home/vmail
|
|
||||||
|
|
||||||
The three ``./data/`` subdirectories cover all persistent state.
|
|
||||||
Everything else is regenerated by the ``configure`` and ``activate``
|
|
||||||
stages on container start.
|
|
||||||
|
|
||||||
Building the image
|
|
||||||
------------------
|
|
||||||
|
|
||||||
Clone the repository and build the Docker image::
|
|
||||||
|
|
||||||
git clone https://github.com/chatmail/relay
|
|
||||||
cd relay
|
|
||||||
docker/build.sh
|
|
||||||
|
|
||||||
The build bakes all binaries, Python packages, and the install stage
|
|
||||||
into the image. After building, only the ``docker/`` directory and a ``.env``
|
|
||||||
with ``MAIL_DOMAIN`` are needed to run the container. The `build.sh` passes the
|
|
||||||
git hash onto the docker build so it can be determined if there has been a
|
|
||||||
change that warrants a redeploy.
|
|
||||||
|
|
||||||
You can transfer a locally built image to your server directly (pigz is parallel `gzip` which can be used instead as well) ::
|
|
||||||
|
|
||||||
docker save chatmail-relay:latest | pigz | ssh chat.example.org 'pigz -d | docker load'
|
|
||||||
|
|
||||||
|
|
||||||
Forcing a full reinstall
|
|
||||||
------------------------
|
|
||||||
|
|
||||||
On container start, only the ``configure`` and ``activate`` stages run by default.
|
|
||||||
|
|
||||||
To force a full reinstall (e.g. after updating the source), either
|
|
||||||
rebuild the image::
|
|
||||||
|
|
||||||
docker compose build chatmail
|
|
||||||
docker compose up -d
|
|
||||||
|
|
||||||
Or override the stages at runtime without rebuilding::
|
|
||||||
|
|
||||||
CMDEPLOY_STAGES="install,configure,activate" docker compose up -d
|
|
||||||
@@ -98,12 +98,6 @@ steps. Please substitute it with your own domain.
|
|||||||
configure at your DNS provider (it can take some time until they are
|
configure at your DNS provider (it can take some time until they are
|
||||||
public).
|
public).
|
||||||
|
|
||||||
Docker installation
|
|
||||||
-------------------
|
|
||||||
|
|
||||||
There is experimental support for running chatmail via Docker Compose.
|
|
||||||
See :doc:`docker` for full setup instructions.
|
|
||||||
|
|
||||||
Other helpful commands
|
Other helpful commands
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
|
|||||||
:maxdepth: 5
|
:maxdepth: 5
|
||||||
|
|
||||||
getting_started
|
getting_started
|
||||||
docker
|
|
||||||
proxy
|
proxy
|
||||||
migrate
|
migrate
|
||||||
overview
|
overview
|
||||||
|
lxc
|
||||||
related
|
related
|
||||||
faq
|
faq
|
||||||
|
|||||||
288
doc/source/lxc.rst
Normal file
288
doc/source/lxc.rst
Normal file
@@ -0,0 +1,288 @@
|
|||||||
|
Local testing with LXC/Incus
|
||||||
|
============================
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
cmdeploy LXC support is geared towards local testing and CI, only.
|
||||||
|
Do not base production setups on it.
|
||||||
|
|
||||||
|
|
||||||
|
The ``cmdeploy`` tool includes support for running
|
||||||
|
chatmail relays inside local
|
||||||
|
`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
|
||||||
|
This is useful for development, testing, and CI
|
||||||
|
without requiring a remote server.
|
||||||
|
LXC system containers behave like lightweight virtual machines.
|
||||||
|
They share the host's kernel but run their own init system
|
||||||
|
(systemd), package manager, and network stack,
|
||||||
|
so the cmdeploy deployment scripts work exactly
|
||||||
|
as they would on a real Debian server or cloud VPS.
|
||||||
|
|
||||||
|
Prerequisites
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Install `Incus <https://linuxcontainers.org/incus/>`_
|
||||||
|
(LXC container manager).
|
||||||
|
See the `official installation guide
|
||||||
|
<https://linuxcontainers.org/incus/docs/main/installing/>`_
|
||||||
|
for full details.
|
||||||
|
|
||||||
|
After installing incus, initialise and grant yourself access::
|
||||||
|
|
||||||
|
sudo incus admin init --minimal
|
||||||
|
sudo usermod -aG incus-admin $USER
|
||||||
|
|
||||||
|
.. warning::
|
||||||
|
|
||||||
|
You **must now log out and back in** (or run ``newgrp incus-admin``)
|
||||||
|
after adding yourself to the group.
|
||||||
|
Without this, all ``cmdeploy lxc-*`` commands
|
||||||
|
will fail with permission errors.
|
||||||
|
|
||||||
|
Verify the installation works by running ``incus list``,
|
||||||
|
which should print an empty table without errors.
|
||||||
|
|
||||||
|
|
||||||
|
Quick start
|
||||||
|
-----------
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
cd relay
|
||||||
|
scripts/initenv.sh # bootstrap venv
|
||||||
|
source venv/bin/activate # activate venv
|
||||||
|
cmdeploy lxc-test # create containers, deploy, test
|
||||||
|
|
||||||
|
|
||||||
|
The ``lxc-test`` command executes each ``cmdeploy`` subprocess command
|
||||||
|
so you can copy-paste and run them individually.
|
||||||
|
A section timing summary is printed at the end.
|
||||||
|
No host DNS delegation or ``~/.ssh/config`` changes are needed
|
||||||
|
because lxc-test passes ssh-related CLI options to
|
||||||
|
``cmdeploy run`` and ``cmdeploy test`` commands.
|
||||||
|
|
||||||
|
|
||||||
|
CLI reference
|
||||||
|
--------------
|
||||||
|
|
||||||
|
``lxc-start [--ipv4-only] [--run] [NAME ...]``
|
||||||
|
Create and start containers.
|
||||||
|
Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
|
||||||
|
Pass one or more ``NAME`` arguments to create user relay containers instead
|
||||||
|
(e.g. ``cmdeploy lxc-start myrelay``).
|
||||||
|
Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
|
||||||
|
producing an IPv4-only relay.
|
||||||
|
Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
|
||||||
|
Generates ``lxconfigs/ssh-config``.
|
||||||
|
It reuses existing containers and resets DNS zones to minimal records.
|
||||||
|
|
||||||
|
``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
|
||||||
|
Stop relay containers.
|
||||||
|
Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
|
||||||
|
Pass ``NAME`` to stop specific containers.
|
||||||
|
Use ``--destroy`` to also delete the containers and their config files.
|
||||||
|
Use ``--destroy-all`` to additionally destroy
|
||||||
|
the ``ns-localchat`` DNS container **and** remove all cached
|
||||||
|
images (``localchat-base``, per-relay images),
|
||||||
|
giving a fully clean slate for the next ``lxc-test``.
|
||||||
|
User containers are **never** destroyed unless named explicitly.
|
||||||
|
|
||||||
|
``lxc-test [--one]``
|
||||||
|
Idempotent full pipeline:
|
||||||
|
|
||||||
|
1. ``lxc-start``: create ``test0`` + ``test1`` containers,
|
||||||
|
configure DNS with readiness check
|
||||||
|
|
||||||
|
2. ``cmdeploy run``: deploy chatmail services
|
||||||
|
on all relays **in parallel**
|
||||||
|
|
||||||
|
3. publish per-relay cached images (``localchat-test0``,
|
||||||
|
``localchat-test1``) after first successful deploy
|
||||||
|
|
||||||
|
4. ``cmdeploy dns --zonefile``: generate standard
|
||||||
|
BIND-format zone files, load full DNS records
|
||||||
|
|
||||||
|
5. ``cmdeploy test``: run full test suite
|
||||||
|
with ``-n4 -x``
|
||||||
|
|
||||||
|
By default creates, deploys, and tests both ``test0`` and ``test1``
|
||||||
|
for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
|
||||||
|
test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
|
||||||
|
Pass ``--one`` to only deploy and test against ``test0``
|
||||||
|
(skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
|
||||||
|
|
||||||
|
``lxc-status``
|
||||||
|
Show live status of all LXC containers (including the DNS container),
|
||||||
|
deploy freshness (comparing ``/etc/chatmail-version``
|
||||||
|
against local ``git rev-parse HEAD`` and ``git diff``),
|
||||||
|
SSH config inclusion, and host DNS forwarding for ``.localchat``.
|
||||||
|
Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
|
||||||
|
**STALE** (different commit), or **NOT DEPLOYED**.
|
||||||
|
|
||||||
|
|
||||||
|
Container types
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
**Test relay containers** (``test0-localchat``, ``test1-localchat``)
|
||||||
|
Created automatically by ``lxc-test``.
|
||||||
|
**test0** has IPv4 and IPv6 configured,
|
||||||
|
**test1** is IPv4-only (``disable_ipv6 = True``).
|
||||||
|
|
||||||
|
**User relay containers** (``<name>-localchat``)
|
||||||
|
Created by ``cmdeploy lxc-start <name>``
|
||||||
|
where ``<name>`` does not start with ``test``.
|
||||||
|
These are personal development instances,
|
||||||
|
never touched by ``lxc-stop --destroy`` unless named explicitly.
|
||||||
|
|
||||||
|
**DNS container** (``ns-localchat``)
|
||||||
|
Singleton container running PowerDNS.
|
||||||
|
Created automatically when any relay is started.
|
||||||
|
|
||||||
|
|
||||||
|
.. _lxc-ssh-config:
|
||||||
|
|
||||||
|
SSH configuration
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
|
||||||
|
a standard OpenSSH config file mapping every container name,
|
||||||
|
its domain, and a short alias to the container's IP address::
|
||||||
|
|
||||||
|
Host test0-localchat _test0.localchat _test0
|
||||||
|
Hostname 10.204.0.42
|
||||||
|
User root
|
||||||
|
IdentityFile /path/to/relay/lxconfigs/id_localchat
|
||||||
|
IdentitiesOnly yes
|
||||||
|
StrictHostKeyChecking accept-new
|
||||||
|
UserKnownHostsFile /dev/null
|
||||||
|
LogLevel ERROR
|
||||||
|
|
||||||
|
All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
|
||||||
|
accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
|
||||||
|
``lxc-test`` passes it automatically.
|
||||||
|
|
||||||
|
**Using containers from the host shell:**
|
||||||
|
|
||||||
|
To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
|
||||||
|
|
||||||
|
Include /absolute/path/to/relay/lxconfigs/ssh-config
|
||||||
|
|
||||||
|
|
||||||
|
.. _lxc-dns-setup:
|
||||||
|
.. _localchat-tld:
|
||||||
|
|
||||||
|
``.localchat`` DNS and name resolution
|
||||||
|
---------------------------------------
|
||||||
|
|
||||||
|
All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
|
||||||
|
(e.g. ``_test0.localchat``, ``_test1.localchat``),
|
||||||
|
a non-delegated suffix that exists only within the local PowerDNS infrastructure.
|
||||||
|
A dedicated DNS container (``ns-localchat``)
|
||||||
|
is created so that local test relays interact
|
||||||
|
with DNS similar to a regular public Internet setup.
|
||||||
|
On first start, ``cmdeploy lxc-start`` creates this container
|
||||||
|
running two `PowerDNS <https://www.powerdns.com/>`_ services:
|
||||||
|
|
||||||
|
* **pdns-server** (authoritative) serves ``.localchat``
|
||||||
|
zones from a local SQLite database.
|
||||||
|
|
||||||
|
* **pdns-recursor** (recursive) listens on the Incus
|
||||||
|
bridge so all containers can use it.
|
||||||
|
Forwards ``.localchat`` queries to the local
|
||||||
|
authoritative server and everything else to Quad9 (``9.9.9.9``).
|
||||||
|
|
||||||
|
After the DNS container is up, ``lxc-start`` configures the Incus bridge
|
||||||
|
to advertise its IP via DHCP and disables Incus's own DNS.
|
||||||
|
DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
|
||||||
|
|
||||||
|
1. **``lxc-start``** resets each relay zone to
|
||||||
|
**SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
|
||||||
|
If host DNS resolution is configured, users can
|
||||||
|
afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
|
||||||
|
--ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
|
||||||
|
LXC subcommands do not depend on host DNS resolution
|
||||||
|
and resolve addresses via ``lxconfigs/ssh-config``.
|
||||||
|
|
||||||
|
2. **``cmdeploy dns --zonefile``** generates a standard
|
||||||
|
BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
|
||||||
|
TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
|
||||||
|
into PowerDNS.
|
||||||
|
|
||||||
|
This two-phase approach prevents premature configuration of mail records
|
||||||
|
before the relay is actually deployed and running.
|
||||||
|
Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
|
||||||
|
inside a relay container, Unbound has a configuration plugin snippet
|
||||||
|
that forwards all ``.localchat`` queries to the PowerDNS recursor,
|
||||||
|
and lets all other queries go through normal recursive resolution.
|
||||||
|
|
||||||
|
|
||||||
|
State outside the repository
|
||||||
|
-----------------------------
|
||||||
|
|
||||||
|
All generated configuration by lxc subcommands live in ``lxconfigs/``
|
||||||
|
(git-ignored), including the SSH key pair (``id_localchat``),
|
||||||
|
per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
|
||||||
|
|
||||||
|
The only state *outside* the repository is the Incus containers and images themselves
|
||||||
|
(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
|
||||||
|
Several cached images are published to the local Incus image store:
|
||||||
|
|
||||||
|
* ``localchat-base``: Debian 12 with openssh-server and Python
|
||||||
|
(built on first run)
|
||||||
|
|
||||||
|
* ``localchat-test0``, ``localchat-test1``: per-relay snapshots
|
||||||
|
published after the first successful ``cmdeploy run``.
|
||||||
|
Subsequent containers launch from these images
|
||||||
|
so the deploy step is mostly no-ops.
|
||||||
|
|
||||||
|
Relay containers are limited to **500 MiB RAM**
|
||||||
|
and the DNS container to **256 MiB**.
|
||||||
|
|
||||||
|
|
||||||
|
.. _lxc-tls:
|
||||||
|
|
||||||
|
TLS handling and underscore domains
|
||||||
|
------------------------------------
|
||||||
|
|
||||||
|
Container domains start with ``_`` (e.g. ``_test0.localchat``).
|
||||||
|
As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
|
||||||
|
underscore domains automatically use self-signed TLS
|
||||||
|
and ``smtp_tls_security_level = encrypt``.
|
||||||
|
This permits cross-relay federation between LXC containers
|
||||||
|
without any external certificate authority.
|
||||||
|
Delta Chat clients connecting to these relays
|
||||||
|
must be configured with
|
||||||
|
``certificateChecks = acceptInvalidCertificates``
|
||||||
|
(the test fixtures handle this automatically).
|
||||||
|
`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
|
||||||
|
is meant to make this special setting unnecessary for chatmail clients
|
||||||
|
that are connecting to underscore domains.
|
||||||
|
|
||||||
|
|
||||||
|
Known limitations
|
||||||
|
------------------
|
||||||
|
|
||||||
|
The LXC environment differs from a production
|
||||||
|
deployment in several ways:
|
||||||
|
|
||||||
|
**No ACME / Let's Encrypt**:
|
||||||
|
Self-signed TLS only (see :ref:`lxc-tls`);
|
||||||
|
ACME code paths are never exercised locally.
|
||||||
|
|
||||||
|
**No inbound connections from the internet**:
|
||||||
|
Containers sit on a private Incus bridge and are not port-forwarded.
|
||||||
|
Only the host and other containers on the same bridge can reach them.
|
||||||
|
|
||||||
|
**Local federation only**:
|
||||||
|
Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
|
||||||
|
but these relays are invisible to any external mail server.
|
||||||
|
|
||||||
|
**DNS is local only**:
|
||||||
|
The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
|
||||||
|
(see :ref:`lxc-dns-setup`).
|
||||||
|
|
||||||
|
**IPv6 is ULA-only**:
|
||||||
|
Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
|
||||||
|
These are not globally routable, but are sufficient for testing IPv6 service binding
|
||||||
|
(Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
|
||||||
|
test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Build the chatmail Docker image with the current git hash baked in.
|
|
||||||
# Usage: ./docker/build.sh [extra docker-compose build args...]
|
|
||||||
#
|
|
||||||
# .git/ is excluded from the build context (.dockerignore) so the hash
|
|
||||||
# must be passed as a build arg from the host.
|
|
||||||
|
|
||||||
export GIT_HASH=$(git rev-parse HEAD)
|
|
||||||
exec docker compose -f docker/docker-compose.yaml build "$@"
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Run container setup commands
|
|
||||||
After=multi-user.target
|
|
||||||
ConditionPathExists=/chatmail-init.sh
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
ExecStart=/bin/bash /chatmail-init.sh
|
|
||||||
RemainAfterExit=true
|
|
||||||
WorkingDirectory=/opt/chatmail
|
|
||||||
PassEnvironment=<envs_list>
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
export CHATMAIL_INI="${CHATMAIL_INI:-/etc/chatmail/chatmail.ini}"
|
|
||||||
|
|
||||||
CMDEPLOY=/opt/cmdeploy/bin/cmdeploy
|
|
||||||
|
|
||||||
if [ -z "$MAIL_DOMAIN" ]; then
|
|
||||||
echo "ERROR: Environment variable 'MAIL_DOMAIN' must be set!" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Generate DKIM keys if not mounted
|
|
||||||
if [ ! -f /etc/dkimkeys/opendkim.private ]; then
|
|
||||||
/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d "$MAIL_DOMAIN" -s opendkim
|
|
||||||
fi
|
|
||||||
# Fix ownership for bind-mounted keys (host opendkim UID may differ from container)
|
|
||||||
chown -R opendkim:opendkim /etc/dkimkeys
|
|
||||||
|
|
||||||
# Create chatmail.ini, skip if mounted
|
|
||||||
mkdir -p "$(dirname "$CHATMAIL_INI")"
|
|
||||||
if [ ! -f "$CHATMAIL_INI" ]; then
|
|
||||||
$CMDEPLOY init --config "$CHATMAIL_INI" "$MAIL_DOMAIN"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Auto-detect IPv6: if the host has no IPv6 connectivity, set disable_ipv6
|
|
||||||
# in the ini so dovecot/postfix/nginx bind to IPv4 only.
|
|
||||||
# Uses network_mode:host so /proc/net/if_inet6 reflects the host's stack.
|
|
||||||
if [ ! -e /proc/net/if_inet6 ]; then
|
|
||||||
if grep -q '^disable_ipv6 = False' "$CHATMAIL_INI"; then
|
|
||||||
sed -i 's/^disable_ipv6 = False/disable_ipv6 = True/' "$CHATMAIL_INI"
|
|
||||||
echo "[INFO] IPv6 not available, set disable_ipv6 = True"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Inject external TLS paths from env var unless defined in chatmail.ini
|
|
||||||
if [ -n "${TLS_EXTERNAL_CERT_AND_KEY:-}" ]; then
|
|
||||||
if ! grep -q '^tls_external_cert_and_key' "$CHATMAIL_INI"; then
|
|
||||||
echo "tls_external_cert_and_key = $TLS_EXTERNAL_CERT_AND_KEY" >> "$CHATMAIL_INI"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Ensure mailboxes directory exists (chatmail-metadata needs it at startup,
|
|
||||||
# but Dovecot only creates it on first mail delivery)
|
|
||||||
mkdir -p "/home/vmail/mail/${MAIL_DOMAIN}"
|
|
||||||
chown vmail:vmail "/home/vmail/mail/${MAIL_DOMAIN}"
|
|
||||||
|
|
||||||
# --- Deploy fingerprint: skip cmdeploy run if nothing changed ---
|
|
||||||
# On restart with identical image+config, systemd already brings up all
|
|
||||||
# enabled services only configure+activate are needed here.
|
|
||||||
IMAGE_VERSION_FILE="/etc/chatmail-image-version"
|
|
||||||
FINGERPRINT_FILE="/etc/chatmail/.deploy-fingerprint"
|
|
||||||
image_ver="none"
|
|
||||||
[ -f "$IMAGE_VERSION_FILE" ] && image_ver=$(cat "$IMAGE_VERSION_FILE")
|
|
||||||
config_hash=$(sha256sum "$CHATMAIL_INI" | cut -c1-16)
|
|
||||||
current_fp="${image_ver}:${config_hash}"
|
|
||||||
|
|
||||||
# CMDEPLOY_STAGES non-empty in env = operator override -> always run.
|
|
||||||
# Otherwise, if fingerprint matches the last successful deploy, skip.
|
|
||||||
if [ -z "${CMDEPLOY_STAGES:-}" ] \
|
|
||||||
&& [ -f "$FINGERPRINT_FILE" ] \
|
|
||||||
&& [ "$(cat "$FINGERPRINT_FILE")" = "$current_fp" ]; then
|
|
||||||
echo "[INFO] No changes detected ($current_fp), skipping deploy."
|
|
||||||
else
|
|
||||||
export CMDEPLOY_STAGES="${CMDEPLOY_STAGES:-configure,activate}"
|
|
||||||
|
|
||||||
# Skip DNS check when MAIL_DOMAIN is a bare IP address
|
|
||||||
SKIP_DNS=""
|
|
||||||
if [[ "$MAIL_DOMAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$MAIL_DOMAIN" =~ : ]]; then
|
|
||||||
SKIP_DNS="--skip-dns-check"
|
|
||||||
fi
|
|
||||||
$CMDEPLOY run --config "$CHATMAIL_INI" --ssh-host @local $SKIP_DNS
|
|
||||||
|
|
||||||
# Restore the build-time hash
|
|
||||||
cp /etc/chatmail-image-version /etc/chatmail-version
|
|
||||||
echo "$current_fp" > "$FINGERPRINT_FILE"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Signal success to Docker healthcheck
|
|
||||||
touch /run/chatmail-init.done
|
|
||||||
|
|
||||||
# Forward journald to console so `docker compose logs` works
|
|
||||||
grep -q '^ForwardToConsole=yes' /etc/systemd/journald.conf \
|
|
||||||
|| echo "ForwardToConsole=yes" >> /etc/systemd/journald.conf
|
|
||||||
systemctl restart systemd-journald
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
# syntax=docker/dockerfile:1
|
|
||||||
FROM jrei/systemd-debian:12 AS base
|
|
||||||
|
|
||||||
ENV LANG=en_US.UTF-8
|
|
||||||
|
|
||||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|
||||||
--mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
|
|
||||||
echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend && \
|
|
||||||
echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend && \
|
|
||||||
apt-get update && \
|
|
||||||
DEBIAN_FRONTEND=noninteractive TZ=UTC \
|
|
||||||
apt-get install -y \
|
|
||||||
ca-certificates \
|
|
||||||
gcc \
|
|
||||||
git \
|
|
||||||
python3 \
|
|
||||||
python3-dev \
|
|
||||||
python3-venv \
|
|
||||||
tzdata \
|
|
||||||
locales && \
|
|
||||||
sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
|
|
||||||
dpkg-reconfigure --frontend=noninteractive locales && \
|
|
||||||
update-locale LANG=$LANG
|
|
||||||
|
|
||||||
# --- Build-time: install cmdeploy venv and run install stage ---
|
|
||||||
# Editable install so importlib.resources reads directly from the source tree.
|
|
||||||
# On container start only "configure,activate" stages run.
|
|
||||||
|
|
||||||
# Copy dependency metadata first so pip install layer is cached
|
|
||||||
COPY cmdeploy/pyproject.toml /opt/chatmail/cmdeploy/pyproject.toml
|
|
||||||
COPY chatmaild/pyproject.toml /opt/chatmail/chatmaild/pyproject.toml
|
|
||||||
|
|
||||||
# Dummy scaffolding so editable install can discover packages
|
|
||||||
RUN mkdir -p /opt/chatmail/cmdeploy/src/cmdeploy \
|
|
||||||
/opt/chatmail/chatmaild/src/chatmaild && \
|
|
||||||
touch /opt/chatmail/cmdeploy/src/cmdeploy/__init__.py \
|
|
||||||
/opt/chatmail/chatmaild/src/chatmaild/__init__.py
|
|
||||||
|
|
||||||
# Dummy git repo: .git/ is excluded from the build context (.dockerignore)
|
|
||||||
# but setuptools calls `git ls-files` when building the sdist.
|
|
||||||
WORKDIR /opt/chatmail
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
||||||
git init -q && \
|
|
||||||
python3 -m venv /opt/cmdeploy && \
|
|
||||||
/opt/cmdeploy/bin/pip install -e chatmaild/ -e cmdeploy/
|
|
||||||
|
|
||||||
# Full source copy (editable install's .egg-link still points here)
|
|
||||||
COPY . /opt/chatmail/
|
|
||||||
|
|
||||||
# Minimal chatmail.ini
|
|
||||||
RUN printf '[params]\nmail_domain = build.local\n' > /tmp/chatmail.ini
|
|
||||||
|
|
||||||
RUN CMDEPLOY_STAGES=install \
|
|
||||||
CHATMAIL_INI=/tmp/chatmail.ini \
|
|
||||||
/opt/cmdeploy/bin/pyinfra @local \
|
|
||||||
/opt/chatmail/cmdeploy/src/cmdeploy/run.py -y
|
|
||||||
|
|
||||||
RUN cp -a www/ /opt/chatmail-www/
|
|
||||||
|
|
||||||
# Remove build-only packages — not needed at runtime.
|
|
||||||
# Keep git: test_deployed_state needs `git rev-parse HEAD` to verify the
|
|
||||||
# deployed version hash matches /etc/chatmail-version.
|
|
||||||
RUN apt-get purge -y gcc python3-dev && \
|
|
||||||
apt-get autoremove -y && \
|
|
||||||
rm -f /tmp/chatmail.ini
|
|
||||||
|
|
||||||
# Record image version (used in deploy fingerprint at runtime).
|
|
||||||
# GIT_HASH is passed as a build arg (from docker-compose or CI) so that
|
|
||||||
# .git/ can be excluded from the build context via .dockerignore.
|
|
||||||
# Two files: chatmail-image-version is the immutable build hash (survives
|
|
||||||
# deploys); chatmail-version is overwritten by cmdeploy run and restored
|
|
||||||
# from the image version after each deploy in chatmail-init.sh.
|
|
||||||
ARG GIT_HASH=unknown
|
|
||||||
RUN echo "$GIT_HASH" > /etc/chatmail-image-version && \
|
|
||||||
echo "$GIT_HASH" > /etc/chatmail-version
|
|
||||||
|
|
||||||
# Mock git HEAD so `git rev-parse HEAD` returns the source repo's commit hash.
|
|
||||||
# The .git/ dir was created by `git init` earlier (for setuptools); we just
|
|
||||||
# write the build hash into whatever branch HEAD points to.
|
|
||||||
RUN head_ref=$(sed 's/^ref: //' /opt/chatmail/.git/HEAD) && \
|
|
||||||
mkdir -p "/opt/chatmail/.git/$(dirname "$head_ref")" && \
|
|
||||||
echo "$GIT_HASH" > "/opt/chatmail/.git/$head_ref"
|
|
||||||
# --- End build-time install ---
|
|
||||||
|
|
||||||
ENV TZ=:/etc/localtime
|
|
||||||
ENV PATH="/opt/cmdeploy/bin:${PATH}"
|
|
||||||
RUN ln -s /etc/chatmail/chatmail.ini /opt/chatmail/chatmail.ini
|
|
||||||
|
|
||||||
ARG CHATMAIL_INIT_SERVICE_PATH=/lib/systemd/system/chatmail-init.service
|
|
||||||
COPY ./docker/chatmail-init.service "$CHATMAIL_INIT_SERVICE_PATH"
|
|
||||||
RUN ln -sf "$CHATMAIL_INIT_SERVICE_PATH" "/etc/systemd/system/multi-user.target.wants/chatmail-init.service"
|
|
||||||
|
|
||||||
# Remove default nginx site config at build time (not in entrypoint)
|
|
||||||
RUN rm -f /etc/nginx/sites-enabled/default
|
|
||||||
|
|
||||||
COPY --chmod=555 ./docker/chatmail-init.sh /chatmail-init.sh
|
|
||||||
COPY --chmod=555 ./docker/entrypoint.sh /entrypoint.sh
|
|
||||||
COPY --chmod=555 ./docker/healthcheck.sh /healthcheck.sh
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=10s --start-period=180s --timeout=10s --retries=3 \
|
|
||||||
CMD /healthcheck.sh
|
|
||||||
|
|
||||||
STOPSIGNAL SIGRTMIN+3
|
|
||||||
|
|
||||||
ENTRYPOINT ["/entrypoint.sh"]
|
|
||||||
|
|
||||||
CMD [ "--default-standard-output=journal+console", \
|
|
||||||
"--default-standard-error=journal+console" ]
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
# Traefik reverse-proxy example — use as a compose override:
|
|
||||||
#
|
|
||||||
# docker compose -f docker-compose.yaml -f docker-compose-traefik.yaml up -d
|
|
||||||
#
|
|
||||||
# Traefik handles HTTP→HTTPS redirect and ACME certificate issuance.
|
|
||||||
# traefik-certs-dumper extracts the certificates to the filesystem so
|
|
||||||
# chatmail's Postfix/Dovecot/nginx can use them via TLS_EXTERNAL_CERT_AND_KEY.
|
|
||||||
#
|
|
||||||
# Prerequisites:
|
|
||||||
# mkdir -p traefik/data traefik/dynamic-configs
|
|
||||||
# touch traefik/data/acme.json && chmod 600 traefik/data/acme.json
|
|
||||||
# cp traefik/config.yaml.example traefik/config.yaml # see below
|
|
||||||
#
|
|
||||||
# Required .env variables (in addition to MAIL_DOMAIN):
|
|
||||||
# ACME_EMAIL=admin@example.org
|
|
||||||
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
environment:
|
|
||||||
# Point chatmail at the certs dumped by traefik-certs-dumper.
|
|
||||||
# The container's tls-cert-reload.path watches for changes.
|
|
||||||
TLS_EXTERNAL_CERT_AND_KEY: >-
|
|
||||||
/traefik-certs/${MAIL_DOMAIN}/certificate.crt
|
|
||||||
/traefik-certs/${MAIL_DOMAIN}/privatekey.key
|
|
||||||
volumes:
|
|
||||||
- traefik-certs:/traefik-certs:ro
|
|
||||||
depends_on:
|
|
||||||
- traefik-certs-dumper
|
|
||||||
labels:
|
|
||||||
- traefik.enable=true
|
|
||||||
- traefik.http.services.chatmail.loadbalancer.server.scheme=https
|
|
||||||
- traefik.http.services.chatmail.loadbalancer.server.port=443
|
|
||||||
- traefik.http.routers.chatmail.rule=Host(`${MAIL_DOMAIN}`) || Host(`mta-sts.${MAIL_DOMAIN}`) || Host(`www.${MAIL_DOMAIN}`)
|
|
||||||
- traefik.http.routers.chatmail.tls=true
|
|
||||||
- traefik.http.routers.chatmail.tls.certresolver=letsEncrypt
|
|
||||||
|
|
||||||
traefik:
|
|
||||||
image: traefik:v3.3
|
|
||||||
container_name: traefik
|
|
||||||
restart: unless-stopped
|
|
||||||
network_mode: host
|
|
||||||
command:
|
|
||||||
- "--configFile=/config.yaml"
|
|
||||||
- "--certificatesresolvers.letsEncrypt.acme.email=${ACME_EMAIL}"
|
|
||||||
volumes:
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- ./traefik/config.yaml:/config.yaml:ro
|
|
||||||
- ./traefik/data/acme.json:/acme.json
|
|
||||||
- ./traefik/dynamic-configs:/dynamic/conf:ro
|
|
||||||
|
|
||||||
traefik-certs-dumper:
|
|
||||||
image: ldez/traefik-certs-dumper:v2.10.0
|
|
||||||
restart: unless-stopped
|
|
||||||
depends_on:
|
|
||||||
- traefik
|
|
||||||
entrypoint: sh -c '
|
|
||||||
apk add openssl
|
|
||||||
&& while ! [ -e /data/acme.json ]
|
|
||||||
|| ! [ $$(jq ".[] | .Certificates | length" /data/acme.json | jq -s "add") != 0 ]; do
|
|
||||||
sleep 1;
|
|
||||||
done
|
|
||||||
&& traefik-certs-dumper file
|
|
||||||
--version v3 --watch --domain-subdir=true
|
|
||||||
--source /data/acme.json --dest /certs'
|
|
||||||
volumes:
|
|
||||||
- ./traefik/data/acme.json:/data/acme.json:ro
|
|
||||||
- traefik-certs:/certs
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
traefik-certs:
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
# Used by .github/workflows/docker-ci.yaml
|
|
||||||
# The GHCR image is set via CHATMAIL_IMAGE env var at deploy time.
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
image: ${CHATMAIL_IMAGE:-chatmail-relay:latest}
|
|
||||||
volumes:
|
|
||||||
- /srv/chatmail/chatmail.ini:/etc/chatmail/chatmail.ini
|
|
||||||
- /srv/chatmail/dkim:/etc/dkimkeys
|
|
||||||
- /srv/chatmail/certs:/var/lib/acme
|
|
||||||
environment:
|
|
||||||
TLS_EXTERNAL_CERT_AND_KEY: /var/lib/acme/live/${MAIL_DOMAIN}/fullchain /var/lib/acme/live/${MAIL_DOMAIN}/privkey
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
# Local overrides: copy to docker-compose.override.yaml in this directory.
|
|
||||||
# Compose automatically merges this with docker-compose.yaml.
|
|
||||||
#
|
|
||||||
# cp docker-compose.override.yaml.example docker-compose.override.yaml
|
|
||||||
#
|
|
||||||
# Volumes are APPENDED to the base file's volumes list, environment and other scalar keys are MERGED by key.
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
volumes:
|
|
||||||
## Data paths — bind-mount to host directories for easy access/backup.
|
|
||||||
|
|
||||||
# - ./data/dkim:/etc/dkimkeys
|
|
||||||
# - ./data/certs:/var/lib/acme
|
|
||||||
|
|
||||||
# - ./data/mail:/home/vmail
|
|
||||||
## Or mount from an existing bare-metal install.
|
|
||||||
# - /home/vmail:/home/vmail
|
|
||||||
|
|
||||||
## Mount your own chatmail.ini (skips auto-generation):
|
|
||||||
# - ./chatmail.ini:/etc/chatmail/chatmail.ini
|
|
||||||
|
|
||||||
## Custom website:
|
|
||||||
# - ./custom/www:/opt/chatmail-www
|
|
||||||
|
|
||||||
## Debug — mount scripts for live editing:
|
|
||||||
# - ./chatmail-init.sh:/chatmail-init.sh
|
|
||||||
# - ./entrypoint.sh:/entrypoint.sh
|
|
||||||
|
|
||||||
# environment:
|
|
||||||
## Mount certs (above) and set TLS_EXTERNAL_CERT_AND_KEY to in-container paths.
|
|
||||||
## A tls-cert-reload.path watcher inside the container reloads services
|
|
||||||
## when the cert file changes. However, inotify does not cross bind-mount
|
|
||||||
## boundaries, so host-side renewals (certbot, acmetool, etc.) must
|
|
||||||
## notify the container explicitly. Add this to your renewal hook:
|
|
||||||
##
|
|
||||||
## docker exec chatmail systemctl start tls-cert-reload.service
|
|
||||||
##
|
|
||||||
## Host acmetool (bare-metal migration): create mount above, and
|
|
||||||
## rsync -a /var/lib/acme/live data/certs
|
|
||||||
# TLS_EXTERNAL_CERT_AND_KEY: "/var/lib/acme/live/${MAIL_DOMAIN}/fullchain /var/lib/acme/live/${MAIL_DOMAIN}/privkey"
|
|
||||||
##
|
|
||||||
## (Untested) Traefik certs-dumper (see docker/docker-compose-traefik.yaml) - also add volume:
|
|
||||||
## - traefik-certs:/certs:ro
|
|
||||||
# TLS_EXTERNAL_CERT_AND_KEY: "/certs/${MAIL_DOMAIN}/certificate.crt /certs/${MAIL_DOMAIN}/privatekey.key"
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
# Base compose file — do not edit. Put customizations (data paths, extra
|
|
||||||
# volumes, env overrides) in docker-compose.override.yaml instead.
|
|
||||||
# See docker-compose.override.yaml.example in this directory for a starting point.
|
|
||||||
#
|
|
||||||
# Security notes: this container uses
|
|
||||||
# - network_mode:host chatmail needs many ports (25, 53, 80, 143, 443, 465,
|
|
||||||
# 587, 993, 3340, 8443) and needs to operate from the real IP, which bridging
|
|
||||||
# would make tricky
|
|
||||||
# - cgroup:host (required for systemd).
|
|
||||||
# Together these give the container near-host-level access. This is acceptable
|
|
||||||
# for a dedicated mail server, but be aware that the container can bind any
|
|
||||||
# port and see all host network traffic.
|
|
||||||
|
|
||||||
services:
|
|
||||||
chatmail:
|
|
||||||
build:
|
|
||||||
context: ../
|
|
||||||
dockerfile: docker/chatmail_relay.dockerfile
|
|
||||||
args:
|
|
||||||
GIT_HASH: ${GIT_HASH:-unknown}
|
|
||||||
image: chatmail-relay:latest
|
|
||||||
restart: unless-stopped
|
|
||||||
container_name: chatmail
|
|
||||||
# Required for systemd — use only one of the following:
|
|
||||||
cgroup: host # compose v2
|
|
||||||
# privileged: true # compose v1 (less restricted)
|
|
||||||
tty: true # required for logs
|
|
||||||
tmpfs: # required for systemd
|
|
||||||
- /tmp
|
|
||||||
- /run
|
|
||||||
- /run/lock
|
|
||||||
logging:
|
|
||||||
driver: none
|
|
||||||
environment:
|
|
||||||
MAIL_DOMAIN: $MAIL_DOMAIN
|
|
||||||
network_mode: "host"
|
|
||||||
volumes:
|
|
||||||
## system (required)
|
|
||||||
- /sys/fs/cgroup:/sys/fs/cgroup:rw
|
|
||||||
## data (defaults — override in docker-compose.override.yaml)
|
|
||||||
- mail:/home/vmail
|
|
||||||
- dkim:/etc/dkimkeys
|
|
||||||
- certs:/var/lib/acme
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
mail:
|
|
||||||
dkim:
|
|
||||||
certs:
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -eo pipefail
|
|
||||||
|
|
||||||
CHATMAIL_INIT_SERVICE_PATH="${CHATMAIL_INIT_SERVICE_PATH:-/lib/systemd/system/chatmail-init.service}"
|
|
||||||
|
|
||||||
env_vars="MAIL_DOMAIN CMDEPLOY_STAGES CHATMAIL_INI TLS_EXTERNAL_CERT_AND_KEY PATH"
|
|
||||||
sed -i "s|<envs_list>|$env_vars|g" "$CHATMAIL_INIT_SERVICE_PATH"
|
|
||||||
|
|
||||||
exec /lib/systemd/systemd "$@"
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
MAIL_DOMAIN=chat.example.com
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# returns 0 when chatmail-init succeeded and all expected services are running.
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
test -f /run/chatmail-init.done
|
|
||||||
|
|
||||||
# Core services
|
|
||||||
services="chatmail-metadata doveauth dovecot filtermail filtermail-incoming nginx postfix unbound"
|
|
||||||
|
|
||||||
# Optional services
|
|
||||||
for svc in iroh-relay turnserver; do
|
|
||||||
systemctl is-enabled "$svc" 2>/dev/null && services="$services $svc"
|
|
||||||
done
|
|
||||||
|
|
||||||
exec systemctl is-active $services
|
|
||||||
Reference in New Issue
Block a user