mirror of
https://github.com/chatmail/relay.git
synced 2026-08-10 18:40:51 +00:00
Compare commits
50 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 33c85ff58a | |||
| 6b872446e1 | |||
| f0fe7256b7 | |||
| ab14cc7319 | |||
| 53b5a8189b | |||
| dc8e0a34a2 | |||
| efc24fcdf3 | |||
| 9a9bda80b1 | |||
| 74f4721f2b | |||
| 14f829003d | |||
| 5da9aaeb37 | |||
| d068052f0c | |||
| fa5afadaaa | |||
| eb862f1645 | |||
| 6f3039509c | |||
| 504fb249a9 | |||
| deca1ef2a3 | |||
| 0323e757a9 | |||
| 752957dae8 | |||
| 3a553f2286 | |||
| cb1e4ff5bb | |||
| 8d7870db55 | |||
| 5a31ff475f | |||
| 8c18aea18e | |||
| ebf5a51964 | |||
| f596d4b56d | |||
| 8e3c18019b | |||
| 9da3f5c235 | |||
| 6def189d16 | |||
| 24612e9121 | |||
| a9dd9fe3e0 | |||
| aa846c3478 | |||
| 921080125f | |||
| d898f41064 | |||
| e9e012234b | |||
| bb40c5bb21 | |||
| a229f1bc45 | |||
| 4ba19b0031 | |||
| 5eab3a5a25 | |||
| 30729d9be0 | |||
| 4b04aae83b | |||
| 0eed92171c | |||
| a5b9a98baa | |||
| ab2d807084 | |||
| ce05b26c77 | |||
| 77ed93fb7a | |||
| 39d1ecaa03 | |||
| a266ffd060 | |||
| a47bb94143 | |||
| 43ae9fee5c |
@@ -20,9 +20,9 @@ concurrency:
|
||||
jobs:
|
||||
no-dns:
|
||||
name: LXC deploy and test
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
|
||||
with:
|
||||
cmlxc_version: v0.14.6
|
||||
cmlxc_version: main
|
||||
cmlxc_commands: |
|
||||
cmlxc init
|
||||
# single cmdeploy relay test
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
persist-credentials: false
|
||||
- name: download filtermail
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.7.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
|
||||
- name: run chatmaild tests
|
||||
working-directory: chatmaild
|
||||
run: pipx run tox
|
||||
@@ -57,9 +57,9 @@ jobs:
|
||||
|
||||
lxc-test:
|
||||
name: LXC deploy and test
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
|
||||
uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
|
||||
with:
|
||||
cmlxc_version: v0.14.6
|
||||
cmlxc_version: main
|
||||
cmlxc_commands: |
|
||||
cmlxc init
|
||||
# single cmdeploy relay test
|
||||
|
||||
@@ -8,7 +8,8 @@ name: Trigger Docker build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, j4n/dovecot-multidist]
|
||||
tags: ['[0-9]+.[0-9]+.[0-9]+']
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
mkdir -p "$HOME/.ssh"
|
||||
echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
|
||||
chmod 600 "$HOME/.ssh/key"
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:${STEPS_PREPARE_OUTPUTS_PRID}/"
|
||||
env:
|
||||
STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
|
||||
|
||||
|
||||
@@ -47,5 +47,5 @@ jobs:
|
||||
mkdir -p "$HOME/.ssh"
|
||||
echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
|
||||
chmod 600 "$HOME/.ssh/key"
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/chatmail.at/doc/relay/"
|
||||
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:"
|
||||
|
||||
|
||||
+98
-1
@@ -1,4 +1,101 @@
|
||||
# Changelog for chatmail deployment
|
||||
# Changelog for chatmail deployment
|
||||
|
||||
## [1.12.0] - 2026-07-31
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- [**breaking**] Introduce configurable system limits to reject new address creation and limit imap/smtp connections.
|
||||
Dovecot default connection limit lowered from 50k to 10k,
|
||||
Postfix default connection limit lowered from 5k to 1k,
|
||||
larger relays need to adjust their settings.
|
||||
|
||||
### Features
|
||||
|
||||
- Reduce maximal_queue_lifetime from 5d to 2d
|
||||
- Disable negative cache in unbound (#992)
|
||||
- *(mtail)* Add incoming_mailer_daemon_mail_count
|
||||
- *(postfix)* Disable processing of MIME headers
|
||||
- *(dovecot)* Advertise privacy_mail as admin contact, drop server comment
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Set relay restrictions per smtpd service with default reject
|
||||
- Reduce maxproc for filtermail-transport LMTP client to 500
|
||||
- Core 2.50.0 does not have delete_server_after config anymore.
|
||||
- Check if all required ports are available for filtermail (#983)
|
||||
- Always deploy unbound.conf.d/chatmail.conf (#993)
|
||||
- Expire empty directories (#994)
|
||||
- Crypt-r dependency was declared for wrong Python version
|
||||
- Always overwrite /etc/resolv.conf, even if it is a symbolic link
|
||||
- Pass kwargs to files.put()
|
||||
- List Iroh proxy endpoints used by 0.35 and 1.0, drop stale /relay/probe from earlier versions
|
||||
- Fix port discovery when ss -tulpn shows dovecot before stats
|
||||
|
||||
### Documentation
|
||||
|
||||
- Add scripts/initenv.sh to upgrade instructions
|
||||
- Update overview diagrams (#995)
|
||||
- *(overview)* Remove mermaid styles from 'Accepting and delivering mail' (#1009)
|
||||
- *(README.md)* Clarify security enforcement (#1011)
|
||||
|
||||
### Miscellaneous Tasks
|
||||
|
||||
- *(ci)* Auto-trigger docker build on release tag push
|
||||
- *(acmetool)* Update let's encrypt ToS link to 1.8
|
||||
- *(ci)* Update doc staging upload path
|
||||
- *(ci)* Fix docs upload path
|
||||
|
||||
### Refactor
|
||||
|
||||
- *(postfix)* Remove unused "filter" lmtp service
|
||||
- Install dns-root-data instead of using unbound-anchor
|
||||
- *(deps)* Remove domain-validator dependency
|
||||
|
||||
### Testing
|
||||
|
||||
- Set socket security for IMAP and SMTP to "TLS" in "dclogin"
|
||||
|
||||
## [1.11.0] - 2026-05-15
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- [**breaking**] Drop passthrough_sender and passthrough_recipients chatmail.ini options to eliminate one more source of unencrypted messages
|
||||
|
||||
### Features
|
||||
|
||||
- Use filtermail for delivery to remote MTAs
|
||||
- Expose metadata "maxsmtprecipients" value
|
||||
- Support setup without domain, with only an IPv4 address (#963)
|
||||
- *(doc/docker)* Introduce docker images in documentation
|
||||
- DKIM-sign bounce messages (mainly "user does not exist")
|
||||
- *(config)* Load default values from Config(), not chatmail.ini.f (#853)
|
||||
- Make turn_socket_path configurable, and cleanup tests and turnserver code.
|
||||
- Warn about any unused chatmail.ini parameter at the end of "cmdeploy run"
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Make www tests work with editable instead of just plain installs
|
||||
- Use path with no leading slash for mxdeliv
|
||||
- Increase filtermail-transport concurrency limit
|
||||
- Fix #972 by increasing file descriptors for filtermail
|
||||
- *(mtail)* Correct boot ordering and deploy restart logic
|
||||
- *(cmdeploy)* Stop and disable unbound-resolvconf
|
||||
- *(nginx)* Properly redirect www to mail_domain
|
||||
- *(dns)* Query correct NS if MNAME server is hidden (#954)
|
||||
- Legacy token metadata storage used list type, but if no new setmetadata happened, the user would not be notified at all.
|
||||
- *(logging)* Log all http requests to syslog
|
||||
|
||||
### Documentation
|
||||
|
||||
- Document how to upgrade to new version (#965)
|
||||
|
||||
### Other
|
||||
|
||||
- *(deps)* Upgrade to filtermail v0.6.4
|
||||
|
||||
### Refactor
|
||||
|
||||
- Introduce automated change-tracking across deployers
|
||||
|
||||
## 1.10.0 2026-04-30
|
||||
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
|
||||
# Chatmail relays for end-to-end encrypted email
|
||||
|
||||
Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for:
|
||||
Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for:
|
||||
|
||||
- **Zero State:** no private data or metadata collected, messages are auto-deleted, low disk usage
|
||||
|
||||
- **Instant/Realtime:** sub-second message delivery, realtime P2P
|
||||
streaming, privacy-preserving Push Notifications for Apple, Google, and Huawei;
|
||||
|
||||
- **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
|
||||
- **Security Enforcement**: Only connections with strict TLS are accepted;
|
||||
all messages must be correctly signed with DKIM and OpenPGP-encrypted with minimized metadata.
|
||||
There are experimental exceptions for no-DNS relays,
|
||||
which are allowed use self-signed TLS certificates
|
||||
and which do not need to DKIM-sign their messages.
|
||||
Unencrypted messages are allowed in neither case.
|
||||
|
||||
- **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
|
||||
depends on established IETF standards and protocols.
|
||||
|
||||
This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host.
|
||||
This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host.
|
||||
For getting started and more information please refer to the web version of this repositories' documentation at
|
||||
|
||||
[https://chatmail.at/doc/relay](https://chatmail.at/doc/relay)
|
||||
|
||||
+4
-6
@@ -1,15 +1,13 @@
|
||||
# Releasing a new version of chatmail relay
|
||||
|
||||
For example, to release version 1.9.0 of chatmail relay, do the following steps.
|
||||
For example, to release version 1.13.0 of chatmail relay, do the following steps.
|
||||
|
||||
1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`.
|
||||
1. Update the changelog: `git cliff --unreleased --tag 1.13.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.13.0 -p CHANGELOG.md`.
|
||||
|
||||
2. Open the changelog in the editor, edit it if required.
|
||||
|
||||
3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`.
|
||||
|
||||
3. Tag the release: `git tag --annotate 1.9.0`.
|
||||
4. Open a PR with the new commit, merge it to main after review.
|
||||
|
||||
4. Push the release tag: `git push origin 1.9.0`.
|
||||
|
||||
5. Create a GitHub release: `gh release create 1.9.0`.
|
||||
5. In the web interface, create a GitHub release, tell it to create a new tag.
|
||||
|
||||
@@ -8,9 +8,9 @@ version = "0.3"
|
||||
dependencies = [
|
||||
"iniconfig",
|
||||
"filelock",
|
||||
"psutil",
|
||||
"requests",
|
||||
"crypt-r >= 3.13.1 ; python_version >= '3.11'",
|
||||
"domain-validator",
|
||||
"crypt-r >= 3.13.1 ; python_version >= '3.13'",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
|
||||
@@ -2,7 +2,6 @@ import ipaddress
|
||||
from pathlib import Path
|
||||
|
||||
import iniconfig
|
||||
from domain_validator import DomainValidator
|
||||
|
||||
from chatmaild.user import User
|
||||
|
||||
@@ -16,7 +15,8 @@ def read_config(inipath):
|
||||
class Config:
|
||||
def __init__(self, inipath, params):
|
||||
self._inipath = inipath
|
||||
raw_domain = params["mail_domain"]
|
||||
params = dict(params)
|
||||
raw_domain = params.pop("mail_domain")
|
||||
self.mail_domain_bare = raw_domain
|
||||
|
||||
if is_valid_ipv4(raw_domain):
|
||||
@@ -24,65 +24,73 @@ class Config:
|
||||
self.mail_domain = f"[{raw_domain}]"
|
||||
self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
|
||||
else:
|
||||
DomainValidator().validate_domain_re(raw_domain)
|
||||
self.ipv4_relay = None
|
||||
self.mail_domain = raw_domain
|
||||
self.postfix_myhostname = raw_domain
|
||||
|
||||
self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
|
||||
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
|
||||
self.max_mailbox_size = params.get("max_mailbox_size", "500M")
|
||||
self.max_message_size = int(params.get("max_message_size", 31457280))
|
||||
self.delete_mails_after = params.get("delete_mails_after", "20")
|
||||
self.delete_large_after = params.get("delete_large_after", "7")
|
||||
self.max_user_send_per_minute = int(params.pop("max_user_send_per_minute", 60))
|
||||
self.max_user_send_burst_size = int(params.pop("max_user_send_burst_size", 10))
|
||||
self.max_mailbox_size = params.pop("max_mailbox_size", "500M")
|
||||
self.max_message_size = int(params.pop("max_message_size", 31457280))
|
||||
self.delete_mails_after = params.pop("delete_mails_after", "20")
|
||||
self.delete_large_after = params.pop("delete_large_after", "7")
|
||||
self.delete_inactive_users_after = int(
|
||||
params.get("delete_inactive_users_after", 90)
|
||||
params.pop("delete_inactive_users_after", 90)
|
||||
)
|
||||
self.username_min_length = int(params.get("username_min_length", 9))
|
||||
self.username_max_length = int(params.get("username_max_length", 9))
|
||||
self.password_min_length = int(params.get("password_min_length", 9))
|
||||
self.passthrough_senders = params.get("passthrough_senders", "").split()
|
||||
self.passthrough_recipients = params.get("passthrough_recipients", "").split()
|
||||
self.www_folder = params.get("www_folder", "")
|
||||
self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
|
||||
self.username_min_length = int(params.pop("username_min_length", 9))
|
||||
self.username_max_length = int(params.pop("username_max_length", 9))
|
||||
self.password_min_length = int(params.pop("password_min_length", 9))
|
||||
self.www_folder = params.pop("www_folder", "")
|
||||
self.filtermail_smtp_port = int(params.pop("filtermail_smtp_port", "10080"))
|
||||
self.filtermail_smtp_port_incoming = int(
|
||||
params.get("filtermail_smtp_port_incoming", "10081")
|
||||
params.pop("filtermail_smtp_port_incoming", "10081")
|
||||
)
|
||||
self.filtermail_http_port_incoming = int(
|
||||
params.get("filtermail_http_port_incoming", "10082")
|
||||
params.pop("filtermail_http_port_incoming", "10082")
|
||||
)
|
||||
self.filtermail_lmtp_port_transport = int(
|
||||
params.get("filtermail_lmtp_port_transport", "10083")
|
||||
params.pop("filtermail_lmtp_port_transport", "10083")
|
||||
)
|
||||
self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
|
||||
self.postfix_reinject_port = int(params.pop("postfix_reinject_port", "10025"))
|
||||
self.postfix_reinject_port_incoming = int(
|
||||
params.get("postfix_reinject_port_incoming", "10026")
|
||||
params.pop("postfix_reinject_port_incoming", "10026")
|
||||
)
|
||||
self.mtail_address = params.get("mtail_address")
|
||||
self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
|
||||
self.acme_email = params.get("acme_email", "")
|
||||
self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
|
||||
self.imap_compress = params.get("imap_compress", "false").lower() == "true"
|
||||
self.turn_socket_path = params.get(
|
||||
self.mtail_address = params.pop("mtail_address", None)
|
||||
self.disable_ipv6 = params.pop("disable_ipv6", "false").lower() == "true"
|
||||
self.acme_email = params.pop("acme_email", "")
|
||||
self.imap_rawlog = params.pop("imap_rawlog", "false").lower() == "true"
|
||||
self.imap_compress = params.pop("imap_compress", "false").lower() == "true"
|
||||
self.turn_socket_path = params.pop(
|
||||
"turn_socket_path", "/run/chatmail-turn/turn.socket"
|
||||
)
|
||||
if "iroh_relay" not in params:
|
||||
iroh_relay = params.pop("iroh_relay", None)
|
||||
if iroh_relay is None:
|
||||
self.iroh_relay = "https://" + raw_domain
|
||||
self.enable_iroh_relay = True
|
||||
else:
|
||||
self.iroh_relay = params["iroh_relay"].strip()
|
||||
self.iroh_relay = iroh_relay.strip()
|
||||
self.enable_iroh_relay = False
|
||||
self.privacy_postal = params.get("privacy_postal")
|
||||
self.privacy_mail = params.get("privacy_mail")
|
||||
self.privacy_pdo = params.get("privacy_pdo")
|
||||
self.privacy_supervisor = params.get("privacy_supervisor")
|
||||
self.privacy_postal = params.pop("privacy_postal", None)
|
||||
self.privacy_mail = params.pop("privacy_mail", None)
|
||||
self.privacy_pdo = params.pop("privacy_pdo", None)
|
||||
self.privacy_supervisor = params.pop("privacy_supervisor", None)
|
||||
|
||||
self.max_load_1m = float(params.pop("max_load_1m", 5))
|
||||
self.min_available_memory_mb = parse_size_mb(
|
||||
params.pop("min_available_memory", "200M")
|
||||
)
|
||||
self.min_free_disk_space_mb = parse_size_mb(
|
||||
params.pop("min_free_disk_space", "1G")
|
||||
)
|
||||
self.max_imap_connections = int(params.pop("max_imap_connections", 10000))
|
||||
self.max_smtp_connections = int(params.pop("max_smtp_connections", 1000))
|
||||
|
||||
# TLS certificate management.
|
||||
# If tls_external_cert_and_key is set, use externally managed certs.
|
||||
# Otherwise derived from the domain name:
|
||||
# - Domains starting with "_" use self-signed certificates
|
||||
# - All other domains use ACME.
|
||||
external = params.get("tls_external_cert_and_key", "").strip()
|
||||
external = params.pop("tls_external_cert_and_key", "").strip()
|
||||
|
||||
if external:
|
||||
parts = external.split()
|
||||
@@ -103,11 +111,12 @@ class Config:
|
||||
self.tls_key_path = f"/var/lib/acme/live/{raw_domain}/privkey"
|
||||
|
||||
# deprecated option
|
||||
mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{raw_domain}")
|
||||
mbdir = params.pop("mailboxes_dir", f"/home/vmail/mail/{raw_domain}")
|
||||
self.mailboxes_dir = Path(mbdir.strip())
|
||||
|
||||
# old unused option (except for first migration from sqlite to maildir store)
|
||||
self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
|
||||
self.passdb_path = Path(params.pop("passdb_path", "/home/vmail/passdb.sqlite"))
|
||||
self._unused_keys = list(params)
|
||||
|
||||
@property
|
||||
def max_mailbox_size_mb(self):
|
||||
|
||||
@@ -14,6 +14,7 @@ except ImportError:
|
||||
from .config import Config, read_config
|
||||
from .dictproxy import DictProxy
|
||||
from .migrate_db import migrate_from_db_to_maildir
|
||||
from .syslimits import has_sufficient_resources
|
||||
|
||||
NOCREATE_FILE = "/etc/chatmail-nocreate"
|
||||
VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
|
||||
@@ -147,6 +148,8 @@ class AuthDictProxy(DictProxy):
|
||||
return userdata
|
||||
if not is_allowed_to_create(self.config, addr, cleartext_password):
|
||||
return
|
||||
if not has_sufficient_resources(self.config):
|
||||
return
|
||||
|
||||
lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
|
||||
with lock:
|
||||
|
||||
@@ -168,6 +168,16 @@ class Expiry:
|
||||
if mbox.last_login and mbox.last_login < cutoff_without_login:
|
||||
self.remove_mailbox(mbox.basedir)
|
||||
return
|
||||
elif mbox.last_login is None:
|
||||
try:
|
||||
if not self.dry:
|
||||
os.rmdir(mbox.basedir)
|
||||
self.del_mboxes += 1
|
||||
except OSError:
|
||||
print_info(
|
||||
f"Skipped deleting {mbox.basedir}, doesn't have last_login but isn't empty"
|
||||
)
|
||||
return
|
||||
|
||||
mboxname = os.path.basename(mbox.basedir)
|
||||
if self.verbose:
|
||||
|
||||
@@ -42,12 +42,32 @@ mail_domain = {mail_domain}
|
||||
# minimum length a password must have
|
||||
#password_min_length = 9
|
||||
|
||||
# list of chatmail addresses which can send outbound un-encrypted mail
|
||||
#passthrough_senders =
|
||||
#
|
||||
# System resource limits
|
||||
#
|
||||
|
||||
# list of e-mail recipients for which to accept outbound un-encrypted mails
|
||||
# (space-separated, item may start with "@" to whitelist whole recipient domains)
|
||||
#passthrough_recipients =
|
||||
# The following three limits refuse creation of new addresses
|
||||
# while existing addresses keep working.
|
||||
# Rejections are logged by the doveauth service.
|
||||
|
||||
# Maximum 1-minute load average, as reported by "uptime";
|
||||
# it counts processes waiting for disk I/O as well as for CPU.
|
||||
#max_load_1m = 5
|
||||
|
||||
# Minimum memory available without swapping.
|
||||
#min_available_memory = 200M
|
||||
|
||||
# Minimum free disk space on the file system holding the mailboxes.
|
||||
#min_free_disk_space = 1G
|
||||
|
||||
# Maximum number of concurrent IMAP connections
|
||||
# (the Dovecot imap process limit).
|
||||
#max_imap_connections = 10000
|
||||
|
||||
# Maximum number of concurrent SMTP connections
|
||||
# on each of the submission and smtps ports (the Postfix process limit).
|
||||
# A single client IP may use up to a fifth of this.
|
||||
#max_smtp_connections = 1000
|
||||
|
||||
# Use externally managed TLS certificates instead of built-in acmetool.
|
||||
# Paths refer to files on the deployment server (not the build machine).
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Detect whether the system is at its limits."""
|
||||
|
||||
import logging
|
||||
|
||||
import psutil
|
||||
|
||||
MB = 1024 * 1024
|
||||
|
||||
|
||||
def read_value(getter):
|
||||
try:
|
||||
return getter()
|
||||
except Exception as e:
|
||||
logging.warning("ignoring unreadable system limit: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def has_sufficient_resources(config):
|
||||
"""Return False if load, memory or disk exceeds a configured limit."""
|
||||
load = read_value(lambda: psutil.getloadavg()[0])
|
||||
mem = read_value(lambda: psutil.virtual_memory().available // MB)
|
||||
disk = read_value(lambda: psutil.disk_usage(str(config.mailboxes_dir)).free // MB)
|
||||
if load is not None and load > config.max_load_1m:
|
||||
msg = f"load avg {load:.2f} > {config.max_load_1m:.2f}"
|
||||
elif mem is not None and mem < config.min_available_memory_mb:
|
||||
msg = f"available memory {mem}MB < {config.min_available_memory_mb}MB"
|
||||
elif disk is not None and disk < config.min_free_disk_space_mb:
|
||||
msg = f"free disk {disk}MB < {config.min_free_disk_space_mb}MB"
|
||||
else:
|
||||
return True
|
||||
logging.warning("registration rejected: %s", msg)
|
||||
return False
|
||||
@@ -20,6 +20,10 @@ def make_config(tmp_path):
|
||||
basedir.mkdir(parents=True, exist_ok=True)
|
||||
overrides = settings.copy() if settings else {}
|
||||
overrides["mailboxes_dir"] = str(basedir)
|
||||
# permissive resource limits so tests never depend on host load/memory/disk
|
||||
overrides.setdefault("max_load_1m", "99999")
|
||||
overrides.setdefault("min_available_memory", "0")
|
||||
overrides.setdefault("min_free_disk_space", "0")
|
||||
write_initial_config(inipath, mail_domain, overrides=overrides)
|
||||
return read_config(inipath)
|
||||
|
||||
|
||||
@@ -45,8 +45,14 @@ def test_read_config_basic_using_defaults(tmp_path, maildomain):
|
||||
assert example_config.username_min_length == 9
|
||||
assert example_config.username_max_length == 9
|
||||
assert example_config.password_min_length == 9
|
||||
assert example_config.passthrough_recipients == []
|
||||
assert example_config.passthrough_senders == []
|
||||
assert example_config.max_imap_connections == 10000
|
||||
assert example_config.max_smtp_connections == 1000
|
||||
assert example_config._unused_keys == []
|
||||
|
||||
|
||||
def test_config_unused_keys(make_config):
|
||||
config = make_config("chat.example.org", {"passthrough_senders": "x@y.org"})
|
||||
assert config._unused_keys == ["passthrough_senders"]
|
||||
|
||||
|
||||
def test_config_userstate_paths(make_config, tmp_path):
|
||||
|
||||
@@ -202,3 +202,17 @@ def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
|
||||
res = results.get()
|
||||
if res is not None:
|
||||
pytest.fail(f"concurrent lookup failed\n{res}")
|
||||
|
||||
|
||||
def test_insufficient_resources_block_creation_not_existing_logins(
|
||||
dictproxy, gencreds, monkeypatch
|
||||
):
|
||||
addr, password = gencreds()
|
||||
assert dictproxy.lookup_passdb(addr, password)
|
||||
|
||||
monkeypatch.setattr(
|
||||
chatmaild.doveauth, "has_sufficient_resources", lambda config: False
|
||||
)
|
||||
newaddr, newpassword = gencreds()
|
||||
assert not dictproxy.lookup_passdb(newaddr, newpassword)
|
||||
assert dictproxy.lookup_passdb(addr, password)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import itertools
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from fnmatch import fnmatch
|
||||
@@ -9,6 +10,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from chatmaild.expire import (
|
||||
Expiry,
|
||||
FileEntry,
|
||||
MailboxStat,
|
||||
expire_to_target,
|
||||
@@ -104,6 +106,32 @@ def test_stats_mailbox(mbox1):
|
||||
assert mbox3.last_login is None
|
||||
|
||||
|
||||
def test_mbox_without_password(mbox1, example_config, capsys):
|
||||
password = Path(mbox1.basedir).joinpath("password")
|
||||
os.remove(password)
|
||||
mbox_rescan = MailboxStat(mbox1.basedir)
|
||||
assert mbox_rescan.last_login is None
|
||||
|
||||
exp = Expiry(
|
||||
example_config, dry=False, now=datetime.now().timestamp(), verbose=False
|
||||
)
|
||||
exp.process_mailbox_stat(mbox_rescan)
|
||||
out, err = capsys.readouterr()
|
||||
assert "doesn't have last_login but isn't empty" in err
|
||||
assert os.path.isdir(mbox_rescan.basedir)
|
||||
|
||||
for entry in os.scandir(mbox_rescan.basedir):
|
||||
if os.path.isdir(entry):
|
||||
shutil.rmtree(entry)
|
||||
else:
|
||||
os.remove(entry)
|
||||
|
||||
exp.process_mailbox_stat(mbox_rescan)
|
||||
out, err = capsys.readouterr()
|
||||
assert "doesn't have last_login but isn't empty" not in err
|
||||
assert not os.path.isdir(mbox_rescan.basedir)
|
||||
|
||||
|
||||
def test_report_no_mailboxes(example_config):
|
||||
args = (str(example_config._inipath),)
|
||||
report_main(args)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import shutil
|
||||
|
||||
import psutil
|
||||
|
||||
from chatmaild.syslimits import has_sufficient_resources
|
||||
|
||||
PERMISSIVE = {
|
||||
"max_load_1m": "99999",
|
||||
"min_available_memory": "0",
|
||||
"min_free_disk_space": "0",
|
||||
}
|
||||
|
||||
|
||||
def test_rejects_constrained_system(make_config, caplog):
|
||||
assert has_sufficient_resources(make_config("chat.example.org", PERMISSIVE))
|
||||
for settings in (
|
||||
{"max_load_1m": "-1.0"},
|
||||
{"min_available_memory": "99999999G"},
|
||||
{"min_free_disk_space": "99999999G"},
|
||||
):
|
||||
config = make_config("chat.example.org", PERMISSIVE | settings)
|
||||
caplog.clear()
|
||||
assert not has_sufficient_resources(config), settings
|
||||
assert "registration rejected" in caplog.text
|
||||
|
||||
|
||||
def test_unreadable_disk_does_not_reject(make_config, caplog):
|
||||
config = make_config(
|
||||
"chat.example.org", PERMISSIVE | {"min_free_disk_space": "99999999G"}
|
||||
)
|
||||
shutil.rmtree(config.mailboxes_dir)
|
||||
assert has_sufficient_resources(config)
|
||||
assert "ignoring" in caplog.text
|
||||
|
||||
|
||||
def test_one_unreadable_value_keeps_other_checks(make_config, monkeypatch, caplog):
|
||||
def raise_error(*args):
|
||||
raise psutil.Error("dud")
|
||||
|
||||
monkeypatch.setattr(psutil, "getloadavg", raise_error)
|
||||
config = make_config(
|
||||
"chat.example.org", PERMISSIVE | {"min_free_disk_space": "99999999G"}
|
||||
)
|
||||
assert not has_sufficient_resources(config)
|
||||
assert "ignoring" in caplog.text
|
||||
@@ -1,2 +1,2 @@
|
||||
"acme-enter-email": "{{ email }}"
|
||||
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true
|
||||
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.8-July-06-2026.pdf": true
|
||||
|
||||
@@ -166,7 +166,7 @@ class Deployer:
|
||||
return self.put_template(src, dest, **kwargs)
|
||||
return self.put_file(src, dest)
|
||||
|
||||
def put_file(self, src, dest, mode="644"):
|
||||
def put_file(self, src, dest, mode="644", **kwargs):
|
||||
if isinstance(src, str):
|
||||
src = get_resource(src)
|
||||
res = files.put(
|
||||
@@ -176,6 +176,7 @@ class Deployer:
|
||||
user="root",
|
||||
group="root",
|
||||
mode=mode,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return self._update_restart_signals(dest, res)
|
||||
|
||||
@@ -84,6 +84,15 @@ def run_cmd_options(parser):
|
||||
add_ssh_host_option(parser)
|
||||
|
||||
|
||||
def _warn_unused_settings(unused_keys, out):
|
||||
if unused_keys:
|
||||
names = ", ".join(unused_keys)
|
||||
out.red(
|
||||
f"WARNING: chatmail.ini contains settings that have no effect: {names}\n"
|
||||
"Please remove them from chatmail.ini."
|
||||
)
|
||||
|
||||
|
||||
def run_cmd(args, out):
|
||||
"""Deploy chatmail services on the remote server."""
|
||||
|
||||
@@ -125,6 +134,7 @@ def run_cmd(args, out):
|
||||
out.green("Deploy completed.")
|
||||
else:
|
||||
out.green("Deploy completed, call `cmdeploy dns` next.")
|
||||
_warn_unused_settings(args.config._unused_keys, out)
|
||||
return 0
|
||||
except subprocess.CalledProcessError:
|
||||
out.red("Deploy failed")
|
||||
|
||||
@@ -98,6 +98,23 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
|
||||
dest=remote_dist_file,
|
||||
)
|
||||
|
||||
# Remove venv if its Python major.minor doesn't match the system Python
|
||||
server.shell(
|
||||
name="remove stale chatmaild venv if python version changed",
|
||||
commands=[
|
||||
"\n".join(
|
||||
[
|
||||
r"re='[0-9]+\.[0-9]+'", # major.minor out of 'Python X.Y.Z'
|
||||
'sys_version=$(python3 --version 2>/dev/null | grep -oE "$re")',
|
||||
f'venv_version=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
|
||||
# an empty sys_version means we could not tell: keep the venv
|
||||
f'[ -z "$sys_version" ] || [ "$sys_version" = "$venv_version" ] '
|
||||
f"|| rm -rf {remote_venv_dir}",
|
||||
]
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
pip.virtualenv(
|
||||
name=f"chatmaild virtualenv {remote_venv_dir}",
|
||||
path=remote_venv_dir,
|
||||
@@ -138,9 +155,22 @@ class UnboundDeployer(Deployer):
|
||||
# On an IPv4-only system, if unbound is started but not configured,
|
||||
# it causes subsequent steps to fail to resolve hosts.
|
||||
with blocked_service_startup():
|
||||
# dns-root-data is an optional package
|
||||
# that contains /usr/share/dns/root.key
|
||||
#
|
||||
# This file is copied into /var/lib/unbound/root.key
|
||||
# at the start of "unbound" systemd unit
|
||||
# by /usr/libexec/unbound-helper shell script
|
||||
# from the "unbound" package as of version 1.17.1-2+deb12u4
|
||||
#
|
||||
# The same /var/lib/unbound/root.key can be retrieved directly
|
||||
# following the procedure from
|
||||
# <https://www.rfc-editor.org/info/rfc7958/#section-3.1>
|
||||
# with "unbound-anchor -a /var/lib/unbound/root.key"
|
||||
# We don't install and use "unbound-anchor".
|
||||
apt.packages(
|
||||
name="Install unbound",
|
||||
packages=["unbound", "unbound-anchor", "dnsutils"],
|
||||
packages=["unbound", "dns-root-data", "dnsutils"],
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
@@ -164,23 +194,16 @@ class UnboundDeployer(Deployer):
|
||||
self.put_file(
|
||||
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
|
||||
dest="/etc/resolv.conf",
|
||||
force=True,
|
||||
)
|
||||
server.shell(
|
||||
name="Generate root keys for validating DNSSEC",
|
||||
commands=[
|
||||
"unbound-anchor -a /var/lib/unbound/root.key || true",
|
||||
],
|
||||
self.ensure_directory(
|
||||
path="/etc/unbound/unbound.conf.d",
|
||||
)
|
||||
self.put_template(
|
||||
"unbound/unbound.conf.j2",
|
||||
"/etc/unbound/unbound.conf.d/chatmail.conf",
|
||||
disable_ipv6=self.config.disable_ipv6,
|
||||
)
|
||||
if self.config.disable_ipv6:
|
||||
self.ensure_directory(
|
||||
path="/etc/unbound/unbound.conf.d",
|
||||
)
|
||||
self.put_template(
|
||||
"unbound/unbound.conf.j2",
|
||||
"/etc/unbound/unbound.conf.d/chatmail.conf",
|
||||
)
|
||||
else:
|
||||
self.remove_file("/etc/unbound/unbound.conf.d/chatmail.conf")
|
||||
|
||||
def activate(self):
|
||||
server.shell(
|
||||
@@ -247,6 +270,13 @@ class LegacyRemoveDeployer(Deployer):
|
||||
def install(self):
|
||||
apt.packages(name="Remove rspamd", packages="rspamd", present=False)
|
||||
|
||||
# unbound-anchor was used to download /var/lib/unbound/root.key
|
||||
# It is replaced by dns-root-data which contains /usr/share/dns/root.key.
|
||||
# unbound systemd unit copies /usr/share/dns/root.key
|
||||
# into /var/lib/unbound/root.key automatically on start
|
||||
# as long as /usr/share/dns/root.key is present.
|
||||
apt.packages(name="Remove unbound-anchor", packages="unbound-anchor", present=False)
|
||||
|
||||
# remove historic expunge script
|
||||
# which is now implemented through a systemd timer (chatmail-expire)
|
||||
self.remove_file("/etc/cron.d/expunge")
|
||||
@@ -357,6 +387,8 @@ class ChatmailVenvDeployer(Deployer):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.units = (
|
||||
# doveauth must restart when chatmaild/ini file changes
|
||||
"doveauth",
|
||||
"chatmail-metadata",
|
||||
"lastlogin",
|
||||
"chatmail-expire",
|
||||
@@ -391,6 +423,12 @@ class ChatmailDeployer(Deployer):
|
||||
src=BytesIO(b'APT::Install-Recommends "false";\n'),
|
||||
dest="/etc/apt/apt.conf.d/00InstallRecommends",
|
||||
)
|
||||
# Pin dovecot-* to priority -1 before any apt operation, apt should
|
||||
# never manage dovecot as our version might be lower than the distro's.
|
||||
self.put_file(
|
||||
src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
|
||||
dest="/etc/apt/preferences.d/pin-dovecot",
|
||||
)
|
||||
apt.update(name="apt update", cache_time=24 * 3600)
|
||||
apt.upgrade(name="upgrade apt packages", auto_remove=True)
|
||||
|
||||
@@ -505,15 +543,17 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
||||
("nginx", 443),
|
||||
(["master", "smtpd"], 465),
|
||||
(["master", "smtpd"], 587),
|
||||
(["imap-login", "dovecot"], 993),
|
||||
(["dovecot", "imap-login"], 993),
|
||||
("iroh-relay", 3340),
|
||||
("mtail", 3903),
|
||||
("stats", 3904),
|
||||
(["dovecot", "stats"], 3904),
|
||||
("nginx", 8443),
|
||||
(["master", "smtpd"], config.postfix_reinject_port),
|
||||
(["master", "smtpd"], config.postfix_reinject_port_incoming),
|
||||
("filtermail", config.filtermail_smtp_port),
|
||||
("filtermail", config.filtermail_smtp_port_incoming),
|
||||
("filtermail", config.filtermail_http_port_incoming),
|
||||
("filtermail", config.filtermail_lmtp_port_transport),
|
||||
]
|
||||
for service, port in port_services:
|
||||
print(f"Checking if port {port} is available for {service}...")
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import io
|
||||
import urllib.request
|
||||
|
||||
from chatmaild.config import Config
|
||||
from pyinfra import host
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
from pyinfra.facts.server import Arch, Command, Sysctl
|
||||
from pyinfra.operations import apt, files, server
|
||||
from pyinfra.operations import files, server
|
||||
|
||||
from cmdeploy.basedeploy import (
|
||||
Deployer,
|
||||
@@ -15,16 +14,31 @@ from cmdeploy.basedeploy import (
|
||||
is_in_container,
|
||||
)
|
||||
|
||||
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
|
||||
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
|
||||
# distro-neutral base version, as committed in chatmail/dovecot debian/changelog
|
||||
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3+chatmail2"
|
||||
|
||||
VERSION_ID_CMD = "grep '^VERSION_ID=' /etc/os-release"
|
||||
|
||||
|
||||
def _stamped_version(deb_release: int) -> str:
|
||||
"""Version as built, including the per-distro suffix stamped by
|
||||
chatmail/dovecot CI into package version and filename."""
|
||||
return f"{DOVECOT_ARCHIVE_VERSION}+deb{deb_release}u1"
|
||||
|
||||
|
||||
DOVECOT_SHA256 = {
|
||||
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
|
||||
("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
|
||||
("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
|
||||
("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
|
||||
("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
|
||||
("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
|
||||
("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e",
|
||||
("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109",
|
||||
("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0",
|
||||
("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3",
|
||||
("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea",
|
||||
("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c",
|
||||
("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5",
|
||||
("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe",
|
||||
("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab",
|
||||
("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908",
|
||||
("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418",
|
||||
("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d",
|
||||
}
|
||||
|
||||
|
||||
@@ -34,38 +48,34 @@ class DovecotDeployer(Deployer):
|
||||
def __init__(self, config, disable_mail):
|
||||
self.config = config
|
||||
self.disable_mail = disable_mail
|
||||
self.units = ["doveauth"]
|
||||
self.units = []
|
||||
|
||||
def install(self):
|
||||
arch = host.get_fact(Arch)
|
||||
deb_release = _parse_version_id(host.get_fact(Command, VERSION_ID_CMD))
|
||||
with blocked_service_startup():
|
||||
debs = []
|
||||
for pkg in ("core", "imapd", "lmtpd"):
|
||||
deb, changed = _download_dovecot_package(pkg, arch)
|
||||
deb, changed = _download_dovecot_package(pkg, arch, deb_release)
|
||||
self.need_restart |= changed
|
||||
if deb:
|
||||
debs.append(deb)
|
||||
if debs:
|
||||
deb_list = " ".join(debs)
|
||||
# First dpkg may fail on missing dependencies (stderr suppressed);
|
||||
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
|
||||
# apt-get install with local .deb paths resolves depends
|
||||
# against the configured repos (e.g. pulls libwrap0),
|
||||
# The pin file written earlier by ChatmailDeployer prevents apt
|
||||
# from installing a 'wrong' version
|
||||
server.shell(
|
||||
name="Install dovecot packages",
|
||||
commands=[
|
||||
f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
|
||||
f"dpkg --force-confdef --force-confold -i {deb_list}",
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get install -y "
|
||||
'-o Dpkg::Options::="--force-confdef" '
|
||||
'-o Dpkg::Options::="--force-confold" '
|
||||
f"--allow-downgrades {deb_list}",
|
||||
],
|
||||
)
|
||||
self.need_restart = True
|
||||
self.put_file(
|
||||
src=io.StringIO(
|
||||
"Package: dovecot-*\n"
|
||||
"Pin: version *\n"
|
||||
"Pin-Priority: -1\n"
|
||||
),
|
||||
dest="/etc/apt/preferences.d/pin-dovecot",
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
configure_remote_units(self, self.config.mail_domain_bare, self.units)
|
||||
@@ -78,7 +88,7 @@ class DovecotDeployer(Deployer):
|
||||
if not self.disable_mail and not self.need_restart:
|
||||
stale = host.get_fact(
|
||||
Command,
|
||||
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
|
||||
"pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
|
||||
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
|
||||
" && echo STALE || true",
|
||||
)
|
||||
@@ -93,6 +103,15 @@ class DovecotDeployer(Deployer):
|
||||
)
|
||||
|
||||
|
||||
def _parse_version_id(version_line: str) -> int:
|
||||
"""Debian major release from an /etc/os-release VERSION_ID line."""
|
||||
_, _, raw = (version_line or "").strip().partition("=")
|
||||
try:
|
||||
return int(raw.strip('"'))
|
||||
except ValueError:
|
||||
raise ValueError(f"cannot determine Debian release from {version_line!r}")
|
||||
|
||||
|
||||
def _pick_url(primary, fallback):
|
||||
try:
|
||||
req = urllib.request.Request(primary, method="HEAD")
|
||||
@@ -102,27 +121,36 @@ def _pick_url(primary, fallback):
|
||||
return fallback
|
||||
|
||||
|
||||
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
|
||||
def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]:
|
||||
"""Download a dovecot .deb if needed, return (path, changed)."""
|
||||
arch = "amd64" if arch == "x86_64" else arch
|
||||
arch = "arm64" if arch == "aarch64" else arch
|
||||
|
||||
pkg_name = f"dovecot-{package}"
|
||||
sha256 = DOVECOT_SHA256.get((package, arch))
|
||||
if sha256 is None:
|
||||
op = apt.packages(packages=[pkg_name])
|
||||
return None, bool(getattr(op, "changed", False))
|
||||
try:
|
||||
# never fall back to the distro package: it is pinned to -1 and would
|
||||
# in any case be a version we did not build and do not support
|
||||
sha256 = DOVECOT_SHA256[(arch, deb_release, package)]
|
||||
except KeyError:
|
||||
raise ValueError(f"no dovecot build for {pkg_name} on deb{deb_release}/{arch}")
|
||||
|
||||
stamped_version = _stamped_version(deb_release)
|
||||
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
|
||||
if DOVECOT_PACKAGE_VERSION in installed_versions:
|
||||
if f"1:{stamped_version}" in installed_versions:
|
||||
return None, False
|
||||
|
||||
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
|
||||
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
|
||||
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
|
||||
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
|
||||
# Primary URL: flat structure with distro suffix in filename
|
||||
primary_deb = f"{pkg_name}_{stamped_version}_{arch}.deb"
|
||||
primary_url = f"https://download.delta.chat/dovecot/{primary_deb}"
|
||||
# GitHub release files: escaped + in filename; the release tag stays
|
||||
# distro-neutral, both distros ship in one combined release
|
||||
tag_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
|
||||
fallback_deb = f"{pkg_name}_{stamped_version.replace('+', '%2B')}_{arch}.deb"
|
||||
fallback_url = (
|
||||
f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{tag_version}/{fallback_deb}"
|
||||
)
|
||||
url = _pick_url(primary_url, fallback_url)
|
||||
deb_filename = f"/root/{deb_base}"
|
||||
deb_filename = f"/root/{primary_deb}"
|
||||
|
||||
files.download(
|
||||
name=f"Download {pkg_name}",
|
||||
@@ -134,6 +162,7 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
|
||||
|
||||
return deb_filename, True
|
||||
|
||||
|
||||
def _configure_dovecot(deployer, config: Config, debug: bool = False):
|
||||
"""Configures Dovecot IMAP server."""
|
||||
deployer.put_template(
|
||||
@@ -144,9 +173,7 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
|
||||
disable_ipv6=config.disable_ipv6,
|
||||
)
|
||||
deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
|
||||
deployer.put_file(
|
||||
"dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
|
||||
)
|
||||
deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
|
||||
|
||||
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
|
||||
# it is recommended to set the following inotify limits
|
||||
|
||||
@@ -37,11 +37,15 @@ default_client_limit = 20000
|
||||
# the following warning will be logged:
|
||||
# Warning: service(imap): process_limit (1024) reached, client connections are being dropped
|
||||
service imap {
|
||||
process_limit = 50000
|
||||
process_limit = {{ config.max_imap_connections }}
|
||||
}
|
||||
|
||||
mail_server_admin = mailto:root@{{ config.mail_domain }}
|
||||
mail_server_comment = Chatmail server
|
||||
{% if config.privacy_mail %}
|
||||
# Advertised to clients as IMAP METADATA /shared/admin (RFC 5464).
|
||||
# The privacy_mail contact from chatmail.ini is used because it is
|
||||
# the only address set by an operator.
|
||||
mail_server_admin = mailto:{{ config.privacy_mail }}
|
||||
{% endif %}
|
||||
|
||||
# `zlib` enables compressing messages stored in the maildir.
|
||||
# See
|
||||
|
||||
@@ -20,10 +20,10 @@ class FiltermailDeployer(Deployer):
|
||||
return
|
||||
|
||||
arch = host.get_fact(facts.server.Arch)
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-{arch}"
|
||||
url = f"https://github.com/chatmail/filtermail/releases/download/v0.7.4/filtermail-{arch}"
|
||||
sha256sum = {
|
||||
"x86_64": "05c7e7ac244606c2eeb275f2d282ffdbc2403e0169f1cdd3110ffcebdb994a92",
|
||||
"aarch64": "8cf8bbda4d907beca547b365cc7e6753532a74b1712492d0d2f3d2d8a553fb3d",
|
||||
"x86_64": "484cb8dff083134aefba9fce4a6b7ef4784a0f0e28e5108ecf8bb9e58a44fd2c",
|
||||
"aarch64": "66aa0ca2ca9add7a12d92883d76f8786384092adfde24a3d3a1d0b1f30d23a9e",
|
||||
}[arch]
|
||||
self.download_executable(url, self.bin_path, sha256sum)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ ExecStart={{ bin_path }} {{ config_path }} transport
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
User=vmail
|
||||
LimitNOFILE=524288
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -28,6 +28,13 @@ counter created_nonci_accounts
|
||||
}
|
||||
}
|
||||
|
||||
# doveauth refusing new addresses because a chatmail.ini
|
||||
# system resource limit is exceeded.
|
||||
counter rejected_registrations
|
||||
/registration rejected: / {
|
||||
rejected_registrations++
|
||||
}
|
||||
|
||||
counter postfix_timeouts
|
||||
/timeout after DATA/ {
|
||||
postfix_timeouts++
|
||||
@@ -73,6 +80,11 @@ counter incoming_unencrypted_mail_count
|
||||
filtered_incoming_mail_count++
|
||||
}
|
||||
|
||||
counter incoming_mailer_daemon_mail_count
|
||||
/Incoming: Filtering mailer-daemon message from/ {
|
||||
incoming_mailer_daemon_mail_count++
|
||||
filtered_incoming_mail_count++
|
||||
}
|
||||
|
||||
counter rejected_unencrypted_mail_count
|
||||
/Rejected unencrypted mail/ {
|
||||
|
||||
@@ -128,7 +128,10 @@ http {
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
|
||||
location /relay/probe {
|
||||
# Endpoints Iroh uses for net_report probes and that clients
|
||||
# probe to tell whether this relay works. Both paths are served
|
||||
# by iroh-relay 0.35 and by the 1.0 line.
|
||||
location /ping {
|
||||
proxy_pass http://127.0.0.1:3340;
|
||||
proxy_http_version 1.1;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
/^From:/ DUNNO
|
||||
/^Message-Id:/ DUNNO
|
||||
/^Chat-Is-Post-Message:/ DUNNO
|
||||
/^Content-Type:/ DUNNO
|
||||
/.*/ IGNORE
|
||||
/^DKIM-Signature:/ IGNORE
|
||||
/^Authentication-Results:/ IGNORE
|
||||
/^Received:/ IGNORE
|
||||
|
||||
@@ -53,7 +53,8 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
|
||||
# See <https://www.postfix.org/FORWARD_SECRECY_README.html#server_fs>.
|
||||
tls_preempt_cipherlist = yes
|
||||
|
||||
smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
|
||||
# Reject by default, override per smtpd in master.cf
|
||||
smtpd_relay_restrictions = reject
|
||||
myhostname = {{ config.postfix_myhostname }}
|
||||
alias_maps = hash:/etc/aliases
|
||||
alias_database = hash:/etc/aliases
|
||||
@@ -84,6 +85,14 @@ inet_protocols = all
|
||||
|
||||
lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
|
||||
|
||||
# Do not apply header checks to MIME headers
|
||||
# and other headers that are actually part of the message body.
|
||||
# Reference:
|
||||
# <https://www.postfix.org/postconf.5.html#disable_mime_input_processing>
|
||||
# <https://www.postfix.org/header_checks.5.html>
|
||||
# <https://stump.io/blog/2020/11/29/a-little-gotcha-with-postfixs-header_checks/>
|
||||
disable_mime_input_processing = yes
|
||||
|
||||
mua_client_restrictions = permit_sasl_authenticated, reject
|
||||
mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
|
||||
mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
|
||||
@@ -101,9 +110,24 @@ smtpd_peername_lookup = no
|
||||
# so instead this is handled in filtermail.
|
||||
# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
|
||||
default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}
|
||||
|
||||
# All deliveries over lmtp-filtermail are treated
|
||||
# as having the same destination [127.0.0.1],
|
||||
# so it is not possible to limit per-destination concurrency here,
|
||||
# it is a job for filtermail-transport.
|
||||
# Total number of parallel deliveries is limited
|
||||
# by "maxproc" column in /etc/postfix/master.cf for lmtp-filtermail.
|
||||
# Settings below are to prevent Postfix queue manager
|
||||
# from limiting the number of LMTP connections to filtermail-transport.
|
||||
# Read <https://www.postfix.org/TUNING_README.html#rope> and
|
||||
# <https://www.postfix.org/SCHEDULER_README.html> for the details
|
||||
# of the Postfix algorithm that we effectively disable here.
|
||||
lmtp-filtermail_initial_destination_concurrency=10000
|
||||
lmtp-filtermail_destination_concurrency_limit=10000
|
||||
|
||||
# Do not try to deliver messages for more than 2 days.
|
||||
maximal_queue_lifetime = 2d
|
||||
|
||||
{% if not config.ipv4_relay %}
|
||||
# DKIM-sign locally generated mail (bounces, DSNs).
|
||||
# These bypass smtpd, so they need explicit milter configuration.
|
||||
|
||||
@@ -17,7 +17,8 @@ smtp inet n - y - - smtpd
|
||||
-o smtpd_tls_security_level=encrypt
|
||||
-o smtpd_tls_mandatory_protocols=>=TLSv1.2
|
||||
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
|
||||
submission inet n - y - 5000 smtpd
|
||||
-o smtpd_relay_restrictions=reject_unauth_destination
|
||||
submission inet n - y - {{ config.max_smtp_connections }} smtpd
|
||||
-o syslog_name=postfix/submission
|
||||
-o smtpd_tls_security_level=encrypt
|
||||
-o smtpd_tls_mandatory_protocols=>=TLSv1.3
|
||||
@@ -31,9 +32,9 @@ submission inet n - y - 5000 smtpd
|
||||
-o smtpd_sender_restrictions=$mua_sender_restrictions
|
||||
-o smtpd_recipient_restrictions=
|
||||
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
|
||||
-o smtpd_client_connection_count_limit=1000
|
||||
-o smtpd_client_connection_count_limit={{ config.max_smtp_connections // 5 }}
|
||||
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
|
||||
smtps inet n - y - 5000 smtpd
|
||||
smtps inet n - y - {{ config.max_smtp_connections }} smtpd
|
||||
-o syslog_name=postfix/smtps
|
||||
-o smtpd_tls_wrappermode=yes
|
||||
-o smtpd_tls_security_level=encrypt
|
||||
@@ -47,7 +48,7 @@ smtps inet n - y - 5000 smtpd
|
||||
-o smtpd_sender_restrictions=$mua_sender_restrictions
|
||||
-o smtpd_recipient_restrictions=
|
||||
-o smtpd_relay_restrictions=permit_sasl_authenticated,reject
|
||||
-o smtpd_client_connection_count_limit=1000
|
||||
-o smtpd_client_connection_count_limit={{ config.max_smtp_connections // 5 }}
|
||||
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
|
||||
#628 inet n - y - - qmqpd
|
||||
pickup unix n - y 60 1 pickup
|
||||
@@ -75,18 +76,19 @@ lmtp unix - - y - - lmtp
|
||||
anvil unix - - y - 1 anvil
|
||||
scache unix - - y - 1 scache
|
||||
postlog unix-dgram n - n - 1 postlogd
|
||||
filter unix - n n - - lmtp
|
||||
# Local SMTP server for reinjecting outgoing filtered mail.
|
||||
127.0.0.1:{{ config.postfix_reinject_port }} inet n - n - 100 smtpd
|
||||
-o syslog_name=postfix/reinject
|
||||
-o milter_macro_daemon_name=ORIGINATING
|
||||
-o cleanup_service_name=authclean
|
||||
-o smtpd_relay_restrictions=permit_mynetworks,reject
|
||||
{% if not config.ipv4_relay %} -o smtpd_milters=unix:opendkim/opendkim.sock
|
||||
{% endif %}
|
||||
|
||||
# Local SMTP server for reinjecting incoming filtered mail
|
||||
127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet n - n - 100 smtpd
|
||||
-o syslog_name=postfix/reinject_incoming
|
||||
-o smtpd_relay_restrictions=reject_unauth_destination
|
||||
|
||||
# Cleanup `Received` headers for authenticated mail
|
||||
# to avoid leaking client IP.
|
||||
@@ -102,7 +104,15 @@ filter unix - n n - - lmtp
|
||||
authclean unix n - - - 0 cleanup
|
||||
-o header_checks=regexp:/etc/postfix/submission_header_cleanup
|
||||
|
||||
lmtp-filtermail unix - - y - 10000 lmtp
|
||||
# Reducing `maxproc` here may result in a head of line blocking
|
||||
# when there are many messages sent to unreachable destinations
|
||||
# at the same time.
|
||||
# LMTP clients here talk to filtermail-transport.
|
||||
# LMTP has no pipelining,
|
||||
# so while filtermail-transport tries to deliver the message,
|
||||
# possibly waiting for a long connection timeout
|
||||
# or talking to a slow server, LMTP client cannot be reused.
|
||||
lmtp-filtermail unix - - y - 500 lmtp
|
||||
-o syslog_name=postfix/lmtp-filtermail
|
||||
-o lmtp_header_checks=
|
||||
-o lmtp_tls_security_level=none
|
||||
|
||||
@@ -232,6 +232,26 @@ def try_n_times(n, f):
|
||||
return f()
|
||||
|
||||
|
||||
def test_rewrite_subject(cmsetup, maildata):
|
||||
"""Test that subject gets replaced with [...]."""
|
||||
user1, user2 = cmsetup.gen_users(2)
|
||||
|
||||
sent_msg = maildata(
|
||||
"encrypted.eml",
|
||||
from_addr=user1.addr,
|
||||
to_addr=user2.addr,
|
||||
subject="Unencrypted subject",
|
||||
).as_string()
|
||||
user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user2.addr], msg=sent_msg)
|
||||
|
||||
# The message may need some time to get delivered by postfix.
|
||||
messages = try_n_times(5, user2.imap.fetch_all_messages)
|
||||
assert len(messages) == 1
|
||||
rcvd_msg = messages[0]
|
||||
assert "Subject: [...]" not in sent_msg
|
||||
assert "Subject: [...]" in rcvd_msg
|
||||
assert "Subject: Unencrypted subject" in sent_msg
|
||||
assert "Subject: Unencrypted subject" not in rcvd_msg
|
||||
|
||||
|
||||
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||
|
||||
@@ -10,13 +10,11 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from chatmaild.config import is_valid_ipv4, read_config
|
||||
from domain_validator import DomainValidator
|
||||
|
||||
|
||||
def format_mail_domain(raw_domain: str) -> str:
|
||||
if is_valid_ipv4(raw_domain):
|
||||
return f"[{raw_domain}]"
|
||||
DomainValidator().validate_domain_re(raw_domain)
|
||||
return raw_domain
|
||||
|
||||
|
||||
@@ -349,9 +347,9 @@ class ChatmailACFactory:
|
||||
qr = (
|
||||
f"dclogin:{addr}"
|
||||
f"?p={password}&v=1"
|
||||
f"&ih={domain}&ip=993"
|
||||
f"&sh={domain}&sp=465"
|
||||
f"&ic=3&ss=default"
|
||||
f"&ih={domain}&ip=993&is=ssl"
|
||||
f"&sh={domain}&sp=465&ss=ssl"
|
||||
f"&ic=3"
|
||||
)
|
||||
future = account.add_transport_from_qr.future(qr)
|
||||
else:
|
||||
@@ -362,7 +360,7 @@ class ChatmailACFactory:
|
||||
|
||||
# ensure messages stay in INBOX so that they can be
|
||||
# concurrently fetched via extra IMAP connections during tests
|
||||
account.set_config("delete_server_after", "10")
|
||||
account.set_config("bcc_self", "1")
|
||||
accounts.append(account)
|
||||
|
||||
for future in futures:
|
||||
|
||||
@@ -3,29 +3,40 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pyinfra.facts.deb import DebPackages
|
||||
from pyinfra.facts.server import Command
|
||||
|
||||
from cmdeploy.dovecot import deployer as dovecot_deployer
|
||||
|
||||
|
||||
def _fact_name(key):
|
||||
if isinstance(key, tuple):
|
||||
return f"{key[0].__name__}{key[1:]!r}"
|
||||
return key.__name__
|
||||
|
||||
|
||||
def make_host(*fact_pairs):
|
||||
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
|
||||
"""Build a mock host; get_fact() dispatches to the provided facts mapping.
|
||||
|
||||
Args:
|
||||
*fact_pairs: tuples of (fact_class, fact_value) to register
|
||||
*fact_pairs: (fact_class, value) to match any call of that fact, or
|
||||
((fact_class, *args), value) to match one specific call. Needed
|
||||
for Command, which install() and check_restart() invoke with
|
||||
different scripts; a bare Command entry would serve both.
|
||||
|
||||
Returns:
|
||||
SimpleNamespace with get_fact that raises a clear error if an
|
||||
unexpected fact type is requested.
|
||||
unregistered fact is requested.
|
||||
"""
|
||||
facts = dict(fact_pairs)
|
||||
|
||||
def get_fact(cls):
|
||||
if cls not in facts:
|
||||
registered = ", ".join(c.__name__ for c in facts)
|
||||
raise LookupError(
|
||||
f"unexpected get_fact({cls.__name__}); only registered: {registered}"
|
||||
)
|
||||
return facts[cls]
|
||||
def get_fact(cls, *args):
|
||||
for key in ((cls, *args), cls):
|
||||
if key in facts:
|
||||
return facts[key]
|
||||
registered = ", ".join(_fact_name(k) for k in facts)
|
||||
raise LookupError(
|
||||
f"unexpected get_fact({_fact_name((cls, *args))}); only registered: {registered}"
|
||||
)
|
||||
|
||||
return SimpleNamespace(get_fact=get_fact)
|
||||
|
||||
@@ -64,7 +75,9 @@ def track_shell(monkeypatch):
|
||||
|
||||
|
||||
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
|
||||
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
|
||||
# what dpkg reports after installing our deb: epoch + the +debNu1 suffix
|
||||
# that chatmail/dovecot CI stamps via dch before building
|
||||
epoch_version = f"1:{dovecot_deployer._stamped_version(12)}"
|
||||
downloads = []
|
||||
monkeypatch.setattr(
|
||||
dovecot_deployer,
|
||||
@@ -82,15 +95,17 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
|
||||
lambda **kwargs: downloads.append(kwargs),
|
||||
)
|
||||
|
||||
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
|
||||
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12)
|
||||
|
||||
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
|
||||
assert changed is False, "should not flag changed when version already installed"
|
||||
assert downloads == [], "should not download when version already installed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deb_release", [12, 13])
|
||||
@pytest.mark.parametrize("arch", ["amd64", "arm64"])
|
||||
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
|
||||
monkeypatch,
|
||||
monkeypatch, deb_release, arch
|
||||
):
|
||||
downloads = []
|
||||
monkeypatch.setattr(
|
||||
@@ -109,18 +124,26 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
|
||||
lambda **kwargs: downloads.append(kwargs),
|
||||
)
|
||||
|
||||
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
|
||||
deb, changed = dovecot_deployer._download_dovecot_package(
|
||||
"core", arch, deb_release=deb_release
|
||||
)
|
||||
|
||||
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
|
||||
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
|
||||
stamped = dovecot_deployer._stamped_version(deb_release)
|
||||
expected_deb = f"/root/dovecot-core_{stamped}_{arch}.deb"
|
||||
|
||||
# Verify the returned path uses archive version, not package version (with epoch)
|
||||
# path uses the stamped version, and deb filenames never carry the epoch
|
||||
assert changed is True, "should flag changed when package not yet installed"
|
||||
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
|
||||
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
|
||||
f"deb path should use archive version (no epoch), got {deb!r}"
|
||||
)
|
||||
assert "1:" not in deb, f"deb filename must not contain the epoch, got {deb!r}"
|
||||
assert len(downloads) == 1, "files.download should be called exactly once"
|
||||
# the checksum is the security boundary: verify the right table row is used
|
||||
assert (
|
||||
downloads[0]["sha256sum"]
|
||||
== dovecot_deployer.DOVECOT_SHA256[(arch, deb_release, "core")]
|
||||
), "must pass the sha256 matching (arch, release, package)"
|
||||
assert f"deb{deb_release}u1" in downloads[0]["src"], (
|
||||
f"download URL should carry the deb{deb_release} suffix, got {downloads[0]['src']!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
|
||||
@@ -133,12 +156,13 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
|
||||
(
|
||||
dovecot_deployer.DebPackages,
|
||||
{
|
||||
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
|
||||
"dovecot-core": [f"1:{dovecot_deployer._stamped_version(12)}"],
|
||||
"dovecot-imapd": [f"1:{dovecot_deployer._stamped_version(12)}"],
|
||||
"dovecot-lmtpd": [f"1:{dovecot_deployer._stamped_version(12)}"],
|
||||
},
|
||||
),
|
||||
(dovecot_deployer.Arch, "x86_64"),
|
||||
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
|
||||
),
|
||||
)
|
||||
downloads = []
|
||||
@@ -152,41 +176,26 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
|
||||
|
||||
assert downloads == [], "should not download when all packages epoch-matched"
|
||||
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
|
||||
assert deployer.need_restart is False, (
|
||||
"need_restart should be False when nothing changed"
|
||||
)
|
||||
assert deployer.need_restart is False, "need_restart should be False when nothing changed"
|
||||
|
||||
|
||||
def test_install_unsupported_arch_falls_back_to_apt(
|
||||
def test_install_unsupported_arch_raises(
|
||||
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
|
||||
):
|
||||
# For unsupported architectures, all fact lookups return the arch string.
|
||||
monkeypatch.setattr(
|
||||
dovecot_deployer,
|
||||
"host",
|
||||
SimpleNamespace(get_fact=lambda cls: "riscv64"),
|
||||
make_host(
|
||||
(dovecot_deployer.Arch, "riscv64"),
|
||||
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
|
||||
),
|
||||
)
|
||||
apt_calls = []
|
||||
|
||||
# Mirrors apt.packages() return value: OperationMeta with .changed property.
|
||||
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
|
||||
def fake_apt(**kwargs):
|
||||
apt_calls.append(kwargs)
|
||||
changed = "lmtpd" in kwargs["packages"][0]
|
||||
return SimpleNamespace(changed=changed)
|
||||
# we never fall back to the pinned distro package
|
||||
with pytest.raises(ValueError, match="no dovecot build for dovecot-core"):
|
||||
deployer.install()
|
||||
|
||||
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
|
||||
|
||||
deployer.install()
|
||||
|
||||
actual_pkgs = [c["packages"] for c in apt_calls]
|
||||
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
|
||||
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
|
||||
)
|
||||
assert track_shell == [], "should not run dpkg for unsupported arch"
|
||||
assert deployer.need_restart is True, (
|
||||
"need_restart should be True when apt installed a package"
|
||||
)
|
||||
assert track_shell == [], "should not run apt-get for unsupported arch"
|
||||
|
||||
|
||||
def test_install_runs_dpkg_when_packages_need_download(
|
||||
@@ -198,6 +207,7 @@ def test_install_runs_dpkg_when_packages_need_download(
|
||||
make_host(
|
||||
(dovecot_deployer.DebPackages, {}),
|
||||
(dovecot_deployer.Arch, "x86_64"),
|
||||
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -213,17 +223,15 @@ def test_install_runs_dpkg_when_packages_need_download(
|
||||
|
||||
deployer.install()
|
||||
|
||||
assert len(track_shell) == 1, (
|
||||
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
|
||||
)
|
||||
assert len(track_shell) == 1, f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
|
||||
cmds = track_shell[0]["commands"]
|
||||
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
|
||||
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
|
||||
assert "apt-get -y --fix-broken install" in cmds[1]
|
||||
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
|
||||
assert deployer.need_restart is True, (
|
||||
"need_restart should be True after dpkg install"
|
||||
)
|
||||
assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
|
||||
assert "apt-get install -y" in cmds[0]
|
||||
assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
|
||||
assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
|
||||
assert "--allow-downgrades" in cmds[0]
|
||||
assert ".deb" in cmds[0]
|
||||
assert deployer.need_restart is True, "need_restart should be True after dpkg install"
|
||||
|
||||
|
||||
def test_pick_url_falls_back_on_primary_error(monkeypatch):
|
||||
@@ -232,6 +240,43 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
|
||||
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
|
||||
assert result == "http://fallback", (
|
||||
f"should fall back when primary fails, got {result!r}"
|
||||
assert result == "http://fallback", f"should fall back when primary fails, got {result!r}"
|
||||
|
||||
|
||||
def test_install_fails_on_unsupported_debian_version(deployer, patch_blocked, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
dovecot_deployer,
|
||||
"host",
|
||||
make_host(
|
||||
(dovecot_deployer.Arch, "x86_64"),
|
||||
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="99"'),
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="no dovecot build for dovecot-core on deb99"):
|
||||
deployer.install()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version_line", ["", None, "ID=debian"], ids=["empty", "none", "no-version-id"]
|
||||
)
|
||||
def test_parse_version_id_raises_without_version_id(version_line):
|
||||
with pytest.raises(ValueError, match="cannot determine Debian release"):
|
||||
dovecot_deployer._parse_version_id(version_line)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deb_release", [12, 13])
|
||||
def test_parse_version_id(deb_release):
|
||||
parsed = dovecot_deployer._parse_version_id(f'VERSION_ID="{deb_release}"\n')
|
||||
assert parsed == deb_release
|
||||
|
||||
|
||||
def test_dovecot_sha256_covers_all_packages_per_release():
|
||||
"""Every release in the table needs all three packages on both arches."""
|
||||
table = dovecot_deployer.DOVECOT_SHA256
|
||||
expected = {
|
||||
(arch, pkg) for arch in ("amd64", "arm64") for pkg in ("core", "imapd", "lmtpd")
|
||||
}
|
||||
for release in {r for _, r, _ in table}:
|
||||
got = {(arch, pkg) for arch, r, pkg in table if r == release}
|
||||
assert got == expected, f"deb{release} incomplete: {sorted(expected - got)}"
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
# Managed by cmdeploy: disable IPv6 in unbound.
|
||||
# Managed by cmdeploy
|
||||
server:
|
||||
{% if disable_ipv6 %}
|
||||
interface: 127.0.0.1
|
||||
do-ip6: no
|
||||
{% endif %}
|
||||
cache-max-negative-ttl: 0
|
||||
|
||||
@@ -60,6 +60,7 @@ and run the following commands:
|
||||
::
|
||||
|
||||
git pull origin main --rebase --autostash
|
||||
scripts/initenv.sh
|
||||
scripts/cmdeploy run
|
||||
|
||||
If you don't want the latest development version,
|
||||
|
||||
@@ -199,6 +199,93 @@ creating addresses, login with ssh to the deployment machine and run:
|
||||
Chatmail address creation will be denied while this file is present.
|
||||
|
||||
|
||||
.. _system-limits:
|
||||
|
||||
Configurable System Limits
|
||||
--------------------------
|
||||
|
||||
Limits for auto-rejecting address creation
|
||||
..........................................
|
||||
|
||||
A relay refuses creation of new addresses
|
||||
when the machine runs low on resources,
|
||||
but existing addresses keep working.
|
||||
|
||||
Three ``chatmail.ini`` settings control this,
|
||||
shown here with their defaults::
|
||||
|
||||
max_load_1m = 5
|
||||
min_available_memory = 200M
|
||||
min_free_disk_space = 1G
|
||||
|
||||
- ``max_load_1m`` is the maximum 1-minute load average,
|
||||
as reported by ``uptime``;
|
||||
it counts processes waiting for disk I/O as well as for CPU.
|
||||
It is deliberately not scaled by the number of CPUs
|
||||
because I/O rather than CPU is what typically limits a relay.
|
||||
|
||||
- ``min_available_memory`` is the minimum memory available without swapping.
|
||||
|
||||
- ``min_free_disk_space`` is the minimum free disk space
|
||||
on the file system holding the mailboxes.
|
||||
|
||||
The defaults suit the small machine described in
|
||||
`Minimal requirements and prerequisites`_.
|
||||
|
||||
.. note::
|
||||
|
||||
If you run a bigger machine,
|
||||
raise ``max_load_1m`` after watching ``uptime`` under typical load.
|
||||
|
||||
Rejections are logged by the ``doveauth`` service,
|
||||
so you can check whether a limit is set too tightly::
|
||||
|
||||
journalctl -u doveauth --grep 'registration rejected'
|
||||
|
||||
If ``mtail_address`` is set, rejections are also counted
|
||||
in the ``rejected_registrations`` metric.
|
||||
|
||||
|
||||
Overall IMAP and SMTP connection limits
|
||||
.......................................
|
||||
|
||||
Two further settings bound how many connections
|
||||
the relay accepts at all, again shown with their defaults::
|
||||
|
||||
max_imap_connections = 10000
|
||||
max_smtp_connections = 1000
|
||||
|
||||
``max_imap_connections`` becomes the Dovecot imap process limit,
|
||||
and ``max_smtp_connections`` the Postfix process limit
|
||||
on each of the submission and smtps ports.
|
||||
A single client IP may use up to a fifth of ``max_smtp_connections``.
|
||||
Each connection costs memory,
|
||||
so these limits defend the relay against running out of RAM.
|
||||
|
||||
Unless ``imap_compress`` is enabled,
|
||||
an IMAP connection that is idle for ``imap_hibernate_timeout``
|
||||
is handed over to the ``imap-hibernate`` process
|
||||
and does not count towards ``max_imap_connections``,
|
||||
which is why a relay can serve far more IMAP clients
|
||||
than this setting suggests.
|
||||
|
||||
If you run a large relay with 10k or 100k's of addresses,
|
||||
check current connection counts before upgrading
|
||||
and set the limits accordingly.
|
||||
|
||||
To see how close a running relay is to these two limits,
|
||||
copy ``scripts/check-connections.sh`` from the relay repository
|
||||
onto the relay and run it there::
|
||||
|
||||
imap 5 ports 143,993 (max_imap_connections)
|
||||
5 dovecot sessions, 0 of them in an active imap process
|
||||
submission 0 ports 465,587 (max_smtp_connections per port)
|
||||
incoming 0 port 25 (from other relays, no chatmail.ini limit)
|
||||
|
||||
It counts established sockets with ``ss``
|
||||
and cross-checks the IMAP number against ``doveadm who``.
|
||||
|
||||
|
||||
Running a relay with self-signed certificates
|
||||
----------------------------------------------
|
||||
|
||||
|
||||
+41
-15
@@ -156,6 +156,7 @@ Chatmail relay dependency diagram
|
||||
postfix --- |10083|filtermail-transport;
|
||||
filtermail-outgoing --- |10025 reinject|postfix;
|
||||
filtermail-incoming --- |10026 reinject|postfix;
|
||||
postfix --- |milter opendkim.sock|OpenDKIM
|
||||
dovecot --- |doveauth.socket|doveauth;
|
||||
dovecot --- |message delivery|maildir["maildir
|
||||
/home/vmail/.../user"];
|
||||
@@ -179,26 +180,51 @@ Chatmail relay dependency diagram
|
||||
style nginx-right fill:#f66;
|
||||
style postfix fill:#f66;
|
||||
style dovecot fill:#f66;
|
||||
style OpenDKIM fill:#f66;
|
||||
style notification-proxy fill:#f66;
|
||||
|
||||
Message between users on the same relay
|
||||
---------------------------------------
|
||||
Accepting and delivering mail
|
||||
-----------------------------
|
||||
|
||||
.. mermaid::
|
||||
:caption: This diagram shows the path a non-federated message takes.
|
||||
:caption: This diagram shows all the paths a message can take.
|
||||
|
||||
graph LR;
|
||||
sender --> |465|smtps/smtpd;
|
||||
sender --> |587|submission/smtpd;
|
||||
smtps/smtpd --> |10080|filtermail;
|
||||
submission/smtpd --> |10080|filtermail;
|
||||
filtermail --> |10025|smtpd_reinject;
|
||||
smtpd_reinject --> cleanup;
|
||||
cleanup --> qmgr;
|
||||
qmgr --> smtpd_accepts_message;
|
||||
qmgr --> |lmtp|dovecot;
|
||||
dovecot --> recipient;
|
||||
dovecot --> sender's_other_devices;
|
||||
flowchart LR
|
||||
subgraph chatmail relay
|
||||
subgraph postfix
|
||||
qmgr .-> lmtp-filtermail["lmtp/lmtp-filtermail (default_transport)"]
|
||||
qmgr .-> lmtp["lmtp (local_transport)"]
|
||||
lmtp --> cleanup["cleanup (lmtp_header_cleanup)"]
|
||||
bounce
|
||||
smtpd-submission["smtpd/submission"]
|
||||
smtpd-smtps["smtpd/smtps"]
|
||||
smtpd-reinject-outgoing["smtpd/reinject-outgoing"] --> authclean["cleanup/authclean (submission_header_cleanup)"]
|
||||
authclean --> qmgr
|
||||
smtpd-smtp["smtpd/smtp"]
|
||||
smtpd-reinject-incoming["smtpd/reinject-incoming"] --> qmgr
|
||||
end
|
||||
lmtp-filtermail --LMTP inet:10083--> filtermail-transport
|
||||
cleanup --LMTP unix:private/dovecot-lmtp --> dovecot
|
||||
dovecot --> maildir
|
||||
smtpd-submission --SMTP inet:10080--> filtermail-outgoing
|
||||
smtpd-smtps --SMTP inet:10080--> filtermail-outgoing
|
||||
filtermail-outgoing --SMTP inet:10025--> smtpd-reinject-outgoing
|
||||
open-dkim["OpenDKIM (signing only)"] <--milter unix:opendkim/opendkim.sock--> smtpd-reinject-outgoing
|
||||
bounce <--milter unix:opendkim/opendkim.sock--> open-dkim
|
||||
bounce --> qmgr
|
||||
nginx
|
||||
smtpd-smtp -.SMTP inet:10081.-> filtermail-incoming
|
||||
nginx -.HTTP inet:10082.-> filtermail-incoming
|
||||
filtermail-incoming --SMTP inet:10026--> smtpd-reinject-incoming
|
||||
end
|
||||
filtermail-transport -.SMTP inet:25.-> mta1[Remote relay]
|
||||
filtermail-transport -.HTTPS /mxdeliv.-> mta1
|
||||
client[Client] -.SMTP inet:587.-> smtpd-submission
|
||||
client -.SMTP inet:465.-> smtpd-smtps
|
||||
client -.SMTP inet:443.-> nginx
|
||||
nginx -.SMTP inet:465.-> smtpd-smtps
|
||||
mta2[Remote relay] -.SMTP inet:25.-> smtpd-smtp
|
||||
mta2 -.HTTPS /mxdeliv.-> nginx
|
||||
|
||||
Operational details of a chatmail relay
|
||||
----------------------------------------
|
||||
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Show current IMAP and SMTP connections on a chatmail relay,
|
||||
# to compare against the max_imap_connections and max_smtp_connections
|
||||
# settings in chatmail.ini. Run this on the relay itself.
|
||||
set -e
|
||||
|
||||
# Count established TCP connections whose local port is one of the arguments.
|
||||
established() {
|
||||
filter=$(printf 'sport = :%s or ' "$@")
|
||||
ss -Htn state established "( ${filter% or } )" | wc -l
|
||||
}
|
||||
|
||||
# doveadm prints a header line and then one line per logged-in user,
|
||||
# with that user's number of connections in the second column.
|
||||
sessions=$(doveadm who | awk 'NR > 1 { n += $2 } END { print n + 0 }')
|
||||
|
||||
# Unless imap_compress is enabled, connections idle for
|
||||
# imap_hibernate_timeout are handed over to the imap-hibernate
|
||||
# process, so they cost no imap process while idle.
|
||||
active=$(pgrep -x imap | wc -l)
|
||||
|
||||
printf 'imap %6d ports 143,993 (max_imap_connections)\n' "$(established 143 993)"
|
||||
printf ' %6d dovecot sessions, %d of them in an active imap process\n' "$sessions" "$active"
|
||||
printf 'submission %6d ports 465,587 (max_smtp_connections per port)\n' "$(established 465 587)"
|
||||
printf 'incoming %6d port 25 (from other relays, no chatmail.ini limit)\n' "$(established 25)"
|
||||
Reference in New Issue
Block a user