Compare commits

..

2 Commits

Author SHA1 Message Date
missytake
5a6f036106 tests: other bots could be in passthrough_recipients 2023-12-29 11:26:10 +01:00
missytake
7d55d98620 config: add xstore and groupsbot to default passthrough_recipients 2023-12-29 11:08:01 +01:00
30 changed files with 128 additions and 491 deletions

View File

@@ -1,20 +0,0 @@
;; Zone file for staging.testrun.org
$ORIGIN staging.testrun.org.
$TTL 300
@ IN SOA ns.testrun.org. root.nine.testrun.org (
2023010101 ; Serial
7200 ; Refresh
3600 ; Retry
1209600 ; Expire
3600 ; Negative response caching TTL
)
;; Nameservers.
@ IN NS ns.testrun.org.
;; DNS records.
@ IN A 37.27.37.98
mta-sts.staging.testrun.org. CNAME staging.testrun.org.
www.staging.testrun.org. CNAME staging.testrun.org.

View File

@@ -1,72 +0,0 @@
name: deploy on staging.testrun.org, and run tests
on:
push:
branches:
- main
- staging-ci
jobs:
deploy:
name: deploy on staging.testrun.org, and run tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: prepare SSH
run: |
mkdir ~/.ssh
echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
# rsync -avz root@staging.testrun.org:/var/lib/acme . || true
# rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
#- name: rebuild staging.testrun.org to have a clean VPS
# run: |
# curl -X POST \
# -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
# -H "Content-Type: application/json" \
# -d '{"image":"debian-12"}' \
# "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
- run: scripts/initenv.sh
- name: append venv/bin to PATH
run: echo venv/bin >>$GITHUB_PATH
- name: run formatting checks
run: cmdeploy fmt -v
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy
#- name: upload TLS cert after rebuilding
# run: |
# echo " --- wait until staging.testrun.org VPS is rebuilt --- "
# rm ~/.ssh/known_hosts
# while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
# ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
# rsync -avz acme root@staging.testrun.org:/var/lib/ || true
# rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
- run: cmdeploy init staging.testrun.org
- run: cmdeploy run
- name: set DNS entries
run: |
#ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
cmdeploy dns --zonefile staging-generated.zone
cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
cat .github/workflows/staging.testrun.org-default.zone
scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
ssh root@ns.testrun.org systemctl reload nsd
- name: cmdeploy test
run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
- name: cmdeploy dns (try 3 times)
run: cmdeploy dns || cmdeploy dns || cmdeploy dns

View File

@@ -157,6 +157,6 @@ While this file is present, account creation will be blocked.
[acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
Delta Chat apps will, however, discover all ports and configurations
automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
automatically by reading the [autoconfig XML file](https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) from the chatmail service.

View File

@@ -91,7 +91,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
VALUES (?, ?, ?)"""
conn.execute(q, (user, encrypted_password, int(time.time())))
return dict(
home=f"/home/vmail/mail/{config.mail_domain}/{user}",
home=f"/home/vmail/{user}",
uid="vmail",
gid="vmail",
password=encrypted_password,
@@ -160,19 +160,6 @@ def handle_dovecot_request(msg, db, config: Config):
return None
def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
while True:
msg = rfile.readline().strip().decode()
if not msg:
break
res = handle_dovecot_request(msg, db, config)
if res:
wfile.write(res.encode("ascii"))
wfile.flush()
else:
logging.warning("request had no answer: %r", msg)
class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
request_queue_size = 100
@@ -186,7 +173,16 @@ def main():
class Handler(StreamRequestHandler):
def handle(self):
try:
handle_dovecot_protocol(self.rfile, self.wfile, db, config)
while True:
msg = self.rfile.readline().strip().decode()
if not msg:
break
res = handle_dovecot_request(msg, db, config)
if res:
self.wfile.write(res.encode("ascii"))
self.wfile.flush()
else:
logging.warn("request had no answer: %r", msg)
except Exception:
logging.exception("Exception in the handler")
raise

View File

@@ -7,7 +7,7 @@ Date: Sun, 15 Oct 2023 16:41:44 +0000
Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
Chat-Version: 1.0
Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;
keydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m
nNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID
FgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I
@@ -20,4 +20,4 @@ Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
Hi!

View File

@@ -1,17 +1,11 @@
import io
import json
import pytest
import queue
import threading
import queue
import traceback
import chatmaild.doveauth
from chatmaild.doveauth import (
get_user_data,
lookup_passdb,
handle_dovecot_request,
handle_dovecot_protocol,
)
from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
from chatmaild.database import DBError
@@ -67,23 +61,11 @@ def test_handle_dovecot_request(db, example_config):
assert res
assert res[0] == "O" and res.endswith("\n")
userdata = json.loads(res[1:].strip())
assert (
userdata["home"]
== "/home/vmail/mail/chat.example.org/some42123@chat.example.org"
)
assert userdata["home"] == "/home/vmail/some42123@chat.example.org"
assert userdata["uid"] == userdata["gid"] == "vmail"
assert userdata["password"].startswith("{SHA512-CRYPT}")
def test_handle_dovecot_protocol(db, example_config):
rfile = io.BytesIO(
b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
)
wfile = io.BytesIO()
handle_dovecot_protocol(rfile, wfile, db, example_config)
assert wfile.getvalue() == b"N\n"
def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
num_threads = 50
req_per_thread = 5

View File

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
name = "cmdeploy"
version = "0.2"
dependencies = [
"pyinfra==3.0b0",
"pyinfra",
"pillow",
"qrcode",
"markdown",

View File

@@ -1,7 +1,6 @@
"""
Chat Mail pyinfra deploy.
"""
import sys
import importlib.resources
import subprocess
@@ -10,15 +9,13 @@ import io
from pathlib import Path
from pyinfra import host
from pyinfra.operations import apt, files, server, systemd, pip, util
from pyinfra.operations import apt, files, server, systemd, pip
from pyinfra.facts.files import File
from pyinfra.facts.systemd import SystemdEnabled
from .acmetool import deploy_acmetool
from chatmaild.config import read_config, Config
from typing import Callable
def _build_chatmaild(dist_dir) -> None:
dist_dir = Path(dist_dir).resolve()
@@ -129,21 +126,9 @@ def _install_remote_venv_with_chatmaild(config) -> None:
)
def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[], bool]:
def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
"""Configures OpenDKIM"""
server.group(name="Create opendkim group", group="opendkim", system=True)
server.user(
name="Create opendkim user",
user="opendkim",
groups=["opendkim"],
system=True,
)
server.user(
name="Add postfix user to opendkim group for socket access",
user="postfix",
groups=["opendkim"],
system=True,
)
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
@@ -153,22 +138,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[]
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
screen_script = files.put(
src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
dest="/etc/opendkim/screen.lua",
user="root",
group="root",
mode="644",
)
final_script = files.put(
src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
dest="/etc/opendkim/final.lua",
user="root",
group="root",
mode="644",
)
need_restart |= main_config.changed
files.directory(
name="Add opendkim directory to /etc",
@@ -187,6 +157,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[]
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= keytable.changed
signing_table = files.template(
src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
@@ -196,6 +167,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[]
mode="644",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= signing_table.changed
files.directory(
name="Add opendkim socket directory to /var/spool/postfix",
@@ -206,11 +178,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[]
present=True,
)
apt.packages(
name="apt install opendkim opendkim-tools",
packages=["opendkim", "opendkim-tools"],
)
if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
server.shell(
name="Generate OpenDKIM domain keys",
@@ -221,12 +188,10 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[]
_sudo_user="opendkim",
)
return util.any_changed(
main_config, screen_script, final_script, keytable, signing_table
)
return need_restart
def _install_mta_sts_daemon() -> Callable[[], bool]:
def _install_mta_sts_daemon() -> bool:
need_restart = False
config = files.put(
@@ -239,6 +204,7 @@ def _install_mta_sts_daemon() -> Callable[[], bool]:
group="root",
mode="644",
)
need_restart |= config.changed
server.shell(
name="install postfix-mta-sts-resolver with pip",
@@ -258,12 +224,15 @@ def _install_mta_sts_daemon() -> Callable[[], bool]:
group="root",
mode="644",
)
need_restart |= systemd_unit.changed
return util.any_changed(config, systemd_unit)
return need_restart
def _configure_postfix(config: Config, debug: bool = False) -> Callable[[], bool]:
def _configure_postfix(config: Config, debug: bool = False) -> bool:
"""Configures Postfix SMTP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
dest="/etc/postfix/main.cf",
@@ -272,6 +241,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> Callable[[], bool
mode="644",
config=config,
)
need_restart |= main_config.changed
master_config = files.template(
src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
@@ -282,31 +252,15 @@ def _configure_postfix(config: Config, debug: bool = False) -> Callable[[], bool
debug=debug,
config=config,
)
need_restart |= master_config.changed
header_cleanup = files.put(
src=importlib.resources.files(__package__).joinpath(
"postfix/submission_header_cleanup"
),
dest="/etc/postfix/submission_header_cleanup",
user="root",
group="root",
mode="644",
)
# Login map that 1:1 maps email address to login.
login_map = files.put(
src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
dest="/etc/postfix/login_map",
user="root",
group="root",
mode="644",
)
return util.any_changed(main_config, master_config, header_cleanup, login_map)
return need_restart
def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool]:
def _configure_dovecot(config: Config, debug: bool = False) -> bool:
"""Configures Dovecot IMAP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
dest="/etc/dovecot/dovecot.conf",
@@ -316,6 +270,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool
config=config,
debug=debug,
)
need_restart |= main_config.changed
auth_config = files.put(
src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
dest="/etc/dovecot/auth.conf",
@@ -323,6 +278,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool
group="root",
mode="644",
)
need_restart |= auth_config.changed
files.template(
src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -344,11 +300,13 @@ def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool
persist=True,
)
return util.any_changed(main_config, auth_config)
return need_restart
def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
def _configure_nginx(domain: str, debug: bool = False) -> bool:
"""Configures nginx HTTP server."""
need_restart = False
main_config = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
dest="/etc/nginx/nginx.conf",
@@ -357,6 +315,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
mode="644",
config={"domain_name": domain},
)
need_restart |= main_config.changed
autoconfig = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
@@ -366,6 +325,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
mode="644",
config={"domain_name": domain},
)
need_restart |= autoconfig.changed
mta_sts_config = files.template(
src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
@@ -375,6 +335,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
mode="644",
config={"domain_name": domain},
)
need_restart |= mta_sts_config.changed
# install CGI newemail script
#
@@ -395,12 +356,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
mode="755",
)
return util.any_changed(main_config, autoconfig, mta_sts_config)
def _remove_rspamd() -> None:
"""Remove rspamd"""
apt.packages(name="Remove rspamd", packages="rspamd", present=False)
return need_restart
def check_config(config):
@@ -430,6 +386,14 @@ def deploy_chatmail(config_path: Path) -> None:
server.group(name="Create vmail group", group="vmail", system=True)
server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
server.group(name="Create opendkim group", group="opendkim", system=True)
server.user(
name="Add postfix user to opendkim group for socket access",
user="postfix",
groups=["opendkim"],
system=True,
)
# Run local DNS resolver `unbound`.
# `resolvconf` takes care of setting up /etc/resolv.conf
# to use 127.0.0.1 as the resolver.
@@ -439,10 +403,7 @@ def deploy_chatmail(config_path: Path) -> None:
)
server.shell(
name="Generate root keys for validating DNSSEC",
commands=[
"unbound-anchor -a /var/lib/unbound/root.key || true",
"systemctl reset-failed unbound.service",
],
commands=["unbound-anchor -a /var/lib/unbound/root.key || true"],
)
systemd.service(
name="Start and enable unbound",
@@ -452,10 +413,7 @@ def deploy_chatmail(config_path: Path) -> None:
)
# Deploy acmetool to have TLS certificates.
deploy_acmetool(
nginx_hook=True,
domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
)
deploy_acmetool(nginx_hook=True, domains=[mail_domain, f"mta-sts.{mail_domain}"])
apt.packages(
name="Install Postfix",
@@ -467,6 +425,14 @@ def deploy_chatmail(config_path: Path) -> None:
packages=["dovecot-imapd", "dovecot-lmtpd"],
)
apt.packages(
name="Install OpenDKIM",
packages=[
"opendkim",
"opendkim-tools",
],
)
apt.packages(
name="Install nginx",
packages=["nginx"],
@@ -488,23 +454,16 @@ def deploy_chatmail(config_path: Path) -> None:
debug = False
dovecot_need_restart = _configure_dovecot(config, debug=debug)
postfix_need_restart = _configure_postfix(config, debug=debug)
opendkim_need_restart = _configure_opendkim(mail_domain)
mta_sts_need_restart = _install_mta_sts_daemon()
nginx_need_restart = _configure_nginx(mail_domain)
_remove_rspamd()
opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
systemd.service(
name="Start and enable OpenDKIM",
service="opendkim.service",
running=True,
enabled=True,
)
systemd.service(
name="Restart OpenDKIM",
service="opendkim.service",
restarted=True,
_if=opendkim_need_restart,
restarted=opendkim_need_restart,
)
systemd.service(
@@ -513,12 +472,7 @@ def deploy_chatmail(config_path: Path) -> None:
daemon_reload=True,
running=True,
enabled=True,
)
systemd.service(
name="Restart MTA-STS daemon",
service="mta-sts-daemon.service",
restarted=True,
_if=mta_sts_need_restart,
restarted=mta_sts_need_restart,
)
systemd.service(
@@ -526,12 +480,7 @@ def deploy_chatmail(config_path: Path) -> None:
service="postfix.service",
running=True,
enabled=True,
)
systemd.service(
name="Restart Postfix",
service="postfix.service",
restarted=True,
_if=postfix_need_restart,
restarted=postfix_need_restart,
)
systemd.service(
@@ -539,12 +488,7 @@ def deploy_chatmail(config_path: Path) -> None:
service="dovecot.service",
running=True,
enabled=True,
)
systemd.service(
name="Restart Dovecot",
service="dovecot.service",
restarted=True,
_if=dovecot_need_restart,
restarted=dovecot_need_restart,
)
systemd.service(
@@ -552,12 +496,7 @@ def deploy_chatmail(config_path: Path) -> None:
service="nginx.service",
running=True,
enabled=True,
)
systemd.service(
name="Restart nginx",
service="nginx.service",
restarted=True,
_if=nginx_need_restart,
restarted=nginx_need_restart,
)
# This file is used by auth proxy.

View File

@@ -69,12 +69,7 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
service="acmetool-redirector.service",
running=True,
enabled=True,
)
systemd.service(
name="Restart acmetool-redirector service",
service="acmetool-redirector.service",
restarted=True,
_if=service_file.did_change,
restarted=service_file.changed,
)
server.shell(

View File

@@ -7,9 +7,8 @@ _imap._tcp.{chatmail_domain}. SRV 0 1 143 {chatmail_domain}.
_imaps._tcp.{chatmail_domain}. SRV 0 1 993 {chatmail_domain}.
{chatmail_domain}. CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
{chatmail_domain}. TXT "v=spf1 a:{chatmail_domain} -all"
_dmarc.{chatmail_domain}. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
_dmarc.{chatmail_domain}. TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
_mta-sts.{chatmail_domain}. TXT "v=STSv1; id={sts_id}"
mta-sts.{chatmail_domain}. CNAME {chatmail_domain}.
www.{chatmail_domain}. CNAME {chatmail_domain}.
_smtp._tls.{chatmail_domain}. TXT "v=TLSRPTv1;rua=mailto:{email}"
{dkim_entry}
_adsp._domainkey.{chatmail_domain}. TXT "dkim=discardable"

View File

@@ -82,8 +82,7 @@ def dns_cmd_options(parser):
def dns_cmd(args, out):
"""Generate dns zone file."""
exit_code = show_dns(args, out)
exit(exit_code)
show_dns(args, out)
def status_cmd(args, out):

View File

@@ -4,6 +4,7 @@ import requests
import importlib
import subprocess
import datetime
from ipaddress import ip_address
class DNS:
@@ -47,8 +48,7 @@ class DNS:
return result == f"{mail_domain}."
def show_dns(args, out) -> int:
"""Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
def show_dns(args, out):
template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
mail_domain = args.config.mail_domain
ssh = f"ssh root@{mail_domain}"
@@ -68,7 +68,7 @@ def show_dns(args, out) -> int:
acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
except subprocess.CalledProcessError:
print("Please run `cmdeploy run` first.")
return 1
return
dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
ipv6 = dns.get_ipv6()
@@ -82,6 +82,7 @@ def show_dns(args, out) -> int:
f.read()
.format(
acme_account_url=acme_account_url,
email=f"root@{args.config.mail_domain}",
sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
chatmail_domain=args.config.mail_domain,
dkim_entry=dkim_entry,
@@ -94,13 +95,14 @@ def show_dns(args, out) -> int:
with open(args.zonefile, "w+") as zf:
zf.write(zonefile)
print(f"DNS records successfully written to: {args.zonefile}")
return 0
return
except TypeError:
pass
started_dkim_parsing = False
for line in zonefile.splitlines():
line = line.format(
acme_account_url=acme_account_url,
email=f"root@{args.config.mail_domain}",
sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
chatmail_domain=args.config.mail_domain,
dkim_entry=dkim_entry,
@@ -133,7 +135,7 @@ def show_dns(args, out) -> int:
continue
if current != value:
to_print.append(line)
if "IN TXT ( " in line:
if " IN TXT ( " in line:
started_dkim_parsing = True
dkim_lines = [line]
if started_dkim_parsing and line.startswith('"'):
@@ -147,7 +149,6 @@ def show_dns(args, out) -> int:
else:
to_print.append(dkim_entry)
exit_code = 0
if to_print:
to_print.insert(
0, "You should configure the following DNS entries at your provider:\n"
@@ -156,7 +157,6 @@ def show_dns(args, out) -> int:
"\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
)
print("\n".join(to_print))
exit_code = 1
else:
out.green("Great! All your DNS entries are correct.")
@@ -176,8 +176,6 @@ def show_dns(args, out) -> int:
print(
"You can do so at your hosting provider (maybe this isn't your DNS provider)."
)
exit_code = 1
return exit_code
def check_necessary_dns(out, mail_domain):
@@ -186,14 +184,14 @@ def check_necessary_dns(out, mail_domain):
ipv4 = dns.get("A", mail_domain)
ipv6 = dns.get("AAAA", mail_domain)
mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
www_entry = dns.get("CNAME", "www." + mail_domain)
mta_ip = dns.get("A", mta_entry)
if not mta_ip:
mta_ip = dns.get("AAAA", mta_entry)
to_print = []
if not (ipv4 or ipv6):
to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
if mta_entry != mail_domain + ".":
if not mta_ip or not (mta_ip == ipv4 or mta_ip == ipv6):
to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
if www_entry != mail_domain + ".":
to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
if to_print:
to_print.insert(
0,

View File

@@ -13,15 +13,13 @@ auth_cache_size = 100M
mail_debug = yes
{% endif %}
mail_server_admin = mailto:root@{{ config.mail_domain }}
mail_server_comment = Chatmail server
mail_plugins = quota
# these are the capabilities Delta Chat cares about actually
# so let's keep the network overhead per login small
# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY
# Authentication for system users.
@@ -75,7 +73,6 @@ mail_privileged_group = vmail
# <https://datatracker.ietf.org/doc/html/rfc4978.html>
protocol imap {
mail_plugins = $mail_plugins imap_zlib imap_quota
imap_metadata = yes
}
protocol lmtp {

View File

@@ -1,10 +1,10 @@
# delete all mails after {{ config.delete_mails_after }} days, in the Inbox
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
# or in any IMAP subfolder
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
# even if they are unseen
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/new -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/new -mtime +{{ config.delete_mails_after }} -type f -delete
# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete

View File

@@ -6,7 +6,7 @@ import io
def gen_qr_png_data(maildomain):
url = f"DCACCOUNT:https://{maildomain}/new"
url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
image = gen_qr(maildomain, url)
temp = io.BytesIO()
image.save(temp, format="png")

View File

@@ -41,44 +41,11 @@ http {
try_files $uri $uri/ =404;
}
location /metrics {
default_type text/plain;
}
location /metrics {
default_type text/plain;
}
location /new {
if ($request_method = GET) {
# Redirect to Delta Chat,
# which will in turn do a POST request.
return 301 dcaccount:https://{{ config.domain_name }}/new;
}
fastcgi_pass unix:/run/fcgiwrap.socket;
include /etc/nginx/fastcgi_params;
fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
}
# Old URL for compatibility with e.g. printed QR codes.
#
# Copy-paste instead of redirect to /new
# because Delta Chat core does not follow redirects.
#
# Redirects are only for browsers.
location /cgi-bin/newemail.py {
if ($request_method = GET) {
return 301 dcaccount:https://{{ config.domain_name }}/new;
}
fastcgi_pass unix:/run/fcgiwrap.socket;
include /etc/nginx/fastcgi_params;
fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
}
}
# Redirect www. to non-www
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name www.{{ config.domain_name }};
return 301 $scheme://{{ config.domain_name }}$request_uri;
# add cgi-bin support
include /usr/share/doc/fcgiwrap/examples/nginx.conf;
}
}

View File

@@ -1 +1 @@
{{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/{{ config.opendkim_selector }}.private
dkim._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/dkim.private

View File

@@ -1,28 +0,0 @@
if odkim.internal_ip(ctx) == 1 then
-- Outgoing message will be signed,
-- no need to look for signatures.
return nil
end
nsigs = odkim.get_sigcount(ctx)
if nsigs == nil then
return nil
end
for i = 1, nsigs do
sig = odkim.get_sighandle(ctx, i - 1)
sigres = odkim.sig_result(sig)
-- All signatures that do not correspond to From:
-- were ignored in screen.lua and return sigres -1.
--
-- Any valid signature that was not ignored like this
-- means the message is acceptable.
if sigres == 0 then
return nil
end
end
odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
odkim.set_result(ctx, SMFIS_REJECT)
return nil

View File

@@ -8,12 +8,10 @@ SyslogSuccess yes
# oversigned, because it is often the identity key used by reputation systems
# and thus somewhat security sensitive.
Canonicalization relaxed/simple
#Mode sv
#SubDomains no
OversignHeaders From
On-BadSignature reject
On-KeyNotFound reject
On-NoSignature reject
# Signing domain, selector, and key (required). For example, perform signing
# for domain "example.com" with selector "2020" (2020._domainkey.example.com),
# using the private key stored in /etc/dkimkeys/example.private. More granular
@@ -24,15 +22,6 @@ KeyFile /etc/dkimkeys/{{ config.opendkim_selector }}.private
KeyTable /etc/dkimkeys/KeyTable
SigningTable refile:/etc/dkimkeys/SigningTable
# Sign Autocrypt header in addition to the default specified in RFC 6376.
SignHeaders *,+autocrypt
# Script to ignore signatures that do not correspond to the From: domain.
ScreenPolicyScript /etc/opendkim/screen.lua
# Script to reject mails without a valid DKIM signature.
FinalPolicyScript /etc/opendkim/final.lua
# In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
# using a local socket with MTAs that access the socket as a non-privileged
# user (for example, Postfix). You may need to add user "postfix" to group
@@ -40,10 +29,22 @@ FinalPolicyScript /etc/opendkim/final.lua
UserID opendkim
UMask 007
# Socket for the MTA connection (required). If the MTA is inside a chroot jail,
# it must be ensured that the socket is accessible. In Debian, Postfix runs in
# a chroot in /var/spool/postfix, therefore a Unix socket would have to be
# configured as shown on the last line below.
#Socket local:/run/opendkim/opendkim.sock
#Socket inet:8891@localhost
#Socket inet:8891
Socket local:/var/spool/postfix/opendkim/opendkim.sock
PidFile /run/opendkim/opendkim.pid
# Hosts for which to sign rather than verify, default is 127.0.0.1. See the
# OPERATION section of opendkim(8) for more information.
#InternalHosts 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12
# The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
# by the package dns-root-data.
TrustAnchorFile /usr/share/dns/root.key
#Nameservers 127.0.0.1

View File

@@ -1,21 +0,0 @@
-- Ignore signatures that do not correspond to the From: domain.
from_domain = odkim.get_fromdomain(ctx)
if from_domain == nil then
return nil
end
n = odkim.get_sigcount(ctx)
if n == nil then
return nil
end
for i = 1, n do
sig = odkim.get_sighandle(ctx, i - 1)
sig_domain = odkim.sig_getdomain(sig)
if from_domain ~= sig_domain then
odkim.sig_ignore(sig)
end
end
return nil

View File

@@ -1 +0,0 @@
/^(.*)$/ ${1}

View File

@@ -23,31 +23,6 @@ smtp_tls_CApath=/etc/ssl/certs
smtp_tls_security_level=may
smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
smtpd_tls_protocols = >=TLSv1.2
# Disable anonymous cipher suites
# and known insecure algorithms.
#
# Disabling anonymous ciphers
# does not generally improve security
# because clients that want to verify certificate
# will not select them anyway,
# but makes cipher suite list shorter and security scanners happy.
# See <https://www.postfix.org/TLS_README.html> for discussion.
#
# Only ancient insecure ciphers should be disabled here
# as MTA clients that do not support more secure cipher
# likely do not support MTA-STS either and will
# otherwise fall back to using plaintext connection.
smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
# Override client's preference order.
# <https://www.postfix.org/postconf.5.html#tls_preempt_cipherlist>
#
# This is mostly to ensure cipher suites with forward secrecy
# are preferred over non cipher suites without forward secrecy.
# See <https://www.postfix.org/FORWARD_SECRECY_README.html#server_fs>.
tls_preempt_cipherlist = yes
smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
myhostname = {{ config.mail_domain }}
@@ -71,9 +46,5 @@ inet_protocols = all
virtual_transport = lmtp:unix:private/dovecot-lmtp
virtual_mailbox_domains = {{ config.mail_domain }}
mua_client_restrictions = permit_sasl_authenticated, reject
mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
# 1:1 map MAIL FROM to SASL login name.
smtpd_sender_login_maps = regexp:/etc/postfix/login_map
smtpd_milters = unix:opendkim/opendkim.sock
non_smtpd_milters = $smtpd_milters

View File

@@ -11,10 +11,13 @@
# ==========================================================================
{% if debug == true %}
smtp inet n - y - - smtpd -v
{%- else %}
{% else %}
smtp inet n - y - - smtpd
{%- endif %}
-o smtpd_milters=unix:opendkim/opendkim.sock
{% endif %}
#smtp inet n - y - 1 postscreen
#smtpd pass - - y - - smtpd
#dnsblog unix - - y - 0 dnsblog
#tlsproxy unix - - y - 0 tlsproxy
submission inet n - y - - smtpd
-o syslog_name=postfix/submission
-o smtpd_tls_security_level=encrypt
@@ -31,7 +34,6 @@ submission inet n - y - - smtpd
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_client_connection_count_limit=1000
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-o cleanup_service_name=authclean
smtps inet n - y - - smtpd
-o syslog_name=postfix/smtps
-o smtpd_tls_wrappermode=yes
@@ -48,7 +50,6 @@ smtps inet n - y - - smtpd
-o smtpd_client_connection_count_limit=1000
-o milter_macro_daemon_name=ORIGINATING
-o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-o cleanup_service_name=authclean
#628 inet n - y - - qmqpd
pickup unix n - y 60 1 pickup
cleanup unix n - y - 0 cleanup
@@ -79,14 +80,3 @@ filter unix - n n - - lmtp
# Local SMTP server for reinjecting filered mail.
localhost:{{ config.postfix_reinject_port }} inet n - n - 10 smtpd
-o syslog_name=postfix/reinject
-o smtpd_milters=unix:opendkim/opendkim.sock
-o cleanup_service_name=authclean
# Cleanup `Received` headers for authenticated mail
# to avoid leaking client IP.
#
# We do not do this for received mails
# as this will break DKIM signatures
# if `Received` header is signed.
authclean unix n - - - 0 cleanup
-o header_checks=regexp:/etc/postfix/submission_header_cleanup

View File

@@ -1,4 +0,0 @@
/^Received:/ IGNORE
/^X-Originating-IP:/ IGNORE
/^X-Mailer:/ IGNORE
/^User-Agent:/ IGNORE

View File

@@ -9,7 +9,7 @@ def test_gen_qr_png_data(maildomain):
def test_fastcgi_working(maildomain, chatmail_config):
url = f"https://{maildomain}/new"
url = f"https://{maildomain}/cgi-bin/newemail.py"
print(url)
res = requests.post(url)
assert maildomain in res.json().get("email")
@@ -18,7 +18,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
def test_newemail_configure(maildomain, rpc):
"""Test configuring accounts by scanning a QR code works."""
url = f"DCACCOUNT:https://{maildomain}/new"
url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
for i in range(3):
account_id = rpc.add_account()
rpc.set_config_from_qr(account_id, url)

View File

@@ -42,28 +42,6 @@ def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
assert "500" in str(e.value)
def test_authenticated_from(cmsetup, maildata):
"""Test that envelope FROM must be the same as login."""
user1, user2, user3 = cmsetup.gen_users(3)
msg = maildata("encrypted.eml", from_addr=user2.addr, to_addr=user3.addr)
with pytest.raises(smtplib.SMTPException) as e:
user1.smtp.sendmail(
from_addr=user2.addr, to_addrs=[user3.addr], msg=msg.as_string()
)
assert e.value.recipients[user3.addr][0] == 553
@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
def test_reject_missing_dkim(cmsetup, maildata, from_addr):
"""Test that emails with missing or wrong DMARC, DKIM, and SPF entries are rejected."""
recipient = cmsetup.gen_users(1)[0]
msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
with smtplib.SMTP(cmsetup.maildomain, 25) as s:
with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
@pytest.mark.slow
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
"""Test that the per-account send-mail limit is exceeded."""

View File

@@ -1,10 +1,7 @@
import time
import re
import random
import pytest
import requests
import ipaddress
class TestEndToEndDeltaChat:
@@ -122,29 +119,3 @@ class TestEndToEndDeltaChat:
for msg in msgs:
assert "error" not in m.get_message_info()
time.sleep(1)
def test_hide_senders_ip_address(cmfactory):
public_ip = requests.get("http://icanhazip.com").content.decode().strip()
assert ipaddress.ip_address(public_ip)
user1, user2 = cmfactory.get_online_accounts(2)
chat = cmfactory.get_accepted_chat(user1, user2)
chat.send_text("testing submission header cleanup")
user2.wait_next_incoming_message()
user2.direct_imap.select_folder("Inbox")
msg = user2.direct_imap.get_all_messages()[0]
assert public_ip not in msg.obj.as_string()
def test_echobot(cmfactory, chatmail_config, lp):
ac = cmfactory.get_online_accounts(1)[0]
lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
text = "hi, I hope you text me back"
chat.send_text(text)
lp.sec("Wait for reply from echobot")
reply = ac.wait_next_incoming_message()
assert reply.text == text

View File

@@ -1,6 +1,6 @@
#!/bin/bash
set -e
python3 -m venv --upgrade-deps venv
python3 -m venv venv
venv/bin/pip install -e chatmaild
venv/bin/pip install -e cmdeploy

View File

@@ -7,7 +7,7 @@ Welcome to instant, interoperable and [privacy-preserving](privacy.html) messagi
👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
<a href="DCACCOUNT:https://{{ config.mail_domain }}/cgi-bin/newemail.py">
<img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
🐣 **Choose** your Avatar and Name

View File

@@ -3,7 +3,7 @@
## More information
{{ config.mail_domain }} provides a low-maintenance, resource efficient and
`nine.testrun.org` provides a low-maintenance, resource efficient and
interoperable e-mail service for everyone. What's behind a `chatmail` is
effectively a normal e-mail address just like any other but optimized
for the usage in chats, especially DeltaChat.