Compare commits

..

1 Commits

Author SHA1 Message Date
link2xt
5591920cdc Document email authentication requirements 2024-04-10 17:46:18 +00:00
36 changed files with 136 additions and 299 deletions

View File

@@ -56,7 +56,7 @@ jobs:
# restore acme & dkim state to staging.testrun.org
rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme
- name: run formatting checks
run: cmdeploy fmt -v

View File

@@ -2,27 +2,6 @@
## untagged
- fix writing of multiple obs repositories in `/etc/apt/sources.list`
([#272](https://github.com/deltachat/chatmail/issues/272))
- metadata: add support for `/shared/vendor/deltachat/irohrelay`
([#284](https://github.com/deltachat/chatmail/pull/284))
- Emit "XCHATMAIL" capability from IMAP server
([#278](https://github.com/deltachat/chatmail/pull/278))
- Move echobot `into /var/lib/echobot`
([#281](https://github.com/deltachat/chatmail/pull/281))
- Accept Let's Encrypt's new Terms of Services
([#275](https://github.com/deltachat/chatmail/pull/276))
- Reload Dovecot and Postfix when TLS certificate updates
([#271](https://github.com/deltachat/chatmail/pull/271))
- Use forked version of dovecot without hardcoded delays
([#270](https://github.com/deltachat/chatmail/pull/270))
## 1.2.0 - 2024-04-04
- Install dig on the server to resolve DNS records

View File

@@ -36,16 +36,6 @@ log_format = "%(asctime)s %(levelname)s %(message)s"
log_date_format = "%Y-%m-%d %H:%M:%S"
log_level = "INFO"
[tool.ruff]
lint.select = [
"F", # Pyflakes
"I", # isort
"PLC", # Pylint Convention
"PLE", # Pylint Error
"PLW", # Pylint Warning
]
[tool.tox]
legacy_tox_ini = """
[tox]
@@ -57,9 +47,10 @@ skipdist = True
skip_install = True
deps =
ruff
black
commands =
ruff format --quiet --diff src/
ruff check src/
black --quiet --check --diff src/
ruff src/
[testenv]
deps = pytest

View File

@@ -20,7 +20,6 @@ class Config:
self.passthrough_recipients = params["passthrough_recipients"].split()
self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
self.postfix_reinject_port = int(params["postfix_reinject_port"])
self.iroh_relay = params.get("iroh_relay")
self.privacy_postal = params.get("privacy_postal")
self.privacy_mail = params.get("privacy_mail")
self.privacy_pdo = params.get("privacy_pdo")

View File

@@ -1,5 +1,5 @@
import contextlib
import sqlite3
import contextlib
import time
from pathlib import Path

View File

@@ -1,18 +1,17 @@
import crypt
import json
import logging
import os
import sys
import time
from pathlib import Path
import sys
import json
import crypt
from socketserver import (
UnixStreamServer,
StreamRequestHandler,
ThreadingMixIn,
UnixStreamServer,
)
from .config import Config, read_config
from .database import Database
from .config import read_config, Config
NOCREATE_FILE = "/etc/chatmail-nocreate"
@@ -46,32 +45,23 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
return False
localpart, domain = parts
if localpart == "echo":
# echobot account should not be created in the database
return False
if (
len(localpart) > config.username_max_length
or len(localpart) < config.username_min_length
):
logging.warning(
"localpart %s has to be between %s and %s chars long",
localpart,
config.username_min_length,
config.username_max_length,
)
if localpart != "echo":
logging.warning(
"localpart %s has to be between %s and %s chars long",
localpart,
config.username_min_length,
config.username_max_length,
)
return False
return True
def get_user_data(db, config: Config, user):
if user == f"echo@{config.mail_domain}":
return dict(
home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
uid="vmail",
gid="vmail",
)
with db.read_connection() as conn:
result = conn.get_user(user)
if result:
@@ -86,21 +76,6 @@ def lookup_userdb(db, config: Config, user):
def lookup_passdb(db, config: Config, user, cleartext_password):
if user == f"echo@{config.mail_domain}":
# Echobot writes password it wants to log in with into /run/echobot/password
try:
password = Path("/run/echobot/password").read_text()
except Exception:
logging.exception("Exception when trying to read /run/echobot/password")
return None
return dict(
home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
uid="vmail",
gid="vmail",
password=encrypt_password(password),
)
with db.write_transaction() as conn:
userdata = conn.get_user(user)
if userdata:

View File

@@ -3,17 +3,14 @@
it will echo back any message that has non-empty text and also supports the /help command.
"""
import logging
import os
import subprocess
import sys
from pathlib import Path
from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
from chatmaild.config import read_config
from chatmaild.newemail import create_newemail_dict
from chatmaild.config import read_config
hooks = events.HookCollection()
@@ -78,23 +75,9 @@ def main():
account = accounts[0] if accounts else deltachat.add_account()
bot = Bot(account, hooks)
config = read_config(sys.argv[1])
# Create password file
if bot.is_configured():
password = bot.account.get_config("mail_pw")
else:
password = create_newemail_dict(config)["password"]
Path("/run/echobot/password").write_text(password)
# Give the user which doveauth runs as access to the password file.
subprocess.run(
["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
check=True,
)
if not bot.is_configured():
config = read_config(sys.argv[1])
password = create_newemail_dict(config).get("password")
email = "echo@" + config.mail_domain
bot.configure(email, password)
bot.run_forever()

View File

@@ -1,9 +1,8 @@
import json
import logging
import os
from contextlib import contextmanager
import logging
import json
import filelock
from contextlib import contextmanager
class FileDict:

View File

@@ -1,14 +1,14 @@
#!/usr/bin/env python3
import asyncio
import logging
import sys
import time
from email import policy
import sys
from email.parser import BytesParser
from email import policy
from email.utils import parseaddr
from smtplib import SMTP as SMTPClient
from aiosmtpd.controller import Controller
from smtplib import SMTP as SMTPClient
from .config import read_config

View File

@@ -1,17 +1,17 @@
import logging
import os
import sys
from pathlib import Path
from socketserver import (
UnixStreamServer,
StreamRequestHandler,
ThreadingMixIn,
UnixStreamServer,
)
import sys
import logging
import os
from .config import read_config
from .filedict import FileDict
from .notifier import Notifier
DICTPROXY_HELLO_CHAR = "H"
DICTPROXY_LOOKUP_CHAR = "L"
DICTPROXY_ITERATE_CHAR = "I"
@@ -49,40 +49,32 @@ class Metadata:
return mdict.get(self.DEVICETOKEN_KEY, [])
def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
def handle_dovecot_protocol(rfile, wfile, notifier, metadata):
transactions = {}
while True:
msg = rfile.readline().strip().decode()
if not msg:
break
res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
res = handle_dovecot_request(msg, transactions, notifier, metadata)
if res:
wfile.write(res.encode("ascii"))
wfile.flush()
def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
def handle_dovecot_request(msg, transactions, notifier, metadata):
# see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
short_command = msg[0]
parts = msg[1:].split("\t")
if short_command == DICTPROXY_LOOKUP_CHAR:
# Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
keyparts = parts[0].split("/", 2)
keyparts = parts[0].split("/")
if keyparts[0] == "priv":
keyname = keyparts[2]
addr = parts[1]
if keyname == metadata.DEVICETOKEN_KEY:
res = " ".join(metadata.get_tokens_for_addr(addr))
return f"O{res}\n"
elif keyparts[0] == "shared":
keyname = keyparts[2]
if (
keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
and iroh_relay
):
# Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
return f"O{iroh_relay}\n"
logging.warning("lookup ignored: %r", msg)
return "N\n"
elif short_command == DICTPROXY_ITERATE_CHAR:
@@ -128,10 +120,7 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
def main():
socket, vmail_dir, config_path = sys.argv[1:]
config = read_config(config_path)
iroh_relay = config.iroh_relay
socket, vmail_dir = sys.argv[1:]
vmail_dir = Path(vmail_dir)
if not vmail_dir.exists():
@@ -147,9 +136,7 @@ def main():
class Handler(StreamRequestHandler):
def handle(self):
try:
handle_dovecot_protocol(
self.rfile, self.wfile, notifier, metadata, iroh_relay
)
handle_dovecot_protocol(self.rfile, self.wfile, notifier, metadata)
except Exception:
logging.exception("Exception in the dovecot dictproxy handler")
raise

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import sys
import time
from pathlib import Path
import time
import sys
def main(vmail_dir=None):

View File

@@ -1,13 +1,13 @@
#!/usr/local/lib/chatmaild/venv/bin/python3
"""CGI script for creating new accounts."""
""" CGI script for creating new accounts. """
import json
import random
import secrets
import string
from chatmaild.config import Config, read_config
from chatmaild.config import read_config, Config
CONFIG_PATH = "/usr/local/lib/chatmaild/chatmail.ini"
ALPHANUMERIC = string.ascii_lowercase + string.digits

View File

@@ -25,16 +25,15 @@ The meaning and format of tokens is basically a matter of Delta-Chat Core and
the `notification.delta.chat` service.
"""
import logging
import math
import os
import time
from dataclasses import dataclass
import math
import logging
from uuid import uuid4
from threading import Thread
from pathlib import Path
from queue import PriorityQueue
from threading import Thread
from uuid import uuid4
from dataclasses import dataclass
import requests

View File

@@ -1,14 +1,14 @@
import random
from pathlib import Path
import os
import importlib.resources
import itertools
import os
import random
from email import policy
from email.parser import BytesParser
from pathlib import Path
from email import policy
import pytest
from chatmaild.config import read_config, write_initial_config
from chatmaild.database import Database
from chatmaild.config import read_config, write_initial_config
@pytest.fixture

View File

@@ -1,18 +1,18 @@
import io
import json
import pytest
import queue
import threading
import traceback
import chatmaild.doveauth
import pytest
from chatmaild.database import DBError
from chatmaild.doveauth import (
get_user_data,
handle_dovecot_protocol,
handle_dovecot_request,
lookup_passdb,
handle_dovecot_request,
handle_dovecot_protocol,
)
from chatmaild.database import DBError
def test_basic(db, example_config):

View File

@@ -1,11 +1,12 @@
import pytest
from chatmaild.filtermail import (
check_encrypted,
BeforeQueueHandler,
SendRateLimiter,
check_encrypted,
check_mdn,
)
import pytest
@pytest.fixture
def maildomain():

View File

@@ -1,12 +1,12 @@
import io
import time
import pytest
import requests
import time
from chatmaild.metadata import (
Metadata,
handle_dovecot_protocol,
handle_dovecot_request,
handle_dovecot_protocol,
Metadata,
)
from chatmaild.notifier import (
Notifier,
@@ -296,17 +296,3 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
item2.delete()
assert not item2.path.exists()
assert not queue_item < item2 and not item2 < queue_item
def test_iroh_relay(metadata):
rfile = io.BytesIO(
b"\n".join(
[
b"H",
b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
]
)
)
wfile = io.BytesIO()
handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
assert wfile.getvalue() == b"Ohttps://example.org/\n"

View File

@@ -16,6 +16,7 @@ dependencies = [
"build",
"tox",
"ruff",
"black",
"pytest",
"pytest-xdist",
"imap_tools",
@@ -30,13 +31,3 @@ cmdeploy = "cmdeploy.cmdeploy:main"
[tool.pytest.ini_options]
addopts = "-v -ra --strict-markers"
[tool.ruff]
lint.select = [
"F", # Pyflakes
"I", # isort
"PLC", # Pylint Convention
"PLE", # Pylint Error
"PLW", # Pylint Warning
]

View File

@@ -2,21 +2,21 @@
Chat Mail pyinfra deploy.
"""
import importlib.resources
import io
import shutil
import subprocess
import sys
import importlib.resources
import subprocess
import shutil
import io
from pathlib import Path
from chatmaild.config import Config, read_config
from pyinfra import host
from pyinfra.operations import apt, files, server, systemd, pip
from pyinfra.facts.files import File
from pyinfra.facts.systemd import SystemdEnabled
from pyinfra.operations import apt, files, pip, server, systemd
from .acmetool import deploy_acmetool
from chatmaild.config import read_config, Config
def _build_chatmaild(dist_dir) -> None:
dist_dir = Path(dist_dir).resolve()
@@ -477,31 +477,12 @@ def deploy_chatmail(config_path: Path) -> None:
groups=["opendkim"],
system=True,
)
server.user(name="Create echobot user", user="echobot", system=True)
server.shell(
name="Fix file owner in /home/vmail",
commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
)
# Add our OBS repository for dovecot_no_delay
files.put(
name="Add Deltachat OBS GPG key to apt keyring",
src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
user="root",
group="root",
mode="644",
)
files.line(
name="Add DeltaChat OBS home repository to sources.list",
path="/etc/apt/sources.list",
line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
escape_regex_characters=True,
ensure_newline=True,
)
apt.update(name="apt update", cache_time=24 * 3600)
apt.packages(
@@ -532,15 +513,10 @@ def deploy_chatmail(config_path: Path) -> None:
# Deploy acmetool to have TLS certificates.
deploy_acmetool(
nginx_hook=True,
domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
)
apt.packages(
# required for setfacl for echobot
name="Install acl",
packages="acl",
)
apt.packages(
name="Install Postfix",
packages="postfix",

View File

@@ -1,11 +1,11 @@
import importlib.resources
from pyinfra.operations import apt, files, systemd, server
from pyinfra import host
from pyinfra.facts.systemd import SystemdStatus
from pyinfra.operations import apt, files, server, systemd
def deploy_acmetool(email="", domains=[]):
def deploy_acmetool(nginx_hook=False, email="", domains=[]):
"""Deploy acmetool."""
apt.packages(
name="Install acmetool",
@@ -20,13 +20,16 @@ def deploy_acmetool(email="", domains=[]):
mode="644",
)
files.put(
src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
dest="/usr/lib/acme/hooks/nginx",
user="root",
group="root",
mode="744",
)
if nginx_hook:
files.put(
src=importlib.resources.files(__package__)
.joinpath("acmetool.hook")
.open("rb"),
dest="/usr/lib/acme/hooks/nginx",
user="root",
group="root",
mode="744",
)
files.template(
src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
@@ -71,5 +74,5 @@ def deploy_acmetool(email="", domains=[]):
server.shell(
name=f"Request certificate for: { ', '.join(domains) }",
commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
commands=[f"acmetool want { ' '.join(domains)}"],
)

View File

@@ -3,5 +3,3 @@ set -e
EVENT_NAME="$1"
[ "$EVENT_NAME" = "live-updated" ] || exit 42
systemctl restart nginx.service
systemctl reload dovecot.service
systemctl reload postfix.service

View File

@@ -1,2 +1,2 @@
"acme-enter-email": "{{ email }}"
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true
"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf": true

View File

@@ -4,18 +4,19 @@ along with command line option and subcommand parsing.
"""
import argparse
import shutil
import subprocess
import importlib.resources
import importlib.util
import os
import shutil
import subprocess
import sys
from pathlib import Path
from chatmaild.config import read_config, write_initial_config
from termcolor import colored
from cmdeploy.dns import check_necessary_dns, show_dns
from termcolor import colored
from chatmaild.config import read_config, write_initial_config
from cmdeploy.dns import show_dns, check_necessary_dns
#
# cmdeploy sub commands and options
@@ -156,26 +157,26 @@ def fmt_cmd_options(parser):
def fmt_cmd(args, out):
"""Run formattting fixes on all chatmail source code."""
"""Run formattting fixes (ruff and black) on all chatmail source code."""
sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
format_args = [shutil.which("ruff"), "format"]
check_args = [shutil.which("ruff"), "check"]
black_args = [shutil.which("black")]
ruff_args = [shutil.which("ruff")]
if args.check:
format_args.append("--diff")
black_args.append("--check")
else:
check_args.append("--fix")
ruff_args.append("--fix")
if not args.verbose:
check_args.append("--quiet")
format_args.append("--quiet")
black_args.append("-q")
ruff_args.append("-q")
format_args.extend(sources)
check_args.extend(sources)
black_args.extend(sources)
ruff_args.extend(sources)
out.check_call(" ".join(format_args), quiet=not args.verbose)
out.check_call(" ".join(check_args), quiet=not args.verbose)
out.check_call(" ".join(black_args), quiet=not args.verbose)
out.check_call(" ".join(ruff_args), quiet=not args.verbose)
return 0
@@ -231,7 +232,7 @@ class Out:
if not quiet:
cmdstring = " ".join(args)
self(f"[$ {cmdstring}]", file=sys.stderr)
proc = subprocess.run(args, env=env, check=False)
proc = subprocess.run(args, env=env)
return proc.returncode

View File

@@ -1,8 +1,6 @@
import importlib.resources
import os
import importlib.resources
import pyinfra
from cmdeploy import deploy_chatmail

View File

@@ -1,9 +1,9 @@
import datetime
import importlib
import subprocess
import sys
import requests
import importlib
import subprocess
import datetime
class DNS:
@@ -104,8 +104,8 @@ def show_dns(args, out) -> int:
return 0
except TypeError:
pass
for raw_line in zonefile.splitlines():
line = raw_line.format(
for line in zonefile.splitlines():
line = line.format(
acme_account_url=acme_account_url,
sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
chatmail_domain=args.config.mail_domain,

View File

@@ -27,7 +27,7 @@ mail_plugins = quota
# these are the capabilities Delta Chat cares about actually
# so let's keep the network overhead per login small
# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
# Authentication for system users.

View File

@@ -1,9 +1,8 @@
import importlib
import io
import os
import qrcode
from PIL import Image, ImageDraw, ImageFont
import os
from PIL import ImageFont, ImageDraw, Image
import io
def gen_qr_png_data(maildomain):

View File

@@ -2,7 +2,7 @@
Description=Chatmail dict proxy for IMAP METADATA
[Service]
ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain}
Restart=always
RestartSec=30
User=vmail

View File

@@ -7,20 +7,6 @@ Environment="PATH={remote_venv_dir}:$PATH"
Restart=always
RestartSec=30
User=echobot
Group=echobot
# Create /var/lib/echobot
StateDirectory=echobot
# Create /run/echobot
#
# echobot stores /run/echobot/password
# with a password there, which doveauth then reads.
RuntimeDirectory=echobot
WorkingDirectory=/var/lib/echobot
# Apply security restrictions suggested by
# systemd-analyze security echobot.service
CapabilityBoundingSet=
@@ -30,10 +16,7 @@ NoNewPrivileges=true
PrivateDevices=true
PrivateMounts=true
PrivateTmp=true
# We need to know about doveauth user to give it access to /run/echobot/password
PrivateUsers=false
PrivateUsers=true
ProtectClock=true
ProtectControlGroups=true
ProtectHostname=true

View File

@@ -1,10 +1,9 @@
import pytest
import threading
import queue
import socket
import threading
import pytest
from chatmaild.config import read_config
from cmdeploy.cmdeploy import main
@@ -15,13 +14,6 @@ def test_init(tmp_path, maildomain):
assert config.mail_domain == maildomain
def test_capabilities(imap):
imap.connect()
capas = imap.conn.capabilities
assert "XCHATMAIL" in capas
assert "XDELTAPUSH" in capas
def test_login_basic_functioning(imap_or_smtp, gencreds, lp):
"""Test a) that an initial login creates a user automatically
and b) verify we can also login a second time with the same password

View File

@@ -1,5 +1,4 @@
import smtplib
import pytest

View File

@@ -1,11 +1,11 @@
import ipaddress
import random
import re
import time
import re
import random
import imap_tools
import pytest
import requests
import ipaddress
import imap_tools
@pytest.fixture

View File

@@ -1,16 +1,17 @@
import imaplib
import io
import itertools
import os
import random
import smtplib
import subprocess
import io
import time
import random
import subprocess
import imaplib
import smtplib
import itertools
from pathlib import Path
import pytest
from chatmaild.config import read_config
from chatmaild.database import Database
from chatmaild.config import read_config
conftestdir = Path(__file__).parent

View File

@@ -1,7 +1,6 @@
import os
import pytest
from cmdeploy.cmdeploy import get_parser, main

View File

@@ -1,14 +1,13 @@
import hashlib
import importlib.resources
import webbrowser
import hashlib
import time
import traceback
import webbrowser
import markdown
from chatmaild.config import read_config
from jinja2 import Template
from .genqr import gen_qr_png_data
from chatmaild.config import read_config
def snapshot_dir_stats(somedir):
@@ -121,8 +120,7 @@ def main():
print(f"watching {src_path} directory for changes")
changenum = 0
count = 0
while True:
for count in range(0, 1000000):
newstats = snapshot_dir_stats(src_path)
if newstats == stats and count % 60 != 0:
count += 1