Compare commits

..

1 Commits

Author SHA1 Message Date
link2xt
569a043065 Add execnet dependency 2025-12-06 17:18:24 +00:00
21 changed files with 354 additions and 116 deletions

View File

@@ -11,9 +11,6 @@ jobs:
scripts:
name: build
runs-on: ubuntu-latest
environment:
name: 'staging.chatmail.at/doc/relay/'
url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
steps:
- uses: actions/checkout@v4
@@ -47,6 +44,36 @@ jobs:
chmod 600 "$HOME/.ssh/key"
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
- name: "Post links to details"
id: details
if: steps.prepare.outputs.uploadtoserver
run: |
# URLs for API connection and uploads
export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
export STATUS_DATA="{\"state\": \"success\", \
\"description\": \"Preview the changed documentation here:\", \
\"context\": \"Documentation Preview\", \
\"target_url\": \"${PREVIEW_LINK}\"}"
curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
#check if comment already exists, if not post it
export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
echo $RESPONSE > response
grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
- name: "Post link to comments"
if: steps.details.outputs.comment
uses: actions/github-script@v7
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
})
- name: check links
working-directory: doc
run: sphinx-build --builder linkcheck source build

View File

@@ -14,9 +14,6 @@ jobs:
scripts:
name: build
runs-on: ubuntu-latest
environment:
name: 'chatmail.at/doc/relay/'
url: https://chatmail.at/doc/relay/
steps:
- uses: actions/checkout@v4

View File

@@ -16,9 +16,6 @@ jobs:
name: deploy on staging-ipv4.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency:
group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}

View File

@@ -16,9 +16,6 @@ jobs:
name: deploy on staging2.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -73,9 +70,6 @@ jobs:
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: add hpk42 key to staging server
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy

View File

@@ -2,11 +2,6 @@
## untagged
- Add imap_compress option to chatmail.ini (#760)
- Remove echobot from relays
([#753](https://github.com/chatmail/relay/pull/753))
- Add robots.txt to exclude all web crawlers
([#732](https://github.com/chatmail/relay/pull/732))

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "chatmaild"
version = "0.3"
version = "0.2"
dependencies = [
"aiosmtpd",
"iniconfig",
@@ -25,6 +25,7 @@ where = ['src']
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
filtermail = "chatmaild.filtermail:main"
echobot = "chatmaild.echo:main"
chatmail-metrics = "chatmaild.metrics:main"
chatmail-expire = "chatmaild.expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"

View File

@@ -4,6 +4,8 @@ import iniconfig
from chatmaild.user import User
echobot_password_path = Path("/run/echobot/password")
def read_config(inipath):
assert Path(inipath).exists(), inipath
@@ -44,7 +46,6 @@ class Config:
self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
self.acme_email = params.get("acme_email", "")
self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
self.imap_compress = params.get("imap_compress", "false").lower() == "true"
if "iroh_relay" not in params:
self.iroh_relay = "https://" + params["mail_domain"]
self.enable_iroh_relay = True
@@ -71,7 +72,10 @@ class Config:
raise ValueError(f"invalid address {addr!r}")
maildir = self.mailboxes_dir.joinpath(addr)
password_path = maildir.joinpath("password")
if addr.startswith("echo@"):
password_path = echobot_password_path
else:
password_path = maildir.joinpath("password")
return User(maildir, addr, password_path, uid="vmail", gid="vmail")

View File

@@ -40,6 +40,10 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
return False
localpart, domain = parts
if localpart == "echo":
# echobot account should not be created in the database
return False
if (
len(localpart) > config.username_max_length
or len(localpart) < config.username_min_length

View File

@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Advanced echo bot example.
it will echo back any message that has non-empty text and also supports the /help command.
"""
import logging
import os
import subprocess
import sys
from pathlib import Path
from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
from chatmaild.config import echobot_password_path, read_config
from chatmaild.doveauth import encrypt_password
from chatmaild.newemail import create_newemail_dict
hooks = events.HookCollection()
@hooks.on(events.RawEvent)
def log_event(event):
if event.kind == EventType.INFO:
logging.info(event.msg)
elif event.kind == EventType.WARNING:
logging.warning(event.msg)
@hooks.on(events.RawEvent(EventType.ERROR))
def log_error(event):
logging.error("%s", event.msg)
@hooks.on(events.MemberListChanged)
def on_memberlist_changed(event):
logging.info(
"member %s was %s", event.member, "added" if event.member_added else "removed"
)
@hooks.on(events.GroupImageChanged)
def on_group_image_changed(event):
logging.info("group image %s", "deleted" if event.image_deleted else "changed")
@hooks.on(events.GroupNameChanged)
def on_group_name_changed(event):
logging.info(f"group name changed, old name: {event.old_name}")
@hooks.on(events.NewMessage(func=lambda e: not e.command))
def echo(event):
snapshot = event.message_snapshot
if snapshot.is_info:
# Ignore info messages
return
if snapshot.text or snapshot.file:
snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
@hooks.on(events.NewMessage(command="/help"))
def help_command(event):
snapshot = event.message_snapshot
snapshot.chat.send_text("Send me any message and I will echo it back")
def main():
logging.basicConfig(level=logging.INFO)
path = os.environ.get("PATH")
venv_path = sys.argv[0].strip("echobot")
os.environ["PATH"] = path + ":" + venv_path
with Rpc() as rpc:
deltachat = DeltaChat(rpc)
system_info = deltachat.get_system_info()
logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
accounts = deltachat.get_all_accounts()
account = accounts[0] if accounts else deltachat.add_account()
bot = Bot(account, hooks)
config = read_config(sys.argv[1])
addr = "echo@" + config.mail_domain
# Create password file
if bot.is_configured():
password = bot.account.get_config("mail_pw")
else:
password = create_newemail_dict(config)["password"]
echobot_password_path.write_text(encrypt_password(password))
# Give the user which doveauth runs as access to the password file.
subprocess.check_call(
["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
)
if not bot.is_configured():
bot.configure(addr, password)
# write invite link to working directory
invitelink = bot.account.get_qr_code()
Path("invite-link.txt").write_text(invitelink)
bot.run_forever()
if __name__ == "__main__":
main()

View File

@@ -45,7 +45,7 @@ passthrough_senders =
# (space-separated, item may start with "@" to whitelist whole recipient domains)
passthrough_recipients = echo@{mail_domain}
# path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
# path to www directory - documented here: https://github.com/chatmail/relay/#custom-web-pages
#www_folder = www
#
@@ -99,12 +99,6 @@ acme_email =
# so use this option with caution on production servers.
imap_rawlog = false
# set to true if you want to enable the IMAP COMPRESS Extension,
# which allows IMAP connections to be efficiently compressed.
# WARNING: Enabling this makes it impossible to hibernate IMAP
# processes which will result in much higher memory/RAM usage.
imap_compress = false
#
# Privacy Policy

View File

@@ -36,3 +36,29 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
assert res == "O\n"
assert len(dictproxy_transactions) == 0
def test_handle_dovecot_request_last_login_echobot(example_config):
dictproxy = LastLoginDictProxy(config=example_config)
authproxy = AuthDictProxy(config=example_config)
testaddr = f"echo@{example_config.mail_domain}"
authproxy.lookup_passdb(testaddr, "ignore")
user = dictproxy.config.get_user(testaddr)
transactions = {}
# set last-login info for user
tx = "1111"
msg = f"B{tx}\t{testaddr}"
res = dictproxy.handle_dovecot_request(msg, transactions)
assert not res
assert transactions == {tx: dict(addr=testaddr, res="O\n")}
timestamp = int(time.time())
msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
res = dictproxy.handle_dovecot_request(msg, transactions)
assert not res
assert len(transactions) == 1
read_timestamp = user.get_last_login_timestamp()
assert read_timestamp is None

View File

@@ -109,6 +109,15 @@ def run_cmd(args, out):
try:
retcode = out.check_call(cmd, env=env)
if retcode == 0:
if not args.disable_mail:
print("\nYou can try out the relay by talking to this echo bot: ")
sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
print(
sshexec(
call=remote.rshell.shell,
kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
)
)
out.green("Deploy completed, call `cmdeploy dns` next.")
elif not remote_data["acme_account_url"]:
out.red("Deploy completed but letsencrypt not configured")

View File

@@ -270,14 +270,6 @@ class LegacyRemoveDeployer(Deployer):
path="/var/log/journal/",
present=False,
)
# remove echobot if it is still running
if host.get_fact(SystemdEnabled).get("echobot.service"):
systemd.service(
name="Disable echobot.service",
service="echobot.service",
running=False,
enabled=False,
)
def check_config(config):
@@ -412,6 +404,30 @@ class JournaldDeployer(Deployer):
self.need_restart = False
class EchobotDeployer(Deployer):
#
# This deployer depends on the dovecot and postfix deployers because
# it needs to base its decision of whether to restart the service on
# whether those two services were restarted.
#
def __init__(self, mail_domain):
self.mail_domain = mail_domain
self.units = ["echobot"]
def install(self):
apt.packages(
# required for setfacl for echobot
name="Install acl",
packages="acl",
)
def configure(self):
configure_remote_units(self.mail_domain, self.units)
def activate(self):
activate_remote_units(self.units)
class ChatmailVenvDeployer(Deployer):
def __init__(self, config):
self.config = config
@@ -574,6 +590,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
PostfixDeployer(config, disable_mail),
FcgiwrapDeployer(),
NginxDeployer(config),
EchobotDeployer(mail_domain),
MtailDeployer(config.mtail_address),
GithashDeployer(),
]

View File

@@ -113,7 +113,7 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
# `imap_zlib` enables IMAP COMPRESS (RFC 4978).
# <https://datatracker.ietf.org/doc/html/rfc4978.html>
protocol imap {
mail_plugins = $mail_plugins imap_quota last_login {% if config.imap_compress %}imap_zlib{% endif %}
mail_plugins = $mail_plugins imap_zlib imap_quota last_login
imap_metadata = yes
}
@@ -252,28 +252,3 @@ protocol imap {
rawlog_dir = %h
}
{% endif %}
{% if not config.imap_compress %}
# Hibernate IDLE users to save memory and CPU resources
# NOTE: this will have no effect if imap_zlib plugin is used
imap_hibernate_timeout = 30s
service imap {
# Note that this change will allow any process running as
# $default_internal_user (dovecot) to access mails as any other user.
# This may be insecure in some installations, which is why this isn't
# done by default.
unix_listener imap-master {
user = $default_internal_user
}
}
# The following is the default already in v2.3.1+:
service imap {
extra_groups = $default_internal_group
}
service imap-hibernate {
unix_listener imap-hibernate {
mode = 0660
group = $default_internal_group
}
}
{% endif %}

View File

@@ -37,10 +37,7 @@ def perform_initial_checks(mail_domain, pre_command=""):
return res
# parse out sts-id if exists, example: "v=STSv1; id=2090123"
mta_sts_txt = query_dns("TXT", f"_mta-sts.{mail_domain}")
if not mta_sts_txt:
return res
parts = mta_sts_txt.split("id=")
parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
return res

View File

@@ -0,0 +1,67 @@
[Unit]
Description=Chatmail echo bot for testing it works
[Service]
ExecStart={execpath} {config_path}
Environment="PATH={remote_venv_dir}:$PATH"
Restart=always
RestartSec=30
User=echobot
Group=echobot
# Create /var/lib/echobot
StateDirectory=echobot
# Create /run/echobot
#
# echobot stores /run/echobot/password
# with a password there, which doveauth then reads.
RuntimeDirectory=echobot
WorkingDirectory=/var/lib/echobot
# Apply security restrictions suggested by
# systemd-analyze security echobot.service
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
NoNewPrivileges=true
PrivateDevices=true
PrivateMounts=true
PrivateTmp=true
# We need to know about doveauth user to give it access to /run/echobot/password
PrivateUsers=false
ProtectClock=true
ProtectControlGroups=true
ProtectHostname=true
ProtectKernelLogs=true
ProtectKernelModules=true
ProtectKernelTunables=true
ProtectProc=noaccess
# Should be "strict", but we currently write /accounts folder in a protected path
ProtectSystem=full
RemoveIPC=true
RestrictAddressFamilies=AF_INET AF_INET6
RestrictNamespaces=true
RestrictRealtime=true
RestrictSUIDSGID=true
SystemCallArchitectures=native
SystemCallFilter=~@clock
SystemCallFilter=~@cpu-emulation
SystemCallFilter=~@debug
SystemCallFilter=~@module
SystemCallFilter=~@mount
SystemCallFilter=~@obsolete
SystemCallFilter=~@raw-io
SystemCallFilter=~@reboot
SystemCallFilter=~@resources
SystemCallFilter=~@swap
UMask=0077
[Install]
WantedBy=multi-user.target

View File

@@ -1,5 +1,5 @@
import datetime
#import os
import os
import smtplib
import socket
import subprocess
@@ -8,7 +8,7 @@ import time
import pytest
from cmdeploy import remote
#from cmdeploy.cmdeploy import main
from cmdeploy.cmdeploy import main
from cmdeploy.sshexec import SSHExec
@@ -70,44 +70,45 @@ class TestSSHExecutor:
assert (now - since_date).total_seconds() < 60 * 60 * 51
#def test_status_cmd(chatmail_config, capsys, request):
# os.chdir(request.config.invocation_params.dir)
# assert main(["status"]) == 0
# status_out = capsys.readouterr()
# print(status_out.out)
#
# services = [
# "acmetool-redirector",
# "chatmail-metadata",
# "doveauth",
# "dovecot",
# "fcgiwrap",
# "filtermail-incoming",
# "filtermail",
# "lastlogin",
# "nginx",
# "opendkim",
# "postfix@-",
# "systemd-journald",
# "turnserver",
# "unbound",
# ]
# not_running = []
# for service in services:
# active = False
# for line in status_out:
# if service in line:
# active = True
# if not "loaded" in line:
# active = False
# if not "active" in line:
# active = False
# if not "running" in line:
# active = False
# break
# if not active:
# not_running.append(service)
# assert not_running == []
def test_status_cmd(chatmail_config, capsys, request):
os.chdir(request.config.invocation_params.dir)
assert main(["status"]) == 0
status_out = capsys.readouterr()
print(status_out.out)
services = [
"acmetool-redirector",
"chatmail-metadata",
"doveauth",
"dovecot",
"echobot",
"fcgiwrap",
"filtermail-incoming",
"filtermail",
"lastlogin",
"nginx",
"opendkim",
"postfix@-",
"systemd-journald",
"turnserver",
"unbound",
]
not_running = []
for service in services:
active = False
for line in status_out:
if service in line:
active = True
if not "loaded" in line:
active = False
if not "active" in line:
active = False
if not "running" in line:
active = False
break
if not active:
not_running.append(service)
assert not_running == []
def test_timezone_env(remote):

View File

@@ -160,3 +160,22 @@ def test_hide_senders_ip_address(cmfactory):
user2.direct_imap.select_folder("Inbox")
msg = user2.direct_imap.get_all_messages()[0]
assert public_ip not in msg.obj.as_string()
def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
ac = cmfactory.get_online_accounts(1)[0]
# establish contact with echobot
sshexec = SSHExec(sshdomain)
command = "cat /var/lib/echobot/invite-link.txt"
echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
chat = ac.qr_setup_contact(echo_invite_link)
ac._evtracker.wait_securejoin_joiner_progress(1000)
# send message and check it gets replied back
lp.sec("Send message to echobot")
text = "hi, I hope you text me back"
chat.send_text(text)
lp.sec("Wait for reply from echobot")
reply = ac._evtracker.wait_next_incoming_message()
assert reply.text == text

View File

@@ -166,7 +166,7 @@ def main():
build_webpages(src_path, build_dir, config)
print(f"[{changenum}] regenerated web pages at: {index_path}")
print(f"URL: file://{index_path.resolve()}\n\n")
time.sleep(debounce_time) # simple debounce

View File

@@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
or receive messages until the migration is completed.
2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
``/etc/dkimkeys``, and ``/var/spool/postfix`` to
``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
the new site. Login to the old site while forwarding your SSH agent
so you can copy directly from the old to the new site with your SSH
key:
@@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
::
ssh -A root@13.37.13.37
tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
This transfers all addresses, the TLS certificate,
and DKIM keys (so DKIM DNS record remains valid).
It also preserves the Postfix mail spool so any messages
This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
DNS record remains valid), and the echobots password so it continues
to function. It also preserves the Postfix mail spool so any messages
pending delivery will still be delivered.
3. Install chatmail on the new machine:
@@ -58,6 +58,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
chown root: -R /var/lib/acme
chown opendkim: -R /etc/dkimkeys
chown vmail: -R /home/vmail/mail
chown echobot: -R /run/echobot
5. Now, update DNS entries.

View File

@@ -109,6 +109,10 @@ short overview of ``chatmaild`` services:
is contacted by Dovecot when a user logs in and stores the date of
the login.
- `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
is a small bot for test purposes. It simply echoes back messages from
users.
- `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
collects some metrics and displays them at
``https://example.org/metrics``.
@@ -272,8 +276,8 @@ by OpenDKIM screen policy script before validating the signatures. This
corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
If there is no valid DKIM signature on the incoming email, the
sender receives a “5.7.1 No valid DKIM signature found” error.
After validating the DKIM signature,
the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
After validating the DKIM signature,
the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
Note that chatmail relays