Compare commits

..

1 Commits

Author SHA1 Message Date
missytake
cf934b966a echobot: require dovecot to be running 2025-12-01 10:31:54 +01:00
21 changed files with 337 additions and 135 deletions

View File

@@ -11,9 +11,6 @@ jobs:
scripts:
name: build
runs-on: ubuntu-latest
environment:
name: 'staging.chatmail.at/doc/relay/'
url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
steps:
- uses: actions/checkout@v4
@@ -47,6 +44,36 @@ jobs:
chmod 600 "$HOME/.ssh/key"
rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
- name: "Post links to details"
id: details
if: steps.prepare.outputs.uploadtoserver
run: |
# URLs for API connection and uploads
export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
export STATUS_DATA="{\"state\": \"success\", \
\"description\": \"Preview the changed documentation here:\", \
\"context\": \"Documentation Preview\", \
\"target_url\": \"${PREVIEW_LINK}\"}"
curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
#check if comment already exists, if not post it
export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
echo $RESPONSE > response
grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
- name: "Post link to comments"
if: steps.details.outputs.comment
uses: actions/github-script@v7
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
})
- name: check links
working-directory: doc
run: sphinx-build --builder linkcheck source build

View File

@@ -14,9 +14,6 @@ jobs:
scripts:
name: build
runs-on: ubuntu-latest
environment:
name: 'chatmail.at/doc/relay/'
url: https://chatmail.at/doc/relay/
steps:
- uses: actions/checkout@v4

View File

@@ -16,9 +16,6 @@ jobs:
name: deploy on staging-ipv4.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging-ipv4.testrun.org
url: https://staging-ipv4.testrun.org/
concurrency:
group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}

View File

@@ -16,9 +16,6 @@ jobs:
name: deploy on staging2.testrun.org, and run tests
runs-on: ubuntu-latest
timeout-minutes: 30
environment:
name: staging2.testrun.org
url: https://staging2.testrun.org/
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -73,9 +70,6 @@ jobs:
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: add hpk42 key to staging server
run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy

View File

@@ -2,9 +2,6 @@
## untagged
- Remove echobot from relays
([#753](https://github.com/chatmail/relay/pull/753))
- Add robots.txt to exclude all web crawlers
([#732](https://github.com/chatmail/relay/pull/732))

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "chatmaild"
version = "0.3"
version = "0.2"
dependencies = [
"aiosmtpd",
"iniconfig",
@@ -25,6 +25,7 @@ where = ['src']
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
filtermail = "chatmaild.filtermail:main"
echobot = "chatmaild.echo:main"
chatmail-metrics = "chatmaild.metrics:main"
chatmail-expire = "chatmaild.expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
@@ -72,6 +73,5 @@ commands =
deps = pytest
pdbpp
pytest-localserver
execnet
commands = pytest -v -rsXx {posargs}
"""

View File

@@ -4,6 +4,8 @@ import iniconfig
from chatmaild.user import User
echobot_password_path = Path("/run/echobot/password")
def read_config(inipath):
assert Path(inipath).exists(), inipath
@@ -70,7 +72,10 @@ class Config:
raise ValueError(f"invalid address {addr!r}")
maildir = self.mailboxes_dir.joinpath(addr)
password_path = maildir.joinpath("password")
if addr.startswith("echo@"):
password_path = echobot_password_path
else:
password_path = maildir.joinpath("password")
return User(maildir, addr, password_path, uid="vmail", gid="vmail")

View File

@@ -40,6 +40,10 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
return False
localpart, domain = parts
if localpart == "echo":
# echobot account should not be created in the database
return False
if (
len(localpart) > config.username_max_length
or len(localpart) < config.username_min_length

View File

@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Advanced echo bot example.
it will echo back any message that has non-empty text and also supports the /help command.
"""
import logging
import os
import subprocess
import sys
from pathlib import Path
from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
from chatmaild.config import echobot_password_path, read_config
from chatmaild.doveauth import encrypt_password
from chatmaild.newemail import create_newemail_dict
hooks = events.HookCollection()
@hooks.on(events.RawEvent)
def log_event(event):
if event.kind == EventType.INFO:
logging.info(event.msg)
elif event.kind == EventType.WARNING:
logging.warning(event.msg)
@hooks.on(events.RawEvent(EventType.ERROR))
def log_error(event):
logging.error("%s", event.msg)
@hooks.on(events.MemberListChanged)
def on_memberlist_changed(event):
logging.info(
"member %s was %s", event.member, "added" if event.member_added else "removed"
)
@hooks.on(events.GroupImageChanged)
def on_group_image_changed(event):
logging.info("group image %s", "deleted" if event.image_deleted else "changed")
@hooks.on(events.GroupNameChanged)
def on_group_name_changed(event):
logging.info(f"group name changed, old name: {event.old_name}")
@hooks.on(events.NewMessage(func=lambda e: not e.command))
def echo(event):
snapshot = event.message_snapshot
if snapshot.is_info:
# Ignore info messages
return
if snapshot.text or snapshot.file:
snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
@hooks.on(events.NewMessage(command="/help"))
def help_command(event):
snapshot = event.message_snapshot
snapshot.chat.send_text("Send me any message and I will echo it back")
def main():
logging.basicConfig(level=logging.INFO)
path = os.environ.get("PATH")
venv_path = sys.argv[0].strip("echobot")
os.environ["PATH"] = path + ":" + venv_path
with Rpc() as rpc:
deltachat = DeltaChat(rpc)
system_info = deltachat.get_system_info()
logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
accounts = deltachat.get_all_accounts()
account = accounts[0] if accounts else deltachat.add_account()
bot = Bot(account, hooks)
config = read_config(sys.argv[1])
addr = "echo@" + config.mail_domain
# Create password file
if bot.is_configured():
password = bot.account.get_config("mail_pw")
else:
password = create_newemail_dict(config)["password"]
echobot_password_path.write_text(encrypt_password(password))
# Give the user which doveauth runs as access to the password file.
subprocess.check_call(
["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
)
if not bot.is_configured():
bot.configure(addr, password)
# write invite link to working directory
invitelink = bot.account.get_qr_code()
Path("invite-link.txt").write_text(invitelink)
bot.run_forever()
if __name__ == "__main__":
main()

View File

@@ -20,7 +20,7 @@ def create_newemail_dict(config: Config):
secrets.choice(ALPHANUMERIC_PUNCT)
for _ in range(config.password_min_length + 3)
)
return dict(email=f"{user}@bloc7.icu", password=f"{password}")
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
def print_new_account():

View File

@@ -36,3 +36,29 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
assert res == "O\n"
assert len(dictproxy_transactions) == 0
def test_handle_dovecot_request_last_login_echobot(example_config):
dictproxy = LastLoginDictProxy(config=example_config)
authproxy = AuthDictProxy(config=example_config)
testaddr = f"echo@{example_config.mail_domain}"
authproxy.lookup_passdb(testaddr, "ignore")
user = dictproxy.config.get_user(testaddr)
transactions = {}
# set last-login info for user
tx = "1111"
msg = f"B{tx}\t{testaddr}"
res = dictproxy.handle_dovecot_request(msg, transactions)
assert not res
assert transactions == {tx: dict(addr=testaddr, res="O\n")}
timestamp = int(time.time())
msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
res = dictproxy.handle_dovecot_request(msg, transactions)
assert not res
assert len(transactions) == 1
read_timestamp = user.get_last_login_timestamp()
assert read_timestamp is None

View File

@@ -109,6 +109,15 @@ def run_cmd(args, out):
try:
retcode = out.check_call(cmd, env=env)
if retcode == 0:
if not args.disable_mail:
print("\nYou can try out the relay by talking to this echo bot: ")
sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
print(
sshexec(
call=remote.rshell.shell,
kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
)
)
out.green("Deploy completed, call `cmdeploy dns` next.")
elif not remote_data["acme_account_url"]:
out.red("Deploy completed but letsencrypt not configured")

View File

@@ -270,14 +270,6 @@ class LegacyRemoveDeployer(Deployer):
path="/var/log/journal/",
present=False,
)
# remove echobot if it is still running
if host.get_fact(SystemdEnabled).get("echobot.service"):
systemd.service(
name="Disable echobot.service",
service="echobot.service",
running=False,
enabled=False,
)
def check_config(config):
@@ -412,6 +404,30 @@ class JournaldDeployer(Deployer):
self.need_restart = False
class EchobotDeployer(Deployer):
#
# This deployer depends on the dovecot and postfix deployers because
# it needs to base its decision of whether to restart the service on
# whether those two services were restarted.
#
def __init__(self, mail_domain):
self.mail_domain = mail_domain
self.units = ["echobot"]
def install(self):
apt.packages(
# required for setfacl for echobot
name="Install acl",
packages="acl",
)
def configure(self):
configure_remote_units(self.mail_domain, self.units)
def activate(self):
activate_remote_units(self.units)
class ChatmailVenvDeployer(Deployer):
def __init__(self, config):
self.config = config
@@ -574,6 +590,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
PostfixDeployer(config, disable_mail),
FcgiwrapDeployer(),
NginxDeployer(config),
EchobotDeployer(mail_domain),
MtailDeployer(config.mtail_address),
GithashDeployer(),
]

View File

@@ -9,10 +9,9 @@ if nsigs == nil then
return nil
end
local valid = false
for i = 1, nsigs do
sig = odkim.get_sighandle(ctx, i - 1)
sigres = odkim.sig_result(sig)
sig = odkim.get_sighandle(ctx, i - 1)
sigres = odkim.sig_result(sig)
-- All signatures that do not correspond to From:
-- were ignored in screen.lua and return sigres -1.
@@ -20,19 +19,10 @@ for i = 1, nsigs do
-- Any valid signature that was not ignored like this
-- means the message is acceptable.
if sigres == 0 then
valid = true
end
end
if valid then
-- Strip all DKIM-Signature headers after successful validation
-- Delete in reverse order to avoid index shifting.
for i = nsigs, 1, -1 do
odkim.del_header(ctx, "DKIM-Signature", i)
end
else
odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
odkim.set_result(ctx, SMFIS_REJECT)
return nil
end
end
odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
odkim.set_result(ctx, SMFIS_REJECT)
return nil

View File

@@ -0,0 +1,68 @@
[Unit]
Description=Chatmail echo bot for testing it works
Requires=dovecot.service
[Service]
ExecStart={execpath} {config_path}
Environment="PATH={remote_venv_dir}:$PATH"
Restart=always
RestartSec=30
User=echobot
Group=echobot
# Create /var/lib/echobot
StateDirectory=echobot
# Create /run/echobot
#
# echobot stores /run/echobot/password
# with a password there, which doveauth then reads.
RuntimeDirectory=echobot
WorkingDirectory=/var/lib/echobot
# Apply security restrictions suggested by
# systemd-analyze security echobot.service
CapabilityBoundingSet=
LockPersonality=true
MemoryDenyWriteExecute=true
NoNewPrivileges=true
PrivateDevices=true
PrivateMounts=true
PrivateTmp=true
# We need to know about doveauth user to give it access to /run/echobot/password
PrivateUsers=false
ProtectClock=true
ProtectControlGroups=true
ProtectHostname=true
ProtectKernelLogs=true
ProtectKernelModules=true
ProtectKernelTunables=true
ProtectProc=noaccess
# Should be "strict", but we currently write /accounts folder in a protected path
ProtectSystem=full
RemoveIPC=true
RestrictAddressFamilies=AF_INET AF_INET6
RestrictNamespaces=true
RestrictRealtime=true
RestrictSUIDSGID=true
SystemCallArchitectures=native
SystemCallFilter=~@clock
SystemCallFilter=~@cpu-emulation
SystemCallFilter=~@debug
SystemCallFilter=~@module
SystemCallFilter=~@mount
SystemCallFilter=~@obsolete
SystemCallFilter=~@raw-io
SystemCallFilter=~@reboot
SystemCallFilter=~@resources
SystemCallFilter=~@swap
UMask=0077
[Install]
WantedBy=multi-user.target

View File

@@ -81,6 +81,7 @@ def test_status_cmd(chatmail_config, capsys, request):
"chatmail-metadata",
"doveauth",
"dovecot",
"echobot",
"fcgiwrap",
"filtermail-incoming",
"filtermail",

View File

@@ -160,3 +160,22 @@ def test_hide_senders_ip_address(cmfactory):
user2.direct_imap.select_folder("Inbox")
msg = user2.direct_imap.get_all_messages()[0]
assert public_ip not in msg.obj.as_string()
def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
ac = cmfactory.get_online_accounts(1)[0]
# establish contact with echobot
sshexec = SSHExec(sshdomain)
command = "cat /var/lib/echobot/invite-link.txt"
echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
chat = ac.qr_setup_contact(echo_invite_link)
ac._evtracker.wait_securejoin_joiner_progress(1000)
# send message and check it gets replied back
lp.sec("Send message to echobot")
text = "hi, I hope you text me back"
chat.send_text(text)
lp.sec("Wait for reply from echobot")
reply = ac._evtracker.wait_next_incoming_message()
assert reply.text == text

View File

@@ -140,34 +140,34 @@ def main():
config.webdev = True
assert config.mail_domain
# start web page generation, open a browser and wait for changes
www_path, src_path, build_dir = get_paths(config)
build_dir = build_webpages(src_path, build_dir, config)
index_path = build_dir.joinpath("index.html")
webbrowser.open(str(index_path))
print(f"\nOpened URL: file://{index_path.resolve()}\n")
print(f"Watching {src_path} directory for changes...")
stats = snapshot_dir_stats(src_path)
print(f"\nOpened URL: file://{index_path.resolve()}\n")
print(f"watching {src_path} directory for changes")
changenum = 0
debounce_time = 0.5 # wait 0.5s after detecting a change
count = 0
while True:
time.sleep(1)
newstats = snapshot_dir_stats(src_path)
if newstats == stats and count % 60 != 0:
count += 1
time.sleep(1.0)
continue
if newstats != stats:
changed_files = [f for f in newstats if stats.get(f) != newstats[f]]
for f in changed_files:
print(f"*** CHANGED: {f}")
for key in newstats:
if stats[key] != newstats[key]:
print(f"*** CHANGED: {key}")
changenum += 1
stats = newstats
changenum += 1
build_webpages(src_path, build_dir, config)
print(f"[{changenum}] regenerated web pages at: {index_path}")
print(f"URL: file://{index_path.resolve()}\n\n")
time.sleep(debounce_time) # simple debounce
stats = newstats
build_webpages(src_path, build_dir, config)
print(f"[{changenum}] regenerated web pages at: {index_path}")
print(f"URL: file://{index_path.resolve()}\n\n")
count = 0
if __name__ == "__main__":

View File

@@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
or receive messages until the migration is completed.
2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
``/etc/dkimkeys``, and ``/var/spool/postfix`` to
``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
the new site. Login to the old site while forwarding your SSH agent
so you can copy directly from the old to the new site with your SSH
key:
@@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
::
ssh -A root@13.37.13.37
tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
This transfers all addresses, the TLS certificate,
and DKIM keys (so DKIM DNS record remains valid).
It also preserves the Postfix mail spool so any messages
This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
DNS record remains valid), and the echobots password so it continues
to function. It also preserves the Postfix mail spool so any messages
pending delivery will still be delivered.
3. Install chatmail on the new machine:
@@ -58,6 +58,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
chown root: -R /var/lib/acme
chown opendkim: -R /etc/dkimkeys
chown vmail: -R /home/vmail/mail
chown echobot: -R /run/echobot
5. Now, update DNS entries.

View File

@@ -109,6 +109,10 @@ short overview of ``chatmaild`` services:
is contacted by Dovecot when a user logs in and stores the date of
the login.
- `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
is a small bot for test purposes. It simply echoes back messages from
users.
- `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
collects some metrics and displays them at
``https://example.org/metrics``.
@@ -269,11 +273,9 @@ Incoming emails must have a valid DKIM signature with
Signing Domain Identifier (SDID, ``d=`` parameter in the DKIM-Signature
header) equal to the ``From:`` header domain. This property is checked
by OpenDKIM screen policy script before validating the signatures. This
corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
correpsonds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
If there is no valid DKIM signature on the incoming email, the
sender receives a “5.7.1 No valid DKIM signature found” error.
After validating the DKIM signature,
the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
Note that chatmail relays

View File

@@ -22,12 +22,7 @@ Note that your chatmail relay still needs to be able to make outgoing
connections on port 25 to send messages outside.
To setup a reverse proxy (or rather Destination NAT, DNAT) for your
chatmail relay, follow these instructions:
Linux
^^^^^
Put the following configuration in
chatmail relay, put the following configuration in
``/etc/nftables.conf``:
::
@@ -115,61 +110,5 @@ Uncomment in ``/etc/sysctl.conf`` the following two lines:
Then reboot the relay or do ``sysctl -p`` and
``nft -f /etc/nftables.conf``.
FreeBSD / pf
^^^^^^^^^^^^
Put the following configuration in
``/etc/pf.conf``:
::
ext_if = "em0"
forward_ports = "{ 25, 80, 143, 443, 465, 587, 993 }"
chatmail_ipv4 = "AAA.BBB.CCC.DDD"
icmp_types = "{ echoreq, echorep, unreach, timex }"
chatmail_ipv6 = "XXX::1"
icmp6_types = "{ echorep, echoreq, neighbradv, neighbrsol, routeradv, routersol, unreach, toobig, timex }"
set skip on lo0
nat on $ext_if inet from any to any -> ($ext_if:0)
nat on $ext_if inet6 from any to any -> ($ext_if:0)
# Define the redirect rules
rdr on $ext_if inet proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv4
rdr on $ext_if inet6 proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv6
# Accept the incoming traffic to the specified ports we will NAT redirect
pass in quick on $ext_if inet proto tcp from any to any port $forward_ports flags S/SA modulate state
pass in quick on $ext_if inet6 proto tcp from any to any port $forward_ports flags S/SA modulate state
# Allow incoming SSH for host mgmt
pass in quick on $ext_if proto tcp from any to ($ext_if) port 22 flags S/SA modulate state
# Allow ICMP
pass in quick on $ext_if inet proto icmp all icmp-type $icmp_types keep state
pass in quick on $ext_if inet6 proto ipv6-icmp all icmp6-type $icmp6_types keep state
# Allow traffic from anyone to go through the NAT
pass on $ext_if inet proto tcp from any to $chatmail_ipv4 flags S/SA modulate state
pass on $ext_if inet6 proto tcp from any to $chatmail_ipv6 flags S/SA modulate state
# Default allow out
pass out quick on $ext_if from any to any
# Default block
block drop in log all
Insert into ``/etc/sysctl.conf.local`` the following two lines:
::
net.inet.ip.forwarding=1
net.inet6.ip6.forwarding=1
Activate the sysctls with ``service sysctl onestart``.
Enable the pf firewall with ``service pf enable``.
Apply the firewall rules with ``service pf start`` or ``pfctl -f /etc/pf.conf``.
Note, enabling the firewall may interrupt your SSH session, but you can reconnect.
Once proxy relay is set up, you can add its IP address to the DNS.