Compare commits

..

1 Commits

Author SHA1 Message Date
missytake
1493acb87b cmdeploy: run SSH commands locally if ssh_host is localhost 2025-08-20 18:17:59 +02:00
18 changed files with 94 additions and 272 deletions

View File

@@ -70,6 +70,9 @@ jobs:
rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
- name: run formatting checks
run: cmdeploy fmt -v
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy

View File

@@ -70,6 +70,9 @@ jobs:
rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
- name: run formatting checks
run: cmdeploy fmt -v
- name: run deploy-chatmail offline tests
run: pytest --pyargs cmdeploy

View File

@@ -1,50 +0,0 @@
This diagram shows components of the chatmail server; this is a draft
overview as of mid-August 2025:
```mermaid
graph LR;
cmdeploy --- sshd;
letsencrypt --- |80|acmetool-redirector;
acmetool-redirector --- |443|nginx-right(["`nginx
(external)`"]);
nginx-external --- |465|postfix;
nginx-external(["`nginx
(external)`"]) --- |8443|nginx-internal["`nginx
(internal)`"];
nginx-internal --- website["`Website
/var/www/html`"];
nginx-internal --- newemail.py;
nginx-internal --- autoconfig.xml;
certs-nginx[("`TLS certs
/var/lib/acme`")] --> nginx-internal;
cron --- chatmail-metrics;
cron --- acmetool;
cron --- expunge;
chatmail-metrics --- website;
acmetool --> certs[("`TLS certs
/var/lib/acme`")];
nginx-external --- |993|dovecot;
autoconfig.xml --- postfix;
autoconfig.xml --- dovecot;
postfix --- echobot;
postfix --- |10080,10081|filtermail;
postfix --- users["`User data
home/vmail/mail`"];
postfix --- |doveauth.socket|doveauth;
dovecot --- |doveauth.socket|doveauth;
dovecot --- users;
dovecot --- |metadata.socket|chatmail-metadata;
doveauth --- users;
expunge --- users;
chatmail-metadata --- iroh-relay;
certs-nginx --> postfix;
certs-nginx --> dovecot;
style certs fill:#ff6;
style certs-nginx fill:#ff6;
style nginx-external fill:#fc9;
style nginx-right fill:#fc9;
```
The edges in this graph should not be taken too literally; they
reflect some sort of communication path or dependency relationship
between components of the chatmail server.

View File

@@ -2,18 +2,6 @@
## untagged
- Enable invite-only chatmail relays with invite tokens
that can override disabled account creation
([#600](https://github.com/chatmail/relay/pull/600))
- dovecot: keep mailbox index only in memory to avoid unnecessary disc usage
([#632](https://github.com/chatmail/relay/pull/632))
## 1.7.0 2025-09-11
- Make www upload path configurable
([#618](https://github.com/chatmail/relay/pull/618))
- Check whether GCC is installed in initenv.sh
([#608](https://github.com/chatmail/relay/pull/608))
@@ -41,9 +29,6 @@
- filtermail: respect config message size limit
([#572](https://github.com/chatmail/relay/pull/572))
- Don't deploy if one of the ports used for chatmail relay services is occupied by an unexpected process
([#568](https://github.com/chatmail/relay/pull/568))
- Add config value after how many days large files are deleted
([#555](https://github.com/chatmail/relay/pull/555))

View File

@@ -255,18 +255,6 @@ This starts a local live development cycle for chatmail web pages:
- Starts a browser window automatically where you can "refresh" as needed.
#### Custom web pages
You can skip uploading a web page
by setting `www_folder=disabled` in `chatmail.ini`.
If you want to manage your web pages outside this git repository,
you can set `www_folder` in `chatmail.ini` to a custom directory on your computer.
`cmdeploy run` will upload it as the server's home page,
and if it contains a `src/index.md` file,
will build it with hugo.
## Mailbox directory layout
Fresh chatmail addresses have a mailbox directory that contains:
@@ -284,23 +272,8 @@ Fresh chatmail addresses have a mailbox directory that contains:
will typically be empty unless the user of that address hasn't been online
for a while.
## Restrict address creation
### Only allow new addresses with an invite token
To restrict address creation for anyone who doesn't have the invite link/QR code:
1. Use the `invite_token` option to add
one or more tokens of your choice to `chatmail.ini`:
`invite_token = s3cr3t privil3g3`
- (recommendation: choose 9 or more letters, or it will be easily bruteforced)
2. Run `scripts/cmdeploy run`
3. Distribute a `dcaccount` invite link/QR code
(like the one on your web page)
with one of your invite tokens added at the end,
for example: `dcaccount:https://example.org/new?s3cr3t`
### Emergency Command to disable automatic address creation
## Emergency Commands to disable automatic address creation
If you need to stop address creation,
e.g. because some script is wildly creating addresses,

View File

@@ -31,10 +31,8 @@ class Config:
self.username_min_length = int(params["username_min_length"])
self.username_max_length = int(params["username_max_length"])
self.password_min_length = int(params["password_min_length"])
self.invite_token = params.get("invite_token", "")
self.passthrough_senders = params["passthrough_senders"].split()
self.passthrough_recipients = params["passthrough_recipients"].split()
self.www_folder = params.get("www_folder", "")
self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
self.filtermail_smtp_port_incoming = int(
params["filtermail_smtp_port_incoming"]

View File

@@ -26,19 +26,8 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
if os.path.exists(NOCREATE_FILE):
logging.warning(f"blocked account creation because {NOCREATE_FILE!r} exists.")
return False
password_length = len(cleartext_password)
if config.invite_token:
for inv_token in config.invite_token.split():
if cleartext_password.startswith(inv_token):
password_length = len(cleartext_password) - len(inv_token)
break
else:
logging.warning(
"blocked account creation because password didn't contain invite token(s)."
)
return False
if password_length < config.password_min_length:
if len(cleartext_password) < config.password_min_length:
logging.warning(
"Password needs to be at least %s characters long",
config.password_min_length,

View File

@@ -3,7 +3,6 @@
"""CGI script for creating new accounts."""
import json
import os
import random
import secrets
import string
@@ -21,11 +20,7 @@ def create_newemail_dict(config: Config):
secrets.choice(ALPHANUMERIC_PUNCT)
for _ in range(config.password_min_length + 3)
)
redirect_uri = os.getenv("REQUEST_URI", "/new")
invite_token = "" if redirect_uri == "/new" else redirect_uri[5:]
return dict(
email=f"{user}@{config.mail_domain}", password=f"{invite_token}{password}"
)
return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
def print_new_account():

View File

@@ -64,38 +64,12 @@ def test_dont_overwrite_password_on_wrong_login(dictproxy):
assert res["password"] == res2["password"]
@pytest.mark.parametrize(
["nocreate_file", "account", "invite_token", "password"],
[
(False, True, "asdf", "asdfasdmaimfelsgwerw"),
(False, False, "asdf", "z9873240187420913798"),
(False, True, "", "dsaiujfw9fjiwf9w"),
(False, False, "asdf", "z987324018742asdf0913798"),
(False, True, "as df", "asj0wiefkj0ofkeefok"),
(False, True, "as df", "dfj0wiefkj0ofkeefok"),
(False, False, "as df", "j0wiefkj0ofas dfkeefok"),
(True, False, "asdf", "asdfmosadkdkfwdofkw"),
(True, False, "asdf", "z9873240187420913798"),
(True, False, "", "dsaiujfw9fjiwf9w"),
],
)
def test_nocreate_file(
monkeypatch,
tmpdir,
dictproxy,
example_config,
nocreate_file: bool,
account: bool,
invite_token: str,
password: str,
):
if nocreate_file:
p = tmpdir.join("nocreate")
p.write("")
monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
example_config.invite_token = invite_token
dictproxy.lookup_passdb("newuser12@chat.example.org", password)
assert bool(dictproxy.lookup_userdb("newuser12@chat.example.org")) == account
def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
p = tmpdir.join("nocreate")
p.write("")
monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik")
assert not dictproxy.lookup_userdb("newuser12@chat.example.org")
def test_handle_dovecot_request(dictproxy):

View File

@@ -11,7 +11,7 @@ from io import StringIO
from pathlib import Path
from chatmaild.config import Config, read_config
from pyinfra import facts, host, logger
from pyinfra import facts, host
from pyinfra.api import FactBase
from pyinfra.facts.files import File
from pyinfra.facts.server import Sysctl
@@ -618,7 +618,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
check_config(config)
mail_domain = config.mail_domain
from .www import build_webpages, get_paths
from .www import build_webpages
server.group(name="Create vmail group", group="vmail", system=True)
server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
@@ -675,30 +675,10 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
# to use 127.0.0.1 as the resolver.
from cmdeploy.cmdeploy import Out
port_services = [
(["master", "smtpd"], 25),
("unbound", 53),
("acmetool", 80),
("imap-login", 143),
("nginx", 443),
(["master", "smtpd"], 465),
(["master", "smtpd"], 587),
("imap-login", 993),
("iroh-relay", 3340),
("nginx", 8443),
(["master", "smtpd"], config.postfix_reinject_port),
(["master", "smtpd"], config.postfix_reinject_port_incoming),
("filtermail", config.filtermail_smtp_port),
("filtermail", config.filtermail_smtp_port_incoming),
]
for service, port in port_services:
print(f"Checking if port {port} is available for {service}...")
running_service = host.get_fact(Port, port=port)
if running_service:
if running_service not in service:
Out().red(f"Deploy failed: port {port} is occupied by: {running_service}")
exit(1)
process_on_53 = host.get_fact(Port, port=53)
if process_on_53 not in (None, "unbound"):
Out().red(f"Can't install unbound: port 53 is occupied by: {process_on_53}")
exit(1)
apt.packages(
name="Install unbound",
packages=["unbound", "unbound-anchor", "dnsutils"],
@@ -751,16 +731,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
packages=["fcgiwrap"],
)
www_path, src_dir, build_dir = get_paths(config)
# if www_folder was set to a non-existing folder, skip upload
if not www_path.is_dir():
logger.warning("Building web pages is disabled in chatmail.ini, skipping")
else:
# if www_folder is a hugo page, build it
if build_dir:
www_path = build_webpages(src_dir, build_dir, config)
# if it is not a hugo page, upload it as is
files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"])
www_path = importlib.resources.files(__package__).joinpath("../../../www").resolve()
build_dir = www_path.joinpath("build")
src_dir = www_path.joinpath("src")
build_webpages(src_dir, build_dir, config)
files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
_install_remote_venv_with_chatmaild(config)
debug = False

View File

@@ -1,5 +1,7 @@
import importlib.resources
from pyinfra import host
from pyinfra.facts.systemd import SystemdStatus
from pyinfra.operations import apt, files, server, systemd
@@ -52,6 +54,12 @@ def deploy_acmetool(email="", domains=[]):
group="root",
mode="644",
)
if host.get_fact(SystemdStatus).get("nginx.service"):
systemd.service(
name="Stop nginx service to free port 80",
service="nginx",
running=False,
)
systemd.service(
name="Setup acmetool-redirector service",

View File

@@ -19,7 +19,7 @@ from packaging import version
from termcolor import colored
from . import dns, remote
from .sshexec import SSHExec
from .sshexec import SSHExec, Local
#
# cmdeploy sub commands and options
@@ -62,13 +62,18 @@ def run_cmd_options(parser):
"--ssh-host",
dest="ssh_host",
help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default",
default=None,
)
def run_cmd(args, out):
"""Deploy chatmail services on the remote server."""
sshexec = args.get_sshexec()
ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
if ssh_host == "localhost":
sshexec = Local(ssh_host)
else:
sshexec = args.get_sshexec(ssh_host)
require_iroh = args.config.enable_iroh_relay
remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
if not dns.check_initial_remote_data(remote_data, print=out.red):
@@ -80,7 +85,7 @@ def run_cmd(args, out):
env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host
ssh_host = "@local" if ssh_host == "localhost" else f"--ssh-host {ssh_host}"
cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
if version.parse(pyinfra.__version__) < version.parse("3"):
out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
@@ -89,14 +94,6 @@ def run_cmd(args, out):
try:
retcode = out.check_call(cmd, env=env)
if retcode == 0:
print("\nYou can try out the relay by talking to this echo bot: ")
sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
print(
sshexec(
call=remote.rshell.shell,
kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
)
)
out.green("Deploy completed, call `cmdeploy dns` next.")
elif not remote_data["acme_account_url"]:
out.red("Deploy completed but letsencrypt not configured")
@@ -338,9 +335,9 @@ def main(args=None):
if not hasattr(args, "func"):
return parser.parse_args(["-h"])
def get_sshexec():
print(f"[ssh] login to {args.config.mail_domain}")
return SSHExec(args.config.mail_domain, verbose=args.verbose)
def get_sshexec(host):
print(f"[ssh] login to {host}")
return SSHExec(host, verbose=args.verbose)
args.get_sshexec = get_sshexec

View File

@@ -68,7 +68,7 @@ userdb {
##
# Mailboxes are stored in the "mail" directory of the vmail user home.
mail_location = maildir:{{ config.mailboxes_dir }}/%u:INDEX=MEMORY
mail_location = maildir:{{ config.mailboxes_dir }}/%u
namespace inbox {
inbox = yes

View File

@@ -84,13 +84,12 @@ http {
if ($request_method = GET) {
# Redirect to Delta Chat,
# which will in turn do a POST request.
return 301 dcaccount:https://{{ config.domain_name }}$request_uri;
return 301 dcaccount:https://{{ config.domain_name }}/new;
}
fastcgi_pass unix:/run/fcgiwrap.socket;
include /etc/nginx/fastcgi_params;
fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
fastcgi_param QUERY_STRING $query_string;
}
# Old URL for compatibility with e.g. printed QR codes.
@@ -101,7 +100,7 @@ http {
# Redirects are only for browsers.
location /cgi-bin/newemail.py {
if ($request_method = GET) {
return 301 dcaccount:https://{{ config.domain_name }}$request_uri;
return 301 dcaccount:https://{{ config.domain_name }}/new;
}
fastcgi_pass unix:/run/fcgiwrap.socket;

View File

@@ -1,5 +1,6 @@
import inspect
import os
import subprocess
import sys
from queue import Queue
@@ -44,30 +45,16 @@ def print_stderr(item="", end="\n"):
print(item, file=sys.stderr, end=end)
class SSHExec:
RemoteError = execnet.RemoteError
class Exec:
FuncError = FuncError
def __init__(self, host, verbose=False, python="python3", timeout=60):
self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
def __init__(self, host, verbose, timeout):
self.host = host
self.timeout = timeout
self.verbose = verbose
def __call__(self, call, kwargs=None, log_callback=None):
if kwargs is None:
kwargs = {}
assert call.__module__.startswith("cmdeploy.remote")
modname = call.__module__.replace("cmdeploy.", "")
self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
while 1:
code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
if log_callback is not None and code == "log":
log_callback(data)
elif code == "finish":
return data
elif code == "error":
raise self.FuncError(data)
return subprocess.check_output(call)
def logged(self, call, kwargs):
def log_progress(data):
@@ -85,3 +72,33 @@ class SSHExec:
res = self(call, kwargs, log_callback=log_progress)
print_stderr()
return res
class Local(Exec):
def __init__(self, host, verbose=False, timeout=60):
super().__init__(host, verbose, timeout)
class SSHExec(Exec):
RemoteError = execnet.RemoteError
def __init__(self, host, verbose=False, timeout=60):
super().__init__(host, verbose, timeout)
self.gateway = execnet.makegateway(f"ssh=root@{host}//python=python3")
self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
def __call__(self, call, kwargs=None, log_callback=None):
if kwargs is None:
kwargs = {}
assert call.__module__.startswith("cmdeploy.remote")
modname = call.__module__.replace("cmdeploy.", "")
self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
while 1:
code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
if log_callback is not None and code == "log":
log_callback(data)
elif code == "finish":
return data
elif code == "error":
raise self.FuncError(data)

View File

@@ -1,10 +1,8 @@
import importlib
import os
import pytest
from cmdeploy.cmdeploy import get_parser, main
from cmdeploy.www import get_paths
@pytest.fixture(autouse=True)
@@ -29,28 +27,3 @@ class TestCmdline:
assert main(["init", "chat.example.org"]) == 1
out, err = capsys.readouterr()
assert "path exists" in out.lower()
def test_www_folder(example_config, tmp_path):
reporoot = importlib.resources.files(__package__).joinpath("../../../../").resolve()
assert not example_config.www_folder
www_path, src_dir, build_dir = get_paths(example_config)
assert www_path.absolute() == reporoot.joinpath("www").absolute()
assert src_dir == reporoot.joinpath("www").joinpath("src")
assert build_dir == reporoot.joinpath("www").joinpath("build")
example_config.www_folder = "disabled"
www_path, _, _ = get_paths(example_config)
assert not www_path.is_dir()
example_config.www_folder = str(tmp_path)
www_path, src_dir, build_dir = get_paths(example_config)
assert www_path == tmp_path
assert not src_dir.exists()
assert not build_dir
src_path = tmp_path.joinpath("src")
os.mkdir(src_path)
with open(src_path / "index.md", "w") as f:
f.write("# Test")
www_path, src_dir, build_dir = get_paths(example_config)
assert www_path == tmp_path
assert src_dir == src_path
assert build_dir == tmp_path.joinpath("build")

View File

@@ -3,7 +3,6 @@ import importlib.resources
import time
import traceback
import webbrowser
from pathlib import Path
import markdown
from chatmaild.config import read_config
@@ -31,25 +30,9 @@ def prepare_template(source):
return render_vars, page_layout
def get_paths(config) -> (Path, Path, Path):
reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
www_path = Path(config.www_folder)
# if www_folder was not set, use default directory
if config.www_folder == "":
www_path = reporoot.joinpath("www")
src_dir = www_path.joinpath("src")
# if www_folder is a hugo page, build it
if src_dir.joinpath("index.md").is_file():
build_dir = www_path.joinpath("build")
# if it is not a hugo page, upload it as is
else:
build_dir = None
return www_path, src_dir, build_dir
def build_webpages(src_dir, build_dir, config) -> Path:
def build_webpages(src_dir, build_dir, config):
try:
return _build_webpages(src_dir, build_dir, config)
_build_webpages(src_dir, build_dir, config)
except Exception:
print(traceback.format_exc())
@@ -123,11 +106,15 @@ def main():
config = read_config(inipath)
config.webdev = True
assert config.mail_domain
www_path = reporoot.joinpath("www")
src_path = www_path.joinpath("src")
stats = None
build_dir = www_path.joinpath("build")
src_dir = www_path.joinpath("src")
index_path = build_dir.joinpath("index.html")
# start web page generation, open a browser and wait for changes
www_path, src_path, build_dir = get_paths(config)
build_dir = build_webpages(src_path, build_dir, config)
index_path = build_dir.joinpath("index.html")
build_webpages(src_dir, build_dir, config)
webbrowser.open(str(index_path))
stats = snapshot_dir_stats(src_path)
print(f"\nOpened URL: file://{index_path.resolve()}\n")
@@ -148,7 +135,7 @@ def main():
changenum += 1
stats = newstats
build_webpages(src_path, build_dir, config)
build_webpages(src_dir, build_dir, config)
print(f"[{changenum}] regenerated web pages at: {index_path}")
print(f"URL: file://{index_path.resolve()}\n\n")
count = 0

View File

@@ -11,7 +11,6 @@ for Delta Chat users. For details how it avoids storing personal information
please see our [privacy policy](privacy.html).
{% endif %}
{% if not config.invite_token %}
<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
If you are viewing this page on a different device
@@ -24,10 +23,6 @@ you can also **scan this QR code** with Delta Chat:
🐣 **Choose** your Avatar and Name
💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
{% else %}
**To join this instance, you need an invite link or QR code -
ask the admin for an invite.**
{% endif %}
{% if config.mail_domain != "nine.testrun.org" %}
<div class="experimental">Note: this is only a temporary development chatmail service</div>