Deploy iroh relay

.github/workflows/staging-ipv4.testrun.org-default.zone

@@ -17,4 +17,5 @@ $TTL 300
 ;; DNS records.
 @       IN      A       37.27.95.249
 mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
+iroh.staging-ipv4.testrun.org.       CNAME   staging-ipv4.testrun.org.
 www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.

.github/workflows/staging.testrun.org-default.zone

@@ -17,5 +17,6 @@ $TTL 300
 ;; DNS records.
 @       IN      A       37.27.24.139
 mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
+iroh.staging2.testrun.org.       CNAME   staging2.testrun.org.
 www.staging2.testrun.org.        CNAME   staging2.testrun.org.
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -38,7 +38,9 @@ jobs:
           if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
+          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: rebuild staging-ipv4.testrun.org to have a clean VPS
@@ -64,8 +66,8 @@ jobs:
           rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true
           rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true
           # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/dkimkeys || true
+          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 

.github/workflows/test-and-deploy.yaml

@@ -38,7 +38,9 @@ jobs:
           if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
           if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: rebuild staging2.testrun.org to have a clean VPS
@@ -64,8 +66,8 @@ jobs:
           rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
           rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
           # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true
+          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## untagged
 
+- deploy `iroh-relay` (requires new "iroh.{mail_domain}" DNS entry), 
+  also update "realtime relay services" in privacy policy. 
+  ([#434](https://github.com/deltachat/chatmail/pull/434))
+
 - add guide to migrate chatmail to a new server
   ([#429](https://github.com/deltachat/chatmail/pull/429))
 

chatmaild/src/chatmaild/config.py

@@ -33,7 +33,12 @@ def __init__(self, inipath, params):
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
-        self.iroh_relay = params.get("iroh_relay")
+        if "iroh_relay" not in params:
+            self.iroh_relay = "https://iroh." + params["mail_domain"]
+            self.enable_iroh_relay = True
+        else:
+            self.iroh_relay = params["iroh_relay"].strip()
+            self.enable_iroh_relay = False
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -55,6 +55,13 @@
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+# Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
+# service.
+# If you set it to anything else, the service will be disabled
+# and users will be directed to use the given iroh relay URL.
+# Set it to empty string if you want users to use their default iroh relay.
+# iroh_relay =
+
 # Address on which `mtail` listens,
 # e.g. 127.0.0.1 or some private network
 # address like 192.168.10.1.

cmdeploy/src/cmdeploy/__init__.py

@@ -10,7 +10,7 @@
 from pathlib import Path
 
 from chatmaild.config import Config, read_config
-from pyinfra import host
+from pyinfra import host, facts
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -479,6 +479,55 @@ def deploy_mtail(config):
     )
 
 
+def deploy_iroh_relay(config) -> None:
+    (url, sha256sum) = {
+            "x86_64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-x86_64-unknown-linux-musl.tar.gz", "8af7f6d29d17476ce5c3053c3161db5793cb2ac49057d0bcaf689436cdccbeab"),
+            "aarch64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-aarch64-unknown-linux-musl.tar.gz", "18039f0d39df78922a5055a0d4a5a8fa98a2a0e19b1eaa4c3fe6db73b8698697")
+    }[host.get_fact(facts.server.Arch)]
+
+    server.shell(
+        name="Download iroh-relay",
+        commands=[
+            f"(echo '{sha256sum} /usr/local/bin/iroh-relay' | sha256sum -c) || curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay",
+            "chmod 755 /usr/local/bin/iroh-relay",
+        ],
+    )
+
+    need_restart = False
+
+    systemd_unit = files.put(
+        name="Upload iroh-relay systemd unit",
+        src=importlib.resources.files(__package__).joinpath(
+            "iroh-relay.service"
+        ),
+        dest="/etc/systemd/system/iroh-relay.service",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= systemd_unit.changed
+
+    iroh_config = files.put(
+        name=f"Upload iroh-relay config",
+        src=importlib.resources.files(__package__).joinpath(
+            "iroh-relay.toml"
+        ),
+        dest=f"/etc/iroh-relay.toml",
+        user="iroh",
+        group="iroh",
+        mode="600",
+    )
+    need_restart |= iroh_config.changed
+
+    systemd.service(
+        name="Start and enable iroh-relay",
+        service="iroh-relay.service",
+        running=True,
+        enabled=config.enable_iroh_relay,
+        restarted=need_restart,
+    )
+
+
 def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     """Deploy a chat-mail instance.
 
@@ -508,6 +557,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         system=True,
     )
     server.user(name="Create echobot user", user="echobot", system=True)
+    server.user(name="Create iroh user", user="iroh", system=True)
 
     # Add our OBS repository for dovecot_no_delay
     files.put(
@@ -556,9 +606,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         enabled=True,
     )
 
+    deploy_iroh_relay(config)
+
     # Deploy acmetool to have TLS certificates.
     deploy_acmetool(
-        domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
+        domains=[mail_domain, f"mta-sts.{mail_domain}", f"iroh.{mail_domain}", f"www.{mail_domain}"],
     )
 
     apt.packages(

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -69,8 +69,9 @@ def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     sshexec = args.get_sshexec()
-    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-    if not dns.check_initial_remote_data(remote_data, print=out.red):
+    require_iroh = args.config.enable_iroh_relay
+    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh)
+    if not dns.check_initial_remote_data(remote_data, require_iroh, print=out.red):
         return 1
 
     env = os.environ.copy()
@@ -109,7 +110,8 @@ def dns_cmd_options(parser):
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     sshexec = args.get_sshexec()
-    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
+    require_iroh = args.config.enable_iroh_relay
+    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh)
     if not remote_data:
         return 1
 

cmdeploy/src/cmdeploy/dns.py

@@ -6,19 +6,22 @@
 from . import remote
 
 
-def get_initial_remote_data(sshexec, mail_domain):
+def get_initial_remote_data(sshexec, mail_domain, iroh_enabled):
     return sshexec.logged(
-        call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
+        call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain, iroh_enabled=iroh_enabled)
     )
 
 
-def check_initial_remote_data(remote_data, print=print):
+def check_initial_remote_data(remote_data, require_iroh, *, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
     elif remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
         print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
+    elif require_iroh and remote_data["IROH"] != f"{mail_domain}.":
+        print("Missing iroh CNAME record:")
+        print(f"iroh.{mail_domain}.   CNAME  {mail_domain}.")
     elif remote_data["WWW"] != f"{mail_domain}.":
         print("Missing www CNAME record:")
         print(f"www.{mail_domain}.   CNAME  {mail_domain}.")

cmdeploy/src/cmdeploy/iroh-relay.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Iroh relay
+
+[Service]
+ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml
+Restart=on-failure
+RestartSec=5s
+User=iroh
+Group=iroh
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/iroh-relay.toml

@@ -0,0 +1,5 @@
+enable_relay = true
+http_bind_addr = "[::]:3340"
+enable_stun = true
+enable_metrics = false
+metrics_bind_addr = "127.0.0.1:9092"

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -108,4 +108,16 @@ http {
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
+
+	# Pass iroh. to iroh-relay service.
+	server {
+		listen 8443 ssl;
+		{% if not disable_ipv6 %}
+		listen [::]:8443 ssl;
+		{% endif %}
+		server_name iroh.{{ config.domain_name }};
+		location / {
+			proxy_pass http://127.0.0.1:3340;
+		}
+	}
 }

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -15,21 +15,22 @@
 from .rshell import CalledProcessError, shell
 
 
-def perform_initial_checks(mail_domain):
+def perform_initial_checks(mail_domain, iroh_enabled):
     """Collecting initial DNS settings."""
     assert mail_domain
     if not shell("dig", fail_ok=True):
         shell("apt-get install -y dnsutils")
     A = query_dns("A", mail_domain)
     AAAA = query_dns("AAAA", mail_domain)
     MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
+    IROH = query_dns("CNAME", f"iroh.{mail_domain}")
     WWW = query_dns("CNAME", f"www.{mail_domain}")
 
-    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
+    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, IROH=IROH, WWW=WWW)
     res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
     res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
 
-    if not MTA_STS or not WWW or (not A and not AAAA):
+    if not MTA_STS or (not IROH and not iroh_enabled) or not WWW or (not A and not AAAA):
         return res
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -18,13 +18,13 @@ def test_ls(self, sshexec):
 
     def test_perform_initial(self, sshexec, maildomain):
         res = sshexec(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True)
         )
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
         sshexec.logged(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True)
         )
         out, err = capsys.readouterr()
         assert err.startswith("Collecting")
@@ -33,7 +33,7 @@ def test_logged(self, sshexec, maildomain, capsys):
 
         sshexec.verbose = True
         sshexec.logged(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True)
         )
         out, err = capsys.readouterr()
         lines = err.split("\n")
@@ -44,7 +44,7 @@ def test_exception(self, sshexec, capsys):
         try:
             sshexec.logged(
                 remote.rdns.perform_initial_checks,
-                kwargs=dict(mail_domain=None),
+                kwargs=dict(mail_domain=None, iroh_enabled=True),
             )
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -26,6 +26,7 @@ def mockdns(mockdns_base):
             "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
             "CNAME": {
                 "mta-sts.some.domain": "some.domain.",
+                "iroh.some.domain": "some.domain.",
                 "www.some.domain": "some.domain.",
             },
         }
@@ -35,30 +36,31 @@ def mockdns(mockdns_base):
 
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns):
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True)
         assert remote_data["A"] == mockdns["A"]["some.domain"]
         assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"]
         assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"]
+        assert remote_data["IROH"] == mockdns["CNAME"]["iroh.some.domain"]
         assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])
     def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop):
         del mockdns[drop]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True)
         assert not remote_data[drop]
 
         l = []
-        res = check_initial_remote_data(remote_data, print=l.append)
+        res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append)
         assert res
         assert not l
 
     def test_perform_initial_checks_no_mta_sts(self, mockdns):
         del mockdns["CNAME"]["mta-sts.some.domain"]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True)
         assert not remote_data["MTA_STS"]
 
         l = []
-        res = check_initial_remote_data(remote_data, print=l.append)
+        res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append)
         assert not res
         assert len(l) == 2