From 2e43f56e1fe41690fd4edbdb236981fb86b697e8 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 27 Oct 2024 13:56:26 +0100 Subject: [PATCH 1/8] cmdeploy: add argument to specify different SSH host than mail_domain --- cmdeploy/src/cmdeploy/cmdeploy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index cb744b37..2d199fa0 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -58,6 +58,11 @@ def run_cmd_options(parser): action="store_true", help="install/upgrade the server, but disable postfix & dovecot for now" ) + parser.add_argument( + "--ssh-host", + dest="ssh_host", + help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default" + ) def run_cmd(args, out): @@ -73,7 +78,8 @@ def run_cmd(args, out): env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else "" deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve() pyinf = "pyinfra --dry" if args.dry_run else "pyinfra" - cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y" + ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host + cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y" if version.parse(pyinfra.__version__) < version.parse("3"): out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.") return 1 From d0a70b620466fa4165fc975ed949a1446f10e88c Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 14:57:16 +0100 Subject: [PATCH 2/8] added changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 071aa202..5f35f925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) +- add argument to `cmdeploy run` for specifying + a different SSH host than `mail_domain` + ([#439](https://github.com/deltachat/chatmail/pull/439)) + - query autoritative nameserver to bypass DNS cache ([#424](https://github.com/deltachat/chatmail/pull/424)) From c2ff9377bc5cd95f19a8ad6e9b5090ba02fbcb00 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 16 Oct 2024 11:59:04 +0200 Subject: [PATCH 3/8] Guide on how to migrate chatmail to a new host This guide doesn't require knowing about firewalls, but utilizes the `cmdeploy run --disable-mail` command from #428. supercedes #417 --- README.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/README.md b/README.md index c9cc13a7..8a538cc2 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,86 @@ and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch `From:` header must correspond to envelope MAIL FROM, this is ensured by `filtermail` proxy. +## Migrating chatmail server to a new host + +If you want to migrate chatmail from an old machine +to a new machine, +you can use these steps. +They were tested with a linux laptop; +you might need to adjust some of the steps to your environment. + +Let's assume that your `mail_domain` is `mail.example.org`, +all involved machines run Debian 12, +your old server's IP address is `13.37.13.37`, +and your new server's IP address is `13.12.23.42`. + +1. First, copy `/var/lib/acme` to your local machine with `rsync -avz mail.example.org:/var/lib/acme .` + +2. Now, in your local `/etc/hosts`, point your domain to the new machine: `13.12.23.42 mail.example.org` + +3. You need to run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` so you can connect to the new machine via SSH. + +4. Upload /var/lib/acme to the new machine with `rsync -avz acme mail.example.org:/var/lib/`. + +5. On the server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. + +6. Run `cmdeploy run --disable-mail` to install chatmail on the new machine. + postfix and dovecot are disabled for now, + we will enable them later. + +7. Now, point DNS to the new IP addresses. + + You can already remove the old IP addresses from DNS. + Existing Delta Chat users will still be able to connect + to the old server, send and receive messages, + but new users will fail to create new profiles + with your chatmail server. + + If other servers try to deliver messages to your new server they will fail, + but normally email servers will retry delivering messages + for at least a week, so messages will not be lost. + +8. Then point the domain to the old machine in your local `/etc/hosts` again: `13.37.13.37 mail.example.org` + +9. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` again so you can connect to the new machine via SSH. + +10. Now you can run `cmdeploy run --disable-mail` to disable your old server. + + Now your users will notice the migration + and will not be able to send or receive messages + until the migration is completed. + +11. After everything is stopped, + you can copy the `/home/vmail/mail` directory to the new server. + It includes all user data, messages, password hashes, etc. + + If you have enough storage on your local machine, + you can simply download it with `rsync -avz mail.example.org:/home/vmail/mail .`, + change `/etc/hosts` and run `ssh-keygen` as in step 11 and 12, + and upload it again with `rsync -avz mail mail.example.org:/home/vmail/`. + + The other way would be copying it + from the old machine to the new machine directly, + which requires setting up an SSH connection + with a new SSH key. + + After this, your new server has all the necessary files to start operating :) + +12. If you haven't done this during the last step, + point your domain to the new machine in your `/etc/hosts` again: `13.12.23.42 mail.example.org` + +13. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` a final time + to make sure you can SSH-connect to the new machine. + +14. To be sure the permissions are still fine, + run `chown vmail: -R /home/vmail` on the new server. + +15. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. + Your users can continue using the chatmail server, + and messages which were sent after step 9 should arrive now. + +16. Voilà! Consider removing the entry in your local `/etc/hosts` to clean up. + ## Setting up a reverse proxy A chatmail server does not depend on the client IP address From 3c9ebe69a095d1488831a84c8d199fa6d125b6e1 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 27 Oct 2024 14:04:05 +0100 Subject: [PATCH 4/8] doc: migration guide should use new --ssh-host command --- README.md | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 8a538cc2..7d2e3bf9 100644 --- a/README.md +++ b/README.md @@ -269,21 +269,21 @@ all involved machines run Debian 12, your old server's IP address is `13.37.13.37`, and your new server's IP address is `13.12.23.42`. -1. First, copy `/var/lib/acme` to your local machine with `rsync -avz mail.example.org:/var/lib/acme .` +During the guide, you might get a warning about changed SSH Host keys; +in this case, just run `ssh-keygen -R "mail.example.org"` as recommended +to make sure you can connect with SSH. -2. Now, in your local `/etc/hosts`, point your domain to the new machine: `13.12.23.42 mail.example.org` +1. First, copy `/var/lib/acme` to your local machine with `rsync -avz root@13.37.13.37:/var/lib/acme .` -3. You need to run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` so you can connect to the new machine via SSH. +2. Upload /var/lib/acme to the new machine with `rsync -avz acme root@13.12.23.42:/var/lib/`. -4. Upload /var/lib/acme to the new machine with `rsync -avz acme mail.example.org:/var/lib/`. +3. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. -5. On the server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. - -6. Run `cmdeploy run --disable-mail` to install chatmail on the new machine. +4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -7. Now, point DNS to the new IP addresses. +5. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -295,24 +295,19 @@ and your new server's IP address is `13.12.23.42`. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -8. Then point the domain to the old machine in your local `/etc/hosts` again: `13.37.13.37 mail.example.org` - -9. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` again so you can connect to the new machine via SSH. - -10. Now you can run `cmdeploy run --disable-mail` to disable your old server. +6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -11. After everything is stopped, +7. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. If you have enough storage on your local machine, - you can simply download it with `rsync -avz mail.example.org:/home/vmail/mail .`, - change `/etc/hosts` and run `ssh-keygen` as in step 11 and 12, - and upload it again with `rsync -avz mail mail.example.org:/home/vmail/`. + you can simply download it with `rsync -avz 13.37.13.37:/home/vmail/mail .`, + and upload it again with `rsync -avz mail 13.12.23.42:/home/vmail/`. The other way would be copying it from the old machine to the new machine directly, @@ -321,20 +316,13 @@ and your new server's IP address is `13.12.23.42`. After this, your new server has all the necessary files to start operating :) -12. If you haven't done this during the last step, - point your domain to the new machine in your `/etc/hosts` again: `13.12.23.42 mail.example.org` - -13. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` a final time - to make sure you can SSH-connect to the new machine. - -14. To be sure the permissions are still fine, +8. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -15. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 9 should arrive now. - -16. Voilà! Consider removing the entry in your local `/etc/hosts` to clean up. + Voilà! ## Setting up a reverse proxy From c7d080e4bb27697043c606a6dc93e0fba1301582 Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 14:59:28 +0100 Subject: [PATCH 5/8] doc: fix step 9 -> step 6 Co-authored-by: holger krekel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d2e3bf9..cea1f38d 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ to make sure you can connect with SSH. 9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, - and messages which were sent after step 9 should arrive now. + and messages which were sent after step 6. should arrive now. Voilà! ## Setting up a reverse proxy From f51468c10d3b3c991600fc0b3d55556b5d463df1 Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 15:00:42 +0100 Subject: [PATCH 6/8] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f35f925..493a2c22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- add guide to migrate chatmail to a new server + ([#429](https://github.com/deltachat/chatmail/pull/429)) + - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) From 3d08ad3ac6fb96e144ac8b27f7edbb5ed9488691 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 28 Oct 2024 18:16:01 +0000 Subject: [PATCH 7/8] Replace rsync with tar --- README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cea1f38d..485a7f30 100644 --- a/README.md +++ b/README.md @@ -273,17 +273,16 @@ During the guide, you might get a warning about changed SSH Host keys; in this case, just run `ssh-keygen -R "mail.example.org"` as recommended to make sure you can connect with SSH. -1. First, copy `/var/lib/acme` to your local machine with `rsync -avz root@13.37.13.37:/var/lib/acme .` +1. First, copy `/var/lib/acme` to your local machine with + `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`. -2. Upload /var/lib/acme to the new machine with `rsync -avz acme root@13.12.23.42:/var/lib/`. +2. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. -3. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. - -4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. +3. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -5. Now, point DNS to the new IP addresses. +4. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -295,13 +294,13 @@ to make sure you can connect with SSH. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. +5. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -7. After everything is stopped, +6. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. @@ -316,10 +315,10 @@ to make sure you can connect with SSH. After this, your new server has all the necessary files to start operating :) -8. To be sure the permissions are still fine, +7. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +8. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 6. should arrive now. Voilà! From e12aecb508b673236ffee5b59c48cffb1744086b Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 29 Oct 2024 16:42:35 +0100 Subject: [PATCH 8/8] doc: use ssh+tar to transfer vmail + dkimkeys as well --- README.md | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 485a7f30..eaf00981 100644 --- a/README.md +++ b/README.md @@ -273,16 +273,21 @@ During the guide, you might get a warning about changed SSH Host keys; in this case, just run `ssh-keygen -R "mail.example.org"` as recommended to make sure you can connect with SSH. -1. First, copy `/var/lib/acme` to your local machine with +1. First, copy `/var/lib/acme` to the new server with `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`. + This transfers your TLS certificate. -2. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. +2. You should also copy `/etc/dkimkeys` to the new server with + `ssh root@13.37.13.37 tar c /etc/dkimkeys | ssh root@13.12.23.42 tar x -C /etc/` + so the DKIM DNS record stays correct. -3. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. +3. On the new server, run `chown root: -R /var/lib/acme` and `chown root: -R /etc/dkimkeys` to make sure the permissions are correct. + +4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -4. Now, point DNS to the new IP addresses. +5. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -294,31 +299,24 @@ to make sure you can connect with SSH. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -5. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. +6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -6. After everything is stopped, +7. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. - If you have enough storage on your local machine, - you can simply download it with `rsync -avz 13.37.13.37:/home/vmail/mail .`, - and upload it again with `rsync -avz mail 13.12.23.42:/home/vmail/`. - - The other way would be copying it - from the old machine to the new machine directly, - which requires setting up an SSH connection - with a new SSH key. + Just run: `ssh root@13.37.13.37 tar c /home/vmail/mail | ssh root@13.12.23.42 tar x -C /home/vmail/` After this, your new server has all the necessary files to start operating :) -7. To be sure the permissions are still fine, +8. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -8. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 6. should arrive now. Voilà!