diff --git a/.ci/auth-requirements.txt b/.ci/auth-requirements.txt new file mode 120000 index 0000000..579667a --- /dev/null +++ b/.ci/auth-requirements.txt @@ -0,0 +1 @@ +../kolombo/docker/auth/requirements.txt \ No newline at end of file diff --git a/.ci/check_security b/.ci/check_security new file mode 100755 index 0000000..efd07b2 --- /dev/null +++ b/.ci/check_security @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +bandit --quiet -r kolombo +safety check --full-report diff --git a/.ci/check_style b/.ci/check_style new file mode 100755 index 0000000..953902d --- /dev/null +++ b/.ci/check_style @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +black --target-version=py39 --check kolombo tests setup.py +mypy kolombo +isort --project=kolombo --check kolombo tests setup.py +flake8 kolombo diff --git a/.ci/security-requirements.txt b/.ci/security-requirements.txt new file mode 100644 index 0000000..a03d61b --- /dev/null +++ b/.ci/security-requirements.txt @@ -0,0 +1,2 @@ +bandit==1.7.0 +safety==1.10.3 diff --git a/.ci/style-requirements.txt b/.ci/style-requirements.txt new file mode 100644 index 0000000..8c9129b --- /dev/null +++ b/.ci/style-requirements.txt @@ -0,0 +1,15 @@ +# Static type check analysis +mypy==0.910 +types-setuptools==57.0.2 +# Flake8 with extensions +flake8==3.9.2 +flake8-bugbear==21.4.3 +flake8-pie==0.14.0 +pep8-naming==0.12.1 +flake8-simplify==0.14.1 +flake8-comprehensions==3.6.1 +flake8-fixme==1.1.1 +flake8-cognitive-complexity==0.1.0 +# Code style enforcing +black==21.6b0 +isort==5.9.3 diff --git a/.gitignore b/.gitignore index 7e5eb21..d430600 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ /venv/ -__pycache__/ -/*.egg-info/ +/.vscode/ +__pycache__/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..1174571 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,58 @@ +stages: + - tests + +variables: + PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip" + +cache: + key: "${CI_JOB_STAGE}-${CI_COMMIT_REF_SLUG}-pip" + paths: + - .cache/pip + - venv/ + +before_script: + - pip install virtualenv && virtualenv venv && source venv/bin/activate + +check_style: + stage: tests + tags: + - docker-other + image: python:3.9-slim + script: + - pip install -r requirements.txt + - pip install -r .ci/auth-requirements.txt + - pip install -r .ci/style-requirements.txt + - bash .ci/check_style + only: + refs: + - main + - merge_requests + changes: + - requirements.txt + - setup.py + - kolombo/**/* + - tests/**/* + - .ci/* + - .gitlab-ci.yml + +check_security: + stage: tests + tags: + - docker-other + image: python:3.9-slim + script: + - pip install -r requirements.txt + - pip install -r .ci/auth-requirements.txt + - pip install -r .ci/security-requirements.txt + - bash .ci/check_security + only: + refs: + - main + - merge_requests + changes: + - requirements.txt + - setup.py + - kolombo/**/* + - tests/**/* + - .ci/* + - .gitlab-ci.yml diff --git a/LICENSE b/LICENSE index 459f72b..3b0c725 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 Igor Nehoroshev + Copyright 2020-now Igor Nehoroshev Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MANIFEST.in b/MANIFEST.in index c367fea..5c9064b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ -include README.md kolombo/py.typed -recursive-include kolombo *.py *.sh *.conf requirements.txt virtual-* Dockerfile +include requirements.txt README.md logo.png kolombo/py.typed +recursive-include kolombo *.py +recursive-include kolombo_auth *.py +recursive-include kolombo/docker * diff --git a/README.md b/README.md index a8954b7..cb301df 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,33 @@ # Kolombo -![Kolombo Logo](https://raw.githubusercontent.com/HarrySky/kolombo/master/logo.png "Kolombo Logo") +![Kolombo Logo](https://raw.githubusercontent.com/HarrySky/kolombo/main/logo.png "Kolombo Logo") -Easy to manage mail server 💌 +CLI for easy mail server managing 💌 -**NB! Work in progress, not ready!** +**NB! Work in progress, not ready for production use!** -## Introduction +## How to use -TODO +Documentation is coming, for now, +this is how to setup mail server for +domain example.com with email info@example.com + +```bash +# Initialize Kolombo +kolombo init + +# Add domain and generate DKIM keys for it +kolombo domain add example.com mx.example.com # MX field is optional +kolombo dkim generate example.com # generates DKIM keys and returns DNS TXT record to add +kolombo dkim txt example.com # returns DNS TXT record to add + +# Add user (email) for domain you just added +kolombo user add info@example.com + +# Deploy Kolombo services one by one... + +kolombo run receiver # Listens on 25 for incoming mail, gives emails to users that come through nginx 993/995 port +kolombo run auth # Authenticates SMTP/POP3/IMAP users from nginx +kolombo run nginx # Listens on 465 (SMTP), 993 (IMAP) and 995 (POP3) +kolombo run senders # Sends emails that come through nginx 465 port +``` diff --git a/kolombo/__init__.py b/kolombo/__init__.py index 957e4e7..493f741 100644 --- a/kolombo/__init__.py +++ b/kolombo/__init__.py @@ -1,5 +1 @@ -from kolombo.auth import api -from kolombo.bin import main as cli - -__version__ = "0.2.0" -__all__ = ["api", "cli"] +__version__ = "0.3.0" diff --git a/kolombo/auth/__init__.py b/kolombo/auth/__init__.py index 09cab05..e69de29 100644 --- a/kolombo/auth/__init__.py +++ b/kolombo/auth/__init__.py @@ -1 +0,0 @@ -from kolombo.auth.api import api # noqa: F401 diff --git a/kolombo/auth/_resolver.py b/kolombo/auth/_resolver.py index e10de8a..d3efb32 100644 --- a/kolombo/auth/_resolver.py +++ b/kolombo/auth/_resolver.py @@ -1,6 +1,6 @@ from socket import gethostbyname -def get_ip_by_host(host: str) -> str: +def get_ip_for_host(host: str) -> str: """Return IP address for provided host (fast in Docker network)""" return gethostbyname(host) diff --git a/kolombo/auth/api.py b/kolombo/auth/api.py index 62f6ee4..7ba3b8b 100644 --- a/kolombo/auth/api.py +++ b/kolombo/auth/api.py @@ -2,14 +2,16 @@ from kolombo import conf from kolombo.auth.endpoints import auth -from kolombo.resources import init_database, init_logger +from kolombo.models import init_database -api = FastAPI( +conf.read_configuration() +app = FastAPI( + title="Kolombo auth API", debug=conf.DEBUG, - on_startup=[init_logger, init_database], - # Disable Swagger UI and other docs + on_startup=[init_database], + # Disable Swagger UI and other API docs openapi_url=None, docs_url=None, redoc_url=None, ) -api.add_api_route("/auth", auth, methods=["GET"]) +app.add_api_route("/auth", auth, methods=["GET"]) diff --git a/kolombo/auth/credentials.py b/kolombo/auth/credentials.py index 01d1e93..74a8cbe 100644 --- a/kolombo/auth/credentials.py +++ b/kolombo/auth/credentials.py @@ -5,8 +5,8 @@ from ormar.exceptions import NoMatch from kolombo import conf +from kolombo.console import debug from kolombo.models import User -from kolombo.resources import log async def check_credentials(email: str, password: str, domain: str) -> bool: @@ -15,11 +15,11 @@ async def check_credentials(email: str, password: str, domain: str) -> bool: user = await User.objects.filter(active=True, email=email, domain=domain).get() expected_key = b64decode(user.password) except NoMatch: - log.debug("No active user '%s' from domain '%s' found", email, domain) + debug(f"No active user '{email}' from domain '{domain}' found") key_material = b"Password that 100% not matching, I swear" expected_key = b"Password that 100% not matching, trust me" - kdf = Scrypt(conf.SALT, length=32, n=2 ** 16, r=8, p=1) # type: ignore[call-arg] + kdf = Scrypt(conf.SALT, length=32, n=2 ** 16, r=8, p=1) try: kdf.verify(key_material, expected_key) return True diff --git a/kolombo/auth/endpoints.py b/kolombo/auth/endpoints.py index 4c8f959..0caae4e 100644 --- a/kolombo/auth/endpoints.py +++ b/kolombo/auth/endpoints.py @@ -1,13 +1,15 @@ +from datetime import datetime + from fastapi import Header, Response from kolombo import conf -from kolombo.auth._resolver import get_ip_by_host +from kolombo.auth._resolver import get_ip_for_host from kolombo.auth.credentials import check_credentials from kolombo.auth.protocol import AuthError, AuthSuccess -from kolombo.resources import log +from kolombo.console import debug, error, info, warning -#: We only put SMTP-send and IMAP behind auth, POP3 is unsupported (and old) -_protocol_to_port_mapping = {"smtp": 25, "imap": 143} +#: We only put SMTP-send, IMAP and POP3 (for GMail. Google, use IMAP in 2021 ffs!) +_protocol_to_port_mapping = {"smtp": 25, "imap": 143, "pop3": 110} async def auth( @@ -21,31 +23,34 @@ async def auth( client_ip: str = Header(...), # noqa: B008 ) -> Response: """Endpoint used for auth of SMTP/IMAP users.""" + now = datetime.now().isoformat() if x_secret_key != conf.NGINX_SECRET_KEY: # This MUST NOT happen if everything is set up properly - log.error("Not nginx trying to use API") + error(f"({now}) Not nginx trying to use API") return AuthError("Go Away", retry=False) # Check for possible usage errors to close connection early if protocol not in _protocol_to_port_mapping: + error(f"({now}) Unsupported protocol: {protocol}") return AuthError("Unsupported protocol", retry=False) elif auth_method != "plain": + error(f"({now}) Unsupported auth method: {auth_method}") return AuthError("Unsupported auth method", retry=False) # Remove newline from credentials email = email.rstrip("%0A") password = password.rstrip("%0A") if not await check_credentials(email, password, domain): - log.warning("Failed %s auth as %s from %s", protocol, email, client_ip) + warning(f"({now}) Failed {protocol} auth as {email} from {client_ip}") retry = login_attempt < conf.MAX_ATTEMPTS return AuthError("Invalid login or password", retry=retry) - log.info("Successful %s auth as %s from %s", protocol, email, client_ip) + info(f"({now}) Successful {protocol} auth as {email} from {client_ip}") server_host = "kolombo-receiver" if protocol == "smtp": server_host = f"kolombo-{domain}-sender" - server_ip = get_ip_by_host(server_host) + server_ip = get_ip_for_host(server_host) server_port = _protocol_to_port_mapping[protocol] - log.debug("Forwarding nginx to %s:%s (%s)", server_ip, server_port, server_host) + debug(f"({now}) Forwarding nginx to {server_ip}:{server_port} ({server_host})") return AuthSuccess(server_ip, server_port) diff --git a/kolombo/auth/protocol.py b/kolombo/auth/protocol.py index fc81c03..8b395bc 100644 --- a/kolombo/auth/protocol.py +++ b/kolombo/auth/protocol.py @@ -9,10 +9,10 @@ class AuthSuccess(Response): """ Response that MUST be returned after successful auth. - **Parameters:** + Parameters: - * **server_ip** - IP that nginx will use to access real SMTP/IMAP server. - * **server_port** - port that nginx will use to access real SMTP/IMAP server. + * server_ip - IP that nginx will use to access real SMTP/IMAP server. + * server_port - port that nginx will use to access real SMTP/IMAP server. """ def __init__(self, server_ip: str, server_port: int) -> None: @@ -28,13 +28,13 @@ class AuthError(Response): """ Response that MUST be returned when auth error occured. - **Parameters:** + Parameters: - * **status** - status that explains error that happened. - * **retry** - *(optional)* whether client can retry (default behaviour) + * status - status that explains error that happened. + * retry - *(optional)* whether client can retry (default behaviour) or connection will be closed. - * **wait** - *(optional)* seconds to wait if client can retry (default - 2). - * **error_code** - *(optional)* error code that will be used in SMTP case + * wait - *(optional)* seconds to wait if client can retry (default - 2). + * error_code - *(optional)* error code that will be used in SMTP case (default - "535 5.7.0", a.k.a. "Authentication Failed"). """ diff --git a/kolombo/bin/__init__.py b/kolombo/bin/__init__.py index a0111c7..c1717de 100644 --- a/kolombo/bin/__init__.py +++ b/kolombo/bin/__init__.py @@ -1 +1,32 @@ -from kolombo.bin.kolombo import main # noqa: F401 +from os import path + +from typer import Context, Typer + +from kolombo import conf +from kolombo.console import error +from kolombo.dkim import dkim_cli +from kolombo.domain import domain_cli +from kolombo.init import init +from kolombo.run import run_cli +from kolombo.stop import stop_cli +from kolombo.user import user_cli + +kolombo_cli = Typer(name="kolombo", add_completion=True) +kolombo_cli.command("init")(init) +kolombo_cli.add_typer(domain_cli, name="domain") +kolombo_cli.add_typer(dkim_cli, name="dkim") +kolombo_cli.add_typer(user_cli, name="user") +kolombo_cli.add_typer(run_cli, name="run") +kolombo_cli.add_typer(stop_cli, name="stop") + + +@kolombo_cli.callback() +def main(ctx: Context) -> None: + if ctx.invoked_subcommand == "init": + return + + if not path.exists("/etc/kolombo/kolombo.conf"): + error("Kolombo is not initialized! Run [code]kolombo init[/] first") + exit(1) + + conf.read_configuration() diff --git a/kolombo/bin/domain.py b/kolombo/bin/domain.py deleted file mode 100644 index 3820ae2..0000000 --- a/kolombo/bin/domain.py +++ /dev/null @@ -1,141 +0,0 @@ -from rich.markdown import Markdown -from typer import Argument, Option, Typer - -from kolombo import conf -from kolombo.bin.util import ( - CliLog, - async_command, - build_kolombo_image, - create_network, - kolombo_image_exists, - run_container, -) -from kolombo.models import Domain -from kolombo.resources import init_database - -cli = Typer() - - -@cli.command("list") -@async_command -async def list_domains( - conf: str = Option(None, help="Path to .env file with configuration") # noqa: B008 -) -> None: - log = CliLog() - await init_database() - - all_domains = await Domain.all_active() - active_pairs = [f"{domain.mx} -> {domain.actual}" for domain in all_domains] - log.step(f"Active domains: {len(active_pairs)}") - if len(active_pairs) > 0: - log.info(Markdown(f"* {'* '.join(active_pairs)}")) - - -async def _update_virtual_domains() -> None: - domains_list = "\n".join(domain.actual for domain in await Domain.all_active()) - with open("/etc/kolombo/virtual/domains", mode="w") as domains_file: - domains_file.write(f"# File is auto-generated\n{domains_list}\n") - - -async def _update_virtual_ssl_map() -> None: - domains_mx = [domain.mx for domain in await Domain.all_active()] - ssl_map = "\n".join( - f"{mx} /etc/letsencrypt/live/{mx}/privkey.pem " - f"/etc/letsencrypt/live/{mx}/fullchain.pem" - for mx in domains_mx - ) - with open("/etc/kolombo/virtual/ssl_map", mode="w") as ssl_map_file: - ssl_map_file.write(f"{ssl_map}\n") - - -_nginx_config = """# File was auto-generated by kolombo -# Receive-only SMTP & IMAP mail reading -server { - server_name {{MX_DOMAIN}}; - ssl_certificate /etc/letsencrypt/live/{{MX_DOMAIN}}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{MX_DOMAIN}}/privkey.pem; - auth_http_header X-Secret-Key {{SECRET_KEY}}; - auth_http_header X-Domain {{ACTUAL_DOMAIN}}; - auth_http kolombo-auth:7089/auth; - - listen 993 ssl; - protocol imap; -} - -# Send-only SMTP -server { - server_name {{MX_DOMAIN}}; - ssl_certificate /etc/letsencrypt/live/{{MX_DOMAIN}}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{MX_DOMAIN}}/privkey.pem; - auth_http_header X-Secret-Key {{SECRET_KEY}}; - auth_http_header X-Domain {{ACTUAL_DOMAIN}}; - auth_http kolombo-auth:7089/auth; - - listen 587 ssl; - protocol smtp; -} -""" - - -def _add_mail_enabled_config(domain: str, mx: str) -> None: - config = _nginx_config.replace("{{ACTUAL_DOMAIN}}", domain) - config = config.replace("{{MX_DOMAIN}}", mx) - config = config.replace("{{SECRET_KEY}}", conf.NGINX_SECRET_KEY) - with open(f"/etc/kolombo/mail-enabled/{mx}.conf", mode="w") as nginx_file: - nginx_file.write(config) - - -def _generate_dkim_key(domain: str) -> str: - create_network("kolombo", subnet="192.168.79.0/24") - return run_container( - "kolombo-sender", - args=f"gen_key {domain}", - name=f"kolombo-{domain}-sender", - net="kolombo", - volumes=["/etc/kolombo/dkim_keys:/etc/opendkim/keys"], - # Remove container after stop, attach interactive TTY - other_flags="--rm -it", - ) - - -def _parse_txt_record(output: str) -> str: - txt_record = output[output.find("(") + 1 : output.find(")")] # noqa: E203 - return txt_record.replace('"\n\t "', "") - - -@cli.command("add") -@async_command -async def add_domain( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - domain: str = Argument(..., help="Domain that come after @ in email"), # noqa: B008 - mx: str = Argument(None, help="Domain from DNS MX record if exists"), # noqa: B008 -) -> None: - log = CliLog() - await init_database() - - if mx is None: - mx = domain - - if not domain or not mx: - log.error("Arguments MUST NOT be empty string") - exit(1) - elif await Domain.objects.filter(actual=domain, mx=mx).exists(): - log.error(f"Domain pair '{mx} -> {domain}' exists") - exit(2) - - log.step("- Adding config to mail-enabled") - _add_mail_enabled_config(domain, mx) - # TODO: Do postfix reload in kolombo-receiver - log.step("- Building kolombo-sender for DKIM key generation (if does not exist)") - if not kolombo_image_exists(component="sender"): - build_kolombo_image(component="sender") - - log.step("- Generating DKIM key ([u]save TXT DNS record from output[/u])") - output = _generate_dkim_key(domain) - log.info(f"mail._domainkey.{domain} TXT: {_parse_txt_record(output)}") - log.step(f"- Adding pair '{mx} -> {domain}' to database") - await Domain.objects.create(actual=domain, mx=mx) - log.step("- Updating virtual domains") - await _update_virtual_domains() - await _update_virtual_ssl_map() - log.step(f"Domain pair '{mx} -> {domain}' added!") diff --git a/kolombo/bin/kolombo.py b/kolombo/bin/kolombo.py deleted file mode 100644 index 154e2b9..0000000 --- a/kolombo/bin/kolombo.py +++ /dev/null @@ -1,73 +0,0 @@ -from os import environ - -from typer import Typer - -from kolombo.bin.domain import cli as domain_cli -from kolombo.bin.run import cli as run_cli -from kolombo.bin.user import cli as user_cli -from kolombo.bin.util import CliLog, run_command - -cli = Typer() -cli.add_typer(user_cli, name="user") -cli.add_typer(domain_cli, name="domain") -cli.add_typer(run_cli, name="run") -#: Default content of /etc/kolombo/kolombo.env file -_default_env = """# This is configuration file for kolombo that is used by default -# To enable debug - change to 1 here -DEBUG=0 -# Secret key that is used to determine that nginx is using API -# NB! Change this secret! -NGINX_SECRET_KEY=changeme -# Maximum auth attempts -MAX_ATTEMPTS=3 -# Salt used for passwords hashing -# NB! Change this secret! -SALT=changeme -# Name of SQLite database file -DATABASE_NAME=kolombo -""" -#: Default content of /etc/kolombo/virtual/ files -_virtual = { - "addresses": "bob@example.com bob@example.com", - "domains": "example.com", - "mailbox": "bob@example.com bob@example.com/", - "ssl_map": ( - "example.com /etc/letsencrypt/live/example.com/privkey.pem " - "/etc/letsencrypt/live/example.com/fullchain.pem" - ), -} - - -@cli.command("setup") -def setup() -> None: - """Set up kolombo for current user""" - log = CliLog() - log.step("- Installing bash completion") - run_command("kolombo --install-completion bash") - - log.step("- Creating /etc/kolombo folders and files") - log.step("-- Creating /etc/kolombo folder (using sudo)") - run_command("mkdir -p -m 770 /etc/kolombo", sudo=True) - - user = environ["USER"] - log.step(f"-- Setting /etc/kolombo owner to {user} (using sudo)") - run_command(f"chown {user}:{user} /etc/kolombo", sudo=True) - - log.step("-- Creating /etc/kolombo folders for volumes") - run_command("mkdir -p -m 770 /etc/kolombo/maildirs") - run_command("mkdir -p -m 770 /etc/kolombo/mail-enabled") - run_command("mkdir -p -m 770 /etc/kolombo/virtual") - run_command("mkdir -p -m 700 /etc/kolombo/dkim_keys") - - log.step("-- Writing default conf to /etc/kolombo/kolombo.env") - with open("/etc/kolombo/kolombo.env", "w") as default_config: - default_config.write(_default_env) - - for file in ("addresses", "domains", "mailbox", "ssl_map"): - log.step(f"-- Writing default file to /etc/kolombo/virtual/{file}") - with open(f"/etc/kolombo/virtual/{file}", "w") as virtual_file: - virtual_file.write(f"{_virtual[file]}\n") - - -def main() -> None: - cli() diff --git a/kolombo/bin/run.py b/kolombo/bin/run.py deleted file mode 100644 index 641ae77..0000000 --- a/kolombo/bin/run.py +++ /dev/null @@ -1,125 +0,0 @@ -from typer import Option, Typer - -from kolombo.bin.util import ( - CliLog, - async_command, - build_kolombo_image, - create_network, - kolombo_image_exists, - run_container, -) -from kolombo.models import Domain - -cli = Typer() -_component_volumes = { - "auth": ["/etc/kolombo:/etc/kolombo"], - "nginx": [ - "/etc/kolombo/mail-enabled:/etc/nginx/mail-enabled", - "/etc/letsencrypt:/etc/letsencrypt:ro", - ], - "receiver": [ - "/etc/kolombo/maildirs:/var/mail", - "/etc/kolombo/virtual:/etc/postfix/virtual", - "/etc/letsencrypt:/etc/letsencrypt:ro", - ], -} - - -def run_kolombo_component(component: str, build: bool) -> None: - log = CliLog() - if build or not kolombo_image_exists(component): - log.step(f"- Building kolombo-{component}") - build_kolombo_image(component=component) - - log.step("- Creating kolombo network (if does not exist)") - create_network("kolombo", subnet="192.168.79.0/24") - - log.step(f"- Running kolombo-{component} container") - ports = None - if component == "receiver": - ports = ["25:25"] - elif component == "nginx": - ports = ["587:587", "993:993"] - - # Attach TTY (for receiver) and run in background - other_flags = "-t -d" if component == "receiver" else "-d" - run_container( - f"kolombo-{component}", - name=f"kolombo-{component}", - net="kolombo", - volumes=_component_volumes[component], - ports=ports, - other_flags=other_flags, - ) - - -@cli.command("receiver") -def run_receiver( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - build: bool = Option(False, help="Whether to skip build step"), # noqa: B008 -) -> None: - run_kolombo_component("receiver", build) - - -@cli.command("auth") -def run_auth( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - build: bool = Option(False, help="Whether to skip build step"), # noqa: B008 -) -> None: - run_kolombo_component("auth", build) - - -@cli.command("nginx") -def run_nginx( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - build: bool = Option(False, help="Whether to skip build step"), # noqa: B008 -) -> None: - run_kolombo_component("nginx", build) - - -async def _run_senders(build: bool) -> None: - log = CliLog() - domains = {domain.actual for domain in await Domain.all_active()} - if len(domains) < 1: - log.error("No active domains to run senders for. Aborting") - exit(0) - - if build or not kolombo_image_exists(component="sender"): - log.step("- Building kolombo-sender") - build_kolombo_image(component="sender") - - log.step("- Creating kolombo network (if does not exist)") - create_network("kolombo", subnet="192.168.79.0/24") - - for domain in domains: - log.step(f"- Running kolombo-sender container for {domain}") - run_container( - "kolombo-sender", - args=domain, - name=f"kolombo-{domain}-sender", - net="kolombo", - volumes=["/etc/kolombo/dkim_keys:/etc/opendkim/keys"], - # Attach TTY and run in background - other_flags="-t -d", - ) - - -@cli.command("senders") -@async_command -async def run_senders( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - build: bool = Option(False, help="Whether to skip build step"), # noqa: B008 -) -> None: - await _run_senders(build) - - -@cli.command("all") -@async_command -async def run_all( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - build: bool = Option(False, help="Whether to skip build step"), # noqa: B008 -) -> None: - run_kolombo_component("receiver", build) - await _run_senders(build) - run_kolombo_component("auth", build) - run_kolombo_component("nginx", build) diff --git a/kolombo/bin/user.py b/kolombo/bin/user.py deleted file mode 100644 index bd9cdc8..0000000 --- a/kolombo/bin/user.py +++ /dev/null @@ -1,77 +0,0 @@ -from base64 import b64encode - -from cryptography.hazmat.primitives.kdf.scrypt import Scrypt -from rich.markdown import Markdown -from typer import Argument, Option, Typer, prompt - -from kolombo import conf -from kolombo.bin.util import CliLog, async_command -from kolombo.models import Domain, User -from kolombo.resources import init_database - -cli = Typer() - - -@cli.command("list") -@async_command -async def list_users( - conf: str = Option(None, help="Path to .env file with configuration") # noqa: B008 -) -> None: - log = CliLog() - await init_database() - - active_users = [user.email for user in await User.all_active()] - log.step(f"Active users: {len(active_users)}") - if len(active_users) > 0: - log.info(Markdown(f"* {'* '.join(active_users)}")) - - -async def _add_user(email: str, password: str, domain: str) -> None: - kdf = Scrypt(conf.SALT, length=32, n=2 ** 16, r=8, p=1) # type: ignore[call-arg] - b64_password = b64encode(kdf.derive(password.encode("utf-8"))) - await User.objects.create(email=email, password=b64_password, domain=domain) - - -async def _update_virtual_files() -> None: - emails = [user.email for user in await User.all_active()] - addresses = "\n".join(f"{email} {email}" for email in emails) - with open("/etc/kolombo/virtual/addresses", mode="w") as addresses_file: - addresses_file.write(f"{addresses}\n") - - mailboxes = "\n".join(f"{email} {email}/" for email in emails) - with open("/etc/kolombo/virtual/mailbox", mode="w") as mailbox_file: - mailbox_file.write(f"{mailboxes}\n") - - -@cli.command("add") -@async_command -async def add_user( - conf: str = Option(None, help="Path to .env file with configuration"), # noqa: B008 - email: str = Argument(..., help="Email for new user"), # noqa: B008 -) -> None: - log = CliLog() - await init_database() - - if "@" not in email: - log.error(f"Email '{email}' does not contain @") - exit(1) - - domain = email.split("@", maxsplit=1)[1].strip() - if not domain: - log.error("Domain part MUST NOT be empty string") - exit(2) - elif not await Domain.objects.filter(active=True, actual=domain).exists(): - log.error( - f"Domain '{domain}' is not added, you can add it via " - f"[code]kolombo domain add {domain} mx.{domain}[/code]", - ) - exit(3) - elif await User.objects.filter(email=email).exists(): - log.error(f"User with email '{email}' exists") - exit(4) - - password = prompt(f"{email} password", hide_input=True, confirmation_prompt=True) - await _add_user(email, password, domain) - await _update_virtual_files() - # TODO: Do postfix reload in kolombo-receiver - log.step(f"User '{email}' added!") diff --git a/kolombo/bin/util.py b/kolombo/bin/util.py deleted file mode 100644 index 810427b..0000000 --- a/kolombo/bin/util.py +++ /dev/null @@ -1,92 +0,0 @@ -from asyncio import run -from functools import wraps -from importlib.resources import path as resource_path - -# Security implications are considered -from subprocess import STDOUT, CalledProcessError, check_output # nosec: B404 -from typing import Any, Awaitable, Callable, List, Optional, Union - -from rich.console import Console -from rich.markdown import Markdown - - -def async_command(func: Callable[..., Awaitable[None]]) -> Callable[..., Any]: - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - return run(func(*args, **kwargs)) - - return wrapper - - -def run_command(command: str, sudo: bool = False) -> str: - cmd = command.split(" ") - if sudo: - cmd.insert(0, "sudo") - # We trust input here, it does not come from user - return check_output(cmd, text=True, stderr=STDOUT) # nosec: B603 - - -def docker(command: str) -> str: - return run_command(f"docker {command}", sudo=True) - - -def kolombo_image_exists(component: str) -> bool: - try: - docker(f"image inspect kolombo-{component}") - return True - except CalledProcessError: - return False - - -def build_kolombo_image(component: str) -> str: - with resource_path(package="kolombo", resource=".") as context: - dockerfile = f"{context}/docker/{component}/Dockerfile" - return docker( - f"build --no-cache -f {dockerfile} -t kolombo-{component} {context}" - ) - - -def run_container( - image: str, - args: Optional[str] = None, - *, - name: Optional[str] = None, - net: Optional[str] = None, - volumes: Optional[List[str]] = None, - ports: Optional[List[str]] = None, - other_flags: str = "-d", -) -> str: - cmd = "run" - if args is not None: - image += f" {args}" - if name is not None: - cmd += f" --name {name}" - if net is not None: - cmd += f" --net {net}" - if volumes: - cmd += f" -v {' -v '.join(volumes)}" - if ports: - cmd += f" -p {' -p '.join(ports)}" - - return docker(f"{cmd} {other_flags} {image}") - - -def create_network(network: str, subnet: str) -> str: - # Create only if does not exist - if not docker(f"network ls -q --filter name={network}"): - return docker(f"network create {network} --subnet={subnet}") - return "Network already created" - - -class CliLog: - def __init__(self) -> None: - self._console = Console() - - def step(self, message: Union[str, Markdown]) -> None: - self._console.print(message, style="bold green") - - def info(self, message: Union[str, Markdown]) -> None: - self._console.print(message, style="bold cyan") - - def error(self, message: Union[str, Markdown]) -> None: - self._console.print(message, style="bold red") diff --git a/kolombo/conf.py b/kolombo/conf.py index 7b2b2b6..fd1555e 100644 --- a/kolombo/conf.py +++ b/kolombo/conf.py @@ -1,26 +1,45 @@ -import sys -from os import R_OK, access, path -from warnings import warn - -from starlette.config import Config - -_config_file = "/etc/kolombo/kolombo.env" -if "--conf" in sys.argv: - _config_file = sys.argv[sys.argv.index("--conf") + 1] - -if not path.exists(_config_file) or not access(_config_file, R_OK): - warn(f"Config file ({_config_file}) does not exist or not readable", UserWarning) - -_config = Config(env_file=_config_file) -#: Whether debug is enabled -DEBUG: bool = _config.get("DEBUG", bool, False) -#: Secret key that should be sent in X-Secret-Key header from nginx -NGINX_SECRET_KEY: str = _config.get("NGINX_SECRET_KEY", str, "changeme") -#: Max number of attempts before API closes connection with client -MAX_ATTEMPTS: int = _config.get("MAX_ATTEMPTS", int, 3) +from kolombo.console import enable_debug + +#: Whether debug mode is enabled +DEBUG: bool = False +#: Secret key that is sent in X-Secret-Key header from nginx +NGINX_SECRET_KEY: str = "changeme" +#: Maximum number of auth attempts before API closes connection with client +MAX_ATTEMPTS: int = 3 #: Salt used for passwords hashing -SALT: bytes = _config.get("SALT", str, "changeme").encode("utf-8") -#: Name of database file without extension -DATABASE_NAME: str = _config.get("DATABASE_NAME", str, "kolombo") +SALT: bytes = b"changeme" #: URL used to connect to database -DATABASE_URL = f"sqlite:////etc/kolombo/{DATABASE_NAME}.sqlite" +DATABASE_URL: str = "sqlite:////etc/kolombo/kolombo.sqlite" + + +def _read_config_file(config_path: str) -> dict[str, str]: + """Copied from starlette.config.Config""" + config: dict[str, str] = {} + with open(config_path) as config_file: + for line in config_file.readlines(): + line = line.strip() + if "=" in line and not line.startswith("#"): + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip("\"'") + config[key] = value + + return config + + +def read_configuration() -> None: + config = _read_config_file("/etc/kolombo/kolombo.conf") + + global DEBUG + DEBUG = bool(config.get("DEBUG", False)) + if DEBUG: + enable_debug() + + global NGINX_SECRET_KEY + NGINX_SECRET_KEY = config.get("NGINX_SECRET_KEY", "changeme") + + global MAX_ATTEMPTS + MAX_ATTEMPTS = int(config.get("MAX_ATTEMPTS", 3)) + + global SALT + SALT = config.get("SALT", "changeme").encode("utf-8") diff --git a/kolombo/configuration_files.py b/kolombo/configuration_files.py new file mode 100644 index 0000000..5346060 --- /dev/null +++ b/kolombo/configuration_files.py @@ -0,0 +1,87 @@ +from typing import Iterable + + +def generate_nginx_config(mx: str, domain: str, secret_key: str) -> str: + from kolombo import __version__ as version + + return ( + f"### Generated by kolombo v{version}\n\n" + "# IMAP (mail reading)\n" + "server {\n" + f" server_name {mx};\n" + f" ssl_certificate /etc/letsencrypt/live/{mx}/fullchain.pem;\n" + f" ssl_certificate_key /etc/letsencrypt/live/{mx}/privkey.pem;\n" + f" auth_http_header X-Secret-Key {secret_key};\n" + f" auth_http_header X-Domain {domain};\n" + f" auth_http kolombo-auth:7089/auth;\n" + " listen 993 ssl;\n" + " protocol imap;\n" + "}\n\n" + "# POP3 (mail reading, use IMAP if possible)\n" + "server {\n" + f" server_name {mx};\n" + f" ssl_certificate /etc/letsencrypt/live/{mx}/fullchain.pem;\n" + f" ssl_certificate_key /etc/letsencrypt/live/{mx}/privkey.pem;\n" + f" auth_http_header X-Secret-Key {secret_key};\n" + f" auth_http_header X-Domain {domain};\n" + f" auth_http kolombo-auth:7089/auth;\n" + " listen 995 ssl;\n" + " protocol pop3;\n" + "}\n\n" + "# Send-only SMTP\n" + "server {\n" + f" server_name {mx};\n" + f" ssl_certificate /etc/letsencrypt/live/{mx}/fullchain.pem;\n" + f" ssl_certificate_key /etc/letsencrypt/live/{mx}/privkey.pem;\n" + f" auth_http_header X-Secret-Key {secret_key};\n" + f" auth_http_header X-Domain {domain};\n" + f" auth_http kolombo-auth:7089/auth;\n" + " listen 465 ssl;\n" + " protocol smtp;\n" + "}\n" + ) + + +def generate_virtual_domains(domains: Iterable[str]) -> str: + return "\n".join(domains) + + +def generate_virtual_ssl_map(mx_domains: Iterable[str]) -> str: + return "\n".join( + f"{mx} /etc/letsencrypt/live/{mx}/privkey.pem " + f"/etc/letsencrypt/live/{mx}/fullchain.pem" + for mx in mx_domains + ) + + +def generate_senders_compose_config(domains: Iterable[str]) -> str: + compose_file = ( + # fmt: off + "version: '3.8'\n" + "services:\n" + # fmt: on + ) + for index, domain in enumerate(domains): + compose_file += ( + f" kolombo-{domain}-sender:\n" + " image: kolombo-sender:latest\n" + " tty: true\n" + " restart: always\n" + f" container_name: kolombo-{domain}-sender\n" + f" hostname: {domain}\n" + " volumes:\n" + f" - /etc/kolombo/dkim_keys/{domain}:/etc/opendkim/keys/{domain}\n" + " networks:\n" + " kolombo-net:\n" + f" ipv4_address: 192.168.79.{10 + index}\n" + f" command: {domain}\n\n" + ) + + compose_file += ( + # fmt: off + "networks:\n" + " kolombo-net:\n" + " external: true\n" + # fmt: on + ) + return compose_file diff --git a/kolombo/console.py b/kolombo/console.py new file mode 100644 index 0000000..c27b4a4 --- /dev/null +++ b/kolombo/console.py @@ -0,0 +1,50 @@ +from typing import Any + +from rich.console import Console +from rich.markdown import Markdown + +_console = Console() +_debug_mode = False + + +def started(message: str) -> None: + """Message to indicate that command is started""" + _console.print(message, style="underline") + _console.line() + + +def step(message: str) -> None: + """Message to indicate current step""" + _console.print(f"> {message}", style="bold green") + + +def finished(message: str) -> None: + """Message to indicate that command is finished""" + _console.line() + _console.print(f":checkered_flag: {message}", style="green") + + +def info(message: str) -> None: + _console.print(message) + + +def print_list(list_to_print: list[Any]) -> None: + _console.print(Markdown(f"* {'* '.join(list_to_print)}")) + + +def warning(message: str) -> None: + _console.print(f":warning-text: {message}", style="underline bold orange1") + + +def error(message: str) -> None: + _console.print(f":x-text: {message}", style="bold red") + + +def enable_debug() -> None: + global _debug_mode + _debug_mode = True + + +def debug(message: str) -> None: + if _debug_mode: + _console.print(f"[DEBUG] {message}", style="italic cyan") diff --git a/kolombo/dkim.py b/kolombo/dkim.py new file mode 100644 index 0000000..5a5df44 --- /dev/null +++ b/kolombo/dkim.py @@ -0,0 +1,55 @@ +from os import path + +from docker import from_env # type: ignore[import] +from typer import Argument, Typer + +from kolombo.console import error, info, step +from kolombo.util import build_kolombo_image, execute_as_root + +dkim_cli = Typer() + + +def read_dkim_txt_record(domain: str) -> str: + with open(f"/etc/kolombo/dkim_keys/{domain}.txt", mode="r") as txt_file: + txt_file_content = txt_file.read() + + paren_open_idx = txt_file_content.find("(") + paren_close_idx = txt_file_content.find(")") + txt_record = txt_file_content[paren_open_idx + 1 : paren_close_idx] # noqa: E203 + return txt_record.replace('"\n\t "', "") + + +@dkim_cli.command("generate") +@execute_as_root +def generate_keys( + domain: str = Argument(..., help="Domain to generate DKIM keys for"), # noqa: B008 +) -> None: + client = from_env() + build_kolombo_image(client, "dkim-gen") + + step(f"Generating DKIM keys for domain: {domain}") + client.containers.run( + "kolombo-dkim-gen", + domain, + stderr=True, + auto_remove=True, + volumes=["/etc/kolombo/dkim_keys:/etc/opendkim/keys"], + ) + + dkim_txt = read_dkim_txt_record(domain) + info(f"[b]TXT[/] record for [b u]mail._domainkey.{domain}[/] is: {dkim_txt}") + + +@dkim_cli.command("txt") +@execute_as_root +def get_txt_record( + domain: str = Argument( # noqa: B008 + ..., help="Domain to get DKIM TXT record for (if it was generated)" + ), +) -> None: + if not path.exists(f"/etc/kolombo/dkim_keys/{domain}.txt"): + error(f"There is no DKIM TXT record generated for {domain}!") + exit(1) + + dkim_txt = read_dkim_txt_record(domain) + info(f"[b]TXT[/] record for [b u]mail._domainkey.{domain}[/] is: {dkim_txt}") diff --git a/kolombo/docker/Dockerfile.auth b/kolombo/docker/Dockerfile.auth new file mode 100644 index 0000000..4f984be --- /dev/null +++ b/kolombo/docker/Dockerfile.auth @@ -0,0 +1,15 @@ +FROM python:3.9-slim +LABEL maintainer="Igor Nehoroshev " + +# Volume MUST be attached to /etc/kolombo +RUN mkdir /app /etc/kolombo +COPY . /app/kolombo +WORKDIR /app + +COPY docker/auth/requirements.txt /app/requirements.txt +COPY docker/auth/run_auth_api.sh /bin/run_auth_api +RUN pip install --no-cache-dir -r requirements.txt + +EXPOSE 7089/tcp + +CMD ["/bin/run_auth_api"] diff --git a/kolombo/docker/Dockerfile.dkim-gen b/kolombo/docker/Dockerfile.dkim-gen new file mode 100644 index 0000000..de38558 --- /dev/null +++ b/kolombo/docker/Dockerfile.dkim-gen @@ -0,0 +1,9 @@ +FROM alpine:3.14 +LABEL maintainer="Igor Nehoroshev " + +RUN apk add --no-cache opendkim-utils +RUN mkdir /etc/opendkim/keys +WORKDIR /etc/opendkim/keys +COPY docker/dkim-gen/gen_key.sh /bin/gen_key + +ENTRYPOINT ["/bin/gen_key"] diff --git a/kolombo/docker/nginx/Dockerfile b/kolombo/docker/Dockerfile.nginx similarity index 75% rename from kolombo/docker/nginx/Dockerfile rename to kolombo/docker/Dockerfile.nginx index 4aa03bb..04e0b85 100644 --- a/kolombo/docker/nginx/Dockerfile +++ b/kolombo/docker/Dockerfile.nginx @@ -1,6 +1,6 @@ -FROM nginx:mainline-alpine +FROM nginx:1.21.1-alpine LABEL maintainer="Igor Nehoroshev " RUN apk add --no-cache nginx-mod-mail && mkdir /etc/nginx/mail-enabled COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf -EXPOSE 587/tcp 993/tcp +EXPOSE 465/tcp 993/tcp 995/tcp diff --git a/kolombo/docker/Dockerfile.receiver b/kolombo/docker/Dockerfile.receiver new file mode 100644 index 0000000..dba536e --- /dev/null +++ b/kolombo/docker/Dockerfile.receiver @@ -0,0 +1,26 @@ +FROM alpine:3.14 +LABEL maintainer="Igor Nehoroshev " + +# Install needed software +RUN apk add --no-cache rsyslog postfix dovecot dovecot-pop3d openssl \ +# Generate SSL certificate (to fallback to if TLS SNI did not found proper one) + && openssl req -x509 -nodes -newkey rsa:4096 -keyout /etc/postfix/tls.key -out /etc/postfix/tls.crt -days 1825 -subj '/CN=Kolombo Receiver' +# Copy script that "checks password" (just returns OK, check MUST be on nginx mail proxy level) +COPY docker/receiver/checkpassword.sh /bin/checkpassword +# Copy script that updates postmap +COPY docker/receiver/update_map.sh /bin/update_map +COPY docker/receiver/start_receiver.sh /bin/start_receiver +# Copy configurations +COPY docker/receiver/rsyslog.conf /etc/rsyslog.conf +COPY docker/receiver/dovecot.conf /etc/dovecot/dovecot.conf +COPY docker/receiver/postfix.conf /etc/postfix/main.cf +# Create aliases, so that Postfix stop giving warnings +RUN touch /etc/postfix/aliases && newaliases +# Volume should be attached to /etc/postfix/virtual_files +RUN mkdir /etc/postfix/virtual_files && cd /etc/postfix/virtual_files && touch domains addresses mailbox ssl_map + +# 25 for SMTP (receive, direct) +# 143 for IMAP (read mail, through nginx mail proxy) +# 110 for POP3 (read mail legacy, through nginx mail proxy) +EXPOSE 25/tcp 143/tcp 110/tcp +CMD ["/bin/start_receiver"] diff --git a/kolombo/docker/sender/Dockerfile b/kolombo/docker/Dockerfile.sender similarity index 64% rename from kolombo/docker/sender/Dockerfile rename to kolombo/docker/Dockerfile.sender index 63ecaf4..abab754 100644 --- a/kolombo/docker/sender/Dockerfile +++ b/kolombo/docker/Dockerfile.sender @@ -1,23 +1,16 @@ -FROM alpine:3.12 +FROM alpine:3.14 LABEL maintainer="Igor Nehoroshev " -ARG TIMEZONE=Europe/Tallinn - -COPY docker/sender/rsyslog.conf /etc/rsyslog.conf -COPY docker/sender/gen_key.sh /bin/gen_key -COPY docker/sender/startup.sh /bin/startup - # Install needed software -RUN apk add --no-cache rsyslog postfix opendkim opendkim-utils tzdata \ +RUN apk add --no-cache rsyslog postfix opendkim \ # Add postfix to opendkim group - && addgroup postfix opendkim \ -# Set timezone - && ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime \ - && echo "${TIMEZONE}" > /etc/timezone + && addgroup postfix opendkim # Copy configurations +COPY docker/sender/rsyslog.conf /etc/rsyslog.conf COPY docker/sender/postfix.conf /etc/postfix/main.cf COPY docker/sender/opendkim.conf /etc/opendkim/opendkim.conf +COPY docker/sender/start_sender.sh /bin/start_sender # Create aliases, so that Postfix stop giving warnings RUN touch /etc/postfix/aliases && newaliases @@ -30,4 +23,5 @@ RUN mkdir -p /run/opendkim /etc/opendkim/keys \ # 25 for SMTP (send) EXPOSE 25/tcp -ENTRYPOINT ["/bin/startup"] + +ENTRYPOINT ["/bin/start_sender"] \ No newline at end of file diff --git a/kolombo/docker/auth/Dockerfile b/kolombo/docker/auth/Dockerfile deleted file mode 100644 index 6d4c176..0000000 --- a/kolombo/docker/auth/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.8-slim -LABEL maintainer="Igor Nehoroshev " - -COPY . /app/kolombo -COPY docker/auth/requirements.txt /app/requirements.txt -WORKDIR /app -RUN pip install --no-cache-dir -r requirements.txt \ - && mkdir /etc/kolombo \ - && find kolombo -type d -name __pycache__ -exec rm -r {} \+ -# Volume should be attached to /etc/kolombo -COPY docker/auth/kolombo.conf /etc/kolombo/kolombo.env -EXPOSE 7089/tcp -CMD uvicorn kolombo:api --loop=uvloop --host=0.0.0.0 --port=7089 diff --git a/kolombo/docker/auth/kolombo.conf b/kolombo/docker/auth/kolombo.conf deleted file mode 100644 index 6a2a320..0000000 --- a/kolombo/docker/auth/kolombo.conf +++ /dev/null @@ -1,5 +0,0 @@ -DEBUG=0 -NGINX_SECRET_KEY=changeme -MAX_ATTEMPTS=3 -SALT=changeme -DATABASE_NAME=kolombo diff --git a/kolombo/docker/auth/requirements.txt b/kolombo/docker/auth/requirements.txt index 361ee44..f815993 100644 --- a/kolombo/docker/auth/requirements.txt +++ b/kolombo/docker/auth/requirements.txt @@ -1,11 +1,16 @@ -uvloop==0.14.0 -ormar==0.7.1 -databases==0.4.1 -aiosqlite==0.16.0 -cryptography==3.2.1 -fastapi==0.62.0 -uvicorn==0.12.3 -httptools==0.1.* - -typer==0.3.2 -rich==9.3.0 +# Fast async event loop +uvloop==0.16.0 +# For better output +rich==10.7.0 +# ORM +ormar==0.10.16 +databases==0.4.3 +aiosqlite==0.17.0 +# Passwords hashing +cryptography==3.4.8 +# For API +fastapi==0.68.1 +pydantic==1.8.2 +uvicorn==0.15.0 +# Since 0.12.0 uvicorn does not install httptools (and uvloop) +httptools==0.3.0 diff --git a/kolombo/docker/auth/run_auth_api.sh b/kolombo/docker/auth/run_auth_api.sh new file mode 100755 index 0000000..f274ba2 --- /dev/null +++ b/kolombo/docker/auth/run_auth_api.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +find kolombo -type d -name __pycache__ -exec rm -r {} \+ + +# Trust nginx with real IP addresses +export FORWARDED_ALLOW_IPS=192.168.79.120 +exec uvicorn kolombo.auth.api:app --host=0.0.0.0 --port 7089 \ + --loop=uvloop --http httptools --interface asgi3 \ + --proxy-headers --use-colors diff --git a/kolombo/docker/dkim-gen/gen_key.sh b/kolombo/docker/dkim-gen/gen_key.sh new file mode 100755 index 0000000..cc9ae40 --- /dev/null +++ b/kolombo/docker/dkim-gen/gen_key.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +if [ -z "$1" ] ; then + echo "Provide domain as first argument" + exit 1 +fi + +opendkim-genkey -b 2048 -d ${1} -s mail -v +mv mail.private ${1} +mv mail.txt ${1}.txt +chown opendkim:opendkim ${1} diff --git a/kolombo/docker/nginx/nginx.conf b/kolombo/docker/nginx/nginx.conf index 848b7c4..54c8350 100644 --- a/kolombo/docker/nginx/nginx.conf +++ b/kolombo/docker/nginx/nginx.conf @@ -23,9 +23,10 @@ mail { proxy_timeout 20s; smtp_capabilities 8BITMIME "SIZE 10485760" DSN ENHANCEDSTATUSCODES; - smtp_auth plain login; + smtp_auth plain; imap_capabilities IMAP4rev1 UIDPLUS IDLE LITERAL+ QUOTA; - imap_auth plain login; + imap_auth plain; + pop3_auth plain; xclient off; diff --git a/kolombo/docker/receiver/Dockerfile b/kolombo/docker/receiver/Dockerfile deleted file mode 100644 index cf9a17e..0000000 --- a/kolombo/docker/receiver/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM alpine:3.12 -LABEL maintainer="Igor Nehoroshev " - -ARG TIMEZONE=Europe/Tallinn - -COPY docker/receiver/rsyslog.conf /etc/rsyslog.conf -# Volume should be attached to /etc/postfix/virtual -COPY docker/receiver/virtual-domains /etc/postfix/virtual/domains -COPY docker/receiver/virtual-addresses /etc/postfix/virtual/addresses -COPY docker/receiver/virtual-mailbox /etc/postfix/virtual/mailbox -COPY docker/receiver/virtual-ssl_map /etc/postfix/virtual/ssl_map - -# Install needed software -RUN apk add --no-cache rsyslog postfix dovecot tzdata openssl \ -# Set timezone - && ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime \ - && echo "${TIMEZONE}" > /etc/timezone \ -# Generate SSL certificates - && openssl req -x509 -nodes -newkey rsa:4096 -keyout /etc/postfix/tls.key -out /etc/postfix/tls.crt -days 1825 -subj '/CN=Kolombo Receiver' -# Copy script that "checks password" (just returns OK, check MUST be on nginx level) -COPY docker/receiver/checkpassword.sh /bin/checkpassword -# Copy configurations -COPY docker/receiver/dovecot.conf /etc/dovecot/dovecot.conf -COPY docker/receiver/postfix.conf /etc/postfix/main.cf -# Create aliases, so that Postfix stop giving warnings -RUN touch /etc/postfix/aliases && newaliases - -# 25 for SMTP (receive), 143 for IMAP -EXPOSE 25/tcp 143/tcp -CMD postmap /etc/postfix/virtual/addresses \ - && postmap /etc/postfix/virtual/mailbox \ - && postmap -oF /etc/postfix/virtual/ssl_map \ - && chgrp -R postfix /etc/postfix/virtual \ - && postfix start && dovecot && rsyslogd -n diff --git a/kolombo/docker/receiver/dovecot.conf b/kolombo/docker/receiver/dovecot.conf index 7ea311a..1ed8616 100644 --- a/kolombo/docker/receiver/dovecot.conf +++ b/kolombo/docker/receiver/dovecot.conf @@ -1,4 +1,4 @@ -protocols = imap +protocols = imap pop3 listen = * mail_location = maildir:/var/mail/%u diff --git a/kolombo/docker/receiver/postfix.conf b/kolombo/docker/receiver/postfix.conf index f0dd7aa..fc419bd 100644 --- a/kolombo/docker/receiver/postfix.conf +++ b/kolombo/docker/receiver/postfix.conf @@ -13,10 +13,10 @@ queue_minfree = 20971520 virtual_minimum_uid = 1000 virtual_uid_maps = static:1000 virtual_gid_maps = static:1000 -virtual_mailbox_domains = /etc/postfix/virtual/domains +virtual_mailbox_domains = /etc/postfix/virtual_files/domains virtual_mailbox_base = /var/mail -virtual_mailbox_maps = texthash:/etc/postfix/virtual/mailbox -virtual_alias_maps = texthash:/etc/postfix/virtual/addresses +virtual_mailbox_maps = texthash:/etc/postfix/virtual_files/mailbox +virtual_alias_maps = texthash:/etc/postfix/virtual_files/addresses local_recipient_maps = $virtual_mailbox_maps # Network parameters: @@ -33,7 +33,7 @@ unknown_address_reject_code = 554 # Other parameters: mail_owner = postfix setgid_group = postdrop -compatibility_level = 2 +compatibility_level = 3.6 biff = no append_dot_mydomain = no @@ -47,7 +47,7 @@ smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, rej smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining # TLS parameters: -# Reduce opportunities for a potential CPU exhaustion attack with `NO_RENEGOTIATION` +# Reduce opportunities for a potential CPU exhaustion attack with NO_RENEGOTIATION tls_ssl_options = NO_COMPRESSION, NO_RENEGOTIATION tls_high_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256 tls_preempt_cipherlist = yes @@ -62,6 +62,6 @@ smtpd_tls_security_level = encrypt smtpd_tls_cert_file = /etc/postfix/tls.crt smtpd_tls_key_file = /etc/postfix/tls.key # But switch to appropriate normal via SNI when possible -tls_server_sni_maps = hash:/etc/postfix/virtual/ssl_map -smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache +tls_server_sni_maps = lmdb:/etc/postfix/virtual_files/ssl_map +smtpd_tls_session_cache_database = lmdb:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = lmdb:${data_directory}/smtp_scache diff --git a/kolombo/docker/receiver/start_receiver.sh b/kolombo/docker/receiver/start_receiver.sh new file mode 100755 index 0000000..adf909a --- /dev/null +++ b/kolombo/docker/receiver/start_receiver.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +/bin/update_map +chgrp -R 1000 /var/mail +chmod 770 /var/mail +chmod 640 /etc/postfix/main.cf + +postfix start +dovecot +exec rsyslogd -n diff --git a/kolombo/docker/receiver/update_map.sh b/kolombo/docker/receiver/update_map.sh new file mode 100755 index 0000000..fa6a17f --- /dev/null +++ b/kolombo/docker/receiver/update_map.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +postmap /etc/postfix/virtual_files/addresses +postmap /etc/postfix/virtual_files/mailbox +postmap -oF lmdb:/etc/postfix/virtual_files/ssl_map + +chmod -R 640 /etc/postfix/virtual_files +chmod 750 /etc/postfix/virtual_files +chown -R root:postfix /etc/postfix/virtual_files diff --git a/kolombo/docker/receiver/virtual-addresses b/kolombo/docker/receiver/virtual-addresses deleted file mode 100644 index 807ef1d..0000000 --- a/kolombo/docker/receiver/virtual-addresses +++ /dev/null @@ -1 +0,0 @@ -bob@example.com bob@example.com diff --git a/kolombo/docker/receiver/virtual-domains b/kolombo/docker/receiver/virtual-domains deleted file mode 100644 index caa12a8..0000000 --- a/kolombo/docker/receiver/virtual-domains +++ /dev/null @@ -1 +0,0 @@ -example.com \ No newline at end of file diff --git a/kolombo/docker/receiver/virtual-mailbox b/kolombo/docker/receiver/virtual-mailbox deleted file mode 100644 index 62c3f40..0000000 --- a/kolombo/docker/receiver/virtual-mailbox +++ /dev/null @@ -1 +0,0 @@ -bob@example.com bob@example.com/ diff --git a/kolombo/docker/receiver/virtual-ssl_map b/kolombo/docker/receiver/virtual-ssl_map deleted file mode 100644 index 80d4ae0..0000000 --- a/kolombo/docker/receiver/virtual-ssl_map +++ /dev/null @@ -1 +0,0 @@ -example.com /etc/letsencrypt/live/example.com/privkey.pem /etc/letsencrypt/live/example.com/fullchain.pem \ No newline at end of file diff --git a/kolombo/docker/sender/gen_key.sh b/kolombo/docker/sender/gen_key.sh deleted file mode 100755 index e3cced7..0000000 --- a/kolombo/docker/sender/gen_key.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -e - -if [ -z "$1" ] ; then - echo "No domain argument provided" - exit 1 -fi - -cd /etc/opendkim/keys -opendkim-genkey -b 2048 -d ${1} -s mail -v -chown opendkim:opendkim mail.private -mv mail.private ${1} -mv mail.txt ${1}.txt - -echo "TXT record for mail._domainkey.${1}" -cat ${1}.txt diff --git a/kolombo/docker/sender/postfix.conf b/kolombo/docker/sender/postfix.conf index cc7a618..9154916 100644 --- a/kolombo/docker/sender/postfix.conf +++ b/kolombo/docker/sender/postfix.conf @@ -32,7 +32,7 @@ non_smtpd_milters = unix:/tmp/opendkim.sock # Other parameters: mail_owner = postfix setgid_group = postdrop -compatibility_level = 2 +compatibility_level = 3.6 biff = no append_dot_mydomain = no diff --git a/kolombo/docker/sender/startup.sh b/kolombo/docker/sender/start_sender.sh similarity index 50% rename from kolombo/docker/sender/startup.sh rename to kolombo/docker/sender/start_sender.sh index 54b4ba8..5c35222 100755 --- a/kolombo/docker/sender/startup.sh +++ b/kolombo/docker/sender/start_sender.sh @@ -2,26 +2,20 @@ set -e if [ -z "$1" ] ; then - echo "No arguments supplied, arguments usage:" - echo "- 'DOMAIN' to start normally" - echo "- 'gen_key DOMAIN' to generate DKIM key for domain" + echo "No arguments supplied, provide DOMAIN to start normally" exit 1 fi -if [ "$1" == "gen_key" ] ; then - /bin/gen_key "${2}" - exit 0 -fi - if [ ! -f "/etc/opendkim/keys/${1}" ] ; then - echo "DKIM key does not exist! Generate one" + echo "DKIM key does not exist! Generate one via 'kolombo dkim generate ${1}'" exit 2 fi chown -R opendkim:opendkim /etc/opendkim/keys sed -i -e "s/doma.in/${1}/g" /etc/postfix/main.cf sed -i -e "s/doma.in/${1}/g" /etc/opendkim/opendkim.conf +chmod 640 /etc/postfix/main.cf opendkim postfix start -rsyslogd -n +exec rsyslogd -n diff --git a/kolombo/docker/services/docker-compose.yml b/kolombo/docker/services/docker-compose.yml new file mode 100644 index 0000000..714a08c --- /dev/null +++ b/kolombo/docker/services/docker-compose.yml @@ -0,0 +1,62 @@ +version: '3.8' +services: + kolombo-receiver: + image: kolombo-receiver:latest + tty: true + restart: always + container_name: kolombo-receiver + hostname: kolombo-receiver + ports: + # Default SMTP port (receive mail) + - 0.0.0.0:25:25/tcp + volumes: + - /etc/kolombo/maildirs:/var/mail + - /etc/kolombo/virtual:/etc/postfix/virtual_files + - /etc/letsencrypt:/etc/letsencrypt:ro + networks: + kolombo-net: + ipv4_address: 192.168.79.100 + + kolombo-auth: + image: kolombo-auth:latest + restart: always + container_name: kolombo-auth + hostname: kolombo-auth + volumes: + - /etc/kolombo:/etc/kolombo + depends_on: + - kolombo-receiver + networks: + kolombo-net: + ipv4_address: 192.168.79.110 + + kolombo-nginx: + image: kolombo-nginx:latest + restart: always + container_name: kolombo-nginx + hostname: kolombo-nginx + ports: + # Encrypted SMTP port (send mail) + - 0.0.0.0:465:465/tcp + # Encrypted IMAP port (read mail) + - 0.0.0.0:993:993/tcp + # Encrypted POP3 port (read mail, legacy) + - 0.0.0.0:995:995/tcp + volumes: + - /etc/kolombo/mail-enabled:/etc/nginx/mail-enabled + - /etc/letsencrypt:/etc/letsencrypt:ro + depends_on: + - kolombo-auth + networks: + kolombo-net: + ipv4_address: 192.168.79.120 + +networks: + kolombo-net: + name: kolombo-net + driver: bridge + ipam: + driver: default + config: + - subnet: 192.168.79.0/24 + gateway: 192.168.79.1 diff --git a/kolombo/domain.py b/kolombo/domain.py new file mode 100644 index 0000000..ada60f5 --- /dev/null +++ b/kolombo/domain.py @@ -0,0 +1,85 @@ +from typing import TYPE_CHECKING + +from typer import Argument, Typer + +from kolombo.configuration_files import ( + generate_nginx_config, + generate_virtual_domains, + generate_virtual_ssl_map, +) +from kolombo.console import error, finished, info, print_list, started, step, warning +from kolombo.util import async_command, needs_database + +if TYPE_CHECKING: + from kolombo.models import Domain + +domain_cli = Typer() + + +@domain_cli.command("list") +@async_command +@needs_database +async def list_domains() -> None: + from kolombo.models import Domain + + all_domains = await Domain.all_active() + active_pairs = [f"(MX) {domain.mx} -> {domain.actual}" for domain in all_domains] + info(f"Active domains: {len(active_pairs)}") + if len(active_pairs) > 0: + print_list(active_pairs) + + +def update_virtual_files(active_domains: list["Domain"]) -> None: + domains = (domain.actual for domain in active_domains) + virtual_domains = generate_virtual_domains(domains) + with open("/etc/kolombo/virtual/domains", mode="w") as virtual_domains_file: + virtual_domains_file.write(f"{virtual_domains}\n") + + mx_domains = (domain.mx for domain in active_domains) + virtual_ssl_map = generate_virtual_ssl_map(mx_domains) + with open("/etc/kolombo/virtual/ssl_map", mode="w") as virtual_ssl_map_file: + virtual_ssl_map_file.write(f"{virtual_ssl_map}\n") + + +@domain_cli.command("add") +@async_command +@needs_database +async def add_domain( + domain: str = Argument( # noqa: B008 + ..., help="Domain that comes after @ in email" + ), + mx: str = Argument(None, help="Domain from DNS MX record if exists"), # noqa: B008 +) -> None: + from kolombo import conf + from kolombo.models import Domain + + if mx is None: + mx = domain + + if not domain or not mx: + error("Arguments MUST NOT be empty strings!") + exit(1) + elif await Domain.objects.filter(actual=domain, mx=mx).exists(): + error(f"Pair [code]{mx} -> {domain}[/] already exists!") + exit(1) + + started(f"Adding [code]{mx} -> {domain}[/] pair") + + step("Adding configuration to [code]mail-enabled[/]") + nginx_config = generate_nginx_config(mx, domain, conf.NGINX_SECRET_KEY) + with open(f"/etc/kolombo/mail-enabled/{mx}.conf", mode="w") as nginx_file: + nginx_file.write(nginx_config) + + step("Saving to database") + await Domain.objects.create(actual=domain, mx=mx) + + step("Updating virtual files (domains and SSL map)") + active_domains = await Domain.all_active() + update_virtual_files(active_domains) + + warning( + f"Run command [code]kolombo dkim generate {domain}[/] to generate DKIM keys" + ) + warning("Run command [code]kolombo run[/] to reload Kolombo") + + finished(f"Pair [code]{mx} -> {domain}[/] added!") diff --git a/kolombo/init.py b/kolombo/init.py new file mode 100644 index 0000000..61b428d --- /dev/null +++ b/kolombo/init.py @@ -0,0 +1,88 @@ +import sys +from os import getgid, getuid +from pathlib import Path +from pwd import getpwuid + +from typer import confirm, prompt + +from kolombo.console import debug, enable_debug, finished, info, started, step +from kolombo.util import run + +_virtual_configs = { + "addresses": "bob@example.com bob@example.com", + "domains": "example.com", + "mailbox": "bob@example.com bob@example.com/", + "ssl_map": ( + "example.com /etc/letsencrypt/live/example.com/privkey.pem " + "/etc/letsencrypt/live/example.com/fullchain.pem" + ), +} + + +def init() -> None: + from kolombo import __version__ as version + + username = getpwuid(getuid()).pw_name + started(f"Setting up Kolombo for current user [b]{username}[/]") + + step("Creating /etc/kolombo folder ([u]need root privileges[/])") + + info("Creating /etc/kolombo folder (as root)") + run(["mkdir", "-p", "-m", "750", "/etc/kolombo"], as_root=True) + info(f"Changing /etc/kolombo owner to {username} (as root)") + run(["chown", f"{getuid()}:{getgid()}", "/etc/kolombo"], as_root=True) + + step("Writing configuration to /etc/kolombo/kolombo.conf") + debug_mode = confirm("Enable debug mode?", default=False, show_default=True) + if debug_mode: + enable_debug() + + nginx_secret_key: str = prompt( + "Enter secret key for communication between NginX and auth API", + default="changeme", + show_default=True, + hide_input=True, + confirmation_prompt=True, + ) + max_auth_attempts: int = prompt( + "Enter maximum auth attempts per one session", + default="3", + show_default=True, + type=int, + ) + passwords_salt: str = prompt( + "Enter secret key to be used as salt for passwords hashing", + default="changeme", + show_default=True, + hide_input=True, + confirmation_prompt=True, + ) + configuration = ( + f"### Generated by kolombo v{version}\n\n" + "# Whether debug mode is enabled (0 - disabled, 1 - enabled)\n" + f"DEBUG={int(debug_mode)}\n" + "# Secret key that is used to determine that nginx is using API\n" + f"NGINX_SECRET_KEY={nginx_secret_key}\n" + "# Maximum auth attempts per one session\n" + f"MAX_ATTEMPTS={max_auth_attempts}\n" + "# Salt used for passwords hashing\n" + f"SALT={passwords_salt}\n" + ) + with open("/etc/kolombo/kolombo.conf", "w") as config_file: + config_file.write(configuration) + + step("Populating /etc/kolombo with default folders and files") + debug("Creating /etc/kolombo folders for volumes") + folders = ("maildirs", "mail-enabled", "virtual", "dkim_keys") + for folder in folders: + Path(f"/etc/kolombo/{folder}").mkdir(mode=0o770, exist_ok=True) + + for file in ("addresses", "domains", "mailbox", "ssl_map"): + debug(f"Writing default file to /etc/kolombo/virtual/{file}") + with open(f"/etc/kolombo/virtual/{file}", "w") as virtual_file: + virtual_file.write(f"# {_virtual_configs[file]}\n") + + step("Installing auto-completion ([u]restart current shell session to use[/])") + run([sys.argv[0], "--install-completion"]) + + finished("Kolombo is set up!") diff --git a/kolombo/models.py b/kolombo/models.py index 0f94035..0355fdf 100644 --- a/kolombo/models.py +++ b/kolombo/models.py @@ -1,8 +1,17 @@ -from typing import List - +from databases import Database from ormar import Boolean, Integer, Model, String +from sqlalchemy import MetaData, create_engine # type: ignore[import] + +from kolombo import conf + +database = Database(conf.DATABASE_URL) +metadata = MetaData() + -from kolombo.resources import database, metadata +async def init_database() -> None: + await database.connect() + engine = create_engine(conf.DATABASE_URL) + metadata.create_all(engine) class Domain(Model): @@ -20,8 +29,8 @@ class Meta: active: bool = Boolean(default=True) @classmethod - async def all_active(cls) -> List["Domain"]: - return await cls.objects.filter(active=True).all() # type: ignore[return-value] + async def all_active(cls) -> list["Domain"]: + return await cls.objects.filter(active=True).all() class User(Model): @@ -41,5 +50,5 @@ class Meta: active: bool = Boolean(default=True) @classmethod - async def all_active(cls) -> List["User"]: - return await cls.objects.filter(active=True).all() # type: ignore[return-value] + async def all_active(cls) -> list["User"]: + return await cls.objects.filter(active=True).all() diff --git a/kolombo/resources.py b/kolombo/resources.py deleted file mode 100644 index 8668637..0000000 --- a/kolombo/resources.py +++ /dev/null @@ -1,24 +0,0 @@ -from logging import DEBUG, INFO, Formatter, StreamHandler, getLogger - -from databases import Database -from sqlalchemy import MetaData, create_engine # type: ignore[import] - -from kolombo import conf - -database = Database(conf.DATABASE_URL) -metadata = MetaData() -log = getLogger("kolombo") - - -def init_logger() -> None: - log.setLevel(DEBUG if conf.DEBUG else INFO) - formatter = Formatter("%(asctime)s [kolombo:%(levelname)s] %(message)s") - stderr_handler = StreamHandler() - stderr_handler.setFormatter(formatter) - log.addHandler(stderr_handler) - - -async def init_database() -> None: - await database.connect() - engine = create_engine(conf.DATABASE_URL) - metadata.create_all(engine) diff --git a/kolombo/run.py b/kolombo/run.py new file mode 100644 index 0000000..82354b7 --- /dev/null +++ b/kolombo/run.py @@ -0,0 +1,77 @@ +from docker import from_env # type: ignore[import] +from typer import Context, Typer + +from kolombo.configuration_files import generate_senders_compose_config +from kolombo.console import debug, error, step +from kolombo.util import ( + async_command, + build_kolombo_image, + execute_as_root, + needs_database, + run, + up_kolombo_service, +) + +run_cli = Typer(invoke_without_command=True) + + +@run_cli.callback() +def run_main(ctx: Context) -> None: + if ctx.invoked_subcommand is None: + debug("Deploying all Kolombo services") + # TODO + + +@run_cli.command("receiver") +@execute_as_root +def run_receiver() -> None: + client = from_env() + build_kolombo_image(client, "receiver") + + step("Bringing up receiver service") + up_kolombo_service("receiver") + + +@run_cli.command("auth") +@execute_as_root +def run_auth() -> None: + client = from_env() + build_kolombo_image(client, "auth") + + step("Bringing up auth service") + up_kolombo_service("auth") + + +@run_cli.command("nginx") +@execute_as_root +def run_nginx() -> None: + client = from_env() + build_kolombo_image(client, "nginx") + + step("Bringing up nginx service") + up_kolombo_service("nginx") + + +@run_cli.command("senders") +@execute_as_root +@async_command +@needs_database +async def run_senders() -> None: + from kolombo.models import Domain + + domains = [domain.actual for domain in await Domain.all_active()] + if len(domains) < 1: + error("No active domains to run senders for") + exit(1) + + client = from_env() + build_kolombo_image(client, "sender") + + senders_compose_config = generate_senders_compose_config(domains) + project_name = "kolombo_senders" + file_path = "/etc/kolombo/senders-compose.yml" + with open(file_path, mode="w") as compose_file: + compose_file.write(senders_compose_config) + + compose_command = ["docker-compose", "-p", project_name, "-f", file_path, "up"] + run([*compose_command, "--force-recreate", "--remove-orphans", "-d"]) diff --git a/kolombo/stop.py b/kolombo/stop.py new file mode 100644 index 0000000..d71191e --- /dev/null +++ b/kolombo/stop.py @@ -0,0 +1,63 @@ +from typer import Context, Typer + +from kolombo.configuration_files import generate_senders_compose_config +from kolombo.console import debug, error, step +from kolombo.util import ( + async_command, + execute_as_root, + needs_database, + run, + stop_kolombo_service, +) + +stop_cli = Typer(invoke_without_command=True) + + +@stop_cli.callback() +def stop_main(ctx: Context) -> None: + if ctx.invoked_subcommand is None: + debug("Stopping all Kolombo services") + # TODO + + +@stop_cli.command("receiver") +@execute_as_root +def stop_receiver() -> None: + step("Stopping receiver service") + stop_kolombo_service("receiver") + + +@stop_cli.command("auth") +@execute_as_root +def stop_auth() -> None: + step("Stopping auth service") + stop_kolombo_service("auth") + + +@stop_cli.command("nginx") +@execute_as_root +def stop_nginx() -> None: + step("Stopping nginx service") + stop_kolombo_service("nginx") + + +@stop_cli.command("senders") +@execute_as_root +@async_command +@needs_database +async def stop_senders() -> None: + from kolombo.models import Domain + + domains = {domain.actual for domain in await Domain.all_active()} + if len(domains) < 1: + error("No active domains to stop senders for") + exit(1) + + senders_compose_config = generate_senders_compose_config(domains) + project_name = "kolombo_senders" + file_path = "/etc/kolombo/senders-compose.yml" + with open(file_path, mode="w") as compose_file: + compose_file.write(senders_compose_config) + + compose_command = ["docker-compose", "-p", project_name, "-f", file_path, "down"] + run([*compose_command, "--remove-orphans"]) diff --git a/kolombo/user.py b/kolombo/user.py new file mode 100644 index 0000000..6cc621b --- /dev/null +++ b/kolombo/user.py @@ -0,0 +1,86 @@ +from base64 import b64encode +from typing import TYPE_CHECKING + +from cryptography.hazmat.primitives.kdf.scrypt import Scrypt +from typer import Argument, Typer, prompt + +from kolombo.console import error, finished, info, print_list, started, step, warning +from kolombo.util import async_command, needs_database + +if TYPE_CHECKING: + from kolombo.models import User + +user_cli = Typer() + + +@user_cli.command("list") +@async_command +@needs_database +async def list_users() -> None: + from kolombo.models import User + + active_users = [user.email for user in await User.all_active()] + info(f"Active users: {len(active_users)}") + if len(active_users) > 0: + print_list(active_users) + + +async def _save_user(email: str, password: str, domain: str) -> None: + from kolombo import conf + from kolombo.models import User + + kdf = Scrypt(conf.SALT, length=32, n=2 ** 16, r=8, p=1) + b64_password = b64encode(kdf.derive(password.encode("utf-8"))) + await User.objects.create(email=email, password=b64_password, domain=domain) + + +def update_virtual_files(active_users: list["User"]) -> None: + emails = [user.email for user in active_users] + addresses = "\n".join(f"{email} {email}" for email in emails) + with open("/etc/kolombo/virtual/addresses", mode="w") as addresses_file: + addresses_file.write(f"{addresses}\n") + + mailboxes = "\n".join(f"{email} {email}/" for email in emails) + with open("/etc/kolombo/virtual/mailbox", mode="w") as mailbox_file: + mailbox_file.write(f"{mailboxes}\n") + + +@user_cli.command("add") +@async_command +@needs_database +async def add_user( + email: str = Argument(..., help="Email for new user"), # noqa: B008 +) -> None: + from kolombo.models import Domain, User + + if "@" not in email: + error(f"Email '{email}' does not contain '@'!") + exit(1) + + domain = email.split("@", maxsplit=1)[1].strip() + if not domain: + error("Domain part MUST NOT be empty string!") + exit(1) + elif not await Domain.objects.filter(active=True, actual=domain).exists(): + error(f"Domain '{domain}' is not added (or inactive)!") + warning( + f"You can add it via [code]kolombo domain add {domain} mx.{domain}[/code]" + ) + exit(1) + elif await User.objects.filter(email=email).exists(): + error(f"User with email '{email}' already exists!") + exit(1) + + started(f"Adding [code]{email}[/] user") + + password = prompt(f"{email} password", hide_input=True, confirmation_prompt=True) + step("Saving to database") + await _save_user(email, password, domain) + + step("Updating virtual files (addresses and mailbox map)") + active_users = await User.all_active() + update_virtual_files(active_users) + + warning("Run command [code]kolombo run[/] to reload Kolombo") + + finished(f"User '{email}' added!") diff --git a/kolombo/util.py b/kolombo/util.py new file mode 100644 index 0000000..40488d4 --- /dev/null +++ b/kolombo/util.py @@ -0,0 +1,87 @@ +import sys +from asyncio import run as run_async +from functools import wraps +from importlib.resources import files +from os import execvp, getuid +from os.path import realpath + +# Security implications are considered +from subprocess import STDOUT, check_output # nosec: B404 +from typing import Any, Awaitable, Callable + +from docker import DockerClient # type: ignore[import] + +from kolombo.console import step, warning + + +def run(command: list[str], as_root: bool = False) -> str: + if as_root and getuid() != 0: + command.insert(0, "sudo") + + # We trust input here, it does not come from user + return check_output(command, text=True, stderr=STDOUT) # nosec: B603 + + +def async_command(func: Callable[..., Awaitable[None]]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + run_async(func(*args, **kwargs)) + + return wrapper + + +def needs_database( + func: Callable[..., Awaitable[None]] +) -> Callable[..., Awaitable[Any]]: + @wraps(func) + async def wrapper(*args: Any, **kwargs: Any) -> Any: + from kolombo.models import init_database + + await init_database() + await func(*args, **kwargs) + + return wrapper + + +def execute_as_root(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + if getuid() == 0: + func(*args, **kwargs) + return + + warning("Executing as root via sudo!") + script_path = realpath(sys.argv[0]) + arguments = sys.argv[1:] + execvp("sudo", ["sudo", script_path, *arguments]) # nosec: B606, B607 + + return wrapper + + +def build_kolombo_image(client: DockerClient, service: str) -> None: + kolombo_folder = files("kolombo") + step(f"Building [code]kolombo-{service}[/] image") + client.images.build( + tag=f"kolombo-{service}", + path=str(kolombo_folder), + pull=True, + nocache=True, + rm=True, + dockerfile=f"docker/Dockerfile.{service}", + ) + + +def up_kolombo_service(service: str) -> None: + project_name = "kolombo_services" + docker_folder = files("kolombo") / "docker" + file_path = str(docker_folder / "services" / "docker-compose.yml") + compose_command = ["docker-compose", "-p", project_name, "-f", file_path, "up"] + run([*compose_command, "--no-deps", "--force-recreate", "-d", f"kolombo-{service}"]) + + +def stop_kolombo_service(service: str) -> None: + project_name = "kolombo_services" + docker_folder = files("kolombo") / "docker" + file_path = str(docker_folder / "services" / "docker-compose.yml") + compose_command = ["docker-compose", "-p", project_name, "-f", file_path, "rm"] + run([*compose_command, "--stop", "--force", f"kolombo-{service}"]) diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..01ba5d2 --- /dev/null +++ b/renovate.json @@ -0,0 +1,24 @@ +{ + "extends": [ + "config:base" + ], + "packageRules": [ + { + "groupName": "all", + "matchManagers": ["pip_requirements"], + "matchFiles": ["requirements.txt", "./kolombo/docker/auth/requirements.txt"], + "schedule": ["before 5pm"] + }, + { + "groupName": "ci-all", + "matchManagers": ["pip_requirements"], + "matchFiles": [".ci/style-requirements.txt", ".ci/security-requirements.txt"], + "schedule": ["before 5pm"] + }, + { + "groupName": "docker-images", + "matchDatasources": ["docker"], + "schedule": ["before 5pm"] + } + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c8ad45 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# For CLI +typer==0.3.2 +shellingham==1.4.0 +rich==10.7.0 +docker==5.0.0 +# ORM +ormar==0.10.16 +databases==0.4.3 +aiosqlite==0.17.0 +# Passwords hashing +cryptography==3.4.8 diff --git a/scripts/install b/scripts/install index e29f7e3..a2ac624 100755 --- a/scripts/install +++ b/scripts/install @@ -2,16 +2,14 @@ set -e rm -rf venv -python3.7 -m virtualenv venv +python3.9 -m virtualenv venv -# Install package +# Install all requirements +./venv/bin/pip install --no-cache-dir -r requirements.txt +./venv/bin/pip install --no-cache-dir -r .ci/auth-requirements.txt +./venv/bin/pip install --no-cache-dir -r .ci/style-requirements.txt +./venv/bin/pip install --no-cache-dir -r .ci/security-requirements.txt +# Install radon for `metrics` script +./venv/bin/pip install --no-cache-dir radon +# Install kolombo package ./venv/bin/pip install --no-cache-dir . - -# Testing -./venv/bin/pip install --no-cache-dir pytest pytest-cov pytest-asyncio httpx -# Static analysis (without changing files) -./venv/bin/pip install --no-cache-dir mypy flake8 flake8-bugbear flake8-pie pep8-naming -# Linting (with changing files) -./venv/bin/pip install --no-cache-dir black isort -# Security checks and other metrics -./venv/bin/pip install --no-cache-dir bandit radon safety diff --git a/scripts/lint b/scripts/lint index 240d8c9..3f48087 100755 --- a/scripts/lint +++ b/scripts/lint @@ -3,12 +3,12 @@ set -e echo -e "Linting (black):" echo -./venv/bin/black --target-version=py37 kolombo tests setup.py +./venv/bin/black --target-version=py39 kolombo tests setup.py echo "----------" echo -e "Sorting Imports (isort):" echo -./venv/bin/isort --project=citylight kolombo tests setup.py +./venv/bin/isort --project=kolombo kolombo tests setup.py echo "----------" echo -e "Static Types Check After Linting (mypy):" diff --git a/scripts/upload b/scripts/upload index dad6d61..f7afb4f 100755 --- a/scripts/upload +++ b/scripts/upload @@ -10,7 +10,7 @@ find kolombo -type d -name __pycache__ -delete rm -rf dist build kolombo.egg-info -python3.7 setup.py sdist bdist_wheel +python3.9 setup.py sdist bdist_wheel twine upload dist/* rm -rf dist build kolombo.egg-info diff --git a/setup.cfg b/setup.cfg index 35a721a..98d17fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,17 @@ -[metadata] -license_file = LICENSE -long_description_content_type = "text/markdown" - -[flake8] -max-line-length = 88 -show-source = True -exclude = __pycache__ -statistics = True -select = C,E,F,W,B,PIE,N,B903 - -[mypy] -plugins = pydantic.mypy -disallow_untyped_defs = True -show_error_codes = True - -[pydantic-mypy] -init_typed = True -warn_required_dynamic_aliases = True -warn_untyped_fields = True - -[tool:isort] -profile = black -combine_as_imports = True -known_first_party = kolombo +[flake8] +max-line-length = 88 +show-source = True +exclude = __pycache__ +statistics = True +select = C,E,F,W,B,PIE,N,SIM,T,CCR,B903 +ignore = W503 +max-cognitive-complexity = 15 + +[mypy] +disallow_untyped_defs = True +show_error_codes = True + +[tool:isort] +profile = black +combine_as_imports = True +known_first_party = kolombo, kolombo_auth diff --git a/setup.py b/setup.py index bc70cfd..4ae2238 100644 --- a/setup.py +++ b/setup.py @@ -3,67 +3,56 @@ from setuptools import find_packages, setup +# Only Kolombo CLI requirements are installed by default +with open("requirements.txt") as reqs: + required = reqs.read().splitlines() + def get_version(package): - """ - Return package version as listed in `__version__` in `__init__.py`. - """ + """Return package version as listed in `__version__` in `__init__.py`""" with open(os.path.join(package, "__init__.py")) as f: return re.search("__version__ = ['\"]([^'\"]+)['\"]", f.read()).group(1) def get_long_description(): - """ - Return the README. - """ + """Return the README""" with open("README.md", encoding="utf8") as f: return f.read() setup( name="kolombo", - python_requires=">=3.7", + python_requires=">=3.9", version=get_version("kolombo"), - description="Kolombo - easy to manage mail server 💌", + description="Kolombo - CLI for easy mail server managing 💌", long_description=get_long_description(), long_description_content_type="text/markdown", - url="https://github/HarrySky/kolombo", + url="https://github.com/HarrySky/kolombo", + project_urls={ + "Documentation": "https://docs.neigor.me/kolombo", + "Changelog": "https://github/HarrySky/kolombo/blob/main/README.md", + "Source": "https://github.com/HarrySky/kolombo", + "Tracker": "https://github.com/HarrySky/kolombo/issues", + }, license="Apache License 2.0", author="Igor Nehoroshev", author_email="hi@neigor.me", maintainer="Igor Nehoroshev", maintainer_email="hi@neigor.me", - packages=find_packages(), + packages=find_packages(exclude=["tests"]), # Use MANIFEST.in for data files include_package_data=True, zip_safe=False, - install_requires=[ - # Fast async event loop - "uvloop==0.14.0", - # For CLI - "typer==0.3.2", - "rich==9.3.0", - # ORM - "ormar==0.7.1", - "databases==0.4.1", - "aiosqlite==0.16.0", - # Passwords hashing - "cryptography==3.2.1", - # For API - "fastapi==0.62.0", - "uvicorn==0.12.3", - # Since 0.12.0 uvicorn does not install httptools (and uvloop) - "httptools==0.1.*", - ], - entry_points={"console_scripts": ["kolombo = kolombo:cli"]}, + install_requires=required, + entry_points={"console_scripts": ["kolombo = kolombo.bin:kolombo_cli"]}, classifiers=[ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: System Administrators", + "Intended Audience :: Information Technology", "License :: OSI Approved :: Apache Software License", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Topic :: Communications :: Email",