From d7f259bb3a536daa83209c5b89dd5e42a6127e4b Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 27 May 2024 20:43:31 +0100 Subject: [PATCH 01/14] Add note about POP/SMTP testing when troubleshooting --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2a00b0d..975ccb1 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,7 @@ It is often helpful to be able to view the raw connection details when debugging This can be achieved using `telnet`, [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/) or similar. For example, to test the Office 365 IMAP server from the [example configuration](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config), first open a connection using `telnet 127.0.0.1 1993`, and then send a login command: `a1 login e@mail.com password`, replacing `e@mail.com` with your email address, and `password` with any value you like during testing (see above for why the password is irrelevant). If you have already authorised your account with the proxy you should see a response starting with `a1 OK`; if not, this command should trigger a notification from the proxy about authorising your account. +Note that POP and SMTP are different protocols, and while they can be tested in this way, they require different commands to be sent – see [this issue comment](https://github.com/simonrob/email-oauth2-proxy/issues/251#issuecomment-2133976839) for further details. If you are using a [secure local connection](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) the interaction with the remote email server is the same as above, but you will need to use a local debugging tool that supports encryption. The easiest approach here is to use [OpenSSL](https://www.openssl.org/): `openssl s_client -crlf -connect 127.0.0.1:1993`. From 3468e1a2ae9e0d0d609985673a2e05c8faa66283 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Tue, 4 Jun 2024 08:56:21 +0100 Subject: [PATCH 02/14] Fix minor documentation typo. Closes #254 --- emailproxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index 65c744f..0f62294 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-05-25' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-06-04' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -2907,7 +2907,7 @@ def create_authorisation_window(self, request): # pywebview 3.6+ moved window events to a separate namespace in a non-backwards-compatible way # noinspection PyDeprecation pywebview_version = pkg_resources.parse_version(pkg_resources.get_distribution('pywebview').version) - # the version zero check is due to a bug in the Ubuntu 22.04 python-pywebview package - see GitHub #242 + # the version zero check is due to a bug in the Ubuntu 24.04 python-pywebview package - see GitHub #242 # noinspection PyDeprecation if pkg_resources.parse_version('0') < pywebview_version < pkg_resources.parse_version('3.6'): # noinspection PyUnresolvedReferences From a5e5415ac73f65f81bc3c8d9282cd1bf6037eb50 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Wed, 5 Jun 2024 08:56:37 +0100 Subject: [PATCH 03/14] Explain alternative scope values for O365 via regional providers --- emailproxy.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/emailproxy.config b/emailproxy.config index e4d55a8..4740d3a 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -109,6 +109,13 @@ documentation = Accounts are specified using your email address as the section h in the example below) in order to allow the proxy to refresh its access token on your behalf. The proxy will still work if this parameter is not included, but you will need to re-authenticate extremely often (about once per hour). + - The example Office 365 configuration entries below use an OAuth 2.0 scope that clearly specifies IMAP, POP and + SMTP permission. If you do not require one or more of these protocols, you may remove the relevant values to ensure + the access tokens obtained on your behalf are as precisely-targeted as possible. Conversely, it is also possible to + replace these specific scopes with the more generic `https://outlook.office365.com/.default`. Switching to a broader + scope value may also be needed if you are using a version of O365 delivered by a regional provider (e.g., 21Vianet). + See: https://github.com/simonrob/email-oauth2-proxy/issues/255 for more details and discussion. + - By default, new Entra (Azure AD) clients are accessible only within your own tenant. If you are registering a new client to use with the proxy (and do not want to make it available outside your own organisation) you will need to replace `common` with your tenant ID in the Office 365 `permission_url` and `token_url` values below. Alternatively, From 48773b086b5b9df70439ba251946dfa0a86ce5c4 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Fri, 28 Jun 2024 09:00:15 +0100 Subject: [PATCH 04/14] Minor lint fixes; signpost to configuration format documentation --- emailproxy.config | 6 +++++- emailproxy.py | 53 ++++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/emailproxy.config b/emailproxy.config index 4740d3a..bf65fdd 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -3,7 +3,11 @@ documentation = This is a sample Email OAuth 2.0 Proxy configuration file. Confi [Server setup] and [Account setup] sections below. You may delete any servers or accounts that you do not intend to use. Documentation is provided inline, with example setups for Gmail and Office 365 (though you will need to enter your own desktop app API client credentials in the accounts section). Use the `Reload configuration file` menu - option or send a SIGHUP signal (or restart the proxy) to apply any changes. + option or send a SIGHUP signal (or quit the proxy before editing, then restart) to apply any changes. +format = This file's format is documented at docs.python.org/library/configparser#supported-ini-file-structure. Values + that span multiple lines should be indented deeper than the first line of their key (as in this comment). Quoting + of values is not required. Documentation sections can be removed if needed (though it is advisable to leave these + in place for reference) - thw only required sections are the individual server and account items of your setup. warning = Do not commit changes to this file into a public repository (e.g., GitHub, etc). While the proxy encrypts the OAuth 2.0 tokens it obtains and saves on your behalf, it cannot protect these against offline brute-force attacks. diff --git a/emailproxy.py b/emailproxy.py index 0f62294..eb53a06 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-06-04' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-06-28' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -319,7 +319,7 @@ def format_host_port(address): host, port, *_ = address with contextlib.suppress(ValueError): ip = ipaddress.ip_address(host) - host = '[%s]' % host if type(ip) is ipaddress.IPv6Address else host + host = '[%s]' % host if isinstance(ip, ipaddress.IPv6Address) else host return '%s:%d' % (host, port) @staticmethod @@ -356,13 +356,13 @@ def _get_boto3_client(store_id): except ModuleNotFoundError: Log.error('Unable to load AWS SDK - please install the `boto3` module: `python -m pip install boto3`') return None, None - else: - # allow a profile to be chosen by prefixing the store_id - the separator used (`||`) will not be in an ARN - # or secret name (see: https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html) - split_id = store_id.split('||', maxsplit=1) - if '||' in store_id: - return split_id[1], boto3.session.Session(profile_name=split_id[0]).client('secretsmanager') - return store_id, boto3.client(service_name='secretsmanager') + + # allow a profile to be chosen by prefixing the store_id - the separator used (`||`) will not be in an ARN + # or secret name (see: https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html) + split_id = store_id.split('||', maxsplit=1) + if '||' in store_id: + return split_id[1], boto3.session.Session(profile_name=split_id[0]).client('secretsmanager') + return store_id, boto3.client(service_name='secretsmanager') @staticmethod def _create_secret(aws_client, store_id): @@ -815,7 +815,7 @@ def get_oauth2_credentials(username, password, reload_remote_accounts=True): headers={ 'x5t#S256': base64.urlsafe_b64encode(jwt_certificate_fingerprint).decode('utf-8') }) - except FileNotFoundError: + except (FileNotFoundError, OSError): # catch OSError due to GitHub issue 257 (quoted paths) return (False, 'Unable to create credentials assertion for account %s - please check that the ' '`jwt_certificate_path` and `jwt_key_path` values are correct' % username) @@ -903,7 +903,7 @@ def get_oauth2_credentials(username, password, reload_remote_accounts=True): except OAuth2Helper.TokenRefreshError as e: # always clear access tokens - can easily request another via the refresh token (with no user interaction) - has_access_token = True if config.get(username, 'access_token', fallback=None) else False + has_access_token = bool(config.get(username, 'access_token', fallback=None)) config.remove_option(username, 'access_token') config.remove_option(username, 'access_token_expiry') @@ -977,6 +977,7 @@ def start_redirection_receiver_server(token_request): Log.format_host_port((parsed_uri.hostname, parsed_port))) class LoggingWSGIRequestHandler(wsgiref.simple_server.WSGIRequestHandler): + # pylint: disable=arguments-differ def log_message(self, _format_string, *args): Log.debug('Local server auth mode (%s): received authentication response' % Log.format_host_port( (parsed_uri.hostname, parsed_port)), *args) @@ -1141,22 +1142,22 @@ def get_service_account_authorisation_token(key_type, key_path_or_contents, oaut import requests import google.oauth2.service_account import google.auth.transport.requests - except ModuleNotFoundError: - raise Exception('Unable to load Google Auth SDK - please install the `requests` and `google-auth` modules: ' - '`python -m pip install requests google-auth`') + except ModuleNotFoundError as e: + raise ModuleNotFoundError('Unable to load Google Auth SDK - please install the `requests` and ' + '`google-auth` modules: `python -m pip install requests google-auth`') from e if key_type == 'file': try: - with open(key_path_or_contents) as key_file: + with open(key_path_or_contents, mode='r', encoding='utf-8') as key_file: service_account = json.load(key_file) except IOError as e: - raise FileNotFoundError('Unable to open service account key file %s for account %s', + raise FileNotFoundError('Unable to open service account key file %s for account %s' % (key_path_or_contents, username)) from e elif key_type == 'key': service_account = json.loads(key_path_or_contents) else: - raise Exception('Service account key type not specified for account %s - `client_id` must be set to ' - '`file` or `key`' % username) + raise KeyError('Service account key type not specified for account %s - `client_id` must be set to ' + '`file` or `key`' % username) credentials = google.oauth2.service_account.Credentials.from_service_account_info(service_account) credentials = credentials.with_scopes(oauth2_scope.split(' ')) @@ -2003,7 +2004,7 @@ def process_data(self, byte_data): if not re.search(' SASL-IR', updated_response, re.IGNORECASE): updated_response = updated_response.replace(' AUTH=PLAIN', ' AUTH=PLAIN SASL-IR') updated_response = re.sub(' LOGINDISABLED', '', updated_response, count=1, flags=re.IGNORECASE) - byte_data = (b'%s\r\n' % updated_response.encode('utf-8')) + byte_data = b'%s\r\n' % updated_response.encode('utf-8') super().process_data(byte_data) @@ -2776,9 +2777,9 @@ def create_config_menu(self): if len(self.proxies) <= 0: # note that we don't actually allow no servers when loading the config, so no need to generate a menu return items # (avoids creating and then immediately regenerating the menu when servers are loaded) - else: - for server_type in ['IMAP', 'POP', 'SMTP']: - items.extend(App.get_config_menu_servers(self.proxies, server_type)) + + for server_type in ['IMAP', 'POP', 'SMTP']: + items.extend(App.get_config_menu_servers(self.proxies, server_type)) config_accounts = AppConfig.accounts() items.append(pystray.MenuItem('Accounts (+ last authenticated activity):', None, enabled=False)) @@ -3104,10 +3105,10 @@ def macos_launchctl(command): output = subprocess.check_output(['/bin/launchctl', command, proxy_command], stderr=subprocess.STDOUT) except subprocess.CalledProcessError: return False - else: - if output and command != 'list': - return False # load/unload gives no output unless unsuccessful (return code is always 0 regardless) - return True + + if output and command != 'list': + return False # load/unload gives no output unless unsuccessful (return code is always 0 regardless) + return True @staticmethod def started_at_login(_): From 23d3b4ac0836912f3a6737cfa1e53dddbb20c776 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Fri, 5 Jul 2024 22:44:05 +0100 Subject: [PATCH 05/14] Update Outlook SMTP server; alternative example O365 scope values --- emailproxy.config | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/emailproxy.config b/emailproxy.config index bf65fdd..e5d9528 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -59,7 +59,7 @@ server_port = 995 local_address = 127.0.0.1 [SMTP-1587] -server_address = smtp.office365.com +server_address = smtp-mail.outlook.com server_port = 587 server_starttls = True local_address = 127.0.0.1 @@ -199,6 +199,14 @@ redirect_uri = http://localhost client_id = *** your client id here *** client_secret = *** your client secret here *** +[your.email@outlook.com] +permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize +token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token +oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access +redirect_uri = http://localhost +client_id = *** your client id here *** +client_secret = *** your client secret here *** + [your.email@gmail.com] permission_url = https://accounts.google.com/o/oauth2/auth token_url = https://oauth2.googleapis.com/token From 4e1dfd80fbf92bd48ff804bac163efa4b8af9d74 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 8 Jul 2024 17:25:11 +0100 Subject: [PATCH 06/14] Minor lint improvement --- emailproxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index eb53a06..97487e8 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-06-28' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-07-08' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -977,7 +977,7 @@ def start_redirection_receiver_server(token_request): Log.format_host_port((parsed_uri.hostname, parsed_port))) class LoggingWSGIRequestHandler(wsgiref.simple_server.WSGIRequestHandler): - # pylint: disable=arguments-differ + # pylint: disable-next=arguments-differ def log_message(self, _format_string, *args): Log.debug('Local server auth mode (%s): received authentication response' % Log.format_host_port( (parsed_uri.hostname, parsed_port)), *args) From 50d07c904b2de16349d4cc267321092cff9467da Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 29 Jul 2024 20:16:45 +0100 Subject: [PATCH 07/14] Switch to `packaging` and `importlib_metadata` (fixes Nuitka build) --- emailproxy.py | 41 ++++++++++++++++++++++------------------- requirements-gui.txt | 3 ++- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index 97487e8..8c0119e 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-07-08' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-07-29' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -99,14 +99,22 @@ class Icon: except ImportError as gui_requirement_import_error: MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) +try: + # pylint: disable-next=ungrouped-imports + import importlib.metadata as importlib_metadata # get package version numbers - available in stdlib from python 3.8 +except ImportError: try: - # noinspection PyDeprecation,PyUnresolvedReferences - import pkg_resources # from setuptools - to change to importlib.metadata and packaging.version once min. is 3.8 + # noinspection PyUnresolvedReferences + import importlib_metadata except ImportError as gui_requirement_import_error: MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) +try: + # noinspection PyUnresolvedReferences + import packaging.version # parse package version numbers - used to work around various GUI-only package issues +except ImportError as gui_requirement_import_error: + MISSING_GUI_REQUIREMENTS.append(gui_requirement_import_error) + # for macOS-specific functionality if sys.platform == 'darwin': try: @@ -2696,13 +2704,12 @@ def macos_nsworkspace_notification_listener_(self, notification): # noinspection PyDeprecation def create_icon(self): # fix pystray <= 0.19.4 incompatibility with PIL 10.0.0+; resolved in 0.19.5 and later via pystray PR #147 - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - pystray_version = pkg_resources.get_distribution('pystray').version - pillow_version = pkg_resources.get_distribution('pillow').version - if pkg_resources.parse_version(pystray_version) <= pkg_resources.parse_version('0.19.4') and \ - pkg_resources.parse_version(pillow_version) >= pkg_resources.parse_version('10.0.0'): - Image.ANTIALIAS = Image.LANCZOS if hasattr(Image, 'LANCZOS') else Image.Resampling.LANCZOS + pystray_version = packaging.version.Version(importlib_metadata.version('pystray')) + pillow_version = packaging.version.Version(importlib_metadata.version('pillow')) + if pystray_version <= packaging.version.Version('0.19.4') and \ + pillow_version >= packaging.version.Version('10.0.0'): + Image.ANTIALIAS = Image.LANCZOS if hasattr(Image, 'LANCZOS') else Image.Resampling.LANCZOS + icon_class = RetinaIcon if sys.platform == 'darwin' else pystray.Icon return icon_class(APP_NAME, App.get_image(), APP_NAME, menu=pystray.Menu( pystray.MenuItem('Servers and accounts', pystray.Menu(self.create_config_menu)), @@ -2763,9 +2770,7 @@ def get_icon_size(text, font_size): font = ImageFont.truetype(io.BytesIO(zlib.decompress(base64.b64decode(APP_ICON))), size=font_size) # pillow's getsize method was deprecated in 9.2.0 (see docs for PIL.ImageFont.ImageFont.getsize) - # noinspection PyDeprecation - if pkg_resources.parse_version( - pkg_resources.get_distribution('pillow').version) < pkg_resources.parse_version('9.2.0'): + if packaging.version.Version(importlib_metadata.version('pillow')) < packaging.version.Version('9.2.0'): font_width, font_height = font.getsize(text) return font, font_width, font_height @@ -2906,11 +2911,9 @@ def create_authorisation_window(self, request): setattr(authorisation_window, 'get_title', lambda window: window.title) # add missing get_title method # pywebview 3.6+ moved window events to a separate namespace in a non-backwards-compatible way - # noinspection PyDeprecation - pywebview_version = pkg_resources.parse_version(pkg_resources.get_distribution('pywebview').version) + pywebview_version = packaging.version.Version(importlib_metadata.version('pywebview')) # the version zero check is due to a bug in the Ubuntu 24.04 python-pywebview package - see GitHub #242 - # noinspection PyDeprecation - if pkg_resources.parse_version('0') < pywebview_version < pkg_resources.parse_version('3.6'): + if packaging.version.Version('0') < pywebview_version < packaging.version.Version('3.6'): # noinspection PyUnresolvedReferences authorisation_window.loaded += self.authorisation_window_loaded else: diff --git a/requirements-gui.txt b/requirements-gui.txt index 0726fce..dc48b84 100644 --- a/requirements-gui.txt +++ b/requirements-gui.txt @@ -1,8 +1,9 @@ # the standard way to install the proxy and dependencies is `python -m pip install emailproxy` (i.e., direct from PyPI) # to install requirements directly, use: `python -m pip install -r requirements-core.txt -r requirements-gui.txt` +importlib_metadata; python_version < '3.8' # to get dependency versions (available in stdlib from 3.8 onwards) +packaging # for dependency version comparisons pillow # to create the menu bar icon image from a TTF icon -setuptools # for pkg_resources (checking dependency versions) timeago # for displaying the last authenticated activity hint # force pystray version with dummy GUI fix for headless deployments (https://github.com/moses-palmer/pystray/issues/118) From 785f6152428c58aed46b9d0d519b7e59f0ae5c43 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 29 Jul 2024 20:19:29 +0100 Subject: [PATCH 08/14] Switch to a minimum of Python 3.7 due to dependency requirements --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 975ccb1..9dd42b7 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ If your network requires connections to use an existing proxy, you can instruct After installing its requirements, the proxy script can be packaged as a single self-contained executable using [pyinstaller](https://pyinstaller.org/) if desired: `pyinstaller --onefile emailproxy.py`. If you are using the GUI version of the proxy, you may need to add `--hidden-import timeago.locales.en_short` until [this `timeago` issue](https://github.com/hustcc/timeago/issues/40) is resolved. -Python 3.6 or later is required to run the proxy. +Python 3.7 or later is required to run the proxy. The [python2 branch](https://github.com/simonrob/email-oauth2-proxy/tree/python2) provides minimal compatibility with python 2.7, but with a limited feature set, and no ongoing maintenance. See [issue 38](https://github.com/simonrob/email-oauth2-proxy/issues/38) for further discussion. diff --git a/pyproject.toml b/pyproject.toml index 4e80d08..20362eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ description = "Transparently add OAuth 2.0 support to IMAP/POP/SMTP clients that don't support this authentication method." readme = { file = "README.md", content-type = "text/markdown" } license = { text = "Apache License 2.0" } -requires-python = ">=3.6" +requires-python = ">=3.7" classifiers = [ "Operating System :: OS Independent", "Operating System :: MacOS", From fd0491b8d11b42de04f0ef5e4387be48c8c4e0c2 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 5 Aug 2024 09:00:13 +0100 Subject: [PATCH 09/14] Update readme link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dd42b7..7399021 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ It will notify you if this is the case. As part of the proxy setup process you need to provide an OAuth 2.0 `client_id` and `client_secret` to allow it to authenticate with email servers on your behalf. If you have an existing client ID and secret for a desktop app, you can use these directly in the proxy. -If this is not possible, you can also reuse the client ID and secret from any email client that supports IMAP/POP/SMTP OAuth 2.0 authentication with the email server you would like to connect to (such as [the](https://github.com/mozilla/releases-comm-central/blob/812b7c9068ca5cac0580b0ddbea8e34c141cd441/mailnews/base/src/OAuth2Providers.jsm) [various](https://github.com/Foundry376/Mailspring/blob/master/app/internal_packages/onboarding/lib/onboarding-constants.ts) [open](https://gitlab.gnome.org/GNOME/evolution-data-server/-/blob/master/CMakeLists.txt) [source](https://gitlab.gnome.org/GNOME/gnome-online-accounts/-/blob/master/meson_options.txt) [clients](https://github.com/M66B/FairEmail/blob/master/app/src/main/res/xml/providers.xml) with OAuth 2.0 support), but please do this with care and restraint as access through reused tokens will be associated with the token owner rather than your own client. +If this is not possible, you can also reuse the client ID and secret from any email client that supports IMAP/POP/SMTP OAuth 2.0 authentication with the email server you would like to connect to (such as [the](https://github.com/mozilla/releases-comm-central/blob/812b7c9068ca5cac0580b0ddbea8e34c141cd441/mailnews/base/src/OAuth2Providers.jsm) [many](https://github.com/mozilla/releases-comm-central/blob/master/mailnews/base/src/OAuth2Providers.sys.mjs) [existing](https://github.com/Foundry376/Mailspring/blob/master/app/internal_packages/onboarding/lib/onboarding-constants.ts) [open](https://gitlab.gnome.org/GNOME/evolution-data-server/-/blob/master/CMakeLists.txt) [source](https://gitlab.gnome.org/GNOME/gnome-online-accounts/-/blob/master/meson_options.txt) [clients](https://github.com/M66B/FairEmail/blob/master/app/src/main/res/xml/providers.xml) with OAuth 2.0 support), but please do this with care and restraint as access through reused tokens will be associated with the token owner rather than your own client. If you do not have access to credentials for an existing client you will need to register your own. The process to do this is different for each provider, but the registration guides for several common ones are linked here. From 32865aefb2cf9be2d8e28171d8f89c016592958f Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 5 Aug 2024 20:36:18 +0100 Subject: [PATCH 10/14] Switch to newer O365 scope values --- emailproxy.config | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/emailproxy.config b/emailproxy.config index e5d9528..7bc0e6a 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -175,9 +175,9 @@ documentation = Accounts are specified using your email address as the section h Advanced account configuration: - For most configurations the default `redirect_uri` value of `http://localhost` is correct, unless you have - explicitly set your OAuth 2.0 client to use a different address here (which you will need to manually redirect to - the proxy). Please also note that this address is `http` and not `https` (which is not supported unless you provide - a secure connection yourself outside of the proxy using an external tool). + explicitly set the OAuth 2.0 client configuration with your provider to use a different address for this purpose + (e.g., redirecting via an external domain). If this is the case, you will need to manually redirect this to the + proxy. Please note that in most cases the address is indeed `http://localhost`, not `https`. - When using the `--local-server-auth` option you will need to either run the proxy with additional privileges to use the implicit default port 80 (e.g., via sudo) or specify a different port (and/or a different host if needed) - @@ -194,14 +194,6 @@ documentation = Accounts are specified using your email address as the section h [your.office365.address@example.com] permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token -oauth2_scope = https://outlook.office365.com/IMAP.AccessAsUser.All https://outlook.office365.com/POP.AccessAsUser.All https://outlook.office365.com/SMTP.Send offline_access -redirect_uri = http://localhost -client_id = *** your client id here *** -client_secret = *** your client secret here *** - -[your.email@outlook.com] -permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize -token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access redirect_uri = http://localhost client_id = *** your client id here *** From 52e729913a53dd05257ee6056670a9aa20906edf Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 12 Aug 2024 21:29:34 +0100 Subject: [PATCH 11/14] Link to O365 SMTP AUTH removal timeframe --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7399021..0eaabb8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The proxy works in the background with a menu bar/taskbar helper or as a headles ### Example use-cases - You need to use an Office 365 email account, but don't get on with Outlook. -The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437). +The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437) ([September 2025 for SMTP](https://techcommunity.microsoft.com/t5/exchange-team-blog/exchange-online-to-retire-basic-auth-for-client-submission-smtp/ba-p/4114750)). - You used to use Gmail via IMAP/POP/SMTP with your raw account credentials (i.e., your real password), but cannot do this now that Google has disabled this method, and don't want to use an [App Password](https://support.google.com/accounts/answer/185833) (or cannot enable this option). - You have an account already set up in an email client, and you need to switch it to OAuth 2.0 authentication. You can edit the server details, but the client forces you to delete and re-add the account to enable OAuth 2.0, and you don't want to do this. From 075752e0450f3619ff6aa736c113e125efaca0b1 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Sat, 7 Sep 2024 20:03:29 +0100 Subject: [PATCH 12/14] Add support for falling back to login password as secret See #271 --- emailproxy.config | 10 ++++++++++ emailproxy.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/emailproxy.config b/emailproxy.config index 7bc0e6a..e4a5ca4 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -292,6 +292,15 @@ documentation = The parameters below control advanced options for the proxy. In using catch-all accounts or the proxy's `--cache-store` parameter you must manually remove unencrypted secrets from the local configuration file after the encrypted secret has been created (i.e., this will not be automatic). + - use_login_password_as_client_credentials_secret (default = False): When using the O365 client credentials grant + (CCG) flow, rather than encrypting the client secret (see above), the proxy can be instructed to use the given + IMAP/POP/SMTP login password as the client secret. This approach removes the risk of storing the unencrypted client + secret in the proxy's configuration file, and also means there is no risk of unauthorised account access when using + the O365 CCG flow in conjunction with the proxy's catch-all mode (see below). To enable this option, set + `use_login_password_as_client_credentials_secret` to True. Note that if a `client_secret` value is present in your + account's configuration entry, that value will be used instead of the given IMAP/POP/SMTP login password even if + this option is enabled. To avoid this, remove the entire `client_secret` line from the configuration entry. + - allow_catch_all_accounts (default = False): The default behaviour of the proxy is to require a full separate configuration file entry for each account. However, when proxying multiple accounts from the same domain it can be cumbersome to have to create multiple near-identical configuration profiles. To simplify this the proxy supports @@ -308,4 +317,5 @@ documentation = The parameters below control advanced options for the proxy. In [emailproxy] delete_account_token_on_password_error = True encrypt_client_secret_on_first_use = False +use_login_password_as_client_credentials_secret = False allow_catch_all_accounts = False diff --git a/emailproxy.py b/emailproxy.py index 8c0119e..738058c 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-07-29' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-09-07' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -1125,6 +1125,11 @@ def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_s params['client_assertion_type'] = 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer' params['client_assertion'] = jwt_client_assertion + # CCG flow can fall back to the login password as the client secret (see GitHub #271 discussion) + elif oauth2_flow == 'client_credentials' and AppConfig.get_global( + 'use_login_password_as_client_credentials_secret', fallback=False): + params['client_secret'] = password + if oauth2_flow != 'authorization_code': del params['code'] # CCG/ROPCG flows have no code, but we need the scope and (for ROPCG) username+password params['scope'] = oauth2_scope From 963ec4b86f781d205e834347e8e22b1145dea386 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Tue, 10 Sep 2024 17:28:38 +0100 Subject: [PATCH 13/14] Make redirect_uri optional for flows that don't use it --- emailproxy.config | 10 +++------- emailproxy.py | 13 +++++++------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/emailproxy.config b/emailproxy.config index e4a5ca4..18b9aa6 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -83,9 +83,9 @@ local_address = 127.0.0.1 [Account setup] documentation = Accounts are specified using your email address as the section heading (e.g., [your.email@gmail.com], etc, below). Account usernames (i.e., email addresses) must be unique - only one entry per account is permitted. - Each account section must provide values for `permission_url`, `token_url`, `oauth2_scope` and `redirect_uri`. If - you are adding an account for a service other than the examples shown below then the provider's documentation should - provide these details. + Each account section must provide values for at least `token_url`, `oauth2_scope` and `client_id`. Depending on the + OAuth 2.0 flow you are using, other values may also be required (see examples below). If you are adding an account + for a service other than the examples shown below then the provider's documentation should provide these details. You will also need to add your own `client_id` and `client_secret` values as indicated below. These can either be reused from an existing source (such as another email client that supports OAuth 2.0), or you can register and use @@ -228,7 +228,6 @@ documentation = *** note: this is an advanced O365 account example; in most case token_url = https://login.microsoftonline.com/*** your tenant id here ***/oauth2/v2.0/token oauth2_scope = https://outlook.office365.com/.default oauth2_flow = client_credentials -redirect_uri = http://localhost client_id = *** your client id here *** client_secret = *** your client secret here *** @@ -237,7 +236,6 @@ documentation = *** note: this is an advanced O365 account example; in most case token_url = https://login.microsoftonline.com/*** your tenant id here ***/oauth2/v2.0/token oauth2_scope = https://outlook.office365.com/IMAP.AccessAsUser.All https://outlook.office365.com/POP.AccessAsUser.All https://outlook.office365.com/SMTP.Send offline_access oauth2_flow = password -redirect_uri = http://localhost client_id = *** your client id here *** client_secret = *** your client secret here *** @@ -246,7 +244,6 @@ documentation = *** note: this is an advanced Google account example; in most ca token_url = https://oauth2.googleapis.com/token oauth2_scope = https://mail.google.com/ oauth2_flow = service_account -redirect_uri = http://localhost client_id = file client_secret = *** your /path/to/service-account-key.json here *** @@ -255,7 +252,6 @@ documentation = *** note: this is an advanced Google account example; in most ca token_url = https://oauth2.googleapis.com/token oauth2_scope = https://mail.google.com/ oauth2_flow = service_account -redirect_uri = http://localhost client_id = key client_secret = *** your pasted service account JSON key file contents here, making sure to indent all lines by at least one space *** diff --git a/emailproxy.py b/emailproxy.py index 738058c..0b64494 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-09-07' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-09-10' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -725,13 +725,12 @@ def get_oauth2_credentials(username, password, reload_remote_accounts=True): jwt_certificate_path = AppConfig.get_option_with_catch_all_fallback(config, username, 'jwt_certificate_path') jwt_key_path = AppConfig.get_option_with_catch_all_fallback(config, username, 'jwt_key_path') - # note that we don't require permission_url here because it is not needed for the client credentials grant flow, - # and likewise for client_secret here because it can be optional for Office 365 configurations - if not (token_url and oauth2_scope and redirect_uri and client_id): + # because the proxy supports a wide range of OAuth 2.0 flows, in addition to the token_url we only mandate the + # core parameters that are required by all methods: oauth2_scope and client_id + if not (token_url and oauth2_scope and client_id): Log.error('Proxy config file entry incomplete for account', username, '- aborting login') return (False, '%s: Incomplete config file entry found for account %s - please make sure all required ' - 'fields are added (permission_url, token_url, oauth2_scope, redirect_uri, client_id ' - 'and client_secret)' % (APP_NAME, username)) + 'fields are added (at least token_url, oauth2_scope and client_id)' % (APP_NAME, username)) # while not technically forbidden (RFC 6749, A.1 and A.2), it is highly unlikely the example value is valid example_client_value = '*** your' @@ -1136,6 +1135,8 @@ def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_s if oauth2_flow == 'password': params['username'] = username params['password'] = password + if not redirect_uri: + del params['redirect_uri'] # redirect_uri is not typically required in non-code flows; remove if empty try: response = urllib.request.urlopen( urllib.request.Request(token_url, data=urllib.parse.urlencode(params).encode('utf-8'), From 561065047d989e79520770ee3082e32f908cc4c0 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Thu, 12 Sep 2024 09:15:12 +0100 Subject: [PATCH 14/14] Minor lint improvement --- emailproxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emailproxy.py b/emailproxy.py index 8c0119e..0a61905 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-07-29' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-09-12' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -2771,6 +2771,7 @@ def get_icon_size(text, font_size): # pillow's getsize method was deprecated in 9.2.0 (see docs for PIL.ImageFont.ImageFont.getsize) if packaging.version.Version(importlib_metadata.version('pillow')) < packaging.version.Version('9.2.0'): + # noinspection PyUnresolvedReferences font_width, font_height = font.getsize(text) return font, font_width, font_height