Update set_profile_displayname to use UserID type

This is required as the function was updated to take this explicit type,
rather than just the localpart, in matrix-org/synapse#15458 as part of
version 1.83.

This is stacked atop #10 to ensure everything is updated.

.gitignore

@@ -0,0 +1,204 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+*.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+

README.md

@@ -1,32 +1,47 @@
-# HTTP JSON REST Authenticator module for synapse
-This synapse authentication module (password provider) allows you to query identity data in existing webapps, like:
+# Synapse REST Password provider
+- [Overview](#overview)
+- [Install](#install)
+- [Configure](#configure)
+- [Integrate](#integrate)
+- [Support](#support)
+
+## Overview
+This synapse's password provider allows you to validate a password for a given username and return a user profile using an existing backend, like:
+
 - Forums (phpBB, Discourse, etc.)
 - Custom Identity stores (Keycloak, ...)
 - CRMs (Wordpress, ...)
 - self-hosted clouds (Nextcloud, ownCloud, ...)
 
-It is mainly used with [mxisd](https://github.com/kamax-io/mxisd), the Federated Matrix Identity Server, to provide
+It is mainly used with [ma1sd](https://github.com/ma1uta/ma1sd), the Federated Matrix Identity Server, to provide
 missing features and offer a fully integrated solution (directory, authentication, search).
 
-## Install
-Copy in whichever directory python2.x can pick it up as a module.  
+**NOTE:** This module doesn't provide direct integration with any backend. If you do not use mxisd, you will need to write
+your own backend, following the [Integration section](#integrate). This module simply translate an anthentication result
+and profile information into actionables in synapse, and adapt your user profile with what is given.
+
+## Install
+Copy in whichever directory python can pick it up as a module.
 
-If you installed synapse using the Matrix debian repos:
 ```
-git clone https://github.com/maxidor/matrix-synapse-rest-auth.git
-cd matrix-synapse-rest-auth
-sudo cp rest_auth_provider.py /usr/lib/python2.7/dist-packages/
+sudo pip install git+https://github.com/ma1uta/matrix-synapse-rest-password-provider
 ```
 
+If the command fail, double check that the python version still matches. If not, please let us know by opening an issue.
+
 ## Configure
-Add or amend the `password_providers` entry like so:
-```
-password_providers:
+Add or amend the `modules` entry like so:
+```yaml
+modules:
   - module: "rest_auth_provider.RestAuthProvider"
     config:
       endpoint: "http://change.me.example.com:12345"
 ```
-Set `endpoint` to the appropriate value.
+Set `endpoint` to the value documented with the endpoint provider.
+
+**NOTE:** This requires Synapse 1.46 or later! If you migrate from the legacy `password_providers`, make sure
+to remove the old `RestAuthProvider` entry. If the `password_providers` list is empty, you can also remove it completely or
+comment it out.
 
 ## Use
 1. Install, configure, restart synapse
@@ -34,17 +49,18 @@ Set `endpoint` to the appropriate value.
 
 ## Next steps
 ### Lowercase username enforcement
+**NOTE**: This is no longer relevant as synapse natively enforces lowercase.
+
 To avoid creating users accounts with uppercase characters in their usernames and running into known
 issues regarding case sensitivity in synapse, attempting to login with such username will fail.
 
 It is highly recommended to keep this feature enable, but in case you would like to disable it:
-```
-[...]
+```yaml
     config:
       policy:
         registration:
           username:
-            enforceLowercase: False
+            enforceLowercase: false
 ```
 
 ### Profile auto-fill
@@ -53,27 +69,36 @@ If none is given, the display name is not set.
 Upon subsequent login, the display name is not changed.
 
 If you would like to change the behaviour, you can use the following configuration items:
-```
-[...]
+```yaml
     config:
       policy:
         registration:
           profile:
-            name: True
+            name: true
         login:
           profile:
-            name: False
+            name: false
 ```
 
 3PIDs received from the backend are merged with the ones already linked to the account.
+If you would like to change this behaviour, you can use the following configuration items:
+```yaml
+    config:
+      policy:
+        all:
+          threepid:
+            update: false
+            replace: false
+```
+If update is set to `false`, the 3PIDs will not be changed at all. If replace is set to `true`, all 3PIDs not available in the backend anymore will be deleted from synapse.
 
 ## Integrate
-To use this module with your backend, you will need to implement a single REST endpoint:
+To use this module with your back-end, you will need to implement a single REST endpoint:
 
 Path: `/_matrix-internal/identity/v1/check_credentials`  
 Method: POST  
 Body as JSON UTF-8:
-```
+```json
 {
   "user": {
     "id": "@matrix.id.of.the.user:example.com",
@@ -82,12 +107,12 @@ Body as JSON UTF-8:
 }
 ```
 
-The following JSON answer will be provided:
-```
+If the credentials are accepted, the following JSON answer will be provided:
+```json
 {
   "auth": {
-    "success": <boolean>
-    "mxid": "@matrix.id.of.the.user:example.com"
+    "success": true,
+    "mxid": "@matrix.id.of.the.user:example.com",
     "profile": {
       "display_name": "John Doe",
       "three_pids": [
@@ -104,6 +129,15 @@ The following JSON answer will be provided:
   }
 }
 ```
+`auth.profile` and any sub-key are optional.
 
-## Support
-For community support, visit our Matrix room [#matrix-synapse-rest-auth:kamax.io](https://matrix.to/#/#matrix-synapse-rest-auth:kamax.io)
+---
+
+If the credentials are refused, the following JSON answer will be provided:
+```json
+{
+  "auth": {
+    "success": false
+  }
+}
+```

rest_auth_provider.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 # REST endpoint Authentication module for Matrix synapse
-# Copyright (C) 2017 Maxime Dor
+# Copyright (C) 2017 Kamax Sarl
 #
-# https://max.kamax.io/
+# https://www.kamax.io/
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -20,16 +20,21 @@
 #
 
 import logging
-from twisted.internet import defer
+from typing import Tuple, Optional, Callable, Awaitable
+
 import requests
-import json
+import time
+import synapse
+from synapse import module_api
+from synapse.types import UserID
 
 logger = logging.getLogger(__name__)
 
+
 class RestAuthProvider(object):
 
-    def __init__(self, config, account_handler):
-        self.account_handler = account_handler
+    def __init__(self, config: dict, api: module_api):
+        self.account_handler = api
 
         if not config.endpoint:
             raise RuntimeError('Missing endpoint config')
@@ -37,15 +42,44 @@ class RestAuthProvider(object):
         self.endpoint = config.endpoint
         self.regLower = config.regLower
         self.config = config
-        
+
         logger.info('Endpoint: %s', self.endpoint)
         logger.info('Enforce lowercase username during registration: %s', self.regLower)
 
-    @defer.inlineCallbacks
-    def check_password(self, user_id, password):
+        # register an auth callback handler
+        # see https://matrix-org.github.io/synapse/latest/modules/password_auth_provider_callbacks.html
+        api.register_password_auth_provider_callbacks(
+            auth_checkers={
+                ("m.login.password", ("password",)): self.check_m_login_password
+            }
+        )
+
+    async def check_m_login_password(self, username: str,
+                                     login_type: str,
+                                     login_dict: "synapse.module_api.JsonDict") -> Optional[
+        Tuple[
+            str,
+            Optional[Callable[["synapse.module_api.LoginResponse"], Awaitable[None]]],
+        ]
+    ]:
+        if login_type != "m.login.password":
+            return None
+
+        # get the complete MXID
+        mxid = self.account_handler.get_qualified_user_id(username)
+
+        # check if the password is valid with the old function
+        password_valid = await self.check_password(mxid, login_dict.get("password"))
+
+        if password_valid:
+            return mxid, None
+        else:
+            return None
+
+    async def check_password(self, user_id, password):
         logger.info("Got password check for " + user_id)
-        data = {'user':{'id':user_id, 'password':password}}
-        r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json = data)
+        data = {'user': {'id': user_id, 'password': password}}
+        r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json=data)
         r.raise_for_status()
         r = r.json()
         if not r["auth"]:
@@ -56,20 +90,22 @@ class RestAuthProvider(object):
         auth = r["auth"]
         if not auth["success"]:
             logger.info("User not authenticated")
-            defer.returnValue(False)
+            return False
 
+        types_user_id = UserID.from_string(user_id)
         localpart = user_id.split(":", 1)[0][1:]
+        domain = user_id.split(":", 1)[1][1:]
         logger.info("User %s authenticated", user_id)
 
         registration = False
-        if not (yield self.account_handler.check_user_exists(user_id)):
+        if not (await self.account_handler.check_user_exists(user_id)):
             logger.info("User %s does not exist yet, creating...", user_id)
-            
+
             if localpart != localpart.lower() and self.regLower:
                 logger.info('User %s was cannot be created due to username lowercase policy', localpart)
-                defer.returnValue(False)
-            
-            user_id, access_token = (yield self.account_handler.register(localpart=localpart))
+                return False
+
+            user_id, access_token = (await self.account_handler.register(localpart=localpart))
             registration = True
             logger.info("Registration based on REST data was successful for %s", user_id)
         else:
@@ -79,37 +115,57 @@ class RestAuthProvider(object):
             logger.info("Handling profile data")
             profile = auth["profile"]
 
-            store = yield self.account_handler.hs.get_handlers().profile_handler.store
+            store = self.account_handler._hs.get_profile_handler().store
+
             if "display_name" in profile and ((registration and self.config.setNameOnRegister) or (self.config.setNameOnLogin)):
                 display_name = profile["display_name"]
                 logger.info("Setting display name to '%s' based on profile data", display_name)
-                yield store.set_profile_displayname(localpart, display_name)
+                await store.set_profile_displayname(types_user_id, display_name)
             else:
                 logger.info("Display name was not set because it was not given or policy restricted it")
-            
-            if "three_pids" in profile:
-                logger.info("Handling 3PIDs")
-                for threepid in profile["three_pids"]:
-                    medium = threepid["medium"].lower()
-                    address = threepid["address"].lower()
-                    logger.info("Looking for 3PID %s:%s in user profile", medium, address)
 
-                    validated_at = self.account_handler.hs.get_clock().time_msec()
-                    if not (yield store.get_user_id_by_threepid(medium, address)):
-                        logger.info("3PID is not present, adding")
-                        yield store.user_add_threepid(
-                            user_id,
-                            medium,
-                            address,
-                            validated_at,
-                            validated_at
-                        )
-                    else:
-                        logger.info("3PID is present, skipping")
+            if (self.config.updateThreepid):
+                if "three_pids" in profile:
+                    logger.info("Handling 3PIDs")
+
+                    external_3pids = []
+                    for threepid in profile["three_pids"]:
+                        medium = threepid["medium"].lower()
+                        address = threepid["address"].lower()
+                        external_3pids.append({"medium": medium, "address": address})
+                        logger.info("Looking for 3PID %s:%s in user profile", medium, address)
+
+                        validated_at = time_msec()
+                        if not (await store.get_user_id_by_threepid(medium, address)):
+                            logger.info("3PID is not present, adding")
+                            await store.user_add_threepid(
+                                user_id,
+                                medium,
+                                address,
+                                validated_at,
+                                validated_at
+                            )
+                        else:
+                            logger.info("3PID is present, skipping")
+
+                    if (self.config.replaceThreepid):
+                        for threepid in (await store.user_get_threepids(user_id)):
+                            medium = threepid["medium"].lower()
+                            address = threepid["address"].lower()
+                            if {"medium": medium, "address": address} not in external_3pids:
+                                logger.info("3PID is not present in external datastore, deleting")
+                                await store.user_delete_threepid(
+                                    user_id,
+                                    medium,
+                                    address
+                                )
+
+            else:
+                logger.info("3PIDs were not updated due to policy")
         else:
             logger.info("No profile data")
 
-        defer.returnValue(True)
+        return True
 
     @staticmethod
     def parse_config(config):
@@ -121,6 +177,8 @@ class RestAuthProvider(object):
             regLower = True
             setNameOnRegister = True
             setNameOnLogin = False
+            updateThreepid = True
+            replaceThreepid = False
 
         rest_config = _RestConfig()
         rest_config.endpoint = config["endpoint"]
@@ -142,7 +200,7 @@ class RestAuthProvider(object):
         except KeyError:
             # we don't care
             pass
-        
+
         try:
             rest_config.setNameOnLogin = config['policy']['login']['profile']['name']
         except TypeError:
@@ -152,8 +210,27 @@ class RestAuthProvider(object):
             # we don't care
             pass
 
+        try:
+            rest_config.updateThreepid = config['policy']['all']['threepid']['update']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
+        try:
+            rest_config.replaceThreepid = config['policy']['all']['threepid']['replace']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
         return rest_config
 
+
 def _require_keys(config, required):
     missing = [key for key in required if key not in config]
     if missing:
@@ -163,3 +240,8 @@ def _require_keys(config, required):
             )
         )
 
+
+def time_msec():
+    """Get the current timestamp in milliseconds
+    """
+    return int(time.time() * 1000)

setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup
+
+setup(
+    name="rest_auth_provider",
+    version="0.0.1",
+    py_modules=['rest_auth_provider'],
+    description="Password Provider for Synapse fetching data from a REST endpoint",
+    include_package_data=True,
+    zip_safe=True,
+    install_requires=[],
+)
+