Temporary fix for authentication. Update readme. Add gitignore.

Add option to replace 3PIDs in synapse if removed/changed in the backend

.gitignore

@@ -0,0 +1,204 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+*.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+

README.md

@@ -1,32 +1,53 @@
-# HTTP JSON REST Authenticator module for synapse
+# Synapse REST Password provider
-This synapse authentication module (password provider) allows you to query identity data in existing webapps, like:
+- [Overview](#overview)
+- [Install](#install)
+- [Configure](#configure)
+- [Integrate](#integrate)
+- [Support](#support)
+
+## Overview
+This synapse's password provider allows you to validate a password for a given username and return a user profile using an existing backend, like:
+
 - Forums (phpBB, Discourse, etc.)
 - Custom Identity stores (Keycloak, ...)
 - CRMs (Wordpress, ...)
 - self-hosted clouds (Nextcloud, ownCloud, ...)
 
-It is mainly used with [mxisd](https://github.com/kamax-io/mxisd), the Federated Matrix Identity Server, to provide
+It is mainly used with [ma1sd](https://github.com/ma1uta/ma1sd), the Federated Matrix Identity Server, to provide
 missing features and offer a fully integrated solution (directory, authentication, search).
 
+**NOTE:** This module doesn't provide direct integration with any backend. If you do not use mxisd, you will need to write
+your own backend, following the [Integration section](#integrate). This module simply translate an anthentication result
+and profile information into actionables in synapse, and adapt your user profile with what is given.
+
 ## Install
+### From Synapse v0.34.0/py3
+Copy in whichever directory python3.x can pick it up as a module.  
+
+If you installed synapse using the Matrix debian repos:
+```
+sudo curl https://raw.githubusercontent.com/ma1uta/matrix-synapse-rest-password-provider/master/rest_auth_provider.py -o /opt/venvs/matrix-synapse/lib/python3.7/site-packages/rest_auth_provider.py
+```
+If the command fail, double check that the python version still matches. If not, please let us know by opening an issue.
+
+### Before Synapse v0.34.0/py3 or any py2-based release
 Copy in whichever directory python2.x can pick it up as a module.  
 
 If you installed synapse using the Matrix debian repos:
 ```
-git clone https://github.com/maxidor/matrix-synapse-rest-auth.git
+sudo curl https://raw.githubusercontent.com/ma1uta/matrix-synapse-rest-password-provider/master/rest_auth_provider.py -o /usr/lib/python2.7/dist-packages/rest_auth_provider.py
-cd matrix-synapse-rest-auth
-sudo cp rest_auth_provider.py /usr/lib/python2.7/dist-packages/
 ```
+If the command fail, double check that the python version still matches. If not, please let us know by opening an issue.
 
 ## Configure
 Add or amend the `password_providers` entry like so:
-```
+```yaml
 password_providers:
   - module: "rest_auth_provider.RestAuthProvider"
     config:
       endpoint: "http://change.me.example.com:12345"
 ```
-Set `endpoint` to the appropriate value.
+Set `endpoint` to the value documented with the endpoint provider.
 
 ## Use
 1. Install, configure, restart synapse
@@ -34,17 +55,18 @@ Set `endpoint` to the appropriate value.
 
 ## Next steps
 ### Lowercase username enforcement
+**NOTE**: This is no longer relevant as synapse natively enforces lowercase.
+
 To avoid creating users accounts with uppercase characters in their usernames and running into known
 issues regarding case sensitivity in synapse, attempting to login with such username will fail.
 
 It is highly recommended to keep this feature enable, but in case you would like to disable it:
-```
+```yaml
-[...]
     config:
       policy:
         registration:
           username:
-            enforceLowercase: False
+            enforceLowercase: false
 ```
 
 ### Profile auto-fill
@@ -53,27 +75,36 @@ If none is given, the display name is not set.
 Upon subsequent login, the display name is not changed.
 
 If you would like to change the behaviour, you can use the following configuration items:
-```
+```yaml
-[...]
     config:
       policy:
         registration:
           profile:
-            name: True
+            name: true
         login:
           profile:
-            name: False
+            name: false
 ```
 
 3PIDs received from the backend are merged with the ones already linked to the account.
+If you would like to change this behaviour, you can use the following configuration items:
+```yaml
+    config:
+      policy:
+        all:
+          threepid:
+            update: false
+            replace: false
+```
+If update is set to `false`, the 3PIDs will not be changed at all. If replace is set to `true`, all 3PIDs not available in the backend anymore will be deleted from synapse.
 
 ## Integrate
-To use this module with your backend, you will need to implement a single REST endpoint:
+To use this module with your back-end, you will need to implement a single REST endpoint:
 
 Path: `/_matrix-internal/identity/v1/check_credentials`  
 Method: POST  
 Body as JSON UTF-8:
-```
+```json
 {
   "user": {
     "id": "@matrix.id.of.the.user:example.com",
@@ -82,12 +113,12 @@ Body as JSON UTF-8:
 }
 ```
 
-The following JSON answer will be provided:
+If the credentials are accepted, the following JSON answer will be provided:
-```
+```json
 {
   "auth": {
-    "success": <boolean>
+    "success": true,
-    "mxid": "@matrix.id.of.the.user:example.com"
+    "mxid": "@matrix.id.of.the.user:example.com",
     "profile": {
       "display_name": "John Doe",
       "three_pids": [
@@ -104,6 +135,15 @@ The following JSON answer will be provided:
   }
 }
 ```
+`auth.profile` and any sub-key are optional.
 
-## Support
+---
-For community support, visit our Matrix room [#matrix-synapse-rest-auth:kamax.io](https://matrix.to/#/#matrix-synapse-rest-auth:kamax.io)
+
+If the credentials are refused, the following JSON answer will be provided:
+```json
+{
+  "auth": {
+    "success": false
+  }
+}
+```

rest_auth_provider.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 # REST endpoint Authentication module for Matrix synapse
-# Copyright (C) 2017 Maxime Dor
+# Copyright (C) 2017 Kamax Sarl
 #
-# https://max.kamax.io/
+# https://www.kamax.io/
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -23,9 +23,11 @@ import logging
 from twisted.internet import defer
 import requests
 import json
+import time
 
 logger = logging.getLogger(__name__)
 
+
 class RestAuthProvider(object):
 
     def __init__(self, config, account_handler):
@@ -37,15 +39,15 @@ class RestAuthProvider(object):
         self.endpoint = config.endpoint
         self.regLower = config.regLower
         self.config = config
-        
+
         logger.info('Endpoint: %s', self.endpoint)
         logger.info('Enforce lowercase username during registration: %s', self.regLower)
 
     @defer.inlineCallbacks
     def check_password(self, user_id, password):
         logger.info("Got password check for " + user_id)
-        data = {'user':{'id':user_id, 'password':password}}
+        data = {'user': {'id': user_id, 'password': password}}
-        r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json = data)
+        r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json=data)
         r.raise_for_status()
         r = r.json()
         if not r["auth"]:
@@ -64,11 +66,11 @@ class RestAuthProvider(object):
         registration = False
         if not (yield self.account_handler.check_user_exists(user_id)):
             logger.info("User %s does not exist yet, creating...", user_id)
-            
+
             if localpart != localpart.lower() and self.regLower:
                 logger.info('User %s was cannot be created due to username lowercase policy', localpart)
                 defer.returnValue(False)
-            
+
             user_id, access_token = (yield self.account_handler.register(localpart=localpart))
             registration = True
             logger.info("Registration based on REST data was successful for %s", user_id)
@@ -79,33 +81,57 @@ class RestAuthProvider(object):
             logger.info("Handling profile data")
             profile = auth["profile"]
 
-            store = yield self.account_handler.hs.get_profile_handler().store
+            # fixme: temporary fix
+            try:
+                store = yield self.account_handler._hs.get_profile_handler().store  # for synapse >= 1.9.0
+            except AttributeError:
+                store = yield self.account_handler.hs.get_profile_handler().store   # for synapse < 1.9.0
+
             if "display_name" in profile and ((registration and self.config.setNameOnRegister) or (self.config.setNameOnLogin)):
                 display_name = profile["display_name"]
                 logger.info("Setting display name to '%s' based on profile data", display_name)
                 yield store.set_profile_displayname(localpart, display_name)
             else:
                 logger.info("Display name was not set because it was not given or policy restricted it")
-            
-            if "three_pids" in profile:
-                logger.info("Handling 3PIDs")
-                for threepid in profile["three_pids"]:
-                    medium = threepid["medium"].lower()
-                    address = threepid["address"].lower()
-                    logger.info("Looking for 3PID %s:%s in user profile", medium, address)
 
-                    validated_at = self.account_handler.hs.get_clock().time_msec()
+            if (self.config.updateThreepid):
-                    if not (yield store.get_user_id_by_threepid(medium, address)):
+                if "three_pids" in profile:
-                        logger.info("3PID is not present, adding")
+                    logger.info("Handling 3PIDs")
-                        yield store.user_add_threepid(
+
-                            user_id,
+                    external_3pids = []
-                            medium,
+                    for threepid in profile["three_pids"]:
-                            address,
+                        medium = threepid["medium"].lower()
-                            validated_at,
+                        address = threepid["address"].lower()
-                            validated_at
+                        external_3pids.append({"medium": medium, "address": address})
-                        )
+                        logger.info("Looking for 3PID %s:%s in user profile", medium, address)
-                    else:
+
-                        logger.info("3PID is present, skipping")
+                        validated_at = time_msec()
+                        if not (yield store.get_user_id_by_threepid(medium, address)):
+                            logger.info("3PID is not present, adding")
+                            yield store.user_add_threepid(
+                                user_id,
+                                medium,
+                                address,
+                                validated_at,
+                                validated_at
+                            )
+                        else:
+                            logger.info("3PID is present, skipping")
+
+                    if (self.config.replaceThreepid):
+                        for threepid in (yield store.user_get_threepids(user_id)):
+                            medium = threepid["medium"].lower()
+                            address = threepid["address"].lower()
+                            if {"medium": medium, "address": address} not in external_3pids:
+                                logger.info("3PID is not present in external datastore, deleting")
+                                yield store.user_delete_threepid(
+                                    user_id,
+                                    medium,
+                                    address
+                                )
+
+            else:
+                logger.info("3PIDs were not updated due to policy")
         else:
             logger.info("No profile data")
 
@@ -121,6 +147,8 @@ class RestAuthProvider(object):
             regLower = True
             setNameOnRegister = True
             setNameOnLogin = False
+            updateThreepid = True
+            replaceThreepid = False
 
         rest_config = _RestConfig()
         rest_config.endpoint = config["endpoint"]
@@ -142,7 +170,7 @@ class RestAuthProvider(object):
         except KeyError:
             # we don't care
             pass
-        
+
         try:
             rest_config.setNameOnLogin = config['policy']['login']['profile']['name']
         except TypeError:
@@ -152,8 +180,27 @@ class RestAuthProvider(object):
             # we don't care
             pass
 
+        try:
+            rest_config.updateThreepid = config['policy']['all']['threepid']['update']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
+        try:
+            rest_config.replaceThreepid = config['policy']['all']['threepid']['replace']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
         return rest_config
 
+
 def _require_keys(config, required):
     missing = [key for key in required if key not in config]
     if missing:
@@ -163,3 +210,8 @@ def _require_keys(config, required):
             )
         )
 
+
+def time_msec():
+    """Get the current timestamp in milliseconds
+    """
+    return int(time.time() * 1000)