Merge pull request #16 from kamax-io/bridge-lookup

Bridge lookup

application.example.yaml

@@ -53,6 +53,47 @@ lookup:
       - '192.168.0.0/16'
       - '::1/128'
 
+    # In case no binding is found, query an application server which implements the single lookup end-point
+    # to return bridge virtual user that would allow the user to be contacted directly by the said bridge.
+    #
+    # If a binding is returned, the application server is not expected to sign the message as it is not meant to be
+    # reachable from the outside.
+    # If a signature is provided, it will be discarded/replaced by this IS implementation (to be implemented).
+    #
+    # IMPORTANT: This bypass the regular Invite system of the Homeserver. It will be up to the Application Server
+    # to handle such invite. Also, if the bridged user were to actually join Matrix later, or if a 3PID binding is found
+    # room rights and history would not be transferred, as it would appear as a regular Matrix user to the Homeserver.
+    #
+    # This configuration is only helpful for Application Services that want to overwrite bridging for 3PID that are
+    # handled by the Homeserver. Do not enable unless the Application Server specifically supports it!
+    bridge:
+
+      # Enable unknown 3PID bridging globally
+      enabled: false
+
+      # Enable unknown 3PID bridging for hosts that are allowed to perform recursive lookups.
+      # Leaving this setting to true is highly recommended in a standard setup, unless this Identity Server
+      # is meant to always return a virtual user MXID even for the outside world.
+      recursiveOnly: true
+
+      # This mechanism can handle the following scenarios:
+      #
+      # - Single Application Server for all 3PID types: only configure the server value, comment out the rest.
+      #
+      # - Specific Application Server for some 3PID types, default server for the rest: configure the server value and
+      #          each specific 3PID type.
+      #
+      # - Only specific 3PID types: do not configure the server value or leave it empty/blank, configure each specific
+      #          3PID type.
+
+      # Default application server to use for all 3PID types. Remove config item or leave empty/blank to disable.
+      server: ''
+
+      # Configure each 3PID type with a specific application server. Remove config item or leave empty/blank to disable.
+      mappings:
+        email: 'http://localhost:8091'
+        msisdn: ''
+
 
 
 ldap:
@@ -72,12 +113,9 @@ ldap:
   # The attribute containing the binding itself. This value will be used differently depending on the type.
   #
   # Typical values:
-  # - For type 'uid':
+  # - For type 'uid': userPrincipalName
-  #   - Samba/AD: userPrincipalName
-  #   - LDAP: If someone knows the most appropriate value, please open an issue
-  #
   # - For type 'mxid', regardless of the directory type, we recommend using 'pager' as it is a standard attribute and
-  #   are typically not used.
+  #   is typically not used.
   attribute: 'userPrincipalName'
 
   # Configure each 3PID type with a dedicated query.

src/main/groovy/io/kamax/mxisd/config/RecursiveLookupBridgeConfig.groovy

@@ -0,0 +1,60 @@
+package io.kamax.mxisd.config
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.InitializingBean
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+@ConfigurationProperties(prefix = "lookup.recursive.bridge")
+class RecursiveLookupBridgeConfig implements InitializingBean {
+
+    private Logger log = LoggerFactory.getLogger(RecursiveLookupBridgeConfig.class)
+
+    private boolean enabled
+    private boolean recursiveOnly
+    private String server
+    private Map<String, String> mappings
+
+    boolean getEnabled() {
+        return enabled
+    }
+
+    void setEnabled(boolean enabled) {
+        this.enabled = enabled
+    }
+
+    boolean getRecursiveOnly() {
+        return recursiveOnly
+    }
+
+    void setRecursiveOnly(boolean recursiveOnly) {
+        this.recursiveOnly = recursiveOnly
+    }
+
+    String getServer() {
+        return server
+    }
+
+    void setServer(String server) {
+        this.server = server
+    }
+
+    Map<String, String> getMappings() {
+        return mappings
+    }
+
+    void setMappings(Map<String, String> mappings) {
+        this.mappings = mappings
+    }
+
+    @Override
+    void afterPropertiesSet() throws Exception {
+        log.info("Enabled: {}", getEnabled())
+        log.info("Recursive only: {}", getRecursiveOnly())
+        log.info("Server: {}", getServer())
+        log.info("Mappings: {}", mappings.size())
+    }
+
+}

src/main/groovy/io/kamax/mxisd/config/RecursiveLookupConfig.groovy

@@ -29,6 +29,7 @@ class RecursiveLookupConfig {
 
     private boolean enabled
     private List<String> allowedCidr
+    private RecursiveLookupBridgeConfig bridge
 
     boolean isEnabled() {
         return enabled
@@ -46,4 +47,12 @@ class RecursiveLookupConfig {
         this.allowedCidr = allowedCidr
     }
 
+    RecursiveLookupBridgeConfig getBridge() {
+        return bridge
+    }
+
+    void setBridge(RecursiveLookupBridgeConfig bridge) {
+        this.bridge = bridge
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/fetcher/IBridgeFetcher.groovy

@@ -0,0 +1,32 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.fetcher
+
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
+
+interface IBridgeFetcher {
+
+    Optional<?> find(SingleLookupRequest request)
+
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings)
+
+}

src/main/groovy/io/kamax/mxisd/lookup/fetcher/IRemoteIdentityServerFetcher.groovy

@@ -0,0 +1,33 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.fetcher
+
+import io.kamax.mxisd.lookup.ThreePidMapping
+
+interface IRemoteIdentityServerFetcher {
+
+    boolean isUsable(String remote)
+
+    Optional<?> find(String remote, String type, String threePid)
+
+    List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings)
+
+}

src/main/groovy/io/kamax/mxisd/lookup/provider/BridgeFetcher.java

@@ -0,0 +1,73 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import io.kamax.mxisd.config.RecursiveLookupBridgeConfig;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.fetcher.IBridgeFetcher;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+@Component
+public class BridgeFetcher implements IBridgeFetcher {
+
+    private Logger log = LoggerFactory.getLogger(BridgeFetcher.class);
+
+    @Autowired
+    private RecursiveLookupBridgeConfig cfg;
+
+    @Autowired
+    private RemoteIdentityServerFetcher fetcher;
+
+    @Override
+    public Optional<?> find(SingleLookupRequest request) {
+        Optional<String> mediumUrl = Optional.ofNullable(cfg.getMappings().get(request.getType()));
+        if (mediumUrl.isPresent() && !StringUtils.isBlank(mediumUrl.get())) {
+            log.info("Using specific medium bridge lookup URL {}", mediumUrl.get());
+
+            return fetcher.find(mediumUrl.get(), request.getType(), request.getThreePid());
+        } else if (!StringUtils.isBlank(cfg.getServer())) {
+            log.info("Using generic bridge lookup URL {}", cfg.getServer());
+
+            return fetcher.find(cfg.getServer(), request.getType(), request.getThreePid());
+        } else {
+            log.info("No bridge lookup URL found/configured, skipping");
+
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        log.warn("Bulk lookup on bridge lookup requested, but not supported - returning empty list");
+
+        return Collections.emptyList();
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy

@@ -23,6 +23,7 @@ package io.kamax.mxisd.lookup.provider
 import io.kamax.mxisd.config.ServerConfig
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher
 import org.apache.commons.lang.StringUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -37,13 +38,21 @@ import java.util.concurrent.RecursiveTask
 import java.util.function.Function
 
 @Component
-class DnsLookupProvider extends RemoteIdentityServerProvider {
+class DnsLookupProvider implements IThreePidProvider {
 
     private Logger log = LoggerFactory.getLogger(DnsLookupProvider.class)
 
     @Autowired
     private ServerConfig srvCfg
 
+    @Autowired
+    private IRemoteIdentityServerFetcher fetcher
+
+    @Override
+    boolean isLocal() {
+        return false
+    }
+
     @Override
     int getPriority() {
         return 10
@@ -87,7 +96,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
             for (SRVRecord record : records) {
                 log.info("Found SRV record: {}", record.toString())
                 String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}"
-                if (isUsableIdentityServer(baseUrl)) {
+                if (fetcher.isUsable(baseUrl)) {
                     log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
                     return Optional.of(baseUrl)
                 } else {
@@ -100,7 +109,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
 
         log.info("Performing basic lookup using domain name {}", domain)
         String baseUrl = "https://" + domain
-        if (isUsableIdentityServer(baseUrl)) {
+        if (fetcher.isUsable(baseUrl)) {
             log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
             return Optional.of(baseUrl)
         } else {
@@ -123,7 +132,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
         Optional<String> baseUrl = findIdentityServerForDomain(domain)
 
         if (baseUrl.isPresent()) {
-            return find(baseUrl.get(), request.getType().toString(), request.getThreePid())
+            return fetcher.find(baseUrl.get(), request.getType().toString(), request.getThreePid())
         }
 
         return Optional.empty()
@@ -205,7 +214,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
             if (!baseUrl.isPresent()) {
                 log.info("No usable Identity server for domain {}", domain)
             } else {
-                domainMappings.addAll(find(baseUrl.get(), mappings))
+                domainMappings.addAll(fetcher.find(baseUrl.get(), mappings))
                 log.info("Found {} mappings in domain {}", domainMappings.size(), domain)
             }
 

src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy

@@ -23,19 +23,28 @@ package io.kamax.mxisd.lookup.provider
 import io.kamax.mxisd.config.ForwardConfig
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
-class ForwarderProvider extends RemoteIdentityServerProvider {
+class ForwarderProvider implements IThreePidProvider {
 
     private Logger log = LoggerFactory.getLogger(ForwarderProvider.class)
 
     @Autowired
     private ForwardConfig cfg
 
+    @Autowired
+    private IRemoteIdentityServerFetcher fetcher
+
+    @Override
+    boolean isLocal() {
+        return false
+    }
+
     @Override
     int getPriority() {
         return 0
@@ -44,7 +53,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
     @Override
     Optional<?> find(SingleLookupRequest request) {
         for (String root : cfg.getServers()) {
-            Optional<?> answer = find(root, request.getType(), request.getThreePid())
+            Optional<?> answer = fetcher.find(root, request.getType(), request.getThreePid())
             if (answer.isPresent()) {
                 return answer
             }
@@ -61,7 +70,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
         for (String root : cfg.getServers()) {
             log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo)
             log.info("Querying {}", root)
-            List<ThreePidMapping> mappingsFound = find(root, mappingsToDo)
+            List<ThreePidMapping> mappingsFound = fetcher.find(root, mappingsToDo)
             log.info("{} returned {} mappings", root, mappingsFound.size())
             mappingsFoundGlobal.addAll(mappingsFound)
             mappingsToDo.removeAll(mappingsFound)

src/main/groovy/io/kamax/mxisd/lookup/provider/IThreePidProvider.groovy

@@ -23,7 +23,7 @@ package io.kamax.mxisd.lookup.provider
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
 
-interface ThreePidProvider {
+interface IThreePidProvider {
 
     boolean isLocal()
 

src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy

@@ -37,7 +37,7 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
-class LdapProvider implements ThreePidProvider {
+class LdapProvider implements IThreePidProvider {
 
     public static final String UID = "uid"
     public static final String MATRIX_ID = "mxid"

src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerFetcher.groovy

@@ -25,6 +25,7 @@ import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher
 import org.apache.http.HttpEntity
 import org.apache.http.HttpResponse
 import org.apache.http.client.HttpClient
@@ -34,22 +35,24 @@ import org.apache.http.entity.ContentType
 import org.apache.http.impl.client.HttpClients
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
+import org.springframework.context.annotation.Lazy
+import org.springframework.context.annotation.Scope
+import org.springframework.stereotype.Component
 
-abstract class RemoteIdentityServerProvider implements ThreePidProvider {
+@Component
+@Scope("prototype")
+@Lazy
+public class RemoteIdentityServerFetcher implements IRemoteIdentityServerFetcher {
 
     public static final String THREEPID_TEST_MEDIUM = "email"
     public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org"
 
-    private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class)
+    private Logger log = LoggerFactory.getLogger(RemoteIdentityServerFetcher.class)
 
     private JsonSlurper json = new JsonSlurper()
 
     @Override
-    boolean isLocal() {
+    boolean isUsable(String remote) {
-        return false
-    }
-
-    boolean isUsableIdentityServer(String remote) {
         try {
             HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
                     "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}"
@@ -73,6 +76,7 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         }
     }
 
+    @Override
     Optional<?> find(String remote, String type, String threePid) {
         log.info("Looking up {} 3PID {} using {}", type, threePid, remote)
 
@@ -98,6 +102,7 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         }
     }
 
+    @Override
     List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
         List<ThreePidMapping> mappingsFound = new ArrayList<>()
 

src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy

@@ -26,7 +26,8 @@ import io.kamax.mxisd.lookup.ALookupRequest
 import io.kamax.mxisd.lookup.BulkLookupRequest
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
-import io.kamax.mxisd.lookup.provider.ThreePidProvider
+import io.kamax.mxisd.lookup.fetcher.IBridgeFetcher
+import io.kamax.mxisd.lookup.provider.IThreePidProvider
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.InitializingBean
@@ -42,7 +43,10 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
     private RecursiveLookupConfig recursiveCfg
 
     @Autowired
-    private List<ThreePidProvider> providers
+    private List<IThreePidProvider> providers
+
+    @Autowired
+    private IBridgeFetcher bridge
 
     private List<CIDRUtils> allowedCidr = new ArrayList<>()
 
@@ -50,10 +54,10 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
     void afterPropertiesSet() throws Exception {
         log.info("Found ${providers.size()} providers")
 
-        providers.sort(new Comparator<ThreePidProvider>() {
+        providers.sort(new Comparator<IThreePidProvider>() {
 
             @Override
-            int compare(ThreePidProvider o1, ThreePidProvider o2) {
+            int compare(IThreePidProvider o1, IThreePidProvider o2) {
                 return Integer.compare(o2.getPriority(), o1.getPriority())
             }
 
@@ -66,25 +70,32 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         }
     }
 
-    List<ThreePidProvider> listUsableProviders(ALookupRequest request) {
+    boolean isAllowedForRecursive(String source) {
-        List<ThreePidProvider> usableProviders = new ArrayList<>()
-
         boolean canRecurse = false
+
         if (recursiveCfg.isEnabled()) {
             log.debug("Checking {} CIDRs for recursion", allowedCidr.size())
             for (CIDRUtils cidr : allowedCidr) {
-                if (cidr.isInRange(request.getRequester())) {
+                if (cidr.isInRange(source)) {
-                    log.debug("{} is in range {}, allowing recursion", request.getRequester(), cidr.getNetworkAddress())
+                    log.debug("{} is in range {}, allowing recursion", source, cidr.getNetworkAddress())
                     canRecurse = true
                     break
                 } else {
-                    log.debug("{} is not in range {}", request.getRequester(), cidr.getNetworkAddress())
+                    log.debug("{} is not in range {}", source, cidr.getNetworkAddress())
                 }
             }
         }
 
+        return canRecurse
+    }
+
+    List<IThreePidProvider> listUsableProviders(ALookupRequest request) {
+        List<IThreePidProvider> usableProviders = new ArrayList<>()
+
+        boolean canRecurse = isAllowedForRecursive(request.getRequester())
+
         log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse)
-        for (ThreePidProvider provider : providers) {
+        for (IThreePidProvider provider : providers) {
             if (provider.isLocal() || canRecurse) {
                 usableProviders.add(provider)
             }
@@ -95,13 +106,18 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
 
     @Override
     Optional<?> find(SingleLookupRequest request) {
-        for (ThreePidProvider provider : listUsableProviders(request)) {
+        for (IThreePidProvider provider : listUsableProviders(request)) {
             Optional<?> lookupDataOpt = provider.find(request)
             if (lookupDataOpt.isPresent()) {
                 return lookupDataOpt
             }
         }
 
+        if (recursiveCfg.getBridge().getEnabled() && (!recursiveCfg.getBridge().getRecursiveOnly() || isAllowedForRecursive(request.getRequester()))) {
+            log.info("Using bridge failover for lookup")
+            return bridge.find(request)
+        }
+
         return Optional.empty()
     }
 
@@ -110,7 +126,7 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         List<ThreePidMapping> mapToDo = new ArrayList<>(request.getMappings())
         List<ThreePidMapping> mapFoundAll = new ArrayList<>()
 
-        for (ThreePidProvider provider : listUsableProviders(request)) {
+        for (IThreePidProvider provider : listUsableProviders(request)) {
             if (mapToDo.isEmpty()) {
                 log.info("No more mappings to lookup")
                 break