Recursive lookup management

application.example.yaml

@@ -1,14 +1,19 @@
 server:
 
-  # Indicate on which port the Identity Server will listen. This is an unencrypted port.
+  # Indicate on which port the Identity Server will listen.
+  #
+  # This is be default an unencrypted port.
   # HTTPS can be configured using Tomcat configuration properties.
   port: 8090
 
-  # Realm under which this Identity Server is authoritative, e.g. domain name in e-mails.
-  # This is used in some recursive lookups to avoid endless loops and avoid bothering other Identity Servers.
+  # Realm under which this Identity Server is authoritative.
+  #
+  # This is used to avoid unnecessary connections and endless recursive lookup.
+  # e.g. domain name in e-mails.
   name: 'example.org'
 
 
+
 key:
 
   # Where the Identity Server signing key will be stored.
@@ -19,6 +24,34 @@ key:
   path: '/var/tmp/mxis-signing.key'
 
 
+
+# This element contains all the configuration item for lookup strategies
+lookup:
+
+  # Configuration items for recursion-type of lookup
+  #
+  # Lookup access are divided into two types:
+  # - Local
+  # - Remote
+  #
+  # This is similar to DNS lookup and recursion and is therefore prone to the same vulnerabilities.
+  # By default, only non-public hosts are allowed to perform recursive lookup.
+  # This will also prevent basic endless loops where:
+  # host A ask host B, which in turn is configured to ask host B, etc.
+  recursive:
+
+    # Enable recursive lookup globally
+    enabled: true
+
+    # Whitelist of CIDR that will trigger a recursive lookup
+    allowedCidr:
+      - '127.0.0.0/8'
+      - '10.0.0.0/8'
+      - '172.16.0.0/16'
+      - '192.168.0.0/16'
+
+
+
 ldap:
   host: 'localhost'
   port: 389
@@ -44,3 +77,15 @@ ldap:
   # - For type 'mxid', regardless of the directory type, we recommend using 'pager' as it is a standard attribute but
   #   shouldn't be used in infrastructures.
   attribute: 'sAMAccountName'
+
+
+
+forward:
+
+  # List of forwarders to use to try to match a 3PID.
+  #
+  # Each server will be tried in the given order, going to the next if no binding was found or an error occurred.
+  # There are the current root Identity Servers of the Matrix network.
+  servers:
+    - "https://matrix.org"
+    - "https://vector.im"

build.gradle

@@ -54,6 +54,9 @@ dependencies {
     // DNS lookups
     compile 'dnsjava:dnsjava:2.1.8'
 
+    // Network utilities for recursive host check
+    compile 'commons-net:commons-net:3.5'
+
     testCompile 'junit:junit:4.12'
 }
 

src/main/groovy/io/kamax/mxisd/config/ForwardConfig.groovy

@@ -0,0 +1,40 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+@ConfigurationProperties(prefix = "forward")
+class ForwardConfig {
+
+    private List<String> servers
+
+    List<String> getServers() {
+        return servers
+    }
+
+    void setServers(List<String> servers) {
+        this.servers = servers
+    }
+
+}

src/main/groovy/io/kamax/mxisd/config/RecursiveLookupConfig.groovy

@@ -0,0 +1,49 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+@ConfigurationProperties(prefix = "lookup.recursive")
+class RecursiveLookupConfig {
+
+    private boolean enabled
+    private List<String> allowedCidr
+
+    boolean isEnabled() {
+        return enabled
+    }
+
+    void setEnabled(boolean enabled) {
+        this.enabled = enabled
+    }
+
+    List<String> getAllowedCidr() {
+        return allowedCidr
+    }
+
+    void setAllowedCidr(List<String> allowedCidr) {
+        this.allowedCidr = allowedCidr
+    }
+
+}

src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy

@@ -22,18 +22,26 @@ package io.kamax.mxisd.controller.v1
 
 import groovy.json.JsonOutput
 import io.kamax.mxisd.api.ThreePidType
-import io.kamax.mxisd.lookup.LookupStrategy
+import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.strategy.LookupStrategy
 import io.kamax.mxisd.signature.SignatureManager
+import org.apache.commons.lang.StringUtils
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RequestParam
 import org.springframework.web.bind.annotation.RestController
 
+import javax.servlet.http.HttpServletRequest
+
 import static org.springframework.web.bind.annotation.RequestMethod.GET
 
 @RestController
 class MappingController {
 
+    private Logger log = LoggerFactory.getLogger(MappingController.class)
+
     @Autowired
     private LookupStrategy strategy
 
@@ -41,15 +49,26 @@ class MappingController {
     private SignatureManager signMgr
 
     @RequestMapping(value = "/_matrix/identity/api/v1/lookup", method = GET)
-    String lookup(@RequestParam String medium, @RequestParam String address) {
+    String lookup(HttpServletRequest request, @RequestParam String medium, @RequestParam String address) {
+        String remote = StringUtils.defaultIfBlank(request.getHeader("X-FORWARDED-FOR"), request.getRemoteAddr())
+        log.info("Got request from {}", remote)
+
         ThreePidType type = ThreePidType.valueOf(medium)
-        Optional<?> lookupOpt = strategy.find(type, address)
+
+        LookupRequest lookupRequest = new LookupRequest()
+        lookupRequest.setRequester(remote)
+        lookupRequest.setType(type)
+        lookupRequest.setThreePid(address)
+
+        Optional<?> lookupOpt = strategy.find(lookupRequest)
         if (!lookupOpt.isPresent()) {
+            log.info("No mapping was found, return empty JSON object")
             return JsonOutput.toJson([])
         }
 
         def lookup = lookupOpt.get()
         if (lookup['signatures'] == null) {
+            log.info("lookup is not signed yet, we sign it")
             lookup['signatures'] = signMgr.signMessage(JsonOutput.toJson(lookup))
         }
 

src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy

@@ -0,0 +1,55 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup
+
+import io.kamax.mxisd.api.ThreePidType
+
+class LookupRequest {
+
+    private String requester
+    private ThreePidType type
+    private String threePid
+
+    String getRequester() {
+        return requester
+    }
+
+    void setRequester(String requester) {
+        this.requester = requester
+    }
+
+    ThreePidType getType() {
+        return type
+    }
+
+    void setType(ThreePidType type) {
+        this.type = type
+    }
+
+    String getThreePid() {
+        return threePid
+    }
+
+    void setThreePid(String threePid) {
+        this.threePid = threePid
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy

@@ -18,10 +18,11 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.provider
 
 import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.ServerConfig
+import io.kamax.mxisd.lookup.LookupRequest
 import org.apache.commons.lang.StringUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -45,15 +46,15 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
     }
 
     @Override
-    Optional<?> find(ThreePidType type, String threePid) {
-        log.info("Performing DNS lookup for {}", threePid)
-        if (ThreePidType.email != type) {
-            log.info("Skipping unsupported type {} for {}", type, threePid)
+    Optional<?> find(LookupRequest request) {
+        log.info("Performing DNS lookup for {}", request.getThreePid())
+        if (ThreePidType.email != request.getType()) {
+            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
             return Optional.empty()
         }
 
-        String domain = threePid.substring(threePid.lastIndexOf("@") + 1)
-        log.info("Domain name for {}: {}", threePid, domain)
+        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        log.info("Domain name for {}: {}", request.getThreePid(), domain)
         if (StringUtils.equals(srvCfg.getName(), domain)) {
             log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?")
             return Optional.empty()
@@ -77,7 +78,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
             for (SRVRecord record : records) {
                 log.info("Found SRV record: {}", record.toString())
                 String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}"
-                Optional<?> answer = find(baseUrl, type, threePid)
+                Optional<?> answer = find(baseUrl, request.getType(), request.getThreePid())
                 if (answer.isPresent()) {
                     return answer
                 } else {
@@ -90,7 +91,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
 
         log.info("Performing basic lookup using domain name {}", domain)
         String baseUrl = "https://" + domain
-        return find(baseUrl, type, threePid)
+        return find(baseUrl, request.getType(), request.getThreePid())
     }
 
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy

@@ -18,15 +18,18 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.provider
 
-import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.config.ForwardConfig
+import io.kamax.mxisd.lookup.LookupRequest
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
-class RootProvider extends RemoteIdentityServerProvider {
+class ForwarderProvider extends RemoteIdentityServerProvider {
 
-    private List<String> roots = Arrays.asList("https://matrix.org", "https://vector.im")
+    @Autowired
+    private ForwardConfig cfg
 
     @Override
     int getPriority() {
@@ -34,9 +37,9 @@ class RootProvider extends RemoteIdentityServerProvider {
     }
 
     @Override
-    Optional<?> find(ThreePidType type, String threePid) {
-        for (String root : roots) {
-            Optional<?> answer = find(root, type, threePid)
+    Optional<?> find(LookupRequest request) {
+        for (String root : cfg.getServers()) {
+            Optional<?> answer = find(root, request.getType(), request.getThreePid())
             if (answer.isPresent()) {
                 return answer
             }

src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy

@@ -18,11 +18,11 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.provider
 
-import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.LdapConfig
 import io.kamax.mxisd.config.ServerConfig
+import io.kamax.mxisd.lookup.LookupRequest
 import org.apache.commons.lang.StringUtils
 import org.apache.directory.api.ldap.model.cursor.EntryCursor
 import org.apache.directory.api.ldap.model.entry.Attribute
@@ -49,11 +49,6 @@ class LdapProvider implements ThreePidProvider, InitializingBean {
     @Autowired
     private LdapConfig ldapCfg
 
-    @Override
-    int getPriority() {
-        return 20
-    }
-
     @Override
     void afterPropertiesSet() throws Exception {
         if (!Arrays.asList(UID, MATRIX_ID).contains(ldapCfg.getType())) {
@@ -62,14 +57,24 @@ class LdapProvider implements ThreePidProvider, InitializingBean {
     }
 
     @Override
-    Optional<?> find(ThreePidType type, String threePid) {
-        log.info("Performing LDAP lookup ${threePid} of type ${type}")
+    boolean isLocal() {
+        return true
+    }
+
+    @Override
+    int getPriority() {
+        return 20
+    }
+
+    @Override
+    Optional<?> find(LookupRequest request) {
+        log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}")
 
         LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
         try {
             conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
 
-            String searchQuery = ldapCfg.getQuery().replaceAll("%3pid", threePid)
+            String searchQuery = ldapCfg.getQuery().replaceAll("%3pid", request.getThreePid())
             EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
             try {
                 if (cursor.next()) {
@@ -96,8 +101,8 @@ class LdapProvider implements ThreePidProvider, InitializingBean {
                         }
 
                         return Optional.of([
-                                address   : threePid,
-                                medium    : type,
+                                address   : request.getThreePid(),
+                                medium    : request.getType(),
                                 mxid      : matrixId.toString(),
                                 not_before: 0,
                                 not_after : 9223372036854775807,

src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy

@@ -18,7 +18,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.provider
 
 import groovy.json.JsonException
 import groovy.json.JsonSlurper
@@ -29,8 +29,14 @@ import org.slf4j.LoggerFactory
 abstract class RemoteIdentityServerProvider implements ThreePidProvider {
 
     private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class)
+
     private JsonSlurper json = new JsonSlurper()
 
+    @Override
+    boolean isLocal() {
+        return false
+    }
+
     Optional<?> find(String remote, ThreePidType type, String threePid) {
         log.info("Looking up {} 3PID {} using {}", type, threePid, remote)
 
@@ -40,7 +46,8 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
 
         try {
             def output = json.parseText(rootSrvConn.getInputStream().getText())
-            if (output['address'] != null) {
+            if (output['address']) {
+                log.info("Found 3PID mapping: {}", output)
                 return Optional.of(output)
             }
 

src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy

@@ -18,17 +18,19 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.provider
 
-import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.lookup.LookupRequest
 
 interface ThreePidProvider {
 
+    boolean isLocal()
+
     /**
      * Higher has more priority
      */
     int getPriority() // Should not be here but let's KISS for now
 
-    Optional<?> find(ThreePidType type, String threePid)
+    Optional<?> find(LookupRequest request)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy

@@ -18,12 +18,12 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.strategy
 
-import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.lookup.LookupRequest
 
 interface LookupStrategy {
 
-    Optional<?> find(ThreePidType type, String threePid)
+    Optional<?> find(LookupRequest request)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy

@@ -18,9 +18,13 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-package io.kamax.mxisd.lookup
+package io.kamax.mxisd.lookup.strategy
 
 import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.config.RecursiveLookupConfig
+import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.provider.ThreePidProvider
+import org.apache.commons.net.util.SubnetUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.InitializingBean
@@ -28,13 +32,18 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
-class PriorityLookupStrategy implements LookupStrategy, InitializingBean {
+class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBean {
 
-    private Logger log = LoggerFactory.getLogger(PriorityLookupStrategy.class)
+    private Logger log = LoggerFactory.getLogger(RecursivePriorityLookupStrategy.class)
+
+    @Autowired
+    private RecursiveLookupConfig recursiveCfg
 
     @Autowired
     private List<ThreePidProvider> providers
 
+    private List<SubnetUtils.SubnetInfo> allowedCidr = new ArrayList<>()
+
     @Override
     void afterPropertiesSet() throws Exception {
         log.info("Found ${providers.size()} providers")
@@ -47,18 +56,37 @@ class PriorityLookupStrategy implements LookupStrategy, InitializingBean {
             }
 
         })
+
+        log.info("Recursive lookup enabled: {}", recursiveCfg.isEnabled())
+        for (String cidr : recursiveCfg.getAllowedCidr()) {
+            log.info("{} is allowed for recursion", cidr)
+            allowedCidr.add(new SubnetUtils(cidr).getInfo())
+        }
     }
 
     @Override
-    Optional<?> find(ThreePidType type, String threePid) {
-        if (ThreePidType.email != type) {
-            throw new IllegalArgumentException("${type} is currently not supported")
+    Optional<?> find(LookupRequest request) {
+        if (ThreePidType.email != request.getType()) {
+            throw new IllegalArgumentException("${request.getType()} is currently not supported")
         }
 
+        boolean canRecurse = false
+        if (recursiveCfg.isEnabled()) {
+            for (SubnetUtils.SubnetInfo cidr : allowedCidr) {
+                if (cidr.isInRange(request.getRequester())) {
+                    canRecurse = true
+                    break
+                }
+            }
+        }
+        log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse)
+
         for (ThreePidProvider provider : providers) {
-            Optional<?> lookupDataOpt = provider.find(type, threePid)
-            if (lookupDataOpt.isPresent()) {
-                return lookupDataOpt
+            if (provider.isLocal() || canRecurse) {
+                Optional<?> lookupDataOpt = provider.find(request)
+                if (lookupDataOpt.isPresent()) {
+                    return lookupDataOpt
+                }
             }
         }