Merge pull request #13 from kamax-io/bulk_lookup

Bulk lookup

build.gradle

@@ -54,6 +54,9 @@ dependencies {
     // DNS lookups
     compile 'dnsjava:dnsjava:2.1.8'
 
+    // HTTP connections
+    compile 'org.apache.httpcomponents:httpclient:4.5.3'
+
     testCompile 'junit:junit:4.12'
 }
 

src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java

@@ -0,0 +1,44 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.controller.v1;
+
+import io.kamax.mxisd.lookup.ThreePidMapping;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+public class ClientBulkLookupAnswer {
+
+    private List<List<String>> threepids = new ArrayList<>();
+
+    public void addAll(Collection<ThreePidMapping> mappings) {
+        for (ThreePidMapping mapping : mappings) {
+            threepids.add(Arrays.asList(mapping.getMedium(), mapping.getValue(), mapping.getMxid()));
+        }
+    }
+
+    public List<List<String>> getThreepids() {
+        return threepids;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy

@@ -0,0 +1,46 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.controller.v1
+
+import io.kamax.mxisd.lookup.ThreePidMapping
+
+class ClientBulkLookupRequest {
+
+    private List<List<String>> threepids = new ArrayList<>()
+
+    List<List<String>> getThreepids() {
+        return threepids
+    }
+
+    void setThreepids(List<List<String>> threepids) {
+        this.threepids = threepids
+    }
+
+    void setMappings(List<ThreePidMapping> mappings) {
+        for (ThreePidMapping mapping : mappings) {
+            List<String> threepid = new ArrayList<>()
+            threepid.add(mapping.getMedium())
+            threepid.add(mapping.getValue())
+            threepids.add(threepid)
+        }
+    }
+
+}

src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy

@@ -21,8 +21,11 @@
 package io.kamax.mxisd.controller.v1
 
 import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import io.kamax.mxisd.lookup.strategy.LookupStrategy
 import io.kamax.mxisd.signature.SignatureManager
 import org.apache.commons.lang.StringUtils
@@ -36,11 +39,13 @@ import org.springframework.web.bind.annotation.RestController
 import javax.servlet.http.HttpServletRequest
 
 import static org.springframework.web.bind.annotation.RequestMethod.GET
+import static org.springframework.web.bind.annotation.RequestMethod.POST
 
 @RestController
 class MappingController {
 
     private Logger log = LoggerFactory.getLogger(MappingController.class)
+    private JsonSlurper json = new JsonSlurper()
 
     @Autowired
     private LookupStrategy strategy
@@ -55,7 +60,7 @@ class MappingController {
 
         ThreePidType type = ThreePidType.valueOf(medium)
 
-        LookupRequest lookupRequest = new LookupRequest()
+        SingleLookupRequest lookupRequest = new SingleLookupRequest()
         lookupRequest.setRequester(remote)
         lookupRequest.setType(type)
         lookupRequest.setThreePid(address)
@@ -75,4 +80,27 @@ class MappingController {
         return JsonOutput.toJson(lookup)
     }
 
+    @RequestMapping(value = "/_matrix/identity/api/v1/bulk_lookup", method = POST)
+    String bulkLookup(HttpServletRequest request) {
+        String remote = StringUtils.defaultIfBlank(request.getHeader("X-FORWARDED-FOR"), request.getRemoteAddr())
+        log.info("Got request from {}", remote)
+
+        BulkLookupRequest lookupRequest = new BulkLookupRequest()
+        lookupRequest.setRequester(remote)
+
+        ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(request.getInputStream().getText())
+        List<ThreePidMapping> mappings = new ArrayList<>()
+        for (List<String> mappingRaw : input.getThreepids()) {
+            ThreePidMapping mapping = new ThreePidMapping()
+            mapping.setMedium(mappingRaw.get(0))
+            mapping.setValue(mappingRaw.get(1))
+            mappings.add(mapping)
+        }
+        lookupRequest.setMappings(mappings)
+
+        ClientBulkLookupAnswer answer = new ClientBulkLookupAnswer()
+        answer.addAll(strategy.find(lookupRequest))
+        return JsonOutput.toJson(answer)
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java

@@ -0,0 +1,35 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+public abstract class ALookupRequest {
+
+    private String requester;
+
+    public String getRequester() {
+        return requester;
+    }
+
+    public void setRequester(String requester) {
+        this.requester = requester;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java

@@ -0,0 +1,37 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import java.util.List;
+
+public class BulkLookupRequest extends ALookupRequest {
+
+    private List<ThreePidMapping> mappings;
+
+    public List<ThreePidMapping> getMappings() {
+        return mappings;
+    }
+
+    public void setMappings(List<ThreePidMapping> mappings) {
+        this.mappings = mappings;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy

@@ -22,20 +22,11 @@ package io.kamax.mxisd.lookup
 
 import io.kamax.mxisd.api.ThreePidType
 
-class LookupRequest {
+class SingleLookupRequest extends ALookupRequest {
 
-    private String requester
     private ThreePidType type
     private String threePid
 
-    String getRequester() {
-        return requester
-    }
-
-    void setRequester(String requester) {
-        this.requester = requester
-    }
-
     ThreePidType getType() {
         return type
     }

src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java

@@ -0,0 +1,77 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import groovy.json.JsonOutput;
+
+public class ThreePidMapping {
+
+    private String medium;
+    private String value;
+    private String mxid;
+
+    public String getMedium() {
+        return medium;
+    }
+
+    public void setMedium(String medium) {
+        this.medium = medium;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+
+    public String getMxid() {
+        return mxid;
+    }
+
+    public void setMxid(String mxid) {
+        this.mxid = mxid;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreePidMapping that = (ThreePidMapping) o;
+
+        if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false;
+        return value != null ? value.equals(that.value) : that.value == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = medium != null ? medium.hashCode() : 0;
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return JsonOutput.toJson(this);
+    }
+}

src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy

@@ -22,7 +22,8 @@ package io.kamax.mxisd.lookup.provider
 
 import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.ServerConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.apache.commons.lang.StringUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -32,6 +33,10 @@ import org.xbill.DNS.Lookup
 import org.xbill.DNS.SRVRecord
 import org.xbill.DNS.Type
 
+import java.util.concurrent.ForkJoinPool
+import java.util.concurrent.RecursiveTask
+import java.util.function.Function
+
 @Component
 class DnsLookupProvider extends RemoteIdentityServerProvider {
 
@@ -45,23 +50,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
         return 10
     }
 
-    @Override
+    String getSrvRecordName(String domain) {
-    Optional<?> find(LookupRequest request) {
+        return "_matrix-identity._tcp." + domain
-        log.info("Performing DNS lookup for {}", request.getThreePid())
+    }
-        if (ThreePidType.email != request.getType()) {
+
-            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+    Optional<String> getDomain(String email) {
+        int atIndex = email.lastIndexOf("@")
+        if (atIndex == -1) {
             return Optional.empty()
         }
 
-        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        return Optional.of(email.substring(atIndex + 1))
-        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+    }
+
+    // TODO use caching mechanism
+    Optional<String> findIdentityServerForDomain(String domain) {
         if (StringUtils.equals(srvCfg.getName(), domain)) {
-            log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?")
+            log.info("We are authoritative for {}, no remote lookup", domain)
             return Optional.empty()
         }
 
         log.info("Performing SRV lookup")
-        String lookupDns = "_matrix-identity._tcp." + domain
+        String lookupDns = getSrvRecordName(domain)
         log.info("Lookup name: {}", lookupDns)
 
         SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run()
@@ -78,11 +88,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
             for (SRVRecord record : records) {
                 log.info("Found SRV record: {}", record.toString())
                 String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}"
-                Optional<?> answer = find(baseUrl, request.getType(), request.getThreePid())
+                if (isUsableIdentityServer(baseUrl)) {
-                if (answer.isPresent()) {
+                    log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
-                    return answer
+                    return Optional.of(baseUrl)
                 } else {
-                    log.info("No mapping found at {}", baseUrl)
+                    log.info("{} is not a usable Identity Server", baseUrl)
                 }
             }
         } else {
@@ -91,7 +101,116 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
 
         log.info("Performing basic lookup using domain name {}", domain)
         String baseUrl = "https://" + domain
-        return find(baseUrl, request.getType(), request.getThreePid())
+        if (isUsableIdentityServer(baseUrl)) {
+            log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
+            return Optional.of(baseUrl)
+        } else {
+            log.info("{} is not a usable Identity Server", baseUrl)
+            return Optional.empty()
+        }
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        log.info("Performing DNS lookup for {}", request.getThreePid())
+        if (ThreePidType.email != request.getType()) {
+            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+            return Optional.empty()
+        }
+
+        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+        Optional<String> baseUrl = findIdentityServerForDomain(domain)
+
+        if (baseUrl.isPresent()) {
+            return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid())
+        }
+
+        return Optional.empty()
+    }
+
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        Map<String, List<ThreePidMapping>> domains = new HashMap<>()
+
+        for (ThreePidMapping mapping : mappings) {
+            if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) {
+                log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue())
+                continue
+            }
+
+            Optional<String> domainOpt = getDomain(mapping.getValue())
+            if (!domainOpt.isPresent()) {
+                log.warn("No domain for 3PID {}", mapping.getValue())
+                continue
+            }
+
+            String domain = domainOpt.get()
+            List<ThreePidMapping> domainMappings = domains.computeIfAbsent(domain, new Function<String, List<ThreePidMapping>>() {
+
+                @Override
+                List<ThreePidMapping> apply(String s) {
+                    return new ArrayList<>()
+                }
+
+            })
+            domainMappings.add(mapping)
+        }
+
+        log.info("Looking mappings across {} domains", domains.keySet().size())
+        ForkJoinPool pool = new ForkJoinPool()
+        RecursiveTask<List<ThreePidMapping>> task = new RecursiveTask<List<ThreePidMapping>>() {
+
+            @Override
+            protected List<ThreePidMapping> compute() {
+                List<ThreePidMapping> mappingsFound = new ArrayList<>()
+                List<DomainBulkLookupTask> tasks = new ArrayList<>()
+
+                for (String domain : domains.keySet()) {
+                    DomainBulkLookupTask domainTask = new DomainBulkLookupTask(domain, domains.get(domain))
+                    domainTask.fork()
+                    tasks.add(domainTask)
+                }
+
+                for (DomainBulkLookupTask task : tasks) {
+                    mappingsFound.addAll(task.join())
+                }
+
+                return mappingsFound
+            }
+        }
+        pool.submit(task)
+        pool.shutdown()
+
+        List<ThreePidMapping> mappingsFound = task.join()
+        log.info("Found {} mappings overall", mappingsFound.size())
+        return mappingsFound
+    }
+
+    private class DomainBulkLookupTask extends RecursiveTask<List<ThreePidMapping>> {
+
+        private String domain
+        private List<ThreePidMapping> mappings
+
+        DomainBulkLookupTask(String domain, List<ThreePidMapping> mappings) {
+            this.domain = domain
+            this.mappings = mappings
+        }
+
+        @Override
+        protected List<ThreePidMapping> compute() {
+            List<ThreePidMapping> domainMappings = new ArrayList<>()
+
+            Optional<String> baseUrl = findIdentityServerForDomain(domain)
+            if (!baseUrl.isPresent()) {
+                log.info("No usable Identity server for domain {}", domain)
+            } else {
+                domainMappings.addAll(find(baseUrl.get(), mappings))
+                log.info("Found {} mappings in domain {}", domainMappings.size(), domain)
+            }
+
+            return domainMappings
+        }
     }
 
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy

@@ -21,13 +21,18 @@
 package io.kamax.mxisd.lookup.provider
 
 import io.kamax.mxisd.config.ForwardConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
 class ForwarderProvider extends RemoteIdentityServerProvider {
 
+    private Logger log = LoggerFactory.getLogger(ForwarderProvider.class)
+
     @Autowired
     private ForwardConfig cfg
 
@@ -37,7 +42,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
     }
 
     @Override
-    Optional<?> find(LookupRequest request) {
+    Optional<?> find(SingleLookupRequest request) {
         for (String root : cfg.getServers()) {
             Optional<?> answer = find(root, request.getType(), request.getThreePid())
             if (answer.isPresent()) {
@@ -48,4 +53,21 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsToDo = new ArrayList<>(mappings)
+        List<ThreePidMapping> mappingsFoundGlobal = new ArrayList<>()
+
+        for (String root : cfg.getServers()) {
+            log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo)
+            log.info("Querying {}", root)
+            List<ThreePidMapping> mappingsFound = find(root, mappingsToDo)
+            log.info("{} returned {} mappings", root, mappingsFound.size())
+            mappingsFoundGlobal.addAll(mappingsFound)
+            mappingsToDo.removeAll(mappingsFound)
+        }
+
+        return mappingsFoundGlobal
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy

@@ -20,9 +20,11 @@
 
 package io.kamax.mxisd.lookup.provider
 
+import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.LdapConfig
 import io.kamax.mxisd.config.ServerConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.apache.commons.lang.StringUtils
 import org.apache.directory.api.ldap.model.cursor.EntryCursor
 import org.apache.directory.api.ldap.model.entry.Attribute
@@ -58,21 +60,14 @@ class LdapProvider implements ThreePidProvider {
         return 20
     }
 
-    @Override
+    Optional<String> lookup(LdapConnection conn, ThreePidType medium, String value) {
-    Optional<?> find(LookupRequest request) {
+        Optional<String> queryOpt = ldapCfg.getMapping(medium)
-        log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}")
-
-        LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
-        try {
-            conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
-
-            Optional<String> queryOpt = ldapCfg.getMapping(request.getType())
         if (!queryOpt.isPresent()) {
-                log.warn("{} is not a supported 3PID type for LDAP lookup", request.getType())
+            log.warn("{} is not a supported 3PID type for LDAP lookup", medium)
             return Optional.empty()
         }
 
-            String searchQuery = queryOpt.get().replaceAll("%3pid", request.getThreePid())
+        String searchQuery = queryOpt.get().replaceAll("%3pid", value)
         EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
         try {
             if (cursor.next()) {
@@ -96,19 +91,35 @@ class LdapProvider implements ThreePidProvider {
                     }
 
                     log.info("Found a match in LDAP")
-                        return Optional.of([
+                    return Optional.of(matrixId.toString())
-                                address   : request.getThreePid(),
-                                medium    : request.getType(),
-                                mxid      : matrixId.toString(),
-                                not_before: 0,
-                                not_after : 9223372036854775807,
-                                ts        : 0
-                        ])
                 }
             }
         } finally {
             cursor.close()
         }
+
+        return Optional.empty()
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}")
+
+        LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
+        try {
+            conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
+
+            Optional<String> mxid = lookup(conn, request.getType(), request.getThreePid())
+            if (mxid.isPresent()) {
+                return Optional.of([
+                        address   : request.getThreePid(),
+                        medium    : request.getType(),
+                        mxid      : mxid.get(),
+                        not_before: 0,
+                        not_after : 9223372036854775807,
+                        ts        : 0
+                ])
+            }
         } finally {
             conn.close()
         }
@@ -117,4 +128,32 @@ class LdapProvider implements ThreePidProvider {
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        log.info("Looking up {} mappings", mappings.size())
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
+        try {
+            conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
+
+            for (ThreePidMapping mapping : mappings) {
+                try {
+                    ThreePidType type = ThreePidType.valueOf(mapping.getMedium())
+                    Optional<String> mxid = lookup(conn, type, mapping.getValue())
+                    if (mxid.isPresent()) {
+                        mapping.setMxid(mxid.get())
+                        mappingsFound.add(mapping)
+                    }
+                } catch (IllegalArgumentException e) {
+                    log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium())
+                }
+            }
+        } finally {
+            conn.close()
+        }
+
+        return mappingsFound
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy

@@ -21,13 +21,26 @@
 package io.kamax.mxisd.lookup.provider
 
 import groovy.json.JsonException
+import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
+import org.apache.http.HttpEntity
+import org.apache.http.HttpResponse
+import org.apache.http.client.HttpClient
+import org.apache.http.client.entity.EntityBuilder
+import org.apache.http.client.methods.HttpPost
+import org.apache.http.entity.ContentType
+import org.apache.http.impl.client.HttpClients
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 abstract class RemoteIdentityServerProvider implements ThreePidProvider {
 
+    public static final String THREEPID_TEST_MEDIUM = "email"
+    public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org"
+
     private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class)
 
     private JsonSlurper json = new JsonSlurper()
@@ -37,6 +50,50 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         return false
     }
 
+    boolean isUsableIdentityServer(String remote) {
+        try {
+            HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                    "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}"
+            ).openConnection()
+            // TODO turn this into a configuration property
+            rootSrvConn.setConnectTimeout(2000)
+
+            if (rootSrvConn.getResponseCode() != 200) {
+                return false
+            }
+
+            def output = json.parseText(rootSrvConn.getInputStream().getText())
+            if (output['address']) {
+                return false
+            }
+
+            return true
+        } catch (IOException | JsonException e) {
+            log.info("{} is not a usable Identity Server: {}", remote, e.getMessage())
+            return false
+        }
+    }
+
+    Optional<ThreePidMapping> performLookup(String remote, String medium, String address) throws IOException, JsonException {
+        HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}"
+        ).openConnection()
+
+        def output = json.parseText(rootSrvConn.getInputStream().getText())
+        if (output['address']) {
+            log.info("Found 3PID mapping: {}", output)
+
+            ThreePidMapping mapping = new ThreePidMapping()
+            mapping.setMedium(output['medium'].toString())
+            mapping.setValue(output['address'].toString())
+            mapping.setMxid(output['mxid'].toString())
+            return Optional.of(mapping)
+        }
+
+        log.info("Empty 3PID mapping from {}", remote)
+        return Optional.empty()
+    }
+
     Optional<?> find(String remote, ThreePidType type, String threePid) {
         log.info("Looking up {} 3PID {} using {}", type, threePid, remote)
 
@@ -62,4 +119,51 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         }
     }
 
+    List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest()
+        mappingRequest.setMappings(mappings)
+
+        String url = "${remote}/_matrix/identity/api/v1/bulk_lookup"
+        HttpClient client = HttpClients.createDefault()
+        try {
+            HttpPost request = new HttpPost(url)
+            request.setEntity(
+                    EntityBuilder.create()
+                            .setText(JsonOutput.toJson(mappingRequest))
+                            .setContentType(ContentType.APPLICATION_JSON)
+                            .build()
+            )
+
+            HttpResponse response = client.execute(request)
+            try {
+                if (response.getStatusLine().getStatusCode() != 200) {
+                    log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode())
+                    return mappingsFound
+                }
+
+                HttpEntity entity = response.getEntity()
+                if (entity != null) {
+                    ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText())
+                    for (List<String> mappingRaw : input.getThreepids()) {
+                        ThreePidMapping mapping = new ThreePidMapping()
+                        mapping.setMedium(mappingRaw.get(0))
+                        mapping.setValue(mappingRaw.get(1))
+                        mapping.setMxid(mappingRaw.get(2))
+                        mappingsFound.add(mapping)
+                    }
+                } else {
+                    log.info("HTTP response from {} was empty", remote)
+                }
+
+                return mappingsFound
+            } finally {
+                response.close()
+            }
+        } finally {
+            client.close()
+        }
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy

@@ -20,7 +20,8 @@
 
 package io.kamax.mxisd.lookup.provider
 
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 
 interface ThreePidProvider {
 
@@ -31,6 +32,8 @@ interface ThreePidProvider {
      */
     int getPriority() // Should not be here but let's KISS for now
 
-    Optional<?> find(LookupRequest request)
+    Optional<?> find(SingleLookupRequest request)
+
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy

@@ -20,10 +20,14 @@
 
 package io.kamax.mxisd.lookup.strategy
 
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 
 interface LookupStrategy {
 
-    Optional<?> find(LookupRequest request)
+    Optional<?> find(SingleLookupRequest request)
+
+    List<ThreePidMapping> find(BulkLookupRequest requests)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy

@@ -22,7 +22,10 @@ package io.kamax.mxisd.lookup.strategy
 
 import edazdarevic.commons.net.CIDRUtils
 import io.kamax.mxisd.config.RecursiveLookupConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.ALookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import io.kamax.mxisd.lookup.provider.ThreePidProvider
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -63,8 +66,9 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         }
     }
 
-    @Override
+    List<ThreePidProvider> listUsableProviders(ALookupRequest request) {
-    Optional<?> find(LookupRequest request) {
+        List<ThreePidProvider> usableProviders = new ArrayList<>()
+
         boolean canRecurse = false
         if (recursiveCfg.isEnabled()) {
             log.debug("Checking {} CIDRs for recursion", allowedCidr.size())
@@ -80,17 +84,48 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         }
 
         log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse)
-
         for (ThreePidProvider provider : providers) {
             if (provider.isLocal() || canRecurse) {
+                usableProviders.add(provider)
+            }
+        }
+
+        return usableProviders
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        for (ThreePidProvider provider : listUsableProviders(request)) {
             Optional<?> lookupDataOpt = provider.find(request)
             if (lookupDataOpt.isPresent()) {
                 return lookupDataOpt
             }
         }
-        }
 
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> find(BulkLookupRequest request) {
+        List<ThreePidMapping> mapToDo = new ArrayList<>(request.getMappings())
+        List<ThreePidMapping> mapFoundAll = new ArrayList<>()
+
+        for (ThreePidProvider provider : listUsableProviders(request)) {
+            if (mapToDo.isEmpty()) {
+                log.info("No more mappings to lookup")
+                break
+            } else {
+                log.info("{} mappings remaining overall", mapToDo.size())
+            }
+
+            log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName())
+            List<ThreePidMapping> mapFound = provider.populate(mapToDo)
+            log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size())
+            mapFoundAll.addAll(mapFound)
+            mapToDo.removeAll(mapFound)
+        }
+
+        return mapFoundAll
+    }
+
 }