Bulk lookup implementation, part 1

- LDAP bulk lookup

src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java

@@ -0,0 +1,44 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.controller.v1;
+
+import io.kamax.mxisd.lookup.ThreePidMapping;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+public class ClientBulkLookupAnswer {
+
+    private List<List<String>> threepids = new ArrayList<>();
+
+    public void addAll(Collection<ThreePidMapping> mappings) {
+        for (ThreePidMapping mapping : mappings) {
+            threepids.add(Arrays.asList(mapping.getMedium(), mapping.getValue(), mapping.getMxid()));
+        }
+    }
+
+    public List<List<String>> getThreepids() {
+        return threepids;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy

@@ -0,0 +1,35 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.controller.v1
+
+class ClientBulkLookupRequest {
+
+    private List<List<String>> threepids
+
+    List<List<String>> getThreepids() {
+        return threepids
+    }
+
+    void setThreepids(List<List<String>> threepids) {
+        this.threepids = threepids
+    }
+
+}

src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy

@@ -21,8 +21,11 @@
 package io.kamax.mxisd.controller.v1
 
 import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import io.kamax.mxisd.lookup.strategy.LookupStrategy
 import io.kamax.mxisd.signature.SignatureManager
 import org.apache.commons.lang.StringUtils
@@ -36,11 +39,13 @@ import org.springframework.web.bind.annotation.RestController
 import javax.servlet.http.HttpServletRequest
 
 import static org.springframework.web.bind.annotation.RequestMethod.GET
+import static org.springframework.web.bind.annotation.RequestMethod.POST
 
 @RestController
 class MappingController {
 
     private Logger log = LoggerFactory.getLogger(MappingController.class)
+    private JsonSlurper json = new JsonSlurper()
 
     @Autowired
     private LookupStrategy strategy
@@ -55,7 +60,7 @@ class MappingController {
 
         ThreePidType type = ThreePidType.valueOf(medium)
 
-        LookupRequest lookupRequest = new LookupRequest()
+        SingleLookupRequest lookupRequest = new SingleLookupRequest()
         lookupRequest.setRequester(remote)
         lookupRequest.setType(type)
         lookupRequest.setThreePid(address)
@@ -75,4 +80,27 @@ class MappingController {
         return JsonOutput.toJson(lookup)
     }
 
+    @RequestMapping(value = "/_matrix/identity/api/v1/bulk_lookup", method = POST)
+    String bulkLookup(HttpServletRequest request) {
+        String remote = StringUtils.defaultIfBlank(request.getHeader("X-FORWARDED-FOR"), request.getRemoteAddr())
+        log.info("Got request from {}", remote)
+
+        BulkLookupRequest lookupRequest = new BulkLookupRequest()
+        lookupRequest.setRequester(remote)
+
+        ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(request.getInputStream().getText())
+        List<ThreePidMapping> mappings = new ArrayList<>()
+        for (List<String> mappingRaw : input.getThreepids()) {
+            ThreePidMapping mapping = new ThreePidMapping()
+            mapping.setMedium(mappingRaw.get(0))
+            mapping.setValue(mappingRaw.get(1))
+            mappings.add(mapping)
+        }
+        lookupRequest.setMappings(mappings)
+
+        ClientBulkLookupAnswer answer = new ClientBulkLookupAnswer()
+        answer.addAll(strategy.find(lookupRequest))
+        return JsonOutput.toJson(answer)
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java

@@ -0,0 +1,35 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+public abstract class ALookupRequest {
+
+    private String requester;
+
+    public String getRequester() {
+        return requester;
+    }
+
+    public void setRequester(String requester) {
+        this.requester = requester;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java

@@ -0,0 +1,37 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import java.util.List;
+
+public class BulkLookupRequest extends ALookupRequest {
+
+    private List<ThreePidMapping> mappings;
+
+    public List<ThreePidMapping> getMappings() {
+        return mappings;
+    }
+
+    public void setMappings(List<ThreePidMapping> mappings) {
+        this.mappings = mappings;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy

@@ -22,20 +22,11 @@ package io.kamax.mxisd.lookup
 
 import io.kamax.mxisd.api.ThreePidType
 
-class LookupRequest {
+class SingleLookupRequest extends ALookupRequest {
 
-    private String requester
     private ThreePidType type
     private String threePid
 
-    String getRequester() {
-        return requester
-    }
-
-    void setRequester(String requester) {
-        this.requester = requester
-    }
-
     ThreePidType getType() {
         return type
     }

src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java

@@ -0,0 +1,53 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+public class ThreePidMapping {
+
+    private String medium;
+    private String value;
+    private String mxid;
+
+    public String getMedium() {
+        return medium;
+    }
+
+    public void setMedium(String medium) {
+        this.medium = medium;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+
+    public String getMxid() {
+        return mxid;
+    }
+
+    public void setMxid(String mxid) {
+        this.mxid = mxid;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy

@@ -22,7 +22,8 @@ package io.kamax.mxisd.lookup.provider
 
 import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.ServerConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.apache.commons.lang.StringUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -46,7 +47,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
     }
 
     @Override
-    Optional<?> find(LookupRequest request) {
+    Optional<?> find(SingleLookupRequest request) {
         log.info("Performing DNS lookup for {}", request.getThreePid())
         if (ThreePidType.email != request.getType()) {
             log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
@@ -94,4 +95,13 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
         return find(baseUrl, request.getType(), request.getThreePid())
     }
 
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        // TODO
+
+        return mappingsFound
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy

@@ -21,7 +21,8 @@
 package io.kamax.mxisd.lookup.provider
 
 import io.kamax.mxisd.config.ForwardConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
@@ -37,7 +38,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
     }
 
     @Override
-    Optional<?> find(LookupRequest request) {
+    Optional<?> find(SingleLookupRequest request) {
         for (String root : cfg.getServers()) {
             Optional<?> answer = find(root, request.getType(), request.getThreePid())
             if (answer.isPresent()) {
@@ -48,4 +49,13 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        // TODO
+
+        return mappingsFound
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy

@@ -20,9 +20,11 @@
 
 package io.kamax.mxisd.lookup.provider
 
+import io.kamax.mxisd.api.ThreePidType
 import io.kamax.mxisd.config.LdapConfig
 import io.kamax.mxisd.config.ServerConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.apache.commons.lang.StringUtils
 import org.apache.directory.api.ldap.model.cursor.EntryCursor
 import org.apache.directory.api.ldap.model.entry.Attribute
@@ -58,56 +60,65 @@ class LdapProvider implements ThreePidProvider {
         return 20
     }
 
+    Optional<String> lookup(LdapConnection conn, ThreePidType medium, String value) {
+        Optional<String> queryOpt = ldapCfg.getMapping(medium)
+        if (!queryOpt.isPresent()) {
+            log.warn("{} is not a supported 3PID type for LDAP lookup", medium)
+            return Optional.empty()
+        }
+
+        String searchQuery = queryOpt.get().replaceAll("%3pid", value)
+        EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
+        try {
+            if (cursor.next()) {
+                Attribute attribute = cursor.get().get(ldapCfg.getAttribute())
+                if (attribute != null) {
+                    String data = attribute.get().toString()
+                    if (data.length() < 1) {
+                        log.warn("Bind was found but value is empty")
+                        return Optional.empty()
+                    }
+
+                    StringBuilder matrixId = new StringBuilder()
+                    // TODO Should we turn this block into a map of functions?
+                    if (StringUtils.equals(UID, ldapCfg.getType())) {
+                        matrixId.append("@").append(data).append(":").append(srvCfg.getName())
+                    } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) {
+                        matrixId.append(data)
+                    } else {
+                        log.warn("Bind was found but type ${ldapCfg.getType()} is not supported")
+                        return Optional.empty()
+                    }
+
+                    log.info("Found a match in LDAP")
+                    return Optional.of(matrixId.toString())
+                }
+            }
+        } finally {
+            cursor.close()
+        }
+
+        return Optional.empty()
+    }
+
     @Override
-    Optional<?> find(LookupRequest request) {
+    Optional<?> find(SingleLookupRequest request) {
         log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}")
 
         LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
         try {
             conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
 
-            Optional<String> queryOpt = ldapCfg.getMapping(request.getType())
+            Optional<String> mxid = lookup(conn, request.getType(), request.getThreePid())
-            if (!queryOpt.isPresent()) {
+            if (mxid.isPresent()) {
-                log.warn("{} is not a supported 3PID type for LDAP lookup", request.getType())
+                return Optional.of([
-                return Optional.empty()
+                        address   : request.getThreePid(),
-            }
+                        medium    : request.getType(),
-
+                        mxid      : mxid.get(),
-            String searchQuery = queryOpt.get().replaceAll("%3pid", request.getThreePid())
+                        not_before: 0,
-            EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
+                        not_after : 9223372036854775807,
-            try {
+                        ts        : 0
-                if (cursor.next()) {
+                ])
-                    Attribute attribute = cursor.get().get(ldapCfg.getAttribute())
-                    if (attribute != null) {
-                        String data = attribute.get().toString()
-                        if (data.length() < 1) {
-                            log.warn("Bind was found but value is empty")
-                            return Optional.empty()
-                        }
-
-                        StringBuilder matrixId = new StringBuilder()
-                        // TODO Should we turn this block into a map of functions?
-                        if (StringUtils.equals(UID, ldapCfg.getType())) {
-                            matrixId.append("@").append(data).append(":").append(srvCfg.getName())
-                        } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) {
-                            matrixId.append(data)
-                        } else {
-                            log.warn("Bind was found but type ${ldapCfg.getType()} is not supported")
-                            return Optional.empty()
-                        }
-
-                        log.info("Found a match in LDAP")
-                        return Optional.of([
-                                address   : request.getThreePid(),
-                                medium    : request.getType(),
-                                mxid      : matrixId.toString(),
-                                not_before: 0,
-                                not_after : 9223372036854775807,
-                                ts        : 0
-                        ])
-                    }
-                }
-            } finally {
-                cursor.close()
             }
         } finally {
             conn.close()
@@ -117,4 +128,32 @@ class LdapProvider implements ThreePidProvider {
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        log.info("Looking up {} mappings", mappings.size())
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
+        try {
+            conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
+
+            for (ThreePidMapping mapping : mappings) {
+                try {
+                    ThreePidType type = ThreePidType.valueOf(mapping.getMedium())
+                    Optional<String> mxid = lookup(conn, type, mapping.getValue())
+                    if (mxid.isPresent()) {
+                        mapping.setMxid(mxid.get())
+                        mappingsFound.add(mapping)
+                    }
+                } catch (IllegalArgumentException e) {
+                    log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium())
+                }
+            }
+        } finally {
+            conn.close()
+        }
+
+        return mappingsFound
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy

@@ -23,6 +23,7 @@ package io.kamax.mxisd.lookup.provider
 import groovy.json.JsonException
 import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.lookup.ThreePidMapping
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
@@ -62,4 +63,12 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         }
     }
 
+    List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+
+        // TODO
+
+        return mappingsFound
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy

@@ -20,7 +20,8 @@
 
 package io.kamax.mxisd.lookup.provider
 
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 
 interface ThreePidProvider {
 
@@ -31,6 +32,8 @@ interface ThreePidProvider {
      */
     int getPriority() // Should not be here but let's KISS for now
 
-    Optional<?> find(LookupRequest request)
+    Optional<?> find(SingleLookupRequest request)
+
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy

@@ -20,10 +20,14 @@
 
 package io.kamax.mxisd.lookup.strategy
 
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 
 interface LookupStrategy {
 
-    Optional<?> find(LookupRequest request)
+    Optional<?> find(SingleLookupRequest request)
+
+    List<ThreePidMapping> find(BulkLookupRequest requests)
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy

@@ -22,7 +22,10 @@ package io.kamax.mxisd.lookup.strategy
 
 import edazdarevic.commons.net.CIDRUtils
 import io.kamax.mxisd.config.RecursiveLookupConfig
-import io.kamax.mxisd.lookup.LookupRequest
+import io.kamax.mxisd.lookup.ALookupRequest
+import io.kamax.mxisd.lookup.BulkLookupRequest
+import io.kamax.mxisd.lookup.SingleLookupRequest
+import io.kamax.mxisd.lookup.ThreePidMapping
 import io.kamax.mxisd.lookup.provider.ThreePidProvider
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -63,8 +66,9 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         }
     }
 
-    @Override
+    List<ThreePidProvider> listUsableProviders(ALookupRequest request) {
-    Optional<?> find(LookupRequest request) {
+        List<ThreePidProvider> usableProviders = new ArrayList<>()
+
         boolean canRecurse = false
         if (recursiveCfg.isEnabled()) {
             log.debug("Checking {} CIDRs for recursion", allowedCidr.size())
@@ -80,17 +84,44 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
         }
 
         log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse)
-
         for (ThreePidProvider provider : providers) {
             if (provider.isLocal() || canRecurse) {
-                Optional<?> lookupDataOpt = provider.find(request)
+                usableProviders.add(provider)
-                if (lookupDataOpt.isPresent()) {
+            }
-                    return lookupDataOpt
+        }
-                }
+
+        return usableProviders
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        for (ThreePidProvider provider : listUsableProviders(request)) {
+            Optional<?> lookupDataOpt = provider.find(request)
+            if (lookupDataOpt.isPresent()) {
+                return lookupDataOpt
             }
         }
 
         return Optional.empty()
     }
 
+    @Override
+    List<ThreePidMapping> find(BulkLookupRequest request) {
+        List<ThreePidMapping> mapToDo = new ArrayList<>(request.getMappings())
+        List<ThreePidMapping> mapFoundAll = new ArrayList<>()
+
+        for (ThreePidProvider provider : listUsableProviders(request)) {
+            if (mapToDo.isEmpty()) {
+                log.info("No more mappings to lookup")
+                break
+            }
+
+            List<ThreePidMapping> mapFound = provider.populate(mapToDo)
+            mapFoundAll.addAll(mapFound)
+            mapToDo.removeAll(mapFound)
+        }
+
+        return mapFoundAll;
+    }
+
 }