Bulk lookup implementation, part 2

- Remote IS bulk lookup

build.gradle

@@ -54,6 +54,9 @@ dependencies {
     // DNS lookups
     compile 'dnsjava:dnsjava:2.1.8'
 
+    // HTTP connections
+    compile 'org.apache.httpcomponents:httpclient:4.5.3'
+
     testCompile 'junit:junit:4.12'
 }
 

src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy

@@ -20,9 +20,11 @@
 
 package io.kamax.mxisd.controller.v1
 
+import io.kamax.mxisd.lookup.ThreePidMapping
+
 class ClientBulkLookupRequest {
 
-    private List<List<String>> threepids
+    private List<List<String>> threepids = new ArrayList<>()
 
     List<List<String>> getThreepids() {
         return threepids
@@ -32,4 +34,13 @@ class ClientBulkLookupRequest {
         this.threepids = threepids
     }
 
+    void setMappings(List<ThreePidMapping> mappings) {
+        for (ThreePidMapping mapping : mappings) {
+            List<String> threepid = new ArrayList<>()
+            threepid.add(mapping.getMedium())
+            threepid.add(mapping.getValue())
+            threepids.add(threepid)
+        }
+    }
+
 }

src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java

@@ -20,6 +20,8 @@
 
 package io.kamax.mxisd.lookup;
 
+import groovy.json.JsonOutput;
+
 public class ThreePidMapping {
 
     private String medium;
@@ -50,4 +52,26 @@ public class ThreePidMapping {
         this.mxid = mxid;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreePidMapping that = (ThreePidMapping) o;
+
+        if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false;
+        return value != null ? value.equals(that.value) : that.value == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = medium != null ? medium.hashCode() : 0;
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return JsonOutput.toJson(this);
+    }
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy

@@ -33,6 +33,8 @@ import org.xbill.DNS.Lookup
 import org.xbill.DNS.SRVRecord
 import org.xbill.DNS.Type
 
+import java.util.function.Function
+
 @Component
 class DnsLookupProvider extends RemoteIdentityServerProvider {
 
@@ -46,23 +48,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
         return 10
     }
 
-    @Override
+    String getSrvRecordName(String domain) {
-    Optional<?> find(SingleLookupRequest request) {
+        return "_matrix-identity._tcp." + domain
-        log.info("Performing DNS lookup for {}", request.getThreePid())
+    }
-        if (ThreePidType.email != request.getType()) {
+
-            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+    Optional<String> getDomain(String email) {
+        int atIndex = email.lastIndexOf("@")
+        if (atIndex == -1) {
             return Optional.empty()
         }
 
-        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        return Optional.of(email.substring(atIndex + 1))
-        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+    }
+
+    // TODO use caching mechanism
+    Optional<String> findIdentityServerForDomain(String domain) {
         if (StringUtils.equals(srvCfg.getName(), domain)) {
-            log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?")
+            log.info("We are authoritative for {}, no remote lookup", domain)
             return Optional.empty()
         }
 
         log.info("Performing SRV lookup")
-        String lookupDns = "_matrix-identity._tcp." + domain
+        String lookupDns = getSrvRecordName(domain)
         log.info("Lookup name: {}", lookupDns)
 
         SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run()
@@ -79,11 +86,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
             for (SRVRecord record : records) {
                 log.info("Found SRV record: {}", record.toString())
                 String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}"
-                Optional<?> answer = find(baseUrl, request.getType(), request.getThreePid())
+                if (isUsableIdentityServer(baseUrl)) {
-                if (answer.isPresent()) {
+                    log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
-                    return answer
+                    return Optional.of(baseUrl)
                 } else {
-                    log.info("No mapping found at {}", baseUrl)
+                    log.info("{} is not a usable Identity Server", baseUrl)
                 }
             }
         } else {
@@ -92,15 +99,77 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
 
         log.info("Performing basic lookup using domain name {}", domain)
         String baseUrl = "https://" + domain
-        return find(baseUrl, request.getType(), request.getThreePid())
+        if (isUsableIdentityServer(baseUrl)) {
+            log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
+            return Optional.of(baseUrl)
+        } else {
+            log.info("{} is not a usable Identity Server", baseUrl)
+            return Optional.empty()
+        }
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        log.info("Performing DNS lookup for {}", request.getThreePid())
+        if (ThreePidType.email != request.getType()) {
+            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+            return Optional.empty()
+        }
+
+        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+        Optional<String> baseUrl = findIdentityServerForDomain(domain)
+
+        if (baseUrl.isPresent()) {
+            return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid())
+        }
+
+        return Optional.empty()
     }
 
     @Override
     List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
         List<ThreePidMapping> mappingsFound = new ArrayList<>()
+        Map<String, List<ThreePidMapping>> domains = new HashMap<>()
 
-        // TODO
+        for (ThreePidMapping mapping : mappings) {
+            if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) {
+                log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue())
+                continue
+            }
 
+            Optional<String> domainOpt = getDomain(mapping.getValue())
+            if (!domainOpt.isPresent()) {
+                log.warn("No domain for 3PID {}", mapping.getValue())
+                continue
+            }
+
+            String domain = domainOpt.get()
+            List<ThreePidMapping> domainMappings = domains.computeIfAbsent(domain, new Function<String, List<ThreePidMapping>>() {
+
+                @Override
+                List<ThreePidMapping> apply(String s) {
+                    return new ArrayList<>()
+                }
+
+            })
+            domainMappings.add(mapping)
+        }
+
+        log.info("Looking mappings across {} domains", domains.keySet().size())
+        for (String domain : domains.keySet()) {
+            Optional<String> baseUrl = findIdentityServerForDomain(domain)
+            if (!baseUrl.isPresent()) {
+                log.info("No usable Identity server for domain {}", domain)
+                continue
+            }
+
+            List<ThreePidMapping> domainMappings = find(baseUrl.get(), domains.get(domain))
+            log.info("Found {} mappings in domain {}", domainMappings.size(), domain)
+            mappingsFound.addAll(domainMappings)
+        }
+
+        log.info("Found {} mappings overall", mappingsFound.size())
         return mappingsFound
     }
 

src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy

@@ -23,12 +23,16 @@ package io.kamax.mxisd.lookup.provider
 import io.kamax.mxisd.config.ForwardConfig
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
 @Component
 class ForwarderProvider extends RemoteIdentityServerProvider {
 
+    private Logger log = LoggerFactory.getLogger(ForwarderProvider.class)
+
     @Autowired
     private ForwardConfig cfg
 
@@ -51,11 +55,19 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
 
     @Override
     List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
-        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+        List<ThreePidMapping> mappingsToDo = new ArrayList<>(mappings)
+        List<ThreePidMapping> mappingsFoundGlobal = new ArrayList<>()
 
-        // TODO
+        for (String root : cfg.getServers()) {
+            log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo)
+            log.info("Querying {}", root)
+            List<ThreePidMapping> mappingsFound = find(root, mappingsToDo)
+            log.info("{} returned {} mappings", root, mappingsFound.size())
+            mappingsFoundGlobal.addAll(mappingsFound)
+            mappingsToDo.removeAll(mappingsFound)
+        }
 
-        return mappingsFound
+        return mappingsFoundGlobal
     }
 
 }

src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy

@@ -21,14 +21,26 @@
 package io.kamax.mxisd.lookup.provider
 
 import groovy.json.JsonException
+import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import org.apache.http.HttpEntity
+import org.apache.http.HttpResponse
+import org.apache.http.client.HttpClient
+import org.apache.http.client.entity.EntityBuilder
+import org.apache.http.client.methods.HttpPost
+import org.apache.http.entity.ContentType
+import org.apache.http.impl.client.HttpClients
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
 abstract class RemoteIdentityServerProvider implements ThreePidProvider {
 
+    public static final String THREEPID_TEST_MEDIUM = "email"
+    public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org"
+
     private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class)
 
     private JsonSlurper json = new JsonSlurper()
@@ -38,6 +50,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
         return false
     }
 
+    boolean isUsableIdentityServer(String remote) {
+        try {
+            HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                    "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}"
+            ).openConnection()
+
+            if (rootSrvConn.getResponseCode() != 200) {
+                return false
+            }
+
+            def output = json.parseText(rootSrvConn.getInputStream().getText())
+            if (output['address']) {
+                return false
+            }
+
+            return true
+        } catch (IOException | JsonException e) {
+            log.info("{} is not a usable Identity Server: {}", remote, e.getMessage())
+            return false
+        }
+    }
+
+    Optional<ThreePidMapping> performLookup(String remote, String medium, String address) throws IOException, JsonException {
+        HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}"
+        ).openConnection()
+
+        def output = json.parseText(rootSrvConn.getInputStream().getText())
+        if (output['address']) {
+            log.info("Found 3PID mapping: {}", output)
+
+            ThreePidMapping mapping = new ThreePidMapping()
+            mapping.setMedium(output['medium'].toString())
+            mapping.setValue(output['address'].toString())
+            mapping.setMxid(output['mxid'].toString())
+            return Optional.of(mapping)
+        }
+
+        log.info("Empty 3PID mapping from {}", remote)
+        return Optional.empty()
+    }
+
     Optional<?> find(String remote, ThreePidType type, String threePid) {
         log.info("Looking up {} 3PID {} using {}", type, threePid, remote)
 
@@ -66,9 +120,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
     List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
         List<ThreePidMapping> mappingsFound = new ArrayList<>()
 
-        // TODO
+        ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest()
+        mappingRequest.setMappings(mappings)
 
-        return mappingsFound
+        String url = "${remote}/_matrix/identity/api/v1/bulk_lookup"
+        HttpClient client = HttpClients.createDefault()
+        try {
+            HttpPost request = new HttpPost(url)
+            request.setEntity(
+                    EntityBuilder.create()
+                            .setText(JsonOutput.toJson(mappingRequest))
+                            .setContentType(ContentType.APPLICATION_JSON)
+                            .build()
+            )
+
+            HttpResponse response = client.execute(request)
+            try {
+                if (response.getStatusLine().getStatusCode() != 200) {
+                    log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode())
+                    return mappingsFound
+                }
+
+                HttpEntity entity = response.getEntity()
+                if (entity != null) {
+                    ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText())
+                    for (List<String> mappingRaw : input.getThreepids()) {
+                        ThreePidMapping mapping = new ThreePidMapping()
+                        mapping.setMedium(mappingRaw.get(0))
+                        mapping.setValue(mappingRaw.get(1))
+                        mapping.setMxid(mappingRaw.get(2))
+                        mappingsFound.add(mapping)
+                    }
+                } else {
+                    log.info("HTTP response from {} was empty", remote)
+                }
+
+                return mappingsFound
+            } finally {
+                response.close()
+            }
+        } finally {
+            client.close()
+        }
     }
 
 }

src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy

@@ -114,14 +114,18 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
             if (mapToDo.isEmpty()) {
                 log.info("No more mappings to lookup")
                 break
+            } else {
+                log.info("{} mappings remaining overall", mapToDo.size())
             }
 
+            log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName())
             List<ThreePidMapping> mapFound = provider.populate(mapToDo)
+            log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size())
             mapFoundAll.addAll(mapFound)
             mapToDo.removeAll(mapFound)
         }
 
-        return mapFoundAll;
+        return mapFoundAll
     }
 
 }