Bulk lookup implementation, part 2

- Remote IS bulk lookup
2017-04-04 01:11:32 +02:00
parent 9cfa008422
commit d0b9f6774d
7 changed files with 238 additions and 22 deletions
--- a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy
+++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy
@@ -20,9 +20,11 @@

 package io.kamax.mxisd.controller.v1

+import io.kamax.mxisd.lookup.ThreePidMapping
+
 class ClientBulkLookupRequest {

-    private List<List<String>> threepids
+    private List<List<String>> threepids = new ArrayList<>()

    List<List<String>> getThreepids() {
        return threepids
@@ -32,4 +34,13 @@ class ClientBulkLookupRequest {
        this.threepids = threepids
    }

+    void setMappings(List<ThreePidMapping> mappings) {
+        for (ThreePidMapping mapping : mappings) {
+            List<String> threepid = new ArrayList<>()
+            threepid.add(mapping.getMedium())
+            threepid.add(mapping.getValue())
+            threepids.add(threepid)
+        }
+    }
+
 }
--- a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java
+++ b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java
@@ -20,6 +20,8 @@

 package io.kamax.mxisd.lookup;

+import groovy.json.JsonOutput;
+
 public class ThreePidMapping {

    private String medium;
@@ -50,4 +52,26 @@ public class ThreePidMapping {
        this.mxid = mxid;
    }

+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreePidMapping that = (ThreePidMapping) o;
+
+        if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false;
+        return value != null ? value.equals(that.value) : that.value == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = medium != null ? medium.hashCode() : 0;
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return JsonOutput.toJson(this);
+    }
 }
--- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy
+++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy
@@ -33,6 +33,8 @@ import org.xbill.DNS.Lookup
 import org.xbill.DNS.SRVRecord
 import org.xbill.DNS.Type

+import java.util.function.Function
+
@Component
 class DnsLookupProvider extends RemoteIdentityServerProvider {

@@ -46,23 +48,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
        return 10
    }

-    @Override
-    Optional<?> find(SingleLookupRequest request) {
-        log.info("Performing DNS lookup for {}", request.getThreePid())
-        if (ThreePidType.email != request.getType()) {
-            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+    String getSrvRecordName(String domain) {
+        return "_matrix-identity._tcp." + domain
+    }
+
+    Optional<String> getDomain(String email) {
+        int atIndex = email.lastIndexOf("@")
+        if (atIndex == -1) {
            return Optional.empty()
        }

-        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
-        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+        return Optional.of(email.substring(atIndex + 1))
+    }
+
+    // TODO use caching mechanism
+    Optional<String> findIdentityServerForDomain(String domain) {
        if (StringUtils.equals(srvCfg.getName(), domain)) {
-            log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?")
+            log.info("We are authoritative for {}, no remote lookup", domain)
            return Optional.empty()
        }

        log.info("Performing SRV lookup")
-        String lookupDns = "_matrix-identity._tcp." + domain
+        String lookupDns = getSrvRecordName(domain)
        log.info("Lookup name: {}", lookupDns)

        SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run()
@@ -79,11 +86,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
            for (SRVRecord record : records) {
                log.info("Found SRV record: {}", record.toString())
                String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}"
-                Optional<?> answer = find(baseUrl, request.getType(), request.getThreePid())
-                if (answer.isPresent()) {
-                    return answer
+                if (isUsableIdentityServer(baseUrl)) {
+                    log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
+                    return Optional.of(baseUrl)
                } else {
-                    log.info("No mapping found at {}", baseUrl)
+                    log.info("{} is not a usable Identity Server", baseUrl)
                }
            }
        } else {
@@ -92,15 +99,77 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {

        log.info("Performing basic lookup using domain name {}", domain)
        String baseUrl = "https://" + domain
-        return find(baseUrl, request.getType(), request.getThreePid())
+        if (isUsableIdentityServer(baseUrl)) {
+            log.info("Found Identity Server for domain {} at {}", domain, baseUrl)
+            return Optional.of(baseUrl)
+        } else {
+            log.info("{} is not a usable Identity Server", baseUrl)
+            return Optional.empty()
+        }
+    }
+
+    @Override
+    Optional<?> find(SingleLookupRequest request) {
+        log.info("Performing DNS lookup for {}", request.getThreePid())
+        if (ThreePidType.email != request.getType()) {
+            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
+            return Optional.empty()
+        }
+
+        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1)
+        log.info("Domain name for {}: {}", request.getThreePid(), domain)
+        Optional<String> baseUrl = findIdentityServerForDomain(domain)
+
+        if (baseUrl.isPresent()) {
+            return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid())
+        }
+
+        return Optional.empty()
    }

    @Override
    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+        Map<String, List<ThreePidMapping>> domains = new HashMap<>()

-        // TODO
+        for (ThreePidMapping mapping : mappings) {
+            if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) {
+                log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue())
+                continue
+            }

+            Optional<String> domainOpt = getDomain(mapping.getValue())
+            if (!domainOpt.isPresent()) {
+                log.warn("No domain for 3PID {}", mapping.getValue())
+                continue
+            }
+
+            String domain = domainOpt.get()
+            List<ThreePidMapping> domainMappings = domains.computeIfAbsent(domain, new Function<String, List<ThreePidMapping>>() {
+
+                @Override
+                List<ThreePidMapping> apply(String s) {
+                    return new ArrayList<>()
+                }
+
+            })
+            domainMappings.add(mapping)
+        }
+
+        log.info("Looking mappings across {} domains", domains.keySet().size())
+        for (String domain : domains.keySet()) {
+            Optional<String> baseUrl = findIdentityServerForDomain(domain)
+            if (!baseUrl.isPresent()) {
+                log.info("No usable Identity server for domain {}", domain)
+                continue
+            }
+
+            List<ThreePidMapping> domainMappings = find(baseUrl.get(), domains.get(domain))
+            log.info("Found {} mappings in domain {}", domainMappings.size(), domain)
+            mappingsFound.addAll(domainMappings)
+        }
+
+        log.info("Found {} mappings overall", mappingsFound.size())
        return mappingsFound
    }

--- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy
+++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy
@@ -23,12 +23,16 @@ package io.kamax.mxisd.lookup.provider
 import io.kamax.mxisd.config.ForwardConfig
 import io.kamax.mxisd.lookup.SingleLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component

@Component
 class ForwarderProvider extends RemoteIdentityServerProvider {

+    private Logger log = LoggerFactory.getLogger(ForwarderProvider.class)
+
    @Autowired
    private ForwardConfig cfg

@@ -51,11 +55,19 @@ class ForwarderProvider extends RemoteIdentityServerProvider {

    @Override
    List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
-        List<ThreePidMapping> mappingsFound = new ArrayList<>()
+        List<ThreePidMapping> mappingsToDo = new ArrayList<>(mappings)
+        List<ThreePidMapping> mappingsFoundGlobal = new ArrayList<>()

-        // TODO
+        for (String root : cfg.getServers()) {
+            log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo)
+            log.info("Querying {}", root)
+            List<ThreePidMapping> mappingsFound = find(root, mappingsToDo)
+            log.info("{} returned {} mappings", root, mappingsFound.size())
+            mappingsFoundGlobal.addAll(mappingsFound)
+            mappingsToDo.removeAll(mappingsFound)
+        }

-        return mappingsFound
+        return mappingsFoundGlobal
    }

 }
--- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy
+++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy
@@ -21,14 +21,26 @@
 package io.kamax.mxisd.lookup.provider

 import groovy.json.JsonException
+import groovy.json.JsonOutput
 import groovy.json.JsonSlurper
 import io.kamax.mxisd.api.ThreePidType
+import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest
 import io.kamax.mxisd.lookup.ThreePidMapping
+import org.apache.http.HttpEntity
+import org.apache.http.HttpResponse
+import org.apache.http.client.HttpClient
+import org.apache.http.client.entity.EntityBuilder
+import org.apache.http.client.methods.HttpPost
+import org.apache.http.entity.ContentType
+import org.apache.http.impl.client.HttpClients
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory

 abstract class RemoteIdentityServerProvider implements ThreePidProvider {

+    public static final String THREEPID_TEST_MEDIUM = "email"
+    public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org"
+
    private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class)

    private JsonSlurper json = new JsonSlurper()
@@ -38,6 +50,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
        return false
    }

+    boolean isUsableIdentityServer(String remote) {
+        try {
+            HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                    "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}"
+            ).openConnection()
+
+            if (rootSrvConn.getResponseCode() != 200) {
+                return false
+            }
+
+            def output = json.parseText(rootSrvConn.getInputStream().getText())
+            if (output['address']) {
+                return false
+            }
+
+            return true
+        } catch (IOException | JsonException e) {
+            log.info("{} is not a usable Identity Server: {}", remote, e.getMessage())
+            return false
+        }
+    }
+
+    Optional<ThreePidMapping> performLookup(String remote, String medium, String address) throws IOException, JsonException {
+        HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}"
+        ).openConnection()
+
+        def output = json.parseText(rootSrvConn.getInputStream().getText())
+        if (output['address']) {
+            log.info("Found 3PID mapping: {}", output)
+
+            ThreePidMapping mapping = new ThreePidMapping()
+            mapping.setMedium(output['medium'].toString())
+            mapping.setValue(output['address'].toString())
+            mapping.setMxid(output['mxid'].toString())
+            return Optional.of(mapping)
+        }
+
+        log.info("Empty 3PID mapping from {}", remote)
+        return Optional.empty()
+    }
+
    Optional<?> find(String remote, ThreePidType type, String threePid) {
        log.info("Looking up {} 3PID {} using {}", type, threePid, remote)

@@ -66,9 +120,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
    List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
        List<ThreePidMapping> mappingsFound = new ArrayList<>()

-        // TODO
+        ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest()
+        mappingRequest.setMappings(mappings)

-        return mappingsFound
+        String url = "${remote}/_matrix/identity/api/v1/bulk_lookup"
+        HttpClient client = HttpClients.createDefault()
+        try {
+            HttpPost request = new HttpPost(url)
+            request.setEntity(
+                    EntityBuilder.create()
+                            .setText(JsonOutput.toJson(mappingRequest))
+                            .setContentType(ContentType.APPLICATION_JSON)
+                            .build()
+            )
+
+            HttpResponse response = client.execute(request)
+            try {
+                if (response.getStatusLine().getStatusCode() != 200) {
+                    log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode())
+                    return mappingsFound
+                }
+
+                HttpEntity entity = response.getEntity()
+                if (entity != null) {
+                    ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText())
+                    for (List<String> mappingRaw : input.getThreepids()) {
+                        ThreePidMapping mapping = new ThreePidMapping()
+                        mapping.setMedium(mappingRaw.get(0))
+                        mapping.setValue(mappingRaw.get(1))
+                        mapping.setMxid(mappingRaw.get(2))
+                        mappingsFound.add(mapping)
+                    }
+                } else {
+                    log.info("HTTP response from {} was empty", remote)
+                }
+
+                return mappingsFound
+            } finally {
+                response.close()
+            }
+        } finally {
+            client.close()
+        }
    }

 }
--- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy
+++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy
@@ -114,14 +114,18 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
            if (mapToDo.isEmpty()) {
                log.info("No more mappings to lookup")
                break
+            } else {
+                log.info("{} mappings remaining overall", mapToDo.size())
            }

+            log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName())
            List<ThreePidMapping> mapFound = provider.populate(mapToDo)
+            log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size())
            mapFoundAll.addAll(mapFound)
            mapToDo.removeAll(mapFound)
        }

-        return mapFoundAll;
+        return mapFoundAll
    }

 }