From d0b9f6774d70258ac76149f5152efeeb7071624d Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Tue, 4 Apr 2017 01:11:32 +0200 Subject: [PATCH] Bulk lookup implementation, part 2 - Remote IS bulk lookup --- build.gradle | 3 + .../v1/ClientBulkLookupRequest.groovy | 13 ++- .../kamax/mxisd/lookup/ThreePidMapping.java | 24 +++++ .../lookup/provider/DnsLookupProvider.groovy | 99 ++++++++++++++++--- .../lookup/provider/ForwarderProvider.groovy | 18 +++- .../RemoteIdentityServerProvider.groovy | 97 +++++++++++++++++- .../RecursivePriorityLookupStrategy.groovy | 6 +- 7 files changed, 238 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index 46ecc13..f169a1a 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,9 @@ dependencies { // DNS lookups compile 'dnsjava:dnsjava:2.1.8' + // HTTP connections + compile 'org.apache.httpcomponents:httpclient:4.5.3' + testCompile 'junit:junit:4.12' } diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy index d3bab59..7109c5c 100644 --- a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy @@ -20,9 +20,11 @@ package io.kamax.mxisd.controller.v1 +import io.kamax.mxisd.lookup.ThreePidMapping + class ClientBulkLookupRequest { - private List> threepids + private List> threepids = new ArrayList<>() List> getThreepids() { return threepids @@ -32,4 +34,13 @@ class ClientBulkLookupRequest { this.threepids = threepids } + void setMappings(List mappings) { + for (ThreePidMapping mapping : mappings) { + List threepid = new ArrayList<>() + threepid.add(mapping.getMedium()) + threepid.add(mapping.getValue()) + threepids.add(threepid) + } + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java index de65058..9d37783 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java +++ b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java @@ -20,6 +20,8 @@ package io.kamax.mxisd.lookup; +import groovy.json.JsonOutput; + public class ThreePidMapping { private String medium; @@ -50,4 +52,26 @@ public class ThreePidMapping { this.mxid = mxid; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ThreePidMapping that = (ThreePidMapping) o; + + if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false; + return value != null ? value.equals(that.value) : that.value == null; + } + + @Override + public int hashCode() { + int result = medium != null ? medium.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return JsonOutput.toJson(this); + } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy index 334b0b7..3d0b328 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy @@ -33,6 +33,8 @@ import org.xbill.DNS.Lookup import org.xbill.DNS.SRVRecord import org.xbill.DNS.Type +import java.util.function.Function + @Component class DnsLookupProvider extends RemoteIdentityServerProvider { @@ -46,23 +48,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { return 10 } - @Override - Optional find(SingleLookupRequest request) { - log.info("Performing DNS lookup for {}", request.getThreePid()) - if (ThreePidType.email != request.getType()) { - log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + String getSrvRecordName(String domain) { + return "_matrix-identity._tcp." + domain + } + + Optional getDomain(String email) { + int atIndex = email.lastIndexOf("@") + if (atIndex == -1) { return Optional.empty() } - String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) - log.info("Domain name for {}: {}", request.getThreePid(), domain) + return Optional.of(email.substring(atIndex + 1)) + } + + // TODO use caching mechanism + Optional findIdentityServerForDomain(String domain) { if (StringUtils.equals(srvCfg.getName(), domain)) { - log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?") + log.info("We are authoritative for {}, no remote lookup", domain) return Optional.empty() } log.info("Performing SRV lookup") - String lookupDns = "_matrix-identity._tcp." + domain + String lookupDns = getSrvRecordName(domain) log.info("Lookup name: {}", lookupDns) SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run() @@ -79,11 +86,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { for (SRVRecord record : records) { log.info("Found SRV record: {}", record.toString()) String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}" - Optional answer = find(baseUrl, request.getType(), request.getThreePid()) - if (answer.isPresent()) { - return answer + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) } else { - log.info("No mapping found at {}", baseUrl) + log.info("{} is not a usable Identity Server", baseUrl) } } } else { @@ -92,15 +99,77 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { log.info("Performing basic lookup using domain name {}", domain) String baseUrl = "https://" + domain - return find(baseUrl, request.getType(), request.getThreePid()) + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) + } else { + log.info("{} is not a usable Identity Server", baseUrl) + return Optional.empty() + } + } + + @Override + Optional find(SingleLookupRequest request) { + log.info("Performing DNS lookup for {}", request.getThreePid()) + if (ThreePidType.email != request.getType()) { + log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + return Optional.empty() + } + + String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) + log.info("Domain name for {}: {}", request.getThreePid(), domain) + Optional baseUrl = findIdentityServerForDomain(domain) + + if (baseUrl.isPresent()) { + return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid()) + } + + return Optional.empty() } @Override List populate(List mappings) { List mappingsFound = new ArrayList<>() + Map> domains = new HashMap<>() - // TODO + for (ThreePidMapping mapping : mappings) { + if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) { + log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue()) + continue + } + Optional domainOpt = getDomain(mapping.getValue()) + if (!domainOpt.isPresent()) { + log.warn("No domain for 3PID {}", mapping.getValue()) + continue + } + + String domain = domainOpt.get() + List domainMappings = domains.computeIfAbsent(domain, new Function>() { + + @Override + List apply(String s) { + return new ArrayList<>() + } + + }) + domainMappings.add(mapping) + } + + log.info("Looking mappings across {} domains", domains.keySet().size()) + for (String domain : domains.keySet()) { + Optional baseUrl = findIdentityServerForDomain(domain) + if (!baseUrl.isPresent()) { + log.info("No usable Identity server for domain {}", domain) + continue + } + + List domainMappings = find(baseUrl.get(), domains.get(domain)) + log.info("Found {} mappings in domain {}", domainMappings.size(), domain) + mappingsFound.addAll(domainMappings) + } + + log.info("Found {} mappings overall", mappingsFound.size()) return mappingsFound } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy index 273d45e..633dafd 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy @@ -23,12 +23,16 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.config.ForwardConfig import io.kamax.mxisd.lookup.SingleLookupRequest import io.kamax.mxisd.lookup.ThreePidMapping +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class ForwarderProvider extends RemoteIdentityServerProvider { + private Logger log = LoggerFactory.getLogger(ForwarderProvider.class) + @Autowired private ForwardConfig cfg @@ -51,11 +55,19 @@ class ForwarderProvider extends RemoteIdentityServerProvider { @Override List populate(List mappings) { - List mappingsFound = new ArrayList<>() + List mappingsToDo = new ArrayList<>(mappings) + List mappingsFoundGlobal = new ArrayList<>() - // TODO + for (String root : cfg.getServers()) { + log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo) + log.info("Querying {}", root) + List mappingsFound = find(root, mappingsToDo) + log.info("{} returned {} mappings", root, mappingsFound.size()) + mappingsFoundGlobal.addAll(mappingsFound) + mappingsToDo.removeAll(mappingsFound) + } - return mappingsFound + return mappingsFoundGlobal } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy index c3ed6a0..85985da 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy @@ -21,14 +21,26 @@ package io.kamax.mxisd.lookup.provider import groovy.json.JsonException +import groovy.json.JsonOutput import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType +import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest import io.kamax.mxisd.lookup.ThreePidMapping +import org.apache.http.HttpEntity +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.entity.EntityBuilder +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.ContentType +import org.apache.http.impl.client.HttpClients import org.slf4j.Logger import org.slf4j.LoggerFactory abstract class RemoteIdentityServerProvider implements ThreePidProvider { + public static final String THREEPID_TEST_MEDIUM = "email" + public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org" + private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class) private JsonSlurper json = new JsonSlurper() @@ -38,6 +50,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { return false } + boolean isUsableIdentityServer(String remote) { + try { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}" + ).openConnection() + + if (rootSrvConn.getResponseCode() != 200) { + return false + } + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + return false + } + + return true + } catch (IOException | JsonException e) { + log.info("{} is not a usable Identity Server: {}", remote, e.getMessage()) + return false + } + } + + Optional performLookup(String remote, String medium, String address) throws IOException, JsonException { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}" + ).openConnection() + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + log.info("Found 3PID mapping: {}", output) + + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(output['medium'].toString()) + mapping.setValue(output['address'].toString()) + mapping.setMxid(output['mxid'].toString()) + return Optional.of(mapping) + } + + log.info("Empty 3PID mapping from {}", remote) + return Optional.empty() + } + Optional find(String remote, ThreePidType type, String threePid) { log.info("Looking up {} 3PID {} using {}", type, threePid, remote) @@ -66,9 +120,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { List find(String remote, List mappings) { List mappingsFound = new ArrayList<>() - // TODO + ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest() + mappingRequest.setMappings(mappings) - return mappingsFound + String url = "${remote}/_matrix/identity/api/v1/bulk_lookup" + HttpClient client = HttpClients.createDefault() + try { + HttpPost request = new HttpPost(url) + request.setEntity( + EntityBuilder.create() + .setText(JsonOutput.toJson(mappingRequest)) + .setContentType(ContentType.APPLICATION_JSON) + .build() + ) + + HttpResponse response = client.execute(request) + try { + if (response.getStatusLine().getStatusCode() != 200) { + log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode()) + return mappingsFound + } + + HttpEntity entity = response.getEntity() + if (entity != null) { + ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText()) + for (List mappingRaw : input.getThreepids()) { + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(mappingRaw.get(0)) + mapping.setValue(mappingRaw.get(1)) + mapping.setMxid(mappingRaw.get(2)) + mappingsFound.add(mapping) + } + } else { + log.info("HTTP response from {} was empty", remote) + } + + return mappingsFound + } finally { + response.close() + } + } finally { + client.close() + } } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy index 50518b6..92d1786 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy @@ -114,14 +114,18 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea if (mapToDo.isEmpty()) { log.info("No more mappings to lookup") break + } else { + log.info("{} mappings remaining overall", mapToDo.size()) } + log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName()) List mapFound = provider.populate(mapToDo) + log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size()) mapFoundAll.addAll(mapFound) mapToDo.removeAll(mapFound) } - return mapFoundAll; + return mapFoundAll } }