From 158a1e635428a043a5701104ce9956575dedefb9 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Mon, 3 Apr 2017 04:11:37 +0200 Subject: [PATCH 1/3] Bulk lookup implementation, part 1 - LDAP bulk lookup --- .../controller/v1/ClientBulkLookupAnswer.java | 44 ++++++ .../v1/ClientBulkLookupRequest.groovy | 35 +++++ .../controller/v1/MappingController.groovy | 32 ++++- .../io/kamax/mxisd/lookup/ALookupRequest.java | 35 +++++ .../kamax/mxisd/lookup/BulkLookupRequest.java | 37 +++++ ...uest.groovy => SingleLookupRequest.groovy} | 11 +- .../kamax/mxisd/lookup/ThreePidMapping.java | 53 ++++++++ .../lookup/provider/DnsLookupProvider.groovy | 14 +- .../lookup/provider/ForwarderProvider.groovy | 14 +- .../mxisd/lookup/provider/LdapProvider.groovy | 127 ++++++++++++------ .../RemoteIdentityServerProvider.groovy | 9 ++ .../lookup/provider/ThreePidProvider.groovy | 7 +- .../lookup/strategy/LookupStrategy.groovy | 8 +- .../RecursivePriorityLookupStrategy.groovy | 47 +++++-- 14 files changed, 401 insertions(+), 72 deletions(-) create mode 100644 src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java create mode 100644 src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy create mode 100644 src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java create mode 100644 src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java rename src/main/groovy/io/kamax/mxisd/lookup/{LookupRequest.groovy => SingleLookupRequest.groovy} (85%) create mode 100644 src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java new file mode 100644 index 0000000..1f0dc34 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java @@ -0,0 +1,44 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.v1; + +import io.kamax.mxisd.lookup.ThreePidMapping; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +public class ClientBulkLookupAnswer { + + private List> threepids = new ArrayList<>(); + + public void addAll(Collection mappings) { + for (ThreePidMapping mapping : mappings) { + threepids.add(Arrays.asList(mapping.getMedium(), mapping.getValue(), mapping.getMxid())); + } + } + + public List> getThreepids() { + return threepids; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy new file mode 100644 index 0000000..d3bab59 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy @@ -0,0 +1,35 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.v1 + +class ClientBulkLookupRequest { + + private List> threepids + + List> getThreepids() { + return threepids + } + + void setThreepids(List> threepids) { + this.threepids = threepids + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy index 80976de..dbc870e 100644 --- a/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy @@ -21,8 +21,11 @@ package io.kamax.mxisd.controller.v1 import groovy.json.JsonOutput +import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import io.kamax.mxisd.lookup.strategy.LookupStrategy import io.kamax.mxisd.signature.SignatureManager import org.apache.commons.lang.StringUtils @@ -36,11 +39,13 @@ import org.springframework.web.bind.annotation.RestController import javax.servlet.http.HttpServletRequest import static org.springframework.web.bind.annotation.RequestMethod.GET +import static org.springframework.web.bind.annotation.RequestMethod.POST @RestController class MappingController { private Logger log = LoggerFactory.getLogger(MappingController.class) + private JsonSlurper json = new JsonSlurper() @Autowired private LookupStrategy strategy @@ -55,7 +60,7 @@ class MappingController { ThreePidType type = ThreePidType.valueOf(medium) - LookupRequest lookupRequest = new LookupRequest() + SingleLookupRequest lookupRequest = new SingleLookupRequest() lookupRequest.setRequester(remote) lookupRequest.setType(type) lookupRequest.setThreePid(address) @@ -75,4 +80,27 @@ class MappingController { return JsonOutput.toJson(lookup) } + @RequestMapping(value = "/_matrix/identity/api/v1/bulk_lookup", method = POST) + String bulkLookup(HttpServletRequest request) { + String remote = StringUtils.defaultIfBlank(request.getHeader("X-FORWARDED-FOR"), request.getRemoteAddr()) + log.info("Got request from {}", remote) + + BulkLookupRequest lookupRequest = new BulkLookupRequest() + lookupRequest.setRequester(remote) + + ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(request.getInputStream().getText()) + List mappings = new ArrayList<>() + for (List mappingRaw : input.getThreepids()) { + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(mappingRaw.get(0)) + mapping.setValue(mappingRaw.get(1)) + mappings.add(mapping) + } + lookupRequest.setMappings(mappings) + + ClientBulkLookupAnswer answer = new ClientBulkLookupAnswer() + answer.addAll(strategy.find(lookupRequest)) + return JsonOutput.toJson(answer) + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java b/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java new file mode 100644 index 0000000..58aa305 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java @@ -0,0 +1,35 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +public abstract class ALookupRequest { + + private String requester; + + public String getRequester() { + return requester; + } + + public void setRequester(String requester) { + this.requester = requester; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java b/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java new file mode 100644 index 0000000..51b6284 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java @@ -0,0 +1,37 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +import java.util.List; + +public class BulkLookupRequest extends ALookupRequest { + + private List mappings; + + public List getMappings() { + return mappings; + } + + public void setMappings(List mappings) { + this.mappings = mappings; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy similarity index 85% rename from src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy rename to src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy index a9b3989..ba256dd 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy @@ -22,20 +22,11 @@ package io.kamax.mxisd.lookup import io.kamax.mxisd.api.ThreePidType -class LookupRequest { +class SingleLookupRequest extends ALookupRequest { - private String requester private ThreePidType type private String threePid - String getRequester() { - return requester - } - - void setRequester(String requester) { - this.requester = requester - } - ThreePidType getType() { return type } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java new file mode 100644 index 0000000..de65058 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java @@ -0,0 +1,53 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +public class ThreePidMapping { + + private String medium; + private String value; + private String mxid; + + public String getMedium() { + return medium; + } + + public void setMedium(String medium) { + this.medium = medium; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getMxid() { + return mxid; + } + + public void setMxid(String mxid) { + this.mxid = mxid; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy index 697d465..334b0b7 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy @@ -22,7 +22,8 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.api.ThreePidType import io.kamax.mxisd.config.ServerConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import org.apache.commons.lang.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -46,7 +47,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { } @Override - Optional find(LookupRequest request) { + Optional find(SingleLookupRequest request) { log.info("Performing DNS lookup for {}", request.getThreePid()) if (ThreePidType.email != request.getType()) { log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) @@ -94,4 +95,13 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { return find(baseUrl, request.getType(), request.getThreePid()) } + @Override + List populate(List mappings) { + List mappingsFound = new ArrayList<>() + + // TODO + + return mappingsFound + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy index df67961..273d45e 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy @@ -21,7 +21,8 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.config.ForwardConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -37,7 +38,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider { } @Override - Optional find(LookupRequest request) { + Optional find(SingleLookupRequest request) { for (String root : cfg.getServers()) { Optional answer = find(root, request.getType(), request.getThreePid()) if (answer.isPresent()) { @@ -48,4 +49,13 @@ class ForwarderProvider extends RemoteIdentityServerProvider { return Optional.empty() } + @Override + List populate(List mappings) { + List mappingsFound = new ArrayList<>() + + // TODO + + return mappingsFound + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy index 9cb8e23..0cd4867 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy @@ -20,9 +20,11 @@ package io.kamax.mxisd.lookup.provider +import io.kamax.mxisd.api.ThreePidType import io.kamax.mxisd.config.LdapConfig import io.kamax.mxisd.config.ServerConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import org.apache.commons.lang.StringUtils import org.apache.directory.api.ldap.model.cursor.EntryCursor import org.apache.directory.api.ldap.model.entry.Attribute @@ -58,56 +60,65 @@ class LdapProvider implements ThreePidProvider { return 20 } + Optional lookup(LdapConnection conn, ThreePidType medium, String value) { + Optional queryOpt = ldapCfg.getMapping(medium) + if (!queryOpt.isPresent()) { + log.warn("{} is not a supported 3PID type for LDAP lookup", medium) + return Optional.empty() + } + + String searchQuery = queryOpt.get().replaceAll("%3pid", value) + EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute()) + try { + if (cursor.next()) { + Attribute attribute = cursor.get().get(ldapCfg.getAttribute()) + if (attribute != null) { + String data = attribute.get().toString() + if (data.length() < 1) { + log.warn("Bind was found but value is empty") + return Optional.empty() + } + + StringBuilder matrixId = new StringBuilder() + // TODO Should we turn this block into a map of functions? + if (StringUtils.equals(UID, ldapCfg.getType())) { + matrixId.append("@").append(data).append(":").append(srvCfg.getName()) + } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) { + matrixId.append(data) + } else { + log.warn("Bind was found but type ${ldapCfg.getType()} is not supported") + return Optional.empty() + } + + log.info("Found a match in LDAP") + return Optional.of(matrixId.toString()) + } + } + } finally { + cursor.close() + } + + return Optional.empty() + } + @Override - Optional find(LookupRequest request) { + Optional find(SingleLookupRequest request) { log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}") LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort()) try { conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword()) - Optional queryOpt = ldapCfg.getMapping(request.getType()) - if (!queryOpt.isPresent()) { - log.warn("{} is not a supported 3PID type for LDAP lookup", request.getType()) - return Optional.empty() - } - - String searchQuery = queryOpt.get().replaceAll("%3pid", request.getThreePid()) - EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute()) - try { - if (cursor.next()) { - Attribute attribute = cursor.get().get(ldapCfg.getAttribute()) - if (attribute != null) { - String data = attribute.get().toString() - if (data.length() < 1) { - log.warn("Bind was found but value is empty") - return Optional.empty() - } - - StringBuilder matrixId = new StringBuilder() - // TODO Should we turn this block into a map of functions? - if (StringUtils.equals(UID, ldapCfg.getType())) { - matrixId.append("@").append(data).append(":").append(srvCfg.getName()) - } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) { - matrixId.append(data) - } else { - log.warn("Bind was found but type ${ldapCfg.getType()} is not supported") - return Optional.empty() - } - - log.info("Found a match in LDAP") - return Optional.of([ - address : request.getThreePid(), - medium : request.getType(), - mxid : matrixId.toString(), - not_before: 0, - not_after : 9223372036854775807, - ts : 0 - ]) - } - } - } finally { - cursor.close() + Optional mxid = lookup(conn, request.getType(), request.getThreePid()) + if (mxid.isPresent()) { + return Optional.of([ + address : request.getThreePid(), + medium : request.getType(), + mxid : mxid.get(), + not_before: 0, + not_after : 9223372036854775807, + ts : 0 + ]) } } finally { conn.close() @@ -117,4 +128,32 @@ class LdapProvider implements ThreePidProvider { return Optional.empty() } + @Override + List populate(List mappings) { + log.info("Looking up {} mappings", mappings.size()) + List mappingsFound = new ArrayList<>() + + LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort()) + try { + conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword()) + + for (ThreePidMapping mapping : mappings) { + try { + ThreePidType type = ThreePidType.valueOf(mapping.getMedium()) + Optional mxid = lookup(conn, type, mapping.getValue()) + if (mxid.isPresent()) { + mapping.setMxid(mxid.get()) + mappingsFound.add(mapping) + } + } catch (IllegalArgumentException e) { + log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium()) + } + } + } finally { + conn.close() + } + + return mappingsFound + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy index a90dd26..c3ed6a0 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy @@ -23,6 +23,7 @@ package io.kamax.mxisd.lookup.provider import groovy.json.JsonException import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType +import io.kamax.mxisd.lookup.ThreePidMapping import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -62,4 +63,12 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { } } + List find(String remote, List mappings) { + List mappingsFound = new ArrayList<>() + + // TODO + + return mappingsFound + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy index 71cd127..d102fad 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy @@ -20,7 +20,8 @@ package io.kamax.mxisd.lookup.provider -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping interface ThreePidProvider { @@ -31,6 +32,8 @@ interface ThreePidProvider { */ int getPriority() // Should not be here but let's KISS for now - Optional find(LookupRequest request) + Optional find(SingleLookupRequest request) + + List populate(List mappings) } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy index a97f796..cfd9e79 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy @@ -20,10 +20,14 @@ package io.kamax.mxisd.lookup.strategy -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping interface LookupStrategy { - Optional find(LookupRequest request) + Optional find(SingleLookupRequest request) + + List find(BulkLookupRequest requests) } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy index 0c8f334..50518b6 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy @@ -22,7 +22,10 @@ package io.kamax.mxisd.lookup.strategy import edazdarevic.commons.net.CIDRUtils import io.kamax.mxisd.config.RecursiveLookupConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.ALookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import io.kamax.mxisd.lookup.provider.ThreePidProvider import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -63,8 +66,9 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea } } - @Override - Optional find(LookupRequest request) { + List listUsableProviders(ALookupRequest request) { + List usableProviders = new ArrayList<>() + boolean canRecurse = false if (recursiveCfg.isEnabled()) { log.debug("Checking {} CIDRs for recursion", allowedCidr.size()) @@ -80,17 +84,44 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea } log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse) - for (ThreePidProvider provider : providers) { if (provider.isLocal() || canRecurse) { - Optional lookupDataOpt = provider.find(request) - if (lookupDataOpt.isPresent()) { - return lookupDataOpt - } + usableProviders.add(provider) + } + } + + return usableProviders + } + + @Override + Optional find(SingleLookupRequest request) { + for (ThreePidProvider provider : listUsableProviders(request)) { + Optional lookupDataOpt = provider.find(request) + if (lookupDataOpt.isPresent()) { + return lookupDataOpt } } return Optional.empty() } + @Override + List find(BulkLookupRequest request) { + List mapToDo = new ArrayList<>(request.getMappings()) + List mapFoundAll = new ArrayList<>() + + for (ThreePidProvider provider : listUsableProviders(request)) { + if (mapToDo.isEmpty()) { + log.info("No more mappings to lookup") + break + } + + List mapFound = provider.populate(mapToDo) + mapFoundAll.addAll(mapFound) + mapToDo.removeAll(mapFound) + } + + return mapFoundAll; + } + } From d0b9f6774d70258ac76149f5152efeeb7071624d Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Tue, 4 Apr 2017 01:11:32 +0200 Subject: [PATCH 2/3] Bulk lookup implementation, part 2 - Remote IS bulk lookup --- build.gradle | 3 + .../v1/ClientBulkLookupRequest.groovy | 13 ++- .../kamax/mxisd/lookup/ThreePidMapping.java | 24 +++++ .../lookup/provider/DnsLookupProvider.groovy | 99 ++++++++++++++++--- .../lookup/provider/ForwarderProvider.groovy | 18 +++- .../RemoteIdentityServerProvider.groovy | 97 +++++++++++++++++- .../RecursivePriorityLookupStrategy.groovy | 6 +- 7 files changed, 238 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index 46ecc13..f169a1a 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,9 @@ dependencies { // DNS lookups compile 'dnsjava:dnsjava:2.1.8' + // HTTP connections + compile 'org.apache.httpcomponents:httpclient:4.5.3' + testCompile 'junit:junit:4.12' } diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy index d3bab59..7109c5c 100644 --- a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy @@ -20,9 +20,11 @@ package io.kamax.mxisd.controller.v1 +import io.kamax.mxisd.lookup.ThreePidMapping + class ClientBulkLookupRequest { - private List> threepids + private List> threepids = new ArrayList<>() List> getThreepids() { return threepids @@ -32,4 +34,13 @@ class ClientBulkLookupRequest { this.threepids = threepids } + void setMappings(List mappings) { + for (ThreePidMapping mapping : mappings) { + List threepid = new ArrayList<>() + threepid.add(mapping.getMedium()) + threepid.add(mapping.getValue()) + threepids.add(threepid) + } + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java index de65058..9d37783 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java +++ b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java @@ -20,6 +20,8 @@ package io.kamax.mxisd.lookup; +import groovy.json.JsonOutput; + public class ThreePidMapping { private String medium; @@ -50,4 +52,26 @@ public class ThreePidMapping { this.mxid = mxid; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ThreePidMapping that = (ThreePidMapping) o; + + if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false; + return value != null ? value.equals(that.value) : that.value == null; + } + + @Override + public int hashCode() { + int result = medium != null ? medium.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return JsonOutput.toJson(this); + } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy index 334b0b7..3d0b328 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy @@ -33,6 +33,8 @@ import org.xbill.DNS.Lookup import org.xbill.DNS.SRVRecord import org.xbill.DNS.Type +import java.util.function.Function + @Component class DnsLookupProvider extends RemoteIdentityServerProvider { @@ -46,23 +48,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { return 10 } - @Override - Optional find(SingleLookupRequest request) { - log.info("Performing DNS lookup for {}", request.getThreePid()) - if (ThreePidType.email != request.getType()) { - log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + String getSrvRecordName(String domain) { + return "_matrix-identity._tcp." + domain + } + + Optional getDomain(String email) { + int atIndex = email.lastIndexOf("@") + if (atIndex == -1) { return Optional.empty() } - String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) - log.info("Domain name for {}: {}", request.getThreePid(), domain) + return Optional.of(email.substring(atIndex + 1)) + } + + // TODO use caching mechanism + Optional findIdentityServerForDomain(String domain) { if (StringUtils.equals(srvCfg.getName(), domain)) { - log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?") + log.info("We are authoritative for {}, no remote lookup", domain) return Optional.empty() } log.info("Performing SRV lookup") - String lookupDns = "_matrix-identity._tcp." + domain + String lookupDns = getSrvRecordName(domain) log.info("Lookup name: {}", lookupDns) SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run() @@ -79,11 +86,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { for (SRVRecord record : records) { log.info("Found SRV record: {}", record.toString()) String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}" - Optional answer = find(baseUrl, request.getType(), request.getThreePid()) - if (answer.isPresent()) { - return answer + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) } else { - log.info("No mapping found at {}", baseUrl) + log.info("{} is not a usable Identity Server", baseUrl) } } } else { @@ -92,15 +99,77 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { log.info("Performing basic lookup using domain name {}", domain) String baseUrl = "https://" + domain - return find(baseUrl, request.getType(), request.getThreePid()) + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) + } else { + log.info("{} is not a usable Identity Server", baseUrl) + return Optional.empty() + } + } + + @Override + Optional find(SingleLookupRequest request) { + log.info("Performing DNS lookup for {}", request.getThreePid()) + if (ThreePidType.email != request.getType()) { + log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + return Optional.empty() + } + + String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) + log.info("Domain name for {}: {}", request.getThreePid(), domain) + Optional baseUrl = findIdentityServerForDomain(domain) + + if (baseUrl.isPresent()) { + return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid()) + } + + return Optional.empty() } @Override List populate(List mappings) { List mappingsFound = new ArrayList<>() + Map> domains = new HashMap<>() - // TODO + for (ThreePidMapping mapping : mappings) { + if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) { + log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue()) + continue + } + Optional domainOpt = getDomain(mapping.getValue()) + if (!domainOpt.isPresent()) { + log.warn("No domain for 3PID {}", mapping.getValue()) + continue + } + + String domain = domainOpt.get() + List domainMappings = domains.computeIfAbsent(domain, new Function>() { + + @Override + List apply(String s) { + return new ArrayList<>() + } + + }) + domainMappings.add(mapping) + } + + log.info("Looking mappings across {} domains", domains.keySet().size()) + for (String domain : domains.keySet()) { + Optional baseUrl = findIdentityServerForDomain(domain) + if (!baseUrl.isPresent()) { + log.info("No usable Identity server for domain {}", domain) + continue + } + + List domainMappings = find(baseUrl.get(), domains.get(domain)) + log.info("Found {} mappings in domain {}", domainMappings.size(), domain) + mappingsFound.addAll(domainMappings) + } + + log.info("Found {} mappings overall", mappingsFound.size()) return mappingsFound } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy index 273d45e..633dafd 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy @@ -23,12 +23,16 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.config.ForwardConfig import io.kamax.mxisd.lookup.SingleLookupRequest import io.kamax.mxisd.lookup.ThreePidMapping +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class ForwarderProvider extends RemoteIdentityServerProvider { + private Logger log = LoggerFactory.getLogger(ForwarderProvider.class) + @Autowired private ForwardConfig cfg @@ -51,11 +55,19 @@ class ForwarderProvider extends RemoteIdentityServerProvider { @Override List populate(List mappings) { - List mappingsFound = new ArrayList<>() + List mappingsToDo = new ArrayList<>(mappings) + List mappingsFoundGlobal = new ArrayList<>() - // TODO + for (String root : cfg.getServers()) { + log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo) + log.info("Querying {}", root) + List mappingsFound = find(root, mappingsToDo) + log.info("{} returned {} mappings", root, mappingsFound.size()) + mappingsFoundGlobal.addAll(mappingsFound) + mappingsToDo.removeAll(mappingsFound) + } - return mappingsFound + return mappingsFoundGlobal } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy index c3ed6a0..85985da 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy @@ -21,14 +21,26 @@ package io.kamax.mxisd.lookup.provider import groovy.json.JsonException +import groovy.json.JsonOutput import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType +import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest import io.kamax.mxisd.lookup.ThreePidMapping +import org.apache.http.HttpEntity +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.entity.EntityBuilder +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.ContentType +import org.apache.http.impl.client.HttpClients import org.slf4j.Logger import org.slf4j.LoggerFactory abstract class RemoteIdentityServerProvider implements ThreePidProvider { + public static final String THREEPID_TEST_MEDIUM = "email" + public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org" + private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class) private JsonSlurper json = new JsonSlurper() @@ -38,6 +50,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { return false } + boolean isUsableIdentityServer(String remote) { + try { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}" + ).openConnection() + + if (rootSrvConn.getResponseCode() != 200) { + return false + } + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + return false + } + + return true + } catch (IOException | JsonException e) { + log.info("{} is not a usable Identity Server: {}", remote, e.getMessage()) + return false + } + } + + Optional performLookup(String remote, String medium, String address) throws IOException, JsonException { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}" + ).openConnection() + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + log.info("Found 3PID mapping: {}", output) + + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(output['medium'].toString()) + mapping.setValue(output['address'].toString()) + mapping.setMxid(output['mxid'].toString()) + return Optional.of(mapping) + } + + log.info("Empty 3PID mapping from {}", remote) + return Optional.empty() + } + Optional find(String remote, ThreePidType type, String threePid) { log.info("Looking up {} 3PID {} using {}", type, threePid, remote) @@ -66,9 +120,48 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { List find(String remote, List mappings) { List mappingsFound = new ArrayList<>() - // TODO + ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest() + mappingRequest.setMappings(mappings) - return mappingsFound + String url = "${remote}/_matrix/identity/api/v1/bulk_lookup" + HttpClient client = HttpClients.createDefault() + try { + HttpPost request = new HttpPost(url) + request.setEntity( + EntityBuilder.create() + .setText(JsonOutput.toJson(mappingRequest)) + .setContentType(ContentType.APPLICATION_JSON) + .build() + ) + + HttpResponse response = client.execute(request) + try { + if (response.getStatusLine().getStatusCode() != 200) { + log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode()) + return mappingsFound + } + + HttpEntity entity = response.getEntity() + if (entity != null) { + ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText()) + for (List mappingRaw : input.getThreepids()) { + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(mappingRaw.get(0)) + mapping.setValue(mappingRaw.get(1)) + mapping.setMxid(mappingRaw.get(2)) + mappingsFound.add(mapping) + } + } else { + log.info("HTTP response from {} was empty", remote) + } + + return mappingsFound + } finally { + response.close() + } + } finally { + client.close() + } } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy index 50518b6..92d1786 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy @@ -114,14 +114,18 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea if (mapToDo.isEmpty()) { log.info("No more mappings to lookup") break + } else { + log.info("{} mappings remaining overall", mapToDo.size()) } + log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName()) List mapFound = provider.populate(mapToDo) + log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size()) mapFoundAll.addAll(mapFound) mapToDo.removeAll(mapFound) } - return mapFoundAll; + return mapFoundAll } } From 84cbf17e9c28b2e153de3eba42057619bacb3394 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Tue, 4 Apr 2017 02:17:41 +0200 Subject: [PATCH 3/3] Bulk lookup implementation, part 3 - Optimize DNS based bulk lookups --- .../lookup/provider/DnsLookupProvider.groovy | 60 +++++++++++++++---- .../RemoteIdentityServerProvider.groovy | 2 + 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy index 3d0b328..d8033f7 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy @@ -33,6 +33,8 @@ import org.xbill.DNS.Lookup import org.xbill.DNS.SRVRecord import org.xbill.DNS.Type +import java.util.concurrent.ForkJoinPool +import java.util.concurrent.RecursiveTask import java.util.function.Function @Component @@ -129,7 +131,6 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { @Override List populate(List mappings) { - List mappingsFound = new ArrayList<>() Map> domains = new HashMap<>() for (ThreePidMapping mapping : mappings) { @@ -157,20 +158,59 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { } log.info("Looking mappings across {} domains", domains.keySet().size()) - for (String domain : domains.keySet()) { - Optional baseUrl = findIdentityServerForDomain(domain) - if (!baseUrl.isPresent()) { - log.info("No usable Identity server for domain {}", domain) - continue + ForkJoinPool pool = new ForkJoinPool() + RecursiveTask> task = new RecursiveTask>() { + + @Override + protected List compute() { + List mappingsFound = new ArrayList<>() + List tasks = new ArrayList<>() + + for (String domain : domains.keySet()) { + DomainBulkLookupTask domainTask = new DomainBulkLookupTask(domain, domains.get(domain)) + domainTask.fork() + tasks.add(domainTask) + } + + for (DomainBulkLookupTask task : tasks) { + mappingsFound.addAll(task.join()) + } + + return mappingsFound } - - List domainMappings = find(baseUrl.get(), domains.get(domain)) - log.info("Found {} mappings in domain {}", domainMappings.size(), domain) - mappingsFound.addAll(domainMappings) } + pool.submit(task) + pool.shutdown() + List mappingsFound = task.join() log.info("Found {} mappings overall", mappingsFound.size()) return mappingsFound } + private class DomainBulkLookupTask extends RecursiveTask> { + + private String domain + private List mappings + + DomainBulkLookupTask(String domain, List mappings) { + this.domain = domain + this.mappings = mappings + } + + @Override + protected List compute() { + List domainMappings = new ArrayList<>() + + Optional baseUrl = findIdentityServerForDomain(domain) + if (!baseUrl.isPresent()) { + log.info("No usable Identity server for domain {}", domain) + } else { + domainMappings.addAll(find(baseUrl.get(), mappings)) + log.info("Found {} mappings in domain {}", domainMappings.size(), domain) + } + + return domainMappings + } + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy index 85985da..abcee20 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy @@ -55,6 +55,8 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}" ).openConnection() + // TODO turn this into a configuration property + rootSrvConn.setConnectTimeout(2000) if (rootSrvConn.getResponseCode() != 200) { return false