diff --git a/build.gradle b/build.gradle index 46ecc13..f169a1a 100644 --- a/build.gradle +++ b/build.gradle @@ -54,6 +54,9 @@ dependencies { // DNS lookups compile 'dnsjava:dnsjava:2.1.8' + // HTTP connections + compile 'org.apache.httpcomponents:httpclient:4.5.3' + testCompile 'junit:junit:4.12' } diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java new file mode 100644 index 0000000..1f0dc34 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupAnswer.java @@ -0,0 +1,44 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.v1; + +import io.kamax.mxisd.lookup.ThreePidMapping; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +public class ClientBulkLookupAnswer { + + private List> threepids = new ArrayList<>(); + + public void addAll(Collection mappings) { + for (ThreePidMapping mapping : mappings) { + threepids.add(Arrays.asList(mapping.getMedium(), mapping.getValue(), mapping.getMxid())); + } + } + + public List> getThreepids() { + return threepids; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy new file mode 100644 index 0000000..7109c5c --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/ClientBulkLookupRequest.groovy @@ -0,0 +1,46 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.v1 + +import io.kamax.mxisd.lookup.ThreePidMapping + +class ClientBulkLookupRequest { + + private List> threepids = new ArrayList<>() + + List> getThreepids() { + return threepids + } + + void setThreepids(List> threepids) { + this.threepids = threepids + } + + void setMappings(List mappings) { + for (ThreePidMapping mapping : mappings) { + List threepid = new ArrayList<>() + threepid.add(mapping.getMedium()) + threepid.add(mapping.getValue()) + threepids.add(threepid) + } + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy b/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy index 80976de..dbc870e 100644 --- a/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy +++ b/src/main/groovy/io/kamax/mxisd/controller/v1/MappingController.groovy @@ -21,8 +21,11 @@ package io.kamax.mxisd.controller.v1 import groovy.json.JsonOutput +import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import io.kamax.mxisd.lookup.strategy.LookupStrategy import io.kamax.mxisd.signature.SignatureManager import org.apache.commons.lang.StringUtils @@ -36,11 +39,13 @@ import org.springframework.web.bind.annotation.RestController import javax.servlet.http.HttpServletRequest import static org.springframework.web.bind.annotation.RequestMethod.GET +import static org.springframework.web.bind.annotation.RequestMethod.POST @RestController class MappingController { private Logger log = LoggerFactory.getLogger(MappingController.class) + private JsonSlurper json = new JsonSlurper() @Autowired private LookupStrategy strategy @@ -55,7 +60,7 @@ class MappingController { ThreePidType type = ThreePidType.valueOf(medium) - LookupRequest lookupRequest = new LookupRequest() + SingleLookupRequest lookupRequest = new SingleLookupRequest() lookupRequest.setRequester(remote) lookupRequest.setType(type) lookupRequest.setThreePid(address) @@ -75,4 +80,27 @@ class MappingController { return JsonOutput.toJson(lookup) } + @RequestMapping(value = "/_matrix/identity/api/v1/bulk_lookup", method = POST) + String bulkLookup(HttpServletRequest request) { + String remote = StringUtils.defaultIfBlank(request.getHeader("X-FORWARDED-FOR"), request.getRemoteAddr()) + log.info("Got request from {}", remote) + + BulkLookupRequest lookupRequest = new BulkLookupRequest() + lookupRequest.setRequester(remote) + + ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(request.getInputStream().getText()) + List mappings = new ArrayList<>() + for (List mappingRaw : input.getThreepids()) { + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(mappingRaw.get(0)) + mapping.setValue(mappingRaw.get(1)) + mappings.add(mapping) + } + lookupRequest.setMappings(mappings) + + ClientBulkLookupAnswer answer = new ClientBulkLookupAnswer() + answer.addAll(strategy.find(lookupRequest)) + return JsonOutput.toJson(answer) + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java b/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java new file mode 100644 index 0000000..58aa305 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java @@ -0,0 +1,35 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +public abstract class ALookupRequest { + + private String requester; + + public String getRequester() { + return requester; + } + + public void setRequester(String requester) { + this.requester = requester; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java b/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java new file mode 100644 index 0000000..51b6284 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java @@ -0,0 +1,37 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +import java.util.List; + +public class BulkLookupRequest extends ALookupRequest { + + private List mappings; + + public List getMappings() { + return mappings; + } + + public void setMappings(List mappings) { + this.mappings = mappings; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy b/src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy similarity index 85% rename from src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy rename to src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy index a9b3989..ba256dd 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy @@ -22,20 +22,11 @@ package io.kamax.mxisd.lookup import io.kamax.mxisd.api.ThreePidType -class LookupRequest { +class SingleLookupRequest extends ALookupRequest { - private String requester private ThreePidType type private String threePid - String getRequester() { - return requester - } - - void setRequester(String requester) { - this.requester = requester - } - ThreePidType getType() { return type } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java new file mode 100644 index 0000000..9d37783 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java @@ -0,0 +1,77 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.lookup; + +import groovy.json.JsonOutput; + +public class ThreePidMapping { + + private String medium; + private String value; + private String mxid; + + public String getMedium() { + return medium; + } + + public void setMedium(String medium) { + this.medium = medium; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getMxid() { + return mxid; + } + + public void setMxid(String mxid) { + this.mxid = mxid; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ThreePidMapping that = (ThreePidMapping) o; + + if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false; + return value != null ? value.equals(that.value) : that.value == null; + } + + @Override + public int hashCode() { + int result = medium != null ? medium.hashCode() : 0; + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return JsonOutput.toJson(this); + } +} diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy index 697d465..d8033f7 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy @@ -22,7 +22,8 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.api.ThreePidType import io.kamax.mxisd.config.ServerConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import org.apache.commons.lang.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -32,6 +33,10 @@ import org.xbill.DNS.Lookup import org.xbill.DNS.SRVRecord import org.xbill.DNS.Type +import java.util.concurrent.ForkJoinPool +import java.util.concurrent.RecursiveTask +import java.util.function.Function + @Component class DnsLookupProvider extends RemoteIdentityServerProvider { @@ -45,23 +50,28 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { return 10 } - @Override - Optional find(LookupRequest request) { - log.info("Performing DNS lookup for {}", request.getThreePid()) - if (ThreePidType.email != request.getType()) { - log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + String getSrvRecordName(String domain) { + return "_matrix-identity._tcp." + domain + } + + Optional getDomain(String email) { + int atIndex = email.lastIndexOf("@") + if (atIndex == -1) { return Optional.empty() } - String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) - log.info("Domain name for {}: {}", request.getThreePid(), domain) + return Optional.of(email.substring(atIndex + 1)) + } + + // TODO use caching mechanism + Optional findIdentityServerForDomain(String domain) { if (StringUtils.equals(srvCfg.getName(), domain)) { - log.warn("We are authoritative for ${domain}, no remote lookup - is your server.name configured properly?") + log.info("We are authoritative for {}, no remote lookup", domain) return Optional.empty() } log.info("Performing SRV lookup") - String lookupDns = "_matrix-identity._tcp." + domain + String lookupDns = getSrvRecordName(domain) log.info("Lookup name: {}", lookupDns) SRVRecord[] records = (SRVRecord[]) new Lookup(lookupDns, Type.SRV).run() @@ -78,11 +88,11 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { for (SRVRecord record : records) { log.info("Found SRV record: {}", record.toString()) String baseUrl = "https://${record.getTarget().toString(true)}:${record.getPort()}" - Optional answer = find(baseUrl, request.getType(), request.getThreePid()) - if (answer.isPresent()) { - return answer + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) } else { - log.info("No mapping found at {}", baseUrl) + log.info("{} is not a usable Identity Server", baseUrl) } } } else { @@ -91,7 +101,116 @@ class DnsLookupProvider extends RemoteIdentityServerProvider { log.info("Performing basic lookup using domain name {}", domain) String baseUrl = "https://" + domain - return find(baseUrl, request.getType(), request.getThreePid()) + if (isUsableIdentityServer(baseUrl)) { + log.info("Found Identity Server for domain {} at {}", domain, baseUrl) + return Optional.of(baseUrl) + } else { + log.info("{} is not a usable Identity Server", baseUrl) + return Optional.empty() + } + } + + @Override + Optional find(SingleLookupRequest request) { + log.info("Performing DNS lookup for {}", request.getThreePid()) + if (ThreePidType.email != request.getType()) { + log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()) + return Optional.empty() + } + + String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1) + log.info("Domain name for {}: {}", request.getThreePid(), domain) + Optional baseUrl = findIdentityServerForDomain(domain) + + if (baseUrl.isPresent()) { + return performLookup(baseUrl.get(), request.getType().toString(), request.getThreePid()) + } + + return Optional.empty() + } + + @Override + List populate(List mappings) { + Map> domains = new HashMap<>() + + for (ThreePidMapping mapping : mappings) { + if (!StringUtils.equals(mapping.getMedium(), ThreePidType.email.toString())) { + log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue()) + continue + } + + Optional domainOpt = getDomain(mapping.getValue()) + if (!domainOpt.isPresent()) { + log.warn("No domain for 3PID {}", mapping.getValue()) + continue + } + + String domain = domainOpt.get() + List domainMappings = domains.computeIfAbsent(domain, new Function>() { + + @Override + List apply(String s) { + return new ArrayList<>() + } + + }) + domainMappings.add(mapping) + } + + log.info("Looking mappings across {} domains", domains.keySet().size()) + ForkJoinPool pool = new ForkJoinPool() + RecursiveTask> task = new RecursiveTask>() { + + @Override + protected List compute() { + List mappingsFound = new ArrayList<>() + List tasks = new ArrayList<>() + + for (String domain : domains.keySet()) { + DomainBulkLookupTask domainTask = new DomainBulkLookupTask(domain, domains.get(domain)) + domainTask.fork() + tasks.add(domainTask) + } + + for (DomainBulkLookupTask task : tasks) { + mappingsFound.addAll(task.join()) + } + + return mappingsFound + } + } + pool.submit(task) + pool.shutdown() + + List mappingsFound = task.join() + log.info("Found {} mappings overall", mappingsFound.size()) + return mappingsFound + } + + private class DomainBulkLookupTask extends RecursiveTask> { + + private String domain + private List mappings + + DomainBulkLookupTask(String domain, List mappings) { + this.domain = domain + this.mappings = mappings + } + + @Override + protected List compute() { + List domainMappings = new ArrayList<>() + + Optional baseUrl = findIdentityServerForDomain(domain) + if (!baseUrl.isPresent()) { + log.info("No usable Identity server for domain {}", domain) + } else { + domainMappings.addAll(find(baseUrl.get(), mappings)) + log.info("Found {} mappings in domain {}", domainMappings.size(), domain) + } + + return domainMappings + } } } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy index df67961..633dafd 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy @@ -21,13 +21,18 @@ package io.kamax.mxisd.lookup.provider import io.kamax.mxisd.config.ForwardConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class ForwarderProvider extends RemoteIdentityServerProvider { + private Logger log = LoggerFactory.getLogger(ForwarderProvider.class) + @Autowired private ForwardConfig cfg @@ -37,7 +42,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider { } @Override - Optional find(LookupRequest request) { + Optional find(SingleLookupRequest request) { for (String root : cfg.getServers()) { Optional answer = find(root, request.getType(), request.getThreePid()) if (answer.isPresent()) { @@ -48,4 +53,21 @@ class ForwarderProvider extends RemoteIdentityServerProvider { return Optional.empty() } + @Override + List populate(List mappings) { + List mappingsToDo = new ArrayList<>(mappings) + List mappingsFoundGlobal = new ArrayList<>() + + for (String root : cfg.getServers()) { + log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo) + log.info("Querying {}", root) + List mappingsFound = find(root, mappingsToDo) + log.info("{} returned {} mappings", root, mappingsFound.size()) + mappingsFoundGlobal.addAll(mappingsFound) + mappingsToDo.removeAll(mappingsFound) + } + + return mappingsFoundGlobal + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy index 9cb8e23..0cd4867 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy @@ -20,9 +20,11 @@ package io.kamax.mxisd.lookup.provider +import io.kamax.mxisd.api.ThreePidType import io.kamax.mxisd.config.LdapConfig import io.kamax.mxisd.config.ServerConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import org.apache.commons.lang.StringUtils import org.apache.directory.api.ldap.model.cursor.EntryCursor import org.apache.directory.api.ldap.model.entry.Attribute @@ -58,56 +60,65 @@ class LdapProvider implements ThreePidProvider { return 20 } + Optional lookup(LdapConnection conn, ThreePidType medium, String value) { + Optional queryOpt = ldapCfg.getMapping(medium) + if (!queryOpt.isPresent()) { + log.warn("{} is not a supported 3PID type for LDAP lookup", medium) + return Optional.empty() + } + + String searchQuery = queryOpt.get().replaceAll("%3pid", value) + EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute()) + try { + if (cursor.next()) { + Attribute attribute = cursor.get().get(ldapCfg.getAttribute()) + if (attribute != null) { + String data = attribute.get().toString() + if (data.length() < 1) { + log.warn("Bind was found but value is empty") + return Optional.empty() + } + + StringBuilder matrixId = new StringBuilder() + // TODO Should we turn this block into a map of functions? + if (StringUtils.equals(UID, ldapCfg.getType())) { + matrixId.append("@").append(data).append(":").append(srvCfg.getName()) + } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) { + matrixId.append(data) + } else { + log.warn("Bind was found but type ${ldapCfg.getType()} is not supported") + return Optional.empty() + } + + log.info("Found a match in LDAP") + return Optional.of(matrixId.toString()) + } + } + } finally { + cursor.close() + } + + return Optional.empty() + } + @Override - Optional find(LookupRequest request) { + Optional find(SingleLookupRequest request) { log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}") LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort()) try { conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword()) - Optional queryOpt = ldapCfg.getMapping(request.getType()) - if (!queryOpt.isPresent()) { - log.warn("{} is not a supported 3PID type for LDAP lookup", request.getType()) - return Optional.empty() - } - - String searchQuery = queryOpt.get().replaceAll("%3pid", request.getThreePid()) - EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute()) - try { - if (cursor.next()) { - Attribute attribute = cursor.get().get(ldapCfg.getAttribute()) - if (attribute != null) { - String data = attribute.get().toString() - if (data.length() < 1) { - log.warn("Bind was found but value is empty") - return Optional.empty() - } - - StringBuilder matrixId = new StringBuilder() - // TODO Should we turn this block into a map of functions? - if (StringUtils.equals(UID, ldapCfg.getType())) { - matrixId.append("@").append(data).append(":").append(srvCfg.getName()) - } else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) { - matrixId.append(data) - } else { - log.warn("Bind was found but type ${ldapCfg.getType()} is not supported") - return Optional.empty() - } - - log.info("Found a match in LDAP") - return Optional.of([ - address : request.getThreePid(), - medium : request.getType(), - mxid : matrixId.toString(), - not_before: 0, - not_after : 9223372036854775807, - ts : 0 - ]) - } - } - } finally { - cursor.close() + Optional mxid = lookup(conn, request.getType(), request.getThreePid()) + if (mxid.isPresent()) { + return Optional.of([ + address : request.getThreePid(), + medium : request.getType(), + mxid : mxid.get(), + not_before: 0, + not_after : 9223372036854775807, + ts : 0 + ]) } } finally { conn.close() @@ -117,4 +128,32 @@ class LdapProvider implements ThreePidProvider { return Optional.empty() } + @Override + List populate(List mappings) { + log.info("Looking up {} mappings", mappings.size()) + List mappingsFound = new ArrayList<>() + + LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort()) + try { + conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword()) + + for (ThreePidMapping mapping : mappings) { + try { + ThreePidType type = ThreePidType.valueOf(mapping.getMedium()) + Optional mxid = lookup(conn, type, mapping.getValue()) + if (mxid.isPresent()) { + mapping.setMxid(mxid.get()) + mappingsFound.add(mapping) + } + } catch (IllegalArgumentException e) { + log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium()) + } + } + } finally { + conn.close() + } + + return mappingsFound + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy index a90dd26..abcee20 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy @@ -21,13 +21,26 @@ package io.kamax.mxisd.lookup.provider import groovy.json.JsonException +import groovy.json.JsonOutput import groovy.json.JsonSlurper import io.kamax.mxisd.api.ThreePidType +import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping +import org.apache.http.HttpEntity +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.entity.EntityBuilder +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.ContentType +import org.apache.http.impl.client.HttpClients import org.slf4j.Logger import org.slf4j.LoggerFactory abstract class RemoteIdentityServerProvider implements ThreePidProvider { + public static final String THREEPID_TEST_MEDIUM = "email" + public static final String THREEPID_TEST_ADDRESS = "john.doe@example.org" + private Logger log = LoggerFactory.getLogger(RemoteIdentityServerProvider.class) private JsonSlurper json = new JsonSlurper() @@ -37,6 +50,50 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { return false } + boolean isUsableIdentityServer(String remote) { + try { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${THREEPID_TEST_MEDIUM}&address=${THREEPID_TEST_ADDRESS}" + ).openConnection() + // TODO turn this into a configuration property + rootSrvConn.setConnectTimeout(2000) + + if (rootSrvConn.getResponseCode() != 200) { + return false + } + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + return false + } + + return true + } catch (IOException | JsonException e) { + log.info("{} is not a usable Identity Server: {}", remote, e.getMessage()) + return false + } + } + + Optional performLookup(String remote, String medium, String address) throws IOException, JsonException { + HttpURLConnection rootSrvConn = (HttpURLConnection) new URL( + "${remote}/_matrix/identity/api/v1/lookup?medium=${medium}&address=${address}" + ).openConnection() + + def output = json.parseText(rootSrvConn.getInputStream().getText()) + if (output['address']) { + log.info("Found 3PID mapping: {}", output) + + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(output['medium'].toString()) + mapping.setValue(output['address'].toString()) + mapping.setMxid(output['mxid'].toString()) + return Optional.of(mapping) + } + + log.info("Empty 3PID mapping from {}", remote) + return Optional.empty() + } + Optional find(String remote, ThreePidType type, String threePid) { log.info("Looking up {} 3PID {} using {}", type, threePid, remote) @@ -62,4 +119,51 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider { } } + List find(String remote, List mappings) { + List mappingsFound = new ArrayList<>() + + ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest() + mappingRequest.setMappings(mappings) + + String url = "${remote}/_matrix/identity/api/v1/bulk_lookup" + HttpClient client = HttpClients.createDefault() + try { + HttpPost request = new HttpPost(url) + request.setEntity( + EntityBuilder.create() + .setText(JsonOutput.toJson(mappingRequest)) + .setContentType(ContentType.APPLICATION_JSON) + .build() + ) + + HttpResponse response = client.execute(request) + try { + if (response.getStatusLine().getStatusCode() != 200) { + log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode()) + return mappingsFound + } + + HttpEntity entity = response.getEntity() + if (entity != null) { + ClientBulkLookupRequest input = (ClientBulkLookupRequest) json.parseText(entity.getContent().getText()) + for (List mappingRaw : input.getThreepids()) { + ThreePidMapping mapping = new ThreePidMapping() + mapping.setMedium(mappingRaw.get(0)) + mapping.setValue(mappingRaw.get(1)) + mapping.setMxid(mappingRaw.get(2)) + mappingsFound.add(mapping) + } + } else { + log.info("HTTP response from {} was empty", remote) + } + + return mappingsFound + } finally { + response.close() + } + } finally { + client.close() + } + } + } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy b/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy index 71cd127..d102fad 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy @@ -20,7 +20,8 @@ package io.kamax.mxisd.lookup.provider -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping interface ThreePidProvider { @@ -31,6 +32,8 @@ interface ThreePidProvider { */ int getPriority() // Should not be here but let's KISS for now - Optional find(LookupRequest request) + Optional find(SingleLookupRequest request) + + List populate(List mappings) } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy index a97f796..cfd9e79 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy @@ -20,10 +20,14 @@ package io.kamax.mxisd.lookup.strategy -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping interface LookupStrategy { - Optional find(LookupRequest request) + Optional find(SingleLookupRequest request) + + List find(BulkLookupRequest requests) } diff --git a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy index 0c8f334..92d1786 100644 --- a/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy +++ b/src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy @@ -22,7 +22,10 @@ package io.kamax.mxisd.lookup.strategy import edazdarevic.commons.net.CIDRUtils import io.kamax.mxisd.config.RecursiveLookupConfig -import io.kamax.mxisd.lookup.LookupRequest +import io.kamax.mxisd.lookup.ALookupRequest +import io.kamax.mxisd.lookup.BulkLookupRequest +import io.kamax.mxisd.lookup.SingleLookupRequest +import io.kamax.mxisd.lookup.ThreePidMapping import io.kamax.mxisd.lookup.provider.ThreePidProvider import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -63,8 +66,9 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea } } - @Override - Optional find(LookupRequest request) { + List listUsableProviders(ALookupRequest request) { + List usableProviders = new ArrayList<>() + boolean canRecurse = false if (recursiveCfg.isEnabled()) { log.debug("Checking {} CIDRs for recursion", allowedCidr.size()) @@ -80,17 +84,48 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea } log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse) - for (ThreePidProvider provider : providers) { if (provider.isLocal() || canRecurse) { - Optional lookupDataOpt = provider.find(request) - if (lookupDataOpt.isPresent()) { - return lookupDataOpt - } + usableProviders.add(provider) + } + } + + return usableProviders + } + + @Override + Optional find(SingleLookupRequest request) { + for (ThreePidProvider provider : listUsableProviders(request)) { + Optional lookupDataOpt = provider.find(request) + if (lookupDataOpt.isPresent()) { + return lookupDataOpt } } return Optional.empty() } + @Override + List find(BulkLookupRequest request) { + List mapToDo = new ArrayList<>(request.getMappings()) + List mapFoundAll = new ArrayList<>() + + for (ThreePidProvider provider : listUsableProviders(request)) { + if (mapToDo.isEmpty()) { + log.info("No more mappings to lookup") + break + } else { + log.info("{} mappings remaining overall", mapToDo.size()) + } + + log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName()) + List mapFound = provider.populate(mapToDo) + log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size()) + mapFoundAll.addAll(mapFound) + mapToDo.removeAll(mapFound) + } + + return mapFoundAll + } + }