cqrenet/mxids
3
0
Fork 0
Code Pull Requests 10 Actions Releases Activity

Bulk lookup implementation, part 1

Browse Source 
- LDAP bulk lookup
This commit is contained in:
Maxime Dor
2017-04-03 04:11:37 +02:00
parent 8bd17d3ffa
commit 158a1e6354
14 changed files with 401 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/main/groovy/io/kamax/mxisd/lookup/ALookupRequest.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* mxisd - Matrix Identity Server Daemon
* Copyright (C) 2017 Maxime Dor
*
* https://max.kamax.io/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package io.kamax.mxisd.lookup;
public abstract class ALookupRequest {
private String requester;
public String getRequester() {
return requester;
}
public void setRequester(String requester) {
this.requester = requester;
}
}

37
src/main/groovy/io/kamax/mxisd/lookup/BulkLookupRequest.java Normal file
View File

@@ -0,0 +1,37 @@
/*
* mxisd - Matrix Identity Server Daemon
* Copyright (C) 2017 Maxime Dor
*
* https://max.kamax.io/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package io.kamax.mxisd.lookup;
import java.util.List;
public class BulkLookupRequest extends ALookupRequest {
private List<ThreePidMapping> mappings;
public List<ThreePidMapping> getMappings() {
return mappings;
}
public void setMappings(List<ThreePidMapping> mappings) {
this.mappings = mappings;
}
}

11
src/main/groovy/io/kamax/mxisd/lookup/LookupRequest.groovy → src/main/groovy/io/kamax/mxisd/lookup/SingleLookupRequest.groovy
View File

@@ -22,20 +22,11 @@ package io.kamax.mxisd.lookup
import io.kamax.mxisd.api.ThreePidType
class LookupRequest {
class SingleLookupRequest extends ALookupRequest {
private String requester
private ThreePidType type
private String threePid
String getRequester() {
return requester
}
void setRequester(String requester) {
this.requester = requester
}
ThreePidType getType() {
return type
}

53
src/main/groovy/io/kamax/mxisd/lookup/ThreePidMapping.java Normal file
View File

@@ -0,0 +1,53 @@
/*
* mxisd - Matrix Identity Server Daemon
* Copyright (C) 2017 Maxime Dor
*
* https://max.kamax.io/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package io.kamax.mxisd.lookup;
public class ThreePidMapping {
private String medium;
private String value;
private String mxid;
public String getMedium() {
return medium;
}
public void setMedium(String medium) {
this.medium = medium;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getMxid() {
return mxid;
}
public void setMxid(String mxid) {
this.mxid = mxid;
}
}

14
src/main/groovy/io/kamax/mxisd/lookup/provider/DnsLookupProvider.groovy
View File

@@ -22,7 +22,8 @@ package io.kamax.mxisd.lookup.provider
import io.kamax.mxisd.api.ThreePidType
import io.kamax.mxisd.config.ServerConfig
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
import org.apache.commons.lang.StringUtils
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@@ -46,7 +47,7 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
}
@Override
Optional<?> find(LookupRequest request) {
Optional<?> find(SingleLookupRequest request) {
log.info("Performing DNS lookup for {}", request.getThreePid())
if (ThreePidType.email != request.getType()) {
log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid())
@@ -94,4 +95,13 @@ class DnsLookupProvider extends RemoteIdentityServerProvider {
return find(baseUrl, request.getType(), request.getThreePid())
}
@Override
List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
List<ThreePidMapping> mappingsFound = new ArrayList<>()
// TODO
return mappingsFound
}
}

14
src/main/groovy/io/kamax/mxisd/lookup/provider/ForwarderProvider.groovy
View File

@@ -21,7 +21,8 @@
package io.kamax.mxisd.lookup.provider
import io.kamax.mxisd.config.ForwardConfig
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
@@ -37,7 +38,7 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
}
@Override
Optional<?> find(LookupRequest request) {
Optional<?> find(SingleLookupRequest request) {
for (String root : cfg.getServers()) {
Optional<?> answer = find(root, request.getType(), request.getThreePid())
if (answer.isPresent()) {
@@ -48,4 +49,13 @@ class ForwarderProvider extends RemoteIdentityServerProvider {
return Optional.empty()
}
@Override
List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
List<ThreePidMapping> mappingsFound = new ArrayList<>()
// TODO
return mappingsFound
}
}

127
src/main/groovy/io/kamax/mxisd/lookup/provider/LdapProvider.groovy
View File

@@ -20,9 +20,11 @@
package io.kamax.mxisd.lookup.provider
import io.kamax.mxisd.api.ThreePidType
import io.kamax.mxisd.config.LdapConfig
import io.kamax.mxisd.config.ServerConfig
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
import org.apache.commons.lang.StringUtils
import org.apache.directory.api.ldap.model.cursor.EntryCursor
import org.apache.directory.api.ldap.model.entry.Attribute
@@ -58,56 +60,65 @@ class LdapProvider implements ThreePidProvider {
return 20
}
Optional<String> lookup(LdapConnection conn, ThreePidType medium, String value) {
Optional<String> queryOpt = ldapCfg.getMapping(medium)
if (!queryOpt.isPresent()) {
log.warn("{} is not a supported 3PID type for LDAP lookup", medium)
return Optional.empty()
}
String searchQuery = queryOpt.get().replaceAll("%3pid", value)
EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
try {
if (cursor.next()) {
Attribute attribute = cursor.get().get(ldapCfg.getAttribute())
if (attribute != null) {
String data = attribute.get().toString()
if (data.length() < 1) {
log.warn("Bind was found but value is empty")
return Optional.empty()
}
StringBuilder matrixId = new StringBuilder()
// TODO Should we turn this block into a map of functions?
if (StringUtils.equals(UID, ldapCfg.getType())) {
matrixId.append("@").append(data).append(":").append(srvCfg.getName())
} else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) {
matrixId.append(data)
} else {
log.warn("Bind was found but type ${ldapCfg.getType()} is not supported")
return Optional.empty()
}
log.info("Found a match in LDAP")
return Optional.of(matrixId.toString())
}
}
} finally {
cursor.close()
}
return Optional.empty()
}
@Override
Optional<?> find(LookupRequest request) {
Optional<?> find(SingleLookupRequest request) {
log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}")
LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
try {
conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
Optional<String> queryOpt = ldapCfg.getMapping(request.getType())
if (!queryOpt.isPresent()) {
log.warn("{} is not a supported 3PID type for LDAP lookup", request.getType())
return Optional.empty()
}
String searchQuery = queryOpt.get().replaceAll("%3pid", request.getThreePid())
EntryCursor cursor = conn.search(ldapCfg.getBaseDn(), searchQuery, SearchScope.SUBTREE, ldapCfg.getAttribute())
try {
if (cursor.next()) {
Attribute attribute = cursor.get().get(ldapCfg.getAttribute())
if (attribute != null) {
String data = attribute.get().toString()
if (data.length() < 1) {
log.warn("Bind was found but value is empty")
return Optional.empty()
}
StringBuilder matrixId = new StringBuilder()
// TODO Should we turn this block into a map of functions?
if (StringUtils.equals(UID, ldapCfg.getType())) {
matrixId.append("@").append(data).append(":").append(srvCfg.getName())
} else if (StringUtils.equals(MATRIX_ID, ldapCfg.getType())) {
matrixId.append(data)
} else {
log.warn("Bind was found but type ${ldapCfg.getType()} is not supported")
return Optional.empty()
}
log.info("Found a match in LDAP")
return Optional.of([
address : request.getThreePid(),
medium : request.getType(),
mxid : matrixId.toString(),
not_before: 0,
not_after : 9223372036854775807,
ts : 0
])
}
}
} finally {
cursor.close()
Optional<String> mxid = lookup(conn, request.getType(), request.getThreePid())
if (mxid.isPresent()) {
return Optional.of([
address : request.getThreePid(),
medium : request.getType(),
mxid : mxid.get(),
not_before: 0,
not_after : 9223372036854775807,
ts : 0
])
}
} finally {
conn.close()
@@ -117,4 +128,32 @@ class LdapProvider implements ThreePidProvider {
return Optional.empty()
}
@Override
List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
log.info("Looking up {} mappings", mappings.size())
List<ThreePidMapping> mappingsFound = new ArrayList<>()
LdapConnection conn = new LdapNetworkConnection(ldapCfg.getHost(), ldapCfg.getPort())
try {
conn.bind(ldapCfg.getBindDn(), ldapCfg.getBindPassword())
for (ThreePidMapping mapping : mappings) {
try {
ThreePidType type = ThreePidType.valueOf(mapping.getMedium())
Optional<String> mxid = lookup(conn, type, mapping.getValue())
if (mxid.isPresent()) {
mapping.setMxid(mxid.get())
mappingsFound.add(mapping)
}
} catch (IllegalArgumentException e) {
log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium())
}
}
} finally {
conn.close()
}
return mappingsFound
}
}

9
src/main/groovy/io/kamax/mxisd/lookup/provider/RemoteIdentityServerProvider.groovy
View File

@@ -23,6 +23,7 @@ package io.kamax.mxisd.lookup.provider
import groovy.json.JsonException
import groovy.json.JsonSlurper
import io.kamax.mxisd.api.ThreePidType
import io.kamax.mxisd.lookup.ThreePidMapping
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@@ -62,4 +63,12 @@ abstract class RemoteIdentityServerProvider implements ThreePidProvider {
}
}
List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
List<ThreePidMapping> mappingsFound = new ArrayList<>()
// TODO
return mappingsFound
}
}

7
src/main/groovy/io/kamax/mxisd/lookup/provider/ThreePidProvider.groovy
View File

@@ -20,7 +20,8 @@
package io.kamax.mxisd.lookup.provider
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
interface ThreePidProvider {
@@ -31,6 +32,8 @@ interface ThreePidProvider {
*/
int getPriority() // Should not be here but let's KISS for now
Optional<?> find(LookupRequest request)
Optional<?> find(SingleLookupRequest request)
List<ThreePidMapping> populate(List<ThreePidMapping> mappings)
}

8
src/main/groovy/io/kamax/mxisd/lookup/strategy/LookupStrategy.groovy
View File

@@ -20,10 +20,14 @@
package io.kamax.mxisd.lookup.strategy
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.BulkLookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
interface LookupStrategy {
Optional<?> find(LookupRequest request)
Optional<?> find(SingleLookupRequest request)
List<ThreePidMapping> find(BulkLookupRequest requests)
}

47
src/main/groovy/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.groovy
View File

@@ -22,7 +22,10 @@ package io.kamax.mxisd.lookup.strategy
import edazdarevic.commons.net.CIDRUtils
import io.kamax.mxisd.config.RecursiveLookupConfig
import io.kamax.mxisd.lookup.LookupRequest
import io.kamax.mxisd.lookup.ALookupRequest
import io.kamax.mxisd.lookup.BulkLookupRequest
import io.kamax.mxisd.lookup.SingleLookupRequest
import io.kamax.mxisd.lookup.ThreePidMapping
import io.kamax.mxisd.lookup.provider.ThreePidProvider
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@@ -63,8 +66,9 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
}
}
@Override
Optional<?> find(LookupRequest request) {
List<ThreePidProvider> listUsableProviders(ALookupRequest request) {
List<ThreePidProvider> usableProviders = new ArrayList<>()
boolean canRecurse = false
if (recursiveCfg.isEnabled()) {
log.debug("Checking {} CIDRs for recursion", allowedCidr.size())
@@ -80,17 +84,44 @@ class RecursivePriorityLookupStrategy implements LookupStrategy, InitializingBea
}
log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse)
for (ThreePidProvider provider : providers) {
if (provider.isLocal() || canRecurse) {
Optional<?> lookupDataOpt = provider.find(request)
if (lookupDataOpt.isPresent()) {
return lookupDataOpt
}
usableProviders.add(provider)
}
}
return usableProviders
}
@Override
Optional<?> find(SingleLookupRequest request) {
for (ThreePidProvider provider : listUsableProviders(request)) {
Optional<?> lookupDataOpt = provider.find(request)
if (lookupDataOpt.isPresent()) {
return lookupDataOpt
}
}
return Optional.empty()
}
@Override
List<ThreePidMapping> find(BulkLookupRequest request) {
List<ThreePidMapping> mapToDo = new ArrayList<>(request.getMappings())
List<ThreePidMapping> mapFoundAll = new ArrayList<>()
for (ThreePidProvider provider : listUsableProviders(request)) {
if (mapToDo.isEmpty()) {
log.info("No more mappings to lookup")
break
}
List<ThreePidMapping> mapFound = provider.populate(mapToDo)
mapFoundAll.addAll(mapFound)
mapToDo.removeAll(mapFound)
}
return mapFoundAll;
}
}