From 182f3c4bc3008a95fd38a9ef0c1c870ba61ae2b0 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Wed, 27 Sep 2017 04:37:45 +0200 Subject: [PATCH 01/14] Skeleton for HS User directory integration --- .../directory/UserDirectoryController.java | 59 +++++++++++++ .../io/UserDirectorySearchRequest.java | 35 ++++++++ .../io/UserDirectorySearchResult.java | 83 +++++++++++++++++++ .../mxisd/directory/DirectoryManager.java | 36 ++++++++ 4 files changed, 213 insertions(+) create mode 100644 src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java create mode 100644 src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java create mode 100644 src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java create mode 100644 src/main/java/io/kamax/mxisd/directory/DirectoryManager.java diff --git a/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java b/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java new file mode 100644 index 0000000..2eaf4a2 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java @@ -0,0 +1,59 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.directory; + +import com.google.gson.Gson; +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchRequest; +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.directory.DirectoryManager; +import io.kamax.mxisd.util.GsonParser; +import io.kamax.mxisd.util.GsonUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.URI; + +@RestController +@CrossOrigin +@RequestMapping(path = "/_matrix/client/r0/user_directory", produces = MediaType.APPLICATION_JSON_UTF8_VALUE) +public class UserDirectoryController { + + private Gson gson = GsonUtil.build(); + private GsonParser parser = new GsonParser(gson); + + @Autowired + private DirectoryManager mgr; + + @RequestMapping(path = "/search") + public String search(HttpServletRequest request, @RequestParam("access_token") String accessToken) throws IOException { + UserDirectorySearchRequest searchQuery = parser.parse(request, UserDirectorySearchRequest.class); + URI target = URI.create(request.getRequestURL().toString()); + UserDirectorySearchResult result = mgr.search(target, accessToken, searchQuery.getSearchTerm()); + return gson.toJson(result); + } + +} diff --git a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java new file mode 100644 index 0000000..e269786 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java @@ -0,0 +1,35 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.directory.io; + +public class UserDirectorySearchRequest { + + private String searchTerm; + + public String getSearchTerm() { + return searchTerm; + } + + public void setSearchTerm(String searchTerm) { + this.searchTerm = searchTerm; + } + +} diff --git a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java new file mode 100644 index 0000000..128385c --- /dev/null +++ b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java @@ -0,0 +1,83 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.controller.directory.io; + +import java.util.ArrayList; +import java.util.List; + +public class UserDirectorySearchResult { + + public static class Result { + + private String displayName; + private String avatarUrl; + private String userId; + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getAvatarUrl() { + return avatarUrl; + } + + public void setAvatarUrl(String avatarUrl) { + this.avatarUrl = avatarUrl; + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + } + + private boolean limited; + private List results = new ArrayList<>(); + + public boolean isLimited() { + return limited; + } + + public void setLimited(boolean limited) { + this.limited = limited; + } + + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } + + public void addResult(Result result) { + this.results.add(result); + } + +} diff --git a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java new file mode 100644 index 0000000..fd609d7 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java @@ -0,0 +1,36 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.directory; + +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.exception.NotImplementedException; +import org.springframework.stereotype.Component; + +import java.net.URI; + +@Component +public class DirectoryManager { + + public UserDirectorySearchResult search(URI target, String accessToken, String query) { + throw new NotImplementedException("user directory - search"); + } + +} From 4f3ecc19f301df1481b48746bc3595d01154ff63 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Fri, 29 Sep 2017 02:52:05 +0200 Subject: [PATCH 02/14] Directory integration prototype using Google Firebase auth + Synapse SQL --- build.gradle | 3 + .../firebase/GoogleFirebaseAuthenticator.java | 57 +----- .../firebase/GoogleFirebaseBackend.java | 84 +++++++++ .../firebase/GoogleFirebaseProvider.java | 58 +----- .../mxisd/backend/sql/SqlConnectionPool.java | 47 +++++ .../backend/sql/SqlDirectoryProvider.java | 116 ++++++++++++ .../backend/sql/SqlThreePidProvider.java | 60 ++++--- .../sql/SynapseSqliteDirectoryProvider.java | 71 ++++++++ .../mxisd/config/DnsOverwriteConfig.java | 106 +++++++++++ .../kamax/mxisd/config/DnsOverwriteEntry.java | 67 ------- .../io/kamax/mxisd/config/FirebaseConfig.java | 14 +- .../config/sql/SqlProviderAuthConfig.java | 21 --- .../mxisd/config/sql/SqlProviderConfig.java | 166 +++++++++++++++++- .../config/sql/SqlProviderIdentityConfig.java | 61 ------- .../synapse/SynapseSqlProviderConfig.java} | 27 +-- .../v1 => }/DefaultExceptionHandler.java | 13 +- .../directory/UserDirectoryController.java | 7 +- .../io/UserDirectorySearchRequest.java | 4 + .../mxisd/directory/DirectoryManager.java | 92 +++++++++- .../mxisd/directory/IDirectoryProvider.java | 33 ++++ .../kamax/mxisd/dns/ClientDnsOverwrite.java | 74 ++++++++ .../mxisd/dns/FederationDnsOverwrite.java | 61 +++++++ .../mxisd/exception/MatrixException.java | 2 +- .../exception/RemoteHomeServerException.java | 11 ++ .../mxisd/invitation/InvitationManager.java | 13 +- .../RecursivePriorityLookupStrategy.java | 30 ++-- src/main/resources/application.yaml | 13 +- 27 files changed, 959 insertions(+), 352 deletions(-) create mode 100644 src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java create mode 100644 src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java create mode 100644 src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java create mode 100644 src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java create mode 100644 src/main/java/io/kamax/mxisd/config/DnsOverwriteConfig.java delete mode 100644 src/main/java/io/kamax/mxisd/config/DnsOverwriteEntry.java delete mode 100644 src/main/java/io/kamax/mxisd/config/sql/SqlProviderAuthConfig.java delete mode 100644 src/main/java/io/kamax/mxisd/config/sql/SqlProviderIdentityConfig.java rename src/main/java/io/kamax/mxisd/config/{DnsOverwrite.java => sql/synapse/SynapseSqlProviderConfig.java} (59%) rename src/main/java/io/kamax/mxisd/controller/{identity/v1 => }/DefaultExceptionHandler.java (89%) create mode 100644 src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java create mode 100644 src/main/java/io/kamax/mxisd/dns/ClientDnsOverwrite.java create mode 100644 src/main/java/io/kamax/mxisd/dns/FederationDnsOverwrite.java create mode 100644 src/main/java/io/kamax/mxisd/exception/RemoteHomeServerException.java diff --git a/build.gradle b/build.gradle index b93d4fa..d5332a1 100644 --- a/build.gradle +++ b/build.gradle @@ -110,6 +110,9 @@ dependencies { // ORMLite compile 'com.j256.ormlite:ormlite-jdbc:5.0' + // Connection Pool + compile 'com.mchange:c3p0:0.9.5.2' + // SQLite compile 'org.xerial:sqlite-jdbc:3.20.0' diff --git a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseAuthenticator.java b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseAuthenticator.java index 5a9178a..441a003 100644 --- a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseAuthenticator.java +++ b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseAuthenticator.java @@ -20,11 +20,6 @@ package io.kamax.mxisd.backend.firebase; -import com.google.firebase.FirebaseApp; -import com.google.firebase.FirebaseOptions; -import com.google.firebase.auth.FirebaseAuth; -import com.google.firebase.auth.FirebaseCredential; -import com.google.firebase.auth.FirebaseCredentials; import com.google.firebase.auth.UserInfo; import com.google.i18n.phonenumbers.NumberParseException; import com.google.i18n.phonenumbers.PhoneNumberUtil; @@ -38,35 +33,17 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileInputStream; -import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -public class GoogleFirebaseAuthenticator implements AuthenticatorProvider { +public class GoogleFirebaseAuthenticator extends GoogleFirebaseBackend implements AuthenticatorProvider { private Logger log = LoggerFactory.getLogger(GoogleFirebaseAuthenticator.class); - private boolean isEnabled; - private FirebaseApp fbApp; - private FirebaseAuth fbAuth; - private PhoneNumberUtil phoneUtil = PhoneNumberUtil.getInstance(); - public GoogleFirebaseAuthenticator(boolean isEnabled) { - this.isEnabled = isEnabled; - } - - public GoogleFirebaseAuthenticator(String credsPath, String db) { - this(true); - try { - fbApp = FirebaseApp.initializeApp(getOpts(credsPath, db), "AuthenticationProvider"); - fbAuth = FirebaseAuth.getInstance(fbApp); - - log.info("Google Firebase Authentication is ready"); - } catch (IOException e) { - throw new RuntimeException("Error when initializing Firebase", e); - } + public GoogleFirebaseAuthenticator(boolean isEnabled, String credsPath, String db) { + super(isEnabled, "AuthenticationProvider", credsPath, db); } private void waitOnLatch(BackendAuthResult result, CountDownLatch l, String purpose) { @@ -105,30 +82,6 @@ public class GoogleFirebaseAuthenticator implements AuthenticatorProvider { } } - private FirebaseCredential getCreds(String credsPath) throws IOException { - if (StringUtils.isNotBlank(credsPath)) { - return FirebaseCredentials.fromCertificate(new FileInputStream(credsPath)); - } else { - return FirebaseCredentials.applicationDefault(); - } - } - - private FirebaseOptions getOpts(String credsPath, String db) throws IOException { - if (StringUtils.isBlank(db)) { - throw new IllegalArgumentException("Firebase database is not configured"); - } - - return new FirebaseOptions.Builder() - .setCredential(getCreds(credsPath)) - .setDatabaseUrl(db) - .build(); - } - - @Override - public boolean isEnabled() { - return isEnabled; - } - private void waitOnLatch(CountDownLatch l) { try { l.await(30, TimeUnit.SECONDS); @@ -149,7 +102,7 @@ public class GoogleFirebaseAuthenticator implements AuthenticatorProvider { String localpart = mxid.getLocalPart(); CountDownLatch l = new CountDownLatch(1); - fbAuth.verifyIdToken(password).addOnSuccessListener(token -> { + getFirebase().verifyIdToken(password).addOnSuccessListener(token -> { try { if (!StringUtils.equals(localpart, token.getUid())) { log.info("Failure to authenticate {}: Matrix ID localpart '{}' does not match Firebase UID '{}'", mxid, localpart, token.getUid()); @@ -161,7 +114,7 @@ public class GoogleFirebaseAuthenticator implements AuthenticatorProvider { log.info("{} was successfully authenticated", mxid); log.info("Fetching profile for {}", mxid); CountDownLatch userRecordLatch = new CountDownLatch(1); - fbAuth.getUser(token.getUid()).addOnSuccessListener(user -> { + getFirebase().getUser(token.getUid()).addOnSuccessListener(user -> { try { toEmail(result, user.getEmail()); toMsisdn(result, user.getPhoneNumber()); diff --git a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java new file mode 100644 index 0000000..9c5bcfc --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java @@ -0,0 +1,84 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.firebase; + +import com.google.firebase.FirebaseApp; +import com.google.firebase.FirebaseOptions; +import com.google.firebase.auth.FirebaseAuth; +import com.google.firebase.auth.FirebaseCredential; +import com.google.firebase.auth.FirebaseCredentials; +import com.google.firebase.database.FirebaseDatabase; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; + +public class GoogleFirebaseBackend { + + private Logger log = LoggerFactory.getLogger(GoogleFirebaseBackend.class); + + private boolean isEnabled; + private FirebaseAuth fbAuth; + protected FirebaseDatabase fbDb; + + GoogleFirebaseBackend(boolean isEnabled, String name, String credsPath, String db) { + this.isEnabled = isEnabled; + try { + FirebaseApp fbApp = FirebaseApp.initializeApp(getOpts(credsPath, db), name); + fbAuth = FirebaseAuth.getInstance(fbApp); + FirebaseDatabase.getInstance(fbApp); + + log.info("Google Firebase Authentication is ready"); + } catch (IOException e) { + throw new RuntimeException("Error when initializing Firebase", e); + } + } + + private FirebaseCredential getCreds(String credsPath) throws IOException { + if (StringUtils.isNotBlank(credsPath)) { + return FirebaseCredentials.fromCertificate(new FileInputStream(credsPath)); + } else { + return FirebaseCredentials.applicationDefault(); + } + } + + private FirebaseOptions getOpts(String credsPath, String db) throws IOException { + if (StringUtils.isBlank(db)) { + throw new IllegalArgumentException("Firebase database is not configured"); + } + + return new FirebaseOptions.Builder() + .setCredential(getCreds(credsPath)) + .setDatabaseUrl(db) + .build(); + } + + FirebaseAuth getFirebase() { + return fbAuth; + } + + public boolean isEnabled() { + return isEnabled; + } + +} diff --git a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseProvider.java b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseProvider.java index e37ff21..38b6a57 100644 --- a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseProvider.java @@ -20,11 +20,6 @@ package io.kamax.mxisd.backend.firebase; -import com.google.firebase.FirebaseApp; -import com.google.firebase.FirebaseOptions; -import com.google.firebase.auth.FirebaseAuth; -import com.google.firebase.auth.FirebaseCredential; -import com.google.firebase.auth.FirebaseCredentials; import com.google.firebase.auth.UserRecord; import com.google.firebase.tasks.OnFailureListener; import com.google.firebase.tasks.OnSuccessListener; @@ -34,72 +29,29 @@ import io.kamax.mxisd.lookup.SingleLookupReply; import io.kamax.mxisd.lookup.SingleLookupRequest; import io.kamax.mxisd.lookup.ThreePidMapping; import io.kamax.mxisd.lookup.provider.IThreePidProvider; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileInputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -public class GoogleFirebaseProvider implements IThreePidProvider { +public class GoogleFirebaseProvider extends GoogleFirebaseBackend implements IThreePidProvider { private Logger log = LoggerFactory.getLogger(GoogleFirebaseProvider.class); - - private boolean isEnabled; private String domain; - private FirebaseAuth fbAuth; - public GoogleFirebaseProvider(boolean isEnabled) { - this.isEnabled = isEnabled; - } - - public GoogleFirebaseProvider(String credsPath, String db, String domain) { - this(true); + public GoogleFirebaseProvider(boolean isEnabled, String credsPath, String db, String domain) { + super(isEnabled, "ThreePidProvider", credsPath, db); this.domain = domain; - - try { - FirebaseApp fbApp = FirebaseApp.initializeApp(getOpts(credsPath, db), "ThreePidProvider"); - fbAuth = FirebaseAuth.getInstance(fbApp); - - log.info("Google Firebase Authentication is ready"); - } catch (IOException e) { - throw new RuntimeException("Error when initializing Firebase", e); - } - } - - private FirebaseCredential getCreds(String credsPath) throws IOException { - if (StringUtils.isNotBlank(credsPath)) { - return FirebaseCredentials.fromCertificate(new FileInputStream(credsPath)); - } else { - return FirebaseCredentials.applicationDefault(); - } - } - - private FirebaseOptions getOpts(String credsPath, String db) throws IOException { - if (StringUtils.isBlank(db)) { - throw new IllegalArgumentException("Firebase database is not configured"); - } - - return new FirebaseOptions.Builder() - .setCredential(getCreds(credsPath)) - .setDatabaseUrl(db) - .build(); } private String getMxid(UserRecord record) { return new MatrixID(record.getUid(), domain).getId(); } - @Override - public boolean isEnabled() { - return isEnabled; - } - @Override public boolean isLocal() { return true; @@ -136,13 +88,13 @@ public class GoogleFirebaseProvider implements IThreePidProvider { if (ThreePidMedium.Email.is(medium)) { log.info("Performing E-mail 3PID lookup for {}", address); - fbAuth.getUserByEmail(address) + getFirebase().getUserByEmail(address) .addOnSuccessListener(success) .addOnFailureListener(failure); waitOnLatch(l); } else if (ThreePidMedium.PhoneNumber.is(medium)) { log.info("Performing msisdn 3PID lookup for {}", address); - fbAuth.getUserByPhoneNumber(address) + getFirebase().getUserByPhoneNumber(address) .addOnSuccessListener(success) .addOnFailureListener(failure); waitOnLatch(l); diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java new file mode 100644 index 0000000..1ec03a6 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java @@ -0,0 +1,47 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.sql; + +import com.mchange.v2.c3p0.ComboPooledDataSource; +import io.kamax.mxisd.config.sql.SqlProviderConfig; +import org.springframework.stereotype.Component; + +import java.sql.Connection; +import java.sql.SQLException; + +@Component +public class SqlConnectionPool { + + private ComboPooledDataSource ds; + + public SqlConnectionPool(SqlProviderConfig cfg) { + ds = new ComboPooledDataSource(); + ds.setJdbcUrl("jdbc:" + cfg.getType() + ":" + cfg.getConnection()); + ds.setMinPoolSize(1); + ds.setMaxPoolSize(10); + ds.setAcquireIncrement(2); + } + + public Connection get() throws SQLException { + return ds.getConnection(); + } + +} diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java new file mode 100644 index 0000000..ce7fcdb --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java @@ -0,0 +1,116 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.sql; + +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.sql.SqlProviderConfig; +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.directory.IDirectoryProvider; +import io.kamax.mxisd.exception.InternalServerError; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; + +import static io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult.Result; + + +@Component +public class SqlDirectoryProvider implements IDirectoryProvider { + + private Logger log = LoggerFactory.getLogger(SqlDirectoryProvider.class); + + protected SqlProviderConfig cfg; + private MatrixConfig mxCfg; + + private SqlConnectionPool pool; + + @Autowired + public SqlDirectoryProvider(SqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { + this.cfg = cfg; + this.pool = pool; + this.mxCfg = mxCfg; + } + + @Override + public boolean isEnabled() { + return cfg.isEnabled(); + } + + protected void setParameters(PreparedStatement stmt, String searchTerm) throws SQLException { + for (int i = 1; i <= stmt.getParameterMetaData().getParameterCount(); i++) { + stmt.setString(i, searchTerm); + } + } + + protected Optional processRow(ResultSet rSet) throws SQLException { + Result item = new Result(); + item.setUserId(rSet.getString(1)); + item.setDisplayName(rSet.getString(2)); + return Optional.of(item); + } + + public UserDirectorySearchResult search(String searchTerm, SqlProviderConfig.Query query) { + try (Connection conn = pool.get()) { + try (PreparedStatement stmt = conn.prepareStatement(query.getValue())) { + setParameters(stmt, searchTerm); + + try (ResultSet rSet = stmt.executeQuery()) { + UserDirectorySearchResult result = new UserDirectorySearchResult(); + result.setLimited(false); + + while (rSet.next()) { + processRow(rSet).ifPresent(e -> { + if (StringUtils.equalsIgnoreCase("localpart", query.getType())) { + e.setUserId("@" + e.getUserId() + mxCfg.getDomain()); + } + result.addResult(e); + }); + } + + return result; + } + } + } catch (SQLException e) { + throw new InternalServerError(e); + } + } + + @Override + public UserDirectorySearchResult searchByDisplayName(String searchTerm) { + log.info("Searching users by display name using '{}'", searchTerm); + return search(searchTerm, cfg.getDirectory().getQuery().getName()); + } + + @Override + public UserDirectorySearchResult searchBy3pid(String searchTerm) { + log.info("Searching users by 3PID using '{}'", searchTerm); + return search(searchTerm, cfg.getDirectory().getQuery().getThreepid()); + } + +} diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java index 0338fe0..b992dc9 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java @@ -33,7 +33,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.sql.*; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -43,11 +46,17 @@ public class SqlThreePidProvider implements IThreePidProvider { private Logger log = LoggerFactory.getLogger(SqlThreePidProvider.class); - @Autowired + private SqlProviderConfig cfg; private MatrixConfig mxCfg; + private SqlConnectionPool pool; + @Autowired - private SqlProviderConfig cfg; + public SqlThreePidProvider(SqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { + this.cfg = cfg; + this.pool = pool; + this.mxCfg = mxCfg; + } @Override public boolean isEnabled() { @@ -64,37 +73,36 @@ public class SqlThreePidProvider implements IThreePidProvider { return 20; } - private Connection getConn() throws SQLException { - return DriverManager.getConnection("jdbc:" + cfg.getType() + ":" + cfg.getConnection()); - } - @Override public Optional find(SingleLookupRequest request) { log.info("SQL lookup"); String stmtSql = StringUtils.defaultIfBlank(cfg.getIdentity().getMedium().get(request.getType()), cfg.getIdentity().getQuery()); log.info("SQL query: {}", stmtSql); - try (PreparedStatement stmt = getConn().prepareStatement(stmtSql)) { - stmt.setString(1, request.getType().toLowerCase()); - stmt.setString(2, request.getThreePid().toLowerCase()); + try (Connection conn = pool.get()) { + try (PreparedStatement stmt = conn.prepareStatement(stmtSql)) { + stmt.setString(1, request.getType().toLowerCase()); + stmt.setString(2, request.getThreePid().toLowerCase()); - ResultSet rSet = stmt.executeQuery(); - while (rSet.next()) { - String uid = rSet.getString("uid"); - log.info("Found match: {}", uid); - if (StringUtils.equals("uid", cfg.getIdentity().getType())) { - log.info("Resolving as localpart"); - return Optional.of(new SingleLookupReply(request, new MatrixID(uid, mxCfg.getDomain()))); - } - if (StringUtils.equals("mxid", cfg.getIdentity().getType())) { - log.info("Resolving as MXID"); - return Optional.of(new SingleLookupReply(request, new MatrixID(uid))); - } + try (ResultSet rSet = stmt.executeQuery()) { + while (rSet.next()) { + String uid = rSet.getString("uid"); + log.info("Found match: {}", uid); + if (StringUtils.equals("uid", cfg.getIdentity().getType())) { + log.info("Resolving as localpart"); + return Optional.of(new SingleLookupReply(request, new MatrixID(uid, mxCfg.getDomain()))); + } + if (StringUtils.equals("mxid", cfg.getIdentity().getType())) { + log.info("Resolving as MXID"); + return Optional.of(new SingleLookupReply(request, new MatrixID(uid))); + } - log.info("Identity type is unknown, skipping"); + log.info("Identity type is unknown, skipping"); + } + + log.info("No match found in SQL"); + return Optional.empty(); + } } - - log.info("No match found in SQL"); - return Optional.empty(); } catch (SQLException e) { throw new RuntimeException(e); } diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java new file mode 100644 index 0000000..8ef931c --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java @@ -0,0 +1,71 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.sql; + +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.sql.SqlProviderConfig; +import io.kamax.mxisd.config.sql.synapse.SynapseSqlProviderConfig; +import io.kamax.mxisd.exception.ConfigurationException; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +@Component +public class SynapseSqliteDirectoryProvider extends SqlDirectoryProvider { + + private SynapseSqlProviderConfig cfg; + + @Autowired + public SynapseSqliteDirectoryProvider(SynapseSqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { + super(cfg, mxCfg, pool); + + if (StringUtils.equals("sqlite", cfg.getType())) { + String userId = "'@' || p.user_id || ':" + mxCfg.getDomain() + "'"; + SqlProviderConfig.Type queries = cfg.getDirectory().getQuery(); + queries.getName().setValue( + "select " + userId + ", displayname from profiles p where displayname like ?"); + queries.getThreepid().setValue( + "select t.user_id, p.displayname " + + "from user_threepids t JOIN profiles p on t.user_id = " + userId + " " + + "where t.address like ?"); + } else if (StringUtils.equals("postgresql", cfg.getType())) { + String userId = "concat('@',p.user_id,':" + mxCfg.getDomain() + "')"; + SqlProviderConfig.Type queries = cfg.getDirectory().getQuery(); + queries.getName().setValue( + "select " + userId + ", displayname from profiles p where displayname ilike ?"); + queries.getThreepid().setValue( + "select t.user_id, p.displayname " + + "from user_threepids t JOIN profiles p on t.user_id = " + userId + " " + + "where t.address ilike ?"); + } else { + throw new ConfigurationException("Invalid SQL type"); + } + } + + @Override + protected void setParameters(PreparedStatement stmt, String searchTerm) throws SQLException { + stmt.setString(1, "%" + searchTerm + "%"); + } + +} diff --git a/src/main/java/io/kamax/mxisd/config/DnsOverwriteConfig.java b/src/main/java/io/kamax/mxisd/config/DnsOverwriteConfig.java new file mode 100644 index 0000000..b9d1922 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/config/DnsOverwriteConfig.java @@ -0,0 +1,106 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.config; + +import com.google.gson.Gson; +import io.kamax.mxisd.util.GsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; + +@Configuration +@ConfigurationProperties("dns.overwrite") +public class DnsOverwriteConfig { + + private Logger log = LoggerFactory.getLogger(DnsOverwriteConfig.class); + + public static class Entry { + + private String name; + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + } + + public static class Type { + + List client = new ArrayList<>(); + List federation = new ArrayList<>(); + + public List getClient() { + return client; + } + + public void setClient(List client) { + this.client = client; + } + + public List getFederation() { + return federation; + } + + public void setFederation(List federation) { + this.federation = federation; + } + + } + + private Type homeserver = new Type(); + + public Type getHomeserver() { + return homeserver; + } + + public void setHomeserver(Type homeserver) { + this.homeserver = homeserver; + } + + @PostConstruct + public void build() { + Gson gson = GsonUtil.build(); + log.info("--- DNS Overwrite config ---"); + log.info("Homeserver:"); + log.info("\tClient: {}", gson.toJson(getHomeserver().getClient())); + log.info("\tFederation: {}", gson.toJson(getHomeserver().getFederation())); + + } + +} diff --git a/src/main/java/io/kamax/mxisd/config/DnsOverwriteEntry.java b/src/main/java/io/kamax/mxisd/config/DnsOverwriteEntry.java deleted file mode 100644 index 70267c4..0000000 --- a/src/main/java/io/kamax/mxisd/config/DnsOverwriteEntry.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * mxisd - Matrix Identity Server Daemon - * Copyright (C) 2017 Maxime Dor - * - * https://max.kamax.io/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.kamax.mxisd.config; - -import org.apache.commons.lang.StringUtils; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -@Configuration -@ConfigurationProperties("dns.overwrite.homeserver") -public class DnsOverwriteEntry { - - private String name; - private String type; - private String value; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public String getTarget() { - if (StringUtils.equals("env", getType())) { - return System.getenv(getValue()); - } else { - return getValue(); - } - } - -} diff --git a/src/main/java/io/kamax/mxisd/config/FirebaseConfig.java b/src/main/java/io/kamax/mxisd/config/FirebaseConfig.java index 03408be..e17ce25 100644 --- a/src/main/java/io/kamax/mxisd/config/FirebaseConfig.java +++ b/src/main/java/io/kamax/mxisd/config/FirebaseConfig.java @@ -71,7 +71,7 @@ public class FirebaseConfig { } @PostConstruct - private void postConstruct() { + public void build() { log.info("--- Firebase configuration ---"); log.info("Enabled: {}", isEnabled()); if (isEnabled()) { @@ -82,20 +82,12 @@ public class FirebaseConfig { @Bean public AuthenticatorProvider getAuthProvider() { - if (!enabled) { - return new GoogleFirebaseAuthenticator(false); - } else { - return new GoogleFirebaseAuthenticator(credentials, database); - } + return new GoogleFirebaseAuthenticator(enabled, credentials, database); } @Bean public IThreePidProvider getLookupProvider() { - if (!enabled) { - return new GoogleFirebaseProvider(false); - } else { - return new GoogleFirebaseProvider(credentials, database, mxCfg.getDomain()); - } + return new GoogleFirebaseProvider(enabled, credentials, database, mxCfg.getDomain()); } } diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderAuthConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderAuthConfig.java deleted file mode 100644 index 4d826a7..0000000 --- a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderAuthConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package io.kamax.mxisd.config.sql; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -// Unused -@Configuration -@ConfigurationProperties("sql.auth") -public class SqlProviderAuthConfig { - - private boolean enabled; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - -} diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java index 2607ad0..b325d93 100644 --- a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java +++ b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java @@ -25,20 +25,149 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; @Configuration @ConfigurationProperties("sql") +@Primary public class SqlProviderConfig { private Logger log = LoggerFactory.getLogger(SqlProviderConfig.class); + public static class Query { + + private String type; + private String value; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + } + + public static class Type { + + private Query name = new Query(); + private Query threepid = new Query(); + + public Query getName() { + return name; + } + + public void setName(Query name) { + this.name = name; + } + + public Query getThreepid() { + return threepid; + } + + public void setThreepid(Query threepid) { + this.threepid = threepid; + } + + } + + public static class Auth { + + private Boolean enabled; + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + } + + public static class Directory { + + private Boolean enabled; + private Type query = new Type(); + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public Type getQuery() { + return query; + } + + public void setQuery(Type query) { + this.query = query; + } + + } + + public static class Identity { + + private Boolean enabled; + private String type; + private String query; + private Map medium = new HashMap<>(); + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Map getMedium() { + return medium; + } + + public void setMedium(Map medium) { + this.medium = medium; + } + + } + private boolean enabled; private String type; private String connection; - private SqlProviderAuthConfig auth; - private SqlProviderIdentityConfig identity; + private Auth auth = new Auth(); + private Directory directory = new Directory(); + private Identity identity = new Identity(); public boolean isEnabled() { return enabled; @@ -64,31 +193,52 @@ public class SqlProviderConfig { this.connection = connection; } - public SqlProviderAuthConfig getAuth() { + public Auth getAuth() { return auth; } - public void setAuth(SqlProviderAuthConfig auth) { + public void setAuth(Auth auth) { this.auth = auth; } - public SqlProviderIdentityConfig getIdentity() { + public Directory getDirectory() { + return directory; + } + + public void setDirectory(Directory directory) { + this.directory = directory; + } + + public Identity getIdentity() { return identity; } - public void setIdentity(SqlProviderIdentityConfig identity) { + public void setIdentity(Identity identity) { this.identity = identity; } @PostConstruct - private void postConstruct() { + public void build() { log.info("--- SQL Provider config ---"); + + if (getAuth().isEnabled() == null) { + getAuth().setEnabled(isEnabled()); + } + + if (getDirectory().isEnabled() == null) { + getDirectory().setEnabled(isEnabled()); + } + + if (getIdentity().isEnabled() == null) { + getIdentity().setEnabled(isEnabled()); + } + log.info("Enabled: {}", isEnabled()); if (isEnabled()) { log.info("Type: {}", getType()); log.info("Connection: {}", getConnection()); log.info("Auth enabled: {}", getAuth().isEnabled()); - log.info("Identy type: {}", getIdentity().getType()); + log.info("Identity type: {}", getIdentity().getType()); log.info("Identity medium queries: {}", new Gson().toJson(getIdentity().getMedium())); } } diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderIdentityConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderIdentityConfig.java deleted file mode 100644 index c55c662..0000000 --- a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderIdentityConfig.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * mxisd - Matrix Identity Server Daemon - * Copyright (C) 2017 Maxime Dor - * - * https://max.kamax.io/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.kamax.mxisd.config.sql; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import java.util.HashMap; -import java.util.Map; - -@Configuration -@ConfigurationProperties("sql.identity") -public class SqlProviderIdentityConfig { - - private String type; - private String query; - private Map medium = new HashMap<>(); - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Map getMedium() { - return medium; - } - - public void setMedium(Map medium) { - this.medium = medium; - } - -} diff --git a/src/main/java/io/kamax/mxisd/config/DnsOverwrite.java b/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java similarity index 59% rename from src/main/java/io/kamax/mxisd/config/DnsOverwrite.java rename to src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java index fab7cd3..9eefbd1 100644 --- a/src/main/java/io/kamax/mxisd/config/DnsOverwrite.java +++ b/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java @@ -18,35 +18,22 @@ * along with this program. If not, see . */ -package io.kamax.mxisd.config; +package io.kamax.mxisd.config.sql.synapse; -import org.apache.commons.lang.StringUtils; +import io.kamax.mxisd.config.sql.SqlProviderConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import java.util.Optional; +import static io.kamax.mxisd.config.sql.synapse.SynapseSqlProviderConfig.NAMESPACE; @Configuration -@ConfigurationProperties("dns.overwrite") -public class DnsOverwrite { +@ConfigurationProperties(NAMESPACE) +public class SynapseSqlProviderConfig extends SqlProviderConfig { - private Logger log = LoggerFactory.getLogger(DnsOverwrite.class); + public static final String NAMESPACE = "synapseSql"; - @Autowired - private ServerConfig srvCfg; - - @Autowired - private DnsOverwriteEntry homeserver; - - public Optional findHost(String lookup) { - if (homeserver != null && StringUtils.equalsIgnoreCase(lookup, homeserver.getName())) { - return Optional.of(homeserver); - } - - return Optional.empty(); - } + private Logger log = LoggerFactory.getLogger(SynapseSqlProviderConfig.class); } diff --git a/src/main/java/io/kamax/mxisd/controller/identity/v1/DefaultExceptionHandler.java b/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java similarity index 89% rename from src/main/java/io/kamax/mxisd/controller/identity/v1/DefaultExceptionHandler.java rename to src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java index 237dd86..79b1242 100644 --- a/src/main/java/io/kamax/mxisd/controller/identity/v1/DefaultExceptionHandler.java +++ b/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java @@ -18,10 +18,11 @@ * along with this program. If not, see . */ -package io.kamax.mxisd.controller.identity.v1; +package io.kamax.mxisd.controller; import com.google.gson.Gson; import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; import io.kamax.mxisd.exception.*; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -72,7 +73,7 @@ public class DefaultExceptionHandler { @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(MissingServletRequestParameterException.class) public String handle(MissingServletRequestParameterException e) { - return handle("M_INVALID_BODY", e.getMessage()); + return handle("M_INCOMPLETE_REQUEST", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @@ -81,6 +82,12 @@ public class DefaultExceptionHandler { return handle("M_INVALID_JSON", e.getMessage()); } + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler(JsonSyntaxException.class) + public String handle(JsonSyntaxException e) { + return handle("M_INVALID_JSON", e.getMessage()); + } + @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(JsonMemberNotFoundException.class) public String handle(JsonMemberNotFoundException e) { @@ -107,7 +114,7 @@ public class DefaultExceptionHandler { "M_UNKNOWN", StringUtils.defaultIfBlank( e.getMessage(), - "An internal server error occured. If this error persists, please contact support with reference #" + + "An internal server error occurred. If this error persists, please contact support with reference #" + Instant.now().toEpochMilli() ) ); diff --git a/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java b/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java index 2eaf4a2..78781f4 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java @@ -28,10 +28,7 @@ import io.kamax.mxisd.util.GsonParser; import io.kamax.mxisd.util.GsonUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; import java.io.IOException; @@ -48,7 +45,7 @@ public class UserDirectoryController { @Autowired private DirectoryManager mgr; - @RequestMapping(path = "/search") + @RequestMapping(path = "/search", method = RequestMethod.POST) public String search(HttpServletRequest request, @RequestParam("access_token") String accessToken) throws IOException { UserDirectorySearchRequest searchQuery = parser.parse(request, UserDirectorySearchRequest.class); URI target = URI.create(request.getRequestURL().toString()); diff --git a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java index e269786..2f78d87 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java @@ -24,6 +24,10 @@ public class UserDirectorySearchRequest { private String searchTerm; + public UserDirectorySearchRequest(String searchTerm) { + setSearchTerm(searchTerm); + } + public String getSearchTerm() { return searchTerm; } diff --git a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java index fd609d7..bf0045e 100644 --- a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java +++ b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java @@ -20,17 +20,105 @@ package io.kamax.mxisd.directory; +import com.google.gson.Gson; +import io.kamax.matrix.MatrixErrorInfo; +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchRequest; import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; -import io.kamax.mxisd.exception.NotImplementedException; +import io.kamax.mxisd.dns.ClientDnsOverwrite; +import io.kamax.mxisd.exception.InternalServerError; +import io.kamax.mxisd.exception.MatrixException; +import io.kamax.mxisd.util.GsonUtil; +import io.kamax.mxisd.util.RestClientUtils; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.io.IOException; import java.net.URI; +import java.nio.charset.Charset; +import java.util.List; +import java.util.stream.Collectors; @Component public class DirectoryManager { + private Logger log = LoggerFactory.getLogger(DirectoryManager.class); + + private List providers; + + private ClientDnsOverwrite dns; + private CloseableHttpClient client; + private Gson gson; + + @Autowired + public DirectoryManager(List providers, ClientDnsOverwrite dns) { + this.dns = dns; + this.client = HttpClients.custom().setUserAgent("mxisd").build(); //FIXME centralize + this.gson = GsonUtil.build(); + this.providers = providers.stream().filter(IDirectoryProvider::isEnabled).collect(Collectors.toList()); + + log.info("Directory providers:"); + this.providers.forEach(p -> log.info("\t- {}", p.getClass().getName())); + } + public UserDirectorySearchResult search(URI target, String accessToken, String query) { - throw new NotImplementedException("user directory - search"); + log.info("Performing search for '{}'", query); + log.info("Original request URL: {}", target); + UserDirectorySearchResult result = new UserDirectorySearchResult(); + + URIBuilder builder = dns.transform(target); + log.info("Querying HS at {}", builder); + builder.setParameter("access_token", accessToken); + HttpPost req = RestClientUtils.post( + builder.toString(), + new UserDirectorySearchRequest(query)); + try (CloseableHttpResponse res = client.execute(req)) { + int status = res.getStatusLine().getStatusCode(); + Charset charset = ContentType.getOrDefault(res.getEntity()).getCharset(); + String body = IOUtils.toString(res.getEntity().getContent(), charset); + + if (status != 200) { + MatrixErrorInfo info = gson.fromJson(body, MatrixErrorInfo.class); + throw new MatrixException(status, info.getErrcode(), info.getError()); + } + + UserDirectorySearchResult resultHs = gson.fromJson(body, UserDirectorySearchResult.class); + log.info("Found {} match(es) in HS for '{}'", resultHs.getResults().size(), query); + result.getResults().addAll(resultHs.getResults()); + if (resultHs.isLimited()) { + result.setLimited(true); + } + } catch (IOException e) { + throw new InternalServerError(e); + } + + for (IDirectoryProvider provider : providers) { + log.info("Using Directory provider {}", provider.getClass().getSimpleName()); + UserDirectorySearchResult resultProvider = provider.searchByDisplayName(query); + log.info("Display name: found {} match(es) for '{}'", resultProvider.getResults().size(), query); + result.getResults().addAll(resultProvider.getResults()); + if (resultProvider.isLimited()) { + result.setLimited(true); + } + + resultProvider = provider.searchBy3pid(query); + log.info("Threepid: found {} match(es) for '{}'", resultProvider.getResults().size(), query); + result.getResults().addAll(resultProvider.getResults()); + if (resultProvider.isLimited()) { + result.setLimited(true); + } + } + + log.info("Total matches: {} - limited? {}", result.getResults().size(), result.isLimited()); + return result; } } diff --git a/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java b/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java new file mode 100644 index 0000000..ddbf025 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java @@ -0,0 +1,33 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.directory; + +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; + +public interface IDirectoryProvider { + + boolean isEnabled(); + + UserDirectorySearchResult searchByDisplayName(String query); + + UserDirectorySearchResult searchBy3pid(String query); + +} diff --git a/src/main/java/io/kamax/mxisd/dns/ClientDnsOverwrite.java b/src/main/java/io/kamax/mxisd/dns/ClientDnsOverwrite.java new file mode 100644 index 0000000..d68a78a --- /dev/null +++ b/src/main/java/io/kamax/mxisd/dns/ClientDnsOverwrite.java @@ -0,0 +1,74 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.dns; + +import io.kamax.mxisd.config.DnsOverwriteConfig; +import io.kamax.mxisd.exception.ConfigurationException; +import org.apache.http.client.utils.URIBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +import static io.kamax.mxisd.config.DnsOverwriteConfig.Entry; + +@Component +public class ClientDnsOverwrite { + + private Logger log = LoggerFactory.getLogger(ClientDnsOverwrite.class); + + private Map mappings; + + @Autowired + public ClientDnsOverwrite(DnsOverwriteConfig cfg) { + mappings = new HashMap<>(); + cfg.getHomeserver().getClient().forEach(e -> mappings.put(e.getName(), e)); + } + + public URIBuilder transform(URI initial) { + URIBuilder builder = new URIBuilder(initial); + Entry mapping = mappings.get(initial.getHost()); + if (mapping == null) { + return builder; + } + + try { + URL target = new URL(mapping.getValue()); + builder.setScheme(target.getProtocol()); + builder.setHost(target.getHost()); + if (target.getPort() != -1) { + builder.setPort(target.getPort()); + } + + return builder; + } catch (MalformedURLException e) { + log.warn("Skipping DNS overwrite entry {} due to invalid value [{}]: {}", mapping.getName(), mapping.getValue(), e.getMessage()); + throw new ConfigurationException("Invalid DNS overwrite entry in homeserver client: " + mapping.getName(), e.getMessage()); + } + } + +} diff --git a/src/main/java/io/kamax/mxisd/dns/FederationDnsOverwrite.java b/src/main/java/io/kamax/mxisd/dns/FederationDnsOverwrite.java new file mode 100644 index 0000000..76772ac --- /dev/null +++ b/src/main/java/io/kamax/mxisd/dns/FederationDnsOverwrite.java @@ -0,0 +1,61 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.dns; + +import io.kamax.mxisd.config.DnsOverwriteConfig; +import io.kamax.mxisd.config.ServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static io.kamax.mxisd.config.DnsOverwriteConfig.Entry; + +@Component +public class FederationDnsOverwrite { + + private Logger log = LoggerFactory.getLogger(FederationDnsOverwrite.class); + + private ServerConfig srvCfg; + private Map mappings; + + @Autowired + public FederationDnsOverwrite(DnsOverwriteConfig cfg, ServerConfig srvCfg) { + this.srvCfg = srvCfg; + + mappings = new HashMap<>(); + cfg.getHomeserver().getFederation().forEach(e -> mappings.put(e.getName(), e)); + } + + public Optional findHost(String lookup) { + Entry mapping = mappings.get(lookup); + if (mapping == null) { + return Optional.empty(); + } + + return Optional.of(mapping.getValue()); + } + +} diff --git a/src/main/java/io/kamax/mxisd/exception/MatrixException.java b/src/main/java/io/kamax/mxisd/exception/MatrixException.java index 7565b15..a328a42 100644 --- a/src/main/java/io/kamax/mxisd/exception/MatrixException.java +++ b/src/main/java/io/kamax/mxisd/exception/MatrixException.java @@ -20,7 +20,7 @@ package io.kamax.mxisd.exception; -public abstract class MatrixException extends MxisdException { +public class MatrixException extends MxisdException { private int status; private String errorCode; diff --git a/src/main/java/io/kamax/mxisd/exception/RemoteHomeServerException.java b/src/main/java/io/kamax/mxisd/exception/RemoteHomeServerException.java new file mode 100644 index 0000000..3ca20f4 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/exception/RemoteHomeServerException.java @@ -0,0 +1,11 @@ +package io.kamax.mxisd.exception; + +import org.apache.http.HttpStatus; + +public class RemoteHomeServerException extends MatrixException { + + public RemoteHomeServerException(String error) { + super(HttpStatus.SC_SERVICE_UNAVAILABLE, "M_REMOTE_HS_ERROR", "Error from remote server: " + error); + } + +} diff --git a/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java b/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java index 3f1ebc5..af19aa8 100644 --- a/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java +++ b/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java @@ -24,8 +24,7 @@ import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.kamax.matrix.MatrixID; -import io.kamax.mxisd.config.DnsOverwrite; -import io.kamax.mxisd.config.DnsOverwriteEntry; +import io.kamax.mxisd.dns.FederationDnsOverwrite; import io.kamax.mxisd.exception.BadRequestException; import io.kamax.mxisd.exception.MappingAlreadyExistsException; import io.kamax.mxisd.lookup.SingleLookupReply; @@ -81,7 +80,7 @@ public class InvitationManager { private SignatureManager signMgr; @Autowired - private DnsOverwrite dns; + private FederationDnsOverwrite dns; private NotificationManager notifMgr; @@ -160,11 +159,11 @@ public class InvitationManager { // TODO use caching mechanism // TODO export in matrix-java-sdk private String findHomeserverForDomain(String domain) { - Optional entryOpt = dns.findHost(domain); + Optional entryOpt = dns.findHost(domain); if (entryOpt.isPresent()) { - DnsOverwriteEntry entry = entryOpt.get(); - log.info("Found DNS overwrite for {} to {}", entry.getName(), entry.getTarget()); - return "https://" + entry.getTarget(); + String entry = entryOpt.get(); + log.info("Found DNS overwrite for {} to {}", domain, entry); + return "https://" + entry; } log.debug("Performing SRV lookup for {}", domain); diff --git a/src/main/java/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.java b/src/main/java/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.java index 7b17378..a5d23f7 100644 --- a/src/main/java/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.java +++ b/src/main/java/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.java @@ -43,26 +43,28 @@ public class RecursivePriorityLookupStrategy implements LookupStrategy { private Logger log = LoggerFactory.getLogger(RecursivePriorityLookupStrategy.class); - @Autowired - private RecursiveLookupConfig recursiveCfg; - - @Autowired + private RecursiveLookupConfig cfg; private List providers; - - @Autowired private IBridgeFetcher bridge; private List allowedCidr = new ArrayList<>(); + @Autowired + public RecursivePriorityLookupStrategy(RecursiveLookupConfig cfg, List providers, IBridgeFetcher bridge) { + this.cfg = cfg; + this.bridge = bridge; + this.providers = providers.stream().filter(IThreePidProvider::isEnabled).collect(Collectors.toList()); + } + @PostConstruct private void build() throws UnknownHostException { try { log.info("Found {} providers", providers.size()); - + providers.forEach(p -> log.info("\t- {}", p.getClass().getName())); providers.sort((o1, o2) -> Integer.compare(o2.getPriority(), o1.getPriority())); - log.info("Recursive lookup enabled: {}", recursiveCfg.isEnabled()); - for (String cidr : recursiveCfg.getAllowedCidr()) { + log.info("Recursive lookup enabled: {}", cfg.isEnabled()); + for (String cidr : cfg.getAllowedCidr()) { log.info("{} is allowed for recursion", cidr); allowedCidr.add(new CIDRUtils(cidr)); } @@ -75,7 +77,7 @@ public class RecursivePriorityLookupStrategy implements LookupStrategy { boolean canRecurse = false; try { - if (recursiveCfg.isEnabled()) { + if (cfg.isEnabled()) { log.debug("Checking {} CIDRs for recursion", allowedCidr.size()); for (CIDRUtils cidr : allowedCidr) { if (cidr.isInRange(source)) { @@ -106,7 +108,7 @@ public class RecursivePriorityLookupStrategy implements LookupStrategy { log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse); for (IThreePidProvider provider : providers) { - if (provider.isEnabled() && (provider.isLocal() || canRecurse || forceRecursive)) { + if (provider.isLocal() || canRecurse || forceRecursive) { usableProviders.add(provider); } } @@ -159,9 +161,9 @@ public class RecursivePriorityLookupStrategy implements LookupStrategy { } if ( - recursiveCfg.getBridge() != null && - recursiveCfg.getBridge().getEnabled() && - (!recursiveCfg.getBridge().getRecursiveOnly() || isAllowedForRecursive(request.getRequester())) + cfg.getBridge() != null && + cfg.getBridge().getEnabled() && + (!cfg.getBridge().getRecursiveOnly() || isAllowedForRecursive(request.getRequester())) ) { log.info("Using bridge failover for lookup"); return bridge.find(request); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f2e6e43..7054d0a 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -66,9 +66,20 @@ sql: connection: '' auth: enabled: false + directory: + enabled: false + query: + name: + value: 'SELECT 1' + threepid: + value: 'SELECT 1' identity: type: 'mxid' - query: "SELECT user_id AS uid FROM user_threepids WHERE medium = ? AND address = ?" + query: 'SELECT user_id AS uid FROM user_threepids WHERE medium = ? AND address = ?' + +synapseSql: + enabled: false + type: 'sqlite' forward: servers: From ed2d13decf146a62de75ae61e964e428e62f8a68 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Fri, 29 Sep 2017 05:34:21 +0200 Subject: [PATCH 03/14] Don't mix up configs --- .../firebase/GoogleFirebaseBackend.java | 4 + .../mxisd/backend/sql/SqlConnectionPool.java | 6 +- .../backend/sql/SqlDirectoryProvider.java | 18 +- .../backend/sql/SqlThreePidProvider.java | 4 +- .../sql/SynapseSqliteDirectoryProvider.java | 4 +- .../io/kamax/mxisd/config/sql/SqlConfig.java | 220 ++++++++++++++++++ .../mxisd/config/sql/SqlProviderConfig.java | 214 +---------------- .../sql/synapse/SynapseSqlProviderConfig.java | 20 +- 8 files changed, 256 insertions(+), 234 deletions(-) create mode 100644 src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java diff --git a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java index 9c5bcfc..4888797 100644 --- a/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java +++ b/src/main/java/io/kamax/mxisd/backend/firebase/GoogleFirebaseBackend.java @@ -43,6 +43,10 @@ public class GoogleFirebaseBackend { GoogleFirebaseBackend(boolean isEnabled, String name, String credsPath, String db) { this.isEnabled = isEnabled; + if (!isEnabled) { + return; + } + try { FirebaseApp fbApp = FirebaseApp.initializeApp(getOpts(credsPath, db), name); fbAuth = FirebaseAuth.getInstance(fbApp); diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java index 1ec03a6..5515cb7 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlConnectionPool.java @@ -21,18 +21,16 @@ package io.kamax.mxisd.backend.sql; import com.mchange.v2.c3p0.ComboPooledDataSource; -import io.kamax.mxisd.config.sql.SqlProviderConfig; -import org.springframework.stereotype.Component; +import io.kamax.mxisd.config.sql.SqlConfig; import java.sql.Connection; import java.sql.SQLException; -@Component public class SqlConnectionPool { private ComboPooledDataSource ds; - public SqlConnectionPool(SqlProviderConfig cfg) { + public SqlConnectionPool(SqlConfig cfg) { ds = new ComboPooledDataSource(); ds.setJdbcUrl("jdbc:" + cfg.getType() + ":" + cfg.getConnection()); ds.setMinPoolSize(1); diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java index ce7fcdb..3be7489 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java @@ -20,7 +20,9 @@ package io.kamax.mxisd.backend.sql; +import io.kamax.matrix.MatrixID; import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.sql.SqlConfig; import io.kamax.mxisd.config.sql.SqlProviderConfig; import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; import io.kamax.mxisd.directory.IDirectoryProvider; @@ -28,8 +30,6 @@ import io.kamax.mxisd.exception.InternalServerError; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import java.sql.Connection; import java.sql.PreparedStatement; @@ -39,21 +39,18 @@ import java.util.Optional; import static io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult.Result; - -@Component -public class SqlDirectoryProvider implements IDirectoryProvider { +public abstract class SqlDirectoryProvider implements IDirectoryProvider { private Logger log = LoggerFactory.getLogger(SqlDirectoryProvider.class); - protected SqlProviderConfig cfg; + protected SqlConfig cfg; private MatrixConfig mxCfg; private SqlConnectionPool pool; - @Autowired - public SqlDirectoryProvider(SqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { + public SqlDirectoryProvider(SqlConfig cfg, MatrixConfig mxCfg) { this.cfg = cfg; - this.pool = pool; + this.pool = new SqlConnectionPool(cfg); this.mxCfg = mxCfg; } @@ -77,6 +74,7 @@ public class SqlDirectoryProvider implements IDirectoryProvider { public UserDirectorySearchResult search(String searchTerm, SqlProviderConfig.Query query) { try (Connection conn = pool.get()) { + log.info("Will execute query: {}", query.getValue()); try (PreparedStatement stmt = conn.prepareStatement(query.getValue())) { setParameters(stmt, searchTerm); @@ -87,7 +85,7 @@ public class SqlDirectoryProvider implements IDirectoryProvider { while (rSet.next()) { processRow(rSet).ifPresent(e -> { if (StringUtils.equalsIgnoreCase("localpart", query.getType())) { - e.setUserId("@" + e.getUserId() + mxCfg.getDomain()); + e.setUserId(new MatrixID(e.getUserId(), mxCfg.getDomain()).getId()); } result.addResult(e); }); diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java index b992dc9..2c62acd 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlThreePidProvider.java @@ -52,9 +52,9 @@ public class SqlThreePidProvider implements IThreePidProvider { private SqlConnectionPool pool; @Autowired - public SqlThreePidProvider(SqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { + public SqlThreePidProvider(SqlProviderConfig cfg, MatrixConfig mxCfg) { this.cfg = cfg; - this.pool = pool; + this.pool = new SqlConnectionPool(cfg); this.mxCfg = mxCfg; } diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java index 8ef931c..2d51c2c 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SynapseSqliteDirectoryProvider.java @@ -37,8 +37,8 @@ public class SynapseSqliteDirectoryProvider extends SqlDirectoryProvider { private SynapseSqlProviderConfig cfg; @Autowired - public SynapseSqliteDirectoryProvider(SynapseSqlProviderConfig cfg, MatrixConfig mxCfg, SqlConnectionPool pool) { - super(cfg, mxCfg, pool); + public SynapseSqliteDirectoryProvider(SynapseSqlProviderConfig cfg, MatrixConfig mxCfg) { + super(cfg, mxCfg); if (StringUtils.equals("sqlite", cfg.getType())) { String userId = "'@' || p.user_id || ':" + mxCfg.getDomain() + "'"; diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java new file mode 100644 index 0000000..ac4ac99 --- /dev/null +++ b/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java @@ -0,0 +1,220 @@ +package io.kamax.mxisd.config.sql; + +import com.google.gson.Gson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +public abstract class SqlConfig { + + private Logger log = LoggerFactory.getLogger(SqlConfig.class); + + public static class Query { + + private String type; + private String value; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + } + + public static class Type { + + private SqlProviderConfig.Query name = new SqlProviderConfig.Query(); + private SqlProviderConfig.Query threepid = new SqlProviderConfig.Query(); + + public SqlProviderConfig.Query getName() { + return name; + } + + public void setName(SqlProviderConfig.Query name) { + this.name = name; + } + + public SqlProviderConfig.Query getThreepid() { + return threepid; + } + + public void setThreepid(SqlProviderConfig.Query threepid) { + this.threepid = threepid; + } + + } + + public static class Auth { + + private Boolean enabled; + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + } + + public static class Directory { + + private Boolean enabled; + private SqlProviderConfig.Type query = new SqlProviderConfig.Type(); + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public SqlProviderConfig.Type getQuery() { + return query; + } + + public void setQuery(SqlProviderConfig.Type query) { + this.query = query; + } + + } + + public static class Identity { + + private Boolean enabled; + private String type; + private String query; + private Map medium = new HashMap<>(); + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Map getMedium() { + return medium; + } + + public void setMedium(Map medium) { + this.medium = medium; + } + + } + + private boolean enabled; + private String type; + private String connection; + private SqlProviderConfig.Auth auth = new SqlProviderConfig.Auth(); + private SqlProviderConfig.Directory directory = new SqlProviderConfig.Directory(); + private SqlProviderConfig.Identity identity = new SqlProviderConfig.Identity(); + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getConnection() { + return connection; + } + + public void setConnection(String connection) { + this.connection = connection; + } + + public SqlProviderConfig.Auth getAuth() { + return auth; + } + + public void setAuth(SqlProviderConfig.Auth auth) { + this.auth = auth; + } + + public SqlProviderConfig.Directory getDirectory() { + return directory; + } + + public void setDirectory(SqlProviderConfig.Directory directory) { + this.directory = directory; + } + + public SqlProviderConfig.Identity getIdentity() { + return identity; + } + + public void setIdentity(SqlProviderConfig.Identity identity) { + this.identity = identity; + } + + protected abstract String getProviderName(); + + public void build() { + log.info("--- " + getProviderName() + " Provider config ---"); + + if (getAuth().isEnabled() == null) { + getAuth().setEnabled(isEnabled()); + } + + if (getDirectory().isEnabled() == null) { + getDirectory().setEnabled(isEnabled()); + } + + if (getIdentity().isEnabled() == null) { + getIdentity().setEnabled(isEnabled()); + } + + log.info("Enabled: {}", isEnabled()); + if (isEnabled()) { + log.info("Type: {}", getType()); + log.info("Connection: {}", getConnection()); + log.info("Auth enabled: {}", getAuth().isEnabled()); + log.info("Identity type: {}", getIdentity().getType()); + log.info("Identity medium queries: {}", new Gson().toJson(getIdentity().getMedium())); + } + } + +} diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java index b325d93..4aa8831 100644 --- a/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java +++ b/src/main/java/io/kamax/mxisd/config/sql/SqlProviderConfig.java @@ -20,227 +20,25 @@ package io.kamax.mxisd.config.sql; -import com.google.gson.Gson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; @Configuration @ConfigurationProperties("sql") @Primary -public class SqlProviderConfig { +public class SqlProviderConfig extends SqlConfig { - private Logger log = LoggerFactory.getLogger(SqlProviderConfig.class); - - public static class Query { - - private String type; - private String value; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - } - - public static class Type { - - private Query name = new Query(); - private Query threepid = new Query(); - - public Query getName() { - return name; - } - - public void setName(Query name) { - this.name = name; - } - - public Query getThreepid() { - return threepid; - } - - public void setThreepid(Query threepid) { - this.threepid = threepid; - } - - } - - public static class Auth { - - private Boolean enabled; - - public Boolean isEnabled() { - return enabled; - } - - public void setEnabled(Boolean enabled) { - this.enabled = enabled; - } - - } - - public static class Directory { - - private Boolean enabled; - private Type query = new Type(); - - public Boolean isEnabled() { - return enabled; - } - - public void setEnabled(Boolean enabled) { - this.enabled = enabled; - } - - public Type getQuery() { - return query; - } - - public void setQuery(Type query) { - this.query = query; - } - - } - - public static class Identity { - - private Boolean enabled; - private String type; - private String query; - private Map medium = new HashMap<>(); - - public Boolean isEnabled() { - return enabled; - } - - public void setEnabled(Boolean enabled) { - this.enabled = enabled; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Map getMedium() { - return medium; - } - - public void setMedium(Map medium) { - this.medium = medium; - } - - } - - private boolean enabled; - private String type; - private String connection; - private Auth auth = new Auth(); - private Directory directory = new Directory(); - private Identity identity = new Identity(); - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getConnection() { - return connection; - } - - public void setConnection(String connection) { - this.connection = connection; - } - - public Auth getAuth() { - return auth; - } - - public void setAuth(Auth auth) { - this.auth = auth; - } - - public Directory getDirectory() { - return directory; - } - - public void setDirectory(Directory directory) { - this.directory = directory; - } - - public Identity getIdentity() { - return identity; - } - - public void setIdentity(Identity identity) { - this.identity = identity; + @Override + protected String getProviderName() { + return "Generic SQL"; } @PostConstruct public void build() { - log.info("--- SQL Provider config ---"); - - if (getAuth().isEnabled() == null) { - getAuth().setEnabled(isEnabled()); - } - - if (getDirectory().isEnabled() == null) { - getDirectory().setEnabled(isEnabled()); - } - - if (getIdentity().isEnabled() == null) { - getIdentity().setEnabled(isEnabled()); - } - - log.info("Enabled: {}", isEnabled()); - if (isEnabled()) { - log.info("Type: {}", getType()); - log.info("Connection: {}", getConnection()); - log.info("Auth enabled: {}", getAuth().isEnabled()); - log.info("Identity type: {}", getIdentity().getType()); - log.info("Identity medium queries: {}", new Gson().toJson(getIdentity().getMedium())); - } + super.build(); } -} +} \ No newline at end of file diff --git a/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java b/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java index 9eefbd1..149c329 100644 --- a/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java +++ b/src/main/java/io/kamax/mxisd/config/sql/synapse/SynapseSqlProviderConfig.java @@ -20,20 +20,24 @@ package io.kamax.mxisd.config.sql.synapse; -import io.kamax.mxisd.config.sql.SqlProviderConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import io.kamax.mxisd.config.sql.SqlConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import static io.kamax.mxisd.config.sql.synapse.SynapseSqlProviderConfig.NAMESPACE; +import javax.annotation.PostConstruct; @Configuration -@ConfigurationProperties(NAMESPACE) -public class SynapseSqlProviderConfig extends SqlProviderConfig { +@ConfigurationProperties("synapseSql") +public class SynapseSqlProviderConfig extends SqlConfig { - public static final String NAMESPACE = "synapseSql"; + @Override + protected String getProviderName() { + return "Synapse SQL"; + } - private Logger log = LoggerFactory.getLogger(SynapseSqlProviderConfig.class); + @PostConstruct + public void build() { + super.build(); + } } From b6008a41f206032d95f61c6017a8c3913d92e843 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Fri, 29 Sep 2017 05:38:58 +0200 Subject: [PATCH 04/14] Be consistent with DNS overwrite (always a URL) --- .../java/io/kamax/mxisd/invitation/InvitationManager.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java b/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java index af19aa8..58770a9 100644 --- a/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java +++ b/src/main/java/io/kamax/mxisd/invitation/InvitationManager.java @@ -57,6 +57,8 @@ import javax.annotation.PreDestroy; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -163,7 +165,11 @@ public class InvitationManager { if (entryOpt.isPresent()) { String entry = entryOpt.get(); log.info("Found DNS overwrite for {} to {}", domain, entry); - return "https://" + entry; + try { + return new URL(entry).toString(); + } catch (MalformedURLException e) { + log.warn("Skipping homeserver Federation DNS overwrite for {} - not a valid URL: {}", domain, entry); + } } log.debug("Performing SRV lookup for {}", domain); From f7984bd36e8928ab4cb2fd1ba8ad5da857625963 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Fri, 29 Sep 2017 20:54:08 +0200 Subject: [PATCH 05/14] LDAP Directory search support --- .../mxisd/backend/ldap/LdapAuthProvider.java | 39 +++--- .../backend/ldap/LdapDirectoryProvider.java | 116 ++++++++++++++++++ .../backend/ldap/LdapGenericBackend.java | 108 +++++++++++++--- .../backend/ldap/LdapThreePidProvider.java | 67 +++------- .../config/ldap/LdapAttributeConfig.java | 13 ++ .../kamax/mxisd/config/ldap/LdapConfig.java | 63 +++++++++- .../mxisd/config/ldap/LdapIdentityConfig.java | 18 +++ .../controller/DefaultExceptionHandler.java | 36 +++--- src/main/resources/application.yaml | 22 +++- 9 files changed, 366 insertions(+), 116 deletions(-) create mode 100644 src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java index 64f38bc..73c3ade 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java @@ -24,6 +24,8 @@ import io.kamax.matrix._MatrixID; import io.kamax.mxisd.UserIdType; import io.kamax.mxisd.auth.provider.AuthenticatorProvider; import io.kamax.mxisd.auth.provider.BackendAuthResult; +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.ldap.LdapConfig; import org.apache.commons.lang.StringUtils; import org.apache.directory.api.ldap.model.cursor.CursorException; import org.apache.directory.api.ldap.model.cursor.CursorLdapReferralException; @@ -35,6 +37,7 @@ import org.apache.directory.api.ldap.model.message.SearchScope; import org.apache.directory.ldap.client.api.LdapConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; @@ -44,8 +47,9 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato private Logger log = LoggerFactory.getLogger(LdapAuthProvider.class); - private String getUidAttribute() { - return getCfg().getAttribute().getUid().getValue(); + @Autowired + public LdapAuthProvider(LdapConfig cfg, MatrixConfig mxCfg) { + super(cfg, mxCfg); } @Override @@ -57,37 +61,34 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato public BackendAuthResult authenticate(_MatrixID mxid, String password) { log.info("Performing auth for {}", mxid); - LdapConnection conn = getConn(); - try { + + try (LdapConnection conn = getConn()) { bind(conn); - String uidType = getCfg().getAttribute().getUid().getType(); - String userFilterValue = StringUtils.equals(LdapThreePidProvider.UID, uidType) ? mxid.getLocalPart() : mxid.getId(); + String uidType = getAt().getUid().getType(); + String userFilterValue = StringUtils.equals(LdapGenericBackend.UID, uidType) ? mxid.getLocalPart() : mxid.getId(); if (StringUtils.isBlank(userFilterValue)) { log.warn("Username is empty, failing auth"); return BackendAuthResult.failure(); } String userFilter = "(" + getCfg().getAttribute().getUid().getValue() + "=" + userFilterValue + ")"; - if (!StringUtils.isBlank(getCfg().getAuth().getFilter())) { - userFilter = "(&" + getCfg().getAuth().getFilter() + userFilter + ")"; - } - EntryCursor cursor = conn.search(getCfg().getConn().getBaseDn(), userFilter, SearchScope.SUBTREE, getUidAttribute(), getCfg().getAttribute().getName()); - try { + userFilter = buildWithFilter(userFilter, getCfg().getAuth().getFilter()); + try (EntryCursor cursor = conn.search(getBaseDn(), userFilter, SearchScope.SUBTREE, getUidAtt(), getAt().getName())) { while (cursor.next()) { Entry entry = cursor.get(); String dn = entry.getDn().getName(); log.info("Checking possible match, DN: {}", dn); - Attribute attribute = entry.get(getUidAttribute()); + Attribute attribute = entry.get(getUidAtt()); if (attribute == null) { - log.info("DN {}: no attribute {}, skpping", dn, getUidAttribute()); + log.info("DN {}: no attribute {}, skpping", dn, getUidAtt()); continue; } String data = attribute.get().toString(); if (data.length() < 1) { - log.info("DN {}: empty attribute {}, skipping", getUidAttribute()); + log.info("DN {}: empty attribute {}, skipping", getUidAtt()); continue; } @@ -99,7 +100,7 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato return BackendAuthResult.failure(); } - Attribute nameAttribute = entry.get(getCfg().getAttribute().getName()); + Attribute nameAttribute = entry.get(getAt().getName()); String name = nameAttribute != null ? nameAttribute.get().toString() : null; log.info("Authentication successful for {}", entry.getDn().getName()); @@ -110,20 +111,12 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato } } catch (CursorLdapReferralException e) { log.warn("Entity for {} is only available via referral, skipping", mxid); - } finally { - cursor.close(); } log.info("No match were found for {}", mxid); return BackendAuthResult.failure(); } catch (LdapException | IOException | CursorException e) { throw new RuntimeException(e); - } finally { - try { - conn.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } } } diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java new file mode 100644 index 0000000..2c25b3f --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java @@ -0,0 +1,116 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.ldap; + +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.ldap.LdapAttributeConfig; +import io.kamax.mxisd.config.ldap.LdapConfig; +import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.directory.IDirectoryProvider; +import io.kamax.mxisd.exception.InternalServerError; +import org.apache.directory.api.ldap.model.cursor.CursorException; +import org.apache.directory.api.ldap.model.cursor.CursorLdapReferralException; +import org.apache.directory.api.ldap.model.cursor.EntryCursor; +import org.apache.directory.api.ldap.model.entry.Entry; +import org.apache.directory.api.ldap.model.exception.LdapException; +import org.apache.directory.api.ldap.model.message.SearchScope; +import org.apache.directory.ldap.client.api.LdapConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Component +public class LdapDirectoryProvider extends LdapGenericBackend implements IDirectoryProvider { + + private Logger log = LoggerFactory.getLogger(LdapDirectoryProvider.class); + + @Autowired + public LdapDirectoryProvider(LdapConfig cfg, MatrixConfig mxCfg) { + super(cfg, mxCfg); + } + + @Override + public boolean isEnabled() { + return getCfg().isEnabled(); + } + + protected UserDirectorySearchResult search(String query, List attributes) { + UserDirectorySearchResult result = new UserDirectorySearchResult(); + result.setLimited(false); + + try (LdapConnection conn = getConn()) { + bind(conn); + + LdapAttributeConfig atCfg = getCfg().getAttribute(); + + attributes = new ArrayList<>(attributes); + attributes.add(getUidAtt()); + String[] attArray = new String[attributes.size()]; + attributes.toArray(attArray); + + String searchQuery = buildOrQueryWithFilter(getCfg().getDirectory().getFilter(), "*" + query + "*", attArray); + try (EntryCursor cursor = conn.search(getBaseDn(), searchQuery, SearchScope.SUBTREE, attArray)) { + while (cursor.next()) { + Entry entry = cursor.get(); + log.info("Found possible match, DN: {}", entry.getDn().getName()); + getAttribute(entry, getUidAtt()).ifPresent(uid -> { + log.info("DN {} is a valid match", entry.getDn().getName()); + try { + UserDirectorySearchResult.Result entryResult = new UserDirectorySearchResult.Result(); + entryResult.setUserId(buildMatrixIdFromUid(uid)); + getAttribute(entry, atCfg.getName()).ifPresent(entryResult::setDisplayName); + result.addResult(entryResult); + } catch (IllegalArgumentException e) { + log.warn("Bind was found but type {} is not supported", atCfg.getUid().getType()); + } + }); + } + } + } catch (CursorLdapReferralException e) { + log.warn("An entry is only available via referral, skipping"); + } catch (IOException | LdapException | CursorException e) { + throw new InternalServerError(e); + } + + return result; + } + + @Override + public UserDirectorySearchResult searchByDisplayName(String query) { + log.info("Performing LDAP directory search on display name using '{}'", query); + return search(query, Collections.singletonList(getCfg().getAttribute().getName())); + } + + @Override + public UserDirectorySearchResult searchBy3pid(String query) { + log.info("Performing LDAP directory search on 3PIDs using '{}'", query); + List attributes = new ArrayList<>(); + getCfg().getAttribute().getThreepid().forEach((k, v) -> attributes.addAll(v)); + return search(query, attributes); + } + +} diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java index ff9a7de..b87d503 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java @@ -20,38 +20,112 @@ package io.kamax.mxisd.backend.ldap; +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.ldap.LdapAttributeConfig; import io.kamax.mxisd.config.ldap.LdapConfig; import org.apache.commons.lang.StringUtils; +import org.apache.directory.api.ldap.model.entry.Attribute; +import org.apache.directory.api.ldap.model.entry.Entry; import org.apache.directory.api.ldap.model.exception.LdapException; import org.apache.directory.ldap.client.api.LdapConnection; import org.apache.directory.ldap.client.api.LdapNetworkConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -@Component -public class LdapGenericBackend { +import java.util.Arrays; +import java.util.Optional; + +public abstract class LdapGenericBackend { + + public static final String UID = "uid"; + public static final String MATRIX_ID = "mxid"; private Logger log = LoggerFactory.getLogger(LdapGenericBackend.class); - @Autowired - private LdapConfig ldapCfg; + private LdapConfig cfg; + private MatrixConfig mxCfg; - protected LdapConnection getConn() { - return new LdapNetworkConnection(ldapCfg.getConn().getHost(), ldapCfg.getConn().getPort(), ldapCfg.getConn().isTls()); - } - - protected void bind(LdapConnection conn) throws LdapException { - if (StringUtils.isBlank(ldapCfg.getConn().getBindDn()) && StringUtils.isBlank(ldapCfg.getConn().getBindPassword())) { - conn.anonymousBind(); - } else { - conn.bind(ldapCfg.getConn().getBindDn(), ldapCfg.getConn().getBindPassword()); - } + public LdapGenericBackend(LdapConfig cfg, MatrixConfig mxCfg) { + this.cfg = cfg; + this.mxCfg = mxCfg; } protected LdapConfig getCfg() { - return ldapCfg; + return cfg; + } + + protected String getBaseDn() { + return cfg.getConn().getBaseDn(); + } + + protected LdapAttributeConfig getAt() { + return cfg.getAttribute(); + } + + protected String getUidAtt() { + return getAt().getUid().getValue(); + } + + protected LdapConnection getConn() { + return new LdapNetworkConnection(cfg.getConn().getHost(), cfg.getConn().getPort(), cfg.getConn().isTls()); + } + + protected void bind(LdapConnection conn) throws LdapException { + if (StringUtils.isBlank(cfg.getConn().getBindDn()) && StringUtils.isBlank(cfg.getConn().getBindPassword())) { + conn.anonymousBind(); + } else { + conn.bind(cfg.getConn().getBindDn(), cfg.getConn().getBindPassword()); + } + } + + protected String buildWithFilter(String base, String filter) { + if (StringUtils.isBlank(filter)) { + return base; + } else { + return "(&" + filter + base + ")"; + } + } + + public static String buildOrQuery(String value, String... attributes) { + StringBuilder builder = new StringBuilder(); + builder.append("(|"); + Arrays.stream(attributes).forEach(s -> { + builder.append("("); + builder.append(s).append("=").append(value).append(")"); + }); + builder.append(")"); + return builder.toString(); + } + + public String buildOrQueryWithFilter(String filter, String value, String... attributes) { + return buildWithFilter(buildOrQuery(value, attributes), filter); + } + + public String buildMatrixIdFromUid(String uid) { + String uidType = getCfg().getAttribute().getUid().getType(); + if (StringUtils.equals(UID, uidType)) { + return "@" + uid + ":" + mxCfg.getDomain(); + } else if (StringUtils.equals(MATRIX_ID, uidType)) { + return uid; + } else { + throw new IllegalArgumentException("Bind type " + uidType + " is not supported"); + } + } + + public Optional getAttribute(Entry entry, String attName) { + Attribute attribute = entry.get(attName); + if (attribute == null) { + log.info("DN {}: no attribute {}, skipping", entry.getDn(), attName); + return Optional.empty(); + } + + String value = attribute.get().toString(); + if (StringUtils.isBlank(value)) { + log.info("DN {}: empty attribute {}, skipping", attName); + return Optional.empty(); + } + + return Optional.of(value); } } diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java index 5dbf139..9715ac4 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java @@ -21,23 +21,21 @@ package io.kamax.mxisd.backend.ldap; import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.ldap.LdapConfig; import io.kamax.mxisd.exception.InternalServerError; import io.kamax.mxisd.lookup.SingleLookupReply; import io.kamax.mxisd.lookup.SingleLookupRequest; import io.kamax.mxisd.lookup.ThreePidMapping; import io.kamax.mxisd.lookup.provider.IThreePidProvider; -import org.apache.commons.lang.StringUtils; import org.apache.directory.api.ldap.model.cursor.CursorException; import org.apache.directory.api.ldap.model.cursor.CursorLdapReferralException; import org.apache.directory.api.ldap.model.cursor.EntryCursor; -import org.apache.directory.api.ldap.model.entry.Attribute; import org.apache.directory.api.ldap.model.entry.Entry; import org.apache.directory.api.ldap.model.exception.LdapException; import org.apache.directory.api.ldap.model.message.SearchScope; import org.apache.directory.ldap.client.api.LdapConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; @@ -48,23 +46,17 @@ import java.util.Optional; @Component public class LdapThreePidProvider extends LdapGenericBackend implements IThreePidProvider { - public static final String UID = "uid"; - public static final String MATRIX_ID = "mxid"; - private Logger log = LoggerFactory.getLogger(LdapThreePidProvider.class); - @Autowired - private MatrixConfig mxCfg; + public LdapThreePidProvider(LdapConfig cfg, MatrixConfig mxCfg) { + super(cfg, mxCfg); + } @Override public boolean isEnabled() { return getCfg().isEnabled(); } - private String getUidAttribute() { - return getCfg().getAttribute().getUid().getValue(); - } - @Override public boolean isLocal() { return true; @@ -76,46 +68,22 @@ public class LdapThreePidProvider extends LdapGenericBackend implements IThreePi } private Optional lookup(LdapConnection conn, String medium, String value) { - String uidAttribute = getUidAttribute(); - Optional queryOpt = getCfg().getIdentity().getQuery(medium); if (!queryOpt.isPresent()) { log.warn("{} is not a configured 3PID type for LDAP lookup", medium); return Optional.empty(); } - String searchQuery = queryOpt.get().replaceAll("%3pid", value); - try (EntryCursor cursor = conn.search(getCfg().getConn().getBaseDn(), searchQuery, SearchScope.SUBTREE, uidAttribute)) { + String searchQuery = queryOpt.get().replaceAll(getCfg().getIdentity().getToken(), value); + try (EntryCursor cursor = conn.search(getBaseDn(), searchQuery, SearchScope.SUBTREE, getUidAtt())) { while (cursor.next()) { Entry entry = cursor.get(); log.info("Found possible match, DN: {}", entry.getDn().getName()); - Attribute attribute = entry.get(uidAttribute); - if (attribute == null) { - log.info("DN {}: no attribute {}, skpping", entry.getDn(), getCfg().getAttribute()); - continue; - } - - String data = attribute.get().toString(); - if (data.length() < 1) { - log.info("DN {}: empty attribute {}, skipping", getCfg().getAttribute()); - continue; - } - - StringBuilder matrixId = new StringBuilder(); - // TODO Should we turn this block into a map of functions? - String uidType = getCfg().getAttribute().getUid().getType(); - if (StringUtils.equals(UID, uidType)) { - matrixId.append("@").append(data).append(":").append(mxCfg.getDomain()); - } else if (StringUtils.equals(MATRIX_ID, uidType)) { - matrixId.append(data); - } else { - log.warn("Bind was found but type {} is not supported", uidType); - continue; - } - - log.info("DN {} is a valid match", entry.getDn().getName()); - return Optional.of(matrixId.toString()); + getAttribute(entry, getUidAtt()).map(uid -> { + log.info("DN {} is a valid match", entry.getDn().getName()); + return buildMatrixIdFromUid(uid); + }); } } catch (CursorLdapReferralException e) { log.warn("3PID {} is only available via referral, skipping", value); @@ -128,15 +96,11 @@ public class LdapThreePidProvider extends LdapGenericBackend implements IThreePi @Override public Optional find(SingleLookupRequest request) { - log.info("Performing LDAP lookup ${request.getThreePid()} of type ${request.getType()}"); + log.info("Performing LDAP lookup {} of type {}", request.getThreePid(), request.getType()); try (LdapConnection conn = getConn()) { bind(conn); - - Optional mxid = lookup(conn, request.getType(), request.getThreePid()); - if (mxid.isPresent()) { - return Optional.of(new SingleLookupReply(request, mxid.get())); - } + lookup(conn, request.getType(), request.getThreePid()).map(id -> new SingleLookupReply(request, id)); } catch (LdapException | IOException e) { throw new InternalServerError(e); } @@ -155,11 +119,10 @@ public class LdapThreePidProvider extends LdapGenericBackend implements IThreePi for (ThreePidMapping mapping : mappings) { try { - Optional mxid = lookup(conn, mapping.getMedium(), mapping.getValue()); - if (mxid.isPresent()) { - mapping.setMxid(mxid.get()); + lookup(conn, mapping.getMedium(), mapping.getValue()).ifPresent(id -> { + mapping.setMxid(id); mappingsFound.add(mapping); - } + }); } catch (IllegalArgumentException e) { log.warn("{} is not a supported 3PID type for LDAP lookup", mapping.getMedium()); } diff --git a/src/main/java/io/kamax/mxisd/config/ldap/LdapAttributeConfig.java b/src/main/java/io/kamax/mxisd/config/ldap/LdapAttributeConfig.java index 9ef4324..fb01de5 100644 --- a/src/main/java/io/kamax/mxisd/config/ldap/LdapAttributeConfig.java +++ b/src/main/java/io/kamax/mxisd/config/ldap/LdapAttributeConfig.java @@ -23,12 +23,17 @@ package io.kamax.mxisd.config.ldap; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @Configuration @ConfigurationProperties(prefix = "ldap.attribute") public class LdapAttributeConfig { private LdapAttributeUidConfig uid; private String name; + private Map> threepid = new HashMap<>(); public LdapAttributeUidConfig getUid() { return uid; @@ -46,4 +51,12 @@ public class LdapAttributeConfig { this.name = name; } + public Map> getThreepid() { + return threepid; + } + + public void setThreepid(Map> threepid) { + this.threepid = threepid; + } + } diff --git a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java index fa9c72c..9c4f9d7 100644 --- a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java +++ b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java @@ -21,7 +21,9 @@ package io.kamax.mxisd.config.ldap; import com.google.gson.Gson; -import io.kamax.mxisd.backend.ldap.LdapThreePidProvider; +import io.kamax.matrix.ThreePidMedium; +import io.kamax.mxisd.backend.ldap.LdapGenericBackend; +import io.kamax.mxisd.exception.ConfigurationException; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,16 +37,31 @@ import javax.annotation.PostConstruct; @ConfigurationProperties(prefix = "ldap") public class LdapConfig { + private Logger log = LoggerFactory.getLogger(LdapConfig.class); private static Gson gson = new Gson(); - private Logger log = LoggerFactory.getLogger(LdapConfig.class); - private boolean enabled; + private String filter; + + public static class Directory { + + private String filter; + + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + } @Autowired private LdapConnectionConfig conn; private LdapAttributeConfig attribute; private LdapAuthConfig auth; + private Directory directory; private LdapIdentityConfig identity; public boolean isEnabled() { @@ -55,6 +72,14 @@ public class LdapConfig { this.enabled = enabled; } + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + public LdapConnectionConfig getConn() { return conn; } @@ -79,6 +104,14 @@ public class LdapConfig { this.auth = auth; } + public Directory getDirectory() { + return directory; + } + + public void setDirectory(Directory directory) { + this.directory = directory; + } + public LdapIdentityConfig getIdentity() { return identity; } @@ -100,7 +133,7 @@ public class LdapConfig { throw new IllegalStateException("LDAP Host must be configured!"); } - if (1 > conn.getPort() || 65535 < conn.getPort()) { + if (conn.getPort() < 1 || conn.getPort() > 65535) { throw new IllegalStateException("LDAP port is not valid"); } @@ -114,10 +147,29 @@ public class LdapConfig { } String uidType = attribute.getUid().getType(); - if (!StringUtils.equals(LdapThreePidProvider.UID, uidType) && !StringUtils.equals(LdapThreePidProvider.MATRIX_ID, uidType)) { + if (!StringUtils.equals(LdapGenericBackend.UID, uidType) && !StringUtils.equals(LdapGenericBackend.MATRIX_ID, uidType)) { throw new IllegalArgumentException("Unsupported LDAP UID type: " + uidType); } + if (StringUtils.isBlank(identity.getToken())) { + throw new ConfigurationException("ldap.identity.token"); + } + + // Build queries + attribute.getThreepid().forEach((k, v) -> { + if (StringUtils.isBlank(identity.getMedium().get(k))) { + if (ThreePidMedium.PhoneNumber.is(k)) { + identity.getMedium().put(k, LdapGenericBackend.buildOrQuery("+" + getIdentity().getToken())); + } else { + identity.getMedium().put(k, LdapGenericBackend.buildOrQuery(getIdentity().getToken())); + } + } + }); + + getAuth().setFilter(StringUtils.defaultIfBlank(getAuth().getFilter(), getFilter())); + getDirectory().setFilter(StringUtils.defaultIfBlank(getDirectory().getFilter(), getFilter())); + getIdentity().setFilter(StringUtils.defaultIfBlank(getIdentity().getFilter(), getFilter())); + log.info("Host: {}", conn.getHost()); log.info("Port: {}", conn.getPort()); log.info("Bind DN: {}", conn.getBindDn()); @@ -125,6 +177,7 @@ public class LdapConfig { log.info("Attribute: {}", gson.toJson(attribute)); log.info("Auth: {}", gson.toJson(auth)); + log.info("Directory: {}", gson.toJson(directory)); log.info("Identity: {}", gson.toJson(identity)); } diff --git a/src/main/java/io/kamax/mxisd/config/ldap/LdapIdentityConfig.java b/src/main/java/io/kamax/mxisd/config/ldap/LdapIdentityConfig.java index 749530f..8bc4110 100644 --- a/src/main/java/io/kamax/mxisd/config/ldap/LdapIdentityConfig.java +++ b/src/main/java/io/kamax/mxisd/config/ldap/LdapIdentityConfig.java @@ -31,8 +31,26 @@ import java.util.Optional; @ConfigurationProperties(prefix = "ldap.identity") public class LdapIdentityConfig { + private String filter; + private String token = "%3pid"; private Map medium = new HashMap<>(); + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + public Map getMedium() { return medium; } diff --git a/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java b/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java index 79b1242..e170f8b 100644 --- a/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java +++ b/src/main/java/io/kamax/mxisd/controller/DefaultExceptionHandler.java @@ -45,65 +45,66 @@ public class DefaultExceptionHandler { private static Gson gson = new Gson(); - static String handle(String erroCode, String error) { + private String handle(HttpServletRequest req, String erroCode, String error) { JsonObject obj = new JsonObject(); obj.addProperty("errcode", erroCode); obj.addProperty("error", error); obj.addProperty("success", false); + log.info("Request {} {} - Error {}: {}", req.getMethod(), req.getRequestURL(), erroCode, error); return gson.toJson(obj); } @ExceptionHandler(InternalServerError.class) - public String handle(InternalServerError e, HttpServletResponse response) { + public String handle(HttpServletRequest req, InternalServerError e, HttpServletResponse response) { if (StringUtils.isNotBlank(e.getInternalReason())) { log.error("Reference #{} - {}", e.getReference(), e.getInternalReason()); } else { log.error("Reference #{}", e); } - return handleGeneric(e, response); + return handleGeneric(req, e, response); } @ExceptionHandler(MatrixException.class) - public String handleGeneric(MatrixException e, HttpServletResponse response) { + public String handleGeneric(HttpServletRequest req, MatrixException e, HttpServletResponse response) { response.setStatus(e.getStatus()); - return handle(e.getErrorCode(), e.getError()); + return handle(req, e.getErrorCode(), e.getError()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(MissingServletRequestParameterException.class) - public String handle(MissingServletRequestParameterException e) { - return handle("M_INCOMPLETE_REQUEST", e.getMessage()); + public String handle(HttpServletRequest req, MissingServletRequestParameterException e) { + return handle(req, "M_INCOMPLETE_REQUEST", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(InvalidResponseJsonException.class) - public String handle(InvalidResponseJsonException e) { - return handle("M_INVALID_JSON", e.getMessage()); + public String handle(HttpServletRequest req, InvalidResponseJsonException e) { + return handle(req, "M_INVALID_JSON", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(JsonSyntaxException.class) - public String handle(JsonSyntaxException e) { - return handle("M_INVALID_JSON", e.getMessage()); + public String handle(HttpServletRequest req, JsonSyntaxException e) { + return handle(req, "M_INVALID_JSON", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(JsonMemberNotFoundException.class) - public String handle(JsonMemberNotFoundException e) { - return handle("M_JSON_MISSING_KEYS", e.getMessage()); + public String handle(HttpServletRequest req, JsonMemberNotFoundException e) { + return handle(req, "M_JSON_MISSING_KEYS", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(MappingAlreadyExistsException.class) - public String handle(MappingAlreadyExistsException e) { - return handle("M_ALREADY_EXISTS", e.getMessage()); + public String handle(HttpServletRequest req, MappingAlreadyExistsException e) { + return handle(req, "M_ALREADY_EXISTS", e.getMessage()); } @ResponseStatus(HttpStatus.BAD_REQUEST) @ExceptionHandler(BadRequestException.class) - public String handle(BadRequestException e) { - return handle("M_BAD_REQUEST", e.getMessage()); + public String handle(HttpServletRequest req, BadRequestException e) { + return handle(req, "M_BAD_REQUEST", e.getMessage()); } @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) @@ -111,6 +112,7 @@ public class DefaultExceptionHandler { public String handle(HttpServletRequest req, RuntimeException e) { log.error("Unknown error when handling {}", req.getRequestURL(), e); return handle( + req, "M_UNKNOWN", StringUtils.defaultIfBlank( e.getMessage(), diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 7054d0a..708c83d 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -44,6 +44,7 @@ rest: ldap: enabled: false + filter: '' connection: tls: false port: 389 @@ -52,10 +53,27 @@ ldap: type: 'uid' value: 'userPrincipalName' name: 'displayName' + threepid: + email: + - 'mailPrimaryAddress' + - 'mail' + - 'otherMailbox' + msisdn: + - 'telephoneNumber' + - 'mobile' + - 'homePhone' + - 'otherTelephone' + - 'otherMobile' + - 'otherHomePhone' + auth: + filter: '' + directory: + filter: '' identity: + filter: '' medium: - email: "(|(mailPrimaryAddress=%3pid)(mail=%3pid)(otherMailbox=%3pid))" - msisdn: "(|(telephoneNumber=+%3pid)(mobile=+%3pid)(homePhone=+%3pid)(otherTelephone=+%3pid)(otherMobile=+%3pid)(otherHomePhone=+%3pid))" + email: '' + msisdn: '' firebase: enabled: false From 69ecef01558f851b5b9d264ffdf74950db9a72ce Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Fri, 29 Sep 2017 22:13:51 +0200 Subject: [PATCH 06/14] Refactored directory package to include API version --- .../io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java | 2 +- .../io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java | 4 ++-- .../directory/{ => v1}/UserDirectoryController.java | 6 +++--- .../directory/{ => v1}/io/UserDirectorySearchRequest.java | 2 +- .../directory/{ => v1}/io/UserDirectorySearchResult.java | 2 +- .../java/io/kamax/mxisd/directory/DirectoryManager.java | 4 ++-- .../java/io/kamax/mxisd/directory/IDirectoryProvider.java | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) rename src/main/java/io/kamax/mxisd/controller/directory/{ => v1}/UserDirectoryController.java (90%) rename src/main/java/io/kamax/mxisd/controller/directory/{ => v1}/io/UserDirectorySearchRequest.java (95%) rename src/main/java/io/kamax/mxisd/controller/directory/{ => v1}/io/UserDirectorySearchResult.java (97%) diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java index 2c25b3f..f6f5254 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java @@ -23,7 +23,7 @@ package io.kamax.mxisd.backend.ldap; import io.kamax.mxisd.config.MatrixConfig; import io.kamax.mxisd.config.ldap.LdapAttributeConfig; import io.kamax.mxisd.config.ldap.LdapConfig; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; import io.kamax.mxisd.directory.IDirectoryProvider; import io.kamax.mxisd.exception.InternalServerError; import org.apache.directory.api.ldap.model.cursor.CursorException; diff --git a/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java index 3be7489..97a36f0 100644 --- a/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/sql/SqlDirectoryProvider.java @@ -24,7 +24,7 @@ import io.kamax.matrix.MatrixID; import io.kamax.mxisd.config.MatrixConfig; import io.kamax.mxisd.config.sql.SqlConfig; import io.kamax.mxisd.config.sql.SqlProviderConfig; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; import io.kamax.mxisd.directory.IDirectoryProvider; import io.kamax.mxisd.exception.InternalServerError; import org.apache.commons.lang.StringUtils; @@ -37,7 +37,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Optional; -import static io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult.Result; +import static io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult.Result; public abstract class SqlDirectoryProvider implements IDirectoryProvider { diff --git a/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java b/src/main/java/io/kamax/mxisd/controller/directory/v1/UserDirectoryController.java similarity index 90% rename from src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java rename to src/main/java/io/kamax/mxisd/controller/directory/v1/UserDirectoryController.java index 78781f4..6af2a9f 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/UserDirectoryController.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/v1/UserDirectoryController.java @@ -18,11 +18,11 @@ * along with this program. If not, see . */ -package io.kamax.mxisd.controller.directory; +package io.kamax.mxisd.controller.directory.v1; import com.google.gson.Gson; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchRequest; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchRequest; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; import io.kamax.mxisd.directory.DirectoryManager; import io.kamax.mxisd.util.GsonParser; import io.kamax.mxisd.util.GsonUtil; diff --git a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java similarity index 95% rename from src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java rename to src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java index 2f78d87..83e5639 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchRequest.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java @@ -18,7 +18,7 @@ * along with this program. If not, see . */ -package io.kamax.mxisd.controller.directory.io; +package io.kamax.mxisd.controller.directory.v1.io; public class UserDirectorySearchRequest { diff --git a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchResult.java similarity index 97% rename from src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java rename to src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchResult.java index 128385c..5c63202 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/io/UserDirectorySearchResult.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchResult.java @@ -18,7 +18,7 @@ * along with this program. If not, see . */ -package io.kamax.mxisd.controller.directory.io; +package io.kamax.mxisd.controller.directory.v1.io; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java index bf0045e..eebd1ac 100644 --- a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java +++ b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java @@ -22,8 +22,8 @@ package io.kamax.mxisd.directory; import com.google.gson.Gson; import io.kamax.matrix.MatrixErrorInfo; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchRequest; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchRequest; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; import io.kamax.mxisd.dns.ClientDnsOverwrite; import io.kamax.mxisd.exception.InternalServerError; import io.kamax.mxisd.exception.MatrixException; diff --git a/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java b/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java index ddbf025..7f173f6 100644 --- a/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/directory/IDirectoryProvider.java @@ -20,7 +20,7 @@ package io.kamax.mxisd.directory; -import io.kamax.mxisd.controller.directory.io.UserDirectorySearchResult; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; public interface IDirectoryProvider { From 52e4a65c3c91f2488997c170f637afa9921de0f8 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sat, 30 Sep 2017 00:27:36 +0200 Subject: [PATCH 07/14] Fix query generation --- .../mxisd/backend/ldap/LdapAuthProvider.java | 12 ++---------- .../mxisd/backend/ldap/LdapGenericBackend.java | 15 ++++++++++++--- .../mxisd/backend/ldap/LdapThreePidProvider.java | 16 ++++++++-------- .../io/kamax/mxisd/config/ldap/LdapConfig.java | 4 ++-- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java index 73c3ade..1079370 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapAuthProvider.java @@ -72,7 +72,7 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato return BackendAuthResult.failure(); } - String userFilter = "(" + getCfg().getAttribute().getUid().getValue() + "=" + userFilterValue + ")"; + String userFilter = "(" + getUidAtt() + "=" + userFilterValue + ")"; userFilter = buildWithFilter(userFilter, getCfg().getAuth().getFilter()); try (EntryCursor cursor = conn.search(getBaseDn(), userFilter, SearchScope.SUBTREE, getUidAtt(), getAt().getName())) { while (cursor.next()) { @@ -80,15 +80,7 @@ public class LdapAuthProvider extends LdapGenericBackend implements Authenticato String dn = entry.getDn().getName(); log.info("Checking possible match, DN: {}", dn); - Attribute attribute = entry.get(getUidAtt()); - if (attribute == null) { - log.info("DN {}: no attribute {}, skpping", dn, getUidAtt()); - continue; - } - - String data = attribute.get().toString(); - if (data.length() < 1) { - log.info("DN {}: empty attribute {}, skipping", getUidAtt()); + if (!getAttribute(entry, getUidAtt()).isPresent()) { continue; } diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java index b87d503..df7ad60 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapGenericBackend.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.List; import java.util.Optional; public abstract class LdapGenericBackend { @@ -66,7 +67,7 @@ public abstract class LdapGenericBackend { return getAt().getUid().getValue(); } - protected LdapConnection getConn() { + protected synchronized LdapConnection getConn() throws LdapException { return new LdapNetworkConnection(cfg.getConn().getHost(), cfg.getConn().getPort(), cfg.getConn().isTls()); } @@ -86,10 +87,14 @@ public abstract class LdapGenericBackend { } } - public static String buildOrQuery(String value, String... attributes) { + public static String buildOrQuery(String value, List attributes) { + if (attributes.size() < 1) { + throw new IllegalArgumentException(); + } + StringBuilder builder = new StringBuilder(); builder.append("(|"); - Arrays.stream(attributes).forEach(s -> { + attributes.forEach(s -> { builder.append("("); builder.append(s).append("=").append(value).append(")"); }); @@ -97,6 +102,10 @@ public abstract class LdapGenericBackend { return builder.toString(); } + public static String buildOrQuery(String value, String... attributes) { + return buildOrQuery(value, Arrays.asList(attributes)); + } + public String buildOrQueryWithFilter(String filter, String value, String... attributes) { return buildWithFilter(buildOrQuery(value, attributes), filter); } diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java index 9715ac4..103eb56 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapThreePidProvider.java @@ -80,10 +80,13 @@ public class LdapThreePidProvider extends LdapGenericBackend implements IThreePi Entry entry = cursor.get(); log.info("Found possible match, DN: {}", entry.getDn().getName()); - getAttribute(entry, getUidAtt()).map(uid -> { - log.info("DN {} is a valid match", entry.getDn().getName()); - return buildMatrixIdFromUid(uid); - }); + Optional data = getAttribute(entry, getUidAtt()); + if (!data.isPresent()) { + continue; + } + + log.info("DN {} is a valid match", entry.getDn().getName()); + return Optional.of(buildMatrixIdFromUid(data.get())); } } catch (CursorLdapReferralException e) { log.warn("3PID {} is only available via referral, skipping", value); @@ -100,13 +103,10 @@ public class LdapThreePidProvider extends LdapGenericBackend implements IThreePi try (LdapConnection conn = getConn()) { bind(conn); - lookup(conn, request.getType(), request.getThreePid()).map(id -> new SingleLookupReply(request, id)); + return lookup(conn, request.getType(), request.getThreePid()).map(id -> new SingleLookupReply(request, id)); } catch (LdapException | IOException e) { throw new InternalServerError(e); } - - log.info("No match found"); - return Optional.empty(); } @Override diff --git a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java index 9c4f9d7..01541bf 100644 --- a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java +++ b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java @@ -159,9 +159,9 @@ public class LdapConfig { attribute.getThreepid().forEach((k, v) -> { if (StringUtils.isBlank(identity.getMedium().get(k))) { if (ThreePidMedium.PhoneNumber.is(k)) { - identity.getMedium().put(k, LdapGenericBackend.buildOrQuery("+" + getIdentity().getToken())); + identity.getMedium().put(k, LdapGenericBackend.buildOrQuery("+" + getIdentity().getToken(), v)); } else { - identity.getMedium().put(k, LdapGenericBackend.buildOrQuery(getIdentity().getToken())); + identity.getMedium().put(k, LdapGenericBackend.buildOrQuery(getIdentity().getToken(), v)); } } }); From 88a37c52c0ffdf60610eff5f49e3563afb17dcf2 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sat, 30 Sep 2017 00:56:16 +0200 Subject: [PATCH 08/14] Skeleton for User directory setup instructions --- docs/features/directory-users.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/features/directory-users.md diff --git a/docs/features/directory-users.md b/docs/features/directory-users.md new file mode 100644 index 0000000..304e758 --- /dev/null +++ b/docs/features/directory-users.md @@ -0,0 +1,14 @@ +- Only work for LDAP and SQL + - For LDAP: Set LDAP config (new global filter, optional: threepids attributes if the default identity queries were changed) + - For SQL: Use `synapseSql` module with `type: {sqlite|postgresql}` and `database` as JDBC url after `jdbc:driver:` + - `/path/to/db` for `sqlite` + - `//host/db?username...` for `postgresql`) +- Configure DNS overwrite for domain name (and mention ${matrix.domain} can be used) +``` +dns.overwrite.homeserver.client: + - name: 'example.org' + value: 'http://localhost:8008' +``` +- Configure reverse proxy + - for `/_matrix/client/r0/user_directory/search` to `http://internalIp:8008/_matrix/client/r0/user_directory/search` + - With `ProxyPreserveHost on` on apache From 8d0b0edad2ff21468a1c97b2a80e296793d826c0 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 00:06:03 +0200 Subject: [PATCH 09/14] Clarify some items thanks to users feedback --- docs/features/directory-users.md | 4 ++-- src/main/java/io/kamax/mxisd/directory/DirectoryManager.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/features/directory-users.md b/docs/features/directory-users.md index 304e758..e77bf1d 100644 --- a/docs/features/directory-users.md +++ b/docs/features/directory-users.md @@ -3,12 +3,12 @@ - For SQL: Use `synapseSql` module with `type: {sqlite|postgresql}` and `database` as JDBC url after `jdbc:driver:` - `/path/to/db` for `sqlite` - `//host/db?username...` for `postgresql`) -- Configure DNS overwrite for domain name (and mention ${matrix.domain} can be used) +- Configure DNS overwrite for the Homeserver hostname used when connecting as a client (and mention ${matrix.domain} can be used) ``` dns.overwrite.homeserver.client: - name: 'example.org' value: 'http://localhost:8008' ``` - Configure reverse proxy - - for `/_matrix/client/r0/user_directory/search` to `http://internalIp:8008/_matrix/client/r0/user_directory/search` + - for `/_matrix/client/r0/user_directory/search` to `http://internalIpOfMxisd:8090/_matrix/client/r0/user_directory/search` - With `ProxyPreserveHost on` on apache diff --git a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java index eebd1ac..b7c2d0b 100644 --- a/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java +++ b/src/main/java/io/kamax/mxisd/directory/DirectoryManager.java @@ -21,6 +21,7 @@ package io.kamax.mxisd.directory; import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import io.kamax.matrix.MatrixErrorInfo; import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchRequest; import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; @@ -96,8 +97,10 @@ public class DirectoryManager { if (resultHs.isLimited()) { result.setLimited(true); } + } catch (JsonSyntaxException e) { + throw new InternalServerError("Invalid JSON reply from the HS: " + e.getMessage()); } catch (IOException e) { - throw new InternalServerError(e); + throw new InternalServerError("Unable to query the HS: I/O error: " + e.getMessage()); } for (IDirectoryProvider provider : providers) { From 786e4a8f916fb0d6bc35217fb8153c900dbeb76b Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 02:20:15 +0200 Subject: [PATCH 10/14] Prepare REST backend for directory flow --- docs/backends/rest.md | 79 +++++++++++++------ .../mxisd/config/rest/RestBackendConfig.java | 29 ++++--- src/main/resources/application.yaml | 7 +- 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/docs/backends/rest.md b/docs/backends/rest.md index 094e85a..b281d50 100644 --- a/docs/backends/rest.md +++ b/docs/backends/rest.md @@ -6,33 +6,33 @@ The REST backend allows you to query identity data in existing webapps, like: - self-hosted clouds (Nextcloud, ownCloud, ...) It supports the following mxisd flows: -- Identity lookup -- Authentication +- [Authentication](#authentication) +- [Directory](#directory) +- [Identity](#identity) To integrate this backend with your webapp, you will need to implement three specific REST endpoints detailed below. ## Configuration -| Key | Default | Description | ----------------------------------|---------------------------------------|------------------------------------------------------| -| rest.enabled | false | Globally enable/disable the REST backend | -| rest.host | *empty* | Default base URL to use for the different endpoints. | -| rest.endpoints.auth | /_mxisd/identity/api/v1/auth | Endpoint to validate credentials | -| rest.endpoints.identity.single | /_mxisd/identity/api/v1/lookup/single | Endpoint to query a single 3PID | -| rest.endpoints.identity.bulk | /_mxisd/identity/api/v1/lookup/bulk | Endpoint to query a list of 3PID | +| Key | Default | Description | +---------------------------------|----------------------------------------------|------------------------------------------------------| +| rest.enabled | false | Globally enable/disable the REST backend | +| rest.host | *empty* | Default base URL to use for the different endpoints. | +| rest.endpoints.auth | /_mxisd/backend/api/v1/auth/login | Validate credentials and get user profile | +| rest.endpoints.directory | /_mxisd/backend/api/v1/directory/user/search | Search for users by arbitrary input | +| rest.endpoints.identity.single | /_mxisd/backend/api/v1/identity/single | Endpoint to query a single 3PID | +| rest.endpoints.identity.bulk | /_mxisd/backend/api/v1/identity/bulk | Endpoint to query a list of 3PID | Endpoint values can handle two formats: - URL Path starting with `/` that gets happened to the `rest.host` - Full URL, if you want each endpoint to go to a specific server/protocol/port -`rest.host` is only mandatory if at least one endpoint is not a full URL. +`rest.host` is mandatory if at least one endpoint is not a full URL. ## Endpoints -### Authenticate -Configured with `rest.endpoints.auth` - +### Authentication HTTP method: `POST` -Encoding: JSON UTF-8 +Content-type: JSON UTF-8 #### Request Body ``` @@ -84,12 +84,47 @@ If the authentication succeed: } ``` -### Lookup -#### Single -Configured with `rest.endpoints.identity.single` +### Directory +HTTP method: `POST` +Content-type: JSON UTF-8 +#### Request Body +``` +{ + "search_term": "doe" +} +``` + +#### Response Body: +If users found: +``` +{ + "limited": false, + "results": [ + { + "display_name": "John Doe", + "avatar_url": "http://domain.tld/path/to/avatar.png", + "user_id": "UserIdLocalpart" + }, + { + ... + } + ] +} +``` + +If no user found: +``` +{ + "limited": false, + "results": [] +} +``` + +### Identity +#### Single 3PID lookup HTTP method: `POST` -Encoding: JSON UTF-8 +Content-type: JSON UTF-8 #### Request Body ``` @@ -122,11 +157,9 @@ If no match was found: {} ``` -#### Bulk -Configured with `rest.endpoints.identity.bulk` - +#### Bulk 3PID lookup HTTP method: `POST` -Encoding: JSON UTF-8 +Content-type: JSON UTF-8 #### Request Body ``` @@ -175,4 +208,4 @@ If no match was found: { "lookup": [] } -``` \ No newline at end of file +``` diff --git a/src/main/java/io/kamax/mxisd/config/rest/RestBackendConfig.java b/src/main/java/io/kamax/mxisd/config/rest/RestBackendConfig.java index d9a4295..005cfff 100644 --- a/src/main/java/io/kamax/mxisd/config/rest/RestBackendConfig.java +++ b/src/main/java/io/kamax/mxisd/config/rest/RestBackendConfig.java @@ -60,16 +60,9 @@ public class RestBackendConfig { public static class Endpoints { - private IdentityEndpoints identity = new IdentityEndpoints(); private String auth; - - public IdentityEndpoints getIdentity() { - return identity; - } - - public void setIdentity(IdentityEndpoints identity) { - this.identity = identity; - } + private String directory; + private IdentityEndpoints identity = new IdentityEndpoints(); public String getAuth() { return auth; @@ -79,6 +72,22 @@ public class RestBackendConfig { this.auth = auth; } + public String getDirectory() { + return directory; + } + + public void setDirectory(String directory) { + this.directory = directory; + } + + public IdentityEndpoints getIdentity() { + return identity; + } + + public void setIdentity(IdentityEndpoints identity) { + this.identity = identity; + } + } private Logger log = LoggerFactory.getLogger(RestBackendConfig.class); @@ -136,11 +145,13 @@ public class RestBackendConfig { if (isEnabled()) { endpoints.setAuth(buildEndpointUrl(endpoints.getAuth())); + endpoints.setDirectory(buildEndpointUrl(endpoints.getDirectory())); endpoints.identity.setSingle(buildEndpointUrl(endpoints.identity.getSingle())); endpoints.identity.setBulk(buildEndpointUrl(endpoints.identity.getBulk())); log.info("Host: {}", getHost()); log.info("Auth endpoint: {}", endpoints.getAuth()); + log.info("Directory endpoint: {}", endpoints.getDirectory()); log.info("Identity Single endpoint: {}", endpoints.identity.getSingle()); log.info("Identity Bulk endpoint: {}", endpoints.identity.getBulk()); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 708c83d..dbe31f7 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -37,10 +37,11 @@ lookup: rest: endpoints: - auth: "/_mxisd/identity/api/v1/auth" + auth: '/_mxisd/backend/api/v1/auth/login' + directory: '/_mxisd/backend/api/v1/directory/user/search' identity: - single: "/_mxisd/identity/api/v1/lookup/single" - bulk: "/_mxisd/identity/api/v1/lookup/bulk" + single: '/_mxisd/backend/api/v1/identity/lookup/single' + bulk: '/_mxisd/backend/api/v1/identity/lookup/bulk' ldap: enabled: false From c702a34aab146ea81b22a24025f352b7f00c140b Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 16:10:05 +0200 Subject: [PATCH 11/14] Fix regression due to bad replace --- src/main/java/io/kamax/mxisd/UserIdType.java | 2 +- .../identity/v1/io/SessionEmailTokenRequestJson.java | 4 +++- .../io/kamax/mxisd/lookup/provider/DnsLookupProvider.java | 5 +++-- src/main/java/io/kamax/mxisd/matrix/IdentityServerUtils.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/kamax/mxisd/UserIdType.java b/src/main/java/io/kamax/mxisd/UserIdType.java index 5b94ec2..e0625bf 100644 --- a/src/main/java/io/kamax/mxisd/UserIdType.java +++ b/src/main/java/io/kamax/mxisd/UserIdType.java @@ -28,7 +28,7 @@ public enum UserIdType { Localpart("localpart"), MatrixID("mxid"), EmailLocalpart("email_localpart"), - Email("threepids/email"); + Email("email"); private String id; diff --git a/src/main/java/io/kamax/mxisd/controller/identity/v1/io/SessionEmailTokenRequestJson.java b/src/main/java/io/kamax/mxisd/controller/identity/v1/io/SessionEmailTokenRequestJson.java index 49b1f76..58c527c 100644 --- a/src/main/java/io/kamax/mxisd/controller/identity/v1/io/SessionEmailTokenRequestJson.java +++ b/src/main/java/io/kamax/mxisd/controller/identity/v1/io/SessionEmailTokenRequestJson.java @@ -20,12 +20,14 @@ package io.kamax.mxisd.controller.identity.v1.io; +import io.kamax.matrix.ThreePidMedium; + public class SessionEmailTokenRequestJson extends GenericTokenRequestJson { private String email; public String getMedium() { - return "threepids/email"; + return ThreePidMedium.Email.getId(); } public String getValue() { diff --git a/src/main/java/io/kamax/mxisd/lookup/provider/DnsLookupProvider.java b/src/main/java/io/kamax/mxisd/lookup/provider/DnsLookupProvider.java index 5564e54..5230f5c 100644 --- a/src/main/java/io/kamax/mxisd/lookup/provider/DnsLookupProvider.java +++ b/src/main/java/io/kamax/mxisd/lookup/provider/DnsLookupProvider.java @@ -20,6 +20,7 @@ package io.kamax.mxisd.lookup.provider; +import io.kamax.matrix.ThreePidMedium; import io.kamax.mxisd.config.MatrixConfig; import io.kamax.mxisd.lookup.SingleLookupReply; import io.kamax.mxisd.lookup.SingleLookupRequest; @@ -83,7 +84,7 @@ class DnsLookupProvider implements IThreePidProvider { @Override public Optional find(SingleLookupRequest request) { - if (!StringUtils.equals("threepids/email", request.getType())) { // TODO use enum + if (!ThreePidMedium.Email.is(request.getType())) { // TODO use enum log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid()); return Optional.empty(); } @@ -106,7 +107,7 @@ class DnsLookupProvider implements IThreePidProvider { Map> domains = new HashMap<>(); for (ThreePidMapping mapping : mappings) { - if (!StringUtils.equals("threepids/email", mapping.getMedium())) { + if (!ThreePidMedium.Email.is(mapping.getMedium())) { log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue()); continue; } diff --git a/src/main/java/io/kamax/mxisd/matrix/IdentityServerUtils.java b/src/main/java/io/kamax/mxisd/matrix/IdentityServerUtils.java index 5441a3f..34b1a6a 100644 --- a/src/main/java/io/kamax/mxisd/matrix/IdentityServerUtils.java +++ b/src/main/java/io/kamax/mxisd/matrix/IdentityServerUtils.java @@ -21,7 +21,7 @@ import java.util.Optional; // FIXME placeholder, this must go in matrix-java-sdk for 1.0 public class IdentityServerUtils { - public static final String THREEPID_TEST_MEDIUM = "threepids/email"; + public static final String THREEPID_TEST_MEDIUM = "email"; public static final String THREEPID_TEST_ADDRESS = "mxisd-email-forever-unknown@forever-invalid.kamax.io"; private static Logger log = LoggerFactory.getLogger(IdentityServerUtils.class); From d0aac5ac5250e3d18aa8082ca714ff5bccc120d1 Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 18:13:01 +0200 Subject: [PATCH 12/14] User Directory support in REST Backend --- docs/backends/rest.md | 6 +- .../backend/rest/RestDirectoryProvider.java | 82 +++++++++ .../v1/io/UserDirectorySearchRequest.java | 9 + .../rest/RestDirectoryProviderTest.java | 167 ++++++++++++++++++ .../rest/RestThreePidProviderTest.java | 2 +- 5 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 src/main/java/io/kamax/mxisd/backend/rest/RestDirectoryProvider.java create mode 100644 src/test/java/io/kamax/mxisd/backend/rest/RestDirectoryProviderTest.java diff --git a/docs/backends/rest.md b/docs/backends/rest.md index b281d50..c59a5f4 100644 --- a/docs/backends/rest.md +++ b/docs/backends/rest.md @@ -91,9 +91,13 @@ Content-type: JSON UTF-8 #### Request Body ``` { + "by": "", "search_term": "doe" } ``` +`by` can be: +- `name` +- `threepid` #### Response Body: If users found: @@ -102,8 +106,8 @@ If users found: "limited": false, "results": [ { - "display_name": "John Doe", "avatar_url": "http://domain.tld/path/to/avatar.png", + "display_name": "John Doe", "user_id": "UserIdLocalpart" }, { diff --git a/src/main/java/io/kamax/mxisd/backend/rest/RestDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/rest/RestDirectoryProvider.java new file mode 100644 index 0000000..b92abbc --- /dev/null +++ b/src/main/java/io/kamax/mxisd/backend/rest/RestDirectoryProvider.java @@ -0,0 +1,82 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.rest; + +import io.kamax.matrix.MatrixID; +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.rest.RestBackendConfig; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchRequest; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; +import io.kamax.mxisd.directory.IDirectoryProvider; +import io.kamax.mxisd.exception.InternalServerError; +import io.kamax.mxisd.util.RestClientUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.client.methods.CloseableHttpResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class RestDirectoryProvider extends RestProvider implements IDirectoryProvider { + + private MatrixConfig mxCfg; + + public RestDirectoryProvider(RestBackendConfig cfg, MatrixConfig mxCfg) { + super(cfg); + this.mxCfg = mxCfg; + } + + @Override + public boolean isEnabled() { + return cfg.isEnabled() && StringUtils.isNotBlank(cfg.getEndpoints().getDirectory()); + } + + private UserDirectorySearchResult search(String by, String query) { + UserDirectorySearchRequest request = new UserDirectorySearchRequest(query); + request.setBy(by); + try (CloseableHttpResponse httpResponse = client.execute(RestClientUtils.post(cfg.getEndpoints().getDirectory(), request))) { + int status = httpResponse.getStatusLine().getStatusCode(); + if (status < 200 || status >= 300) { + throw new InternalServerError("REST backend: Error: " + IOUtils.toString(httpResponse.getEntity().getContent(), StandardCharsets.UTF_8)); + } + + UserDirectorySearchResult response = parser.parse(httpResponse, UserDirectorySearchResult.class); + for (UserDirectorySearchResult.Result result : response.getResults()) { + result.setUserId(new MatrixID(result.getUserId(), mxCfg.getDomain()).getId()); + } + + return response; + } catch (IOException e) { + throw new InternalServerError("REST backend: I/O error: " + e.getMessage()); + } + } + + @Override + public UserDirectorySearchResult searchByDisplayName(String query) { + return search("name", query); + } + + @Override + public UserDirectorySearchResult searchBy3pid(String query) { + return search("threepid", query); + } + +} diff --git a/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java index 83e5639..1bd333a 100644 --- a/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java +++ b/src/main/java/io/kamax/mxisd/controller/directory/v1/io/UserDirectorySearchRequest.java @@ -22,12 +22,21 @@ package io.kamax.mxisd.controller.directory.v1.io; public class UserDirectorySearchRequest { + private String by; private String searchTerm; public UserDirectorySearchRequest(String searchTerm) { setSearchTerm(searchTerm); } + public String getBy() { + return by; + } + + public void setBy(String by) { + this.by = by; + } + public String getSearchTerm() { return searchTerm; } diff --git a/src/test/java/io/kamax/mxisd/backend/rest/RestDirectoryProviderTest.java b/src/test/java/io/kamax/mxisd/backend/rest/RestDirectoryProviderTest.java new file mode 100644 index 0000000..a60bc92 --- /dev/null +++ b/src/test/java/io/kamax/mxisd/backend/rest/RestDirectoryProviderTest.java @@ -0,0 +1,167 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.backend.rest; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import io.kamax.matrix.MatrixID; +import io.kamax.mxisd.config.MatrixConfig; +import io.kamax.mxisd.config.rest.RestBackendConfig; +import io.kamax.mxisd.controller.directory.v1.io.UserDirectorySearchResult; +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class RestDirectoryProviderTest { + + @Rule + public WireMockRule wireMockRule = new WireMockRule(65000); + + private RestDirectoryProvider p; + + private String domain = "example.org"; + private String endpoint = "/directory/search"; + private String byNameSearch = "doe"; + private String byNameAvatar = "http://domain.tld/path/to/avatar.png"; + private String byNameDisplay = "John Doe"; + private String byNameId = "john.doe"; + private String byNameRequest = "{\"by\":\"name\",\"search_term\":\"" + byNameSearch + "\"}"; + private String byNameResponse = "{\"limited\":false,\"results\":[{\"avatar_url\":\"" + byNameAvatar + + "\",\"display_name\":\"" + byNameDisplay + "\",\"user_id\":\"" + byNameId + "\"}]}"; + private String byNameEmptyResponse = "{\"limited\":false,\"results\":[]}"; + + private String byThreepidSearch = "jane"; + private String byThreepidAvatar = "http://domain.tld/path/to/avatar.png"; + private String byThreepidDisplay = "John Doe"; + private String byThreepidId = "john.doe"; + private String byThreepidRequest = "{\"by\":\"threepid\",\"search_term\":\"" + byThreepidSearch + "\"}"; + private String byThreepidResponse = "{\"limited\":false,\"results\":[{\"avatar_url\":\"" + byThreepidAvatar + + "\",\"display_name\":\"" + byThreepidDisplay + "\",\"user_id\":\"" + byThreepidId + "\"}]}"; + private String byThreepidEmptyResponse = "{\"limited\":false,\"results\":[]}"; + + @Before + public void before() { + MatrixConfig mxCfg = new MatrixConfig(); + mxCfg.setDomain(domain); + mxCfg.build(); + + RestBackendConfig cfg = new RestBackendConfig(); + cfg.setEnabled(true); + cfg.setHost("http://localhost:65000"); + cfg.getEndpoints().setDirectory(endpoint); + cfg.build(); + + p = new RestDirectoryProvider(cfg, mxCfg); + } + + @Test + public void byNameFound() { + stubFor(post(urlEqualTo(endpoint)) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(byNameResponse) + ) + ); + + UserDirectorySearchResult result = p.searchByDisplayName(byNameSearch); + assertTrue(!result.isLimited()); + assertTrue(result.getResults().size() == 1); + UserDirectorySearchResult.Result entry = result.getResults().get(0); + assertNotNull(entry); + assertTrue(StringUtils.equals(byNameAvatar, entry.getAvatarUrl())); + assertTrue(StringUtils.equals(byNameDisplay, entry.getDisplayName())); + assertTrue(StringUtils.equals(new MatrixID(byNameId, domain).getId(), entry.getUserId())); + + verify(postRequestedFor(urlMatching(endpoint)) + .withHeader("Content-Type", containing("application/json")) + .withRequestBody(equalTo(byNameRequest)) + ); + } + + @Test + public void byNameNotFound() { + stubFor(post(urlEqualTo(endpoint)) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(byNameEmptyResponse) + ) + ); + + UserDirectorySearchResult result = p.searchByDisplayName(byNameSearch); + assertTrue(!result.isLimited()); + assertTrue(result.getResults().isEmpty()); + + verify(postRequestedFor(urlMatching(endpoint)) + .withHeader("Content-Type", containing("application/json")) + .withRequestBody(equalTo(byNameRequest)) + ); + } + + @Test + public void byThreepidFound() { + stubFor(post(urlEqualTo(endpoint)) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(new String(byThreepidResponse.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8)) + ) + ); + + UserDirectorySearchResult result = p.searchBy3pid(byThreepidSearch); + assertTrue(!result.isLimited()); + assertTrue(result.getResults().size() == 1); + UserDirectorySearchResult.Result entry = result.getResults().get(0); + assertNotNull(entry); + assertTrue(StringUtils.equals(byThreepidAvatar, entry.getAvatarUrl())); + assertTrue(StringUtils.equals(byThreepidDisplay, entry.getDisplayName())); + assertTrue(StringUtils.equals(new MatrixID(byThreepidId, domain).getId(), entry.getUserId())); + + verify(postRequestedFor(urlMatching(endpoint)) + .withHeader("Content-Type", containing("application/json")) + .withRequestBody(equalTo(byThreepidRequest)) + ); + } + + @Test + public void byThreepidNotFound() { + stubFor(post(urlEqualTo(endpoint)) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(byThreepidEmptyResponse) + ) + ); + + UserDirectorySearchResult result = p.searchBy3pid(byThreepidSearch); + assertTrue(!result.isLimited()); + assertTrue(result.getResults().isEmpty()); + + verify(postRequestedFor(urlMatching(endpoint)) + .withHeader("Content-Type", containing("application/json")) + .withRequestBody(equalTo(byThreepidRequest)) + ); + } + +} diff --git a/src/test/java/io/kamax/mxisd/backend/rest/RestThreePidProviderTest.java b/src/test/java/io/kamax/mxisd/backend/rest/RestThreePidProviderTest.java index 0289b12..97dda60 100644 --- a/src/test/java/io/kamax/mxisd/backend/rest/RestThreePidProviderTest.java +++ b/src/test/java/io/kamax/mxisd/backend/rest/RestThreePidProviderTest.java @@ -53,7 +53,7 @@ public class RestThreePidProviderTest { cfg.setEnabled(true); cfg.setHost("http://localhost:65000"); cfg.getEndpoints().getIdentity().setSingle(lookupSinglePath); - cfg.getEndpoints().getIdentity().setBulk("/lookup/bulk"); + cfg.getEndpoints().getIdentity().setBulk(lookupBulkPath); cfg.build(); p = new RestThreePidProvider(cfg, mxCfg); From 8662b3f39f915479ec3bbde774fc45456daf3a4e Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 19:36:11 +0200 Subject: [PATCH 13/14] Stable implementation of Directory integration - Documentation - Allow to specific other attributes in LDAP to include in the search --- docs/features/directory-users.md | 150 ++++++++++++++++-- .../backend/ldap/LdapDirectoryProvider.java | 8 +- .../kamax/mxisd/config/ldap/LdapConfig.java | 25 +++ .../io/kamax/mxisd/config/sql/SqlConfig.java | 5 +- src/main/resources/application.yaml | 4 + 5 files changed, 179 insertions(+), 13 deletions(-) diff --git a/docs/features/directory-users.md b/docs/features/directory-users.md index e77bf1d..a7187f1 100644 --- a/docs/features/directory-users.md +++ b/docs/features/directory-users.md @@ -1,14 +1,146 @@ -- Only work for LDAP and SQL - - For LDAP: Set LDAP config (new global filter, optional: threepids attributes if the default identity queries were changed) - - For SQL: Use `synapseSql` module with `type: {sqlite|postgresql}` and `database` as JDBC url after `jdbc:driver:` - - `/path/to/db` for `sqlite` - - `//host/db?username...` for `postgresql`) -- Configure DNS overwrite for the Homeserver hostname used when connecting as a client (and mention ${matrix.domain} can be used) +# User Directory +This feature allows you to search for existing and/or potential users that are already present in your Identity backend +or that already share a room with you on the Homeserver. + +Without any integration, synapse: +- Only search within the users **already** known to you +- Only search on the Display Name and the Matrix ID + +With mxisd integration, you can: +- Search on Matrix ID, Display name and 3PIDs (Email, phone numbers) of any users already in your configured backend +- Search for users which you are not in contact with yet. Super useful for corporations who want to give Matrix access +internally, so users can just find themselves **prior** to having any common room(s) +- Use any attribute of your backend to extend the search! + +## Overview +This is performed by intercepting the Homeserver endpoint `/_matrix/client/r0/user_directory/search` like so: +``` + +----------------------------------------------+ +client --> | Reverse proxy Step 2 + | Step 1 +-------------------------+ + | /_matrix/client/r0/user_directory/search ----------> | | Search in +---------+ + | /\ | mxisd - Identity server | -----------> | Backend | + | /_matrix/* \----------------------------- | | all users +---------+ + | | Step 4: Send back merged results +-------------------------+ + +--------|------- | + | Step 3 + | | + | +------------+ Search in known users + \--> | Homeserver | <----------------------------------------/ + +------------+ /_matrix/client/r0/user_directory/search +``` + +## Requirements +- Reverse proxy setup, which you should already have in place if you use mxisd +- Compatible backends: + - LDAP + - SQL + - REST + +## Configuration +### Reverse proxy +Apache2 configuration to put under the relevant virtual domain: +``` +ProxyPreserveHost on +ProxyPass /_matrix/identity/ http://mxisdInternalIpAddress:8090/_matrix/identity/ +ProxyPass /_matrix/client/r0/user_directory/ http://mxisdInternalIpAddress:8090/_matrix/client/r0/user_directory/ +ProxyPass /_matrix/ http://HomeserverInternalIpAddress:8008/_matrix/ + +``` +`ProxyPreserveHost` or equivalent must be enabled to detect to which Homeserver mxisd should talk to when building +results. + +### Backend +#### LDAP +Configuration structure has been altered so queries are automatically built from a global or specific filter and a list +of attributes. To ensure Directory feature works, here how the LDAP configuration should look like: +``` +ldap: + enabled: false + filter: '(memberOf=CN=Matrix Users,OU=Groups,DC=example,DC=org)' + connection: + host: 'ldapIpOrDomain' + bindDn: 'CN=Matrix Identity Server,OU=Accounts,DC=example,DC=org' + bindPassword: 'mxisd' + baseDn: 'OU=Accounts,DC=example,DC=org' + attribute: + uid: + type: 'uid' + value: 'userPrincipalName' + name: 'displayName' + threepid: + email: + - 'mailPrimaryAddress' + - 'mail' + - 'otherMailbox' + msisdn: + - 'telephoneNumber' + - 'mobile' + - 'homePhone' + - 'otherTelephone' + - 'otherMobile' + - 'otherHomePhone' + directory: + attribute: + other: + - 'employeeNumber' + - 'someOtherAttribute' +``` +Previous configuration entries that contained queries with the `%3pid` placeholder should not be used anymore, unless +specifically overwritten. Instead, add all attributes to the relevant sections. + +If you would like to include an attribute which is not a display name or a 3PID, you can use the +`directory.attribute.other` to list any extra attributes you want included in searches. +If you do not want to include any extra attribute, that configuration section can be skipped. + +#### SQL +If you plan to integrate directory search directly with synapse, use the `synapseSql` provider, based on the following +config: +``` +synapseSql: + enabled: true + type: + connection: `` +``` +`type` and `connection`, including any other configuration item, follow the same values as the regular `sql` backend. + +--- + +For the regular SQL backend, the following configuration items are available: +``` +sql: + directory: + enabled: true + query: + name: + type: 'localpart' + value: 'SELECT idColumn, displayNameColumn FROM table WHERE displayNameColumn LIKE ?' + threepid: + type: 'localpart' + value: 'SELECT idColumn, displayNameColumn FROM table WHERE threepidColumn LIKE ?' +``` +For each query, `type` can be used to tell mxisd how to process the ID column: +- `localpart` will append the `matrix.domain` to it +- `mxid` will use the ID as-is. If it is not a valid Matrix ID, the search will fail. + +`value` is the SQL query and must return two columns: +- The first being the User ID +- The second being its display name + +#### REST +See the [dedicated document](../backends/rest.md) +### DNS Overwrite +Just like you need to configure a reverse proxy to send client requests to mxisd, you also need to configure mxisd with +the internal IP of the Homeserver so it can talk to it directly to integrate its directory search. + +To do so, use the following configuration: ``` dns.overwrite.homeserver.client: - name: 'example.org' value: 'http://localhost:8008' ``` -- Configure reverse proxy - - for `/_matrix/client/r0/user_directory/search` to `http://internalIpOfMxisd:8090/_matrix/client/r0/user_directory/search` - - With `ProxyPreserveHost on` on apache +`name` must be the hostname of the URL that clients use when connecting to the Homeserver. +In case the hostname is the same as your Matrix domain, you can use `${matrix.domain}` to auto-populate the value using +the `matrix.domain` configuration option and avoid duplicating it. + +`value` is the base intenral URL of the Homeserver, without any `/_matrix/..` or trailing `/`. diff --git a/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java index f6f5254..a5fbad9 100644 --- a/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java +++ b/src/main/java/io/kamax/mxisd/backend/ldap/LdapDirectoryProvider.java @@ -40,7 +40,6 @@ import org.springframework.stereotype.Component; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; @Component @@ -73,6 +72,7 @@ public class LdapDirectoryProvider extends LdapGenericBackend implements IDirect attributes.toArray(attArray); String searchQuery = buildOrQueryWithFilter(getCfg().getDirectory().getFilter(), "*" + query + "*", attArray); + log.debug("Query: {}", searchQuery); try (EntryCursor cursor = conn.search(getBaseDn(), searchQuery, SearchScope.SUBTREE, attArray)) { while (cursor.next()) { Entry entry = cursor.get(); @@ -102,13 +102,17 @@ public class LdapDirectoryProvider extends LdapGenericBackend implements IDirect @Override public UserDirectorySearchResult searchByDisplayName(String query) { log.info("Performing LDAP directory search on display name using '{}'", query); - return search(query, Collections.singletonList(getCfg().getAttribute().getName())); + List attributes = new ArrayList<>(); + attributes.add(getAt().getName()); + attributes.addAll(getCfg().getDirectory().getAttribute().getOther()); + return search(query, attributes); } @Override public UserDirectorySearchResult searchBy3pid(String query) { log.info("Performing LDAP directory search on 3PIDs using '{}'", query); List attributes = new ArrayList<>(); + attributes.add(getAt().getName()); getCfg().getAttribute().getThreepid().forEach((k, v) -> attributes.addAll(v)); return search(query, attributes); } diff --git a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java index 01541bf..3211499 100644 --- a/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java +++ b/src/main/java/io/kamax/mxisd/config/ldap/LdapConfig.java @@ -32,6 +32,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; @Configuration @ConfigurationProperties(prefix = "ldap") @@ -45,8 +47,31 @@ public class LdapConfig { public static class Directory { + public static class Attribute { + + private List other = new ArrayList<>(); + + public List getOther() { + return other; + } + + public void setOther(List other) { + this.other = other; + } + + } + + private Attribute attribute = new Attribute(); private String filter; + public Attribute getAttribute() { + return attribute; + } + + public void setAttribute(Attribute attribute) { + this.attribute = attribute; + } + public String getFilter() { return filter; } diff --git a/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java b/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java index ac4ac99..793226a 100644 --- a/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java +++ b/src/main/java/io/kamax/mxisd/config/sql/SqlConfig.java @@ -1,6 +1,6 @@ package io.kamax.mxisd.config.sql; -import com.google.gson.Gson; +import io.kamax.mxisd.util.GsonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -212,8 +212,9 @@ public abstract class SqlConfig { log.info("Type: {}", getType()); log.info("Connection: {}", getConnection()); log.info("Auth enabled: {}", getAuth().isEnabled()); + log.info("Directory queries: {}", GsonUtil.build().toJson(getDirectory().getQuery())); log.info("Identity type: {}", getIdentity().getType()); - log.info("Identity medium queries: {}", new Gson().toJson(getIdentity().getMedium())); + log.info("Identity medium queries: {}", GsonUtil.build().toJson(getIdentity().getMedium())); } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index dbe31f7..5c0abd9 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -69,6 +69,8 @@ ldap: auth: filter: '' directory: + attribute: + other: [] filter: '' identity: filter: '' @@ -89,8 +91,10 @@ sql: enabled: false query: name: + type: 'localpart' value: 'SELECT 1' threepid: + type: 'localpart' value: 'SELECT 1' identity: type: 'mxid' From 88e86cd0d522d24ef1391697369765517854eb1a Mon Sep 17 00:00:00 2001 From: Maxime Dor Date: Sun, 1 Oct 2017 21:09:18 +0200 Subject: [PATCH 14/14] Improve Directory documentation --- docs/features/directory-users.md | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/docs/features/directory-users.md b/docs/features/directory-users.md index a7187f1..327adcd 100644 --- a/docs/features/directory-users.md +++ b/docs/features/directory-users.md @@ -1,4 +1,16 @@ # User Directory +- [Description](#description) +- [Overview](#overview) +- [Requirements](#requirements) +- [Configuration](#configuration) + - [Reverse Proxy](#reverse-proxy) + - [DNS Overwrite](#dns-overwrite) + - [Backends](#backends) + - [LDAP](#ldap) + - [SQL](#sql) + - [REST](#rest) + +## Description This feature allows you to search for existing and/or potential users that are already present in your Identity backend or that already share a room with you on the Homeserver. @@ -11,24 +23,34 @@ With mxisd integration, you can: - Search for users which you are not in contact with yet. Super useful for corporations who want to give Matrix access internally, so users can just find themselves **prior** to having any common room(s) - Use any attribute of your backend to extend the search! +- Include your homeserver search results to those found by mxisd (default behaviour, no configuration required) + +By integrating mxisd, you get the default behaviour with all the extras, ensuring your users will always find each other. ## Overview This is performed by intercepting the Homeserver endpoint `/_matrix/client/r0/user_directory/search` like so: ``` +----------------------------------------------+ -client --> | Reverse proxy Step 2 - | Step 1 +-------------------------+ +Client --> | Reverse proxy Step 2 + | Step 1 +-------------------------+ | /_matrix/client/r0/user_directory/search ----------> | | Search in +---------+ | /\ | mxisd - Identity server | -----------> | Backend | | /_matrix/* \----------------------------- | | all users +---------+ | | Step 4: Send back merged results +-------------------------+ - +--------|------- | + + | | | Step 3 | | | +------------+ Search in known users \--> | Homeserver | <----------------------------------------/ +------------+ /_matrix/client/r0/user_directory/search ``` +Steps: +1. The intercepted request is directly sent to mxisd instead of the Homeserver. +2. Enabled backends are queried for any math on the search value sent by the client. +3. The Homeserver, from which the request was intercepted, is queried using the same request as the client. +Its address is resolved using the DNS Overwrite feature to reach its internal address on a non-encrypted port. +4. Results from backends and the Homeserver are merged together and sent back to the client, believing it was the HS +which directly answered the request. ## Requirements - Reverse proxy setup, which you should already have in place if you use mxisd @@ -38,19 +60,34 @@ client --> | Reverse proxy - REST ## Configuration -### Reverse proxy +### Reverse Proxy Apache2 configuration to put under the relevant virtual domain: ``` ProxyPreserveHost on ProxyPass /_matrix/identity/ http://mxisdInternalIpAddress:8090/_matrix/identity/ ProxyPass /_matrix/client/r0/user_directory/ http://mxisdInternalIpAddress:8090/_matrix/client/r0/user_directory/ ProxyPass /_matrix/ http://HomeserverInternalIpAddress:8008/_matrix/ - ``` `ProxyPreserveHost` or equivalent must be enabled to detect to which Homeserver mxisd should talk to when building results. -### Backend +### DNS Overwrite +Just like you need to configure a reverse proxy to send client requests to mxisd, you also need to configure mxisd with +the internal IP of the Homeserver so it can talk to it directly to integrate its directory search. + +To do so, use the following configuration: +``` +dns.overwrite.homeserver.client: + - name: 'example.org' + value: 'http://localhost:8008' +``` +`name` must be the hostname of the URL that clients use when connecting to the Homeserver. +In case the hostname is the same as your Matrix domain, you can use `${matrix.domain}` to auto-populate the value using +the `matrix.domain` configuration option and avoid duplicating it. + +`value` is the base intenral URL of the Homeserver, without any `/_matrix/..` or trailing `/`. + +### Backends #### LDAP Configuration structure has been altered so queries are automatically built from a global or specific filter and a list of attributes. To ensure Directory feature works, here how the LDAP configuration should look like: @@ -129,18 +166,3 @@ For each query, `type` can be used to tell mxisd how to process the ID column: #### REST See the [dedicated document](../backends/rest.md) -### DNS Overwrite -Just like you need to configure a reverse proxy to send client requests to mxisd, you also need to configure mxisd with -the internal IP of the Homeserver so it can talk to it directly to integrate its directory search. - -To do so, use the following configuration: -``` -dns.overwrite.homeserver.client: - - name: 'example.org' - value: 'http://localhost:8008' -``` -`name` must be the hostname of the URL that clients use when connecting to the Homeserver. -In case the hostname is the same as your Matrix domain, you can use `${matrix.domain}` to auto-populate the value using -the `matrix.domain` configuration option and avoid duplicating it. - -`value` is the base intenral URL of the Homeserver, without any `/_matrix/..` or trailing `/`.