Support new Homeserver federation discovery with well-known (Fix #127)

src/main/java/io/kamax/mxisd/Mxisd.java

@@ -41,6 +41,7 @@ import io.kamax.mxisd.lookup.provider.BridgeFetcher;
 import io.kamax.mxisd.lookup.provider.RemoteIdentityServerFetcher;
 import io.kamax.mxisd.lookup.strategy.LookupStrategy;
 import io.kamax.mxisd.lookup.strategy.RecursivePriorityLookupStrategy;
+import io.kamax.mxisd.matrix.HomeserverFederationResolver;
 import io.kamax.mxisd.matrix.IdentityServerUtils;
 import io.kamax.mxisd.notification.NotificationHandlerSupplier;
 import io.kamax.mxisd.notification.NotificationHandlers;
@@ -99,6 +100,8 @@ public class Mxisd {
                 .setMaxConnTotal(Integer.MAX_VALUE)
                 .build();
 
+        FederationDnsOverwrite fedDns = new FederationDnsOverwrite(cfg.getDns().getOverwrite());
+        HomeserverFederationResolver resolver = new HomeserverFederationResolver(fedDns, httpClient);
         IdentityServerUtils.setHttpClient(httpClient);
         srvFetcher = new RemoteIdentityServerFetcher(httpClient);
 
@@ -106,10 +109,9 @@ public class Mxisd {
         keyMgr = CryptoFactory.getKeyManager(cfg.getKey());
         signMgr = CryptoFactory.getSignatureManager(keyMgr);
         clientDns = new ClientDnsOverwrite(cfg.getDns().getOverwrite());
-        FederationDnsOverwrite fedDns = new FederationDnsOverwrite(cfg.getDns().getOverwrite());
+
         synapse = new Synapse(cfg.getSynapseSql());
         BridgeFetcher bridgeFetcher = new BridgeFetcher(cfg.getLookup().getRecursive().getBridge(), srvFetcher);
-
         ServiceLoader.load(IdentityStoreSupplier.class).iterator().forEachRemaining(p -> p.accept(this));
         ServiceLoader.load(NotificationHandlerSupplier.class).iterator().forEachRemaining(p -> p.accept(this));
 
@@ -117,7 +119,7 @@ public class Mxisd {
         pMgr = new ProfileManager(ProfileProviders.get(), clientDns, httpClient);
         notifMgr = new NotificationManager(cfg.getNotification(), NotificationHandlers.get());
         sessMgr = new SessionManager(cfg.getSession(), cfg.getMatrix(), store, notifMgr, idStrategy, httpClient);
-        invMgr = new InvitationManager(cfg, store, idStrategy, keyMgr, signMgr, fedDns, notifMgr, pMgr);
+        invMgr = new InvitationManager(cfg, store, idStrategy, keyMgr, signMgr, resolver, notifMgr, pMgr);
         authMgr = new AuthManager(cfg, AuthProviders.get(), idStrategy, invMgr, clientDns, httpClient);
         dirMgr = new DirectoryManager(cfg.getDirectory(), clientDns, httpClient, DirectoryProviders.get());
         regMgr = new RegistrationManager(cfg.getRegister(), httpClient, clientDns, invMgr);

src/main/java/io/kamax/mxisd/invitation/InvitationManager.java

@@ -30,7 +30,6 @@ import io.kamax.mxisd.config.InvitationConfig;
 import io.kamax.mxisd.config.MxisdConfig;
 import io.kamax.mxisd.config.ServerConfig;
 import io.kamax.mxisd.crypto.*;
-import io.kamax.mxisd.dns.FederationDnsOverwrite;
 import io.kamax.mxisd.exception.BadRequestException;
 import io.kamax.mxisd.exception.ConfigurationException;
 import io.kamax.mxisd.exception.MappingAlreadyExistsException;
@@ -38,6 +37,7 @@ import io.kamax.mxisd.exception.ObjectNotFoundException;
 import io.kamax.mxisd.lookup.SingleLookupReply;
 import io.kamax.mxisd.lookup.ThreePidMapping;
 import io.kamax.mxisd.lookup.strategy.LookupStrategy;
+import io.kamax.mxisd.matrix.HomeserverFederationResolver;
 import io.kamax.mxisd.notification.NotificationManager;
 import io.kamax.mxisd.profile.ProfileManager;
 import io.kamax.mxisd.storage.IStorage;
@@ -57,13 +57,10 @@ import org.apache.http.impl.client.HttpClients;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.xbill.DNS.*;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.DateTimeException;
 import java.time.Instant;
@@ -85,7 +82,7 @@ public class InvitationManager {
     private LookupStrategy lookupMgr;
     private KeyManager keyMgr;
     private SignatureManager signMgr;
-    private FederationDnsOverwrite dns;
+    private HomeserverFederationResolver resolver;
     private NotificationManager notifMgr;
     private ProfileManager profileMgr;
 
@@ -100,7 +97,7 @@ public class InvitationManager {
             LookupStrategy lookupMgr,
             KeyManager keyMgr,
             SignatureManager signMgr,
-            FederationDnsOverwrite dns,
+            HomeserverFederationResolver resolver,
             NotificationManager notifMgr,
             ProfileManager profileMgr
     ) {
@@ -110,7 +107,7 @@ public class InvitationManager {
         this.lookupMgr = lookupMgr;
         this.keyMgr = keyMgr;
         this.signMgr = signMgr;
-        this.dns = dns;
+        this.resolver = resolver;
         this.notifMgr = notifMgr;
         this.profileMgr = profileMgr;
 
@@ -207,56 +204,6 @@ public class InvitationManager {
         return reply.getInvite().getSender().getId() + ":" + reply.getInvite().getRoomId() + ":" + reply.getInvite().getMedium() + ":" + reply.getInvite().getAddress();
     }
 
-    private String getSrvRecordName(String domain) {
-        return "_matrix._tcp." + domain;
-    }
-
-    // TODO use caching mechanism
-    // TODO export in matrix-java-sdk
-    private String findHomeserverForDomain(String domain) {
-        Optional<String> entryOpt = dns.findHost(domain);
-        if (entryOpt.isPresent()) {
-            String entry = entryOpt.get();
-            log.info("Found DNS overwrite for {} to {}", domain, entry);
-            try {
-                return new URL(entry).toString();
-            } catch (MalformedURLException e) {
-                log.warn("Skipping homeserver Federation DNS overwrite for {} - not a valid URL: {}", domain, entry);
-            }
-        }
-
-        log.debug("Performing SRV lookup for {}", domain);
-        String lookupDns = getSrvRecordName(domain);
-        log.info("Lookup name: {}", lookupDns);
-
-        try {
-            List<SRVRecord> srvRecords = new ArrayList<>();
-            Record[] rawRecords = new Lookup(lookupDns, Type.SRV).run();
-            if (rawRecords != null && rawRecords.length > 0) {
-                for (Record record : rawRecords) {
-                    if (Type.SRV == record.getType()) {
-                        srvRecords.add((SRVRecord) record);
-                    } else {
-                        log.info("Got non-SRV record: {}", record.toString());
-                    }
-                }
-
-                srvRecords.sort(Comparator.comparingInt(SRVRecord::getPriority));
-                for (SRVRecord record : srvRecords) {
-                    log.info("Found SRV record: {}", record.toString());
-                    return "https://" + record.getTarget().toString(true) + ":" + record.getPort();
-                }
-            } else {
-                log.info("No SRV record for {}", lookupDns);
-            }
-        } catch (TextParseException e) {
-            log.warn("Unable to perform DNS SRV query for {}: {}", lookupDns, e.getMessage());
-        }
-
-        log.info("Performing basic lookup using domain name {}", domain);
-        return "https://" + domain + ":8448";
-    }
-
     private Optional<SingleLookupReply> lookup3pid(String medium, String address) {
         if (!cfg.getResolution().isRecursive()) {
             log.warn("/!\\ /!\\ --- RECURSIVE INVITE RESOLUTION HAS BEEN DISABLED --- /!\\ /!\\");
@@ -413,7 +360,7 @@ public class InvitationManager {
                     continue;
                 }
 
-                log.info("Invite {} has expired at TS {} - Expiring and resolving to {}", targetTs, targetMxid);
+                log.info("Invite {} has expired at TS {} - Expiring and resolving to {}", reply.getId(), targetTs, targetMxid);
                 publishMapping(reply, targetMxid);
             } catch (NumberFormatException | DateTimeException e) {
                 log.warn("Invite {} has an invalid creation TS, setting to default value of {}", reply.getId(), defaultCreateTs);
@@ -475,7 +422,7 @@ public class InvitationManager {
         String address = reply.getInvite().getAddress();
         String domain = reply.getInvite().getSender().getDomain();
         log.info("Discovering HS for domain {}", domain);
-        String hsUrlOpt = findHomeserverForDomain(domain);
+        String hsUrlOpt = resolver.resolve(domain).toString();
 
         // TODO this is needed as this will block if called during authentication cycle due to synapse implementation
         new Thread(() -> { // FIXME need to make this retry-able and within a general background working pool

src/main/java/io/kamax/mxisd/matrix/HomeserverFederationResolver.java

@@ -0,0 +1,215 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2019 Kamax Sarl
+ *
+ * https://www.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.matrix;
+
+import com.google.gson.JsonObject;
+import io.kamax.matrix.json.GsonUtil;
+import io.kamax.matrix.json.InvalidJsonException;
+import io.kamax.mxisd.dns.FederationDnsOverwrite;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xbill.DNS.*;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.*;
+
+public class HomeserverFederationResolver {
+
+    private static final Logger log = LoggerFactory.getLogger(HomeserverFederationResolver.class);
+
+    private FederationDnsOverwrite dns;
+    private CloseableHttpClient client;
+
+    public HomeserverFederationResolver(FederationDnsOverwrite dns, CloseableHttpClient client) {
+        this.dns = dns;
+        this.client = client;
+    }
+
+    private String getDefaultScheme() {
+        return "https";
+    }
+
+    private int getDefaultPort() {
+        return 8448;
+    }
+
+    private String getDnsSrvPrefix() {
+        return "_matrix._tcp.";
+    }
+
+    private String buildSrvRecordName(String domain) {
+        return getDnsSrvPrefix() + domain;
+    }
+
+    private Optional<URL> resolveOverwrite(String domain) {
+        Optional<String> entryOpt = dns.findHost(domain);
+        if (!entryOpt.isPresent()) {
+            log.info("No DNS overwrite for {}", domain);
+            return Optional.empty();
+        }
+
+        try {
+            return Optional.of(new URL(entryOpt.get()));
+        } catch (MalformedURLException e) {
+            log.warn("Skipping homeserver Federation DNS overwrite for {} - not a valid URL: {}", domain, entryOpt.get());
+            return Optional.empty();
+        }
+    }
+
+    private Optional<URL> resolveLiteral(String domain) {
+        if (domain.contains("[") && domain.contains("]")) {
+            // This is an IPv6
+            if (domain.contains("]:")) {
+                // With a custom port, we return as is
+                return Optional.of(build(domain));
+            } else {
+                return Optional.of(build(domain + ":" + getDefaultPort()));
+            }
+        }
+
+        if (domain.contains(":")) {
+            // This is a domain or IPv4 with an explicit port, we return as is
+            return Optional.of(build(domain));
+        }
+
+        // At this point, we do not account for the provided string to be an IPv4 without a port. We will therefore
+        // perform well-known lookup and SRV record. While this is not needed, we don't expect the SRV to return anything
+        // and the well-known shouldn't either, but it might, leading to a wrong destination potentially.
+        //
+        // We accept this risk as mxisd is not meant to be used without DNS domain as per FAQ. We also provide resolution
+        // override facilities. Therefore, we accept to not handle this case until we get report of such unwanted behaviour
+        // that still fix mxisd use case and can't be resolved via override.
+
+        return Optional.empty();
+    }
+
+    private Optional<URL> resolveWellKnown(String domain) {
+        log.debug("Performing Well-known lookup for {}", domain);
+        HttpGet wnReq = new HttpGet("https://" + domain + "/.well-known/matrix/server");
+        try (CloseableHttpResponse wnRes = client.execute(wnReq)) {
+            int status = wnRes.getStatusLine().getStatusCode();
+            if (status == 200) {
+                try {
+                    JsonObject body = GsonUtil.parseObj(EntityUtils.toString(wnRes.getEntity()));
+                    String server = GsonUtil.getStringOrNull(body, "m.server");
+                    if (StringUtils.isNotBlank(server)) {
+                        log.debug("Found well-known entry: {}", server);
+                        return Optional.of(build(server));
+                    }
+                } catch (InvalidJsonException e) {
+                    log.info("Could not parse well-known resource: {}", e.getMessage());
+                }
+            } else {
+                log.info("Well-known did not return status code 200 but {}, ignoring", status);
+            }
+
+            return Optional.empty();
+        } catch (IOException e) {
+            throw new RuntimeException("Error while trying to lookup well-known for " + domain, e);
+        }
+    }
+
+    private Optional<URL> resolveDnsSrv(String domain) {
+        log.debug("Performing SRV lookup for {}", domain);
+        String lookupDns = buildSrvRecordName(domain);
+        log.debug("Lookup name: {}", lookupDns);
+
+        try {
+            List<SRVRecord> srvRecords = new ArrayList<>();
+            Record[] rawRecords = new Lookup(lookupDns, Type.SRV).run();
+            if (Objects.isNull(rawRecords) || rawRecords.length == 0) {
+                log.debug("No SRV record for {}", domain);
+                return Optional.empty();
+            }
+
+            for (Record record : rawRecords) {
+                if (Type.SRV == record.getType()) {
+                    srvRecords.add((SRVRecord) record);
+                } else {
+                    log.debug("Ignoring non-SRV record: {}", record.toString());
+                }
+            }
+
+            if (srvRecords.size() < 1) {
+                log.warn("DNS SRV records were found for {} but none is usable", lookupDns);
+                return Optional.empty();
+            }
+
+            srvRecords.sort(Comparator.comparingInt(SRVRecord::getPriority));
+            SRVRecord record = srvRecords.get(0);
+            return Optional.of(build(record.getTarget().toString(true) + ":" + record.getPort()));
+        } catch (TextParseException e) {
+            log.warn("Unable to perform DNS SRV query for {}: {}", lookupDns, e.getMessage());
+        }
+
+        return Optional.empty();
+    }
+
+    public URL build(String authority) {
+        try {
+            return new URL(getDefaultScheme() + "://" + authority);
+        } catch (MalformedURLException e) {
+            throw new RuntimeException("Could not build URL for " + authority, e);
+        }
+    }
+
+    public URL resolve(String domain) {
+        Optional<URL> s1 = resolveOverwrite(domain);
+        if (s1.isPresent()) {
+            URL dest = s1.get();
+            log.info("Resolution of {} via DNS overwrite to {}", domain, dest);
+            return dest;
+        }
+
+        Optional<URL> s2 = resolveLiteral(domain);
+        if (s2.isPresent()) {
+            URL dest = s2.get();
+            log.info("Resolution of {} as IP literal or IP/hostname with explicit port to {}", domain, dest);
+            return dest;
+        }
+
+        Optional<URL> s3 = resolveWellKnown(domain);
+        if (s3.isPresent()) {
+            URL dest = s3.get();
+            log.info("Resolution of {} via well-known to {}", domain, dest);
+            return dest;
+        }
+        // The domain needs to be resolved
+
+        Optional<URL> s4 = resolveDnsSrv(domain);
+        if (s4.isPresent()) {
+            URL dest = s4.get();
+            log.info("Resolution of {} via DNS SRV record to {}", domain, dest);
+        }
+
+        URL dest = build(domain + ":" + getDefaultPort());
+        log.info("Resolution of {} to {}", domain, dest);
+        return dest;
+    }
+
+}

src/test/java/io/kamax/mxisd/test/matrix/HomeserverFederationResolverTest.java

@@ -0,0 +1,64 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2019 Kamax Sarl
+ *
+ * https://www.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.test.matrix;
+
+import io.kamax.mxisd.Mxisd;
+import io.kamax.mxisd.config.MxisdConfig;
+import io.kamax.mxisd.dns.FederationDnsOverwrite;
+import io.kamax.mxisd.matrix.HomeserverFederationResolver;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.net.URL;
+
+import static org.junit.Assert.assertEquals;
+
+public class HomeserverFederationResolverTest {
+
+    private static HomeserverFederationResolver resolver;
+
+    @BeforeClass
+    public static void beforeClass() {
+        CloseableHttpClient client = HttpClients.custom()
+                .setUserAgent(Mxisd.Agent)
+                .setMaxConnPerRoute(Integer.MAX_VALUE)
+                .setMaxConnTotal(Integer.MAX_VALUE)
+                .build();
+
+        FederationDnsOverwrite fedDns = new FederationDnsOverwrite(new MxisdConfig().getDns().getOverwrite());
+        resolver = new HomeserverFederationResolver(fedDns, client);
+    }
+
+    @Test
+    public void hostnameWithoutPort() {
+        URL url = resolver.resolve("example.org");
+        assertEquals("https://example.org:8448", url.toString());
+    }
+
+    @Test
+    public void hostnameWithPort() {
+        URL url = resolver.resolve("example.org:443");
+        assertEquals("https://example.org:443", url.toString());
+    }
+
+}