Bye bye Groovy, you won't be missed :(

src/main/java/io/kamax/mxisd/lookup/ALookupRequest.java

@@ -0,0 +1,73 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import java.util.List;
+
+public abstract class ALookupRequest {
+
+    private String id;
+    private String requester;
+    private String userAgent;
+    private boolean isRecursive;
+    private List<String> recurseHosts;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getRequester() {
+        return requester;
+    }
+
+    public void setRequester(String requester) {
+        this.requester = requester;
+    }
+
+    public String getUserAgent() {
+        return userAgent;
+    }
+
+    public void setUserAgent(String userAgent) {
+        this.userAgent = userAgent;
+    }
+
+    public boolean isRecursive() {
+        return isRecursive;
+    }
+
+    public void setRecursive(boolean recursive) {
+        isRecursive = recursive;
+    }
+
+    public List<String> getRecurseHosts() {
+        return recurseHosts;
+    }
+
+    public void setRecurseHosts(List<String> recurseHosts) {
+        this.recurseHosts = recurseHosts;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/BulkLookupRequest.java

@@ -0,0 +1,37 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import java.util.List;
+
+public class BulkLookupRequest extends ALookupRequest {
+
+    private List<ThreePidMapping> mappings;
+
+    public List<ThreePidMapping> getMappings() {
+        return mappings;
+    }
+
+    public void setMappings(List<ThreePidMapping> mappings) {
+        this.mappings = mappings;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/SingleLookupReply.java

@@ -0,0 +1,116 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import io.kamax.matrix.MatrixID;
+import io.kamax.matrix._MatrixID;
+import io.kamax.mxisd.controller.v1.io.SingeLookupReplyJson;
+
+import java.time.Instant;
+
+public class SingleLookupReply {
+
+    private static Gson gson = new Gson();
+
+    private boolean isRecursive;
+    private boolean isSigned;
+    private String body;
+    private SingleLookupRequest request;
+    private _MatrixID mxid;
+    private Instant notBefore;
+    private Instant notAfter;
+    private Instant timestamp;
+
+    public static SingleLookupReply fromRecursive(SingleLookupRequest request, String body) {
+        SingleLookupReply reply = new SingleLookupReply();
+        reply.isRecursive = true;
+        reply.request = request;
+        reply.body = body;
+
+        try {
+            SingeLookupReplyJson json = gson.fromJson(body, SingeLookupReplyJson.class);
+            reply.mxid = new MatrixID(json.getMxid());
+            reply.notAfter = Instant.ofEpochMilli(json.getNot_after());
+            reply.notBefore = Instant.ofEpochMilli(json.getNot_before());
+            reply.timestamp = Instant.ofEpochMilli(json.getTs());
+            reply.isSigned = json.isSigned();
+        } catch (JsonSyntaxException e) {
+            // stub - we only want to try, nothing more
+        }
+
+        return reply;
+    }
+
+    private SingleLookupReply() {
+        // stub
+    }
+
+    public SingleLookupReply(SingleLookupRequest request, String mxid) {
+        this(request, new MatrixID(mxid));
+    }
+
+    public SingleLookupReply(SingleLookupRequest request, _MatrixID mxid) {
+        this(request, mxid, Instant.now(), Instant.ofEpochMilli(0), Instant.ofEpochMilli(253402300799000L));
+    }
+
+    public SingleLookupReply(SingleLookupRequest request, _MatrixID mxid, Instant timestamp, Instant notBefore, Instant notAfter) {
+        this.request = request;
+        this.mxid = mxid;
+        this.timestamp = timestamp;
+        this.notBefore = notBefore;
+        this.notAfter = notAfter;
+    }
+
+    public boolean isRecursive() {
+        return isRecursive;
+    }
+
+    public boolean isSigned() {
+        return isSigned;
+    }
+
+    public String getBody() {
+        return body;
+    }
+
+    public SingleLookupRequest getRequest() {
+        return request;
+    }
+
+    public _MatrixID getMxid() {
+        return mxid;
+    }
+
+    public Instant getNotBefore() {
+        return notBefore;
+    }
+
+    public Instant getNotAfter() {
+        return notAfter;
+    }
+
+    public Instant getTimestamp() {
+        return timestamp;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/SingleLookupRequest.java

@@ -0,0 +1,44 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+public class SingleLookupRequest extends ALookupRequest {
+
+    private String type;
+    private String threePid;
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public String getThreePid() {
+        return threePid;
+    }
+
+    public void setThreePid(String threePid) {
+        this.threePid = threePid;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/ThreePidMapping.java

@@ -0,0 +1,95 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import com.google.gson.Gson;
+import io.kamax.mxisd.ThreePid;
+
+public class ThreePidMapping {
+
+    private static Gson gson = new Gson();
+
+    private String medium;
+    private String value;
+    private String mxid;
+
+    public ThreePidMapping() {
+        // stub
+    }
+
+    public ThreePidMapping(ThreePid threePid, String mxid) {
+        this(threePid.getMedium(), threePid.getAddress(), mxid);
+    }
+
+    public ThreePidMapping(String medium, String value, String mxid) {
+        setMedium(medium);
+        setValue(value);
+        setMxid(mxid);
+    }
+
+    public String getMedium() {
+        return medium;
+    }
+
+    public void setMedium(String medium) {
+        this.medium = medium;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+
+    public String getMxid() {
+        return mxid;
+    }
+
+    public void setMxid(String mxid) {
+        this.mxid = mxid;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ThreePidMapping that = (ThreePidMapping) o;
+
+        if (medium != null ? !medium.equals(that.medium) : that.medium != null) return false;
+        return value != null ? value.equals(that.value) : that.value == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = medium != null ? medium.hashCode() : 0;
+        result = 31 * result + (value != null ? value.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return gson.toJson(this);
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/ThreePidValidation.java

@@ -0,0 +1,40 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup;
+
+import io.kamax.mxisd.ThreePid;
+
+import java.time.Instant;
+
+public class ThreePidValidation extends ThreePid {
+
+    private Instant validation;
+
+    public ThreePidValidation(ThreePid tpid, Instant validation) {
+        super(tpid);
+        this.validation = validation;
+    }
+
+    public Instant getValidation() {
+        return validation;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/fetcher/IBridgeFetcher.java

@@ -0,0 +1,36 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.fetcher;
+
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface IBridgeFetcher {
+
+    Optional<SingleLookupReply> find(SingleLookupRequest request);
+
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings);
+
+}

src/main/java/io/kamax/mxisd/lookup/fetcher/IRemoteIdentityServerFetcher.java

@@ -0,0 +1,38 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.fetcher;
+
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface IRemoteIdentityServerFetcher {
+
+    boolean isUsable(String remote);
+
+    Optional<SingleLookupReply> find(String remote, SingleLookupRequest request);
+
+    List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings);
+
+}

src/main/java/io/kamax/mxisd/lookup/provider/BridgeFetcher.java

@@ -0,0 +1,74 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import io.kamax.mxisd.config.RecursiveLookupBridgeConfig;
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.fetcher.IBridgeFetcher;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+@Component
+public class BridgeFetcher implements IBridgeFetcher {
+
+    private Logger log = LoggerFactory.getLogger(BridgeFetcher.class);
+
+    @Autowired
+    private RecursiveLookupBridgeConfig cfg;
+
+    @Autowired
+    private RemoteIdentityServerFetcher fetcher;
+
+    @Override
+    public Optional<SingleLookupReply> find(SingleLookupRequest request) {
+        Optional<String> mediumUrl = Optional.ofNullable(cfg.getMappings().get(request.getType()));
+        if (mediumUrl.isPresent() && !StringUtils.isBlank(mediumUrl.get())) {
+            log.info("Using specific medium bridge lookup URL {}", mediumUrl.get());
+
+            return fetcher.find(mediumUrl.get(), request);
+        } else if (!StringUtils.isBlank(cfg.getServer())) {
+            log.info("Using generic bridge lookup URL {}", cfg.getServer());
+
+            return fetcher.find(cfg.getServer(), request);
+        } else {
+            log.info("No bridge lookup URL found/configured, skipping");
+
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        log.warn("Bulk lookup on bridge lookup requested, but not supported - returning empty list");
+
+        return Collections.emptyList();
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/provider/DnsLookupProvider.java

@@ -0,0 +1,181 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import io.kamax.mxisd.config.MatrixConfig;
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher;
+import io.kamax.mxisd.matrix.IdentityServerUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RecursiveTask;
+
+@Component
+class DnsLookupProvider implements IThreePidProvider {
+
+    private Logger log = LoggerFactory.getLogger(DnsLookupProvider.class);
+
+    @Autowired
+    private MatrixConfig mxCfg;
+
+    @Autowired
+    private IRemoteIdentityServerFetcher fetcher;
+
+    @Override
+    public boolean isEnabled() {
+        return true;
+    }
+
+    @Override
+    public boolean isLocal() {
+        return false;
+    }
+
+    @Override
+    public int getPriority() {
+        return 10;
+    }
+
+    private Optional<String> getDomain(String email) {
+        int atIndex = email.lastIndexOf("@");
+        if (atIndex == -1) {
+            return Optional.empty();
+        }
+
+        return Optional.of(email.substring(atIndex + 1));
+    }
+
+    // TODO use caching mechanism
+    private Optional<String> findIdentityServerForDomain(String domain) {
+        if (StringUtils.equals(mxCfg.getDomain(), domain)) {
+            log.info("We are authoritative for {}, no remote lookup", domain);
+            return Optional.empty();
+        }
+
+        return IdentityServerUtils.findIsUrlForDomain(domain);
+    }
+
+    @Override
+    public Optional<SingleLookupReply> find(SingleLookupRequest request) {
+        if (!StringUtils.equals("email", request.getType())) { // TODO use enum
+            log.info("Skipping unsupported type {} for {}", request.getType(), request.getThreePid());
+            return Optional.empty();
+        }
+
+        log.info("Performing DNS lookup for {}", request.getThreePid());
+
+        String domain = request.getThreePid().substring(request.getThreePid().lastIndexOf("@") + 1);
+        log.info("Domain name for {}: {}", request.getThreePid(), domain);
+        Optional<String> baseUrl = findIdentityServerForDomain(domain);
+
+        if (baseUrl.isPresent()) {
+            return fetcher.find(baseUrl.get(), request);
+        }
+
+        return Optional.empty();
+    }
+
+    @Override
+    public List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        Map<String, List<ThreePidMapping>> domains = new HashMap<>();
+
+        for (ThreePidMapping mapping : mappings) {
+            if (!StringUtils.equals("email", mapping.getMedium())) {
+                log.info("Skipping unsupported type {} for {}", mapping.getMedium(), mapping.getValue());
+                continue;
+            }
+
+            Optional<String> domainOpt = getDomain(mapping.getValue());
+            if (!domainOpt.isPresent()) {
+                log.warn("No domain for 3PID {}", mapping.getValue());
+                continue;
+            }
+
+            String domain = domainOpt.get();
+            List<ThreePidMapping> domainMappings = domains.computeIfAbsent(domain, s -> new ArrayList<>());
+            domainMappings.add(mapping);
+        }
+
+        log.info("Looking mappings across {} domains", domains.keySet().size());
+        ForkJoinPool pool = ForkJoinPool.commonPool();
+        RecursiveTask<List<ThreePidMapping>> task = new RecursiveTask<List<ThreePidMapping>>() {
+
+            @Override
+            protected List<ThreePidMapping> compute() {
+                List<ThreePidMapping> mappingsFound = new ArrayList<>();
+                List<DomainBulkLookupTask> tasks = new ArrayList<>();
+
+                for (String domain : domains.keySet()) {
+                    DomainBulkLookupTask domainTask = new DomainBulkLookupTask(domain, domains.get(domain));
+                    domainTask.fork();
+                    tasks.add(domainTask);
+                }
+
+                for (DomainBulkLookupTask task : tasks) {
+                    mappingsFound.addAll(task.join());
+                }
+
+                return mappingsFound;
+            }
+        };
+        pool.submit(task);
+        pool.shutdown();
+
+        List<ThreePidMapping> mappingsFound = task.join();
+        log.info("Found {} mappings overall", mappingsFound.size());
+        return mappingsFound;
+    }
+
+    private class DomainBulkLookupTask extends RecursiveTask<List<ThreePidMapping>> {
+
+        private String domain;
+        private List<ThreePidMapping> mappings;
+
+        DomainBulkLookupTask(String domain, List<ThreePidMapping> mappings) {
+            this.domain = domain;
+            this.mappings = mappings;
+        }
+
+        @Override
+        protected List<ThreePidMapping> compute() {
+            List<ThreePidMapping> domainMappings = new ArrayList<>();
+
+            Optional<String> baseUrl = findIdentityServerForDomain(domain);
+            if (!baseUrl.isPresent()) {
+                log.info("No usable Identity server for domain {}", domain);
+            } else {
+                domainMappings.addAll(fetcher.find(baseUrl.get(), mappings));
+                log.info("Found {} mappings in domain {}", domainMappings.size(), domain);
+            }
+
+            return domainMappings;
+        }
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/provider/ForwarderProvider.java

@@ -0,0 +1,92 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import io.kamax.mxisd.config.ForwardConfig;
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@Component
+class ForwarderProvider implements IThreePidProvider {
+
+    private Logger log = LoggerFactory.getLogger(ForwarderProvider.class);
+
+    @Autowired
+    private ForwardConfig cfg;
+
+    @Autowired
+    private IRemoteIdentityServerFetcher fetcher;
+
+    @Override
+    public boolean isEnabled() {
+        return true;
+    }
+
+    @Override
+    public boolean isLocal() {
+        return false;
+    }
+
+    @Override
+    public int getPriority() {
+        return 0;
+    }
+
+    @Override
+    public Optional<SingleLookupReply> find(SingleLookupRequest request) {
+        for (String root : cfg.getServers()) {
+            Optional<SingleLookupReply> answer = fetcher.find(root, request);
+            if (answer.isPresent()) {
+                return answer;
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    @Override
+    public List<ThreePidMapping> populate(List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsToDo = new ArrayList<>(mappings);
+        List<ThreePidMapping> mappingsFoundGlobal = new ArrayList<>();
+
+        for (String root : cfg.getServers()) {
+            log.info("{} mappings remaining: {}", mappingsToDo.size(), mappingsToDo);
+            log.info("Querying {}", root);
+            List<ThreePidMapping> mappingsFound = fetcher.find(root, mappingsToDo);
+            log.info("{} returned {} mappings", root, mappingsFound.size());
+            mappingsFoundGlobal.addAll(mappingsFound);
+            mappingsToDo.removeAll(mappingsFound);
+        }
+
+        return mappingsFoundGlobal;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/provider/IThreePidProvider.java

@@ -0,0 +1,45 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface IThreePidProvider {
+
+    boolean isEnabled();
+
+    boolean isLocal();
+
+    /**
+     * Higher has more priority
+     */
+    int getPriority(); // Should not be here but let's KISS for now
+
+    Optional<SingleLookupReply> find(SingleLookupRequest request);
+
+    List<ThreePidMapping> populate(List<ThreePidMapping> mappings);
+
+}

src/main/java/io/kamax/mxisd/lookup/provider/RemoteIdentityServerFetcher.java

@@ -0,0 +1,128 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.provider;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import io.kamax.mxisd.controller.v1.ClientBulkLookupRequest;
+import io.kamax.mxisd.exception.InvalidResponseJsonException;
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.fetcher.IRemoteIdentityServerFetcher;
+import io.kamax.mxisd.matrix.IdentityServerUtils;
+import io.kamax.mxisd.util.GsonParser;
+import io.kamax.mxisd.util.RestClientUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@Component
+@Scope("prototype")
+@Lazy
+public class RemoteIdentityServerFetcher implements IRemoteIdentityServerFetcher {
+
+    private Logger log = LoggerFactory.getLogger(RemoteIdentityServerFetcher.class);
+
+    private Gson gson = new Gson();
+    private GsonParser parser = new GsonParser(gson);
+
+    @Override
+    public boolean isUsable(String remote) {
+        return IdentityServerUtils.isUsable(remote);
+    }
+
+    @Override
+    public Optional<SingleLookupReply> find(String remote, SingleLookupRequest request) {
+        log.info("Looking up {} 3PID {} using {}", request.getType(), request.getThreePid(), remote);
+
+        try {
+            HttpURLConnection rootSrvConn = (HttpURLConnection) new URL(
+                    remote + "/_matrix/identity/api/v1/lookup?medium=" + request.getType() + "&address=" + request.getThreePid()
+            ).openConnection();
+            JsonObject obj = parser.parse(rootSrvConn.getInputStream());
+            if (obj.has("address")) {
+                log.info("Found 3PID mapping: {}", gson.toJson(obj));
+
+                return Optional.of(SingleLookupReply.fromRecursive(request, gson.toJson(obj)));
+            }
+
+            log.info("Empty 3PID mapping from {}", remote);
+            return Optional.empty();
+        } catch (IOException e) {
+            log.warn("Error looking up 3PID mapping {}: {}", request.getThreePid(), e.getMessage());
+            return Optional.empty();
+        } catch (JsonParseException e) {
+            log.warn("Invalid JSON answer from {}", remote);
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public List<ThreePidMapping> find(String remote, List<ThreePidMapping> mappings) {
+        List<ThreePidMapping> mappingsFound = new ArrayList<>();
+
+        ClientBulkLookupRequest mappingRequest = new ClientBulkLookupRequest();
+        mappingRequest.setMappings(mappings);
+
+        String url = remote + "/_matrix/identity/api/v1/bulk_lookup";
+        CloseableHttpClient client = HttpClients.createDefault();
+        try {
+            HttpPost request = RestClientUtils.post(url, mappingRequest);
+            try (CloseableHttpResponse response = client.execute(request)) {
+                if (response.getStatusLine().getStatusCode() != 200) {
+                    log.info("Could not perform lookup at {} due to HTTP return code: {}", url, response.getStatusLine().getStatusCode());
+                    return mappingsFound;
+                }
+
+                ClientBulkLookupRequest input = parser.parse(response, ClientBulkLookupRequest.class);
+                for (List<String> mappingRaw : input.getThreepids()) {
+                    ThreePidMapping mapping = new ThreePidMapping();
+                    mapping.setMedium(mappingRaw.get(0));
+                    mapping.setValue(mappingRaw.get(1));
+                    mapping.setMxid(mappingRaw.get(2));
+                    mappingsFound.add(mapping);
+                }
+            }
+        } catch (IOException e) {
+            log.warn("Unable to fetch remote lookup data: {}", e.getMessage());
+        } catch (InvalidResponseJsonException e) {
+            log.info("HTTP response from {} was empty/invalid", remote);
+        }
+
+        return mappingsFound;
+    }
+
+}

src/main/java/io/kamax/mxisd/lookup/strategy/LookupStrategy.java

@@ -0,0 +1,48 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.strategy;
+
+import io.kamax.mxisd.lookup.BulkLookupRequest;
+import io.kamax.mxisd.lookup.SingleLookupReply;
+import io.kamax.mxisd.lookup.SingleLookupRequest;
+import io.kamax.mxisd.lookup.ThreePidMapping;
+import io.kamax.mxisd.lookup.provider.IThreePidProvider;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface LookupStrategy {
+
+    List<IThreePidProvider> getLocalProviders();
+
+    Optional<SingleLookupReply> find(String medium, String address, boolean recursive);
+
+    Optional<SingleLookupReply> findLocal(String medium, String address);
+
+    Optional<SingleLookupReply> findRemote(String medium, String address);
+
+    Optional<SingleLookupReply> find(SingleLookupRequest request);
+
+    Optional<SingleLookupReply> findRecursive(SingleLookupRequest request);
+
+    List<ThreePidMapping> find(BulkLookupRequest requests);
+
+}

src/main/java/io/kamax/mxisd/lookup/strategy/RecursivePriorityLookupStrategy.java

@@ -0,0 +1,206 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.lookup.strategy;
+
+import edazdarevic.commons.net.CIDRUtils;
+import io.kamax.mxisd.config.RecursiveLookupConfig;
+import io.kamax.mxisd.exception.ConfigurationException;
+import io.kamax.mxisd.lookup.*;
+import io.kamax.mxisd.lookup.fetcher.IBridgeFetcher;
+import io.kamax.mxisd.lookup.provider.IThreePidProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Component
+public class RecursivePriorityLookupStrategy implements LookupStrategy {
+
+    private Logger log = LoggerFactory.getLogger(RecursivePriorityLookupStrategy.class);
+
+    @Autowired
+    private RecursiveLookupConfig recursiveCfg;
+
+    @Autowired
+    private List<IThreePidProvider> providers;
+
+    @Autowired
+    private IBridgeFetcher bridge;
+
+    private List<CIDRUtils> allowedCidr = new ArrayList<>();
+
+    @PostConstruct
+    private void build() throws UnknownHostException {
+        try {
+            log.info("Found {} providers", providers.size());
+
+            providers.sort((o1, o2) -> Integer.compare(o2.getPriority(), o1.getPriority()));
+
+            log.info("Recursive lookup enabled: {}", recursiveCfg.isEnabled());
+            for (String cidr : recursiveCfg.getAllowedCidr()) {
+                log.info("{} is allowed for recursion", cidr);
+                allowedCidr.add(new CIDRUtils(cidr));
+            }
+        } catch (UnknownHostException e) {
+            throw new ConfigurationException("lookup.recursive.allowedCidrs", "Allowed CIDRs");
+        }
+    }
+
+    private boolean isAllowedForRecursive(String source) {
+        boolean canRecurse = false;
+
+        try {
+            if (recursiveCfg.isEnabled()) {
+                log.debug("Checking {} CIDRs for recursion", allowedCidr.size());
+                for (CIDRUtils cidr : allowedCidr) {
+                    if (cidr.isInRange(source)) {
+                        log.debug("{} is in range {}, allowing recursion", source, cidr.getNetworkAddress());
+                        canRecurse = true;
+                        break;
+                    } else {
+                        log.debug("{} is not in range {}", source, cidr.getNetworkAddress());
+                    }
+                }
+            }
+        } catch (UnknownHostException e) {
+            // this should never happened as we should have only IP ranges!
+            log.error("Unexpected {} exception: {}", e.getClass().getSimpleName(), e.getMessage());
+        }
+
+        return canRecurse;
+    }
+
+    private List<IThreePidProvider> listUsableProviders(ALookupRequest request) {
+        return listUsableProviders(request, false);
+    }
+
+    private List<IThreePidProvider> listUsableProviders(ALookupRequest request, boolean forceRecursive) {
+        List<IThreePidProvider> usableProviders = new ArrayList<>();
+
+        boolean canRecurse = forceRecursive || isAllowedForRecursive(request.getRequester());
+
+        log.info("Host {} allowed for recursion: {}", request.getRequester(), canRecurse);
+        for (IThreePidProvider provider : providers) {
+            if (provider.isEnabled() && (provider.isLocal() || canRecurse || forceRecursive)) {
+                usableProviders.add(provider);
+            }
+        }
+
+        return usableProviders;
+    }
+
+    @Override
+    public List<IThreePidProvider> getLocalProviders() {
+        return providers.stream().filter(iThreePidProvider -> iThreePidProvider.isEnabled() && iThreePidProvider.isLocal()).collect(Collectors.toList());
+    }
+
+    public List<IThreePidProvider> getRemoteProviders() {
+        return providers.stream().filter(iThreePidProvider -> iThreePidProvider.isEnabled() && !iThreePidProvider.isLocal()).collect(Collectors.toList());
+    }
+
+    private static SingleLookupRequest build(String medium, String address) {
+        SingleLookupRequest req = new SingleLookupRequest();
+        req.setType(medium);
+        req.setThreePid(address);
+        req.setRequester("Internal");
+        return req;
+    }
+
+    @Override
+    public Optional<SingleLookupReply> find(String medium, String address, boolean recursive) {
+        return find(build(medium, address), recursive);
+    }
+
+    @Override
+    public Optional<SingleLookupReply> findLocal(String medium, String address) {
+        return find(build(medium, address), getLocalProviders());
+    }
+
+    @Override
+    public Optional<SingleLookupReply> findRemote(String medium, String address) {
+        return find(build(medium, address), getRemoteProviders());
+    }
+
+    public Optional<SingleLookupReply> find(SingleLookupRequest request, boolean forceRecursive) {
+        return find(request, listUsableProviders(request, forceRecursive));
+    }
+
+    public Optional<SingleLookupReply> find(SingleLookupRequest request, List<IThreePidProvider> providers) {
+        for (IThreePidProvider provider : providers) {
+            Optional<SingleLookupReply> lookupDataOpt = provider.find(request);
+            if (lookupDataOpt.isPresent()) {
+                return lookupDataOpt;
+            }
+        }
+
+        if (
+                recursiveCfg.getBridge() != null &&
+                        recursiveCfg.getBridge().getEnabled() &&
+                        (!recursiveCfg.getBridge().getRecursiveOnly() || isAllowedForRecursive(request.getRequester()))
+                ) {
+            log.info("Using bridge failover for lookup");
+            return bridge.find(request);
+        }
+
+        return Optional.empty();
+    }
+
+    @Override
+    public Optional<SingleLookupReply> find(SingleLookupRequest request) {
+        return find(request, false);
+    }
+
+    @Override
+    public Optional<SingleLookupReply> findRecursive(SingleLookupRequest request) {
+        return find(request, true);
+    }
+
+    @Override
+    public List<ThreePidMapping> find(BulkLookupRequest request) {
+        List<ThreePidMapping> mapToDo = new ArrayList<>(request.getMappings());
+        List<ThreePidMapping> mapFoundAll = new ArrayList<>();
+
+        for (IThreePidProvider provider : listUsableProviders(request)) {
+            if (mapToDo.isEmpty()) {
+                log.info("No more mappings to lookup");
+                break;
+            } else {
+                log.info("{} mappings remaining overall", mapToDo.size());
+            }
+
+            log.info("Using provider {} for remaining mappings", provider.getClass().getSimpleName());
+            List<ThreePidMapping> mapFound = provider.populate(mapToDo);
+            log.info("Provider {} returned {} mappings", provider.getClass().getSimpleName(), mapFound.size());
+            mapFoundAll.addAll(mapFound);
+            mapToDo.removeAll(mapFound);
+        }
+
+        return mapFoundAll;
+    }
+
+}