Add mechanism for 3PID invites expiration (#120)

build.gradle

@@ -101,7 +101,7 @@ dependencies {
     compile 'com.j256.ormlite:ormlite-jdbc:5.0'
 
     // ed25519 handling
-    compile 'net.i2p.crypto:eddsa:0.3.0'
+    compile 'net.i2p.crypto:eddsa:0.1.0'
 
     // LDAP connector
     compile 'org.apache.directory.api:api-all:1.0.0'

src/main/java/io/kamax/mxisd/config/InvitationConfig.java

@@ -31,10 +31,42 @@ public class InvitationConfig {
 
     private static final Logger log = LoggerFactory.getLogger(InvitationConfig.class);
 
+    public static class Expiration {
+
+        private Boolean enabled;
+        private long after;
+        private String resolveTo;
+
+        public Boolean isEnabled() {
+            return enabled;
+        }
+
+        public void setEnabled(boolean enabled) {
+            this.enabled = enabled;
+        }
+
+        public long getAfter() {
+            return after;
+        }
+
+        public void setAfter(long after) {
+            this.after = after;
+        }
+
+        public String getResolveTo() {
+            return resolveTo;
+        }
+
+        public void setResolveTo(String resolveTo) {
+            this.resolveTo = resolveTo;
+        }
+
+    }
+
     public static class Resolution {
 
         private boolean recursive = true;
-        private long timer = 1;
+        private long timer = 5;
 
         public boolean isRecursive() {
             return recursive;
@@ -80,9 +112,18 @@ public class InvitationConfig {
         }
     }
 
+    private Expiration expiration = new Expiration();
     private Resolution resolution = new Resolution();
     private Policies policy = new Policies();
 
+    public Expiration getExpiration() {
+        return expiration;
+    }
+
+    public void setExpiration(Expiration expiration) {
+        this.expiration = expiration;
+    }
+
     public Resolution getResolution() {
         return resolution;
     }
@@ -101,6 +142,7 @@ public class InvitationConfig {
 
     public void build() {
         log.info("--- Invite config ---");
+        log.info("Expiration: {}", GsonUtil.get().toJson(getExpiration()));
         log.info("Resolution: {}", GsonUtil.get().toJson(getResolution()));
         log.info("Policies: {}", GsonUtil.get().toJson(getPolicy()));
     }

src/main/java/io/kamax/mxisd/invitation/InvitationManager.java

@@ -31,6 +31,7 @@ import io.kamax.mxisd.config.MxisdConfig;
 import io.kamax.mxisd.config.ServerConfig;
 import io.kamax.mxisd.dns.FederationDnsOverwrite;
 import io.kamax.mxisd.exception.BadRequestException;
+import io.kamax.mxisd.exception.ConfigurationException;
 import io.kamax.mxisd.exception.MappingAlreadyExistsException;
 import io.kamax.mxisd.exception.ObjectNotFoundException;
 import io.kamax.mxisd.lookup.SingleLookupReply;
@@ -41,6 +42,7 @@ import io.kamax.mxisd.profile.ProfileManager;
 import io.kamax.mxisd.storage.IStorage;
 import io.kamax.mxisd.storage.crypto.*;
 import io.kamax.mxisd.storage.ormlite.dao.ThreePidInviteIO;
+import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -63,6 +65,8 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.time.DateTimeException;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ForkJoinPool;
@@ -70,7 +74,10 @@ import java.util.concurrent.TimeUnit;
 
 public class InvitationManager {
 
-    private transient final Logger log = LoggerFactory.getLogger(InvitationManager.class);
+    private static final Logger log = LoggerFactory.getLogger(InvitationManager.class);
+    private static final String CreatedAtPropertyKey = "created_at";
+
+    private final String defaultCreateTs = Long.toString(Instant.now().toEpochMilli());
 
     private InvitationConfig cfg;
     private ServerConfig srvCfg;
@@ -97,7 +104,7 @@ public class InvitationManager {
             NotificationManager notifMgr,
             ProfileManager profileMgr
     ) {
-        this.cfg = mxisdCfg.getInvite();
+        this.cfg = requireValid(mxisdCfg);
         this.srvCfg = mxisdCfg.getServer();
         this.storage = storage;
         this.lookupMgr = lookupMgr;
@@ -110,6 +117,7 @@ public class InvitationManager {
         log.info("Loading saved invites");
         Collection<ThreePidInviteIO> ioList = storage.getInvites();
         ioList.forEach(io -> {
+            io.getProperties().putIfAbsent(CreatedAtPropertyKey, defaultCreateTs);
             log.info("Processing invite {}", GsonUtil.get().toJson(io));
             ThreePidInvite invite = new ThreePidInvite(
                     MatrixID.asAcceptable(io.getSender()),
@@ -119,7 +127,7 @@ public class InvitationManager {
                     io.getProperties()
             );
 
-            ThreePidInviteReply reply = new ThreePidInviteReply(getId(invite), invite, io.getToken(), "", Collections.emptyList());
+            ThreePidInviteReply reply = new ThreePidInviteReply(io.getId(), invite, io.getToken(), "", Collections.emptyList());
             invitations.put(reply.getId(), reply);
         });
 
@@ -136,25 +144,64 @@ public class InvitationManager {
 
         log.info("Setting up invitation mapping refresh timer");
         refreshTimer = new Timer();
-        refreshTimer.scheduleAtFixedRate(new TimerTask() {
-            @Override
-            public void run() {
-                try {
-                    lookupMappingsForInvites();
-                } catch (Throwable t) {
-                    log.error("Error when running background mapping refresh", t);
-                }
-            }
-        }, 5000L, TimeUnit.MILLISECONDS.convert(cfg.getResolution().getTimer(), TimeUnit.MINUTES));
 
+        // We add a shutdown hook to cancel the hook and wait for pending resolutions
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             refreshTimer.cancel();
             ForkJoinPool.commonPool().awaitQuiescence(1, TimeUnit.MINUTES);
         }));
+
+        // We set the refresh timer for background tasks
+        refreshTimer.scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                try {
+                    doMaintenance();
+                } catch (Throwable t) {
+                    log.error("Error when running background maintenance", t);
+                }
+            }
+        }, 5000L, TimeUnit.MILLISECONDS.convert(cfg.getResolution().getTimer(), TimeUnit.MINUTES));
     }
 
-    private String getId(IThreePidInvite invite) {
+    private InvitationConfig requireValid(MxisdConfig cfg) {
-        return invite.getSender().getDomain().toLowerCase() + invite.getMedium().toLowerCase() + invite.getAddress().toLowerCase();
+        // This is not configured, we'll apply a default configuration
+        if (Objects.isNull(cfg.getInvite().getExpiration().isEnabled())) {
+            // We compute our own user, so it can be used if we bridge as well
+            String mxId = MatrixID.asAcceptable("_mxisd-expired_invite", cfg.getMatrix().getDomain()).getId();
+
+            // Enabled by default
+            cfg.getInvite().getExpiration().setEnabled(true);
+
+            // We'll resolve to our computed User ID
+            cfg.getInvite().getExpiration().setResolveTo(mxId);
+
+            // One calendar week (60min/1h * 24 = 1d * 7 = 1w)
+            cfg.getInvite().getExpiration().setAfter(60 * 24 * 7);
+        }
+
+        if (cfg.getInvite().getExpiration().isEnabled()) {
+            if (cfg.getInvite().getExpiration().getAfter() < 1) {
+                throw new ConfigurationException("Invitation expiration delay must be greater or equal to 1");
+            }
+
+            if (StringUtils.isBlank(cfg.getInvite().getExpiration().getResolveTo())) {
+                throw new ConfigurationException("Invitation expiration resolution target cannot be empty/blank");
+            }
+
+            try {
+                MatrixID.asAcceptable(cfg.getInvite().getExpiration().getResolveTo());
+            } catch (IllegalArgumentException e) {
+                throw new ConfigurationException("Invitation expiration resolution target is not a valid Matrix ID: " + e.getMessage());
+            }
+        }
+
+        return cfg.getInvite();
+    }
+
+    private String computeId(IThreePidInvite invite) {
+        String rawId = invite.getSender().getDomain().toLowerCase() + invite.getMedium().toLowerCase() + invite.getAddress().toLowerCase();
+        return Base64.encodeBase64URLSafeString(rawId.getBytes(StandardCharsets.UTF_8));
     }
 
     private String getIdForLog(IThreePidInviteReply reply) {
@@ -248,7 +295,7 @@ public class InvitationManager {
             throw new BadRequestException("Medium type " + invitation.getMedium() + " is not supported");
         }
 
-        String invId = getId(invitation);
+        String invId = computeId(invitation);
         log.info("Handling invite for {}:{} from {} in room {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender(), invitation.getRoomId());
         IThreePidInviteReply reply = invitations.get(invId);
         if (reply != null) {
@@ -276,6 +323,7 @@ public class InvitationManager {
         String pPubKey = keyMgr.getPublicKeyBase64(pKeyId);
         String ePubKey = keyMgr.getPublicKeyBase64(eKeyId);
 
+        invitation.getProperties().put(CreatedAtPropertyKey, Long.toString(Instant.now().toEpochMilli()));
         invitation.getProperties().put("p_key_algo", pKeyId.getAlgorithm());
         invitation.getProperties().put("p_key_serial", pKeyId.getSerial());
         invitation.getProperties().put("p_key_public", pPubKey);
@@ -312,6 +360,58 @@ public class InvitationManager {
         return false;
     }
 
+    private void removeInvite(IThreePidInviteReply reply) {
+        invitations.remove(reply.getId());
+        storage.deleteInvite(reply.getId());
+    }
+
+    /**
+     * Trigger the periodic maintenance tasks
+     */
+    public void doMaintenance() {
+        lookupMappingsForInvites();
+        expireInvites();
+    }
+
+    public void expireInvites() {
+        log.debug("Invite expiration: started");
+
+        if (!cfg.getExpiration().isEnabled()) {
+            log.debug("Invite expiration is disabled, skipping");
+            return;
+        }
+
+        if (invitations.isEmpty()) {
+            log.debug("No invite to expired, skipping");
+            return;
+        }
+
+        String targetMxid = cfg.getExpiration().getResolveTo();
+        for (IThreePidInviteReply reply : invitations.values()) {
+            log.debug("Processing invite {}", reply.getId());
+
+            String tsRaw = reply.getInvite().getProperties().computeIfAbsent(CreatedAtPropertyKey, k -> defaultCreateTs);
+            try {
+                Instant ts = Instant.ofEpochMilli(Long.parseLong(tsRaw));
+                Instant targetTs = ts.plusSeconds(cfg.getExpiration().getAfter() * 60);
+                Instant now = Instant.now();
+                log.debug("Invite {} - Created at {} - Expire at {} - Current time is {}", reply.getId(), ts, targetTs, now);
+                if (targetTs.isBefore(Instant.now())) {
+                    log.debug("Invite {} has not expired yet, skipping", reply.getId());
+                    continue;
+                }
+
+                log.info("Invite {} has expired at TS {} - Expiring and resolving to {}", targetTs, targetMxid);
+                publishMapping(reply, targetMxid);
+            } catch (NumberFormatException | DateTimeException e) {
+                log.warn("Invite {} has an invalid creation TS, setting to default value of {}", reply.getId(), defaultCreateTs);
+                reply.getInvite().getProperties().put(CreatedAtPropertyKey, defaultCreateTs);
+            }
+        }
+
+        log.debug("Invite expiration: finished");
+    }
+
     public void lookupMappingsForInvites() {
         if (!invitations.isEmpty()) {
             log.info("Checking for existing mapping for pending invites");
@@ -391,25 +491,32 @@ public class InvitationManager {
             StringEntity entity = new StringEntity(content.toString(), StandardCharsets.UTF_8);
             entity.setContentType("application/json");
             req.setEntity(entity);
+
+            Instant resolvedAt = Instant.now();
+            boolean couldPublish = false;
             try {
                 log.info("Posting onBind event to {}", req.getURI());
                 CloseableHttpResponse response = client.execute(req);
                 int statusCode = response.getStatusLine().getStatusCode();
                 log.info("Answer code: {}", statusCode);
                 if (statusCode >= 300 && statusCode != 403) {
-                    log.warn("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8));
+                    log.info("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8));
+                    log.warn("HS returned an error. Invite can be found in historical storage for manual re-processing");
                 } else {
+                    couldPublish = true;
                     if (statusCode == 403) {
-                        log.info("Invite was obsolete");
+                        log.info("Invite is obsolete or no longer under our control");
                     }
-
-                    invitations.remove(getId(reply.getInvite()));
-                    storage.deleteInvite(reply.getId());
-                    log.info("Removed invite from internal store");
                 }
                 response.close();
             } catch (IOException e) {
                 log.warn("Unable to tell HS {} about invite being mapped", domain, e);
+            } finally {
+                synchronized (this) {
+                    storage.insertHistoricalInvite(reply, mxid, resolvedAt, couldPublish);
+                    removeInvite(reply);
+                    log.info("Moved invite {} to historical table", reply.getId());
+                }
             }
         }).start();
     }
@@ -425,7 +532,7 @@ public class InvitationManager {
         @Override
         public void run() {
             try {
-                log.info("Searching for mapping created since invite {} was created", getIdForLog(reply));
+                log.info("Searching for mapping created after invite {} was created", getIdForLog(reply));
                 Optional<SingleLookupReply> result = lookup3pid(reply.getInvite().getMedium(), reply.getInvite().getAddress());
                 if (result.isPresent()) {
                     SingleLookupReply lookup = result.get();

src/main/java/io/kamax/mxisd/storage/IStorage.java

@@ -38,6 +38,8 @@ public interface IStorage {
 
     void deleteInvite(String id);
 
+    void insertHistoricalInvite(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish);
+
     Optional<IThreePidSessionDao> getThreePidSession(String sid);
 
     Optional<IThreePidSessionDao> findThreePidSession(ThreePid tpid, String secret);

src/main/java/io/kamax/mxisd/storage/crypto/Ed25519KeyManager.java

@@ -47,7 +47,7 @@ public class Ed25519KeyManager implements KeyManager {
     private final KeyStore store;
 
     public Ed25519KeyManager(KeyStore store) {
-        this.keySpecs = EdDSANamedCurveTable.getByName(EdDSANamedCurveTable.ED_25519);
+        this.keySpecs = EdDSANamedCurveTable.getByName(EdDSANamedCurveTable.CURVE_ED25519_SHA512);
         this.store = store;
 
         if (!store.getCurrentKey().isPresent()) {
@@ -106,7 +106,7 @@ public class Ed25519KeyManager implements KeyManager {
     }
 
     public EdDSAPrivateKeySpec getPrivateKeySpecs(KeyIdentifier id) {
-        return new EdDSAPrivateKeySpec(java.util.Base64.getDecoder().decode(getKey(id).getPrivateKeyBase64()), keySpecs);
+        return new EdDSAPrivateKeySpec(Base64.decodeBase64(getKey(id).getPrivateKeyBase64()), keySpecs);
     }
 
     public EdDSAPrivateKey getPrivateKey(KeyIdentifier id) {

src/main/java/io/kamax/mxisd/storage/crypto/Ed25519SignatureManager.java

@@ -43,7 +43,6 @@ public class Ed25519SignatureManager implements SignatureManager {
         Signature sign = sign(message);
 
         JsonObject keySignature = new JsonObject();
-        // FIXME should create a signing key object what would give this ed and index values
         keySignature.addProperty(sign.getKey().getAlgorithm() + ":" + sign.getKey().getSerial(), sign.getSignature());
         JsonObject signature = new JsonObject();
         signature.add(domain, keySignature);
@@ -53,7 +52,6 @@ public class Ed25519SignatureManager implements SignatureManager {
 
     @Override
     public Signature sign(JsonObject obj) {
-
         return sign(MatrixJson.encodeCanonical(obj));
     }
 

src/main/java/io/kamax/mxisd/storage/ormlite/OrmLiteSqlStorage.java

@@ -34,24 +34,18 @@ import io.kamax.mxisd.invitation.IThreePidInviteReply;
 import io.kamax.mxisd.storage.IStorage;
 import io.kamax.mxisd.storage.dao.IThreePidSessionDao;
 import io.kamax.mxisd.storage.ormlite.dao.ASTransactionDao;
+import io.kamax.mxisd.storage.ormlite.dao.HistoricalThreePidInviteIO;
 import io.kamax.mxisd.storage.ormlite.dao.ThreePidInviteIO;
 import io.kamax.mxisd.storage.ormlite.dao.ThreePidSessionDao;
 import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.sql.SQLException;
 import java.time.Instant;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
 
 public class OrmLiteSqlStorage implements IStorage {
 
-    private transient final Logger log = LoggerFactory.getLogger(OrmLiteSqlStorage.class);
-
     @FunctionalInterface
     private interface Getter<T> {
 
@@ -67,6 +61,7 @@ public class OrmLiteSqlStorage implements IStorage {
     }
 
     private Dao<ThreePidInviteIO, String> invDao;
+    private Dao<HistoricalThreePidInviteIO, String> expInvDao;
     private Dao<ThreePidSessionDao, String> sessionDao;
     private Dao<ASTransactionDao, String> asTxnDao;
 
@@ -86,6 +81,7 @@ public class OrmLiteSqlStorage implements IStorage {
         withCatcher(() -> {
             ConnectionSource connPool = new JdbcConnectionSource("jdbc:" + backend + ":" + path);
             invDao = createDaoAndTable(connPool, ThreePidInviteIO.class);
+            expInvDao = createDaoAndTable(connPool, HistoricalThreePidInviteIO.class);
             sessionDao = createDaoAndTable(connPool, ThreePidSessionDao.class);
             asTxnDao = createDaoAndTable(connPool, ASTransactionDao.class);
         });
@@ -150,6 +146,24 @@ public class OrmLiteSqlStorage implements IStorage {
         });
     }
 
+    @Override
+    public void insertHistoricalInvite(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish) {
+        withCatcher(() -> {
+            HistoricalThreePidInviteIO io = new HistoricalThreePidInviteIO(data, resolvedTo, resolvedAt, couldPublish);
+            int updated = expInvDao.create(io);
+            if (updated != 1) {
+                throw new RuntimeException("Unexpected row count after DB action: " + updated);
+            }
+
+            // Ugly, but it avoids touching the structure of the historical parent class
+            // and avoid any possible regression at this point.
+            updated = expInvDao.updateId(io, UUID.randomUUID().toString().replace("-", ""));
+            if (updated != 1) {
+                throw new RuntimeException("Unexpected row count after DB action: " + updated);
+            }
+        });
+    }
+
     @Override
     public Optional<IThreePidSessionDao> getThreePidSession(String sid) {
         return withCatcher(() -> Optional.ofNullable(sessionDao.queryForId(sid)));

src/main/java/io/kamax/mxisd/storage/ormlite/dao/HistoricalThreePidInviteIO.java

@@ -0,0 +1,72 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2019 Kamax Sarl
+ *
+ * https://www.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.storage.ormlite.dao;
+
+import com.j256.ormlite.field.DatabaseField;
+import com.j256.ormlite.table.DatabaseTable;
+import io.kamax.mxisd.invitation.IThreePidInviteReply;
+
+import java.time.Instant;
+
+@DatabaseTable(tableName = "invite_3pid_history")
+public class HistoricalThreePidInviteIO extends ThreePidInviteIO {
+
+    @DatabaseField(canBeNull = false)
+    private String resolvedTo;
+
+    @DatabaseField(canBeNull = false)
+    private long resolvedAt;
+
+    @DatabaseField(canBeNull = false)
+    private boolean couldPublish;
+
+    @DatabaseField(canBeNull = false)
+    private long publishAttempts = 1; // Placeholder for retry mechanism, if ever implemented
+
+    public HistoricalThreePidInviteIO() {
+        // Needed for ORMLite
+    }
+
+    public HistoricalThreePidInviteIO(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish) {
+        super(data);
+
+        this.resolvedTo = resolvedTo;
+        this.resolvedAt = resolvedAt.toEpochMilli();
+        this.couldPublish = couldPublish;
+    }
+
+    public String getResolvedTo() {
+        return resolvedTo;
+    }
+
+    public Instant getResolvedAt() {
+        return Instant.ofEpochMilli(resolvedAt);
+    }
+
+    public boolean isCouldPublish() {
+        return couldPublish;
+    }
+
+    public long getPublishAttempts() {
+        return publishAttempts;
+    }
+
+}