Add persistence storage for invites

.gitignore

@@ -8,3 +8,6 @@ out/
 
 # Local dev config
 /application.yaml
+
+# Local dev storage
+/mxisd.db

application.example.yaml

@@ -244,7 +244,7 @@ invite:
       # - Message-Id
       # - X-Mailer
       #
-      # The following placeholders are possible:
+      # The following placeholders are available:
       # - %DOMAIN%                Domain name as per server.name config item
       # - %DOMAIN_PRETTY%         Word capitalize version of the domain. e.g. example.org -> Example.org
       # - %FROM_EMAIL%            Value of this section's email config item
@@ -258,3 +258,24 @@ invite:
       # - %ROOM_NAME%             Name of the room, empty if not available
       # - %ROOM_NAME_OR_ID%       Value of %ROOM_NAME% or, if empty, value of %ROOM_ID%
       template: "/absolute/path/to/file"
+
+
+
+# Configure persistence settings
+storage:
+
+  # Configure the storage backend, usually a DB
+  # Possible built-in values:
+  #   sqlite                      SQLite backend, default
+  #
+  #backend: 'sqlite'
+
+  # Specific configuration for each provider, refer to their documentation for specifics.
+  provider:
+
+    # Generic SQLite provider config
+    sqlite:
+
+      # Path to the SQLite DB file, required
+      #
+      #database:'%SQLITE_DATABASE_PATH%'

build.gradle

@@ -104,6 +104,12 @@ dependencies {
     // Google Firebase Authentication backend
     compile 'com.google.firebase:firebase-admin:5.3.0'
 
+    // ORMLite
+    compile 'com.j256.ormlite:ormlite-jdbc:5.0'
+
+    // SQLite
+    compile 'org.xerial:sqlite-jdbc:3.20.0'
+
     testCompile 'junit:junit:4.12'
 }
 
@@ -161,6 +167,12 @@ task buildDeb(dependsOn: build) {
             value: "${debDataPath}/signing.key"
         )
 
+        ant.replaceregexp(
+                file: "${debBuildConfPath}/${debConfFileName}",
+                match: "#?database:\\s*'%SQLITE_DATABASE_PATH%'",
+                replace: "database: '${debDataPath}/mxisd.db'"
+        )
+
         copy {
             from project.file('src/debian')
             into debBuildDebianPath

src/main/groovy/io/kamax/mxisd/config/SQLiteStorageConfig.java

@@ -0,0 +1,40 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties("storage.provider.sqlite")
+public class SQLiteStorageConfig {
+
+    private String database;
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/config/StorageConfig.java

@@ -0,0 +1,51 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.config;
+
+import io.kamax.mxisd.exception.ConfigurationException;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import javax.annotation.PostConstruct;
+
+@Configuration
+@ConfigurationProperties("storage")
+public class StorageConfig {
+
+    private String backend;
+
+    public String getBackend() {
+        return backend;
+    }
+
+    public void setBackend(String backend) {
+        this.backend = backend;
+    }
+
+    @PostConstruct
+    private void postConstruct() {
+        if (StringUtils.isBlank(getBackend())) {
+            throw new ConfigurationException("storage.backend");
+        }
+    }
+
+}

src/main/groovy/io/kamax/mxisd/invitation/IThreePidInviteReply.java

@@ -22,6 +22,8 @@ package io.kamax.mxisd.invitation;
 
 public interface IThreePidInviteReply {
 
+    String getId();
+
     IThreePidInvite getInvite();
 
     String getToken();

src/main/groovy/io/kamax/mxisd/invitation/InvitationManager.java

@@ -21,7 +21,7 @@
 package io.kamax.mxisd.invitation;
 
 import com.google.gson.Gson;
-import io.kamax.matrix.ThreePid;
+import io.kamax.matrix.MatrixID;
 import io.kamax.mxisd.exception.BadRequestException;
 import io.kamax.mxisd.exception.MappingAlreadyExistsException;
 import io.kamax.mxisd.invitation.sender.IInviteSender;
@@ -29,8 +29,11 @@ import io.kamax.mxisd.lookup.SingleLookupReply;
 import io.kamax.mxisd.lookup.ThreePidMapping;
 import io.kamax.mxisd.lookup.strategy.LookupStrategy;
 import io.kamax.mxisd.signature.SignatureManager;
+import io.kamax.mxisd.storage.IStorage;
+import io.kamax.mxisd.storage.ormlite.ThreePidInviteIO;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
@@ -64,7 +67,10 @@ public class InvitationManager {
 
     private Logger log = LoggerFactory.getLogger(InvitationManager.class);
 
-    private Map<ThreePid, IThreePidInviteReply> invitations = new ConcurrentHashMap<>();
+    private Map<String, IThreePidInviteReply> invitations = new ConcurrentHashMap<>();
+
+    @Autowired
+    private IStorage storage;
 
     @Autowired
     private LookupStrategy lookupMgr;
@@ -78,10 +84,30 @@ public class InvitationManager {
     private Gson gson;
     private Timer refreshTimer;
 
+    private String getId(IThreePidInvite invite) {
+        return invite.getSender().getDomain() + invite.getMedium() + invite.getAddress();
+    }
+
     @PostConstruct
     private void postConstruct() {
         gson = new Gson();
 
+        log.info("Loading saved invites");
+        Collection<ThreePidInviteIO> ioList = storage.getInvites();
+        ioList.forEach(io -> {
+            log.info("Processing invite {}", gson.toJson(io));
+            ThreePidInvite invite = new ThreePidInvite(
+                    new MatrixID(io.getSender()),
+                    io.getMedium(),
+                    io.getAddress(),
+                    io.getRoomId(),
+                    io.getProperties()
+            );
+
+            ThreePidInviteReply reply = new ThreePidInviteReply(getId(invite), invite, io.getToken(), "");
+            invitations.put(reply.getId(), reply);
+        });
+
         // FIXME export such madness into matrix-java-sdk with a nice wrapper to talk to a homeserver
         try {
             SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustSelfSignedStrategy()).build();
@@ -93,6 +119,7 @@ public class InvitationManager {
             throw new RuntimeException(e);
         }
 
+        log.info("Setting up invitation mapping refresh timer");
         refreshTimer = new Timer();
         refreshTimer.scheduleAtFixedRate(new TimerTask() {
             @Override
@@ -166,30 +193,31 @@ public class InvitationManager {
             throw new BadRequestException("Medium type " + invitation.getMedium() + " is not supported");
         }
 
-        ThreePid pid = new ThreePid(invitation.getMedium(), invitation.getAddress());
+        String invId = getId(invitation);
-
+        log.info("Handling invite for {}:{} from {} in room {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender(), invitation.getRoomId());
-        log.info("Handling invite for {}:{} from {} in room {}", pid.getMedium(), pid.getAddress(), invitation.getSender(), invitation.getRoomId());
+        if (invitations.containsKey(invId)) { // FIXME we need to lookup using the HS domain too!!
-        if (invitations.containsKey(pid)) {
+            log.info("Invite is already pending for {}:{}, returning data", invitation.getMedium(), invitation.getAddress());
-            log.info("Invite is already pending for {}:{}, returning data", pid.getMedium(), pid.getAddress());
+            return invitations.get(invId);
-            return invitations.get(pid);
         }
 
         Optional<?> result = lookupMgr.find(invitation.getMedium(), invitation.getAddress(), true);
         if (result.isPresent()) {
-            log.info("Mapping for {}:{} already exists, refusing to store invite", pid.getMedium(), pid.getAddress());
+            log.info("Mapping for {}:{} already exists, refusing to store invite", invitation.getMedium(), invitation.getAddress());
             throw new MappingAlreadyExistsException();
         }
 
         String token = RandomStringUtils.randomAlphanumeric(64);
         String displayName = invitation.getAddress().substring(0, 3) + "...";
 
-        IThreePidInviteReply reply = new ThreePidInviteReply(invitation, token, displayName);
+        IThreePidInviteReply reply = new ThreePidInviteReply(invId, invitation, token, displayName);
 
-        log.info("Performing invite to {}:{}", pid.getMedium(), pid.getAddress());
+        log.info("Performing invite to {}:{}", invitation.getMedium(), invitation.getAddress());
         sender.send(reply);
 
-        invitations.put(pid, reply);
+        log.info("Storing invite under ID {}", invId);
-        log.info("A new invite has been created for {}:{}", pid.getMedium(), pid.getAddress());
+        storage.insertInvite(reply);
+        invitations.put(invId, reply);
+        log.info("A new invite has been created for {}:{} on HS {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender().getDomain());
 
         return reply;
     }
@@ -203,13 +231,12 @@ public class InvitationManager {
     }
 
     public void publishMappingIfInvited(ThreePidMapping threePid) {
-        ThreePid key = new ThreePid(threePid.getMedium(), threePid.getValue());
+        log.info("Looking up possible pending invites for {}:{}", threePid.getMedium(), threePid.getValue());
-        IThreePidInviteReply reply = invitations.get(key);
+        for (IThreePidInviteReply reply : invitations.values()) {
-        if (reply == null) {
+            if (StringUtils.equals(reply.getInvite().getMedium(), threePid.getMedium()) && StringUtils.equals(reply.getInvite().getAddress(), threePid.getValue())) {
-            log.info("{}:{} does not have a pending invite, no mapping to publish", threePid.getMedium(), threePid.getValue());
+                log.info("{}:{} has an invite pending on HS {}, publishing mapping", threePid.getMedium(), threePid.getValue(), reply.getInvite().getSender().getDomain());
-        } else {
+                publishMapping(reply, threePid.getMxid());
-            log.info("{}:{} has an invite pending, publishing mapping", threePid.getMedium(), threePid.getValue());
+            }
-            publishMapping(reply, threePid.getMxid());
         }
     }
 
@@ -255,15 +282,17 @@ public class InvitationManager {
                 CloseableHttpResponse response = client.execute(req);
                 int statusCode = response.getStatusLine().getStatusCode();
                 log.info("Answer code: {}", statusCode);
-                if (statusCode >= 400) {
+                if (statusCode >= 300) {
                     log.warn("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8));
+                } else {
+                    invitations.remove(getId(reply));
+                    storage.deleteInvite(reply.getId());
+                    log.info("Removed invite from internal store");
                 }
                 response.close();
             } catch (IOException e) {
                 log.warn("Unable to tell HS {} about invite being mapped", domain, e);
             }
-            invitations.remove(new ThreePid(medium, address));
-            log.info("Removed invite from internal store");
         }).start();
     }
 

src/main/groovy/io/kamax/mxisd/invitation/ThreePidInviteReply.java

@@ -22,16 +22,23 @@ package io.kamax.mxisd.invitation;
 
 public class ThreePidInviteReply implements IThreePidInviteReply {
 
+    private String id;
     private IThreePidInvite invite;
     private String token;
     private String displayName;
 
-    public ThreePidInviteReply(IThreePidInvite invite, String token, String displayName) {
+    public ThreePidInviteReply(String id, IThreePidInvite invite, String token, String displayName) {
+        this.id = id;
         this.invite = invite;
         this.token = token;
         this.displayName = displayName;
     }
 
+    @Override
+    public String getId() {
+        return id;
+    }
+
     @Override
     public IThreePidInvite getInvite() {
         return invite;

src/main/groovy/io/kamax/mxisd/storage/IStorage.java

@@ -0,0 +1,36 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.storage;
+
+import io.kamax.mxisd.invitation.IThreePidInviteReply;
+import io.kamax.mxisd.storage.ormlite.ThreePidInviteIO;
+
+import java.util.Collection;
+
+public interface IStorage {
+
+    Collection<ThreePidInviteIO> getInvites();
+
+    void insertInvite(IThreePidInviteReply data);
+
+    void deleteInvite(String id);
+
+}

src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorage.java

@@ -0,0 +1,87 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.storage.ormlite;
+
+import com.j256.ormlite.dao.CloseableWrappedIterable;
+import com.j256.ormlite.dao.Dao;
+import com.j256.ormlite.dao.DaoManager;
+import com.j256.ormlite.jdbc.JdbcConnectionSource;
+import com.j256.ormlite.support.ConnectionSource;
+import com.j256.ormlite.table.TableUtils;
+import io.kamax.mxisd.invitation.IThreePidInviteReply;
+import io.kamax.mxisd.storage.IStorage;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class OrmLiteSqliteStorage implements IStorage {
+
+    private Dao<ThreePidInviteIO, String> invDao;
+
+    OrmLiteSqliteStorage(String path) {
+        try {
+            ConnectionSource connPool = new JdbcConnectionSource("jdbc:sqlite:" + path);
+            invDao = DaoManager.createDao(connPool, ThreePidInviteIO.class);
+            TableUtils.createTableIfNotExists(connPool, ThreePidInviteIO.class);
+        } catch (SQLException e) {
+            throw new RuntimeException(e); // FIXME do better
+        }
+    }
+
+    @Override
+    public Collection<ThreePidInviteIO> getInvites() {
+        try (CloseableWrappedIterable<ThreePidInviteIO> t = invDao.getWrappedIterable()) {
+            List<ThreePidInviteIO> ioList = new ArrayList<>();
+            t.forEach(ioList::add);
+            return ioList;
+        } catch (IOException e) {
+            throw new RuntimeException(e); // FIXME do better
+        }
+    }
+
+    @Override
+    public void insertInvite(IThreePidInviteReply data) {
+        try {
+            int updated = invDao.create(new ThreePidInviteIO(data));
+            if (updated != 1) {
+                throw new RuntimeException("Unexpected row count after DB action: " + updated);
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e); // FIXME do better
+        }
+    }
+
+    @Override
+    public void deleteInvite(String id) {
+        try {
+            int updated = invDao.deleteById(id);
+            if (updated != 1) {
+                throw new RuntimeException("Unexpected row count after DB action: " + updated);
+            }
+        } catch (SQLException e) {
+            throw new RuntimeException(e); // FIXME do better
+        }
+    }
+
+}

src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorageBeanFactory.java

@@ -0,0 +1,76 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.storage.ormlite;
+
+import io.kamax.mxisd.config.SQLiteStorageConfig;
+import io.kamax.mxisd.config.StorageConfig;
+import io.kamax.mxisd.exception.ConfigurationException;
+import io.kamax.mxisd.storage.IStorage;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.FactoryBeanNotInitializedException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+
+@Component
+public class OrmLiteSqliteStorageBeanFactory implements FactoryBean<IStorage> {
+
+    @Autowired
+    private StorageConfig storagecfg;
+
+    @Autowired
+    private SQLiteStorageConfig cfg;
+
+    private OrmLiteSqliteStorage storage;
+
+    @PostConstruct
+    private void postConstruct() {
+        if (StringUtils.equals("sqlite", storagecfg.getBackend())) {
+            if (StringUtils.isBlank(cfg.getDatabase())) {
+                throw new ConfigurationException("storage.provider.sqlite.database");
+            }
+
+            storage = new OrmLiteSqliteStorage(cfg.getDatabase());
+        }
+    }
+
+    @Override
+    public IStorage getObject() throws Exception {
+        if (storage == null) {
+            throw new FactoryBeanNotInitializedException();
+        }
+
+        return storage;
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        return OrmLiteSqliteStorage.class;
+    }
+
+    @Override
+    public boolean isSingleton() {
+        return true;
+    }
+
+}

src/main/groovy/io/kamax/mxisd/storage/ormlite/ThreePidInviteIO.java

@@ -0,0 +1,106 @@
+/*
+ * mxisd - Matrix Identity Server Daemon
+ * Copyright (C) 2017 Maxime Dor
+ *
+ * https://max.kamax.io/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.kamax.mxisd.storage.ormlite;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.j256.ormlite.field.DatabaseField;
+import com.j256.ormlite.table.DatabaseTable;
+import io.kamax.mxisd.invitation.IThreePidInviteReply;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@DatabaseTable(tableName = "invite_3pid")
+public class ThreePidInviteIO {
+
+    private static Gson gson = new Gson();
+
+    @DatabaseField(id = true)
+    private String id;
+
+    @DatabaseField(canBeNull = false)
+    private String token;
+
+    @DatabaseField(canBeNull = false)
+    private String sender;
+
+    @DatabaseField(canBeNull = false)
+    private String medium;
+
+    @DatabaseField(canBeNull = false)
+    private String address;
+
+    @DatabaseField(canBeNull = false)
+    private String roomId;
+
+    @DatabaseField
+    private String properties;
+
+    public ThreePidInviteIO() {
+        // needed for ORMlite
+    }
+
+    public ThreePidInviteIO(IThreePidInviteReply data) {
+        this.id = data.getId();
+        this.token = data.getToken();
+        this.sender = data.getInvite().getSender().getId();
+        this.medium = data.getInvite().getMedium();
+        this.address = data.getInvite().getAddress();
+        this.roomId = data.getInvite().getRoomId();
+        this.properties = gson.toJson(data.getInvite().getProperties());
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getToken() {
+        return token;
+    }
+
+    public String getSender() {
+        return sender;
+    }
+
+    public String getMedium() {
+        return medium;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public String getRoomId() {
+        return roomId;
+    }
+
+    public Map<String, String> getProperties() {
+        if (StringUtils.isBlank(properties)) {
+            return new HashMap<>();
+        }
+
+        return gson.fromJson(properties, new TypeToken<Map<String, String>>() {
+        }.getType());
+    }
+
+}

src/main/resources/application.yaml

@@ -40,3 +40,6 @@ invite:
       tls: 1
       name: "mxisd Identity Server"
       template: "classpath:email/invite-template.eml"
+
+storage:
+  backend: 'sqlite'