diff --git a/.gitignore b/.gitignore index 5a642a0..93fc794 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ out/ # Local dev config /application.yaml + +# Local dev storage +/mxisd.db diff --git a/application.example.yaml b/application.example.yaml index 67d8e3e..8b0a46c 100644 --- a/application.example.yaml +++ b/application.example.yaml @@ -244,7 +244,7 @@ invite: # - Message-Id # - X-Mailer # - # The following placeholders are possible: + # The following placeholders are available: # - %DOMAIN% Domain name as per server.name config item # - %DOMAIN_PRETTY% Word capitalize version of the domain. e.g. example.org -> Example.org # - %FROM_EMAIL% Value of this section's email config item @@ -258,3 +258,24 @@ invite: # - %ROOM_NAME% Name of the room, empty if not available # - %ROOM_NAME_OR_ID% Value of %ROOM_NAME% or, if empty, value of %ROOM_ID% template: "/absolute/path/to/file" + + + +# Configure persistence settings +storage: + + # Configure the storage backend, usually a DB + # Possible built-in values: + # sqlite SQLite backend, default + # + #backend: 'sqlite' + + # Specific configuration for each provider, refer to their documentation for specifics. + provider: + + # Generic SQLite provider config + sqlite: + + # Path to the SQLite DB file, required + # + #database:'%SQLITE_DATABASE_PATH%' diff --git a/build.gradle b/build.gradle index e538ddf..2a53882 100644 --- a/build.gradle +++ b/build.gradle @@ -104,6 +104,12 @@ dependencies { // Google Firebase Authentication backend compile 'com.google.firebase:firebase-admin:5.3.0' + // ORMLite + compile 'com.j256.ormlite:ormlite-jdbc:5.0' + + // SQLite + compile 'org.xerial:sqlite-jdbc:3.20.0' + testCompile 'junit:junit:4.12' } @@ -161,6 +167,12 @@ task buildDeb(dependsOn: build) { value: "${debDataPath}/signing.key" ) + ant.replaceregexp( + file: "${debBuildConfPath}/${debConfFileName}", + match: "#?database:\\s*'%SQLITE_DATABASE_PATH%'", + replace: "database: '${debDataPath}/mxisd.db'" + ) + copy { from project.file('src/debian') into debBuildDebianPath diff --git a/src/main/groovy/io/kamax/mxisd/config/SQLiteStorageConfig.java b/src/main/groovy/io/kamax/mxisd/config/SQLiteStorageConfig.java new file mode 100644 index 0000000..8ca0680 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/config/SQLiteStorageConfig.java @@ -0,0 +1,40 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties("storage.provider.sqlite") +public class SQLiteStorageConfig { + + private String database; + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/config/StorageConfig.java b/src/main/groovy/io/kamax/mxisd/config/StorageConfig.java new file mode 100644 index 0000000..247a424 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/config/StorageConfig.java @@ -0,0 +1,51 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.config; + +import io.kamax.mxisd.exception.ConfigurationException; +import org.apache.commons.lang.StringUtils; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; + +@Configuration +@ConfigurationProperties("storage") +public class StorageConfig { + + private String backend; + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } + + @PostConstruct + private void postConstruct() { + if (StringUtils.isBlank(getBackend())) { + throw new ConfigurationException("storage.backend"); + } + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/invitation/IThreePidInviteReply.java b/src/main/groovy/io/kamax/mxisd/invitation/IThreePidInviteReply.java index cbe351f..cd80dba 100644 --- a/src/main/groovy/io/kamax/mxisd/invitation/IThreePidInviteReply.java +++ b/src/main/groovy/io/kamax/mxisd/invitation/IThreePidInviteReply.java @@ -22,6 +22,8 @@ package io.kamax.mxisd.invitation; public interface IThreePidInviteReply { + String getId(); + IThreePidInvite getInvite(); String getToken(); diff --git a/src/main/groovy/io/kamax/mxisd/invitation/InvitationManager.java b/src/main/groovy/io/kamax/mxisd/invitation/InvitationManager.java index 83e9459..fd9560a 100644 --- a/src/main/groovy/io/kamax/mxisd/invitation/InvitationManager.java +++ b/src/main/groovy/io/kamax/mxisd/invitation/InvitationManager.java @@ -21,7 +21,7 @@ package io.kamax.mxisd.invitation; import com.google.gson.Gson; -import io.kamax.matrix.ThreePid; +import io.kamax.matrix.MatrixID; import io.kamax.mxisd.exception.BadRequestException; import io.kamax.mxisd.exception.MappingAlreadyExistsException; import io.kamax.mxisd.invitation.sender.IInviteSender; @@ -29,8 +29,11 @@ import io.kamax.mxisd.lookup.SingleLookupReply; import io.kamax.mxisd.lookup.ThreePidMapping; import io.kamax.mxisd.lookup.strategy.LookupStrategy; import io.kamax.mxisd.signature.SignatureManager; +import io.kamax.mxisd.storage.IStorage; +import io.kamax.mxisd.storage.ormlite.ThreePidInviteIO; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.StringUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.ssl.NoopHostnameVerifier; @@ -64,7 +67,10 @@ public class InvitationManager { private Logger log = LoggerFactory.getLogger(InvitationManager.class); - private Map invitations = new ConcurrentHashMap<>(); + private Map invitations = new ConcurrentHashMap<>(); + + @Autowired + private IStorage storage; @Autowired private LookupStrategy lookupMgr; @@ -78,10 +84,30 @@ public class InvitationManager { private Gson gson; private Timer refreshTimer; + private String getId(IThreePidInvite invite) { + return invite.getSender().getDomain() + invite.getMedium() + invite.getAddress(); + } + @PostConstruct private void postConstruct() { gson = new Gson(); + log.info("Loading saved invites"); + Collection ioList = storage.getInvites(); + ioList.forEach(io -> { + log.info("Processing invite {}", gson.toJson(io)); + ThreePidInvite invite = new ThreePidInvite( + new MatrixID(io.getSender()), + io.getMedium(), + io.getAddress(), + io.getRoomId(), + io.getProperties() + ); + + ThreePidInviteReply reply = new ThreePidInviteReply(getId(invite), invite, io.getToken(), ""); + invitations.put(reply.getId(), reply); + }); + // FIXME export such madness into matrix-java-sdk with a nice wrapper to talk to a homeserver try { SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustSelfSignedStrategy()).build(); @@ -93,6 +119,7 @@ public class InvitationManager { throw new RuntimeException(e); } + log.info("Setting up invitation mapping refresh timer"); refreshTimer = new Timer(); refreshTimer.scheduleAtFixedRate(new TimerTask() { @Override @@ -166,30 +193,31 @@ public class InvitationManager { throw new BadRequestException("Medium type " + invitation.getMedium() + " is not supported"); } - ThreePid pid = new ThreePid(invitation.getMedium(), invitation.getAddress()); - - log.info("Handling invite for {}:{} from {} in room {}", pid.getMedium(), pid.getAddress(), invitation.getSender(), invitation.getRoomId()); - if (invitations.containsKey(pid)) { - log.info("Invite is already pending for {}:{}, returning data", pid.getMedium(), pid.getAddress()); - return invitations.get(pid); + String invId = getId(invitation); + log.info("Handling invite for {}:{} from {} in room {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender(), invitation.getRoomId()); + if (invitations.containsKey(invId)) { // FIXME we need to lookup using the HS domain too!! + log.info("Invite is already pending for {}:{}, returning data", invitation.getMedium(), invitation.getAddress()); + return invitations.get(invId); } Optional result = lookupMgr.find(invitation.getMedium(), invitation.getAddress(), true); if (result.isPresent()) { - log.info("Mapping for {}:{} already exists, refusing to store invite", pid.getMedium(), pid.getAddress()); + log.info("Mapping for {}:{} already exists, refusing to store invite", invitation.getMedium(), invitation.getAddress()); throw new MappingAlreadyExistsException(); } String token = RandomStringUtils.randomAlphanumeric(64); String displayName = invitation.getAddress().substring(0, 3) + "..."; - IThreePidInviteReply reply = new ThreePidInviteReply(invitation, token, displayName); + IThreePidInviteReply reply = new ThreePidInviteReply(invId, invitation, token, displayName); - log.info("Performing invite to {}:{}", pid.getMedium(), pid.getAddress()); + log.info("Performing invite to {}:{}", invitation.getMedium(), invitation.getAddress()); sender.send(reply); - invitations.put(pid, reply); - log.info("A new invite has been created for {}:{}", pid.getMedium(), pid.getAddress()); + log.info("Storing invite under ID {}", invId); + storage.insertInvite(reply); + invitations.put(invId, reply); + log.info("A new invite has been created for {}:{} on HS {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender().getDomain()); return reply; } @@ -203,13 +231,12 @@ public class InvitationManager { } public void publishMappingIfInvited(ThreePidMapping threePid) { - ThreePid key = new ThreePid(threePid.getMedium(), threePid.getValue()); - IThreePidInviteReply reply = invitations.get(key); - if (reply == null) { - log.info("{}:{} does not have a pending invite, no mapping to publish", threePid.getMedium(), threePid.getValue()); - } else { - log.info("{}:{} has an invite pending, publishing mapping", threePid.getMedium(), threePid.getValue()); - publishMapping(reply, threePid.getMxid()); + log.info("Looking up possible pending invites for {}:{}", threePid.getMedium(), threePid.getValue()); + for (IThreePidInviteReply reply : invitations.values()) { + if (StringUtils.equals(reply.getInvite().getMedium(), threePid.getMedium()) && StringUtils.equals(reply.getInvite().getAddress(), threePid.getValue())) { + log.info("{}:{} has an invite pending on HS {}, publishing mapping", threePid.getMedium(), threePid.getValue(), reply.getInvite().getSender().getDomain()); + publishMapping(reply, threePid.getMxid()); + } } } @@ -255,15 +282,17 @@ public class InvitationManager { CloseableHttpResponse response = client.execute(req); int statusCode = response.getStatusLine().getStatusCode(); log.info("Answer code: {}", statusCode); - if (statusCode >= 400) { + if (statusCode >= 300) { log.warn("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); + } else { + invitations.remove(getId(reply)); + storage.deleteInvite(reply.getId()); + log.info("Removed invite from internal store"); } response.close(); } catch (IOException e) { log.warn("Unable to tell HS {} about invite being mapped", domain, e); } - invitations.remove(new ThreePid(medium, address)); - log.info("Removed invite from internal store"); }).start(); } diff --git a/src/main/groovy/io/kamax/mxisd/invitation/ThreePidInviteReply.java b/src/main/groovy/io/kamax/mxisd/invitation/ThreePidInviteReply.java index 5a02c2e..c67139e 100644 --- a/src/main/groovy/io/kamax/mxisd/invitation/ThreePidInviteReply.java +++ b/src/main/groovy/io/kamax/mxisd/invitation/ThreePidInviteReply.java @@ -22,16 +22,23 @@ package io.kamax.mxisd.invitation; public class ThreePidInviteReply implements IThreePidInviteReply { + private String id; private IThreePidInvite invite; private String token; private String displayName; - public ThreePidInviteReply(IThreePidInvite invite, String token, String displayName) { + public ThreePidInviteReply(String id, IThreePidInvite invite, String token, String displayName) { + this.id = id; this.invite = invite; this.token = token; this.displayName = displayName; } + @Override + public String getId() { + return id; + } + @Override public IThreePidInvite getInvite() { return invite; diff --git a/src/main/groovy/io/kamax/mxisd/storage/IStorage.java b/src/main/groovy/io/kamax/mxisd/storage/IStorage.java new file mode 100644 index 0000000..b77c50b --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/storage/IStorage.java @@ -0,0 +1,36 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.storage; + +import io.kamax.mxisd.invitation.IThreePidInviteReply; +import io.kamax.mxisd.storage.ormlite.ThreePidInviteIO; + +import java.util.Collection; + +public interface IStorage { + + Collection getInvites(); + + void insertInvite(IThreePidInviteReply data); + + void deleteInvite(String id); + +} diff --git a/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorage.java b/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorage.java new file mode 100644 index 0000000..1a37579 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorage.java @@ -0,0 +1,87 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.storage.ormlite; + +import com.j256.ormlite.dao.CloseableWrappedIterable; +import com.j256.ormlite.dao.Dao; +import com.j256.ormlite.dao.DaoManager; +import com.j256.ormlite.jdbc.JdbcConnectionSource; +import com.j256.ormlite.support.ConnectionSource; +import com.j256.ormlite.table.TableUtils; +import io.kamax.mxisd.invitation.IThreePidInviteReply; +import io.kamax.mxisd.storage.IStorage; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class OrmLiteSqliteStorage implements IStorage { + + private Dao invDao; + + OrmLiteSqliteStorage(String path) { + try { + ConnectionSource connPool = new JdbcConnectionSource("jdbc:sqlite:" + path); + invDao = DaoManager.createDao(connPool, ThreePidInviteIO.class); + TableUtils.createTableIfNotExists(connPool, ThreePidInviteIO.class); + } catch (SQLException e) { + throw new RuntimeException(e); // FIXME do better + } + } + + @Override + public Collection getInvites() { + try (CloseableWrappedIterable t = invDao.getWrappedIterable()) { + List ioList = new ArrayList<>(); + t.forEach(ioList::add); + return ioList; + } catch (IOException e) { + throw new RuntimeException(e); // FIXME do better + } + } + + @Override + public void insertInvite(IThreePidInviteReply data) { + try { + int updated = invDao.create(new ThreePidInviteIO(data)); + if (updated != 1) { + throw new RuntimeException("Unexpected row count after DB action: " + updated); + } + } catch (SQLException e) { + throw new RuntimeException(e); // FIXME do better + } + } + + @Override + public void deleteInvite(String id) { + try { + int updated = invDao.deleteById(id); + if (updated != 1) { + throw new RuntimeException("Unexpected row count after DB action: " + updated); + } + } catch (SQLException e) { + throw new RuntimeException(e); // FIXME do better + } + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorageBeanFactory.java b/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorageBeanFactory.java new file mode 100644 index 0000000..724dada --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/storage/ormlite/OrmLiteSqliteStorageBeanFactory.java @@ -0,0 +1,76 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.storage.ormlite; + +import io.kamax.mxisd.config.SQLiteStorageConfig; +import io.kamax.mxisd.config.StorageConfig; +import io.kamax.mxisd.exception.ConfigurationException; +import io.kamax.mxisd.storage.IStorage; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.FactoryBeanNotInitializedException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class OrmLiteSqliteStorageBeanFactory implements FactoryBean { + + @Autowired + private StorageConfig storagecfg; + + @Autowired + private SQLiteStorageConfig cfg; + + private OrmLiteSqliteStorage storage; + + @PostConstruct + private void postConstruct() { + if (StringUtils.equals("sqlite", storagecfg.getBackend())) { + if (StringUtils.isBlank(cfg.getDatabase())) { + throw new ConfigurationException("storage.provider.sqlite.database"); + } + + storage = new OrmLiteSqliteStorage(cfg.getDatabase()); + } + } + + @Override + public IStorage getObject() throws Exception { + if (storage == null) { + throw new FactoryBeanNotInitializedException(); + } + + return storage; + } + + @Override + public Class getObjectType() { + return OrmLiteSqliteStorage.class; + } + + @Override + public boolean isSingleton() { + return true; + } + +} diff --git a/src/main/groovy/io/kamax/mxisd/storage/ormlite/ThreePidInviteIO.java b/src/main/groovy/io/kamax/mxisd/storage/ormlite/ThreePidInviteIO.java new file mode 100644 index 0000000..91a9198 --- /dev/null +++ b/src/main/groovy/io/kamax/mxisd/storage/ormlite/ThreePidInviteIO.java @@ -0,0 +1,106 @@ +/* + * mxisd - Matrix Identity Server Daemon + * Copyright (C) 2017 Maxime Dor + * + * https://max.kamax.io/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.kamax.mxisd.storage.ormlite; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.j256.ormlite.field.DatabaseField; +import com.j256.ormlite.table.DatabaseTable; +import io.kamax.mxisd.invitation.IThreePidInviteReply; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +@DatabaseTable(tableName = "invite_3pid") +public class ThreePidInviteIO { + + private static Gson gson = new Gson(); + + @DatabaseField(id = true) + private String id; + + @DatabaseField(canBeNull = false) + private String token; + + @DatabaseField(canBeNull = false) + private String sender; + + @DatabaseField(canBeNull = false) + private String medium; + + @DatabaseField(canBeNull = false) + private String address; + + @DatabaseField(canBeNull = false) + private String roomId; + + @DatabaseField + private String properties; + + public ThreePidInviteIO() { + // needed for ORMlite + } + + public ThreePidInviteIO(IThreePidInviteReply data) { + this.id = data.getId(); + this.token = data.getToken(); + this.sender = data.getInvite().getSender().getId(); + this.medium = data.getInvite().getMedium(); + this.address = data.getInvite().getAddress(); + this.roomId = data.getInvite().getRoomId(); + this.properties = gson.toJson(data.getInvite().getProperties()); + } + + public String getId() { + return id; + } + + public String getToken() { + return token; + } + + public String getSender() { + return sender; + } + + public String getMedium() { + return medium; + } + + public String getAddress() { + return address; + } + + public String getRoomId() { + return roomId; + } + + public Map getProperties() { + if (StringUtils.isBlank(properties)) { + return new HashMap<>(); + } + + return gson.fromJson(properties, new TypeToken>() { + }.getType()); + } + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 74984af..3f11732 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -40,3 +40,6 @@ invite: tls: 1 name: "mxisd Identity Server" template: "classpath:email/invite-template.eml" + +storage: + backend: 'sqlite'