cqrenet/mxids
3
0
Fork 0
Code Pull Requests 10 Actions Releases Activity

Add mechanism for 3PID invites expiration (#120)

Browse Source
This commit is contained in:
Max Dor
2019-03-01 06:51:18 +01:00
parent 96155c1876
commit c302789898
8 changed files with 272 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@@ -101,7 +101,7 @@ dependencies {
compile 'com.j256.ormlite:ormlite-jdbc:5.0'
// ed25519 handling
compile 'net.i2p.crypto:eddsa:0.3.0'
compile 'net.i2p.crypto:eddsa:0.1.0'
// LDAP connector
compile 'org.apache.directory.api:api-all:1.0.0'

44
src/main/java/io/kamax/mxisd/config/InvitationConfig.java
View File

@@ -31,10 +31,42 @@ public class InvitationConfig {
private static final Logger log = LoggerFactory.getLogger(InvitationConfig.class);
public static class Expiration {
private Boolean enabled;
private long after;
private String resolveTo;
public Boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public long getAfter() {
return after;
}
public void setAfter(long after) {
this.after = after;
}
public String getResolveTo() {
return resolveTo;
}
public void setResolveTo(String resolveTo) {
this.resolveTo = resolveTo;
}
}
public static class Resolution {
private boolean recursive = true;
private long timer = 1;
private long timer = 5;
public boolean isRecursive() {
return recursive;
@@ -80,9 +112,18 @@ public class InvitationConfig {
}
}
private Expiration expiration = new Expiration();
private Resolution resolution = new Resolution();
private Policies policy = new Policies();
public Expiration getExpiration() {
return expiration;
}
public void setExpiration(Expiration expiration) {
this.expiration = expiration;
}
public Resolution getResolution() {
return resolution;
}
@@ -101,6 +142,7 @@ public class InvitationConfig {
public void build() {
log.info("--- Invite config ---");
log.info("Expiration: {}", GsonUtil.get().toJson(getExpiration()));
log.info("Resolution: {}", GsonUtil.get().toJson(getResolution()));
log.info("Policies: {}", GsonUtil.get().toJson(getPolicy()));
}

153
src/main/java/io/kamax/mxisd/invitation/InvitationManager.java
View File

@@ -31,6 +31,7 @@ import io.kamax.mxisd.config.MxisdConfig;
import io.kamax.mxisd.config.ServerConfig;
import io.kamax.mxisd.dns.FederationDnsOverwrite;
import io.kamax.mxisd.exception.BadRequestException;
import io.kamax.mxisd.exception.ConfigurationException;
import io.kamax.mxisd.exception.MappingAlreadyExistsException;
import io.kamax.mxisd.exception.ObjectNotFoundException;
import io.kamax.mxisd.lookup.SingleLookupReply;
@@ -41,6 +42,7 @@ import io.kamax.mxisd.profile.ProfileManager;
import io.kamax.mxisd.storage.IStorage;
import io.kamax.mxisd.storage.crypto.*;
import io.kamax.mxisd.storage.ormlite.dao.ThreePidInviteIO;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -63,6 +65,8 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.DateTimeException;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
@@ -70,7 +74,10 @@ import java.util.concurrent.TimeUnit;
public class InvitationManager {
private transient final Logger log = LoggerFactory.getLogger(InvitationManager.class);
private static final Logger log = LoggerFactory.getLogger(InvitationManager.class);
private static final String CreatedAtPropertyKey = "created_at";
private final String defaultCreateTs = Long.toString(Instant.now().toEpochMilli());
private InvitationConfig cfg;
private ServerConfig srvCfg;
@@ -97,7 +104,7 @@ public class InvitationManager {
NotificationManager notifMgr,
ProfileManager profileMgr
) {
this.cfg = mxisdCfg.getInvite();
this.cfg = requireValid(mxisdCfg);
this.srvCfg = mxisdCfg.getServer();
this.storage = storage;
this.lookupMgr = lookupMgr;
@@ -110,6 +117,7 @@ public class InvitationManager {
log.info("Loading saved invites");
Collection<ThreePidInviteIO> ioList = storage.getInvites();
ioList.forEach(io -> {
io.getProperties().putIfAbsent(CreatedAtPropertyKey, defaultCreateTs);
log.info("Processing invite {}", GsonUtil.get().toJson(io));
ThreePidInvite invite = new ThreePidInvite(
MatrixID.asAcceptable(io.getSender()),
@@ -119,7 +127,7 @@ public class InvitationManager {
io.getProperties()
);
ThreePidInviteReply reply = new ThreePidInviteReply(getId(invite), invite, io.getToken(), "", Collections.emptyList());
ThreePidInviteReply reply = new ThreePidInviteReply(io.getId(), invite, io.getToken(), "", Collections.emptyList());
invitations.put(reply.getId(), reply);
});
@@ -136,25 +144,64 @@ public class InvitationManager {
log.info("Setting up invitation mapping refresh timer");
refreshTimer = new Timer();
refreshTimer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
try {
lookupMappingsForInvites();
} catch (Throwable t) {
log.error("Error when running background mapping refresh", t);
}
}
}, 5000L, TimeUnit.MILLISECONDS.convert(cfg.getResolution().getTimer(), TimeUnit.MINUTES));
// We add a shutdown hook to cancel the hook and wait for pending resolutions
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
refreshTimer.cancel();
ForkJoinPool.commonPool().awaitQuiescence(1, TimeUnit.MINUTES);
}));
// We set the refresh timer for background tasks
refreshTimer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
try {
doMaintenance();
} catch (Throwable t) {
log.error("Error when running background maintenance", t);
}
}
}, 5000L, TimeUnit.MILLISECONDS.convert(cfg.getResolution().getTimer(), TimeUnit.MINUTES));
}
private String getId(IThreePidInvite invite) {
return invite.getSender().getDomain().toLowerCase() + invite.getMedium().toLowerCase() + invite.getAddress().toLowerCase();
private InvitationConfig requireValid(MxisdConfig cfg) {
// This is not configured, we'll apply a default configuration
if (Objects.isNull(cfg.getInvite().getExpiration().isEnabled())) {
// We compute our own user, so it can be used if we bridge as well
String mxId = MatrixID.asAcceptable("_mxisd-expired_invite", cfg.getMatrix().getDomain()).getId();
// Enabled by default
cfg.getInvite().getExpiration().setEnabled(true);
// We'll resolve to our computed User ID
cfg.getInvite().getExpiration().setResolveTo(mxId);
// One calendar week (60min/1h * 24 = 1d * 7 = 1w)
cfg.getInvite().getExpiration().setAfter(60 * 24 * 7);
}
if (cfg.getInvite().getExpiration().isEnabled()) {
if (cfg.getInvite().getExpiration().getAfter() < 1) {
throw new ConfigurationException("Invitation expiration delay must be greater or equal to 1");
}
if (StringUtils.isBlank(cfg.getInvite().getExpiration().getResolveTo())) {
throw new ConfigurationException("Invitation expiration resolution target cannot be empty/blank");
}
try {
MatrixID.asAcceptable(cfg.getInvite().getExpiration().getResolveTo());
} catch (IllegalArgumentException e) {
throw new ConfigurationException("Invitation expiration resolution target is not a valid Matrix ID: " + e.getMessage());
}
}
return cfg.getInvite();
}
private String computeId(IThreePidInvite invite) {
String rawId = invite.getSender().getDomain().toLowerCase() + invite.getMedium().toLowerCase() + invite.getAddress().toLowerCase();
return Base64.encodeBase64URLSafeString(rawId.getBytes(StandardCharsets.UTF_8));
}
private String getIdForLog(IThreePidInviteReply reply) {
@@ -248,7 +295,7 @@ public class InvitationManager {
throw new BadRequestException("Medium type " + invitation.getMedium() + " is not supported");
}
String invId = getId(invitation);
String invId = computeId(invitation);
log.info("Handling invite for {}:{} from {} in room {}", invitation.getMedium(), invitation.getAddress(), invitation.getSender(), invitation.getRoomId());
IThreePidInviteReply reply = invitations.get(invId);
if (reply != null) {
@@ -276,6 +323,7 @@ public class InvitationManager {
String pPubKey = keyMgr.getPublicKeyBase64(pKeyId);
String ePubKey = keyMgr.getPublicKeyBase64(eKeyId);
invitation.getProperties().put(CreatedAtPropertyKey, Long.toString(Instant.now().toEpochMilli()));
invitation.getProperties().put("p_key_algo", pKeyId.getAlgorithm());
invitation.getProperties().put("p_key_serial", pKeyId.getSerial());
invitation.getProperties().put("p_key_public", pPubKey);
@@ -312,6 +360,58 @@ public class InvitationManager {
return false;
}
private void removeInvite(IThreePidInviteReply reply) {
invitations.remove(reply.getId());
storage.deleteInvite(reply.getId());
}
/**
* Trigger the periodic maintenance tasks
*/
public void doMaintenance() {
lookupMappingsForInvites();
expireInvites();
}
public void expireInvites() {
log.debug("Invite expiration: started");
if (!cfg.getExpiration().isEnabled()) {
log.debug("Invite expiration is disabled, skipping");
return;
}
if (invitations.isEmpty()) {
log.debug("No invite to expired, skipping");
return;
}
String targetMxid = cfg.getExpiration().getResolveTo();
for (IThreePidInviteReply reply : invitations.values()) {
log.debug("Processing invite {}", reply.getId());
String tsRaw = reply.getInvite().getProperties().computeIfAbsent(CreatedAtPropertyKey, k -> defaultCreateTs);
try {
Instant ts = Instant.ofEpochMilli(Long.parseLong(tsRaw));
Instant targetTs = ts.plusSeconds(cfg.getExpiration().getAfter() * 60);
Instant now = Instant.now();
log.debug("Invite {} - Created at {} - Expire at {} - Current time is {}", reply.getId(), ts, targetTs, now);
if (targetTs.isBefore(Instant.now())) {
log.debug("Invite {} has not expired yet, skipping", reply.getId());
continue;
}
log.info("Invite {} has expired at TS {} - Expiring and resolving to {}", targetTs, targetMxid);
publishMapping(reply, targetMxid);
} catch (NumberFormatException | DateTimeException e) {
log.warn("Invite {} has an invalid creation TS, setting to default value of {}", reply.getId(), defaultCreateTs);
reply.getInvite().getProperties().put(CreatedAtPropertyKey, defaultCreateTs);
}
}
log.debug("Invite expiration: finished");
}
public void lookupMappingsForInvites() {
if (!invitations.isEmpty()) {
log.info("Checking for existing mapping for pending invites");
@@ -391,25 +491,32 @@ public class InvitationManager {
StringEntity entity = new StringEntity(content.toString(), StandardCharsets.UTF_8);
entity.setContentType("application/json");
req.setEntity(entity);
Instant resolvedAt = Instant.now();
boolean couldPublish = false;
try {
log.info("Posting onBind event to {}", req.getURI());
CloseableHttpResponse response = client.execute(req);
int statusCode = response.getStatusLine().getStatusCode();
log.info("Answer code: {}", statusCode);
if (statusCode >= 300 && statusCode != 403) {
log.warn("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8));
log.info("Answer body: {}", IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8));
log.warn("HS returned an error. Invite can be found in historical storage for manual re-processing");
} else {
couldPublish = true;
if (statusCode == 403) {
log.info("Invite was obsolete");
log.info("Invite is obsolete or no longer under our control");
}
invitations.remove(getId(reply.getInvite()));
storage.deleteInvite(reply.getId());
log.info("Removed invite from internal store");
}
response.close();
} catch (IOException e) {
log.warn("Unable to tell HS {} about invite being mapped", domain, e);
} finally {
synchronized (this) {
storage.insertHistoricalInvite(reply, mxid, resolvedAt, couldPublish);
removeInvite(reply);
log.info("Moved invite {} to historical table", reply.getId());
}
}
}).start();
}
@@ -425,7 +532,7 @@ public class InvitationManager {
@Override
public void run() {
try {
log.info("Searching for mapping created since invite {} was created", getIdForLog(reply));
log.info("Searching for mapping created after invite {} was created", getIdForLog(reply));
Optional<SingleLookupReply> result = lookup3pid(reply.getInvite().getMedium(), reply.getInvite().getAddress());
if (result.isPresent()) {
SingleLookupReply lookup = result.get();

2
src/main/java/io/kamax/mxisd/storage/IStorage.java
View File

@@ -38,6 +38,8 @@ public interface IStorage {
void deleteInvite(String id);
void insertHistoricalInvite(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish);
Optional<IThreePidSessionDao> getThreePidSession(String sid);
Optional<IThreePidSessionDao> findThreePidSession(ThreePid tpid, String secret);

4
src/main/java/io/kamax/mxisd/storage/crypto/Ed25519KeyManager.java
View File

@@ -47,7 +47,7 @@ public class Ed25519KeyManager implements KeyManager {
private final KeyStore store;
public Ed25519KeyManager(KeyStore store) {
this.keySpecs = EdDSANamedCurveTable.getByName(EdDSANamedCurveTable.ED_25519);
this.keySpecs = EdDSANamedCurveTable.getByName(EdDSANamedCurveTable.CURVE_ED25519_SHA512);
this.store = store;
if (!store.getCurrentKey().isPresent()) {
@@ -106,7 +106,7 @@ public class Ed25519KeyManager implements KeyManager {
}
public EdDSAPrivateKeySpec getPrivateKeySpecs(KeyIdentifier id) {
return new EdDSAPrivateKeySpec(java.util.Base64.getDecoder().decode(getKey(id).getPrivateKeyBase64()), keySpecs);
return new EdDSAPrivateKeySpec(Base64.decodeBase64(getKey(id).getPrivateKeyBase64()), keySpecs);
}
public EdDSAPrivateKey getPrivateKey(KeyIdentifier id) {

2
src/main/java/io/kamax/mxisd/storage/crypto/Ed25519SignatureManager.java
View File

@@ -43,7 +43,6 @@ public class Ed25519SignatureManager implements SignatureManager {
Signature sign = sign(message);
JsonObject keySignature = new JsonObject();
// FIXME should create a signing key object what would give this ed and index values
keySignature.addProperty(sign.getKey().getAlgorithm() + ":" + sign.getKey().getSerial(), sign.getSignature());
JsonObject signature = new JsonObject();
signature.add(domain, keySignature);
@@ -53,7 +52,6 @@ public class Ed25519SignatureManager implements SignatureManager {
@Override
public Signature sign(JsonObject obj) {
return sign(MatrixJson.encodeCanonical(obj));
}

30
src/main/java/io/kamax/mxisd/storage/ormlite/OrmLiteSqlStorage.java
View File

@@ -34,24 +34,18 @@ import io.kamax.mxisd.invitation.IThreePidInviteReply;
import io.kamax.mxisd.storage.IStorage;
import io.kamax.mxisd.storage.dao.IThreePidSessionDao;
import io.kamax.mxisd.storage.ormlite.dao.ASTransactionDao;
import io.kamax.mxisd.storage.ormlite.dao.HistoricalThreePidInviteIO;
import io.kamax.mxisd.storage.ormlite.dao.ThreePidInviteIO;
import io.kamax.mxisd.storage.ormlite.dao.ThreePidSessionDao;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.SQLException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.*;
public class OrmLiteSqlStorage implements IStorage {
private transient final Logger log = LoggerFactory.getLogger(OrmLiteSqlStorage.class);
@FunctionalInterface
private interface Getter<T> {
@@ -67,6 +61,7 @@ public class OrmLiteSqlStorage implements IStorage {
}
private Dao<ThreePidInviteIO, String> invDao;
private Dao<HistoricalThreePidInviteIO, String> expInvDao;
private Dao<ThreePidSessionDao, String> sessionDao;
private Dao<ASTransactionDao, String> asTxnDao;
@@ -86,6 +81,7 @@ public class OrmLiteSqlStorage implements IStorage {
withCatcher(() -> {
ConnectionSource connPool = new JdbcConnectionSource("jdbc:" + backend + ":" + path);
invDao = createDaoAndTable(connPool, ThreePidInviteIO.class);
expInvDao = createDaoAndTable(connPool, HistoricalThreePidInviteIO.class);
sessionDao = createDaoAndTable(connPool, ThreePidSessionDao.class);
asTxnDao = createDaoAndTable(connPool, ASTransactionDao.class);
});
@@ -150,6 +146,24 @@ public class OrmLiteSqlStorage implements IStorage {
});
}
@Override
public void insertHistoricalInvite(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish) {
withCatcher(() -> {
HistoricalThreePidInviteIO io = new HistoricalThreePidInviteIO(data, resolvedTo, resolvedAt, couldPublish);
int updated = expInvDao.create(io);
if (updated != 1) {
throw new RuntimeException("Unexpected row count after DB action: " + updated);
}
// Ugly, but it avoids touching the structure of the historical parent class
// and avoid any possible regression at this point.
updated = expInvDao.updateId(io, UUID.randomUUID().toString().replace("-", ""));
if (updated != 1) {
throw new RuntimeException("Unexpected row count after DB action: " + updated);
}
});
}
@Override
public Optional<IThreePidSessionDao> getThreePidSession(String sid) {
return withCatcher(() -> Optional.ofNullable(sessionDao.queryForId(sid)));

72
src/main/java/io/kamax/mxisd/storage/ormlite/dao/HistoricalThreePidInviteIO.java Normal file
View File

@@ -0,0 +1,72 @@
/*
* mxisd - Matrix Identity Server Daemon
* Copyright (C) 2019 Kamax Sarl
*
* https://www.kamax.io/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package io.kamax.mxisd.storage.ormlite.dao;
import com.j256.ormlite.field.DatabaseField;
import com.j256.ormlite.table.DatabaseTable;
import io.kamax.mxisd.invitation.IThreePidInviteReply;
import java.time.Instant;
@DatabaseTable(tableName = "invite_3pid_history")
public class HistoricalThreePidInviteIO extends ThreePidInviteIO {
@DatabaseField(canBeNull = false)
private String resolvedTo;
@DatabaseField(canBeNull = false)
private long resolvedAt;
@DatabaseField(canBeNull = false)
private boolean couldPublish;
@DatabaseField(canBeNull = false)
private long publishAttempts = 1; // Placeholder for retry mechanism, if ever implemented
public HistoricalThreePidInviteIO() {
// Needed for ORMLite
}
public HistoricalThreePidInviteIO(IThreePidInviteReply data, String resolvedTo, Instant resolvedAt, boolean couldPublish) {
super(data);
this.resolvedTo = resolvedTo;
this.resolvedAt = resolvedAt.toEpochMilli();
this.couldPublish = couldPublish;
}
public String getResolvedTo() {
return resolvedTo;
}
public Instant getResolvedAt() {
return Instant.ofEpochMilli(resolvedAt);
}
public boolean isCouldPublish() {
return couldPublish;
}
public long getPublishAttempts() {
return publishAttempts;
}
}