diff --git a/online-tests/test_online_login.py b/online-tests/test_online_login.py
new file mode 100644
index 00000000..24d3a57b
--- /dev/null
+++ b/online-tests/test_online_login.py
@@ -0,0 +1,44 @@
+import pytest
+import imaplib
+import smtplib
+
+
+class TestDovecot:
+    def test_login_ok(self, imap, gencreds):
+        user, password = gencreds()
+        imap.connect()
+        imap.login(user, password)
+        # verify it works on another connection
+        imap.connect()
+        imap.login(user, password)
+
+    def test_login_fail(self, imap, gencreds):
+        user, password = gencreds()
+        imap.connect()
+        imap.login(user, password)
+        imap.connect()
+        with pytest.raises(imaplib.IMAP4.error) as excinfo:
+            imap.login(user, password + "wrong")
+        assert "AUTHENTICATIONFAILED" in str(excinfo)
+
+
+class TestPostfix:
+    def test_login_ok(self, smtp, gencreds):
+        user, password = gencreds()
+        smtp.connect()
+        smtp.login(user, password)
+        # verify it works on another connection
+        smtp.connect()
+        smtp.login(user, password)
+
+    def test_login_fail(self, smtp, gencreds):
+        user, password = gencreds()
+        smtp.connect()
+        smtp.login(user, password)
+        smtp.connect()
+        with pytest.raises(smtplib.SMTPAuthenticationError) as excinfo:
+            smtp.login(user, password + "wrong")
+        assert excinfo.value.smtp_code == 535
+        assert "authentication failed" in str(excinfo)
+
+
diff --git a/scripts/test.sh b/scripts/test.sh
index 0bca51c8..ba949357 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -3,4 +3,4 @@ pushd doveauth/src/doveauth
 ../../venv/bin/pytest
 popd
 
-online-tests/venv/bin/pytest online-tests/ -vrx
+online-tests/venv/bin/pytest online-tests/ -vrx --durations=5