diff --git a/pretix_matrix_inviter/tasks.py b/pretix_matrix_inviter/tasks.py
index 1a3f73709f014bdf1511171b9ee0d5675a616fcd..0756c20341ca7ad4bc3740de0edbe72b25e9d597 100644
--- a/pretix_matrix_inviter/tasks.py
+++ b/pretix_matrix_inviter/tasks.py
@@ -63,7 +63,12 @@ def matrix_inviter_invite(
             )
         else:
             try:
-                self.retry()
+                if r.status_code == 429:
+                    response_dump = r.json()
+                    backoff = response_dump["retry_after_ms"] / 1000
+                    self.retry(countdown=backoff)
+                else:
+                    self.retry()
             except MaxRetriesExceededError:
                 order.log_action(
                     "pretix_matrix_inviter.error",