diff --git a/pretix_matrix_inviter/tasks.py b/pretix_matrix_inviter/tasks.py
index ce199012880b5f963def99c439002f6d96bb27d3..1a3f73709f014bdf1511171b9ee0d5675a616fcd 100644
--- a/pretix_matrix_inviter/tasks.py
+++ b/pretix_matrix_inviter/tasks.py
@@ -49,23 +49,36 @@ def matrix_inviter_invite(
         )
         r.raise_for_status()
     except (requests.ConnectionError, requests.HTTPError) as e:
-        try:
-            self.retry()
-        except MaxRetriesExceededError:
+        if r.status_code in (400, 403):
+            response_dump = r.json()
             order.log_action(
                 "pretix_matrix_inviter.error",
                 data={
                     "matrix_id": user_matrix_id,
                     "matrix_room": room_matrix_id,
-                    "error": "HTTP Code {}".format(r.status_code),
+                    "error": "HTTP Code {} ({})".format(
+                        r.status_code, response_dump["error"]
+                    ),
                 },
             )
-            raise e
-
-    order.log_action(
-        "pretix_matrix_inviter.invite_sent",
-        data={
-            "matrix_id": user_matrix_id,
-            "matrix_room": room_matrix_id,
-        },
-    )
+        else:
+            try:
+                self.retry()
+            except MaxRetriesExceededError:
+                order.log_action(
+                    "pretix_matrix_inviter.error",
+                    data={
+                        "matrix_id": user_matrix_id,
+                        "matrix_room": room_matrix_id,
+                        "error": "HTTP Code {}".format(r.status_code),
+                    },
+                )
+                raise e
+    else:
+        order.log_action(
+            "pretix_matrix_inviter.invite_sent",
+            data={
+                "matrix_id": user_matrix_id,
+                "matrix_room": room_matrix_id,
+            },
+        )