diff --git a/src/udprelay.c b/src/udprelay.c
index 8b1f7f28..9ac63c38 100644
--- a/src/udprelay.c
+++ b/src/udprelay.c
@@ -460,11 +460,14 @@ create_server_socket(const char *host, const char *port)
             ERROR("[udp] setsockopt IP_TRANSPARENT");
             exit(EXIT_FAILURE);
         }
-        if (setsockopt(server_sock, SOL_IP, IP_RECVORIGDSTADDR, &opt, sizeof(opt))) {
-            FATAL("[udp] setsockopt IP_RECVORIGDSTADDR");
-        }
-        if (setsockopt(server_sock, SOL_IPV6, IPV6_RECVORIGDSTADDR, &opt, sizeof(opt))) {
-            FATAL("[udp] setsockopt IPV6_RECVORIGDSTADDR");
+        if (rp->ai_family == AF_INET) {
+            if (setsockopt(server_sock, SOL_IP, IP_RECVORIGDSTADDR, &opt, sizeof(opt))) {
+                FATAL("[udp] setsockopt IP_RECVORIGDSTADDR");
+            }
+        } else if (rp->ai_family == AF_INET6) {
+            if (setsockopt(server_sock, SOL_IPV6, IPV6_RECVORIGDSTADDR, &opt, sizeof(opt))) {
+                FATAL("[udp] setsockopt IPV6_RECVORIGDSTADDR");
+            }
         }
 #endif