--- a/net/Kconfig
+++ b/net/Kconfig
@@ -92,6 +92,9 @@ source "net/netlabel/Kconfig"
 
 endif # if INET
 
+config SOCK_DIAG
+	bool
+
 config NETWORK_SECMARK
 	bool "Security Marking"
 	help
--- a/net/core/Makefile
+++ b/net/core/Makefile
@@ -9,8 +9,9 @@ obj-$(CONFIG_SYSCTL) += sysctl_net_core.
 
 obj-y		     += dev.o ethtool.o dev_addr_lists.o dst.o netevent.o \
 			neighbour.o rtnetlink.o utils.o link_watch.o filter.o \
-			sock_diag.o dev_ioctl.o tso.o
+			dev_ioctl.o tso.o
 
+obj-$(CONFIG_SOCK_DIAG) += sock_diag.o
 obj-$(CONFIG_XFRM) += flow.o
 obj-y += net-sysfs.o
 obj-$(CONFIG_PROC_FS) += net-procfs.o
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1475,9 +1475,11 @@ void sk_destruct(struct sock *sk)
 
 static void __sk_free(struct sock *sk)
 {
+#ifdef CONFIG_SOCK_DIAG
 	if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
 		sock_diag_broadcast_destroy(sk);
 	else
+#endif
 		sk_destruct(sk);
 }
 
--- a/net/ipv4/Kconfig
+++ b/net/ipv4/Kconfig
@@ -416,6 +416,7 @@ config INET_LRO
 
 config INET_DIAG
 	tristate "INET: socket monitoring interface"
+	select SOCK_DIAG
 	default y
 	---help---
 	  Support for INET (TCP, DCCP, etc) socket monitoring interface used by
--- a/net/unix/Kconfig
+++ b/net/unix/Kconfig
@@ -22,6 +22,7 @@ config UNIX
 config UNIX_DIAG
 	tristate "UNIX: socket monitoring interface"
 	depends on UNIX
+	select SOCK_DIAG
 	default n
 	---help---
 	  Support for UNIX socket monitoring interface used by the ss tool.
--- a/net/netlink/Kconfig
+++ b/net/netlink/Kconfig
@@ -4,6 +4,7 @@
 
 config NETLINK_DIAG
 	tristate "NETLINK: socket monitoring interface"
+	select SOCK_DIAG
 	default n
 	---help---
 	  Support for NETLINK socket monitoring interface used by the ss tool.
--- a/net/packet/Kconfig
+++ b/net/packet/Kconfig
@@ -18,6 +18,7 @@ config PACKET
 config PACKET_DIAG
 	tristate "Packet: sockets monitoring interface"
 	depends on PACKET
+	select SOCK_DIAG
 	default n
 	---help---
 	  Support for PF_PACKET sockets monitoring interface used by the ss tool.
