[PATCH v2 10/11] doc: Add program to document and test assumptions about SO_REUSEADDR

[David Gibson <david@gibson.dropbear.id.au>]

For the approach we intend to use for handling UDP flows, we have some
pretty specific requirements about how SO_REUSEADDR works with UDP sockets.
Specifically SO_REUSEADDR allows multiple sockets with overlapping bind()s,
and therefore there can be multiple sockets which are eligible to receive
the same datagram.  Which one will actually receive it is important to us.

Add a test program which verifies things work the way we expect, which
documents what those expectations are in the process.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 doc/platform-requirements/.gitignore          |   1 +
 doc/platform-requirements/Makefile            |  45 ++++
 doc/platform-requirements/README              |  18 ++
 doc/platform-requirements/common.c            |  66 +++++
 doc/platform-requirements/common.h            |  47 ++++
 .../reuseaddr-priority.c                      | 240 ++++++++++++++++++
 6 files changed, 417 insertions(+)
 create mode 100644 doc/platform-requirements/.gitignore
 create mode 100644 doc/platform-requirements/Makefile
 create mode 100644 doc/platform-requirements/README
 create mode 100644 doc/platform-requirements/common.c
 create mode 100644 doc/platform-requirements/common.h
 create mode 100644 doc/platform-requirements/reuseaddr-priority.c

diff --git a/doc/platform-requirements/.gitignore b/doc/platform-requirements/.gitignore
new file mode 100644
index 00000000..c1baa98e
--- /dev/null
+++ b/doc/platform-requirements/.gitignore
@@ -0,0 +1 @@
+/reuseaddr-priority
diff --git a/doc/platform-requirements/Makefile b/doc/platform-requirements/Makefile
new file mode 100644
index 00000000..6e1d966c
--- /dev/null
+++ b/doc/platform-requirements/Makefile
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Copyright Red Hat
+# Author: David Gibson <david@gibson.dropbear.id.au>
+
+TARGETS = reuseaddr-priority
+SRCS = reuseaddr-priority.c
+CFLAGS = -Wall
+
+all: cppcheck clang-tidy $(TARGETS:%=check-%)
+
+$(TARGETS): %: %.c common.c common.h
+
+check-%: %
+	./$<
+
+cppcheck:
+	cppcheck --std=c11 --error-exitcode=1 --enable=all --force \
+		--check-level=exhaustive \
+		--inconclusive --library=posix --quiet \
+		--suppress=missingIncludeSystem \
+		$(SRCS)
+
+clang-tidy:
+	clang-tidy --checks=*,\
+	-altera-id-dependent-backward-branch,\
+	-altera-unroll-loops,\
+	-bugprone-easily-swappable-parameters,\
+	-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,\
+	-concurrency-mt-unsafe,\
+	-cppcoreguidelines-avoid-non-const-global-variables,\
+	-cppcoreguidelines-init-variables,\
+	-cppcoreguidelines-macro-to-enum,\
+	-google-readability-braces-around-statements,\
+	-hicpp-braces-around-statements,\
+	-llvmlibc-restrict-system-libc-headers,\
+	-misc-include-cleaner,\
+	-modernize-macro-to-enum,\
+	-readability-braces-around-statements,\
+	-readability-identifier-length,\
+	-readability-isolate-declaration \
+	$(SRCS)
+
+clean:
+	rm -f $(TARGETS) *.o *~
diff --git a/doc/platform-requirements/README b/doc/platform-requirements/README
new file mode 100644
index 00000000..3914d223
--- /dev/null
+++ b/doc/platform-requirements/README
@@ -0,0 +1,18 @@
+Platform Requirements
+=====================
+
+TODO: document the various Linux specific features we currently require
+
+
+Test Programs
+-------------
+
+In some places we rely on quite specific behaviour of sockets.
+Although Linux, at least, seems to behave as required, It's not always
+clear from the available documentation if this is required by POSIX or
+some other specification.
+
+To specifically document those expectations this directory has some
+test programs which explicitly check for the behaviour we need.
+When/if we attempt a port to a new platform, running these to check
+behaviour would be a good place to start.
diff --git a/doc/platform-requirements/common.c b/doc/platform-requirements/common.c
new file mode 100644
index 00000000..d687377a
--- /dev/null
+++ b/doc/platform-requirements/common.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/* common.c
+ *
+ * Common helper functions for testing SO_REUSEADDR behaviour
+ *
+ * Copyright Red Hat
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+
+#include <errno.h>
+#include <netinet/in.h>
+#include <string.h>
+#include <sys/socket.h>
+
+#include "common.h"
+
+int sock_reuseaddr(void)
+{
+	int y = 1;
+	int s;
+	
+
+	s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+	if (s < 0)
+		die("socket(): %s\n", strerror(errno));
+
+	if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &y, sizeof(y)) , 0)
+		die("SO_REUSEADDR: %s\n", strerror(errno));
+
+	return s;
+}
+
+/* Send a token via the given connected socket */
+void send_token(int s, long token)
+{
+	ssize_t rc;
+
+	rc = send(s, &token, sizeof(token), 0);
+	if (rc < 0)
+		die("send(): %s\n", strerror(errno));
+	if (rc < sizeof(token))
+		die("short send()\n");
+}
+
+/* Attempt to receive a token via the given socket.
+ *
+ * Returns true if we received the token, false if we got an EAGAIN, dies in any
+ * other case */
+bool recv_token(int s, long token)
+{
+	ssize_t rc;
+	long buf;
+
+	rc = recv(s, &buf, sizeof(buf), MSG_DONTWAIT);
+	if (rc < 0) {
+		if (errno == EWOULDBLOCK)
+			return false;
+		die("recv(): %s\n", strerror(errno));
+	}
+	if (rc < sizeof(buf))
+		die("short recv()\n");
+	if (buf != token)
+		die("data mismatch\n");
+	return true;
+}
diff --git a/doc/platform-requirements/common.h b/doc/platform-requirements/common.h
new file mode 100644
index 00000000..8844b1ed
--- /dev/null
+++ b/doc/platform-requirements/common.h
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/* common.h
+ *
+ * Useful shared functions
+ *
+ * Copyright Red Hat
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+#ifndef REUSEADDR_COMMON_H
+#define REUSEADDR_COMMON_H
+
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static inline void die(const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	(void)vfprintf(stderr, fmt, ap);
+	va_end(ap);
+	exit(EXIT_FAILURE);
+}
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define htons_constant(x)       (x)
+#define htonl_constant(x)       (x)
+#else
+#define htons_constant(x)       (__bswap_constant_16(x))
+#define htonl_constant(x)       (__bswap_constant_32(x))
+#endif
+
+#define SOCKADDR_INIT(addr, port)					\
+	{								\
+		.sin_family = AF_INET,					\
+		.sin_addr = { .s_addr = htonl_constant(addr) },		\
+		.sin_port = htons_constant(port),			\
+	}
+
+int sock_reuseaddr(void);
+void send_token(int s, long token);
+bool recv_token(int s, long token);
+
+#endif /* REUSEADDR_COMMON_H */
diff --git a/doc/platform-requirements/reuseaddr-priority.c b/doc/platform-requirements/reuseaddr-priority.c
new file mode 100644
index 00000000..644553f8
--- /dev/null
+++ b/doc/platform-requirements/reuseaddr-priority.c
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/* reuseaddr-priority.c
+ *
+ * Verify which SO_REUSEADDR UDP sockets get priority to receive
+ * =============================================================
+ *
+ * SO_REUSEADDR allows multiple sockets to bind to overlapping addresses, so
+ * there can be multiple sockets eligible to receive the same packet.  The exact
+ * semantics of which socket will receive in this circumstance isn't very well
+ * documented.
+ *
+ * This program verifies that things behave the way we expect.  Specifically we
+ * expect:
+ *
+ * - If both a connected and an unconnected socket could receive a datagram, the
+ *   connected one will receive it in preference to the unconnected one.
+ *
+ * - If an unconnected socket bound to a specific address and an unconnected
+ *   socket bound to the "any" address (0.0.0.0 or ::) could receive a datagram,
+ *   then the one with a specific address will receive it in preference to the
+ *   other.
+ *
+ * These should be true regardless of the order the sockets are created in, or
+ * the order they're polled in.
+ *
+ * Copyright Red Hat
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+
+#include <arpa/inet.h>
+#include <errno.h>
+#include <net/if.h>
+#include <netinet/in.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "common.h"
+
+#define SRCPORT	13246U
+#define DSTPORT	13247U
+
+/* Different cases for receiving socket configuration */
+enum sock_type {
+	/* Socket is bound to 0.0.0.0:DSTPORT and not connected */
+	SOCK_BOUND_ANY = 0,
+
+	/* Socket is bound to 127.0.0.1:DSTPORT and not connected */
+	SOCK_BOUND_LO = 1,
+
+	/* Socket is bound to 0.0.0.0:DSTPORT and connected to 127.0.0.1:SRCPORT */
+	SOCK_CONNECTED = 2,
+
+	NUM_SOCK_TYPES,
+};
+
+typedef enum sock_type order_t[NUM_SOCK_TYPES];
+
+static order_t orders[] = {
+	{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0},
+};
+
+/* 127.0.0.2 */
+#define INADDR_LOOPBACK2	((in_addr_t)(0x7f000002))
+
+/* 0.0.0.0:DSTPORT */
+static const struct sockaddr_in any_dst = SOCKADDR_INIT(INADDR_ANY, DSTPORT);
+/* 127.0.0.1:DSTPORT */
+static const struct sockaddr_in lo_dst = SOCKADDR_INIT(INADDR_LOOPBACK, DSTPORT);
+
+/* 127.0.0.2:DSTPORT */
+static const struct sockaddr_in lo2_dst = SOCKADDR_INIT(INADDR_LOOPBACK2, DSTPORT);
+
+/* 127.0.0.1:SRCPORT */
+static const struct sockaddr_in lo_src = SOCKADDR_INIT(INADDR_LOOPBACK, SRCPORT);
+
+/* Random token to send in datagram */
+static long token;
+
+/* Get a socket of the specified type for receiving */
+static int sock_recv(enum sock_type type)
+{
+	const struct sockaddr *connect_sa = NULL;
+	const struct sockaddr *bind_sa = NULL;
+	int s;
+
+	s = sock_reuseaddr();
+
+	switch (type) {
+	case SOCK_CONNECTED:
+		connect_sa = (struct sockaddr *)&lo_src;
+		/* fallthrough */
+	case SOCK_BOUND_ANY:
+		bind_sa = (struct sockaddr *)&any_dst;
+		break;
+
+	case SOCK_BOUND_LO:
+		bind_sa = (struct sockaddr *)&lo_dst;
+		break;
+
+	default:
+		die("bug");
+	}
+
+	if (bind_sa)
+		if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0)
+			die("bind(): %s\n", strerror(errno));
+	if (connect_sa)
+		if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0)
+			die("connect(): %s\n", strerror(errno));
+
+	return s;
+}
+
+/* Get a socket suitable for sending to the given type of receiving socket */
+static int sock_send(enum sock_type type)
+{
+	const struct sockaddr *connect_sa = NULL;
+	const struct sockaddr *bind_sa = NULL;
+	int s;
+
+	s = sock_reuseaddr();
+
+	switch (type) {
+	case SOCK_BOUND_ANY:
+		connect_sa = (struct sockaddr *)&lo2_dst;
+		break;
+
+	case SOCK_CONNECTED:
+		bind_sa = (struct sockaddr *)&lo_src;
+		/* fallthrough */
+	case SOCK_BOUND_LO:
+		connect_sa = (struct sockaddr *)&lo_dst;
+		break;
+
+	default:
+		die("bug");
+	}
+
+	if (bind_sa)
+		if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0)
+			die("bind(): %s\n", strerror(errno));
+	if (connect_sa)
+		if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0)
+			die("connect(): %s\n", strerror(errno));
+
+	return s;
+}
+
+/* Check for expected behaviour with one specific ordering for various operations:
+ *
+ * @recv_create_order:	Order to create receiving sockets in
+ * @send_create_order:	Order to create sending sockets in
+ * @test_order:		Order to test the behaviour of different types
+ * @recv_order:		Order to check the receiving sockets
+ */
+static void check_one_order(const order_t recv_create_order,
+			    const order_t send_create_order,
+			    const order_t test_order,
+			    const order_t recv_order)
+{
+	int rs[NUM_SOCK_TYPES];
+	int ss[NUM_SOCK_TYPES];
+	int nfds = 0;
+	int i, j;
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type t = recv_create_order[i];
+		int s;
+
+		s = sock_recv(t);
+		if (s >= nfds)
+			nfds = s + 1;
+
+		rs[t] = s;
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type t = send_create_order[i];
+
+		ss[t] = sock_send(t);
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type ti = test_order[i];
+		int recv_via = -1;
+
+		send_token(ss[ti], token);
+
+		for (j = 0; j < NUM_SOCK_TYPES; j++) {
+			enum sock_type tj = recv_order[j];
+
+			if (recv_token(rs[tj], token)) {
+				if (recv_via != -1)
+					die("Received token more than once\n");
+				recv_via = tj;
+			}
+		}
+
+		if (recv_via == -1)
+			die("Didn't receive token at all\n");
+		if (recv_via != ti)
+			die("Received token via unexpected socket\n");
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		close(rs[i]);
+		close(ss[i]);
+	}
+}
+
+static void check_all_orders(void)
+{
+	int norders = sizeof(orders) / sizeof(orders[0]);
+	int i, j, k, l;
+
+	for (i = 0; i < norders; i++)
+		for (j = 0; j < norders; j++)
+			for (k = 0; k < norders; k++)
+				for (l = 0; l < norders; l++)
+					check_one_order(orders[i], orders[j],
+							orders[j], orders[l]);
+}
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+	(void)argv;
+
+	token = random();
+
+	check_all_orders();
+
+	printf("SO_REUSEADDR receive priorities seem to work as expected\n");
+
+	exit(0);
+}
-- 
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/* reuseaddr-priority.c
+ *
+ * Verify which SO_REUSEADDR UDP sockets get priority to receive
+ * =============================================================
+ *
+ * SO_REUSEADDR allows multiple sockets to bind to overlapping addresses, so
+ * there can be multiple sockets eligible to receive the same packet.  The exact
+ * semantics of which socket will receive in this circumstance isn't very well
+ * documented.
+ *
+ * This program verifies that things behave the way we expect.  Specifically we
+ * expect:
+ *
+ * - If both a connected and an unconnected socket could receive a datagram, the
+ *   connected one will receive it in preference to the unconnected one.
+ *
+ * - If an unconnected socket bound to a specific address and an unconnected
+ *   socket bound to the "any" address (0.0.0.0 or ::) could receive a datagram,
+ *   then the one with a specific address will receive it in preference to the
+ *   other.
+ *
+ * These should be true regardless of the order the sockets are created in, or
+ * the order they're polled in.
+ *
+ * Copyright Red Hat
+ * Author: David Gibson <david@gibson.dropbear.id.au>
+ */
+
+#include <arpa/inet.h>
+#include <errno.h>
+#include <net/if.h>
+#include <netinet/in.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "common.h"
+
+#define SRCPORT	13246U
+#define DSTPORT	13247U
+
+/* Different cases for receiving socket configuration */
+enum sock_type {
+	/* Socket is bound to 0.0.0.0:DSTPORT and not connected */
+	SOCK_BOUND_ANY = 0,
+
+	/* Socket is bound to 127.0.0.1:DSTPORT and not connected */
+	SOCK_BOUND_LO = 1,
+
+	/* Socket is bound to 0.0.0.0:DSTPORT and connected to 127.0.0.1:SRCPORT */
+	SOCK_CONNECTED = 2,
+
+	NUM_SOCK_TYPES,
+};
+
+typedef enum sock_type order_t[NUM_SOCK_TYPES];
+
+static order_t orders[] = {
+	{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0},
+};
+
+/* 127.0.0.2 */
+#define INADDR_LOOPBACK2	((in_addr_t)(0x7f000002))
+
+/* 0.0.0.0:DSTPORT */
+static const struct sockaddr_in any_dst = SOCKADDR_INIT(INADDR_ANY, DSTPORT);
+/* 127.0.0.1:DSTPORT */
+static const struct sockaddr_in lo_dst = SOCKADDR_INIT(INADDR_LOOPBACK, DSTPORT);
+
+/* 127.0.0.2:DSTPORT */
+static const struct sockaddr_in lo2_dst = SOCKADDR_INIT(INADDR_LOOPBACK2, DSTPORT);
+
+/* 127.0.0.1:SRCPORT */
+static const struct sockaddr_in lo_src = SOCKADDR_INIT(INADDR_LOOPBACK, SRCPORT);
+
+/* Random token to send in datagram */
+static long token;
+
+/* Get a socket of the specified type for receiving */
+static int sock_recv(enum sock_type type)
+{
+	const struct sockaddr *connect_sa = NULL;
+	const struct sockaddr *bind_sa = NULL;
+	int s;
+
+	s = sock_reuseaddr();
+
+	switch (type) {
+	case SOCK_CONNECTED:
+		connect_sa = (struct sockaddr *)&lo_src;
+		/* fallthrough */
+	case SOCK_BOUND_ANY:
+		bind_sa = (struct sockaddr *)&any_dst;
+		break;
+
+	case SOCK_BOUND_LO:
+		bind_sa = (struct sockaddr *)&lo_dst;
+		break;
+
+	default:
+		die("bug");
+	}
+
+	if (bind_sa)
+		if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0)
+			die("bind(): %s\n", strerror(errno));
+	if (connect_sa)
+		if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0)
+			die("connect(): %s\n", strerror(errno));
+
+	return s;
+}
+
+/* Get a socket suitable for sending to the given type of receiving socket */
+static int sock_send(enum sock_type type)
+{
+	const struct sockaddr *connect_sa = NULL;
+	const struct sockaddr *bind_sa = NULL;
+	int s;
+
+	s = sock_reuseaddr();
+
+	switch (type) {
+	case SOCK_BOUND_ANY:
+		connect_sa = (struct sockaddr *)&lo2_dst;
+		break;
+
+	case SOCK_CONNECTED:
+		bind_sa = (struct sockaddr *)&lo_src;
+		/* fallthrough */
+	case SOCK_BOUND_LO:
+		connect_sa = (struct sockaddr *)&lo_dst;
+		break;
+
+	default:
+		die("bug");
+	}
+
+	if (bind_sa)
+		if (bind(s, bind_sa, sizeof(struct sockaddr_in)) < 0)
+			die("bind(): %s\n", strerror(errno));
+	if (connect_sa)
+		if (connect(s, connect_sa, sizeof(struct sockaddr_in)) < 0)
+			die("connect(): %s\n", strerror(errno));
+
+	return s;
+}
+
+/* Check for expected behaviour with one specific ordering for various operations:
+ *
+ * @recv_create_order:	Order to create receiving sockets in
+ * @send_create_order:	Order to create sending sockets in
+ * @test_order:		Order to test the behaviour of different types
+ * @recv_order:		Order to check the receiving sockets
+ */
+static void check_one_order(const order_t recv_create_order,
+			    const order_t send_create_order,
+			    const order_t test_order,
+			    const order_t recv_order)
+{
+	int rs[NUM_SOCK_TYPES];
+	int ss[NUM_SOCK_TYPES];
+	int nfds = 0;
+	int i, j;
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type t = recv_create_order[i];
+		int s;
+
+		s = sock_recv(t);
+		if (s >= nfds)
+			nfds = s + 1;
+
+		rs[t] = s;
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type t = send_create_order[i];
+
+		ss[t] = sock_send(t);
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		enum sock_type ti = test_order[i];
+		int recv_via = -1;
+
+		send_token(ss[ti], token);
+
+		for (j = 0; j < NUM_SOCK_TYPES; j++) {
+			enum sock_type tj = recv_order[j];
+
+			if (recv_token(rs[tj], token)) {
+				if (recv_via != -1)
+					die("Received token more than once\n");
+				recv_via = tj;
+			}
+		}
+
+		if (recv_via == -1)
+			die("Didn't receive token at all\n");
+		if (recv_via != ti)
+			die("Received token via unexpected socket\n");
+	}
+
+	for (i = 0; i < NUM_SOCK_TYPES; i++) {
+		close(rs[i]);
+		close(ss[i]);
+	}
+}
+
+static void check_all_orders(void)
+{
+	int norders = sizeof(orders) / sizeof(orders[0]);
+	int i, j, k, l;
+
+	for (i = 0; i < norders; i++)
+		for (j = 0; j < norders; j++)
+			for (k = 0; k < norders; k++)
+				for (l = 0; l < norders; l++)
+					check_one_order(orders[i], orders[j],
+							orders[j], orders[l]);
+}
+
+int main(int argc, char *argv[])
+{
+	(void)argc;
+	(void)argv;
+
+	token = random();
+
+	check_all_orders();
+
+	printf("SO_REUSEADDR receive priorities seem to work as expected\n");
+
+	exit(0);
+}
-- 
2.45.2