[PATCH] Implement kqueue(2) support via CONFIG_ELOOP_KQUEUE

Roy Marples roy at marples.name
Wed Jan 20 09:17:10 PST 2016


NOTE: kqueue has to be closed and re-build after forking.
epoll *should* do the same, but it seems that wpa_supplicant doesn't
need it at least.

I have re-worked a little bit of the epoll code (moved into a similar
kqueue function) so it's trivial to requeue epoll if needed in the future.

Roy
-------------- next part --------------
commit 785bd6035f31b2918050df2a08aa07fb23dcb30b
Author: Roy Marples <roy at marples.name>
Date:   Wed Jan 20 17:10:08 2016 +0000

    Implement kqueue(2) support for eloop via CONFIG_ELOOP_QUEUE.

diff --git a/hostapd/Makefile b/hostapd/Makefile
index 45afedf..fd3105e 100644
--- a/hostapd/Makefile
+++ b/hostapd/Makefile
@@ -140,6 +140,10 @@ ifdef CONFIG_ELOOP_EPOLL
 CFLAGS += -DCONFIG_ELOOP_EPOLL
 endif
 
+ifdef CONFIG_ELOOP_KQUEUE
+CFLAGS += -DCONFIG_ELOOP_KQUEUE
+endif
+
 OBJS += ../src/utils/common.o
 OBJS_c += ../src/utils/common.o
 OBJS += ../src/utils/wpa_debug.o
diff --git a/hostapd/hostapd_cli.c b/hostapd/hostapd_cli.c
index 52dfc9f..7e6ac23 100644
--- a/hostapd/hostapd_cli.c
+++ b/hostapd/hostapd_cli.c
@@ -1444,7 +1444,7 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	if (daemonize && os_daemonize(pid_file))
+	if (daemonize && os_daemonize(pid_file) && eloop_sock_requeue())
 		return -1;
 
 	if (interactive)
diff --git a/hostapd/main.c b/hostapd/main.c
index 4913cbd..1b9002c 100644
--- a/hostapd/main.c
+++ b/hostapd/main.c
@@ -408,9 +408,16 @@ static int hostapd_global_run(struct hapd_interfaces *ifaces, int daemonize,
 	}
 #endif /* EAP_SERVER_TNC */
 
-	if (daemonize && os_daemonize(pid_file)) {
-		wpa_printf(MSG_ERROR, "daemon: %s", strerror(errno));
-		return -1;
+	if (daemonize) {
+		if (os_daemonize(pid_file)) {
+			wpa_printf(MSG_ERROR, "daemon: %s", strerror(errno));
+			return -1;
+		}
+		if (eloop_sock_requeue()) {
+			wpa_printf(MSG_ERROR, "eloop_sock_requeue: %s",
+				   strerror(errno));
+			return -1;
+		}
 	}
 
 	eloop_run();
diff --git a/src/utils/eloop.c b/src/utils/eloop.c
index 8647229..656f001 100644
--- a/src/utils/eloop.c
+++ b/src/utils/eloop.c
@@ -18,7 +18,12 @@
 #error Do not define both of poll and epoll
 #endif
 
-#if !defined(CONFIG_ELOOP_POLL) && !defined(CONFIG_ELOOP_EPOLL)
+#if defined(CONFIG_ELOOP_POLL) && defined(CONFIG_ELOOP_KQUEUE)
+#error Do not define both of poll and kqueue
+#endif
+
+#if !defined(CONFIG_ELOOP_POLL) && !defined(CONFIG_ELOOP_EPOLL) && \
+    !defined(CONFIG_ELOOP_KQUEUE)
 #define CONFIG_ELOOP_SELECT
 #endif
 
@@ -30,6 +35,10 @@
 #include <sys/epoll.h>
 #endif /* CONFIG_ELOOP_EPOLL */
 
+#ifdef CONFIG_ELOOP_KQUEUE
+#include <sys/event.h>
+#endif /* CONFIG_ELOOP_KQUEUE */
+
 struct eloop_sock {
 	int sock;
 	void *eloop_data;
@@ -75,13 +84,20 @@ struct eloop_data {
 	struct pollfd *pollfds;
 	struct pollfd **pollfds_map;
 #endif /* CONFIG_ELOOP_POLL */
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
+	int max_fd;
+	struct eloop_sock *fd_table;
+#endif
 #ifdef CONFIG_ELOOP_EPOLL
 	int epollfd;
 	int epoll_max_event_num;
-	int epoll_max_fd;
-	struct eloop_sock *epoll_table;
 	struct epoll_event *epoll_events;
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	int kqueuefd;
+	int kqueue_nevents;
+	struct kevent *kqueue_events;
+#endif /* CONFIG_ELOOP_KQUEUE */
 	struct eloop_sock_table readers;
 	struct eloop_sock_table writers;
 	struct eloop_sock_table exceptions;
@@ -153,26 +169,95 @@ int eloop_init(void)
 			   __func__, strerror(errno));
 		return -1;
 	}
+#endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	eloop.kqueuefd = kqueue();
+	if (eloop.kqueuefd < 0) {
+		wpa_printf(MSG_ERROR, "%s: kqueue failed. %s\n",
+			   __func__, strerror(errno));
+		return -1;
+	}
+#endif /* CONFIG_ELOOP_KQUEUE */
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
 	eloop.readers.type = EVENT_TYPE_READ;
 	eloop.writers.type = EVENT_TYPE_WRITE;
 	eloop.exceptions.type = EVENT_TYPE_EXCEPTION;
-#endif /* CONFIG_ELOOP_EPOLL */
+#endif
 #ifdef WPA_TRACE
 	signal(SIGSEGV, eloop_sigsegv_handler);
 #endif /* WPA_TRACE */
 	return 0;
 }
 
+#ifdef CONFIG_ELOOP_EPOLL
+static int eloop_sock_queue(int sock, eloop_event_type type)
+{
+	struct epoll_event ev;
+
+	os_memset(&ev, 0, sizeof(ev));
+	switch (type) {
+	case EVENT_TYPE_READ:
+		ev.events = EPOLLIN;
+		break;
+	case EVENT_TYPE_WRITE:
+		ev.events = EPOLLOUT;
+		break;
+	/*
+	 * Exceptions are always checked when using epoll, but I suppose it's
+	 * possible that someone registered a socket *only* for exception
+	 * handling.
+	 */
+	case EVENT_TYPE_EXCEPTION:
+		ev.events = EPOLLERR | EPOLLHUP;
+		break;
+	}
+	ev.data.fd = sock;
+	if (epoll_ctl(eloop.epollfd, EPOLL_CTL_ADD, sock, &ev) < 0) {
+		wpa_printf(MSG_ERROR, "%s: epoll_ctl(ADD) for fd=%d "
+			   "failed. %s\n", __func__, sock, strerror(errno));
+		return -1;
+	}
+	return 0;
+}
+#endif /* CONFIG_ELOOP_EPOLL */
+
+#ifdef CONFIG_ELOOP_KQUEUE
+static int eloop_sock_queue(int sock, eloop_event_type type)
+{
+	int filter;
+	struct kevent ke;
+
+	switch (type) {
+	case EVENT_TYPE_READ:
+		filter = EVFILT_READ;
+		break;
+	case EVENT_TYPE_WRITE:
+		filter = EVFILT_WRITE;
+		break;
+	default:
+		filter = 0;
+	}
+	EV_SET(&ke, sock, filter, EV_ADD, 0, 0, NULL);
+	if (kevent(eloop.kqueuefd, &ke, 1, NULL, 0, NULL) == -1) {
+		wpa_printf(MSG_ERROR, "%s: kevent(ADD) for fd=%d "
+			   "failed. %s\n", __func__, sock, strerror(errno));
+		return -1;
+	}
+	return 0;
+}
+#endif /* CONFIG_ELOOP_KQUEUE */
 
 static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
                                      int sock, eloop_sock_handler handler,
                                      void *eloop_data, void *user_data)
 {
 #ifdef CONFIG_ELOOP_EPOLL
+	struct epoll_event *temp_events;
+#endif /* CONFIG_ELOOP_EPOLL */
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
 	struct eloop_sock *temp_table;
-	struct epoll_event ev, *temp_events;
 	int next;
-#endif /* CONFIG_ELOOP_EPOLL */
+#endif
 	struct eloop_sock *tmp;
 	int new_max_sock;
 
@@ -208,18 +293,20 @@ static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
 		eloop.pollfds = n;
 	}
 #endif /* CONFIG_ELOOP_POLL */
-#ifdef CONFIG_ELOOP_EPOLL
-	if (new_max_sock >= eloop.epoll_max_fd) {
-		next = eloop.epoll_max_fd == 0 ? 16 : eloop.epoll_max_fd * 2;
-		temp_table = os_realloc_array(eloop.epoll_table, next,
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
+	if (new_max_sock >= eloop.max_fd) {
+		next = eloop.max_fd == 0 ? 16 : eloop.max_fd * 2;
+		temp_table = os_realloc_array(eloop.fd_table, next,
 					      sizeof(struct eloop_sock));
 		if (temp_table == NULL)
 			return -1;
 
-		eloop.epoll_max_fd = next;
-		eloop.epoll_table = temp_table;
+		eloop.max_fd = next;
+		eloop.fd_table = temp_table;
 	}
+#endif
 
+#ifdef CONFIG_ELOOP_EPOLL
 	if (eloop.count + 1 > eloop.epoll_max_event_num) {
 		next = eloop.epoll_max_event_num == 0 ? 8 :
 			eloop.epoll_max_event_num * 2;
@@ -235,6 +322,21 @@ static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
 		eloop.epoll_events = temp_events;
 	}
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	if (eloop.count + 1 > eloop.kqueue_nevents) {
+		next = eloop.kqueue_nevents == 0 ? 8 : eloop.kqueue_nevents * 2;
+		os_free(eloop.kqueue_events);
+		eloop.kqueue_events = os_malloc(next *
+					        sizeof(eloop.kqueue_events));
+		if (eloop.kqueue_events == NULL) {
+			wpa_printf(MSG_ERROR, "%s: malloc for kqueue failed. "
+				   "%s\n", __func__, strerror(errno));
+			return -1;
+		}
+
+		eloop.kqueue_nevents = next;
+	}
+#endif /* CONFIG_ELOOP_KQUEUE */
 
 	eloop_trace_sock_remove_ref(table);
 	tmp = os_realloc_array(table->table, table->count + 1,
@@ -256,33 +358,12 @@ static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
 	table->changed = 1;
 	eloop_trace_sock_add_ref(table);
 
-#ifdef CONFIG_ELOOP_EPOLL
-	os_memset(&ev, 0, sizeof(ev));
-	switch (table->type) {
-	case EVENT_TYPE_READ:
-		ev.events = EPOLLIN;
-		break;
-	case EVENT_TYPE_WRITE:
-		ev.events = EPOLLOUT;
-		break;
-	/*
-	 * Exceptions are always checked when using epoll, but I suppose it's
-	 * possible that someone registered a socket *only* for exception
-	 * handling.
-	 */
-	case EVENT_TYPE_EXCEPTION:
-		ev.events = EPOLLERR | EPOLLHUP;
-		break;
-	}
-	ev.data.fd = sock;
-	if (epoll_ctl(eloop.epollfd, EPOLL_CTL_ADD, sock, &ev) < 0) {
-		wpa_printf(MSG_ERROR, "%s: epoll_ctl(ADD) for fd=%d "
-			   "failed. %s\n", __func__, sock, strerror(errno));
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
+	if (eloop_sock_queue(sock, table->type) == -1)
 		return -1;
-	}
-	os_memcpy(&eloop.epoll_table[sock], &table->table[table->count - 1],
+	os_memcpy(&eloop.fd_table[sock], &table->table[table->count - 1],
 		  sizeof(struct eloop_sock));
-#endif /* CONFIG_ELOOP_EPOLL */
+#endif
 	return 0;
 }
 
@@ -290,6 +371,9 @@ static int eloop_sock_table_add_sock(struct eloop_sock_table *table,
 static void eloop_sock_table_remove_sock(struct eloop_sock_table *table,
                                          int sock)
 {
+#ifdef CONFIG_ELOOP_KQUEUE
+	struct kevent ke;
+#endif
 	int i;
 
 	if (table == NULL || table->table == NULL || table->count == 0)
@@ -317,8 +401,17 @@ static void eloop_sock_table_remove_sock(struct eloop_sock_table *table,
 			   "failed. %s\n", __func__, sock, strerror(errno));
 		return;
 	}
-	os_memset(&eloop.epoll_table[sock], 0, sizeof(struct eloop_sock));
+	os_memset(&eloop.fd_table[sock], 0, sizeof(struct eloop_sock));
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	EV_SET(&ke, sock, 0, EV_DELETE, 0, 0, NULL);
+	if (kevent(eloop.kqueuefd, &ke, 1, NULL, 0, NULL) == -1) {
+		wpa_printf(MSG_ERROR, "%s: kevent(DEL) for fd=%d "
+			   "failed. %s\n", __func__, sock, strerror(errno));
+		return;
+	}
+	os_memset(&eloop.fd_table[sock], 0, sizeof(struct eloop_sock));
+#endif /* CONFIG_ELOOP_KQUEUE */
 }
 
 
@@ -511,7 +604,7 @@ static void eloop_sock_table_dispatch(struct epoll_event *events, int nfds)
 	int i;
 
 	for (i = 0; i < nfds; i++) {
-		table = &eloop.epoll_table[events[i].data.fd];
+		table = &eloop.fd_table[events[i].data.fd];
 		if (table->handler == NULL)
 			continue;
 		table->handler(table->sock, table->eloop_data,
@@ -525,6 +618,67 @@ static void eloop_sock_table_dispatch(struct epoll_event *events, int nfds)
 #endif /* CONFIG_ELOOP_EPOLL */
 
 
+#ifdef CONFIG_ELOOP_KQUEUE
+static void eloop_sock_table_dispatch(struct kevent *events, int nfds)
+{
+	struct eloop_sock *table;
+	int i;
+
+	for (i = 0; i < nfds; i++) {
+		table = &eloop.fd_table[events[i].ident];
+		if (table->handler == NULL)
+			continue;
+		table->handler(table->sock, table->eloop_data,
+			       table->user_data);
+		if (eloop.readers.changed ||
+		    eloop.writers.changed ||
+		    eloop.exceptions.changed)
+			break;
+	}
+}
+
+static int eloop_sock_table_requeue(struct eloop_sock_table *table)
+{
+	int i, r;
+	struct kevent ke;
+
+	r = 0;
+	for (i = 0; i < table->count && table->table; i++) {
+		if (eloop_sock_queue(table->table[i].sock, table->type) == -1)
+			r = -1;
+	}
+	return r;
+}
+
+int eloop_sock_requeue(void)
+{
+	int r = 0;
+
+	close(eloop.kqueuefd);
+	eloop.kqueuefd = kqueue();
+	if (eloop.kqueuefd < 0) {
+		wpa_printf(MSG_ERROR, "%s: kqueue failed. %s\n",
+			   __func__, strerror(errno));
+		return -1;
+	}
+
+	if (eloop_sock_table_requeue(&eloop.readers) == -1)
+		r = -1;
+	if (eloop_sock_table_requeue(&eloop.writers) == -1)
+		r = -1;
+	if (eloop_sock_table_requeue(&eloop.exceptions) == -1)
+		r = -1;
+
+	return r;
+}
+#else /* CONFIG_ELOOP_KQUEUE */
+int eloop_sock_requeue(void)
+{
+
+	return 0;
+}
+#endif /* !CONFIG_ELOOP_KQUEUE */
+
 static void eloop_sock_table_destroy(struct eloop_sock_table *table)
 {
 	if (table) {
@@ -905,6 +1059,9 @@ void eloop_run(void)
 #ifdef CONFIG_ELOOP_EPOLL
 	int timeout_ms = -1;
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	struct timespec ts;
+#endif /* CONFIG_ELOOP_KQUEUE */
 	int res;
 	struct os_reltime tv, now;
 
@@ -949,6 +1106,10 @@ void eloop_run(void)
 			_tv.tv_sec = tv.sec;
 			_tv.tv_usec = tv.usec;
 #endif /* CONFIG_ELOOP_SELECT */
+#ifdef CONFIG_ELOOP_KQUEUE
+			ts.tv_sec = tv.sec;
+			ts.tv_nsec = tv.usec * 1000L;
+#endif /* CONFIG_ELOOP_KQUEUE */
 		}
 
 #ifdef CONFIG_ELOOP_POLL
@@ -974,6 +1135,15 @@ void eloop_run(void)
 					 eloop.count, timeout_ms);
 		}
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+		if (eloop.count == 0) {
+			res = 0;
+		} else {
+			res = kevent(eloop.kqueuefd, NULL, 0,
+				     eloop.kqueue_events, eloop.kqueue_nevents,
+				     timeout ? &ts : NULL);
+		}
+#endif /* CONFIG_ELOOP_KQUEUE */
 		if (res < 0 && errno != EINTR && errno != 0) {
 			wpa_printf(MSG_ERROR, "eloop: %s: %s",
 #ifdef CONFIG_ELOOP_POLL
@@ -985,6 +1155,10 @@ void eloop_run(void)
 #ifdef CONFIG_ELOOP_EPOLL
 				   "epoll"
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+				   "kqueue"
+#endif /* CONFIG_ELOOP_EKQUEUE */
+
 				   , strerror(errno));
 			goto out;
 		}
@@ -1040,6 +1214,9 @@ void eloop_run(void)
 #ifdef CONFIG_ELOOP_EPOLL
 		eloop_sock_table_dispatch(eloop.epoll_events, res);
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+		eloop_sock_table_dispatch(eloop.kqueue_events, res);
+#endif /* CONFIG_ELOOP_KQUEUE */
 	}
 
 	eloop.terminate = 0;
@@ -1092,11 +1269,17 @@ void eloop_destroy(void)
 	os_free(eloop.pollfds);
 	os_free(eloop.pollfds_map);
 #endif /* CONFIG_ELOOP_POLL */
+#if defined(CONFIG_ELOOP_EPOLL) || defined(CONFIG_ELOOP_KQUEUE)
+	os_free(eloop.fd_table);
+#endif
 #ifdef CONFIG_ELOOP_EPOLL
-	os_free(eloop.epoll_table);
 	os_free(eloop.epoll_events);
 	close(eloop.epollfd);
 #endif /* CONFIG_ELOOP_EPOLL */
+#ifdef CONFIG_ELOOP_KQUEUE
+	os_free(eloop.kqueue_events);
+	close(eloop.kqueuefd);
+#endif /* CONFIG_ELOOP_KQUEUE */
 }
 
 
@@ -1135,6 +1318,17 @@ void eloop_wait_for_read_sock(int sock)
 	FD_SET(sock, &rfds);
 	select(sock + 1, &rfds, NULL, NULL, NULL);
 #endif /* defined(CONFIG_ELOOP_SELECT) || defined(CONFIG_ELOOP_EPOLL) */
+#ifdef CONFIG_ELOOP_KQUEUE
+	int kfd;
+	struct kevent ke1, ke2;
+
+	kfd = kqueue();
+	if (kfd == -1)
+		return;
+	EV_SET(&ke1, sock, EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, NULL);
+	kevent(kfd, &ke1, 1, &ke2, 1, NULL);
+	close(kfd);
+#endif /* CONFIG_ELOOP_KQUEUE */
 }
 
 #ifdef CONFIG_ELOOP_SELECT
diff --git a/src/utils/eloop.h b/src/utils/eloop.h
index 07b8c0d..97af16f 100644
--- a/src/utils/eloop.h
+++ b/src/utils/eloop.h
@@ -313,6 +313,14 @@ int eloop_register_signal_reconfig(eloop_signal_handler handler,
 				   void *user_data);
 
 /**
+ * eloop_sock_requeue - Requeue sockets
+ *
+ * Requeue sockets after forking because some implementations require this,
+ * such as epoll and kqueue.
+ */
+int eloop_sock_requeue(void);
+
+/**
  * eloop_run - Start the event loop
  *
  * Start the event loop and continue running as long as there are any
diff --git a/wpa_supplicant/Makefile b/wpa_supplicant/Makefile
index 8fa35e5..234a0bf 100644
--- a/wpa_supplicant/Makefile
+++ b/wpa_supplicant/Makefile
@@ -170,6 +170,10 @@ ifdef CONFIG_ELOOP_EPOLL
 CFLAGS += -DCONFIG_ELOOP_EPOLL
 endif
 
+ifdef CONFIG_ELOOP_KQUEUE
+CFLAGS += -DCONFIG_ELOOP_KQUEUE
+endif
+
 ifdef CONFIG_EAPOL_TEST
 CFLAGS += -Werror -DEAPOL_TEST
 endif
diff --git a/wpa_supplicant/wpa_cli.c b/wpa_supplicant/wpa_cli.c
index 1aea0a6..265f72c 100644
--- a/wpa_supplicant/wpa_cli.c
+++ b/wpa_supplicant/wpa_cli.c
@@ -4441,7 +4441,7 @@ int main(int argc, char *argv[])
 			}
 		}
 
-		if (daemonize && os_daemonize(pid_file))
+		if (daemonize && os_daemonize(pid_file) && eloop_sock_requeue())
 			return -1;
 
 		if (action_file)
diff --git a/wpa_supplicant/wpa_priv.c b/wpa_supplicant/wpa_priv.c
index fa5a6de..3f91cc1 100644
--- a/wpa_supplicant/wpa_priv.c
+++ b/wpa_supplicant/wpa_priv.c
@@ -1128,7 +1128,7 @@ int main(int argc, char *argv[])
 		interfaces = iface;
 	}
 
-	if (daemonize && os_daemonize(pid_file))
+	if (daemonize && os_daemonize(pid_file) && eloop_sock_requeue())
 		goto out;
 
 	eloop_register_signal_terminate(wpa_priv_terminate, NULL);
diff --git a/wpa_supplicant/wpa_supplicant.c b/wpa_supplicant/wpa_supplicant.c
index 03b1353..b71c459 100644
--- a/wpa_supplicant/wpa_supplicant.c
+++ b/wpa_supplicant/wpa_supplicant.c
@@ -5221,7 +5221,8 @@ int wpa_supplicant_run(struct wpa_global *global)
 	struct wpa_supplicant *wpa_s;
 
 	if (global->params.daemonize &&
-	    wpa_supplicant_daemon(global->params.pid_file))
+	    (wpa_supplicant_daemon(global->params.pid_file) ||
+	    eloop_sock_requeue()))
 		return -1;
 
 	if (global->params.wait_for_monitor) {


More information about the Hostap mailing list