[PATCH 23/23] Temporary hack to remove cache state on client disconnect

[Valerie Aurora]

This is a racy inefficient version of removing a client's block cache
state when it disconnects, but it allows the devd server to continue
functioning after a client disconnects. The correct version would
remove the cache state when it received a message from the mapd server
that the client was evicted from the cluster. Clients would also be
identified by something other than the IP/port of their connection.

Signed-off-by: Valerie Aurora <val at versity.com>
---
 devd/cache-mode.c | 25 +++++++++++++++++++++++++
 devd/cache-mode.h |  1 +
 devd/ngnfs-devd.c |  3 ++-
 utask/net.c       |  9 ++++++++-
 utask/net.h       |  3 ++-
 5 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/devd/cache-mode.c b/devd/cache-mode.c
index f6e4bac..909bfc1 100644
--- a/devd/cache-mode.c
+++ b/devd/cache-mode.c
@@ -472,6 +472,31 @@ out:
 	return ret;
 }
 
+/*
+ * The client has been evicted from the cluster; remove all of its
+ * cache modes on all blocks.
+ */
+void cache_release_all(struct sockaddr_in *addr)
+{
+	struct cache_mode_instance *inst = &global_cache_mode_inst;
+	struct client_mode *cli;
+	struct rb_node *walk;
+	int cmp;
+
+	walk = rb_first(&inst->cli_root);
+
+	while (walk) {
+		cli = cli_container(walk);
+		walk = rb_next(walk); /* before we free it! */
+		cmp = ngnfs_compare(ntohl(addr->sin_addr.s_addr),
+				    ntohl(cli->addr.sin_addr.s_addr)) ?:
+		      ngnfs_compare(ntohs(addr->sin_port), ntohs(cli->addr.sin_port));
+
+		if (cmp == 0)
+			free_client_mode(inst, cli);
+	}
+}
+
 /*
  * Process requests for a given block without returning processing
  * errors to the caller.  There may be no recorded client modes for the
diff --git a/devd/cache-mode.h b/devd/cache-mode.h
index 589421b..cfdbdfe 100644
--- a/devd/cache-mode.h
+++ b/devd/cache-mode.h
@@ -5,5 +5,6 @@
 int cache_mode_request(struct sockaddr_in *addr, u64 bnr, u8 mode, bool no_data);
 int cache_mode_ack(struct sockaddr_in *addr, u64 bnr, u8 mode);
 void cache_mode_process(u64 bnr);
+void cache_release_all(struct sockaddr_in *addr);
 
 #endif
diff --git a/devd/ngnfs-devd.c b/devd/ngnfs-devd.c
index ef1d08d..29c04f2 100644
--- a/devd/ngnfs-devd.c
+++ b/devd/ngnfs-devd.c
@@ -32,6 +32,7 @@
 #include "utask/utask.h"
 
 #include "devd/proc.h"
+#include "devd/cache-mode.h"
 
 struct devd_options {
 	char *dev_path;
@@ -92,7 +93,7 @@ int main(int argc, char **argv)
 
 	ret = utask_init(BLOCK_QUEUE_DEPTH + NET_QUEUE_DEPTH) ?:
 	      net_register_recv(proc_recv) ?:
-	      net_listen(&opts.listen_addr) ?:
+	      net_listen(&opts.listen_addr, cache_release_all) ?:
 	      block_init(opts.dev_path, BLOCK_QUEUE_DEPTH) ?:
 	      utask_run();
 
diff --git a/utask/net.c b/utask/net.c
index 53a3995..d7539cd 100644
--- a/utask/net.c
+++ b/utask/net.c
@@ -56,6 +56,8 @@ struct net_socket {
 	struct utask *connect_tsk;
 	struct utask *send_tsk;
 	struct utask *recv_tsk;
+
+	net_close_fn_t close_fn;
 };
 
 /*
@@ -123,6 +125,9 @@ static void destroy_sock(struct net_instance *inst, struct net_socket *sock)
 	if (sock->fd >= 0)
 		close(sock->fd);
 
+	if (sock->close_fn)
+		sock->close_fn(&sock->addr);
+
 	if (!RB_EMPTY_NODE(&sock->node))
 		rb_erase(&sock->node, &inst->peer_root);
 	free(sock);
@@ -421,6 +426,7 @@ static void accept_utask(void *data)
 
 		accepted->addr = addr;
 		accepted->fd = fd;
+		accepted->close_fn = sock->close_fn;
 
 		ret = utask_create(send_utask, accepted, &accepted->send_tsk) ?:
 		      utask_create(recv_utask, accepted, &accepted->recv_tsk);
@@ -439,7 +445,7 @@ out:
  * accepting new connected sockets on the listening socket.  This
  * doesn't need to be called from a utask.
  */
-int net_listen(struct sockaddr_in *addr)
+int net_listen(struct sockaddr_in *addr, net_close_fn_t close_fn)
 {
 	struct net_instance *inst = &global_net_inst;
 	struct net_socket *sock = NULL;
@@ -478,6 +484,7 @@ int net_listen(struct sockaddr_in *addr)
 		goto out;
 	}
 
+	sock->close_fn = close_fn;
 	ret = utask_create(accept_utask, sock, &sock->accept_tsk);
 	if (ret < 0) {
 		destroy_sock(inst, sock);
diff --git a/utask/net.h b/utask/net.h
index 422e5aa..b68f230 100644
--- a/utask/net.h
+++ b/utask/net.h
@@ -10,11 +10,12 @@ struct sockaddr_in;
 
 typedef int (*net_recv_fn_t)(struct sockaddr_in *addr, struct ngnfs_msg_header *hdr,
 			     void *ctl_buf, struct page *data_page);
+typedef void (*net_close_fn_t)(struct sockaddr_in *addr);
 
 int net_send(struct sockaddr_in *addr, struct ngnfs_msg_header *hdr, void *ctl_buf,
 	     struct page *data_page);
 
-int net_listen(struct sockaddr_in *addr);
+int net_listen(struct sockaddr_in *addr, net_close_fn_t close_fn);
 int net_connect(struct sockaddr_in *addr);
 
 int net_register_recv(net_recv_fn_t recv_fn);
-- 
2.48.1