[Ocfs2-tools-devel] [PATCH 14/39] ocfs2_controld: Steal a better
client loop
Joel Becker
joel.becker at oracle.com
Fri Mar 14 16:52:37 PDT 2008
groupd has a smarter client_add() implementation. It uses callbacks to
match client fd to worker function. Let's use it.
While we're at it, rename member_cman. It was just too long.
Signed-off-by: Joel Becker <joel.becker at oracle.com>
---
ocfs2_controld/Makefile | 2 +-
ocfs2_controld/cman.c | 232 +++++++++++++++++++++++++++++++++++++++
ocfs2_controld/main.c | 162 ++++++++++++++++------------
ocfs2_controld/member_cman.c | 220 -------------------------------------
ocfs2_controld/ocfs2_controld.h | 6 +-
5 files changed, 331 insertions(+), 291 deletions(-)
create mode 100644 ocfs2_controld/cman.c
delete mode 100644 ocfs2_controld/member_cman.c
diff --git a/ocfs2_controld/Makefile b/ocfs2_controld/Makefile
index ec7bb41..00a3b04 100644
--- a/ocfs2_controld/Makefile
+++ b/ocfs2_controld/Makefile
@@ -29,7 +29,7 @@ DEFINES = -DOCFS2_FLAT_INCLUDES -DO2DLM_FLAT_INCLUDES \
UNINST_HFILES = ocfs2_controld.h
-DAEMON_CFILES = main.c member_cman.c
+DAEMON_CFILES = main.c cman.c
TEST_CFILES = test_client.c
DAEMON_OBJS = $(subst .c,.o,$(DAEMON_CFILES))
diff --git a/ocfs2_controld/cman.c b/ocfs2_controld/cman.c
new file mode 100644
index 0000000..cbe3d95
--- /dev/null
+++ b/ocfs2_controld/cman.c
@@ -0,0 +1,232 @@
+/* -*- mode: c; c-basic-offset: 8; -*-
+ * vim: noexpandtab sw=8 ts=8 sts=0:
+ */
+
+/******************************************************************************
+*******************************************************************************
+**
+** Copyright (C) 2005 Red Hat, Inc. All rights reserved.
+**
+** This copyrighted material is made available to anyone wishing to use,
+** modify, copy, or redistribute it subject to the terms and conditions
+** of the GNU General Public License v.2.
+**
+*******************************************************************************
+******************************************************************************/
+
+/*
+ * Copyright (C) 2007 Oracle. All rights reserved.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v.2.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+#include <errno.h>
+#include <unistd.h>
+#include <syslog.h>
+#include <libcman.h>
+
+#include "ocfs2-kernel/kernel-list.h"
+#include "o2cb/o2cb.h"
+
+#include "ocfs2_controld.h"
+
+int our_nodeid;
+int cman_ci;
+char * clustername;
+cman_cluster_t cluster;
+static cman_handle_t ch;
+extern struct list_head mounts;
+static cman_node_t old_nodes[O2NM_MAX_NODES];
+static int old_node_count;
+static cman_node_t cman_nodes[O2NM_MAX_NODES];
+static int cman_node_count;
+
+
+static int is_member(cman_node_t *node_list, int count, int nodeid)
+{
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if (node_list[i].cn_nodeid == nodeid)
+ return node_list[i].cn_member;
+ }
+ return 0;
+}
+
+static int is_old_member(int nodeid)
+{
+ return is_member(old_nodes, old_node_count, nodeid);
+}
+
+static int is_cman_member(int nodeid)
+{
+ return is_member(cman_nodes, cman_node_count, nodeid);
+}
+
+static cman_node_t *find_cman_node(int nodeid)
+{
+ int i;
+
+ for (i = 0; i < cman_node_count; i++) {
+ if (cman_nodes[i].cn_nodeid == nodeid)
+ return &cman_nodes[i];
+ }
+ return NULL;
+}
+
+char *nodeid2name(int nodeid)
+{
+ cman_node_t *cn;
+
+ cn = find_cman_node(nodeid);
+ if (!cn)
+ return NULL;
+ return cn->cn_name;
+}
+
+/* keep track of the nodes */
+static void statechange(void)
+{
+ int i, rv;
+
+ old_node_count = cman_node_count;
+ memcpy(&old_nodes, &cman_nodes, sizeof(old_nodes));
+
+ cman_node_count = 0;
+ memset(&cman_nodes, 0, sizeof(cman_nodes));
+ rv = cman_get_nodes(ch, O2NM_MAX_NODES, &cman_node_count,
+ cman_nodes);
+ if (rv < 0) {
+ log_debug("cman_get_nodes error %d %d", rv, errno);
+ return;
+ }
+
+ for (i = 0; i < old_node_count; i++) {
+ if (old_nodes[i].cn_member &&
+ !is_cman_member(old_nodes[i].cn_nodeid)) {
+
+ log_debug("cman: node %d removed",
+ old_nodes[i].cn_nodeid);
+ }
+ }
+
+ for (i = 0; i < cman_node_count; i++) {
+ if (cman_nodes[i].cn_member &&
+ !is_old_member(cman_nodes[i].cn_nodeid)) {
+
+ log_debug("cman: node %d added",
+ cman_nodes[i].cn_nodeid);
+ }
+ }
+}
+
+static void cman_callback(cman_handle_t h, void *private, int reason, int arg)
+{
+ switch (reason) {
+ case CMAN_REASON_TRY_SHUTDOWN:
+#if 0
+ if (list_empty(&mounts))
+#endif
+ cman_replyto_shutdown(ch, 1);
+#if 0
+ else {
+ log_debug("no to cman shutdown");
+ cman_replyto_shutdown(ch, 0);
+ }
+#endif
+ break;
+
+ case CMAN_REASON_STATECHANGE:
+ statechange();
+ break;
+ }
+}
+
+static void exit_cman(int ci)
+{
+ if (ci != cman_ci) {
+ log_error("Unknown connection %d", ci);
+ return;
+ }
+
+ log_error("cman connection died");
+ shutdown_daemon();
+ client_dead(ci);
+}
+
+static void process_cman(int ci)
+{
+ int rv;
+
+ if (ci != cman_ci) {
+ log_error("Unknown connection %d", ci);
+ return;
+ }
+
+ rv = cman_dispatch(ch, CMAN_DISPATCH_ALL);
+ if (rv == -1 && errno == EHOSTDOWN) {
+ log_error("cman connection died");
+ shutdown_daemon();
+ }
+}
+
+int setup_cman(void)
+{
+ cman_node_t node;
+ int rv, fd;
+
+ ch = cman_init(NULL);
+ if (!ch) {
+ log_error("cman_init error %d", errno);
+ return -ENOTCONN;
+ }
+
+ rv = cman_start_notification(ch, cman_callback);
+ if (rv < 0) {
+ log_error("cman_start_notification error %d %d", rv, errno);
+ goto fail_finish;
+ }
+
+ /* FIXME: wait here for us to be a member of the cluster */
+
+ memset(&cluster, 0, sizeof(cluster));
+ rv = cman_get_cluster(ch, &cluster);
+ if (rv < 0) {
+ log_error("cman_get_cluster error %d %d", rv, errno);
+ goto fail_stop;
+ }
+ clustername = cluster.ci_name;
+
+ memset(&node, 0, sizeof(node));
+ rv = cman_get_node(ch, CMAN_NODEID_US, &node);
+ if (rv < 0) {
+ log_error("cman_get_node error %d %d", rv, errno);
+ goto fail_stop;
+ }
+ our_nodeid = node.cn_nodeid;
+
+ fd = cman_get_fd(ch);
+
+ old_node_count = 0;
+ memset(&old_nodes, 0, sizeof(old_nodes));
+ cman_node_count = 0;
+ memset(&cman_nodes, 0, sizeof(cman_nodes));
+
+ /* Fill the node list */
+ statechange();
+
+ cman_ci = client_add(fd, process_cman, exit_cman);
+ return 0;
+
+ fail_stop:
+ cman_stop_notification(ch);
+ fail_finish:
+ cman_finish(ch);
+ return rv;
+}
+
diff --git a/ocfs2_controld/main.c b/ocfs2_controld/main.c
index 2136534..f0ec2fc 100644
--- a/ocfs2_controld/main.c
+++ b/ocfs2_controld/main.c
@@ -48,18 +48,20 @@
struct client {
int fd;
char type[32];
+ void (*work)(int ci);
+ void (*dead)(int ci);
+#if 0
struct mountgroup *mg;
int another_mount;
+#endif
};
static int client_maxi;
static int client_size = 0;
static struct client *client = NULL;
static struct pollfd *pollfd = NULL;
+static int time_to_die = 0;
-static int cman_fd;
-static int listen_fd;
-static int sigpipe_fd;
static int sigpipe_write_fd;
static int groupd_fd;
@@ -74,6 +76,11 @@ char dump_buf[DUMP_SIZE];
int dump_point;
int dump_wrap;
+void shutdown_daemon(void)
+{
+ time_to_die = 1;
+}
+
static void handler(int signum)
{
log_debug("Caught signal %d", signum);
@@ -81,12 +88,19 @@ static void handler(int signum)
log_error("Problem writing signal: %s", strerror(-errno));
}
-static int handle_signal(void)
+static void dead_sigpipe(int ci)
+{
+ log_error("Error on the signal pipe");
+ client_dead(ci);
+ shutdown_daemon();
+}
+
+static void handle_signal(int ci)
{
int rc, caught_sig, abortp = 0;
static int segv_already = 0;
- rc = read(sigpipe_fd, (char *)&caught_sig, sizeof(caught_sig));
+ rc = read(client[ci].fd, (char *)&caught_sig, sizeof(caught_sig));
if (rc < 0) {
rc = -errno;
log_error("Error reading from signal pipe: %s",
@@ -144,7 +158,8 @@ static int handle_signal(void)
abort();
out:
- return rc;
+ if (rc)
+ shutdown_daemon();
}
static int setup_sigpipe(void)
@@ -161,7 +176,6 @@ static int setup_sigpipe(void)
goto out;
}
- sigpipe_fd = signal_pipe[0];
sigpipe_write_fd = signal_pipe[1];
act.sa_sigaction = NULL;
@@ -182,6 +196,8 @@ static int setup_sigpipe(void)
if (rc)
log_error("Unable to set up signal handlers");
+ else
+ client_add(signal_pipe[0], handle_signal, dead_sigpipe);
out:
return rc;
@@ -225,8 +241,20 @@ int do_write(int fd, void *buf, size_t count)
return 0;
}
+void client_dead(int ci)
+{
+ log_debug("client %d fd %d dead", ci, client[ci].fd);
+ close(client[ci].fd);
+ client[ci].work = NULL;
+ client[ci].fd = -1;
+ pollfd[ci].fd = -1;
+#if 0
+ client[ci].mg = NULL;
+#endif
+}
-static int client_add(int fd)
+
+int client_add(int fd, void (*work)(int ci), void (*dead)(int ci))
{
int i;
@@ -235,6 +263,8 @@ static int client_add(int fd)
for (i = 0; i < client_size; i++) {
if (client[i].fd == -1) {
client[i].fd = fd;
+ client[i].work = work;
+ client[i].dead = dead ? dead : client_dead;
pollfd[i].fd = fd;
pollfd[i].events = POLLIN;
if (i > client_maxi)
@@ -265,15 +295,6 @@ static int client_add(int fd)
}
}
-static void client_dead(int ci)
-{
- log_debug("client %d fd %d dead", ci, client[ci].fd);
- close(client[ci].fd);
- client[ci].fd = -1;
- pollfd[ci].fd = -1;
- client[ci].mg = NULL;
-}
-
static int dump_debug(int ci)
{
int len = DUMP_SIZE;
@@ -288,7 +309,7 @@ static int dump_debug(int ci)
return 0;
}
-static int process_client(int ci)
+static void process_client(int ci)
{
#if 0
struct mountgroup *mg;
@@ -298,16 +319,18 @@ static int process_client(int ci)
char buf[OCFS2_CONTROLD_MAXLINE];
int rv, fd = client[ci].fd;
+ log_debug("client msg");
/* receive_message ensures we have the proper number of arguments */
rv = receive_message(fd, buf, &message, argv);
if (rv == -EPIPE) {
client_dead(ci);
- return 0;
+ return;
}
+
if (rv < 0) {
/* XXX: Should print better errors matching our returns */
log_debug("client %d fd %d read error %d", ci, fd, -rv);
- return rv;
+ return;
}
log_debug("client message %d from %d: %s", message, ci,
@@ -368,7 +391,7 @@ static int process_client(int ci)
}
#endif
- return rv;
+ return;
}
#if 0
@@ -393,28 +416,58 @@ static void bail_on_mounts(void)
}
#endif
+static void process_listener(int ci)
+{
+ int fd, i;
+ fd = accept(client[ci].fd, NULL, NULL);
+ if (fd < 0) {
+ log_debug("accept error %d %d", fd, errno);
+ return;
+ }
-static int loop(void)
+ i = client_add(fd, process_client, NULL);
+ log_debug("new client connection %d", i);
+}
+
+static void dead_listener(int ci)
+{
+ log_error("Error on the listening socket");
+ client_dead(ci);
+ shutdown_daemon();
+}
+
+static int setup_listener(void)
{
- int rv, i, f, poll_timeout = -1;
+ int fd, i;
- rv = listen_fd = ocfs2_client_listen();
- if (rv < 0) {
+ fd = ocfs2_client_listen();
+ if (fd < 0) {
log_error("Unable to start listening socket: %s",
- strerror(-rv));
- goto out;
+ strerror(-fd));
+ return 1;
}
- client_add(listen_fd);
+
+ i = client_add(fd, process_listener, dead_listener);
+ log_debug("new listening connection %d", i);
+
+ return 0;
+}
+
+static int loop(void)
+{
+ int rv, i, poll_timeout = -1;
+
+ rv = setup_listener();
+ if (rv < 0)
+ goto out;
rv = setup_sigpipe();
if (rv < 0)
goto out;
- client_add(sigpipe_fd);
- rv = cman_fd = setup_cman();
+ rv = setup_cman();
if (rv < 0)
goto out;
- client_add(cman_fd);
#if 0
rv = groupd_fd = setup_groupd();
@@ -431,51 +484,24 @@ static int loop(void)
log_error("poll error %d errno %d", rv, errno);
rv = 0;
- /* client[0] is listening for new connections */
-
- if (pollfd[0].revents & POLLIN) {
- f = accept(client[0].fd, NULL, NULL);
- if (f < 0)
- log_debug("accept error %d %d", f, errno);
- else
- client_add(f);
- }
-
- for (i = 1; i <= client_maxi; i++) {
+ for (i = 0; i <= client_maxi; i++) {
if (client[i].fd < 0)
continue;
+ /*
+ * We handle POLLIN before POLLHUP so clients can
+ * finish what they were doing
+ */
if (pollfd[i].revents & POLLIN) {
-#if 0
- if (pollfd[i].fd == groupd_fd)
- process_groupd();
- else if (pollfd[i].fd == cman_fd) {
-#endif
- if (pollfd[i].fd == cman_fd) {
- rv = process_cman();
- if (rv) {
- log_error("cman connection died");
- goto stop;
- }
- } else if (pollfd[i].fd == sigpipe_fd) {
- rv = handle_signal();
- if (rv)
- goto stop;
- } else
- process_client(i);
+ client[i].work(i);
+ if (time_to_die)
+ goto stop;
}
if (pollfd[i].revents & POLLHUP) {
- if (pollfd[i].fd == cman_fd) {
- log_error("cman connection died");
- goto stop;
-#if 0
- } else if (pollfd[i].fd == groupd_fd) {
- log_error("groupd connection died");
+ client[i].dead(i);
+ if (time_to_die)
goto stop;
-#endif
- }
- client_dead(i);
}
}
}
diff --git a/ocfs2_controld/member_cman.c b/ocfs2_controld/member_cman.c
deleted file mode 100644
index d5829ac..0000000
--- a/ocfs2_controld/member_cman.c
+++ /dev/null
@@ -1,220 +0,0 @@
-/* -*- mode: c; c-basic-offset: 8; -*-
- * vim: noexpandtab sw=8 ts=8 sts=0:
- */
-
-/******************************************************************************
-*******************************************************************************
-**
-** Copyright (C) 2005 Red Hat, Inc. All rights reserved.
-**
-** This copyrighted material is made available to anyone wishing to use,
-** modify, copy, or redistribute it subject to the terms and conditions
-** of the GNU General Public License v.2.
-**
-*******************************************************************************
-******************************************************************************/
-
-/*
- * Copyright (C) 2007 Oracle. All rights reserved.
- *
- * This copyrighted material is made available to anyone wishing to use,
- * modify, copy, or redistribute it subject to the terms and conditions
- * of the GNU General Public License v.2.
- */
-
-#include <string.h>
-#include <stdlib.h>
-#include <time.h>
-#include <errno.h>
-#include <unistd.h>
-#include <syslog.h>
-#include <libcman.h>
-
-#include "ocfs2-kernel/kernel-list.h"
-#include "o2cb/o2cb.h"
-
-#include "ocfs2_controld.h"
-
-int our_nodeid;
-char * clustername;
-cman_cluster_t cluster;
-static cman_handle_t ch;
-extern struct list_head mounts;
-static cman_node_t old_nodes[O2NM_MAX_NODES];
-static int old_node_count;
-static cman_node_t cman_nodes[O2NM_MAX_NODES];
-static int cman_node_count;
-
-
-static int is_member(cman_node_t *node_list, int count, int nodeid)
-{
- int i;
-
- for (i = 0; i < count; i++) {
- if (node_list[i].cn_nodeid == nodeid)
- return node_list[i].cn_member;
- }
- return 0;
-}
-
-static int is_old_member(int nodeid)
-{
- return is_member(old_nodes, old_node_count, nodeid);
-}
-
-static int is_cman_member(int nodeid)
-{
- return is_member(cman_nodes, cman_node_count, nodeid);
-}
-
-static cman_node_t *find_cman_node(int nodeid)
-{
- int i;
-
- for (i = 0; i < cman_node_count; i++) {
- if (cman_nodes[i].cn_nodeid == nodeid)
- return &cman_nodes[i];
- }
- return NULL;
-}
-
-char *nodeid2name(int nodeid)
-{
- cman_node_t *cn;
-
- cn = find_cman_node(nodeid);
- if (!cn)
- return NULL;
- return cn->cn_name;
-}
-
-/* keep track of the nodes */
-static void statechange(void)
-{
- int i, rv;
-
- old_node_count = cman_node_count;
- memcpy(&old_nodes, &cman_nodes, sizeof(old_nodes));
-
- cman_node_count = 0;
- memset(&cman_nodes, 0, sizeof(cman_nodes));
- rv = cman_get_nodes(ch, O2NM_MAX_NODES, &cman_node_count,
- cman_nodes);
- if (rv < 0) {
- log_debug("cman_get_nodes error %d %d", rv, errno);
- return;
- }
-
- for (i = 0; i < old_node_count; i++) {
- if (old_nodes[i].cn_member &&
- !is_cman_member(old_nodes[i].cn_nodeid)) {
-
- log_debug("cman: node %d removed",
- old_nodes[i].cn_nodeid);
- }
- }
-
- for (i = 0; i < cman_node_count; i++) {
- if (cman_nodes[i].cn_member &&
- !is_old_member(cman_nodes[i].cn_nodeid)) {
-
- log_debug("cman: node %d added",
- cman_nodes[i].cn_nodeid);
- }
- }
-}
-
-static void cman_callback(cman_handle_t h, void *private, int reason, int arg)
-{
- switch (reason) {
- case CMAN_REASON_TRY_SHUTDOWN:
-#if 0
- if (list_empty(&mounts))
-#endif
- cman_replyto_shutdown(ch, 1);
-#if 0
- else {
- log_debug("no to cman shutdown");
- cman_replyto_shutdown(ch, 0);
- }
-#endif
- break;
-
- case CMAN_REASON_STATECHANGE:
- statechange();
- break;
- }
-}
-
-void exit_cman(void)
-{
- log_error("cluster is down, exiting");
- exit(1);
-}
-
-int process_cman(void)
-{
- int rv;
-
- rv = cman_dispatch(ch, CMAN_DISPATCH_ALL);
-
- if (rv == -1 && errno == EHOSTDOWN)
- return 1;
-
- return 0;
-}
-
-int setup_cman(void)
-{
- cman_node_t node;
- int rv, fd;
-
- ch = cman_init(NULL);
- if (!ch) {
- log_error("cman_init error %d", errno);
- return -ENOTCONN;
- }
-
- rv = cman_start_notification(ch, cman_callback);
- if (rv < 0) {
- log_error("cman_start_notification error %d %d", rv, errno);
- goto fail_finish;
- }
-
- /* FIXME: wait here for us to be a member of the cluster */
-
- memset(&cluster, 0, sizeof(cluster));
- rv = cman_get_cluster(ch, &cluster);
- if (rv < 0) {
- log_error("cman_get_cluster error %d %d", rv, errno);
- goto fail_stop;
- }
- clustername = cluster.ci_name;
-
- memset(&node, 0, sizeof(node));
- rv = cman_get_node(ch, CMAN_NODEID_US, &node);
- if (rv < 0) {
- log_error("cman_get_node error %d %d", rv, errno);
- goto fail_stop;
- }
- our_nodeid = node.cn_nodeid;
-
- fd = cman_get_fd(ch);
-
- old_node_count = 0;
- memset(&old_nodes, 0, sizeof(old_nodes));
- cman_node_count = 0;
- memset(&cman_nodes, 0, sizeof(cman_nodes));
-
- /* Fill the node list */
- statechange();
-
- return fd;
-
- fail_stop:
- cman_stop_notification(ch);
- fail_finish:
- cman_finish(ch);
- return rv;
-}
-
diff --git a/ocfs2_controld/ocfs2_controld.h b/ocfs2_controld/ocfs2_controld.h
index a3e4068..ed57fd9 100644
--- a/ocfs2_controld/ocfs2_controld.h
+++ b/ocfs2_controld/ocfs2_controld.h
@@ -61,9 +61,11 @@ do { \
} while (0)
+int client_add(int fd, void (*work)(int ci), void (*dead)(int ci));
+void client_dead(int ci);
+void shutdown_daemon(void);
+
int setup_cman(void);
-int process_cman(void);
char *nodeid2name(int nodeid);
-void exit_cman(void);
#endif
--
1.5.3.8
More information about the Ocfs2-tools-devel
mailing list