[Ocfs2-tools-devel] [PATCH 2/2] ocfs2_controld: pacemaker support
Mark Fasheh
mfasheh at suse.com
Thu Aug 14 15:38:06 PDT 2008
From: Andrew Beekhof <abeekhof at suse.de>
This adds the build changes, and the core support for the pacemaker cluster
project.
Pacemaker is a next generation cluster resource manager, formerly part of
the heartbeat project. For more details, see:
http://www.clusterlabs.org/
Since pacemaker uses the openais api's, integration is fairly simple - we only
need drop in a pacemaker.c file which fills out the required ocfs2_controld
api's.
[ Whitespace, coding style fixes, naming fixes --Mark ]
Signed-off-by: Andrew Beekhof <abeekhof at suse.de>
Signed-off-by: Mark Fasheh <mfasheh at suse.com>
---
Config.make.in | 1 +
configure.in | 12 ++
ocfs2_controld/Makefile | 17 ++-
ocfs2_controld/pacemaker.c | 308 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 336 insertions(+), 2 deletions(-)
create mode 100644 ocfs2_controld/pacemaker.c
diff --git a/Config.make.in b/Config.make.in
index 10bc790..2e838bb 100644
--- a/Config.make.in
+++ b/Config.make.in
@@ -77,6 +77,7 @@ BUILD_OCFS2CONSOLE = @BUILD_OCFS2CONSOLE@
BUILD_DEBUGOCFS2 = @BUILD_DEBUGOCFS2@
BUILD_OCFS2_CONTROLD = @BUILD_OCFS2_CONTROLD@
+BUILD_PCMK_SUPPORT = @BUILD_PCMK_SUPPORT@
BUILD_CMAN_SUPPORT = @BUILD_CMAN_SUPPORT@
BUILD_FSDLM_SUPPORT = @BUILD_FSDLM_SUPPORT@
CPG_LDFLAGS = @CPG_LDFLAGS@
diff --git a/configure.in b/configure.in
index 41415f9..146a78d 100644
--- a/configure.in
+++ b/configure.in
@@ -203,6 +203,12 @@ LIBS="$ocfs_tools_save_LIBS"
AC_SUBST(BUILD_DEBUGOCFS2)
+pcmk_found=
+AC_CHECK_LIB(crmcluster, crm_get_peer,
+ [AC_CHECK_HEADER(pacemaker/crm_config.h, pcmk_found=yes,
+ [AC_MSG_WARN([Pacemaker headers not found, pacemaker support will not be built])])],
+ [AC_MSG_WARN([libcrmcluster not found, pacemaker support will not be built])])
+
# We use cman_replyto_shutdown to insure a new enough libcman
cman_found=
AC_CHECK_LIB(cman, cman_replyto_shutdown,
@@ -298,6 +304,12 @@ if test "x$cpg_found" = "xyes" -a "x$libdlmcontrol_found" = "xyes"; then
fi
AC_SUBST(BUILD_OCFS2_CONTROLD)
+BUILD_PCMK_SUPPORT=
+if test "x$pcmk_found" = "xyes" -a "x$BUILD_OCFS2_CONTROLD" = "xyes"; then
+ BUILD_PCMK_SUPPORT=yes
+fi
+AC_SUBST(BUILD_PCMK_SUPPORT)
+
BUILD_CMAN_SUPPORT=
if test "x$cman_found" = "xyes" -a "x$BUILD_OCFS2_CONTROLD" = "xyes"; then
BUILD_CMAN_SUPPORT=yes
diff --git a/ocfs2_controld/Makefile b/ocfs2_controld/Makefile
index 5db2cb9..fdb46f8 100644
--- a/ocfs2_controld/Makefile
+++ b/ocfs2_controld/Makefile
@@ -6,12 +6,18 @@ sbindir = $(root_sbindir)
ifneq ($(BUILD_OCFS2_CONTROLD),)
ifneq ($(BUILD_CMAN_SUPPORT),)
-SBIN_PROGRAMS = ocfs2_controld.cman
+SBIN_PROGRAMS += ocfs2_controld.cman
endif
UNINST_PROGRAMS = test_client
endif
-INCLUDES = -I$(TOPDIR)/include -I.
+ifneq ($(BUILD_PCMK_SUPPORT),)
+SBIN_PROGRAMS += ocfs2_controld.pcmk
+# Some pacemaker headers which pacemaker.c includes want this.
+PCMK_INCLUDES = -I/usr/include/pacemaker -I/usr/include/heartbeat/ $(GLIB_CFLAGS)
+endif
+
+INCLUDES = -I$(TOPDIR)/include -I. $(PCMK_INCLUDES)
LIBO2CB_LIBS = -L$(TOPDIR)/libo2cb -lo2cb
LIBO2CB_DEPS = $(TOPDIR)/libo2cb/libo2cb.a
LIBOCFS2_LIBS = -L$(TOPDIR)/libocfs2 -locfs2
@@ -39,6 +45,10 @@ CMAN_CFILES = cman.c
CMAN_DAEMON_CFILES = $(DAEMON_CFILES) $(CMAN_CFILES)
CMAN_DAEMON_OBJS = $(subst .c,.o,$(CMAN_DAEMON_CFILES))
+PCMK_CFILES = pacemaker.c
+PCMK_DAEMON_CFILES = $(DAEMON_CFILES) $(PCMK_CFILES)
+PCMK_DAEMON_OBJS = $(subst .c,.o,$(PCMK_DAEMON_CFILES))
+
TEST_CFILES = test_client.c
TEST_OBJS = $(subst .c,.o,$(TEST_CFILES) $(PROTO_CFILES))
MANS =
@@ -49,6 +59,9 @@ DIST_FILES = \
$(TEST_CFILES) \
$(UNINST_HFILES) \
$(addsuffix .in,$(MANS))
+ocfs2_controld.pcmk: $(PCMK_DAEMON_OBJS) $(LIBO2CB_DEPS)
+ $(LINK) $(GLIB_LIBS) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(OPENAIS_LIBS) \
+ -lcrmcluster
ocfs2_controld.cman: $(CMAN_DAEMON_OBJS) $(LIBO2CB_DEPS)
$(LINK) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(OPENAIS_LIBS) \
diff --git a/ocfs2_controld/pacemaker.c b/ocfs2_controld/pacemaker.c
new file mode 100644
index 0000000..9fe5a3f
--- /dev/null
+++ b/ocfs2_controld/pacemaker.c
@@ -0,0 +1,308 @@
+/* -*- mode: c; c-basic-offset: 8; -*-
+ * vim: noexpandtab sw=8 ts=8 sts=0:
+ */
+
+/*
+ * Copyright (C) 2008 Novell.
+ *
+ * Some portions Copyright Oracle.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v.2.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+#include <errno.h>
+#include <unistd.h>
+#include <syslog.h>
+
+#include <crm/crm.h>
+#include <crm/common/cluster.h>
+
+#include "ocfs2-kernel/kernel-list.h"
+#include "o2cb/o2cb.h"
+
+#include "ocfs2_controld.h"
+
+#include <bzlib.h>
+#include <crm/crm.h>
+#include <crm/ais.h>
+#include <sys/utsname.h>
+
+int our_nodeid = 0;
+static int pcmk_ci;
+static char * clustername = "pacemaker";
+extern struct list_head mounts;
+const char *stackname = "pcmk";
+
+extern int ais_fd_async;
+char *local_node_uname = NULL;
+
+int kill_stack_node(int nodeid)
+{
+ int error = 1;
+
+ log_debug("killing node %d", nodeid);
+
+ /* error = cman_kill_node(ch_admin, nodeid); */
+ if (error)
+ log_debug("Unable to kill node %d, %d %d", nodeid, error,
+ errno);
+
+ return error;
+}
+
+char *nodeid2name(int nodeid) {
+ crm_node_t *node = crm_get_peer(nodeid, NULL);
+
+ if(node->uname == NULL)
+ return NULL;
+
+ return strdup(node->uname);
+}
+
+int validate_cluster(const char *cluster)
+{
+ if (!clustername) {
+ log_error("Trying to validate before pacemaker is alive");
+ return 0;
+ }
+
+ if (!cluster)
+ return 0;
+
+ return !strcmp(cluster, clustername);
+}
+
+int get_clustername(const char **cluster)
+{
+ if (!clustername) {
+ log_error("Trying to validate before pacemaker is alive");
+ return -EIO;
+ }
+
+ if (!cluster) {
+ log_error("NULL passed!");
+ return -EINVAL;
+ }
+
+ *cluster = clustername;
+ return 0;
+}
+
+static void dead_pcmk(int ci)
+{
+ if (ci != pcmk_ci) {
+ log_error("Unknown connection %d", ci);
+ return;
+ }
+
+ log_error("pacemaker connection died");
+ shutdown_daemon();
+ connection_dead(ci);
+}
+
+void exit_stack(void)
+{
+ log_debug("closing pacemaker connection");
+ if (ais_fd_async) {
+ close(ais_fd_async);
+ ais_fd_async = 0;
+ }
+ if (ais_fd_sync) {
+ close(ais_fd_sync);
+ ais_fd_sync = 0;
+ }
+}
+
+static void process_pcmk(int ci)
+{
+ /* ci ::= client number */
+ char *data = NULL;
+ char *uncompressed = NULL;
+ AIS_Message *msg = NULL;
+ SaAisErrorT rc = SA_AIS_OK;
+ mar_res_header_t *header = NULL;
+ static int header_len = sizeof(mar_res_header_t);
+
+ header = malloc(header_len);
+ memset(header, 0, header_len);
+
+ errno = 0;
+ rc = saRecvRetry(ais_fd_async, header, header_len);
+ if (rc != SA_AIS_OK) {
+ cl_perror("Receiving message header failed: (%d) %s", rc,
+ ais_error2text(rc));
+ goto bail;
+ } else if(header->size == header_len) {
+ log_error("Empty message: id=%d, size=%d, error=%d, header_len=%d",
+ header->id, header->size, header->error, header_len);
+ goto done;
+ } else if(header->size == 0 || header->size < header_len) {
+ log_error("Mangled header: size=%d, header=%d, error=%d",
+ header->size, header_len, header->error);
+ goto done;
+ } else if(header->error != 0) {
+ log_error("Header contined error: %d", header->error);
+ }
+
+ header = realloc(header, header->size);
+ /* Use a char* so we can store the remainder into an offset */
+ data = (char*)header;
+
+ errno = 0;
+ rc = saRecvRetry(ais_fd_async, data+header_len, header->size - header_len);
+ msg = (AIS_Message*)data;
+
+ if (rc != SA_AIS_OK) {
+ cl_perror("Receiving message body failed: (%d) %s", rc, ais_error2text(rc));
+ goto bail;
+ }
+
+ data = msg->data;
+ if(msg->is_compressed && msg->size > 0) {
+ int rc = BZ_OK;
+ unsigned int new_size = msg->size;
+
+ if (check_message_sanity(msg, NULL) == FALSE)
+ goto badmsg;
+
+ log_debug("Decompressing message data");
+ uncompressed = malloc(new_size);
+ memset(uncompressed, 0, new_size);
+
+ rc = BZ2_bzBuffToBuffDecompress(
+ uncompressed, &new_size, data, msg->compressed_size,
+ 1, 0);
+
+ if(rc != BZ_OK) {
+ log_error("Decompression failed: %d", rc);
+ goto badmsg;
+ }
+
+ CRM_ASSERT(rc == BZ_OK);
+ CRM_ASSERT(new_size == msg->size);
+
+ data = uncompressed;
+
+ } else if(check_message_sanity(msg, data) == FALSE) {
+ goto badmsg;
+
+ } else if(safe_str_eq("identify", data)) {
+ int pid = getpid();
+ char *pid_s = crm_itoa(pid);
+
+ send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
+ crm_free(pid_s);
+ goto done;
+ }
+
+ if (msg->header.id == crm_class_members) {
+ xmlNode *xml = string2xml(data);
+
+ if(xml != NULL) {
+ const char *value = crm_element_value(xml, "id");
+ if(value)
+ crm_peer_seq = crm_int_helper(value, NULL);
+
+ log_debug("Updating membership %llu", crm_peer_seq);
+ /* crm_log_xml_info(xml, __PRETTY_FUNCTION__); */
+ xml_child_iter(xml, node, crm_update_ais_node(node, crm_peer_seq));
+ crm_calculate_quorum();
+ free_xml(xml);
+ } else {
+ log_error("Invalid peer update: %s", data);
+ }
+ } else {
+ log_error("Unexpected AIS message type: %d", msg->header.id);
+ }
+
+done:
+ free(uncompressed);
+ free(msg);
+ return;
+
+badmsg:
+ log_error("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
+ " min=%d, total=%d, size=%d, bz2_size=%d",
+ msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
+ ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
+ msg->sender.pid, (int)sizeof(AIS_Message),
+ msg->header.size, msg->size, msg->compressed_size);
+ free(uncompressed);
+ free(msg);
+ return;
+
+bail:
+ log_error("AIS connection failed");
+ return;
+}
+
+int setup_stack(void)
+{
+ int retries = 0;
+ int pid;
+ char *pid_s;
+ int rc = SA_AIS_OK;
+ struct utsname name;
+
+ crm_peer_init();
+
+ if (local_node_uname == NULL) {
+ if (uname(&name) < 0) {
+ cl_perror("uname(2) call failed");
+ exit(100);
+ }
+ local_node_uname = crm_strdup(name.nodename);
+ log_debug("Local node name: %s", local_node_uname);
+ }
+
+ /* 16 := CRM_SERVICE */
+retry:
+ log_debug("Creating connection to our AIS plugin");
+ rc = saServiceConnect (&ais_fd_sync, &ais_fd_async, 16);
+ if (rc != SA_AIS_OK) {
+ log_error("Connection to our AIS plugin failed: %s (%d)",
+ ais_error2text(rc), rc);
+ }
+
+ switch(rc) {
+ case SA_AIS_OK:
+ break;
+ case SA_AIS_ERR_TRY_AGAIN:
+ if(retries < 30) {
+ sleep(1);
+ retries++;
+ goto retry;
+ }
+ log_error("Retry count exceeded");
+ return 0;
+ default:
+ return 0;
+ }
+
+ log_debug("AIS connection established");
+
+ pid = getpid();
+ pid_s = crm_itoa(pid);
+ send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
+ crm_free(pid_s);
+
+ /* Sign up for membership updates */
+ send_ais_text(crm_class_notify, "true", TRUE, NULL, crm_msg_ais);
+
+ /* Requesting the current list of known nodes */
+ send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais);
+
+ pcmk_ci = connection_add(ais_fd_async, process_pcmk, dead_pcmk);
+ if (pcmk_ci >= 0)
+ return ais_fd_async;
+
+ log_error("Unable to add pacemaker client: %s", strerror(-pcmk_ci));
+ exit_stack();
+ return pcmk_ci;
+}
--
1.5.4.1
More information about the Ocfs2-tools-devel
mailing list