[Ocfs2-tools-devel] [PATCH 2/2] ocfs2_controld: pacemaker support
Joel Becker
Joel.Becker at oracle.com
Thu Aug 14 15:46:04 PDT 2008
On Thu, Aug 14, 2008 at 03:38:06PM -0700, Mark Fasheh wrote:
> From: Andrew Beekhof <abeekhof at suse.de>
>
> This adds the build changes, and the core support for the pacemaker cluster
> project.
>
> Pacemaker is a next generation cluster resource manager, formerly part of
> the heartbeat project. For more details, see:
>
> http://www.clusterlabs.org/
>
> Since pacemaker uses the openais api's, integration is fairly simple - we only
> need drop in a pacemaker.c file which fills out the required ocfs2_controld
> api's.
>
> [ Whitespace, coding style fixes, naming fixes --Mark ]
>
> Signed-off-by: Andrew Beekhof <abeekhof at suse.de>
> Signed-off-by: Mark Fasheh <mfasheh at suse.com>
Signed-off-by: Joel Becker <joel.becker at oracle.com>
> ---
> Config.make.in | 1 +
> configure.in | 12 ++
> ocfs2_controld/Makefile | 17 ++-
> ocfs2_controld/pacemaker.c | 308 ++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 336 insertions(+), 2 deletions(-)
> create mode 100644 ocfs2_controld/pacemaker.c
>
> diff --git a/Config.make.in b/Config.make.in
> index 10bc790..2e838bb 100644
> --- a/Config.make.in
> +++ b/Config.make.in
> @@ -77,6 +77,7 @@ BUILD_OCFS2CONSOLE = @BUILD_OCFS2CONSOLE@
> BUILD_DEBUGOCFS2 = @BUILD_DEBUGOCFS2@
>
> BUILD_OCFS2_CONTROLD = @BUILD_OCFS2_CONTROLD@
> +BUILD_PCMK_SUPPORT = @BUILD_PCMK_SUPPORT@
> BUILD_CMAN_SUPPORT = @BUILD_CMAN_SUPPORT@
> BUILD_FSDLM_SUPPORT = @BUILD_FSDLM_SUPPORT@
> CPG_LDFLAGS = @CPG_LDFLAGS@
> diff --git a/configure.in b/configure.in
> index 41415f9..146a78d 100644
> --- a/configure.in
> +++ b/configure.in
> @@ -203,6 +203,12 @@ LIBS="$ocfs_tools_save_LIBS"
>
> AC_SUBST(BUILD_DEBUGOCFS2)
>
> +pcmk_found=
> +AC_CHECK_LIB(crmcluster, crm_get_peer,
> + [AC_CHECK_HEADER(pacemaker/crm_config.h, pcmk_found=yes,
> + [AC_MSG_WARN([Pacemaker headers not found, pacemaker support will not be built])])],
> + [AC_MSG_WARN([libcrmcluster not found, pacemaker support will not be built])])
> +
> # We use cman_replyto_shutdown to insure a new enough libcman
> cman_found=
> AC_CHECK_LIB(cman, cman_replyto_shutdown,
> @@ -298,6 +304,12 @@ if test "x$cpg_found" = "xyes" -a "x$libdlmcontrol_found" = "xyes"; then
> fi
> AC_SUBST(BUILD_OCFS2_CONTROLD)
>
> +BUILD_PCMK_SUPPORT=
> +if test "x$pcmk_found" = "xyes" -a "x$BUILD_OCFS2_CONTROLD" = "xyes"; then
> + BUILD_PCMK_SUPPORT=yes
> +fi
> +AC_SUBST(BUILD_PCMK_SUPPORT)
> +
> BUILD_CMAN_SUPPORT=
> if test "x$cman_found" = "xyes" -a "x$BUILD_OCFS2_CONTROLD" = "xyes"; then
> BUILD_CMAN_SUPPORT=yes
> diff --git a/ocfs2_controld/Makefile b/ocfs2_controld/Makefile
> index 5db2cb9..fdb46f8 100644
> --- a/ocfs2_controld/Makefile
> +++ b/ocfs2_controld/Makefile
> @@ -6,12 +6,18 @@ sbindir = $(root_sbindir)
>
> ifneq ($(BUILD_OCFS2_CONTROLD),)
> ifneq ($(BUILD_CMAN_SUPPORT),)
> -SBIN_PROGRAMS = ocfs2_controld.cman
> +SBIN_PROGRAMS += ocfs2_controld.cman
> endif
> UNINST_PROGRAMS = test_client
> endif
>
> -INCLUDES = -I$(TOPDIR)/include -I.
> +ifneq ($(BUILD_PCMK_SUPPORT),)
> +SBIN_PROGRAMS += ocfs2_controld.pcmk
> +# Some pacemaker headers which pacemaker.c includes want this.
> +PCMK_INCLUDES = -I/usr/include/pacemaker -I/usr/include/heartbeat/ $(GLIB_CFLAGS)
> +endif
> +
> +INCLUDES = -I$(TOPDIR)/include -I. $(PCMK_INCLUDES)
> LIBO2CB_LIBS = -L$(TOPDIR)/libo2cb -lo2cb
> LIBO2CB_DEPS = $(TOPDIR)/libo2cb/libo2cb.a
> LIBOCFS2_LIBS = -L$(TOPDIR)/libocfs2 -locfs2
> @@ -39,6 +45,10 @@ CMAN_CFILES = cman.c
> CMAN_DAEMON_CFILES = $(DAEMON_CFILES) $(CMAN_CFILES)
> CMAN_DAEMON_OBJS = $(subst .c,.o,$(CMAN_DAEMON_CFILES))
>
> +PCMK_CFILES = pacemaker.c
> +PCMK_DAEMON_CFILES = $(DAEMON_CFILES) $(PCMK_CFILES)
> +PCMK_DAEMON_OBJS = $(subst .c,.o,$(PCMK_DAEMON_CFILES))
> +
> TEST_CFILES = test_client.c
> TEST_OBJS = $(subst .c,.o,$(TEST_CFILES) $(PROTO_CFILES))
> MANS =
> @@ -49,6 +59,9 @@ DIST_FILES = \
> $(TEST_CFILES) \
> $(UNINST_HFILES) \
> $(addsuffix .in,$(MANS))
> +ocfs2_controld.pcmk: $(PCMK_DAEMON_OBJS) $(LIBO2CB_DEPS)
> + $(LINK) $(GLIB_LIBS) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(OPENAIS_LIBS) \
> + -lcrmcluster
>
> ocfs2_controld.cman: $(CMAN_DAEMON_OBJS) $(LIBO2CB_DEPS)
> $(LINK) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(OPENAIS_LIBS) \
> diff --git a/ocfs2_controld/pacemaker.c b/ocfs2_controld/pacemaker.c
> new file mode 100644
> index 0000000..9fe5a3f
> --- /dev/null
> +++ b/ocfs2_controld/pacemaker.c
> @@ -0,0 +1,308 @@
> +/* -*- mode: c; c-basic-offset: 8; -*-
> + * vim: noexpandtab sw=8 ts=8 sts=0:
> + */
> +
> +/*
> + * Copyright (C) 2008 Novell.
> + *
> + * Some portions Copyright Oracle.
> + *
> + * This copyrighted material is made available to anyone wishing to use,
> + * modify, copy, or redistribute it subject to the terms and conditions
> + * of the GNU General Public License v.2.
> + */
> +
> +#include <stdio.h>
> +#include <string.h>
> +#include <stdlib.h>
> +#include <time.h>
> +#include <errno.h>
> +#include <unistd.h>
> +#include <syslog.h>
> +
> +#include <crm/crm.h>
> +#include <crm/common/cluster.h>
> +
> +#include "ocfs2-kernel/kernel-list.h"
> +#include "o2cb/o2cb.h"
> +
> +#include "ocfs2_controld.h"
> +
> +#include <bzlib.h>
> +#include <crm/crm.h>
> +#include <crm/ais.h>
> +#include <sys/utsname.h>
> +
> +int our_nodeid = 0;
> +static int pcmk_ci;
> +static char * clustername = "pacemaker";
> +extern struct list_head mounts;
> +const char *stackname = "pcmk";
> +
> +extern int ais_fd_async;
> +char *local_node_uname = NULL;
> +
> +int kill_stack_node(int nodeid)
> +{
> + int error = 1;
> +
> + log_debug("killing node %d", nodeid);
> +
> + /* error = cman_kill_node(ch_admin, nodeid); */
> + if (error)
> + log_debug("Unable to kill node %d, %d %d", nodeid, error,
> + errno);
> +
> + return error;
> +}
> +
> +char *nodeid2name(int nodeid) {
> + crm_node_t *node = crm_get_peer(nodeid, NULL);
> +
> + if(node->uname == NULL)
> + return NULL;
> +
> + return strdup(node->uname);
> +}
> +
> +int validate_cluster(const char *cluster)
> +{
> + if (!clustername) {
> + log_error("Trying to validate before pacemaker is alive");
> + return 0;
> + }
> +
> + if (!cluster)
> + return 0;
> +
> + return !strcmp(cluster, clustername);
> +}
> +
> +int get_clustername(const char **cluster)
> +{
> + if (!clustername) {
> + log_error("Trying to validate before pacemaker is alive");
> + return -EIO;
> + }
> +
> + if (!cluster) {
> + log_error("NULL passed!");
> + return -EINVAL;
> + }
> +
> + *cluster = clustername;
> + return 0;
> +}
> +
> +static void dead_pcmk(int ci)
> +{
> + if (ci != pcmk_ci) {
> + log_error("Unknown connection %d", ci);
> + return;
> + }
> +
> + log_error("pacemaker connection died");
> + shutdown_daemon();
> + connection_dead(ci);
> +}
> +
> +void exit_stack(void)
> +{
> + log_debug("closing pacemaker connection");
> + if (ais_fd_async) {
> + close(ais_fd_async);
> + ais_fd_async = 0;
> + }
> + if (ais_fd_sync) {
> + close(ais_fd_sync);
> + ais_fd_sync = 0;
> + }
> +}
> +
> +static void process_pcmk(int ci)
> +{
> + /* ci ::= client number */
> + char *data = NULL;
> + char *uncompressed = NULL;
> + AIS_Message *msg = NULL;
> + SaAisErrorT rc = SA_AIS_OK;
> + mar_res_header_t *header = NULL;
> + static int header_len = sizeof(mar_res_header_t);
> +
> + header = malloc(header_len);
> + memset(header, 0, header_len);
> +
> + errno = 0;
> + rc = saRecvRetry(ais_fd_async, header, header_len);
> + if (rc != SA_AIS_OK) {
> + cl_perror("Receiving message header failed: (%d) %s", rc,
> + ais_error2text(rc));
> + goto bail;
> + } else if(header->size == header_len) {
> + log_error("Empty message: id=%d, size=%d, error=%d, header_len=%d",
> + header->id, header->size, header->error, header_len);
> + goto done;
> + } else if(header->size == 0 || header->size < header_len) {
> + log_error("Mangled header: size=%d, header=%d, error=%d",
> + header->size, header_len, header->error);
> + goto done;
> + } else if(header->error != 0) {
> + log_error("Header contined error: %d", header->error);
> + }
> +
> + header = realloc(header, header->size);
> + /* Use a char* so we can store the remainder into an offset */
> + data = (char*)header;
> +
> + errno = 0;
> + rc = saRecvRetry(ais_fd_async, data+header_len, header->size - header_len);
> + msg = (AIS_Message*)data;
> +
> + if (rc != SA_AIS_OK) {
> + cl_perror("Receiving message body failed: (%d) %s", rc, ais_error2text(rc));
> + goto bail;
> + }
> +
> + data = msg->data;
> + if(msg->is_compressed && msg->size > 0) {
> + int rc = BZ_OK;
> + unsigned int new_size = msg->size;
> +
> + if (check_message_sanity(msg, NULL) == FALSE)
> + goto badmsg;
> +
> + log_debug("Decompressing message data");
> + uncompressed = malloc(new_size);
> + memset(uncompressed, 0, new_size);
> +
> + rc = BZ2_bzBuffToBuffDecompress(
> + uncompressed, &new_size, data, msg->compressed_size,
> + 1, 0);
> +
> + if(rc != BZ_OK) {
> + log_error("Decompression failed: %d", rc);
> + goto badmsg;
> + }
> +
> + CRM_ASSERT(rc == BZ_OK);
> + CRM_ASSERT(new_size == msg->size);
> +
> + data = uncompressed;
> +
> + } else if(check_message_sanity(msg, data) == FALSE) {
> + goto badmsg;
> +
> + } else if(safe_str_eq("identify", data)) {
> + int pid = getpid();
> + char *pid_s = crm_itoa(pid);
> +
> + send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
> + crm_free(pid_s);
> + goto done;
> + }
> +
> + if (msg->header.id == crm_class_members) {
> + xmlNode *xml = string2xml(data);
> +
> + if(xml != NULL) {
> + const char *value = crm_element_value(xml, "id");
> + if(value)
> + crm_peer_seq = crm_int_helper(value, NULL);
> +
> + log_debug("Updating membership %llu", crm_peer_seq);
> + /* crm_log_xml_info(xml, __PRETTY_FUNCTION__); */
> + xml_child_iter(xml, node, crm_update_ais_node(node, crm_peer_seq));
> + crm_calculate_quorum();
> + free_xml(xml);
> + } else {
> + log_error("Invalid peer update: %s", data);
> + }
> + } else {
> + log_error("Unexpected AIS message type: %d", msg->header.id);
> + }
> +
> +done:
> + free(uncompressed);
> + free(msg);
> + return;
> +
> +badmsg:
> + log_error("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
> + " min=%d, total=%d, size=%d, bz2_size=%d",
> + msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
> + ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
> + msg->sender.pid, (int)sizeof(AIS_Message),
> + msg->header.size, msg->size, msg->compressed_size);
> + free(uncompressed);
> + free(msg);
> + return;
> +
> +bail:
> + log_error("AIS connection failed");
> + return;
> +}
> +
> +int setup_stack(void)
> +{
> + int retries = 0;
> + int pid;
> + char *pid_s;
> + int rc = SA_AIS_OK;
> + struct utsname name;
> +
> + crm_peer_init();
> +
> + if (local_node_uname == NULL) {
> + if (uname(&name) < 0) {
> + cl_perror("uname(2) call failed");
> + exit(100);
> + }
> + local_node_uname = crm_strdup(name.nodename);
> + log_debug("Local node name: %s", local_node_uname);
> + }
> +
> + /* 16 := CRM_SERVICE */
> +retry:
> + log_debug("Creating connection to our AIS plugin");
> + rc = saServiceConnect (&ais_fd_sync, &ais_fd_async, 16);
> + if (rc != SA_AIS_OK) {
> + log_error("Connection to our AIS plugin failed: %s (%d)",
> + ais_error2text(rc), rc);
> + }
> +
> + switch(rc) {
> + case SA_AIS_OK:
> + break;
> + case SA_AIS_ERR_TRY_AGAIN:
> + if(retries < 30) {
> + sleep(1);
> + retries++;
> + goto retry;
> + }
> + log_error("Retry count exceeded");
> + return 0;
> + default:
> + return 0;
> + }
> +
> + log_debug("AIS connection established");
> +
> + pid = getpid();
> + pid_s = crm_itoa(pid);
> + send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
> + crm_free(pid_s);
> +
> + /* Sign up for membership updates */
> + send_ais_text(crm_class_notify, "true", TRUE, NULL, crm_msg_ais);
> +
> + /* Requesting the current list of known nodes */
> + send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais);
> +
> + pcmk_ci = connection_add(ais_fd_async, process_pcmk, dead_pcmk);
> + if (pcmk_ci >= 0)
> + return ais_fd_async;
> +
> + log_error("Unable to add pacemaker client: %s", strerror(-pcmk_ci));
> + exit_stack();
> + return pcmk_ci;
> +}
> --
> 1.5.4.1
>
>
--
"But then she looks me in the eye
And says, 'We're going to last forever,'
And man you know I can't begin to doubt it.
Cause it just feels so good and so free and so right,
I know we ain't never going to change our minds about it, Hey!
Here comes my girl."
Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127
More information about the Ocfs2-tools-devel
mailing list