[Ocfs2-tools-devel] [PATCH 1/1] o2hbmonitor: Disk heartbeat monitor
Sunil Mushran
sunil.mushran at oracle.com
Mon Oct 25 15:21:32 PDT 2010
o2hbmonitor is a very light utility that monitors o2cb disk heartbeat.
It periodically reads o2hb debugfs file, elapsed_time_in_ms, and checks
whether the time is greater than the warn threshold. if so, it prints a
message in syslog.
Signed-off-by: Sunil Mushran <sunil.mushran at oracle.com>
---
Makefile | 2 +-
o2monitor/.gitignore | 3 +
o2monitor/Makefile | 26 ++
o2monitor/o2hbmonitor.c | 430 +++++++++++++++++++++++++++++
vendor/common/ocfs2-tools.spec-generic.in | 1 +
5 files changed, 461 insertions(+), 1 deletions(-)
create mode 100644 o2monitor/.gitignore
create mode 100644 o2monitor/Makefile
create mode 100644 o2monitor/o2hbmonitor.c
diff --git a/Makefile b/Makefile
index 88106fb..65c13f9 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ CHKCONFIG_DEP = chkconfig
COMPILE_PY = 1
endif
-SUBDIRS = include libtools-internal libo2dlm libo2cb libocfs2 fsck.ocfs2 mkfs.ocfs2 mounted.ocfs2 tunefs.ocfs2 debugfs.ocfs2 o2cb_ctl ocfs2_hb_ctl mount.ocfs2 ocfs2_controld o2image listuuid sizetest extras fswreck patches
+SUBDIRS = include libtools-internal libo2dlm libo2cb libocfs2 fsck.ocfs2 mkfs.ocfs2 mounted.ocfs2 tunefs.ocfs2 debugfs.ocfs2 o2cb_ctl ocfs2_hb_ctl mount.ocfs2 ocfs2_controld o2image o2monitor listuuid sizetest extras fswreck patches
ifdef BUILD_OCFS2CONSOLE
SUBDIRS += ocfs2console
diff --git a/o2monitor/.gitignore b/o2monitor/.gitignore
new file mode 100644
index 0000000..323fba6
--- /dev/null
+++ b/o2monitor/.gitignore
@@ -0,0 +1,3 @@
+.*.sw?
+*.d
+o2hbmonitor
diff --git a/o2monitor/Makefile b/o2monitor/Makefile
new file mode 100644
index 0000000..961eafa
--- /dev/null
+++ b/o2monitor/Makefile
@@ -0,0 +1,26 @@
+TOPDIR = ..
+
+include $(TOPDIR)/Preamble.make
+
+sbindir = $(root_sbindir)
+SBIN_PROGRAMS = o2hbmonitor
+
+WARNINGS = -Wall -Wstrict-prototypes -Wno-format -Wmissing-prototypes \
+ -Wmissing-declarations
+
+CFLAGS = $(OPTS) $(WARNINGS)
+
+INCLUDES = -I$(TOPDIR)/include -I.
+
+DEFINES = -DVERSION=\"$(VERSION)\"
+
+CFILES = o2hbmonitor.c
+
+OBJS = $(subst .c,.o,$(CFILES))
+
+DIST_FILES = $(CFILES) $(HFILES)
+
+o2hbmonitor: $(OBJS)
+ $(LINK)
+
+include $(TOPDIR)/Postamble.make
diff --git a/o2monitor/o2hbmonitor.c b/o2monitor/o2hbmonitor.c
new file mode 100644
index 0000000..dd74f44
--- /dev/null
+++ b/o2monitor/o2hbmonitor.c
@@ -0,0 +1,430 @@
+/* -*- mode: c; c-basic-offset: 8; -*-
+ * vim: noexpandtab sw=8 ts=8 sts=0:
+ *
+ * o2hbmonitor.c
+ *
+ * Monitors o2hb
+ *
+ * Copyright (C) 2010 Oracle. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/*
+ * This utility requires the o2hb debugfs file elapsed_time_in_ms which shows
+ * the time since the o2hb heartbeat timer was last armed. This file was added
+ * in the mainline kernel via commit 43695d095dfaf266a8a940d9b07eed7f46076b49.
+ *
+ * This utility scans the configfs to load all the heartbeat regions and the
+ * dead_threshold. If no cluster is found, it checks after CONFIG_POLL_IN_SECS.
+ *
+ * The debugfs file, elapsed_time_in_ms, for each region is scanned every
+ * SLOW_POLL_IN_SECS. If a region is found to have an elpased time > warn
+ * threshold, a message is logged to syslog.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <linux/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <string.h>
+#include <libgen.h>
+#include <syslog.h>
+#include <errno.h>
+
+#define SYS_CONFIG_DIR "/sys/kernel/config"
+#define O2HB_CLUSTER_DIR SYS_CONFIG_DIR"/cluster"
+#define O2HB_HEARTBEAT_DIR O2HB_CLUSTER_DIR"/%s/heartbeat"
+#define O2HB_DEAD_THRESHOLD O2HB_HEARTBEAT_DIR"/dead_threshold"
+#define O2HB_DEVICE O2HB_HEARTBEAT_DIR"/%s/dev"
+
+#define SYS_DEBUG_DIR "/sys/kernel/debug"
+#define O2HB_DEBUG_DIR SYS_DEBUG_DIR"/o2hb"
+#define O2HB_ELAPSED_TIME O2HB_DEBUG_DIR"/%s/elapsed_time_in_ms"
+
+#define MAX_REGIONS 200
+
+#define DEAD_THRESHOLD_IN_MSECS(a) (((a) - 1) * 2000)
+#define WARN_THRESHOLD_PERCENT 50
+
+#define CONFIG_POLL_IN_SECS 60
+#define SLOW_POLL_IN_SECS 10
+#define FAST_POLL_IN_SECS 2
+
+char *progname;
+int interactive;
+int warn_threshold_percent;
+int verbose;
+
+struct o2hb_region {
+ char *name;
+ char *device;
+ unsigned long elapsed_in_ms;
+};
+
+struct o2hb_config {
+ char *cluster_name;
+ unsigned long dead_threshold_in_ms;
+ unsigned long warn_threshold_in_ms;
+ unsigned long poll_in_secs;
+ int num_regions;
+ struct o2hb_region regions[MAX_REGIONS];
+};
+
+static void show_version(void)
+{
+ fprintf(stderr, "%s %s\n", progname, VERSION);
+}
+
+static void free_config(struct o2hb_config **oc)
+{
+ int i;
+
+ if (!*oc)
+ return;
+
+ if ((*oc)->cluster_name)
+ free((*oc)->cluster_name);
+
+ for (i = 0; i < (*oc)->num_regions; ++i) {
+ if ((*oc)->regions[i].name)
+ free((*oc)->regions[i].name);
+ if ((*oc)->regions[i].device)
+ free((*oc)->regions[i].device);
+ }
+
+ free(*oc);
+ *oc = NULL;
+}
+
+static int alloc_config(struct o2hb_config **oc)
+{
+ *oc = calloc(sizeof(struct o2hb_config), 1);
+ if (!*oc)
+ return -1;
+
+ return 0;
+}
+
+static char *do_strchomp(char *str)
+{
+ int len = strlen(str);
+ char *p;
+
+ if (!len)
+ return str;
+
+ p = str + len - 1;
+ while ((isspace(*p) || *p == '\n') && len--)
+ *p-- = '\0';
+
+ return str;
+}
+
+static int get_value(char *path, char *value, int count)
+{
+ int fd = -1, ret = -1;
+ char *p = value;
+
+ fd = open(path, O_RDONLY);
+ if (fd > 0)
+ ret = read(fd, value, count);
+ if (ret > 0) {
+ p += ret;
+ *p = '\0';
+ ret = 0;
+ }
+
+ if (!ret)
+ do_strchomp(value);
+
+ if (fd > -1)
+ close(fd);
+ return ret;
+}
+
+static int populate_elapsed_time(struct o2hb_config *oc)
+{
+ int i, ret;
+ char val[32];
+ char path[PATH_MAX];
+ struct o2hb_region *reg;
+
+ for (i = 0; i < oc->num_regions; ++i) {
+ reg = &(oc->regions[i]);
+ sprintf(path, O2HB_ELAPSED_TIME, reg->name);
+ ret = get_value(path, val, sizeof(val));
+ if (ret)
+ goto bail;
+ reg->elapsed_in_ms = strtoul(val, NULL, 0);
+ }
+
+ ret = 0;
+
+bail:
+ return ret;
+
+}
+
+static int populate_thresholds(struct o2hb_config *oc)
+{
+ int ret;
+ char val[32];
+ char path[PATH_MAX];
+
+ sprintf(path, O2HB_DEAD_THRESHOLD, oc->cluster_name);
+ ret = get_value(path, val, sizeof(val));
+ if (!ret) {
+ oc->dead_threshold_in_ms =
+ DEAD_THRESHOLD_IN_MSECS(strtoul(val, NULL, 0));
+ oc->warn_threshold_in_ms =
+ (oc->dead_threshold_in_ms * warn_threshold_percent / 100);
+ }
+
+ return ret;
+}
+
+static int populate_devices(struct o2hb_config *oc)
+{
+ int i, ret;
+ char val[255];
+ char path[PATH_MAX];
+ struct o2hb_region *reg;
+
+ for (i = 0; i < oc->num_regions; ++i) {
+ reg = &(oc->regions[i]);
+ sprintf(path, O2HB_DEVICE, oc->cluster_name, reg->name);
+ ret = get_value(path, val, sizeof(val));
+ if (ret)
+ goto bail;
+ reg->device = strdup(val);
+ if (!reg->device) {
+ ret = -1;
+ goto bail;
+ }
+ }
+
+ ret = 0;
+
+bail:
+ return ret;
+}
+
+static int populate_regions(struct o2hb_config *oc)
+{
+ int ret = -1;
+ DIR *dir = NULL;
+ struct dirent *ent;
+ struct o2hb_region *reg;
+ char path[PATH_MAX];
+
+ sprintf(path, O2HB_HEARTBEAT_DIR, oc->cluster_name);
+
+ dir = opendir(path);
+ if (!dir)
+ goto bail;
+
+ do {
+ ent = readdir(dir);
+
+ if (ent && ent->d_type == 4 && strcmp(ent->d_name, ".") &&
+ strcmp(ent->d_name, "..")) {
+ reg = &(oc->regions[oc->num_regions]);
+ reg->name = strdup(ent->d_name);
+ if (!reg->name)
+ goto bail;
+ ++oc->num_regions;
+ }
+ } while (ent);
+
+ ret = 0;
+
+bail:
+ if (dir)
+ closedir(dir);
+ return ret;
+}
+
+static int populate_cluster_name(struct o2hb_config *oc)
+{
+ DIR *dir;
+ struct dirent *ent;
+
+ dir = opendir(O2HB_CLUSTER_DIR);
+ if (!dir)
+ goto bail;
+
+ do {
+ ent = readdir(dir);
+ if (ent && ent->d_type == 4 && strcmp(ent->d_name, ".") &&
+ strcmp(ent->d_name, "..")) {
+ oc->cluster_name = strdup(ent->d_name);
+ break;
+ }
+ } while (ent);
+
+ closedir(dir);
+
+bail:
+ if (!oc->cluster_name)
+ return -1;
+ return 0;
+}
+
+/*
+ * Reads config for all the heartbeat regions
+ */
+static int load_config(struct o2hb_config **oc)
+{
+ struct stat buf;
+ int ret;
+ static int warn_count = 0;
+
+ ret = stat(O2HB_CLUSTER_DIR, &buf);
+ if (ret)
+ goto bail;
+
+ ret = stat(O2HB_DEBUG_DIR, &buf);
+ if (ret) {
+ if (!(warn_count++ % 60))
+ syslog(LOG_WARNING,
+ "mount debugfs at /sys/kernel/debug");
+ goto bail;
+ }
+
+ ret = alloc_config(oc);
+ if (!ret)
+ ret = populate_cluster_name(*oc);
+ if (!ret)
+ ret = populate_thresholds(*oc);
+ if (!ret)
+ ret = populate_regions(*oc);
+ if (!ret)
+ ret = populate_devices(*oc);
+
+bail:
+ return ret;
+}
+
+static void scan_elapsed_time(struct o2hb_config *oc)
+{
+ int i;
+ struct o2hb_region *reg;
+
+ oc->poll_in_secs = SLOW_POLL_IN_SECS;
+ for (i = 0; i < oc->num_regions; ++i) {
+ reg = &(oc->regions[i]);
+ if (verbose)
+ fprintf(stdout, "%s, %s, %lu\n", reg->name, reg->device,
+ reg->elapsed_in_ms);
+ if (reg->elapsed_in_ms < oc->warn_threshold_in_ms)
+ continue;
+ oc->poll_in_secs = FAST_POLL_IN_SECS;
+ syslog(LOG_WARNING, "Last ping %lu msecs ago on /dev/%s, %s\n",
+ reg->elapsed_in_ms, reg->device, reg->name);
+ }
+}
+
+static void monitor(void)
+{
+ int ret = -1;
+ struct o2hb_config *oc = NULL;
+
+ while (1) {
+ if (!oc) {
+ ret = load_config(&oc);
+ if (ret) {
+ free_config(&oc);
+ sleep(CONFIG_POLL_IN_SECS);
+ continue;
+ }
+ }
+
+ ret = populate_elapsed_time(oc);
+ if (ret) {
+ free_config(&oc);
+ continue;
+ }
+
+ scan_elapsed_time(oc);
+
+ sleep(oc->poll_in_secs);
+ }
+}
+
+static void usage(void)
+{
+ fprintf(stderr, "usage: %s [-w percent] -[ivV]\n", progname);
+ fprintf(stderr, "\t -w, Warn threshold percent (default 50%%)\n");
+ fprintf(stderr, "\t -i, Interactive\n");
+ fprintf(stderr, "\t -v, Verbose\n");
+ fprintf(stderr, "\t -V, Version\n");
+ exit(1);
+}
+
+int main(int argc, char **argv)
+{
+ int c, ret, version = 0;
+
+ /* init globals */
+ progname = basename(argv[0]);
+ interactive = 0;
+ warn_threshold_percent = WARN_THRESHOLD_PERCENT;
+ verbose = 0;
+
+ while (1) {
+ c = getopt(argc, argv, "w:i?hvV");
+ if (c == -1)
+ break;
+ switch (c) {
+ case 'i':
+ interactive = 1;
+ break;
+ case 'v':
+ ++verbose;
+ break;
+ case 'w':
+ warn_threshold_percent = strtoul(optarg, NULL, 0);
+ if (warn_threshold_percent < 1 ||
+ warn_threshold_percent > 99)
+ warn_threshold_percent = WARN_THRESHOLD_PERCENT;
+ break;
+ case 'V':
+ version = 1;
+ break;
+ case '?':
+ case 'h':
+ default:
+ usage();
+ break;
+ }
+ }
+
+ if (version)
+ show_version();
+
+ if (!interactive) {
+ ret = daemon(0, verbose);
+ if (ret)
+ fprintf(stderr, "Unable to daemonize, %s\n",
+ strerror(errno));
+ }
+
+ openlog(progname, LOG_CONS|LOG_NDELAY, LOG_DAEMON);
+ monitor();
+ closelog();
+
+ return 0;
+}
diff --git a/vendor/common/ocfs2-tools.spec-generic.in b/vendor/common/ocfs2-tools.spec-generic.in
index 3e9b46a..44a65cd 100644
--- a/vendor/common/ocfs2-tools.spec-generic.in
+++ b/vendor/common/ocfs2-tools.spec-generic.in
@@ -118,6 +118,7 @@ fi
/sbin/o2cb_ctl
/sbin/mount.ocfs2
/sbin/o2image
+/usr/sbin/o2hbmonitor
/sbin/ocfs2_hb_ctl
/etc/init.d/o2cb
/etc/init.d/ocfs2
--
1.7.1
More information about the Ocfs2-tools-devel
mailing list