[Ocfs2-tools-devel] [PATCH 39/39] ocfs2_controld: Integrate ckpt
with the daemon
Joel Becker
joel.becker at oracle.com
Fri Mar 14 16:53:02 PDT 2008
The daemon code should now install and fill its checkpoint objects.
Signed-off-by: Joel Becker <joel.becker at oracle.com>
---
ocfs2_controld/Makefile | 6 +-
ocfs2_controld/ckpt.c | 6 +-
ocfs2_controld/cpg.c | 6 +-
ocfs2_controld/main.c | 316 ++++++++++++++++++++++++++++++++++++++-
ocfs2_controld/ocfs2_controld.h | 18 ++-
5 files changed, 335 insertions(+), 17 deletions(-)
diff --git a/ocfs2_controld/Makefile b/ocfs2_controld/Makefile
index 1888728..af70ac4 100644
--- a/ocfs2_controld/Makefile
+++ b/ocfs2_controld/Makefile
@@ -14,7 +14,7 @@ LIBO2CB_LIBS = -L$(TOPDIR)/libo2cb -lo2cb
LIBO2CB_DEPS = $(TOPDIR)/libo2cb/libo2cb.a
LIBOCFS2_LIBS = -L$(TOPDIR)/libocfs2 -locfs2
LIBOCFS2_DEPS = $(TOPDIR)/libocfs2/libocfs2.a
-LIBCPG_LIBS = $(CPG_LDFLAGS) -lcpg
+OPENAIS_LIBS = $(CPG_LDFLAGS) -lcpg -lSaCkpt
ifdef OCFS2_DEBUG
OPTS += -ggdb
@@ -30,7 +30,7 @@ DEFINES = -DOCFS2_FLAT_INCLUDES -DO2DLM_FLAT_INCLUDES \
UNINST_HFILES = ocfs2_controld.h
-DAEMON_CFILES = main.c cman.c cpg.c mount.c
+DAEMON_CFILES = main.c cman.c cpg.c mount.c ckpt.c
TEST_CFILES = test_client.c
DAEMON_OBJS = $(subst .c,.o,$(DAEMON_CFILES))
@@ -44,7 +44,7 @@ DIST_FILES = \
$(addsuffix .in,$(MANS))
ocfs2_controld.cman: $(DAEMON_OBJS) $(LIBO2CB_DEPS)
- $(LINK) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(LIBCPG_LIBS) -lcman
+ $(LINK) $(LIBO2CB_LIBS) $(COM_ERR_LIBS) $(OPENAIS_LIBS) -lcman
test_client: $(TEST_OBJS) $(LIBO2CB_DEPS) $(LIBOCFS2_DEPS)
$(LINK) $(LIBOCFS2_LIBS) $(LIBO2CB_LIBS) $(COM_ERR_LIBS)
diff --git a/ocfs2_controld/ckpt.c b/ocfs2_controld/ckpt.c
index 87b94e0..f73990f 100644
--- a/ocfs2_controld/ckpt.c
+++ b/ocfs2_controld/ckpt.c
@@ -525,7 +525,7 @@ static void ckpt_free(struct ckpt_handle *handle)
free(handle);
}
-static int ckpt_open_global(int write)
+int ckpt_open_global(int write)
{
if (global_handle)
return 0;
@@ -533,7 +533,7 @@ static int ckpt_open_global(int write)
return ckpt_new("controld", write, &global_handle);
}
-static void ckpt_close_global(void)
+void ckpt_close_global(void)
{
if (global_handle) {
ckpt_free(global_handle);
@@ -624,6 +624,7 @@ void exit_ckpt(void)
}
}
+#ifdef DEBUG_EXE
int dump_point, dump_wrap, daemon_debug_opt = 1;
char daemon_debug_buf[1024];
char dump_buf[DUMP_SIZE];
@@ -701,3 +702,4 @@ out_exit:
out:
return rc;
}
+#endif /* DEBUG_EXE */
diff --git a/ocfs2_controld/cpg.c b/ocfs2_controld/cpg.c
index 7cd22ea..0c2f48c 100644
--- a/ocfs2_controld/cpg.c
+++ b/ocfs2_controld/cpg.c
@@ -724,17 +724,17 @@ out:
static void daemon_set_cgroup(struct cgroup *cg, void *user_data)
{
- void (*daemon_joined)(void) = user_data;
+ void (*daemon_joined)(int first) = user_data;
if (cg != &daemon_group) {
log_error("Somehow, daemon_set_cgroup is called on a different group!");
return;
}
- daemon_joined();
+ daemon_joined(daemon_group.cg_member_count == 1);
}
-int setup_cpg(void (*daemon_joined)(void))
+int setup_cpg(void (*daemon_joined)(int first))
{
cpg_error_t error;
diff --git a/ocfs2_controld/main.c b/ocfs2_controld/main.c
index a13d6ad..c8126f2 100644
--- a/ocfs2_controld/main.c
+++ b/ocfs2_controld/main.c
@@ -46,6 +46,13 @@
#define LOCKFILE_NAME "/var/run/ocfs2_controld.pid"
#define NALLOC 8
+#define CONTROLD_PROTOCOL_MAJOR 1
+#define CONTROLD_PROTOCOL_MINOR 0
+#define DAEMON_MAX_PROTOCOL_SECTION "daemon_max_protocol"
+#define DAEMON_PROTOCOL_SECTION "daemon_protocol"
+#define FS_MAX_PROTOCOL_SECTION "ocfs2_max_protocol"
+#define FS_PROTOCOL_SECTION "ocfs2_protocol"
+
struct client {
int fd;
char type[32];
@@ -72,6 +79,45 @@ char dump_buf[DUMP_SIZE];
int dump_point;
int dump_wrap;
+/*
+ * Protocol negotiation.
+ *
+ * This is the maximum protocol supported by the daemon for inter-daemon
+ * communication. The negotiated value daemon_running_proto is what the
+ * daemon uses at runtime.
+ *
+ * All daemons must support the initial protocol, which works as follows:
+ * Prior to starting CPG, daemons store two values in the local node
+ * checkpoint. The maximum daemon protocol is stored in the
+ * "daemon_max_protocol" section, and the ocfs2 maximum protocol is stored
+ * in the "ocfs2_max_protocol" section. The protocols are stored in the
+ * format:
+ *
+ * <2-char-hex-major><space><2-char-hex-minor><null>
+ *
+ * These sections MUST be created before CPG is started. Other sections
+ * MUST NOT be created at this time.
+ *
+ * Once CPG is started, the daemon reads the "daemon_protocol" and
+ * "ocfs2_protocol" sections from the daemon's global checkpoint. The
+ * values are stored as the running versions. All interaction takes place
+ * based on the running versions. At this point, the daemon may add
+ * other sections to the local node checkpoint that are part of the
+ * running protocol.
+ *
+ * If the daemon is the first node to join the group, it sets the
+ * "daemon_protocol" and "ocfs2_protocol" sections of the global checkpoint
+ * to the maximum values this daemon supports.
+ */
+static struct ocfs2_protocol_version daemon_max_proto = {
+ .pv_major = CONTROLD_PROTOCOL_MAJOR,
+ .pv_minor = CONTROLD_PROTOCOL_MINOR,
+};
+static struct ocfs2_protocol_version fs_max_proto;
+struct ocfs2_protocol_version daemon_running_proto;
+struct ocfs2_protocol_version fs_running_proto;
+struct ckpt_handle *node_handle;
+
void shutdown_daemon(void)
{
time_to_die = 1;
@@ -568,23 +614,267 @@ static int setup_listener(void)
return 0;
}
-static void cpg_joined(void)
+static int proto_version_to_checkpoint(struct ocfs2_protocol_version *proto,
+ char **checkpoint_data)
{
- int rv;
- errcode_t err;
- struct ocfs2_protocol_version proto;
+ size_t len;
+ char *buf;
+
+ len = snprintf(NULL, 0, "%02x %02x", proto->pv_major,
+ proto->pv_minor);
+ buf = malloc(sizeof(char) * (len + 1));
+ if (!buf) {
+ log_error("Unable to allocate memory for checkpoint data");
+ return -ENOMEM;
+ }
- log_debug("CPG is live, opening control device");
+ snprintf(buf, len + 1, "%02x %02x", proto->pv_major,
+ proto->pv_minor);
+ *checkpoint_data = buf;
- err = o2cb_get_max_locking_protocol(&proto);
+ return 0;
+}
+
+static int checkpoint_to_proto_version(char *data, size_t data_len,
+ struct ocfs2_protocol_version *proto)
+{
+ long major, minor;
+ char *ptr;
+ struct proto_version_str {
+ char major[2];
+ char space;
+ char minor[2];
+ char null;
+ } str;
+
+ if (data_len != sizeof(struct proto_version_str)) {
+ log_error("Protocol version string \"%.*s\" has incorrect "
+ "length",
+ data_len, data);
+ return -EINVAL;
+ }
+ memcpy((char *)&str, data, data_len);
+
+ if ((str.space != ' ') || (str.null != '\0')) {
+ log_error("Protocol version string \"%.*s\" has invalid "
+ "separators",
+ data_len, data);
+ return -EINVAL;
+ }
+ str.space = '\0';
+
+ major = strtol(str.major, &ptr, 16);
+ if (!ptr || *ptr) {
+ log_error("Protocol request has bad version 0x%s 0x%s",
+ str.major, str.minor);
+ return -EINVAL;
+ }
+ minor = strtol(str.minor, &ptr, 16);
+ if (!ptr || *ptr) {
+ log_error("Protocol version string has bad version 0x%s 0x%s",
+ str.major, str.minor);
+ return -EINVAL;
+ }
+
+ /* The major and minor must be between 0 and 255, inclusive. */
+ if ((major == LONG_MIN) || (major == LONG_MAX) ||
+ (minor == LONG_MIN) || (minor == LONG_MAX) ||
+ (major > (uint8_t)-1) || (major < 0) ||
+ (minor > (uint8_t)-1) || (minor < 0)) {
+ log_error("Protocol version string has bad version 0x%s 0x%s",
+ str.major, str.minor);
+ return -ERANGE;
+ }
+
+ proto->pv_major = major;
+ proto->pv_minor = minor;
+
+ return 0;
+}
+
+static int install_node_checkpoint(void)
+{
+ int rc;
+ char *buf;
+ errcode_t err;
+
+ err = o2cb_get_max_locking_protocol(&fs_max_proto);
if (err) {
- log_error("Error querying maximum locking protocol: %s",
+ log_error("Error querying maximum filesystem locking "
+ "protocol: %s",
error_message(err));
+ rc = -EIO;
+ goto out;
+ }
+
+ rc = ckpt_open_this_node(&node_handle);
+ if (rc)
+ goto out;
+
+ rc = proto_version_to_checkpoint(&daemon_max_proto, &buf);
+ if (rc)
+ goto out;
+
+ rc = ckpt_section_store(node_handle, DAEMON_MAX_PROTOCOL_SECTION,
+ buf, strlen(buf) + 1);
+ free(buf);
+ if (rc)
+ goto out;
+
+ rc = proto_version_to_checkpoint(&fs_max_proto, &buf);
+ if (rc)
+ goto out;
+
+ rc = ckpt_section_store(node_handle, FS_MAX_PROTOCOL_SECTION,
+ buf, strlen(buf) + 1);
+ free(buf);
+
+out:
+ return rc;
+}
+
+static void drop_node_checkpoint(void)
+{
+ if (node_handle)
+ ckpt_close(node_handle);
+}
+
+/*
+ * If we're the only daemon running, install our maximum protocols
+ * as the running values.
+ */
+static int install_global_checkpoint(void)
+{
+ int rc;
+ char *buf;
+
+ daemon_running_proto = daemon_max_proto;
+ fs_running_proto = fs_max_proto;
+
+ rc = ckpt_open_global(1);
+ if (rc)
+ goto out;
+
+ rc = proto_version_to_checkpoint(&daemon_running_proto, &buf);
+ if (rc)
+ goto out;
+
+ rc = ckpt_global_store(DAEMON_PROTOCOL_SECTION, buf,
+ strlen(buf) + 1);
+ free(buf);
+ if (rc)
+ goto out;
+
+ rc = proto_version_to_checkpoint(&fs_running_proto, &buf);
+ if (rc)
+ goto out;
+
+ rc = ckpt_global_store(FS_PROTOCOL_SECTION, buf, strlen(buf) + 1);
+ free(buf);
+
+out:
+ return rc;
+}
+
+/*
+ * Compare the cluster's locking protocol version against our maximum.
+ *
+ * If the major numbers are different, they are incompatible.
+ * If the cluster's minor is greater than our maximum minor, they are
+ * incompatible.
+ */
+static int protocol_compatible(struct ocfs2_protocol_version *cluster,
+ struct ocfs2_protocol_version *our_max)
+{
+ if (cluster->pv_major != our_max->pv_major)
+ return 0;
+
+ if (cluster->pv_minor > our_max->pv_minor)
+ return 0;
+
+ return 1;
+}
+
+static int read_global_checkpoint(void)
+{
+ int rc;
+ char *buf;
+ size_t len;
+
+ rc = ckpt_open_global(0);
+ if (rc)
+ goto out;
+
+ rc = ckpt_global_get(DAEMON_PROTOCOL_SECTION, &buf, &len);
+ if (rc)
+ goto out;
+
+ rc = checkpoint_to_proto_version(buf, len, &daemon_running_proto);
+ free(buf);
+ if (rc)
+ goto out;
+ if (!protocol_compatible(&daemon_running_proto,
+ &daemon_max_proto)) {
+ log_error("Our maximum daemon protocol (%d.%d) is not "
+ "compatible with the cluster's protocol (%d.%d)",
+ daemon_max_proto.pv_major,
+ daemon_max_proto.pv_minor,
+ daemon_running_proto.pv_major,
+ daemon_running_proto.pv_minor);
+ rc = -EPROTONOSUPPORT;
+ goto out;
+ }
+
+ rc = ckpt_global_get(FS_PROTOCOL_SECTION, &buf, &len);
+ if (rc)
+ goto out;
+
+ rc = checkpoint_to_proto_version(buf, len, &fs_running_proto);
+ free(buf);
+ if (rc)
+ goto out;
+
+ if (!protocol_compatible(&fs_running_proto,
+ &fs_max_proto)) {
+ log_error("Our maximum fs protocol (%d.%d) is not "
+ "compatible with the cluster's protocol (%d.%d)",
+ fs_max_proto.pv_major,
+ fs_max_proto.pv_minor,
+ fs_running_proto.pv_major,
+ fs_running_proto.pv_minor);
+ rc = -EPROTONOSUPPORT;
+ }
+
+out:
+ return rc;
+}
+
+static void cpg_joined(int first)
+{
+ int rv;
+ errcode_t err;
+
+ log_debug("CPG is live, we are %s first daemon",
+ first ? "the" : "not the");
+
+ if (first)
+ rv = install_global_checkpoint();
+ else
+ rv = read_global_checkpoint();
+
+ if (rv) {
shutdown_daemon();
return;
}
- err = o2cb_control_open(our_nodeid, &proto);
+ log_debug("Daemon protocol is %d.%d",
+ daemon_running_proto.pv_major,
+ daemon_running_proto.pv_minor);
+ log_debug("fs protocol is %d.%d",
+ fs_running_proto.pv_major, fs_running_proto.pv_minor);
+
+ log_debug("Opening control device");
+ err = o2cb_control_open(our_nodeid, &fs_running_proto);
if (err) {
log_error("Error opening control device: %s",
error_message(err));
@@ -612,6 +902,14 @@ static int loop(void)
if (rv < 0)
goto out;
+ rv = setup_ckpt();
+ if (rv < 0)
+ goto out;
+
+ rv = install_node_checkpoint();
+ if (rv < 0)
+ goto out;
+
rv = setup_cpg(cpg_joined);
if (rv < 0)
goto out;
@@ -654,6 +952,8 @@ stop:
o2cb_control_close();
exit_cpg();
+ drop_node_checkpoint();
+ exit_ckpt();
exit_cman();
out:
diff --git a/ocfs2_controld/ocfs2_controld.h b/ocfs2_controld/ocfs2_controld.h
index 6c6e77d..101bba2 100644
--- a/ocfs2_controld/ocfs2_controld.h
+++ b/ocfs2_controld/ocfs2_controld.h
@@ -29,6 +29,7 @@
struct cgroup;
+struct ckpt_handle;
extern char *prog_name;
extern int daemon_debug_opt;
@@ -60,6 +61,21 @@ int connection_add(int fd, void (*work)(int ci), void (*dead)(int ci));
void connection_dead(int ci);
void shutdown_daemon(void);
+/* ckpt.c */
+int setup_ckpt(void);
+void exit_ckpt(void);
+int ckpt_open_global(int write);
+void ckpt_close_global(void);
+int ckpt_open_node(int nodeid, struct ckpt_handle **handle);
+int ckpt_open_this_node(struct ckpt_handle **handle);
+void ckpt_close(struct ckpt_handle *handle);
+int ckpt_global_store(const char *section, const char *data, size_t data_len);
+int ckpt_global_get(const char *section, char **data, size_t *data_len);
+int ckpt_section_store(struct ckpt_handle *handle, const char *section,
+ const char *data, size_t data_len);
+int ckpt_section_get(struct ckpt_handle *handle, const char *section,
+ char **data, size_t *data_len);
+
/* cman.c */
int setup_cman(void);
char *nodeid2name(int nodeid);
@@ -69,7 +85,7 @@ int kill_cman(int nodeid);
void exit_cman(void);
/* cpg.c */
-int setup_cpg(void (*daemon_joined)(void));
+int setup_cpg(void (*daemon_joined)(int first));
void exit_cpg(void);
void for_each_node(struct cgroup *cg,
void (*func)(int nodeid,
--
1.5.3.8
More information about the Ocfs2-tools-devel
mailing list