[Ocfs2-commits] smushran commits r2990 -
branches/ocfs2-1.2/fs/ocfs2/cluster
svn-commits at oss.oracle.com
svn-commits at oss.oracle.com
Tue Mar 13 12:03:25 PDT 2007
Author: smushran
Date: 2007-03-13 12:03:23 -0700 (Tue, 13 Mar 2007)
New Revision: 2990
Modified:
branches/ocfs2-1.2/fs/ocfs2/cluster/nodemanager.c
branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c
branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.h
branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h
Log:
ocfs2: [patch 3/3] OCFS2 Configurable timeouts - Protocol changes
Modify the OCFS2 handshake to ensure essential timeouts are configured
identically on all nodes.
Only allow changes when there are no connected peers
Improves the logic in o2net_advance_rx() which broke now that
sizeof(struct o2net_handshake) is greater than sizeof(struct o2net_msg)
Included is the field for userspace-heartbeat timeout to avoid the need for
further protocol changes.
Uses a global spinlock to ensure the decisions to update configfs entries
are made on the correct value. The region covered by the spinlock when
incrimenting the counter is much larger as this is the more critical case.
Authored by: Andrew Beekhof <abeekhof at suse.de>
Ported to 1.2 by: Jeff Mahoney <jmahoney at suse.de>
Signed-off-by: mfasheh
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/nodemanager.c
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/nodemanager.c 2007-03-13 18:57:50 UTC (rev 2989)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/nodemanager.c 2007-03-13 19:03:23 UTC (rev 2990)
@@ -571,12 +571,21 @@
ret = o2nm_cluster_attr_write(page, count, &val);
if (ret > 0) {
- if (val <= cluster->cl_keepalive_delay_ms) {
+ if (cluster->cl_idle_timeout_ms != val
+ && o2net_num_connected_peers()) {
+ mlog(ML_NOTICE,
+ "o2net: cannot change idle timeout after "
+ "the first peer has agreed to it."
+ " %d connected peers\n",
+ o2net_num_connected_peers());
+ ret = -EINVAL;
+ } else if (val <= cluster->cl_keepalive_delay_ms) {
mlog(ML_NOTICE, "o2net: idle timeout must be larger "
"than keepalive delay\n");
- return -EINVAL;
+ ret = -EINVAL;
+ } else {
+ cluster->cl_idle_timeout_ms = val;
}
- cluster->cl_idle_timeout_ms = val;
}
return ret;
@@ -597,12 +606,21 @@
ret = o2nm_cluster_attr_write(page, count, &val);
if (ret > 0) {
- if (val >= cluster->cl_idle_timeout_ms) {
+ if (cluster->cl_keepalive_delay_ms != val
+ && o2net_num_connected_peers()) {
+ mlog(ML_NOTICE,
+ "o2net: cannot change keepalive delay after"
+ " the first peer has agreed to it."
+ " %d connected peers\n",
+ o2net_num_connected_peers());
+ ret = -EINVAL;
+ } else if (val >= cluster->cl_idle_timeout_ms) {
mlog(ML_NOTICE, "o2net: keepalive delay must be "
"smaller than idle timeout\n");
- return -EINVAL;
+ ret = -EINVAL;
+ } else {
+ cluster->cl_keepalive_delay_ms = val;
}
- cluster->cl_keepalive_delay_ms = val;
}
return ret;
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c 2007-03-13 18:57:50 UTC (rev 2989)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c 2007-03-13 19:03:23 UTC (rev 2990)
@@ -381,6 +381,13 @@
sc_put(sc);
}
+atomic_t o2net_connected_peers = ATOMIC_INIT(0);
+
+int o2net_num_connected_peers(void)
+{
+ return atomic_read(&o2net_connected_peers);
+}
+
static void o2net_set_nn_state(struct o2net_node *nn,
struct o2net_sock_container *sc,
unsigned valid, int err)
@@ -391,6 +398,13 @@
assert_spin_locked(&nn->nn_lock);
+ if (old_sc != sc) {
+ if (old_sc)
+ atomic_dec(&o2net_connected_peers);
+ else
+ atomic_inc(&o2net_connected_peers);
+ }
+
/* the node num comparison and single connect/accept path should stop
* an non-null sc from being overwritten with another */
BUG_ON(sc && nn->nn_sc && nn->nn_sc != sc);
@@ -1145,6 +1159,44 @@
return -1;
}
+ /*
+ * Ensure timeouts are consistent with other nodes, otherwise
+ * we can end up with one node thinking that the other must be down,
+ * but isn't. This can ultimately cause corruption.
+ */
+ if (be32_to_cpu(hand->o2net_idle_timeout_ms) !=
+ o2net_idle_timeout(sc->sc_node)) {
+ mlog(ML_NOTICE, SC_NODEF_FMT " uses a network idle timeout of "
+ "%u ms, but we use %u ms locally. disconnecting\n",
+ SC_NODEF_ARGS(sc),
+ be32_to_cpu(hand->o2net_idle_timeout_ms),
+ o2net_idle_timeout(sc->sc_node));
+ o2net_ensure_shutdown(nn, sc, -ENOTCONN);
+ return -1;
+ }
+
+ if (be32_to_cpu(hand->o2net_keepalive_delay_ms) !=
+ o2net_keepalive_delay(sc->sc_node)) {
+ mlog(ML_NOTICE, SC_NODEF_FMT " uses a keepalive delay of "
+ "%u ms, but we use %u ms locally. disconnecting\n",
+ SC_NODEF_ARGS(sc),
+ be32_to_cpu(hand->o2net_keepalive_delay_ms),
+ o2net_keepalive_delay(sc->sc_node));
+ o2net_ensure_shutdown(nn, sc, -ENOTCONN);
+ return -1;
+ }
+
+ if (be32_to_cpu(hand->o2hb_heartbeat_timeout_ms) !=
+ O2HB_MAX_WRITE_TIMEOUT_MS) {
+ mlog(ML_NOTICE, SC_NODEF_FMT " uses a heartbeat timeout of "
+ "%u ms, but we use %u ms locally. disconnecting\n",
+ SC_NODEF_ARGS(sc),
+ be32_to_cpu(hand->o2hb_heartbeat_timeout_ms),
+ O2HB_MAX_WRITE_TIMEOUT_MS);
+ o2net_ensure_shutdown(nn, sc, -ENOTCONN);
+ return -1;
+ }
+
sc->sc_handshake_ok = 1;
spin_lock(&nn->nn_lock);
@@ -1177,6 +1229,23 @@
sclog(sc, "receiving\n");
do_gettimeofday(&sc->sc_tv_advance_start);
+ if (unlikely(sc->sc_handshake_ok == 0)) {
+ if(sc->sc_page_off < sizeof(struct o2net_handshake)) {
+ data = page_address(sc->sc_page) + sc->sc_page_off;
+ datalen = sizeof(struct o2net_handshake) - sc->sc_page_off;
+ ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen);
+ if (ret > 0)
+ sc->sc_page_off += ret;
+ }
+
+ if (sc->sc_page_off == sizeof(struct o2net_handshake)) {
+ o2net_check_handshake(sc);
+ if (unlikely(sc->sc_handshake_ok == 0))
+ ret = -EPROTO;
+ }
+ goto out;
+ }
+
/* do we need more header? */
if (sc->sc_page_off < sizeof(struct o2net_msg)) {
data = page_address(sc->sc_page) + sc->sc_page_off;
@@ -1184,15 +1253,6 @@
ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen);
if (ret > 0) {
sc->sc_page_off += ret;
-
- /* this working relies on the handshake being
- * smaller than the normal message header */
- if (sc->sc_page_off >= sizeof(struct o2net_handshake)&&
- !sc->sc_handshake_ok && o2net_check_handshake(sc)) {
- ret = -EPROTO;
- goto out;
- }
-
/* only swab incoming here.. we can
* only get here once as we cross from
* being under to over */
@@ -1293,6 +1353,18 @@
return ret;
}
+static void o2net_initialize_handshake(void)
+{
+ o2net_hand->o2hb_heartbeat_timeout_ms = cpu_to_be32(
+ O2HB_MAX_WRITE_TIMEOUT_MS);
+ o2net_hand->o2net_idle_timeout_ms = cpu_to_be32(
+ o2net_idle_timeout(NULL));
+ o2net_hand->o2net_keepalive_delay_ms = cpu_to_be32(
+ o2net_keepalive_delay(NULL));
+ o2net_hand->o2net_reconnect_delay_ms = cpu_to_be32(
+ o2net_reconnect_delay(NULL));
+}
+
/* ------------------------------------------------------------ */
/* called when a connect completes and after a sock is accepted. the
@@ -1305,6 +1377,7 @@
(unsigned long long)O2NET_PROTOCOL_VERSION,
(unsigned long long)be64_to_cpu(o2net_hand->connector_id));
+ o2net_initialize_handshake();
o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand));
sc_put(sc);
}
@@ -1527,6 +1600,8 @@
if (node_num != o2nm_this_node())
o2net_disconnect_node(node);
+
+ BUG_ON(atomic_read(&o2net_connected_peers) < 0);
}
static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num,
@@ -1706,6 +1781,7 @@
o2net_register_callbacks(sc->sc_sock->sk, sc);
o2net_sc_queue_work(sc, &sc->sc_rx_work);
+ o2net_initialize_handshake();
o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand));
out:
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.h
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.h 2007-03-13 18:57:50 UTC (rev 2989)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.h 2007-03-13 19:03:23 UTC (rev 2990)
@@ -115,6 +115,7 @@
int o2net_start_listening(struct o2nm_node *node);
void o2net_stop_listening(struct o2nm_node *node);
void o2net_disconnect_node(struct o2nm_node *node);
+int o2net_num_connected_peers(void);
int o2net_init(void);
void o2net_exit(void);
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h 2007-03-13 18:57:50 UTC (rev 2989)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h 2007-03-13 19:03:23 UTC (rev 2990)
@@ -39,11 +39,16 @@
* 1 - initial release, up through v1.2.3
* 101 - DLM lockres remote refcount change, beginning with v1.2.4
* 102 - DLM join domain includes the live nodemap, beginning with v1.2.4
+ * 103 - Handshake includes timeouts, beginning with v1.2.4
*/
-#define O2NET_PROTOCOL_VERSION 102ULL
+#define O2NET_PROTOCOL_VERSION 103ULL
struct o2net_handshake {
__be64 protocol_version;
__be64 connector_id;
+ __be32 o2hb_heartbeat_timeout_ms;
+ __be32 o2net_idle_timeout_ms;
+ __be32 o2net_keepalive_delay_ms;
+ __be32 o2net_reconnect_delay_ms;
};
struct o2net_node {
More information about the Ocfs2-commits
mailing list