[Ocfs2-commits] smushran commits r3092 - branches/ocfs2-1.2/fs/ocfs2/cluster
svn-commits at oss.oracle.com
svn-commits at oss.oracle.com
Fri May 2 14:25:57 PDT 2008
Author: smushran
Date: 2008-05-02 14:25:57 -0700 (Fri, 02 May 2008)
New Revision: 3092
Modified:
branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c
branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h
Log:
ocfs2: Reconnect after idle time out
Mainline commit 5cc3bf2786f63cceb191c3c02ddd83c6f38a7d64
Author: Tao Ma <tao.ma at oracle.com>
Date: Wed, 5 Mar 2008 15:50:12 +0800
Currently, o2net connects to a node on hb_up and disconnects on
hb_down and net timeout.
It disconnects on net timeout is ok, but it should attempt to
reconnect back. This is because sometimes nodes get overloaded
enough that the network connection breaks but the disk hb does not.
And if we get into that situation, we either fence (unnecessarily)
or wait for its disk hb to die (and sometimes hang in the process).
So in this updated scheme, when the network disconnects, we keep
attempting to reconnect till we succeed or we get a disk hb down
event.
If the other node is really dead, then we will eventually get a
node down event. If not, we should be able to connect again and
continue.
Signed-off-by: zab
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c 2008-05-02 21:24:49 UTC (rev 3091)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/tcp.c 2008-05-02 21:25:57 UTC (rev 3092)
@@ -411,8 +411,6 @@
mlog_bug_on_msg(err && valid, "err %d valid %u\n", err, valid);
mlog_bug_on_msg(valid && !sc, "valid %u sc %p\n", valid, sc);
- /* we won't reconnect after our valid conn goes away for
- * this hb iteration.. here so it shows up in the logs */
if (was_valid && !valid && err == 0)
err = -ENOTCONN;
@@ -442,11 +440,6 @@
if (!was_valid && valid) {
o2quo_conn_up(o2net_num_from_nn(nn));
- /* this is a bit of a hack. we only try reconnecting
- * when heartbeating starts until we get a connection.
- * if that connection then dies we don't try reconnecting.
- * the only way to start connecting again is to down
- * heartbeat and bring it back up. */
cancel_delayed_work(&nn->nn_connect_expired);
printk(KERN_INFO "o2net: %s " SC_NODEF_FMT "\n",
o2nm_this_node() > sc->sc_node->nd_num ?
@@ -469,6 +462,18 @@
delay = 0;
mlog(ML_CONN, "queueing conn attempt in %lu jiffies\n", delay);
queue_delayed_work(o2net_wq, &nn->nn_connect_work, delay);
+
+ /*
+ * Delay the expired work after idle timeout.
+ *
+ * We might have lots of failed connection attempts that run
+ * through here but we only cancel the connect_expired work when
+ * a connection attempt succeeds. So only the first enqueue of
+ * the connect_expired work will do anything. The rest will see
+ * that it's already queued and do nothing.
+ */
+ delay += msecs_to_jiffies(o2net_idle_timeout(NULL));
+ queue_delayed_work(o2net_wq, &nn->nn_connect_expired, delay);
}
/* keep track of the nn's sc ref for the caller */
@@ -1212,6 +1217,7 @@
* shut down already */
if (nn->nn_sc == sc) {
o2net_sc_reset_idle_timer(sc);
+ atomic_set(&nn->nn_timeout, 0);
o2net_set_nn_state(nn, sc, 1, 0);
}
spin_unlock(&nn->nn_lock);
@@ -1405,6 +1411,7 @@
static void o2net_idle_timer(unsigned long data)
{
struct o2net_sock_container *sc = (struct o2net_sock_container *)data;
+ struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num);
struct timeval now;
do_gettimeofday(&now);
@@ -1425,6 +1432,12 @@
sc->sc_tv_func_start.tv_sec, sc->sc_tv_func_start.tv_usec,
sc->sc_tv_func_stop.tv_sec, sc->sc_tv_func_stop.tv_usec);
+ /*
+ * Initialize the nn_timeout so that the next connection attempt
+ * will continue in o2net_start_connect.
+ */
+ atomic_set(&nn->nn_timeout, 1);
+
o2net_sc_queue_work(sc, &sc->sc_shutdown_work);
}
@@ -1457,7 +1470,8 @@
struct o2nm_node *node = NULL, *mynode = NULL;
struct socket *sock = NULL;
struct sockaddr_in myaddr = {0, }, remoteaddr = {0, };
- int ret = 0;
+ int ret = 0, stop;
+ unsigned int timeout;
/* if we're greater we initiate tx, otherwise we accept */
if (o2nm_this_node() <= o2net_num_from_nn(nn))
@@ -1477,11 +1491,19 @@
}
spin_lock(&nn->nn_lock);
- /* see if we already have one pending or have given up */
- if (nn->nn_sc || nn->nn_persistent_error)
- arg = NULL;
+ /*
+ * see if we already have one pending or have given up.
+ * For nn_timeout, it is set when we close the connection
+ * because of the idle time out. So it means that we have
+ * at least connected to that node successfully once,
+ * now try to connect to it again.
+ */
+ timeout = atomic_read(&nn->nn_timeout);
+ stop = (nn->nn_sc ||
+ (nn->nn_persistent_error &&
+ (nn->nn_persistent_error != -ENOTCONN || timeout == 0)));
spin_unlock(&nn->nn_lock);
- if (arg == NULL) /* *shrug*, needed some indicator */
+ if (stop)
goto out;
nn->nn_last_connect_attempt = jiffies;
@@ -1589,6 +1611,7 @@
/* don't reconnect until it's heartbeating again */
spin_lock(&nn->nn_lock);
+ atomic_set(&nn->nn_timeout, 0);
o2net_set_nn_state(nn, NULL, 0, -ENOTCONN);
spin_unlock(&nn->nn_lock);
@@ -1623,17 +1646,12 @@
(msecs_to_jiffies(o2net_reconnect_delay(node)) + 1);
if (node_num != o2nm_this_node()) {
- /* heartbeat doesn't work unless a local node number is
- * configured and doing so brings up the o2net_wq, so we can
- * use it.. */
- queue_delayed_work(o2net_wq, &nn->nn_connect_expired,
- msecs_to_jiffies(o2net_idle_timeout(node)));
-
/* believe it or not, accept and node hearbeating testing
* can succeed for this node before we got here.. so
* only use set_nn_state to clear the persistent error
* if that hasn't already happened */
spin_lock(&nn->nn_lock);
+ atomic_set(&nn->nn_timeout, 0);
if (nn->nn_persistent_error)
o2net_set_nn_state(nn, NULL, 0, 0);
spin_unlock(&nn->nn_lock);
@@ -1782,6 +1800,7 @@
new_sock = NULL;
spin_lock(&nn->nn_lock);
+ atomic_set(&nn->nn_timeout, 0);
o2net_set_nn_state(nn, sc, 0, 0);
spin_unlock(&nn->nn_lock);
@@ -1976,6 +1995,7 @@
for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) {
struct o2net_node *nn = o2net_nn_from_num(i);
+ atomic_set(&nn->nn_timeout, 0);
spin_lock_init(&nn->nn_lock);
INIT_WORK(&nn->nn_connect_work, o2net_start_connect, nn);
INIT_WORK(&nn->nn_connect_expired, o2net_connect_expired, nn);
Modified: branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h
===================================================================
--- branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h 2008-05-02 21:24:49 UTC (rev 3091)
+++ branches/ocfs2-1.2/fs/ocfs2/cluster/tcp_internal.h 2008-05-02 21:25:57 UTC (rev 3092)
@@ -61,6 +61,8 @@
unsigned nn_sc_valid:1;
/* if this is set tx just returns it */
int nn_persistent_error;
+ /* It is only set to 1 after the idle time out. */
+ atomic_t nn_timeout;
/* threads waiting for an sc to arrive wait on the wq for generation
* to increase. it is increased when a connecting socket succeeds
More information about the Ocfs2-commits
mailing list