[Ocfs2-commits] khackel commits r1950 - trunk/fs/ocfs2/dlm

svn-commits at oss.oracle.com svn-commits at oss.oracle.com
Mon Mar 7 12:36:28 CST 2005


Author: khackel
Signed-off-by: mfasheh
Date: 2005-03-07 12:36:26 -0600 (Mon, 07 Mar 2005)
New Revision: 1950

Modified:
   trunk/fs/ocfs2/dlm/dlmmaster.c
   trunk/fs/ocfs2/dlm/dlmmod.c
   trunk/fs/ocfs2/dlm/dlmmod.h
   trunk/fs/ocfs2/dlm/dlmrecovery.c
Log:
* Merged 1926 from dlm-reco-mig branch
        - [1926] move mle heartbeat events onto main dlm hearbeat callbacks

Signed-off-by: mfasheh



Modified: trunk/fs/ocfs2/dlm/dlmmaster.c
===================================================================
--- trunk/fs/ocfs2/dlm/dlmmaster.c	2005-03-07 18:18:47 UTC (rev 1949)
+++ trunk/fs/ocfs2/dlm/dlmmaster.c	2005-03-07 18:36:26 UTC (rev 1950)
@@ -51,20 +51,15 @@
 #include "dlmmod.h"
 
 
-static int dlm_init_mle(dlm_master_list_entry *mle,
+static void dlm_init_mle(dlm_master_list_entry *mle,
 			enum dlm_mle_type type,
 			dlm_ctxt *dlm,
 			dlm_lock_resource *res,
 			const char *name,
-			unsigned int namelen,
-			int locked);
+			unsigned int namelen);
 static void dlm_put_mle(dlm_master_list_entry *mle);
 static int dlm_find_mle(dlm_ctxt *dlm, dlm_master_list_entry **mle,
 			char *name, unsigned int namelen);
-static void dlm_mle_node_up(struct inode *group, struct inode *node, 
-		     int idx, void *data);
-static void dlm_mle_node_down(struct inode *group, struct inode *node, 
-		       int idx, void *data);
 
 static int dlm_do_master_request(dlm_master_list_entry *mle, int to);
 static int dlm_do_master_request_resp(dlm_ctxt *dlm, char *name,
@@ -90,6 +85,40 @@
  * MASTER LIST FUNCTIONS
  */
 
+
+/*
+ * regarding master list entries and heartbeat callbacks:
+ *
+ * in order to avoid sleeping and allocation that occurs in
+ * heartbeat, master list entries are simply attached to the
+ * dlm's established heartbeat callbacks.  the mle is attached
+ * when it is created, and since the dlm->spinlock is held at
+ * that time, any heartbeat event will be properly discovered
+ * by the mle.  the mle needs to be detached from the
+ * dlm->mle_hb_events list as soon as heartbeat events are no
+ * longer useful to the mle, and before the mle is freed.
+ *
+ * as a general rule, heartbeat events are no longer needed by
+ * the mle once an "answer" regarding the lock master has been
+ * received.
+ */
+static inline void __dlm_mle_attach_hb_events(dlm_ctxt *dlm, 
+					      dlm_master_list_entry *mle)
+{
+       list_add_tail(&mle->hb_events, &dlm->mle_hb_events);
+}
+
+
+static inline void dlm_mle_detach_hb_events(dlm_ctxt *dlm, 
+					    dlm_master_list_entry *mle)
+{
+	spin_lock(&dlm->spinlock);
+	if (!list_empty(&mle->hb_events))
+		list_del_init(&mle->hb_events);
+	spin_unlock(&dlm->spinlock);
+}
+
+
 /* remove from list and free */
 static void dlm_put_mle(dlm_master_list_entry *mle)
 {
@@ -99,10 +128,13 @@
 	dlm = mle->dlm;
 
 	if (atomic_dec_and_lock(&mle->refcnt, &dlm->master_lock)) {
-		list_del(&mle->list);
+		/* remove from list if not already */
+		if (!list_empty(&mle->list))
+			list_del(&mle->list);
 		spin_unlock(&dlm->master_lock);
-		hb_unregister_callback(&mle->mle_hb_down);
-		hb_unregister_callback(&mle->mle_hb_up);
+
+		/* detach the mle from the domain node up/down events */
+		dlm_mle_detach_hb_events(dlm, mle);
 		kfree(mle);
 	}
 }
@@ -112,19 +144,18 @@
 	atomic_inc(&mle->refcnt);
 }
 
-static int dlm_init_mle(dlm_master_list_entry *mle,
+
+static void dlm_init_mle(dlm_master_list_entry *mle,
 			enum dlm_mle_type type,
 			dlm_ctxt *dlm,
 			dlm_lock_resource *res,
 			const char *name,
-			unsigned int namelen,
-			int locked)
+			unsigned int namelen)
 {
-	int ret = 0;
-
 	mle->dlm = dlm;
 	mle->type = type;
 	INIT_LIST_HEAD(&mle->list);
+	INIT_LIST_HEAD(&mle->hb_events);
 	memset(mle->maybe_map, 0, sizeof(mle->maybe_map));
 	spin_lock_init(&mle->spinlock);
 	init_waitqueue_head(&mle->wq);
@@ -139,29 +170,14 @@
 	else 
 		strncpy(mle->u.name.name, name, namelen);
 
-	if (!locked)
-		spin_lock(&dlm->spinlock);
-
 	/* copy off the node_map and register hb callbacks on our copy */
 	memcpy(mle->node_map, dlm->node_map, sizeof(mle->node_map));
 	memcpy(mle->vote_map, dlm->node_map, sizeof(mle->vote_map));
 	clear_bit(dlm->group_index, mle->vote_map);
 	clear_bit(dlm->group_index, mle->node_map);
 
-	hb_setup_callback(&mle->mle_hb_down, HB_NODE_DOWN_CB,
-			  dlm_mle_node_down, mle, DLM_HB_NODE_DOWN_PRI+1);
-	hb_setup_callback(&mle->mle_hb_up, HB_NODE_UP_CB,
-			  dlm_mle_node_up, mle, DLM_HB_NODE_UP_PRI+1);
-
-	if (hb_register_callback(&mle->mle_hb_down) || 
-	    hb_register_callback(&mle->mle_hb_up)) {
-		ret = -EINVAL;
-	}
-
-	if (!locked)
-		spin_unlock(&dlm->spinlock);
-
-	return ret;
+	/* attach the mle to the domain node up/down events */
+	__dlm_mle_attach_hb_events(dlm, mle);
 }
 
 
@@ -183,25 +199,13 @@
 	return 0;
 }
 
-static void dlm_mle_node_down(struct inode *group, struct inode *node, 
-		       int idx, void *data)
+
+void dlm_mle_node_down(dlm_ctxt *dlm, dlm_master_list_entry *mle,
+		       struct inode *group, struct inode *node, int idx)
 {
-	//int ret;
-	//struct inode *node = ptr2;
+	DLM_ASSERT(mle);
+	DLM_ASSERT(dlm);
 
-	dlm_master_list_entry *mle;
-	dlm_ctxt *dlm;
-
-	mle = data;
-	if (!mle) {
-		dlmprintk0("eek! NULL mle!\n");
-		return;
-	}
-	if (!mle->dlm) {
-		dlmprintk0("eek! NULL dlm\n");
-		return;
-	}
-       	dlm = mle->dlm;
 	if (dlm->group != group)
 		return;
 
@@ -221,23 +225,12 @@
 	spin_unlock(&mle->spinlock);
 }
 
-static void dlm_mle_node_up(struct inode *group, struct inode *node, 
-		     int idx, void *data)
+void dlm_mle_node_up(dlm_ctxt *dlm, dlm_master_list_entry *mle,
+		       struct inode *group, struct inode *node, int idx)
 {
-	//struct inode *node = ptr2;
-	dlm_master_list_entry *mle;
-	dlm_ctxt *dlm;
+	DLM_ASSERT(mle);
+	DLM_ASSERT(dlm);
 
-	mle = data;
-	if (!mle) {
-		dlmprintk0("eek! NULL mle!\n");
-		return;
-	}
-	if (!mle->dlm) {
-		dlmprintk0("eek! NULL dlm\n");
-		return;
-	}
-       	dlm = mle->dlm;
 	if (dlm->group != group)
 		return;
 
@@ -481,10 +474,7 @@
 	}
 	if (!blocked) {
 		/* go ahead and try to master lock on this node */
-		if (dlm_init_mle(mle, DLM_MLE_MASTER, dlm, res, NULL, 0, 1)) {
-			dlmprintk0("bug! failed to register hb callbacks\n");
-			BUG();
-		}
+		dlm_init_mle(mle, DLM_MLE_MASTER, dlm, res, NULL, 0);
 		list_add(&mle->list, &dlm->master_list);
 	}
 	spin_unlock(&dlm->master_lock);
@@ -538,7 +528,8 @@
 	if (ret == 0)
 		dlmprintk("lockres mastered by %u\n", res->owner);
 
-
+	/* master is known, detach if not already detached */
+	dlm_mle_detach_hb_events(dlm, mle);
 	dlm_put_mle(mle);
 
 wake_waiters:
@@ -834,6 +825,7 @@
 		spin_unlock(&dlm->master_lock);
 		spin_unlock(&res->spinlock);
 
+		/* keep the mle attached to heartbeat events */
 		dlm_put_mle(tmpmle);
 		if (mle)
 			kfree(mle);
@@ -862,13 +854,10 @@
 				response = DLM_MASTER_RESP_ERROR;
 				goto send_response;
 			}
-			if (dlm_init_mle(mle, DLM_MLE_BLOCK, dlm, NULL, 
-					 name, namelen, 0)) {
-				dlmprintk0("eeek!\n");
-				response = DLM_MASTER_RESP_ERROR;
-				dlm_put_mle(mle);
-				goto send_response;
-			}
+			spin_lock(&dlm->spinlock);
+			dlm_init_mle(mle, DLM_MLE_BLOCK, dlm, NULL, 
+					 name, namelen);
+			spin_unlock(&dlm->spinlock);
 			goto way_up_top;
 		}
 
@@ -891,6 +880,7 @@
 	spin_unlock(&dlm->spinlock);
 
 	if (found) {
+		/* keep the mle attached to heartbeat events */
 		dlm_put_mle(tmpmle);
 	}
 send_response:
@@ -1022,9 +1012,10 @@
 		wake_up(&mle->wq);
 	}
 
-	if (mle)
+	if (mle) {
+		/* keep the mle attached to heartbeat events */
 		dlm_put_mle(mle);
-	else
+	} else
 		dlmprintk0("hrrm... got a master resp but found no matching "
 			   "request\n");
 
@@ -1185,6 +1176,8 @@
 		spin_unlock(&mle->spinlock);
 	
 		/* if this is the last put, it will be removed from the list */
+		/* master is known, detach if not already detached */
+		dlm_mle_detach_hb_events(dlm, mle);
 		dlm_put_mle(mle);
 	}
 

Modified: trunk/fs/ocfs2/dlm/dlmmod.c
===================================================================
--- trunk/fs/ocfs2/dlm/dlmmod.c	2005-03-07 18:18:47 UTC (rev 1949)
+++ trunk/fs/ocfs2/dlm/dlmmod.c	2005-03-07 18:36:26 UTC (rev 1950)
@@ -647,6 +647,7 @@
 	dlm->dlm_thread_task = NULL;
 	init_waitqueue_head(&dlm->dlm_thread_wq);
 	INIT_LIST_HEAD(&dlm->master_list);
+	INIT_LIST_HEAD(&dlm->mle_hb_events);
 	init_rwsem(&dlm->recovery_sem);
 
 	/* this eats the reference we got above. */

Modified: trunk/fs/ocfs2/dlm/dlmmod.h
===================================================================
--- trunk/fs/ocfs2/dlm/dlmmod.h	2005-03-07 18:18:47 UTC (rev 1949)
+++ trunk/fs/ocfs2/dlm/dlmmod.h	2005-03-07 18:36:26 UTC (rev 1950)
@@ -216,6 +216,7 @@
 	dlm_recovery_ctxt reco;
 	spinlock_t master_lock;
 	struct list_head master_list;
+	struct list_head mle_hb_events;
 
 	/* these give a really vague idea of the system load */
 	atomic_t local_resources;
@@ -331,6 +332,7 @@
 typedef struct _dlm_master_list_entry
 {
 	struct list_head list;
+	struct list_head hb_events;
 	dlm_ctxt *dlm;
 	spinlock_t spinlock;
 	wait_queue_head_t wq;
@@ -670,6 +672,10 @@
 
 void dlm_init_lock(dlm_lock *newlock, int type, u8 node, u64 cookie);
 
+void dlm_mle_node_down(dlm_ctxt *dlm, dlm_master_list_entry *mle,
+		       struct inode *group, struct inode *node, int idx);
+void dlm_mle_node_up(dlm_ctxt *dlm, dlm_master_list_entry *mle,
+		       struct inode *group, struct inode *node, int idx);
 
 
 static inline const char * dlm_lock_mode_name(int mode)

Modified: trunk/fs/ocfs2/dlm/dlmrecovery.c
===================================================================
--- trunk/fs/ocfs2/dlm/dlmrecovery.c	2005-03-07 18:18:47 UTC (rev 1949)
+++ trunk/fs/ocfs2/dlm/dlmrecovery.c	2005-03-07 18:36:26 UTC (rev 1950)
@@ -834,11 +834,18 @@
 void dlm_hb_node_down_cb(struct inode *group, struct inode *node, int idx, void *data)
 {
 	dlm_ctxt *dlm = data;
+	dlm_master_list_entry *mle;
+	struct list_head *iter;
 
 	if (!dlm_grab(dlm))
 		return;
 
 	spin_lock(&dlm->spinlock);
+	/* notify any mles attached to the heartbeat events */
+	list_for_each(iter, &dlm->mle_hb_events) {
+		mle = list_entry(iter, dlm_master_list_entry, hb_events);
+		dlm_mle_node_down(dlm, mle, group, node, idx);
+	}
 
 	if (!test_bit(idx, dlm->node_map))
 		dlmprintk("node %u already removed from nodemap!\n", idx);
@@ -861,12 +868,20 @@
 void dlm_hb_node_up_cb(struct inode *group, struct inode *node, int idx, void *data)
 {
 	dlm_ctxt *dlm = data;
+	dlm_master_list_entry *mle;
+	struct list_head *iter;
 
 	if (!dlm_grab(dlm))
 		return;
 
 	spin_lock(&dlm->spinlock);
+	/* notify any mles attached to the heartbeat events */
+	list_for_each(iter, &dlm->mle_hb_events) {
+		mle = list_entry(iter, dlm_master_list_entry, hb_events);
+		dlm_mle_node_up(dlm, mle, group, node, idx);
+	}
 
+
 	if (test_bit(idx, dlm->recovery_map)) {
 		dlmprintk("BUG!!! node up message on node in recovery (%u)!!!\n", idx);
 	} else {



More information about the Ocfs2-commits mailing list