[Ocfs2-commits] zab commits r1987 - branches/usysfsify/fs/ocfs2/cluster

svn-commits at oss.oracle.com svn-commits at oss.oracle.com
Wed Mar 16 18:21:06 CST 2005


Author: zab
Date: 2005-03-16 18:21:05 -0600 (Wed, 16 Mar 2005)
New Revision: 1987

Modified:
   branches/usysfsify/fs/ocfs2/cluster/nodemanager.c
   branches/usysfsify/fs/ocfs2/cluster/nodemanager.h
Log:
flesh out nm_get_node*()
                                                                                
o track our single cluster in a global
o store node pointers in an array by node number
o store nodes in a tree keyed by their ip
o don't allow node dir removal yet


Modified: branches/usysfsify/fs/ocfs2/cluster/nodemanager.c
===================================================================
--- branches/usysfsify/fs/ocfs2/cluster/nodemanager.c	2005-03-16 22:46:41 UTC (rev 1986)
+++ branches/usysfsify/fs/ocfs2/cluster/nodemanager.c	2005-03-17 00:21:05 UTC (rev 1987)
@@ -75,9 +75,10 @@
 static DECLARE_MUTEX(nm_cb_sem);
 struct list_head nm_callbacks[NM_NUM_CB];
 
-int nm_create_node(char *buf, nm_op *data);
-int nm_name_cluster(char *buf, nm_op *data);
-int nm_get_node_info(char *buf, nm_op *data);
+/* for now we operate under the assertion that there can be only one
+ * cluster active at a time.  Changing this will require trickling
+ * cluster references throughout where nodes are looked up */
+static struct nm_cluster *nm_single_cluster = NULL;
 
 #define nmprintk(x, arg...)    printk("(nm:%d) " x, current->pid, ##arg)
 #define nmprintk0(x)           printk("(nm:%d) " x, current->pid)
@@ -88,79 +89,51 @@
 
 struct nm_cluster {
 	struct ukset	cl_ukset;
-	char		cl_name[NM_MAX_NAME_LEN+1];
-	spinlock_t	cl_bitmap_lock;
-	unsigned long	cl_node_bitmap[BITS_TO_LONGS(NM_MAX_NODES)];
 	unsigned	cl_has_local:1;
 	u8		cl_local_node;
+	rwlock_t	cl_nodes_lock;
+	struct nm_node  *cl_nodes[NM_MAX_NODES];
+	struct rb_root	cl_node_ip_tree;
 };
 
-static int nm_find_next_slot(spinlock_t *lock, void *bitmap, int max,
-			     int request)
+struct nm_node * nm_get_node_by_num(u8 node_num)
 {
-	int start = 0, slot_num, ret = -1;
+	struct nm_node *node = NULL;
 
-	if (request != NM_INVALID_SLOT_NUM)
-		start = request;
-
-	spin_lock(lock);
-
-	slot_num = find_next_zero_bit (bitmap, max, start);
-	if (slot_num >= max)
+	if (node_num >= NM_MAX_NODES || nm_single_cluster == NULL)
 		goto out;
-	if (request != NM_INVALID_SLOT_NUM && slot_num != request)
-		goto out;
 
-	set_bit(slot_num, bitmap);
-	ret = slot_num;
-
+	read_lock(&nm_single_cluster->cl_nodes_lock);
+	node = nm_single_cluster->cl_nodes[node_num];
+	if (node) /* XXX get a ref */
+		;
+	read_unlock(&nm_single_cluster->cl_nodes_lock);
 out:
-	spin_unlock(lock);
-	return ret;
+	return node;
 }
-
-/* XXX return nm_node, see why people are calling it in the first place.. */
-struct nm_node * nm_get_node_by_num(u8 node_num)
-{
-	BUG();
-	return NULL;
-}
 EXPORT_SYMBOL(nm_get_node_by_num);
 
-/* tcp calls this when a node connects so that it can deref the inode
- * to find the inode_private->net state.. hmm.  it probably just wants
- * to look it up so that it can verify that it's a node we like and
- * allocate its own state?  hmm, that's a pickle. */
-struct nm_node * nm_get_node_by_ip(u32 addr)
-{
-	BUG();
-	return NULL;
-}
-EXPORT_SYMBOL(nm_get_node_by_ip);
-
-#if 0
-/* this tree is only used by get_group_by_name */
-static struct nm_group * nm_group_tree_lookup(nm_cluster *cluster,
-					      const char *group_name,
+static struct nm_node * nm_node_ip_tree_lookup(struct nm_cluster *cluster,
+					      u32 ip_needle,
 					      struct rb_node ***ret_p,
 					      struct rb_node **ret_parent)
 {
-        struct rb_node **p = &cluster->group_name_tree.rb_node;
+        struct rb_node **p = &cluster->cl_node_ip_tree.rb_node;
         struct rb_node *parent = NULL;
-	struct nm_group *group, *ret = NULL;
+	struct nm_node *node, *ret = NULL;
 	int cmp;
 
         while (*p) {
                 parent = *p;
-                group = rb_entry(parent, struct nm_group, gr_name_rb_node);
-		cmp = strcmp(group_name, group->gr_name);
+                node = rb_entry(parent, struct nm_node, nd_ip_node);
+		cmp = ip_needle < node->nd_ipv4_address;
 
                 if (cmp < 0)
                         p = &(*p)->rb_left;
-                else if (cmp > 0) {
+                else if (cmp > 0)
                         p = &(*p)->rb_right;
                 else {
-			ret = group;
+			ret = node;
                         break;
 		}
         }
@@ -170,31 +143,36 @@
         if (ret_parent != NULL)
                 *ret_parent = parent;
 
-	if (ret)
-		/* get ref */;
-
         return ret;
 }
 
-struct nm_group * nm_get_group_by_name(nm_cluster *cluster,
-				       const char *group_name)
+struct nm_node * nm_get_node_by_ip(u32 addr)
 {
-	struct nm_group *group;
+	struct nm_node *node = NULL;
+	struct nm_cluster *cluster = nm_single_cluster;
 
-	read_lock(&cluster->rbtree_lock);
-	group = nm_group_tree_lookup(cluster, group_name, NULL, NULL);
-	read_unlock(&cluster->rbtree_lock);
+	if (cluster == NULL)
+		goto out;
 
-	return group;
+	read_lock(&cluster->cl_nodes_lock);
+	node = nm_node_ip_tree_lookup(cluster, addr, NULL, NULL);
+	if (node) /* XXX get a ref */
+		;
+	read_unlock(&cluster->cl_nodes_lock);
+
+out:
+	return node;
 }
-EXPORT_SYMBOL(nm_get_group_by_name);
-#endif
+EXPORT_SYMBOL(nm_get_node_by_ip);
 
 u8 nm_this_node(void)
 {
-	/* sigh, in our one cluster one node one ip.. */
-	BUG();
-	return 0;
+	u8 node_num = NM_MAX_NODES;
+
+	if (nm_single_cluster && nm_single_cluster->cl_has_local)
+		node_num = nm_single_cluster->cl_local_node;
+
+	return node_num;
 }
 EXPORT_SYMBOL(nm_this_node);
 
@@ -287,7 +265,6 @@
 	struct nm_cluster *cluster = to_nm_cluster(node->nd_kobj.parent);
 	unsigned long tmp;
 	char *p = (char *)page;
-	int node_num;
 
 	tmp = simple_strtoul(p, &p, 0);
 	if (!p || (*p && (*p != '\n')))
@@ -296,13 +273,17 @@
 	if (tmp >= NM_MAX_NODES)
 		return -ERANGE;
 
-	node_num = nm_find_next_slot(&cluster->cl_bitmap_lock,
-				     &(cluster->cl_node_bitmap[0]), 255, tmp);
-	if (node_num < 0)
-		return node_num;
+	write_lock(&cluster->cl_nodes_lock);
+	if (cluster->cl_nodes[tmp])
+		p = NULL;
+	else  {
+		cluster->cl_nodes[tmp] = node;
+		node->nd_num = tmp;
+	}
+	write_unlock(&cluster->cl_nodes_lock);
+	if (p == NULL)
+		return -EEXIST;
 
-	node->nd_num = tmp;
-
 	return count;
 }
 static ssize_t nm_node_ipv4_port_read(struct nm_node *node, char *page)
@@ -341,17 +322,32 @@
 					  const char *page,
 					  size_t count)
 {
+	struct nm_cluster *cluster = to_nm_cluster(node->nd_kobj.parent);
 	int ret;
 	union {
 		u32 ipv4_addr; /* network order */
 		unsigned char bytes[4];
 	} u;
+	struct rb_node **p, *parent;
 
 	ret = sscanf(page, "%c.%c.%c.%c", &u.bytes[0], &u.bytes[1], &u.bytes[2],
 		     &u.bytes[3]);
 	if (ret != 4)
 		return -EINVAL;
 
+	ret = 0;
+	write_lock(&cluster->cl_nodes_lock);
+	node = nm_node_ip_tree_lookup(cluster, u.ipv4_addr, &p, &parent);
+	if (node)
+		ret = -EEXIST;
+	else {
+	        rb_link_node(&node->nd_ip_node, parent, p);
+		rb_insert_color(&node->nd_ip_node, &cluster->cl_node_ip_tree);
+	}
+	write_lock(&cluster->cl_nodes_lock);
+	if (ret)
+		return ret;
+
 	memcpy(&node->nd_ipv4_address, &u.ipv4_addr, sizeof(u.ipv4_addr));
 
 #if 0
@@ -504,7 +500,6 @@
 	struct nm_cluster *cluster = to_nm_cluster(kset->kobj.parent);
 	struct kobject *ret = NULL;
 	net_inode_private *nip;
-	struct page *page;
 
 	printk("trying to make a node object under cluster %p\n", cluster);
 
@@ -516,27 +511,20 @@
 		goto out; /* ENOMEM */
 
 	strcpy(node->nd_name, name); /* use kobj.name instead? */
+	node->nd_num = NM_MAX_NODES;
 
 	/* this should be somewhere else */
 	nip = &node->nd_net_inode_private;
 	spin_lock_init(&nip->sock_lock);
-	nip->sock = NULL;
-	nip->sock_refs = 0;
-	nip->sock_pending = 0;
-	nip->defer_release = 0;
 	INIT_LIST_HEAD(&nip->pending_waiters);
 	init_waitqueue_head(&nip->waitq);
 	INIT_LIST_HEAD(&nip->handlers);
 	INIT_LIST_HEAD(&nip->active_item);
-	nip->page = NULL;
-	nip->page_off = 0;
-
-	page = alloc_page(GFP_KERNEL);
-	if (page == NULL) {
+	nip->page = alloc_page(GFP_KERNEL);
+	if (nip->page == NULL) {
 		nmprintk("page allocation failed\n");
 		goto out; /* ENOMEM */
 	}
-	nip->page = page;
 
 	kobject_set_name(&node->nd_kobj, name);
 	node->nd_kobj.ktype = &nm_node_type.ktype;
@@ -555,12 +543,33 @@
 {
 	struct nm_node *node = to_nm_node(kobj);
 	struct nm_cluster *cluster = to_nm_cluster(node->nd_kobj.parent);
+	int node_not_in_nodes_array = 0;
 
+	/* please don't try this yet, needs proper refcounts of nodes too */
+	BUG();
+
 	if (cluster->cl_has_local)
 		net_stop_rx_thread(node);
 
+	/* XXX sloppy */
+	if (node->nd_ipv4_address)
+		rb_erase(&node->nd_ip_node, &cluster->cl_node_ip_tree);
+
 	/* XXX call into net to stop this node from trading messages */
 
+	write_lock(&cluster->cl_nodes_lock);
+	if (node->nd_num != NM_MAX_NODES) {
+		if (cluster->cl_nodes[node->nd_num] != node)
+			node_not_in_nodes_array = 1;
+		else  {
+			cluster->cl_nodes[node->nd_num] = NULL;
+			node->nd_num = NM_MAX_NODES;
+		}
+	}
+	write_unlock(&cluster->cl_nodes_lock);
+
+	BUG_ON(node_not_in_nodes_array);
+
 	kobject_put(kobj);
 }
 
@@ -578,7 +587,11 @@
 static void nm_cluster_release(struct kobject *kobj)
 {
 	struct nm_cluster *cluster = to_nm_cluster(kobj);
+
 	printk("releasing cluster %p\n", cluster);
+
+	kfree(cluster->cl_ukset.default_sets);
+	kfree(cluster);
 }
 
 static struct ukobj_type nm_cluster_type = {
@@ -597,12 +610,14 @@
 	/* some stuff? */
 };
 
+#if 0
 static struct nm_cluster_set *to_nm_cluster_set(struct kset *kset)
 {
 	return kset ?
 		container_of(to_ukset(kset), struct nm_cluster_set, cs_ukset)
 	       : NULL;
 }
+#endif
 
 static struct kset *nm_cluster_set_make_kset(struct kset *kset,
 					     const char *name)
@@ -614,6 +629,11 @@
 
 	printk("trying to make a cluster object\n");
 
+	/* this runs under the parent dir's i_sem; there can be only
+	 * one caller in here at a time */
+	if (nm_single_cluster)
+		goto out; /* ENOSPC */
+
 	cluster = kcalloc(1, sizeof(struct nm_cluster), GFP_KERNEL);
 	ns = kcalloc(1, sizeof(struct nm_node_set), GFP_KERNEL);
 	defs = kcalloc(3, sizeof(struct kset *), GFP_KERNEL);
@@ -624,13 +644,15 @@
 	ukset_init_type_name(&cluster->cl_ukset, name, &nm_cluster_type.ktype);
 	ukset_init_type_name(&ns->ns_ukset, "node", &nm_node_set_type.ktype);
 
-	spin_lock_init(&cluster->cl_bitmap_lock);
 	cluster->cl_ukset.default_sets = defs;
 	cluster->cl_ukset.default_sets[0] = &ns->ns_ukset.kset;
 	cluster->cl_ukset.default_sets[1] = hb_kset;
 	cluster->cl_ukset.default_sets[2] = NULL;
+	rwlock_init(&cluster->cl_nodes_lock);
+	cluster->cl_node_ip_tree = RB_ROOT;
 
 	ret = &cluster->cl_ukset.kset;
+	nm_single_cluster = cluster;
 
 out:
 	if (ret == NULL) {
@@ -645,10 +667,11 @@
 
 static void nm_cluster_set_drop_object(struct kset *kset, struct kobject *kobj)
 {
-	struct nm_cluster_set *cs = to_nm_cluster_set(kset);
+	struct nm_cluster *cluster = to_nm_cluster(kobj);
 
-	printk("dropping cluster obj set %p from nm_cluster_set %p\n",
-	       kset, cs);
+	BUG_ON(nm_single_cluster != cluster);
+	nm_single_cluster = NULL;
+
 	kobject_put(kobj);
 }
 

Modified: branches/usysfsify/fs/ocfs2/cluster/nodemanager.h
===================================================================
--- branches/usysfsify/fs/ocfs2/cluster/nodemanager.h	2005-03-16 22:46:41 UTC (rev 1986)
+++ branches/usysfsify/fs/ocfs2/cluster/nodemanager.h	2005-03-17 00:21:05 UTC (rev 1987)
@@ -82,6 +82,7 @@
 	 * in network order */
 	__u32			nd_ipv4_address;
 	__u16			nd_ipv4_port;
+	struct rb_node		nd_ip_node;
 	/* there can be only one local node for now */
 	int			nd_local;
 



More information about the Ocfs2-commits mailing list