[rds-devel] [PATCH v2] RDS: Correctly reset connections to the nodev list.

[Steve Wise]

From: Steve Wise <swise at opengridcomputing.com>

If a connection gets shut down, then the connection struct needs to be
put back on the nodev list for proper reconnect handling.

Implementation:

- add a ptr to the rds_xx_device struct in the rds_xx_connection struct

- set this ptr when the connection gets put on the device conn list

- in rds_xx_conn_shutdown, if this ptr is set, then:
	remove the connection from the conn list 
	put it back on the nodev list
	null out the ptr
    
Signed-off-by: Steve Wise <swise at opengridcomputing.com>
---

 drivers/infiniband/ulp/rds/ib.h      |    3 +++
 drivers/infiniband/ulp/rds/ib_cm.c   |   17 +++++++++++++++++
 drivers/infiniband/ulp/rds/ib_rdma.c |    2 ++
 drivers/infiniband/ulp/rds/iw.h      |    3 +++
 drivers/infiniband/ulp/rds/iw_cm.c   |   17 +++++++++++++++++
 drivers/infiniband/ulp/rds/iw_rdma.c |    2 ++
 6 files changed, 44 insertions(+), 0 deletions(-)

diff --git a/drivers/infiniband/ulp/rds/ib.h b/drivers/infiniband/ulp/rds/ib.h
index f7ee4d6..eff70f0 100644
--- a/drivers/infiniband/ulp/rds/ib.h
+++ b/drivers/infiniband/ulp/rds/ib.h
@@ -74,9 +74,12 @@ struct rds_ib_work_ring {
 	atomic_t	w_free_ctr;
 };
 
+struct rds_ib_device;
+
 struct rds_ib_connection {
 
 	struct list_head	ib_node;
+	struct rds_ib_device	*rds_ibdev;
 	struct rds_connection	*conn;
 
 	/* alphabet soup, IBTA style */
diff --git a/drivers/infiniband/ulp/rds/ib_cm.c b/drivers/infiniband/ulp/rds/ib_cm.c
index 79afc66..85444e2 100644
--- a/drivers/infiniband/ulp/rds/ib_cm.c
+++ b/drivers/infiniband/ulp/rds/ib_cm.c
@@ -707,6 +707,22 @@ void rds_ib_conn_shutdown(struct rds_connection *conn)
 			ib_destroy_cq(ic->i_recv_cq);
 		rdma_destroy_id(ic->i_cm_id);
 
+		/*
+	 	 * Move connection back to the nodev list.
+		 */
+		if (ic->rds_ibdev) {
+
+			spin_lock_irq(&ic->rds_ibdev->spinlock);
+			BUG_ON(list_empty(&ic->ib_node));
+			list_del(&ic->ib_node);
+			spin_unlock_irq(&ic->rds_ibdev->spinlock);
+			
+			spin_lock_irq(&ib_nodev_conns_lock);
+			list_add_tail(&ic->ib_node, &ib_nodev_conns);
+			spin_unlock_irq(&ib_nodev_conns_lock);
+			ic->rds_ibdev = NULL;
+		}
+			
 		ic->i_cm_id = NULL;
 		ic->i_pd = NULL;
 		ic->i_mr = NULL;
@@ -716,6 +732,7 @@ void rds_ib_conn_shutdown(struct rds_connection *conn)
 		ic->i_recv_hdrs = NULL;
 		ic->i_ack = NULL;
 	}
+	BUG_ON(ic->rds_ibdev);
 
 	/* Clear pending transmit */
 	if (ic->i_rm) {
diff --git a/drivers/infiniband/ulp/rds/ib_rdma.c b/drivers/infiniband/ulp/rds/ib_rdma.c
index 79d92ee..2cc3204 100644
--- a/drivers/infiniband/ulp/rds/ib_rdma.c
+++ b/drivers/infiniband/ulp/rds/ib_rdma.c
@@ -153,6 +153,8 @@ int rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn
 	spin_lock_irq(&rds_ibdev->spinlock);
 	list_add_tail(&ic->ib_node, &rds_ibdev->conn_list);
 	spin_unlock_irq(&rds_ibdev->spinlock);
+	
+	ic->rds_ibdev = rds_ibdev;
 
 	return 0;
 }
diff --git a/drivers/infiniband/ulp/rds/iw.h b/drivers/infiniband/ulp/rds/iw.h
index 10eebe4..fa0cd2f 100644
--- a/drivers/infiniband/ulp/rds/iw.h
+++ b/drivers/infiniband/ulp/rds/iw.h
@@ -97,9 +97,12 @@ struct rds_iw_work_ring {
 	atomic_t	w_free_ctr;
 };
 
+struct rds_iw_device;
+
 struct rds_iw_connection {
 
 	struct list_head	iw_node;
+	struct rds_iw_device 	*rds_ibdev;
 	struct rds_connection	*conn;
 
 	/* alphabet soup, IBTA style */
diff --git a/drivers/infiniband/ulp/rds/iw_cm.c b/drivers/infiniband/ulp/rds/iw_cm.c
index d540e21..11df585 100644
--- a/drivers/infiniband/ulp/rds/iw_cm.c
+++ b/drivers/infiniband/ulp/rds/iw_cm.c
@@ -774,6 +774,22 @@ void rds_iw_conn_shutdown(struct rds_connection *conn)
 			ib_destroy_cq(ic->i_recv_cq);
 		rdma_destroy_id(ic->i_cm_id);
 
+		/*
+	 	 * Move connection back to the nodev list.
+		 */
+		if (ic->rds_ibdev) {
+
+			spin_lock_irq(&ic->rds_ibdev->spinlock);
+			BUG_ON(list_empty(&ic->iw_node));
+			list_del(&ic->iw_node);
+			spin_unlock_irq(&ic->rds_ibdev->spinlock);
+
+			spin_lock_irq(&iw_nodev_conns_lock);
+			list_add_tail(&ic->iw_node, &iw_nodev_conns);
+			spin_unlock_irq(&iw_nodev_conns_lock);
+			ic->rds_ibdev = NULL;
+		}
+
 		ic->i_cm_id = NULL;
 		ic->i_pd = NULL;
 		ic->i_mr = NULL;
@@ -783,6 +799,7 @@ void rds_iw_conn_shutdown(struct rds_connection *conn)
 		ic->i_recv_hdrs = NULL;
 		ic->i_ack = NULL;
 	}
+	BUG_ON(ic->rds_ibdev);
 
 	/* Clear pending transmit */
 	if (ic->i_rm) {
diff --git a/drivers/infiniband/ulp/rds/iw_rdma.c b/drivers/infiniband/ulp/rds/iw_rdma.c
index def6ee6..898821b 100644
--- a/drivers/infiniband/ulp/rds/iw_rdma.c
+++ b/drivers/infiniband/ulp/rds/iw_rdma.c
@@ -209,6 +209,8 @@ int rds_iw_add_conn(struct rds_iw_device *rds_ibdev, struct rds_connection *conn
 	list_add_tail(&ic->iw_node, &rds_ibdev->conn_list);
 	spin_unlock_irq(&rds_ibdev->spinlock);
 
+	ic->rds_ibdev = rds_ibdev;
+
 	return 0;
 }