[rds-commits] zab commits r122 - trunk/linux/net/rds

[svn-commits@oss.oracle.com]

Author: zab
Date: 2006-07-03 18:18:08 -0500 (Mon, 03 Jul 2006)
New Revision: 122

Modified:
   trunk/linux/net/rds/ib.h
   trunk/linux/net/rds/ib_send.c
Log:
More minor IB send and receive fixes.

We don't need a pointer from rds_ib_data_send to the connection, we can get
there from the callbacks.

We're not doing anything with immediate data at the moment so stop sending
it.

When allocating rds_ib_data_send we weren't calculating the space consumed by
ib_sge elements properly.  We also initialize all that's allocated so we don't
need to use kzalloc().  And we can clean up the pointer assignments and
initialization loop a little.

Initialize the data_send refcount just once to include the ref held by
the caller.

Check for failure to allocate the data_send.


Modified: trunk/linux/net/rds/ib.h
===================================================================
--- trunk/linux/net/rds/ib.h	2006-06-29 00:27:54 UTC (rev 121)
+++ trunk/linux/net/rds/ib.h	2006-07-03 23:18:08 UTC (rev 122)
@@ -45,7 +45,6 @@
  */
 struct rds_ib_data_send {
 	struct rds_message		*id_rm;
-	struct rds_ib_connection	*id_ic;
 	atomic_t			id_refcount;
 	struct ib_sge			*id_sge;
 	struct ib_send_wr		*id_wr;

Modified: trunk/linux/net/rds/ib_send.c
===================================================================
--- trunk/linux/net/rds/ib_send.c	2006-06-29 00:27:54 UTC (rev 121)
+++ trunk/linux/net/rds/ib_send.c	2006-07-03 23:18:08 UTC (rev 122)
@@ -84,7 +84,7 @@
 	ih->ih_wr.wr_id = ((unsigned long)ih) | RDS_IB_WR_ID_HEADER;
 	ih->ih_wr.sg_list = &ih->ih_sge;
 	ih->ih_wr.num_sge = 1;
-	ih->ih_wr.opcode = IB_WR_SEND_WITH_IMM;
+	ih->ih_wr.opcode = IB_WR_SEND;
 	ih->ih_wr.send_flags = 0;  /* XXX what could these be? */
 	ih->ih_wr.imm_data = 0;
 
@@ -109,14 +109,14 @@
 	return ret;
 }
 
-void rds_ib_ds_put(struct rds_ib_data_send *ds)
+void rds_ib_ds_put(struct rds_connection *conn, struct rds_ib_data_send *ds)
 {
+	struct rds_ib_connection *ic = conn->c_transport_data;
+	struct rds_message *rm = ds->id_rm;
+
 	rdsdebug("decref ds %p ref %d\n", ds, atomic_read(&ds->id_refcount));
 	if (atomic_dec_and_test(&ds->id_refcount)) {
-		if (ds->id_rm) {
-			struct rds_ib_connection *ic = ds->id_ic;
-			struct rds_message *rm = ds->id_rm;
-
+		if (rm) {
 			dma_unmap_sg(ic->i_cm_id->device->dma_device, rm->m_sg,
 				     rm->m_nents, DMA_TO_DEVICE);
 			rds_message_put(rm);
@@ -128,15 +128,11 @@
 void rds_ib_data_complete(struct rds_connection *conn, u64 wr_id)
 {
 	struct rds_ib_data_send *ds = (void *)(unsigned long)wr_id;
-
-	rdsdebug("conn %p wr_id 0x%llx\n", conn, (unsigned long long)wr_id);
-	rds_ib_ds_put(ds);
+	rds_ib_ds_put(conn, ds);
 }
 
 static struct rds_ib_data_send *rds_ib_ds_get(struct rds_ib_connection *ic,
 					      struct rds_message *rm,
-					      unsigned int sg,
-					      unsigned int off,
 					      unsigned long nr_work)
 {
 	struct rds_ib_data_send *ds;
@@ -145,11 +141,12 @@
 	struct ib_sge *sge;
 	unsigned long len;
 	unsigned long i;
+	unsigned int off;
 	int ret;
 
-	ds = kzalloc(sizeof(struct rds_ib_data_send) + (nr_work *
-		     (sizeof(struct ib_send_wr) +
-		      sizeof(struct ib_send_wr))), GFP_KERNEL);
+	ds = kmalloc(sizeof(struct rds_ib_data_send) +
+		     (nr_work * (sizeof(struct ib_sge) +
+			         sizeof(struct ib_send_wr))), GFP_KERNEL);
 	if (ds == NULL)
 		goto out;
 
@@ -167,18 +164,16 @@
 
 	rds_message_addref(rm);
 	ds->id_rm = rm;
-	ds->id_ic = ic;
-	atomic_set(&ds->id_refcount, nr_work);
-	ds->id_sge = (void *)ds + sizeof(struct rds_ib_data_send);
-	ds->id_wr = (void *)ds->id_sge +
-		    (nr_work * sizeof(struct ib_sge));
+	atomic_set(&ds->id_refcount, nr_work + 1);
+	ds->id_sge = (struct ib_sge *)(ds + 1);
+	ds->id_wr = (struct ib_send_wr *)(&ds->id_sge[nr_work]);
 
+	scat = rm->m_sg;
+	sge = ds->id_sge;
+	wr = ds->id_wr;
+	off = 0;
 
-	for (i = 0; i < nr_work; i++) {
-		sge = &ds->id_sge[i];
-		wr = &ds->id_wr[i];
-		scat = &rm->m_sg[sg];
-
+	for (i = 0; i < nr_work; i++, sge++, wr++) {
 		len = min(RDS_FRAG_SIZE, sg_dma_len(scat) - off);
 
 		sge->addr = sg_dma_address(scat) + off;
@@ -187,8 +182,8 @@
 
 		off += len;
 		if (len == sg_dma_len(scat)) {
+			scat++;
 			off = 0;
-			sg++;
 		}
 
 		if (i != nr_work - 1)
@@ -198,7 +193,9 @@
 		wr->wr_id = ((unsigned long)ds) | RDS_IB_WR_ID_DATA;
 		wr->sg_list = sge;
 		wr->num_sge = 1;
-		wr->opcode = IB_WR_SEND_WITH_IMM;
+		wr->opcode = IB_WR_SEND;
+		wr->send_flags = 0;
+		wr->imm_data = 0;
 	}
 out:
 	return ds;
@@ -226,9 +223,12 @@
 	nr_work = ceil(be32_to_cpu(rm->m_inc.i_hdr.h_len), RDS_FRAG_SIZE);
 
 	if (ds == NULL) {
-		ds = rds_ib_ds_get(ic, rm, sg, off, nr_work);
+		ds = rds_ib_ds_get(ic, rm, nr_work);
+		if (ds == NULL) {
+			ret = -ENOMEM;
+			goto out;
+		}
 		ic->i_ds = ds;
-		atomic_set(&ds->id_refcount, nr_work + 1);
 	}
 
 	first = (sg * (PAGE_SIZE / RDS_FRAG_SIZE)) + (off / RDS_FRAG_SIZE);
@@ -251,7 +251,7 @@
 	/* XXX this leaves the ds in flight, conn reset will have to 
 	 * force completion of posted work reqs somehow */
 	ic->i_ds = NULL;
-	rds_ib_ds_put(ds);
+	rds_ib_ds_put(conn, ds);
 out:
 	return ret;
 }