[rds-commits] zab commits r120 - trunk/linux/net/rds
svn-commits@oss.oracle.com
svn-commits at oss.oracle.com
Wed Jun 28 16:54:14 CDT 2006
Author: zab
Date: 2006-06-28 16:54:13 -0500 (Wed, 28 Jun 2006)
New Revision: 120
Modified:
trunk/linux/net/rds/ib.h
trunk/linux/net/rds/ib_cm.c
trunk/linux/net/rds/ib_recv.c
trunk/linux/net/rds/ib_send.c
Log:
More progress towards working IB send and receive.
Initialize and destroy the dma pool we'll be using for headers.
Lose our device pointer in the IB connection and use the one embedded in
the CM ID.
Tear down the recv structs as the IB incoming is freed.
Make sure to drop the count of posted receives as one completes and unmap it if
we're not immediately reposting it.
Modified: trunk/linux/net/rds/ib.h
===================================================================
--- trunk/linux/net/rds/ib.h 2006-06-28 20:58:12 UTC (rev 119)
+++ trunk/linux/net/rds/ib.h 2006-06-28 21:54:13 UTC (rev 120)
@@ -60,7 +60,6 @@
struct rds_ib_connection {
struct dma_pool *i_pool;
- struct device *i_device;
/* alphabet soup, IBTA style */
struct rdma_cm_id *i_cm_id;
Modified: trunk/linux/net/rds/ib_cm.c
===================================================================
--- trunk/linux/net/rds/ib_cm.c 2006-06-28 20:58:12 UTC (rev 119)
+++ trunk/linux/net/rds/ib_cm.c 2006-06-28 21:54:13 UTC (rev 120)
@@ -165,6 +165,15 @@
goto out;
}
+ ic->i_pool = dma_pool_create("rds_header_send",
+ ic->i_cm_id->device->dma_device,
+ sizeof(struct rds_ib_header_send),
+ 0, 0);
+ if (ic->i_pool == NULL) {
+ rdsdebug("dma_pool_create failed\n");
+ goto out;
+ }
+
rdsdebug("conn %p pd %p mr %p cq %p\n", conn, ic->i_pd, ic->i_mr,
ic->i_cq);
@@ -352,6 +361,9 @@
ic->i_pd, ic->i_cq,
ic->i_cm_id ? ic->i_cm_id->qp : NULL);
+ if (ic->i_pool)
+ dma_pool_destroy(ic->i_pool);
+
if (ic->i_cm_id) {
if (ic->i_cm_id->qp)
rdma_destroy_qp(ic->i_cm_id);
Modified: trunk/linux/net/rds/ib_recv.c
===================================================================
--- trunk/linux/net/rds/ib_recv.c 2006-06-28 20:58:12 UTC (rev 119)
+++ trunk/linux/net/rds/ib_recv.c 2006-06-28 21:54:13 UTC (rev 120)
@@ -32,19 +32,12 @@
static kmem_cache_t *rds_ib_incoming_slab;
static kmem_cache_t *rds_ib_recv_slab;
-void rds_ib_inc_free(struct rds_incoming *inc)
-{
- struct rds_ib_incoming *ibinc;
- ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
- rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
- BUG_ON(!list_empty(&ibinc->ii_recvs));
- kmem_cache_free(rds_ib_incoming_slab, ibinc);
-}
-
static void rds_ib_recv_free(struct rds_ib_recv *recv)
{
BUG_ON(!list_empty(&recv->ir_item));
BUG_ON(recv->ir_sge.addr != 0);
+
+ rdsdebug("recv %p page %p\n", recv, recv->ir_page);
__free_page(recv->ir_page);
kmem_cache_free(rds_ib_recv_slab, recv);
}
@@ -52,11 +45,28 @@
static void rds_ib_recv_unmap(struct rds_ib_connection *ic,
struct rds_ib_recv *recv)
{
+ rdsdebug("recv %p page %p\n", recv, recv->ir_page);
dma_unmap_page(ic->i_cm_id->device->dma_device, recv->ir_sge.addr,
PAGE_SIZE, DMA_FROM_DEVICE);
recv->ir_sge.addr = 0;
}
+void rds_ib_inc_free(struct rds_incoming *inc)
+{
+ struct rds_ib_incoming *ibinc;
+ struct rds_ib_recv *recv, *pos;
+
+ ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
+ rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
+
+ list_for_each_entry_safe(recv, pos, &ibinc->ii_recvs, ir_item) {
+ list_del_init(&recv->ir_item);
+ rds_ib_recv_free(recv);
+ }
+
+ kmem_cache_free(rds_ib_incoming_slab, ibinc);
+}
+
/*
* allcates a new rds_ib_incoming and initializes its header from the
* header at the start of the give page.
@@ -176,7 +186,9 @@
/* XXX we could be posting lots of these at once */
atomic_inc(&ic->i_recv_posted);
ret = ib_post_recv(ic->i_cm_id->qp, &recv->ir_wr, &failed);
+ rdsdebug("recv %p post ret %d\n", recv, ret);
if (ret) {
+ /* XXX if posting fails kick the thread to refill? */
rds_ib_recv_unmap(ic, recv);
rds_ib_recv_free(recv);
atomic_dec(&ic->i_recv_posted);
@@ -194,8 +206,8 @@
struct rds_ib_recv *recv = (void *)(unsigned long)wr_id;
struct rds_ib_incoming *ibinc = ic->i_ibinc;
- rdsdebug("recv ic %p recv %p sge len %u\n", ic, recv,
- recv->ir_sge.length);
+ rdsdebug("ic %p recv %p sge len %u\n", ic, recv, recv->ir_sge.length);
+ atomic_dec(&ic->i_recv_posted);
/*
* XXX worry about how we clean these up.. if qp shutdown leads
@@ -212,6 +224,7 @@
return;
}
+ rds_ib_recv_unmap(ic, recv);
list_add_tail(&recv->ir_item, &ibinc->ii_recvs);
/* XXX use sge len in case sender does weird things */
Modified: trunk/linux/net/rds/ib_send.c
===================================================================
--- trunk/linux/net/rds/ib_send.c 2006-06-28 20:58:12 UTC (rev 119)
+++ trunk/linux/net/rds/ib_send.c 2006-06-28 21:54:13 UTC (rev 120)
@@ -103,8 +103,7 @@
out:
if (ret < 0 && ih) {
if (ih->ih_vaddr)
- dma_pool_free(ic->i_pool, ih->ih_vaddr,
- ih->ih_dma);
+ dma_pool_free(ic->i_pool, ih->ih_vaddr, ih->ih_dma);
kmem_cache_free(rds_ib_header_send_slab, ih);
}
return ret;
@@ -118,7 +117,7 @@
struct rds_ib_connection *ic = ds->id_ic;
struct rds_message *rm = ds->id_rm;
- dma_unmap_sg(ic->i_device, rm->m_sg,
+ dma_unmap_sg(ic->i_cm_id->device->dma_device, rm->m_sg,
rm->m_nents, DMA_TO_DEVICE);
rds_message_put(rm);
}
@@ -155,11 +154,11 @@
goto out;
/* only set id_rm after we map it */
- ret = dma_map_sg(ic->i_device, rm->m_sg, rm->m_nents,
- DMA_TO_DEVICE);
+ ret = dma_map_sg(ic->i_cm_id->device->dma_device, rm->m_sg,
+ rm->m_nents, DMA_TO_DEVICE);
if (ret != rm->m_nents) {
if (ret > 0) /* XXX not sure this is possible */
- dma_unmap_sg(ic->i_device, rm->m_sg,
+ dma_unmap_sg(ic->i_cm_id->device->dma_device, rm->m_sg,
rm->m_nents, DMA_TO_DEVICE);
kfree(ds);
ds = NULL;
More information about the rds-commits
mailing list