[rds-devel] dma_sync [was: Has anyone tried RDS on IA64 Linux?]

[Olaf Kirch]

On Monday 05 May 2008 08:03:44 Olaf Kirch wrote:
> You mean rather than using dma_map_page once on the page and using
> dma address+offset, we should call dma_map_page() with the appropriate
> offset for each fragment? That would work, too, and save us the hassles
> with dma_sync - where I'm now calling dma_sync, I'd just unmap
> the fragment instead.

For instance like this?

Olaf
---------------------------------
From: Olaf Kirch <olaf.kirch at oracle.com>
Subject: [RDS] Map recvbuf fragments separately

The RDS code currently uses 2k receive buffers. It allocates
one page, calls ib_dma_map_page() on it, and sets up the recv
WRs by putting dma_addr + offset into their SGE list.

When all WRs using the same page have been received, the page
is unmapped.

This creates a problem on platforms that require an explicit
dma_sync call after DMA receives - it is not clear whether is is
allowed to use dma_sync() on fragments of the mapped dma address
range.  So to avoid any ambiguities, this patch changes the code
to dma_map all fragments of the buffer page individually.

Signed-off-by: Olaf Kirch <olaf.kirch at oracle.com>
---
 net/rds/ib.h      |    1 -
 net/rds/ib_recv.c |   31 ++++++++++++-------------------
 2 files changed, 12 insertions(+), 20 deletions(-)

Index: ofa_kernel-1.3/net/rds/ib_recv.c
===================================================================
--- ofa_kernel-1.3.orig/net/rds/ib_recv.c
+++ ofa_kernel-1.3/net/rds/ib_recv.c
@@ -70,10 +70,9 @@ static void rds_ib_recv_unmap_page(struc
 {
 	rdsdebug("recv %p frag %p page %p\n", recv, recv->r_frag,
 		 recv->r_frag->f_page);
-	BUG_ON(recv->r_sge[0].addr == 0);
-	if (recv->r_frag->f_offset == RDS_PAGE_LAST_OFF)
+	if (recv->r_sge[0].addr)
 		ib_dma_unmap_page(ic->i_cm_id->device,
-			       recv->r_sge[0].addr - recv->r_frag->f_offset,
+			       recv->r_sge[0].addr,
 			       PAGE_SIZE, DMA_FROM_DEVICE);
 	recv->r_sge[0].addr = 0;
 }
@@ -127,11 +126,6 @@ void rds_ib_recv_clear_ring(struct rds_i
 	for(i = 0; i < ic->i_recv_ring.w_nr; i++)
 		rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
 
-	if (ic->i_addr) {
-		ib_dma_unmap_page(ic->i_cm_id->device,
-			       ic->i_addr, PAGE_SIZE, DMA_FROM_DEVICE);
-		ic->i_addr = 0;
-	}
 	if (ic->i_frag.f_page)
 		rds_ib_frag_drop_page(&ic->i_frag);
 }
@@ -141,6 +135,7 @@ static int rds_ib_recv_refill_one(struct
 				  gfp_t kptr_gfp, gfp_t page_gfp)
 {
 	struct rds_ib_connection *ic = conn->c_transport_data;
+	dma_addr_t dma_addr;
 	int ret = -ENOMEM;
 
 	if (recv->r_ibinc == NULL) {
@@ -169,17 +164,16 @@ static int rds_ib_recv_refill_one(struct
 		ic->i_frag.f_page = alloc_page(page_gfp);
 		if (ic->i_frag.f_page == NULL)
 			goto out;
+		ic->i_frag.f_offset = 0;
 	}
 
-	if (ic->i_addr == 0) {
-		ic->i_addr = ib_dma_map_page(ic->i_cm_id->device,
-					  ic->i_frag.f_page, 0, PAGE_SIZE,
-					  DMA_FROM_DEVICE);
-		if (ib_dma_mapping_error(ic->i_cm_id->device, ic->i_addr)) {
-			ic ->i_addr = 0;
-			goto out;
-		}
-	}
+	dma_addr = ib_dma_map_page(ic->i_cm_id->device,
+				  ic->i_frag.f_page,
+				  ic->i_frag.f_offset,
+				  RDS_FRAG_SIZE,
+				  DMA_FROM_DEVICE);
+	if (ib_dma_mapping_error(ic->i_cm_id->device, dma_addr))
+		goto out;
 
 	/*
 	 * Once we get the RDS_PAGE_LAST_OFF frag then rds_ib_frag_unmap()
@@ -188,7 +182,7 @@ static int rds_ib_recv_refill_one(struct
 	 */
 	recv->r_frag->f_page = ic->i_frag.f_page;
 	recv->r_frag->f_offset = ic->i_frag.f_offset;
-	recv->r_sge[0].addr = ic->i_addr + ic->i_frag.f_offset;
+	recv->r_sge[0].addr = dma_addr;
 	get_page(recv->r_frag->f_page);
 
 	if (ic->i_frag.f_offset < RDS_PAGE_LAST_OFF) {
@@ -197,7 +191,6 @@ static int rds_ib_recv_refill_one(struct
 		put_page(ic->i_frag.f_page);
 		ic->i_frag.f_page = NULL;
 		ic->i_frag.f_offset = 0;
-		ic->i_addr = 0;
 	}
 
 	ret = 0;
Index: ofa_kernel-1.3/net/rds/ib.h
===================================================================
--- ofa_kernel-1.3.orig/net/rds/ib.h
+++ ofa_kernel-1.3/net/rds/ib.h
@@ -85,7 +85,6 @@ struct rds_ib_connection {
 	u64			i_recv_hdrs_dma;
 	struct rds_ib_recv_work *i_recvs;
 	struct rds_page_frag	i_frag;
-	dma_addr_t 		i_addr;
 	u64			i_ack_recv;	/* last ACK received */
 
 	/* sending acks */

-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir at lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax