[rds-devel] [PATCH 10/19] Make RDS global statistics works. - Teach stats_info_copy() aware of struct net. - Modify stats_inc macro aware of struct net. - Make others stats related stuff can be compiled with this change.
Jie Liu
jeff.liu at oracle.com
Sun Oct 7 05:18:02 PDT 2012
Signed-off-by: Jie Liu <jeff.liu at oracle.com>
---
net/rds/cong.c | 8 ++++----
net/rds/connection.c | 6 ++++--
net/rds/ib.h | 4 ++--
net/rds/ib_recv.c | 15 +++++++++------
net/rds/ib_send.c | 6 ++++--
net/rds/iw.h | 4 ++--
net/rds/iw_recv.c | 44 ++++++++++++++++++++++++--------------------
net/rds/iw_send.c | 14 +++++++-------
net/rds/message.c | 20 ++++++++++++--------
net/rds/page.c | 14 +++++++-------
net/rds/rdma.c | 15 ++++++++-------
net/rds/rds.h | 38 +++++++++++++++++++++-----------------
net/rds/recv.c | 19 ++++++++++---------
net/rds/send.c | 22 +++++++++++++---------
net/rds/stats.c | 9 ++++-----
net/rds/tcp.h | 4 ++--
net/rds/tcp_recv.c | 8 +++++---
net/rds/threads.c | 21 ++++++++++++++-------
net/rds/transport.c | 3 ++-
19 files changed, 154 insertions(+), 120 deletions(-)
diff --git a/net/rds/cong.c b/net/rds/cong.c
index 585415e..518d662 100644
--- a/net/rds/cong.c
+++ b/net/rds/cong.c
@@ -204,7 +204,7 @@ void rds_cong_queue_updates(struct net *net, struct rds_cong_map *map)
list_for_each_entry(conn, &map->m_conn_list, c_map_item) {
if (!test_and_set_bit(0, &conn->c_map_queued)) {
- rds_stats_inc(s_cong_update_queued);
+ rds_stats_inc(net, s_cong_update_queued);
rds_send_xmit(conn);
}
}
@@ -219,7 +219,7 @@ void rds_cong_map_updated(struct net *net, struct rds_cong_map *map,
rdsdebug("waking map %p for %pI4\n", map, &map->m_addr);
- rds_stats_inc(s_cong_update_received);
+ rds_stats_inc(net, s_cong_update_received);
atomic_inc(&rds_cong_generation);
@@ -357,11 +357,11 @@ int rds_cong_wait(struct net *net, struct rds_cong_map *map,
if (!rds_cong_test_bit(map, port))
return 0;
}
- rds_stats_inc(s_cong_send_error);
+ rds_stats_inc(net, s_cong_send_error);
return -ENOBUFS;
}
- rds_stats_inc(s_cong_send_blocked);
+ rds_stats_inc(net, s_cong_send_blocked);
rdsdebug("waiting on map %p for port %u\n", map, be16_to_cpu(port));
return wait_event_interruptible(map->m_waitq,
diff --git a/net/rds/connection.c b/net/rds/connection.c
index d118c9d..6ff5529 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -92,10 +92,12 @@ static struct rds_connection *rds_conn_lookup(struct hlist_head *head,
*/
static void rds_conn_reset(struct rds_connection *conn)
{
+ struct net *net = rds_conn_to_net(conn);
+
rdsdebug("connection %pI4 to %pI4 reset\n",
- &conn->c_laddr, &conn->c_faddr);
+ &conn->c_laddr, &conn->c_faddr);
- rds_stats_inc(s_conn_reset);
+ rds_stats_inc(net, s_conn_reset);
rds_send_reset(conn);
conn->c_flags = 0;
diff --git a/net/rds/ib.h b/net/rds/ib.h
index 71175dc..3c8dd95 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -316,8 +316,8 @@ int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic);
void rds_ib_recv_free_caches(struct rds_ib_connection *ic);
void rds_ib_recv_refill(struct rds_connection *conn, int prefill);
void rds_ib_inc_free(struct rds_incoming *inc);
-int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
- size_t size);
+int rds_ib_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *iov, size_t size);
void rds_ib_recv_cq_comp_handler(struct ib_cq *cq, void *context);
void rds_ib_recv_tasklet_fn(unsigned long data);
void rds_ib_recv_init_ring(struct rds_ib_connection *ic);
diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index 6053182..ea0fe33 100644
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -278,12 +278,14 @@ static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic
if (cache_item) {
frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
} else {
+ struct net *net = rds_conn_to_net(ic->conn);
+
frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
if (!frag)
return NULL;
sg_init_table(&frag->f_sg, 1);
- ret = rds_page_remainder_alloc(&frag->f_sg,
+ ret = rds_page_remainder_alloc(net, &frag->f_sg,
RDS_FRAG_SIZE, page_mask);
if (ret) {
kmem_cache_free(rds_ib_frag_slab, frag);
@@ -468,8 +470,8 @@ static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache
return head;
}
-int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
- size_t size)
+int rds_ib_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *first_iov, size_t size)
{
struct rds_ib_incoming *ibinc;
struct rds_page_frag *frag;
@@ -506,7 +508,7 @@ int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
sg_page(&frag->f_sg), frag->f_sg.offset, frag_off);
/* XXX needs + offset for multiple recvs per page */
- ret = rds_page_copy_to_user(sg_page(&frag->f_sg),
+ ret = rds_page_copy_to_user(net, sg_page(&frag->f_sg),
frag->f_sg.offset + frag_off,
iov->iov_base + iov_off,
to_copy);
@@ -816,6 +818,7 @@ static void rds_ib_process_recv(struct rds_connection *conn,
struct rds_ib_recv_work *recv, u32 data_len,
struct rds_ib_ack_state *state)
{
+ struct net *net = rds_conn_to_net(conn);
struct rds_ib_connection *ic = conn->c_transport_data;
struct rds_ib_incoming *ibinc = ic->i_ibinc;
struct rds_header *ihdr, *hdr;
@@ -843,7 +846,7 @@ static void rds_ib_process_recv(struct rds_connection *conn,
"from %pI4 has corrupted header - "
"forcing a reconnect\n",
&conn->c_faddr);
- rds_stats_inc(s_recv_drop_bad_checksum);
+ rds_stats_inc(net, s_recv_drop_bad_checksum);
return;
}
@@ -929,7 +932,7 @@ static void rds_ib_process_recv(struct rds_connection *conn,
* the complete frame, and after bumping the next_rx
* sequence. */
if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
- rds_stats_inc(s_recv_ack_required);
+ rds_stats_inc(net, s_recv_ack_required);
state->ack_required = 1;
}
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index 94f43e2..f118ebd 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -116,6 +116,8 @@ static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
struct rm_rdma_op *op,
int wc_status)
{
+ struct net *net = rds_conn_to_net(ic->conn);
+
if (op->op_mapped) {
ib_dma_unmap_sg(ic->i_cm_id->device,
op->op_sg, op->op_nents,
@@ -147,9 +149,9 @@ static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,
wc_status, rds_rdma_send_complete);
if (op->op_write)
- rds_stats_add(s_send_rdma_bytes, op->op_bytes);
+ rds_stats_add(net, s_send_rdma_bytes, op->op_bytes);
else
- rds_stats_add(s_recv_rdma_bytes, op->op_bytes);
+ rds_stats_add(net, s_recv_rdma_bytes, op->op_bytes);
}
static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,
diff --git a/net/rds/iw.h b/net/rds/iw.h
index 8c5d8c5..b963d2a 100644
--- a/net/rds/iw.h
+++ b/net/rds/iw.h
@@ -325,8 +325,8 @@ int rds_iw_recv(struct rds_connection *conn);
int rds_iw_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
gfp_t page_gfp, int prefill);
void rds_iw_inc_free(struct rds_incoming *inc);
-int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
- size_t size);
+int rds_iw_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *iov, size_t size);
void rds_iw_recv_cq_comp_handler(struct ib_cq *cq, void *context);
void rds_iw_recv_tasklet_fn(unsigned long data);
void rds_iw_recv_init_ring(struct rds_iw_connection *ic);
diff --git a/net/rds/iw_recv.c b/net/rds/iw_recv.c
index 02b0dca..98f4ffc 100644
--- a/net/rds/iw_recv.c
+++ b/net/rds/iw_recv.c
@@ -144,7 +144,8 @@ static int rds_iw_recv_refill_one(struct rds_connection *conn,
int ret = -ENOMEM;
if (!recv->r_iwinc) {
- if (!atomic_add_unless(&rds_iw_allocation, 1, rds_iw_sysctl_max_recv_allocation)) {
+ if (!atomic_add_unless(&rds_iw_allocation, 1,
+ rds_iw_sysctl_max_recv_allocation)) {
rds_iw_stats_inc(s_iw_rx_alloc_limit);
goto out;
}
@@ -303,8 +304,8 @@ void rds_iw_inc_free(struct rds_incoming *inc)
BUG_ON(atomic_read(&rds_iw_allocation) < 0);
}
-int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
- size_t size)
+int rds_iw_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *first_iov, size_t size)
{
struct rds_iw_incoming *iwinc;
struct rds_page_frag *frag;
@@ -341,7 +342,7 @@ int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
frag->f_page, frag->f_offset, frag_off);
/* XXX needs + offset for multiple recvs per page */
- ret = rds_page_copy_to_user(frag->f_page,
+ ret = rds_page_copy_to_user(net, frag->f_page,
frag->f_offset + frag_off,
iov->iov_base + iov_off,
to_copy);
@@ -461,8 +462,11 @@ static void rds_iw_send_ack(struct rds_iw_connection *ic, unsigned int adv_credi
ic->i_ack_queued = jiffies;
ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr);
- if (unlikely(ret)) {
- /* Failed to send. Release the WR, and
+ if (!ret)
+ rds_iw_stats_inc(s_iw_ack_sent);
+ else {
+ /*
+ * Failed to send. Release the WR, and
* force another ACK.
*/
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
@@ -471,8 +475,7 @@ static void rds_iw_send_ack(struct rds_iw_connection *ic, unsigned int adv_credi
rds_iw_stats_inc(s_iw_ack_send_failure);
rds_iw_conn_error(ic->conn, "sending ack failed\n");
- } else
- rds_iw_stats_inc(s_iw_ack_sent);
+ }
}
/*
@@ -526,7 +529,8 @@ void rds_iw_attempt_ack(struct rds_iw_connection *ic)
}
/* Can we get a send credit? */
- if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
+ if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0,
+ RDS_MAX_ADV_CREDIT)) {
rds_iw_stats_inc(s_iw_tx_throttle);
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
return;
@@ -554,6 +558,7 @@ u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic)
{
if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
rds_iw_stats_inc(s_iw_ack_send_piggybacked);
+
return rds_iw_get_ack(ic);
}
@@ -651,6 +656,7 @@ static void rds_iw_process_recv(struct rds_connection *conn,
struct rds_iw_recv_work *recv, u32 byte_len,
struct rds_iw_ack_state *state)
{
+ struct net *net = rds_conn_to_net(conn);
struct rds_iw_connection *ic = conn->c_transport_data;
struct rds_iw_incoming *iwinc = ic->i_iwinc;
struct rds_header *ihdr, *hdr;
@@ -661,11 +667,9 @@ static void rds_iw_process_recv(struct rds_connection *conn,
byte_len);
if (byte_len < sizeof(struct rds_header)) {
- rds_iw_conn_error(conn, "incoming message "
- "from %pI4 didn't include a "
- "header, disconnecting and "
- "reconnecting\n",
- &conn->c_faddr);
+ rds_iw_conn_error(conn, "incoming message from %pI4 didn't "
+ "include a header, disconnecting and "
+ "reconnecting\n", &conn->c_faddr);
return;
}
byte_len -= sizeof(struct rds_header);
@@ -675,10 +679,9 @@ static void rds_iw_process_recv(struct rds_connection *conn,
/* Validate the checksum. */
if (!rds_message_verify_checksum(ihdr)) {
rds_iw_conn_error(conn, "incoming message "
- "from %pI4 has corrupted header - "
- "forcing a reconnect\n",
- &conn->c_faddr);
- rds_stats_inc(s_recv_drop_bad_checksum);
+ "from %pI4 has corrupted header - "
+ "forcing a reconnect\n", &conn->c_faddr);
+ rds_stats_inc(net, s_recv_drop_bad_checksum);
return;
}
@@ -764,7 +767,7 @@ static void rds_iw_process_recv(struct rds_connection *conn,
* the complete frame, and after bumping the next_rx
* sequence. */
if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
- rds_stats_inc(s_recv_ack_required);
+ rds_stats_inc(net, s_recv_ack_required);
state->ack_required = 1;
}
@@ -797,8 +800,9 @@ static inline void rds_poll_cq(struct rds_iw_connection *ic,
struct rds_iw_ack_state *state)
{
struct rds_connection *conn = ic->conn;
- struct ib_wc wc;
+ struct net *net = rds_conn_to_net(conn);
struct rds_iw_recv_work *recv;
+ struct ib_wc wc;
while (ib_poll_cq(ic->i_recv_cq, 1, &wc) > 0) {
rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
diff --git a/net/rds/iw_send.c b/net/rds/iw_send.c
index e40c3c5..fbe827a 100644
--- a/net/rds/iw_send.c
+++ b/net/rds/iw_send.c
@@ -75,16 +75,16 @@ static void rds_iw_send_unmap_rdma(struct rds_iw_connection *ic,
}
static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic,
- struct rds_iw_send_work *send,
- int wc_status)
+ struct rds_iw_send_work *send,
+ int wc_status)
{
+ struct net *net = rds_conn_to_net(ic->conn);
struct rds_message *rm = send->s_rm;
rdsdebug("ic %p send %p rm %p\n", ic, send, rm);
- ib_dma_unmap_sg(ic->i_cm_id->device,
- rm->data.op_sg, rm->data.op_nents,
- DMA_TO_DEVICE);
+ ib_dma_unmap_sg(ic->i_cm_id->device, rm->data.op_sg,
+ rm->data.op_nents, DMA_TO_DEVICE);
if (rm->rdma.op_active) {
rds_iw_send_unmap_rdma(ic, &rm->rdma);
@@ -112,9 +112,9 @@ static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic,
rds_iw_send_rdma_complete(rm, wc_status);
if (rm->rdma.op_write)
- rds_stats_add(s_send_rdma_bytes, rm->rdma.op_bytes);
+ rds_stats_add(net, s_send_rdma_bytes, rm->rdma.op_bytes);
else
- rds_stats_add(s_recv_rdma_bytes, rm->rdma.op_bytes);
+ rds_stats_add(net, s_recv_rdma_bytes, rm->rdma.op_bytes);
}
/* If anyone waited for this message to get flushed out, wake
diff --git a/net/rds/message.c b/net/rds/message.c
index f0a4658..ade4244 100644
--- a/net/rds/message.c
+++ b/net/rds/message.c
@@ -82,10 +82,12 @@ static void rds_message_purge(struct rds_message *rm)
void rds_message_put(struct rds_message *rm)
{
rdsdebug("put rm %p ref %d\n", rm, atomic_read(&rm->m_refcount));
+
if (atomic_read(&rm->m_refcount) == 0) {
-printk(KERN_CRIT "danger refcount zero on %p\n", rm);
-WARN_ON(1);
+ printk(KERN_CRIT "danger refcount zero on %p\n", rm);
+ WARN_ON(1);
}
+
if (atomic_dec_and_test(&rm->m_refcount)) {
BUG_ON(!list_empty(&rm->m_sock_item));
BUG_ON(!list_empty(&rm->m_conn_item));
@@ -264,8 +266,8 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in
return rm;
}
-int rds_message_copy_from_user(struct rds_message *rm, struct iovec *first_iov,
- size_t total_len)
+int rds_message_copy_from_user(struct net *net, struct rds_message *rm,
+ struct iovec *first_iov, size_t total_len)
{
unsigned long to_copy;
unsigned long iov_off;
@@ -286,7 +288,7 @@ int rds_message_copy_from_user(struct rds_message *rm, struct iovec *first_iov,
while (total_len) {
if (!sg_page(sg)) {
- ret = rds_page_remainder_alloc(sg, total_len,
+ ret = rds_page_remainder_alloc(net, sg, total_len,
GFP_HIGHUSER);
if (ret)
goto out;
@@ -307,7 +309,8 @@ int rds_message_copy_from_user(struct rds_message *rm, struct iovec *first_iov,
to_copy, iov->iov_base, iov->iov_len, iov_off,
(void *)sg_page(sg), sg->offset, sg->length, sg_off);
- ret = rds_page_copy_from_user(sg_page(sg), sg->offset + sg_off,
+ ret = rds_page_copy_from_user(net, sg_page(sg),
+ sg->offset + sg_off,
iov->iov_base + iov_off,
to_copy);
if (ret)
@@ -325,7 +328,7 @@ out:
return ret;
}
-int rds_message_inc_copy_to_user(struct rds_incoming *inc,
+int rds_message_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
struct iovec *first_iov, size_t size)
{
struct rds_message *rm;
@@ -362,7 +365,8 @@ int rds_message_inc_copy_to_user(struct rds_incoming *inc,
to_copy, iov->iov_base, iov->iov_len, iov_off,
sg_page(sg), sg->offset, sg->length, vec_off);
- ret = rds_page_copy_to_user(sg_page(sg), sg->offset + vec_off,
+ ret = rds_page_copy_to_user(net, sg_page(sg),
+ sg->offset + vec_off,
iov->iov_base + iov_off,
to_copy);
if (ret) {
diff --git a/net/rds/page.c b/net/rds/page.c
index 9005a2c..dc8d18a 100644
--- a/net/rds/page.c
+++ b/net/rds/page.c
@@ -53,7 +53,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder,
* user pages we'd have to worry more about cache coherence. (Though
* the flush_dcache_page() in get_user_pages() would probably be enough).
*/
-int rds_page_copy_user(struct page *page, unsigned long offset,
+int rds_page_copy_user(struct net *net, struct page *page, unsigned long offset,
void __user *ptr, unsigned long bytes,
int to_user)
{
@@ -62,10 +62,10 @@ int rds_page_copy_user(struct page *page, unsigned long offset,
addr = kmap(page);
if (to_user) {
- rds_stats_add(s_copy_to_user, bytes);
+ rds_stats_add(net, s_copy_to_user, bytes);
ret = copy_to_user(ptr, addr + offset, bytes);
} else {
- rds_stats_add(s_copy_from_user, bytes);
+ rds_stats_add(net, s_copy_from_user, bytes);
ret = copy_from_user(addr + offset, ptr, bytes);
}
kunmap(page);
@@ -94,8 +94,8 @@ EXPORT_SYMBOL_GPL(rds_page_copy_user);
* path passes read-only page regions down to devices. They hold a page
* reference until they are done with the region.
*/
-int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
- gfp_t gfp)
+int rds_page_remainder_alloc(struct net *net, struct scatterlist *scat,
+ unsigned long bytes, gfp_t gfp)
{
struct rds_page_remainder *rem;
unsigned long flags;
@@ -122,7 +122,7 @@ int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
while (1) {
/* avoid a tiny region getting stuck by tossing it */
if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) {
- rds_stats_inc(s_page_remainder_miss);
+ rds_stats_inc(net, s_page_remainder_miss);
__free_page(rem->r_page);
rem->r_page = NULL;
}
@@ -133,7 +133,7 @@ int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
get_page(sg_page(scat));
if (rem->r_offset != 0)
- rds_stats_inc(s_page_remainder_hit);
+ rds_stats_inc(net, s_page_remainder_hit);
rem->r_offset += bytes;
if (rem->r_offset == PAGE_SIZE) {
diff --git a/net/rds/rdma.c b/net/rds/rdma.c
index 4e37c1c..e4a3e46 100644
--- a/net/rds/rdma.c
+++ b/net/rds/rdma.c
@@ -544,16 +544,17 @@ int rds_rdma_extra_size(struct rds_rdma_args *args)
* Extract all arguments and set up the rdma_op
*/
int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
- struct cmsghdr *cmsg)
+ struct cmsghdr *cmsg)
{
- struct rds_rdma_args *args;
+ struct net *net = sock_net(&rs->rs_sk);
struct rm_rdma_op *op = &rm->rdma;
- int nr_pages;
- unsigned int nr_bytes;
+ struct rds_rdma_args *args;
struct page **pages = NULL;
struct rds_iovec iovstack[UIO_FASTIOV], *iovs = iovstack;
- int iov_size;
+ unsigned int nr_bytes;
unsigned int i, j;
+ int iov_size;
+ int nr_pages;
int ret = 0;
if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_rdma_args))
@@ -699,7 +700,7 @@ out:
if (ret)
rds_rdma_free_op(op);
else
- rds_stats_inc(s_send_rdma);
+ rds_stats_inc(net, s_send_rdma);
return ret;
}
@@ -709,7 +710,7 @@ out:
* to the remote
*/
int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
- struct cmsghdr *cmsg)
+ struct cmsghdr *cmsg)
{
unsigned long flags;
struct rds_mr *mr;
diff --git a/net/rds/rds.h b/net/rds/rds.h
index bb91d04..fdf8103 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -438,8 +438,8 @@ struct rds_transport {
int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
int (*recv)(struct rds_connection *conn);
- int (*inc_copy_to_user)(struct rds_incoming *inc, struct iovec *iov,
- size_t size);
+ int (*inc_copy_to_user)(struct net *net, struct rds_incoming *inc,
+ struct iovec *iov, size_t size);
void (*inc_free)(struct rds_incoming *inc);
int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
@@ -669,8 +669,8 @@ rds_conn_connecting(struct rds_connection *conn)
/* message.c */
struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
-int rds_message_copy_from_user(struct rds_message *rm, struct iovec *first_iov,
- size_t total_len);
+int rds_message_copy_from_user(struct net *net, struct rds_message *rm,
+ struct iovec *first_iov, size_t total_len);
struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
__be16 dport, u64 seq);
@@ -679,7 +679,7 @@ int rds_message_add_extension(struct rds_header *hdr,
int rds_message_next_extension(struct rds_header *hdr,
unsigned int *pos, void *buf, unsigned int *buflen);
int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
-int rds_message_inc_copy_to_user(struct rds_incoming *inc,
+int rds_message_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
struct iovec *first_iov, size_t size);
void rds_message_inc_free(struct rds_incoming *inc);
void rds_message_addref(struct rds_message *rm);
@@ -700,15 +700,15 @@ static inline int rds_message_verify_checksum(const struct rds_header *hdr)
/* page.c */
-int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
- gfp_t gfp);
-int rds_page_copy_user(struct page *page, unsigned long offset,
+int rds_page_remainder_alloc(struct net *net, struct scatterlist *scat,
+ unsigned long bytes, gfp_t gfp);
+int rds_page_copy_user(struct net *net, struct page *page, unsigned long offset,
void __user *ptr, unsigned long bytes,
int to_user);
-#define rds_page_copy_to_user(page, offset, ptr, bytes) \
- rds_page_copy_user(page, offset, ptr, bytes, 1)
-#define rds_page_copy_from_user(page, offset, ptr, bytes) \
- rds_page_copy_user(page, offset, ptr, bytes, 0)
+#define rds_page_copy_to_user(net, page, offset, ptr, bytes) \
+ rds_page_copy_user(net, page, offset, ptr, bytes, 1)
+#define rds_page_copy_from_user(net, page, offset, ptr, bytes) \
+ rds_page_copy_user(net, page, offset, ptr, bytes, 0)
void rds_page_exit(void);
/* recv.c */
@@ -769,17 +769,20 @@ static inline void rds_mr_put(struct rds_mr *mr)
}
/* stats.c */
-DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
#define rds_stats_inc_which(which, member) do { \
per_cpu(which, get_cpu()).member++; \
put_cpu(); \
} while (0)
-#define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
-#define rds_stats_add_which(which, member, count) do { \
+#define rds_stats_inc(net, member) \
+ rds_stats_inc_which((*(net->rds.rds_stats)), member)
+
+#define rds_stats_add_which(which, member, count) do { \
per_cpu(which, get_cpu()).member += count; \
put_cpu(); \
} while (0)
-#define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
+#define rds_stats_add(net, member, count) \
+ rds_stats_add_which((*(net->rds.rds_stats)), member, count)
+
int rds_stats_init(void);
void rds_stats_exit(void);
void rds_stats_info_copy(struct rds_info_iterator *iter,
@@ -816,7 +819,8 @@ int rds_trans_register(struct rds_transport *trans);
void rds_trans_unregister(struct rds_transport *trans);
struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
void rds_trans_put(struct rds_transport *trans);
-unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
+unsigned int rds_trans_stats_info_copy(struct net *net,
+ struct rds_info_iterator *iter,
unsigned int avail);
int rds_trans_init(void);
void rds_trans_exit(void);
diff --git a/net/rds/recv.c b/net/rds/recv.c
index ad83230..5df2469 100644
--- a/net/rds/recv.c
+++ b/net/rds/recv.c
@@ -199,20 +199,20 @@ void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
*/
if (be64_to_cpu(inc->i_hdr.h_sequence) < conn->c_next_rx_seq &&
(inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) {
- rds_stats_inc(s_recv_drop_old_seq);
+ rds_stats_inc(net, s_recv_drop_old_seq);
goto out;
}
conn->c_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1;
if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) {
- rds_stats_inc(s_recv_ping);
+ rds_stats_inc(net, s_recv_ping);
rds_send_pong(conn, inc->i_hdr.h_sport);
goto out;
}
rs = rds_find_bound(net, daddr, inc->i_hdr.h_dport);
if (!rs) {
- rds_stats_inc(s_recv_drop_no_sock);
+ rds_stats_inc(net, s_recv_drop_no_sock);
goto out;
}
@@ -226,7 +226,7 @@ void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
write_lock_irqsave(&rs->rs_recv_lock, flags);
if (!sock_flag(sk, SOCK_DEAD)) {
rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);
- rds_stats_inc(s_recv_queued);
+ rds_stats_inc(net, s_recv_queued);
rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,
be32_to_cpu(inc->i_hdr.h_len),
inc->i_hdr.h_dport);
@@ -234,7 +234,7 @@ void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
list_add_tail(&inc->i_item, &rs->rs_recv_queue);
__rds_wake_sk_sleep(sk);
} else {
- rds_stats_inc(s_recv_drop_dead_sock);
+ rds_stats_inc(net, s_recv_drop_dead_sock);
}
write_unlock_irqrestore(&rs->rs_recv_lock, flags);
@@ -401,6 +401,7 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
size_t size, int msg_flags)
{
struct sock *sk = sock->sk;
+ struct net *net = sock_net(sk);
struct rds_sock *rs = rds_sk_to_rs(sk);
long timeo;
int ret = 0, nonblock = msg_flags & MSG_DONTWAIT;
@@ -453,8 +454,8 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
rdsdebug("copying inc %p from %pI4:%u to user\n", inc,
&inc->i_conn->c_faddr,
ntohs(inc->i_hdr.h_sport));
- ret = inc->i_conn->c_trans->inc_copy_to_user(inc, msg->msg_iov,
- size);
+ ret = inc->i_conn->c_trans->inc_copy_to_user(net, inc,
+ msg->msg_iov, size);
if (ret < 0)
break;
@@ -466,7 +467,7 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {
rds_inc_put(inc);
inc = NULL;
- rds_stats_inc(s_recv_deliver_raced);
+ rds_stats_inc(net, s_recv_deliver_raced);
continue;
}
@@ -481,7 +482,7 @@ int rds_recvmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
goto out;
}
- rds_stats_inc(s_recv_delivered);
+ rds_stats_inc(net, s_recv_delivered);
sin = (struct sockaddr_in *)msg->msg_name;
if (sin) {
diff --git a/net/rds/send.c b/net/rds/send.c
index da70396..1286e5d 100644
--- a/net/rds/send.c
+++ b/net/rds/send.c
@@ -134,6 +134,7 @@ static void release_in_xmit(struct rds_connection *conn)
*/
int rds_send_xmit(struct rds_connection *conn)
{
+ struct net *net = rds_conn_to_net(conn);
struct rds_message *rm;
unsigned long flags;
unsigned int tmp;
@@ -151,7 +152,7 @@ restart:
* caches per message.
*/
if (!acquire_in_xmit(conn)) {
- rds_stats_inc(s_send_lock_contention);
+ rds_stats_inc(net, s_send_lock_contention);
ret = -ENOMEM;
goto out;
}
@@ -246,7 +247,7 @@ restart:
conn->c_unacked_packets = rds_sysctl_max_unacked_packets;
conn->c_unacked_bytes = rds_sysctl_max_unacked_bytes;
- rds_stats_inc(s_send_ack_required);
+ rds_stats_inc(net, s_send_ack_required);
} else {
conn->c_unacked_bytes -= len;
conn->c_unacked_packets--;
@@ -384,7 +385,7 @@ restart:
if (ret == 0) {
smp_mb();
if (!list_empty(&conn->c_send_queue)) {
- rds_stats_inc(s_send_lock_queue_raced);
+ rds_stats_inc(net, s_send_lock_queue_raced);
goto restart;
}
}
@@ -394,6 +395,7 @@ out:
static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)
{
+ struct net *net = sock_net(&rs->rs_sk);
u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len);
assert_spin_locked(&rs->rs_lock);
@@ -402,7 +404,7 @@ static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)
rs->rs_snd_bytes -= len;
if (rs->rs_snd_bytes == 0)
- rds_stats_inc(s_send_queue_empty);
+ rds_stats_inc(net, s_send_queue_empty);
}
static inline int rds_send_is_acked(struct rds_message *rm, u64 ack,
@@ -980,7 +982,9 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
ret = -ENOMEM;
goto out;
}
- ret = rds_message_copy_from_user(rm, msg->msg_iov, payload_len);
+
+ ret = rds_message_copy_from_user(net, rm, msg->msg_iov,
+ payload_len);
if (ret)
goto out;
}
@@ -1032,7 +1036,7 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,
dport, &queued)) {
- rds_stats_inc(s_send_queue_full);
+ rds_stats_inc(net, s_send_queue_full);
/* XXX make sure this is reasonable */
if (payload_len > rds_sk_sndbuf(rs)) {
ret = -EMSGSIZE;
@@ -1063,7 +1067,7 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
* By now we've committed to the send. We reuse rds_send_worker()
* to retry sends in the rds thread if the transport asks us to.
*/
- rds_stats_inc(s_send_queued);
+ rds_stats_inc(net, s_send_queued);
if (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags))
rds_send_xmit(conn);
@@ -1120,8 +1124,8 @@ rds_send_pong(struct rds_connection *conn, __be16 dport)
conn->c_next_tx_seq++;
spin_unlock_irqrestore(&conn->c_lock, flags);
- rds_stats_inc(s_send_queued);
- rds_stats_inc(s_send_pong);
+ rds_stats_inc(net, s_send_queued);
+ rds_stats_inc(net, s_send_pong);
if (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags))
rds_send_xmit(conn);
diff --git a/net/rds/stats.c b/net/rds/stats.c
index 7be790d..6505fb9 100644
--- a/net/rds/stats.c
+++ b/net/rds/stats.c
@@ -37,9 +37,6 @@
#include "rds.h"
-DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
-EXPORT_PER_CPU_SYMBOL_GPL(rds_stats);
-
/* :.,$s/unsigned long\>.*\<s_\(.*\);/"\1",/g */
static const char *const rds_stat_names[] = {
@@ -108,6 +105,8 @@ static void rds_stats_info(struct socket *sock, unsigned int len,
struct rds_info_iterator *iter,
struct rds_info_lengths *lens)
{
+ struct net *net = sock_net(sock->sk);
+ struct rds_statistics *rds_stats = net->rds.rds_stats;
struct rds_statistics stats = {0, };
uint64_t *src;
uint64_t *sum;
@@ -123,7 +122,7 @@ static void rds_stats_info(struct socket *sock, unsigned int len,
}
for_each_online_cpu(cpu) {
- src = (uint64_t *)&(per_cpu(rds_stats, cpu));
+ src = (uint64_t *)per_cpu_ptr(rds_stats, cpu);
sum = (uint64_t *)&stats;
for (i = 0; i < sizeof(stats) / sizeof(uint64_t); i++)
*(sum++) += *(src++);
@@ -135,7 +134,7 @@ static void rds_stats_info(struct socket *sock, unsigned int len,
trans:
lens->each = sizeof(struct rds_info_counter);
- lens->nr = rds_trans_stats_info_copy(iter, avail) +
+ lens->nr = rds_trans_stats_info_copy(net, iter, avail) +
ARRAY_SIZE(rds_stat_names);
}
diff --git a/net/rds/tcp.h b/net/rds/tcp.h
index 9cf2927..fee5ef6 100644
--- a/net/rds/tcp.h
+++ b/net/rds/tcp.h
@@ -69,8 +69,8 @@ void rds_tcp_recv_exit(void);
void rds_tcp_data_ready(struct sock *sk, int bytes);
int rds_tcp_recv(struct rds_connection *conn);
void rds_tcp_inc_free(struct rds_incoming *inc);
-int rds_tcp_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
- size_t size);
+int rds_tcp_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *iov, size_t size);
/* tcp_send.c */
void rds_tcp_xmit_prepare(struct rds_connection *conn);
diff --git a/net/rds/tcp_recv.c b/net/rds/tcp_recv.c
index 7ea92b6..641ab5d 100644
--- a/net/rds/tcp_recv.c
+++ b/net/rds/tcp_recv.c
@@ -42,6 +42,7 @@ static struct kmem_cache *rds_tcp_incoming_slab;
static void rds_tcp_inc_purge(struct rds_incoming *inc)
{
struct rds_tcp_incoming *tinc;
+
tinc = container_of(inc, struct rds_tcp_incoming, ti_inc);
rdsdebug("purging tinc %p inc %p\n", tinc, inc);
skb_queue_purge(&tinc->ti_skb_list);
@@ -50,6 +51,7 @@ static void rds_tcp_inc_purge(struct rds_incoming *inc)
void rds_tcp_inc_free(struct rds_incoming *inc)
{
struct rds_tcp_incoming *tinc;
+
tinc = container_of(inc, struct rds_tcp_incoming, ti_inc);
rds_tcp_inc_purge(inc);
rdsdebug("freeing tinc %p inc %p\n", tinc, inc);
@@ -59,8 +61,8 @@ void rds_tcp_inc_free(struct rds_incoming *inc)
/*
* this is pretty lame, but, whatever.
*/
-int rds_tcp_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
- size_t size)
+int rds_tcp_inc_copy_to_user(struct net *net, struct rds_incoming *inc,
+ struct iovec *first_iov, size_t size)
{
struct rds_tcp_incoming *tinc;
struct iovec *iov, tmp;
@@ -98,7 +100,7 @@ int rds_tcp_inc_copy_to_user(struct rds_incoming *inc, struct iovec *first_iov,
goto out;
}
- rds_stats_add(s_copy_to_user, to_copy);
+ rds_stats_add(net, s_copy_to_user, to_copy);
size -= to_copy;
ret += to_copy;
skb_off += to_copy;
diff --git a/net/rds/threads.c b/net/rds/threads.c
index 65eaefc..ca4c0f9 100644
--- a/net/rds/threads.c
+++ b/net/rds/threads.c
@@ -159,19 +159,22 @@ void rds_connect_worker(struct work_struct *work)
void rds_send_worker(struct work_struct *work)
{
- struct rds_connection *conn = container_of(work, struct rds_connection, c_send_w.work);
+ struct rds_connection *conn = container_of(work, struct rds_connection,
+ c_send_w.work);
int ret;
if (rds_conn_state(conn) == RDS_CONN_UP) {
+ struct net *net = rds_conn_to_net(conn);
+
ret = rds_send_xmit(conn);
rdsdebug("conn %p ret %d\n", conn, ret);
switch (ret) {
case -EAGAIN:
- rds_stats_inc(s_send_immediate_retry);
+ rds_stats_inc(net, s_send_immediate_retry);
queue_delayed_work(rds_wq, &conn->c_send_w, 0);
break;
case -ENOMEM:
- rds_stats_inc(s_send_delayed_retry);
+ rds_stats_inc(net, s_send_delayed_retry);
queue_delayed_work(rds_wq, &conn->c_send_w, 2);
default:
break;
@@ -181,19 +184,22 @@ void rds_send_worker(struct work_struct *work)
void rds_recv_worker(struct work_struct *work)
{
- struct rds_connection *conn = container_of(work, struct rds_connection, c_recv_w.work);
+ struct rds_connection *conn = container_of(work, struct rds_connection,
+ c_recv_w.work);
int ret;
if (rds_conn_state(conn) == RDS_CONN_UP) {
+ struct net *net = rds_conn_to_net(conn);
+
ret = conn->c_trans->recv(conn);
rdsdebug("conn %p ret %d\n", conn, ret);
switch (ret) {
case -EAGAIN:
- rds_stats_inc(s_recv_immediate_retry);
+ rds_stats_inc(net, s_recv_immediate_retry);
queue_delayed_work(rds_wq, &conn->c_recv_w, 0);
break;
case -ENOMEM:
- rds_stats_inc(s_recv_delayed_retry);
+ rds_stats_inc(net, s_recv_delayed_retry);
queue_delayed_work(rds_wq, &conn->c_recv_w, 2);
default:
break;
@@ -203,7 +209,8 @@ void rds_recv_worker(struct work_struct *work)
void rds_shutdown_worker(struct work_struct *work)
{
- struct rds_connection *conn = container_of(work, struct rds_connection, c_down_w);
+ struct rds_connection *conn = container_of(work, struct rds_connection,
+ c_down_w);
rds_conn_shutdown(conn);
}
diff --git a/net/rds/transport.c b/net/rds/transport.c
index a5b187b..28dd771 100644
--- a/net/rds/transport.c
+++ b/net/rds/transport.c
@@ -107,7 +107,8 @@ struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr)
* caller passes in the global stats so that we can size and copy while
* holding the lock.
*/
-unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
+unsigned int rds_trans_stats_info_copy(struct net *net,
+ struct rds_info_iterator *iter,
unsigned int avail)
{
--
1.7.4.1
More information about the rds-devel
mailing list