[rds-devel] rds-rdma sample program help
Jon Mason
jon at opengridcomputing.com
Thu Aug 14 12:43:38 PDT 2008
I am in the process of writing a simple RDS RDMA test program to help me
debug the RDS RDMA fastreg problems I am seeing. I have the basic RDS
support working, but I am not able to get rds-rdma (over IB) to work.
The test program is able to send a message from the client to the server
with cookie, address, etc. The server sends back a message to perform a
RDMA READ, but the client never recvs the message and the server never
receives a response saying the RDMA occurred (and the data is never
transferred).
Any insight would be appreciated. I have included the program source
below.
Thanks,
Jon
/* a Simple RDS Test Program
* Created on 08/08/08
* by Jon Mason
*/
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/* FIXME - this is a hack to getaround RDS not exporting any header files.
* This is a local copy.
*/
#include "ib_rds.h"
/* These are defined in rds.h....but that file is not happily included */
#define SOL_RDS 272
#define PF_RDS 28
#define TESTPORT 4000
#define BUFSIZE 94
static int do_rdma_read(int sock, struct msghdr *msg, void *buf)
{
struct rds_rdma_args *args;
struct rds_iovec iov;
struct cmsghdr *cmsg;
int rc;
cmsg = CMSG_FIRSTHDR(msg);
args = (struct rds_rdma_args *)CMSG_DATA(cmsg);
printf("remote cookie = %lx\n", args->cookie);
/* do a sendmsg call to do the RDMA */
/* Just fill in our bits, as the rds_rdma_args from the rmeote side already has its bits in. */
cmsg->cmsg_level = SOL_RDS;
cmsg->cmsg_type = RDS_CMSG_RDMA_ARGS;
cmsg->cmsg_len = CMSG_LEN(sizeof(struct rds_rdma_args));
iov.addr = (uint64_t) buf;
iov.bytes = BUFSIZE * sizeof(char);
args->local_vec_addr = (uint64_t) &iov;
args->nr_local = 1;
args->flags = RDS_RDMA_NOTIFY_ME;
args->user_token = 0;
msg->msg_controllen = CMSG_SPACE(sizeof(struct rds_rdma_args));
rc = sendmsg(sock, msg, 0);
if (rc < 0) {
printf("%s: Error sending message: %d %d\n", __func__, rc, errno);
return -1;
}
printf("%s: sendmsg successful to %s\n", __func__, inet_ntoa(((struct sockaddr_in *)msg->msg_name)->sin_addr));
rc = recvmsg(sock, msg, 0);
if (rc < 0) {
printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
return -1;
}
printf("%s: recvmsg successful\n", __func__);
return 0;
}
static void server(char *address)
{
struct sockaddr_in sin, din;
char ctlbuf[1024];
struct msghdr msg;
struct iovec *iov;
int rc, sock;
void *buf;
buf = calloc(BUFSIZE, sizeof(char));
if (!buf) {
printf("%s: calloc failed\n", __func__);
return;
}
sock = socket(PF_RDS, SOCK_SEQPACKET, 0);
if (sock < 0) {
printf("%s: Error creating Socket: %d\n", __func__, sock);
goto out;
}
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = inet_addr(address);
sin.sin_port = TESTPORT;
rc = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
if (rc < 0) {
printf("%s: Error binding to address: %d %d\n", __func__, rc, errno);
goto out;
}
/* The recv iov could contain a regular RDS packet or an RDMA RDS
* packet, so set it up for the worst case for both.
*/
iov = calloc(1, sizeof(struct iovec));
if (!iov) {
printf("%s: calloc failed\n", __func__);
goto out;
}
iov[0].iov_base = buf;
iov[0].iov_len = BUFSIZE * sizeof(char);
memset(&msg, 0, sizeof(msg));
msg.msg_name = &din;
msg.msg_namelen = sizeof(din);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
msg.msg_control = &ctlbuf;
msg.msg_controllen = CMSG_SPACE(sizeof(struct rds_rdma_args));
printf("server listening on %s\n", inet_ntoa(sin.sin_addr));
rc = recvmsg(sock, &msg, 0);
if (rc < 0) {
printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
goto out1;
}
printf("%s received a packet from %s of len %d, cmsg len %d, on port %d\n",
inet_ntoa(sin.sin_addr),
inet_ntoa(din.sin_addr),
(uint32_t) iov[0].iov_len,
(uint32_t) msg.msg_controllen,
din.sin_port);
if (msg.msg_controllen) {
rc = do_rdma_read(sock, &msg, buf);
if (rc < 0)
goto out1;
}
printf("payload contains: %s\n", (char *)buf);
out1:
free(iov);
out:
free(buf);
}
static void create_message(char *buf)
{
int i;
for (i = 0; i < BUFSIZE; i++)
buf[i] = i + 0x21;
}
static int build_rds_rdma_packet(int sock, struct msghdr *msg, char *buf, uint64_t *cookie)
{
struct rds_get_mr_args mr_args;
struct cmsghdr *cmsg;
void *ctlbuf;
*cookie = 0;
mr_args.vec.addr = (uint64_t) buf;
mr_args.vec.bytes = BUFSIZE * sizeof(char);
mr_args.cookie_addr = (uint64_t) cookie;
mr_args.flags = RDS_RDMA_READWRITE;
ctlbuf = calloc(1, CMSG_SPACE(sizeof(mr_args)));
if (!ctlbuf) {
printf("%s: calloc failed\n", __func__);
return -1;
}
msg->msg_control = ctlbuf;
msg->msg_controllen = CMSG_SPACE(sizeof(mr_args));
cmsg = CMSG_FIRSTHDR(msg);
cmsg->cmsg_level = SOL_RDS;
cmsg->cmsg_type = RDS_CMSG_RDMA_MAP;
cmsg->cmsg_len = CMSG_LEN(sizeof(mr_args));
memcpy(CMSG_DATA(cmsg), &mr_args, sizeof(mr_args));
msg->msg_iov = NULL;
msg->msg_iovlen = 0;
return 0;
}
static int build_rds_packet(struct msghdr *msg, char *buf)
{
struct iovec *iov;
iov = calloc(1, sizeof(struct iovec));
if (!iov) {
printf("%s: calloc failed\n", __func__);
return -1;
}
msg->msg_iov = iov;
msg->msg_iovlen = 1;
iov[0].iov_base = buf;
iov[0].iov_len = BUFSIZE * sizeof(char);
return 0;
}
static void client(char *localaddr, char *remoteaddr, int rdma)
{
struct sockaddr_in sin, din;
struct msghdr msg;
uint64_t cookie;
int rc, sock;
char *buf;
buf = calloc(BUFSIZE, sizeof(char));
if (!buf) {
printf("%s: calloc failed\n", __func__);
return;
}
create_message(buf);
sock = socket(PF_RDS, SOCK_SEQPACKET, 0);
if (sock < 0) {
printf("%s: Error creating Socket: %d\n", __func__, sock);
goto out;
}
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = inet_addr(localaddr);
sin.sin_port = TESTPORT; //HACK
rc = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
if (rc < 0) {
printf("%s: Error binding to address: %d %d\n", __func__, rc, errno);
goto out;
}
memset(&msg, 0, sizeof(msg));
msg.msg_name = &din;
msg.msg_namelen = sizeof(din);
memset(&din, 0, sizeof(din));
din.sin_family = AF_INET;
din.sin_addr.s_addr = inet_addr(remoteaddr);
din.sin_port = TESTPORT;
if (rdma) {
rc = build_rds_rdma_packet(sock, &msg, buf, &cookie);
if (rc < 0)
goto out;
printf("Client Sending RDMA message from %s to %s on port %d\n",
localaddr, remoteaddr, sin.sin_port);
} else {
rc = build_rds_packet(&msg, buf);
if (rc < 0)
goto out;
printf("client sending %d byte message %s from %s to %s on port %d\n",
(uint32_t) msg.msg_iov->iov_len, buf, localaddr, remoteaddr, sin.sin_port);
}
rc = sendmsg(sock, &msg, 0);
if (rc < 0) {
printf("%s: Error sending message: %d %d\n", __func__, rc, errno);
goto out1;
}
if (rdma) {
printf("Cookie = %lx\n", cookie);
/* reuse the same msg, as it should no longer be necessary and this incoming msg should be empty */
rc = recvmsg(sock, &msg, 0);
if (rc < 0) {
printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
}
}
out1:
if (msg.msg_control)
free(msg.msg_control);
if (msg.msg_iov)
free(msg.msg_iov);
out:
free(buf);
}
int main(int argc, char **argv)
{
char *serveraddr = NULL, *clientaddr = NULL;
int i, rdma = 0;
if (argc < 3) {
printf("not enough args\n");
return -1;
}
for (i = 1; i < argc; i++) {
if (!strcmp("-s", argv[i])) {
serveraddr = argv[i+1];
i++;
} else if (!strcmp("-c", argv[i])) {
clientaddr = argv[i+1];
i++;
} else if (!strcmp("--rdma", argv[i])) {
rdma = 1;
} else
printf("Invalid param\n");
}
if (serveraddr && !clientaddr) {
server(serveraddr);
} else if (serveraddr && clientaddr) {
client(clientaddr, serveraddr, rdma);
}
return 0;
}
More information about the rds-devel
mailing list