[rds-devel] rds-rdma sample program help

Jon Mason jon at opengridcomputing.com
Thu Aug 14 12:43:38 PDT 2008


I am in the process of writing a simple RDS RDMA test program to help me
debug the RDS RDMA fastreg problems I am seeing.  I have the basic RDS
support working, but I am not able to get rds-rdma (over IB) to work.

The test program is able to send a message from the client to the server
with cookie, address, etc.  The server sends back a message to perform a
RDMA READ, but the client never recvs the message and the server never
receives a response saying the RDMA occurred (and the data is never
transferred).

Any insight would be appreciated.  I have included the program source
below.  

Thanks,
Jon


/* a Simple RDS Test Program 
 * Created on 08/08/08
 * by Jon Mason
 */
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

/* FIXME - this is a hack to getaround RDS not exporting any header files.
 * This is a local copy.
 */
#include "ib_rds.h"
/* These are defined in rds.h....but that file is not happily included */
#define SOL_RDS		272
#define PF_RDS		28


#define TESTPORT	4000
#define BUFSIZE		94

static int do_rdma_read(int sock, struct msghdr *msg, void *buf)
{
	struct rds_rdma_args *args;
	struct rds_iovec iov;
	struct cmsghdr *cmsg;
	int rc;

	cmsg = CMSG_FIRSTHDR(msg);
	args = (struct rds_rdma_args *)CMSG_DATA(cmsg);

	printf("remote cookie = %lx\n", args->cookie);

	/* do a sendmsg call to do the RDMA */
	/* Just fill in our bits, as the rds_rdma_args from the rmeote side already has its bits in. */
	cmsg->cmsg_level = SOL_RDS;
	cmsg->cmsg_type = RDS_CMSG_RDMA_ARGS;
	cmsg->cmsg_len = CMSG_LEN(sizeof(struct rds_rdma_args));

	iov.addr = (uint64_t) buf;
	iov.bytes = BUFSIZE * sizeof(char);

	args->local_vec_addr = (uint64_t) &iov;
	args->nr_local = 1;
	args->flags = RDS_RDMA_NOTIFY_ME;
	args->user_token = 0;

	msg->msg_controllen = CMSG_SPACE(sizeof(struct rds_rdma_args));

	rc = sendmsg(sock, msg, 0);
	if (rc < 0) {
		printf("%s: Error sending message: %d %d\n", __func__, rc, errno);
		return -1;
	}

	printf("%s: sendmsg successful to %s\n", __func__, inet_ntoa(((struct sockaddr_in *)msg->msg_name)->sin_addr));

	rc = recvmsg(sock, msg, 0);
	if (rc < 0) {
		printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
		return -1;
	}

	printf("%s: recvmsg successful\n", __func__);

	return 0;
}

static void server(char *address)
{
	struct sockaddr_in sin, din;
	char ctlbuf[1024];
	struct msghdr msg;
	struct iovec *iov;
	int rc, sock;
	void *buf;

	buf = calloc(BUFSIZE, sizeof(char));
	if (!buf) {
		printf("%s: calloc failed\n", __func__);
		return;
	}

	sock = socket(PF_RDS, SOCK_SEQPACKET, 0);
	if (sock < 0) {
		printf("%s: Error creating Socket: %d\n", __func__, sock);
		goto out;
	}

	memset(&sin, 0, sizeof(sin));
	sin.sin_family = AF_INET;
	sin.sin_addr.s_addr = inet_addr(address);
	sin.sin_port = TESTPORT;

	rc = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
	if (rc < 0) {
		printf("%s: Error binding to address: %d %d\n", __func__, rc, errno);
		goto out;
	}

	/* The recv iov could contain a regular RDS packet or an RDMA RDS
	 * packet, so set it up for the worst case for both.
	 */
	iov = calloc(1, sizeof(struct iovec));
	if (!iov) {
		printf("%s: calloc failed\n", __func__);
		goto out;
	}

	iov[0].iov_base = buf;
	iov[0].iov_len = BUFSIZE * sizeof(char);

	memset(&msg, 0, sizeof(msg));
	msg.msg_name = &din;
	msg.msg_namelen = sizeof(din);
	msg.msg_iov = iov;
	msg.msg_iovlen = 1;
	msg.msg_control = &ctlbuf;
	msg.msg_controllen = CMSG_SPACE(sizeof(struct rds_rdma_args));

	printf("server listening on %s\n", inet_ntoa(sin.sin_addr));

	rc = recvmsg(sock, &msg, 0);
	if (rc < 0) {
		printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
		goto out1;
	}

	printf("%s received a packet from %s of len %d, cmsg len %d, on port %d\n",
	       inet_ntoa(sin.sin_addr),
	       inet_ntoa(din.sin_addr),
	       (uint32_t) iov[0].iov_len,
	       (uint32_t) msg.msg_controllen,
	       din.sin_port);

	if (msg.msg_controllen) {
		rc = do_rdma_read(sock, &msg, buf);
		if (rc < 0)
			goto out1;
	}
	printf("payload contains:  %s\n", (char *)buf);

out1:
	free(iov);
out:
	free(buf);
}

static void create_message(char *buf)
{
	int i;

	for (i = 0; i < BUFSIZE; i++)
		buf[i] = i + 0x21;
}

static int build_rds_rdma_packet(int sock, struct msghdr *msg, char *buf, uint64_t *cookie)
{
	struct rds_get_mr_args mr_args;
	struct cmsghdr *cmsg;
	void *ctlbuf;

	*cookie = 0;

	mr_args.vec.addr = (uint64_t) buf;
	mr_args.vec.bytes = BUFSIZE * sizeof(char);
	mr_args.cookie_addr = (uint64_t) cookie;
	mr_args.flags = RDS_RDMA_READWRITE;

	ctlbuf = calloc(1, CMSG_SPACE(sizeof(mr_args)));
	if (!ctlbuf) {
		printf("%s: calloc failed\n", __func__);
		return -1;
	}

	msg->msg_control = ctlbuf;
	msg->msg_controllen = CMSG_SPACE(sizeof(mr_args));

	cmsg = CMSG_FIRSTHDR(msg);
	cmsg->cmsg_level = SOL_RDS;
	cmsg->cmsg_type = RDS_CMSG_RDMA_MAP;
	cmsg->cmsg_len = CMSG_LEN(sizeof(mr_args));
	memcpy(CMSG_DATA(cmsg), &mr_args, sizeof(mr_args));

	msg->msg_iov = NULL;
	msg->msg_iovlen = 0;

	return 0;
}

static int build_rds_packet(struct msghdr *msg, char *buf)
{
	struct iovec *iov;

	iov = calloc(1, sizeof(struct iovec));
	if (!iov) {
		printf("%s: calloc failed\n", __func__);
		return -1;
	}

	msg->msg_iov = iov;
	msg->msg_iovlen = 1;

	iov[0].iov_base = buf;
	iov[0].iov_len = BUFSIZE * sizeof(char);

	return 0;
}

static void client(char *localaddr, char *remoteaddr, int rdma)
{
	struct sockaddr_in sin, din;
	struct msghdr msg;
	uint64_t cookie;
	int rc, sock;
	char *buf;

	buf = calloc(BUFSIZE, sizeof(char));
	if (!buf) {
		printf("%s: calloc failed\n", __func__);
		return;
	}

	create_message(buf);

	sock = socket(PF_RDS, SOCK_SEQPACKET, 0);
	if (sock < 0) {
		printf("%s: Error creating Socket: %d\n", __func__, sock);
		goto out;
	}

	memset(&sin, 0, sizeof(sin));
	sin.sin_family = AF_INET;
	sin.sin_addr.s_addr = inet_addr(localaddr);
	sin.sin_port = TESTPORT; //HACK

	rc = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
	if (rc < 0) {
		printf("%s: Error binding to address: %d %d\n", __func__, rc, errno);
		goto out;
	}

	memset(&msg, 0, sizeof(msg));
	msg.msg_name = &din;
	msg.msg_namelen = sizeof(din);

	memset(&din, 0, sizeof(din));
	din.sin_family = AF_INET;
	din.sin_addr.s_addr = inet_addr(remoteaddr);
	din.sin_port = TESTPORT;

	if (rdma) {
		rc = build_rds_rdma_packet(sock, &msg, buf, &cookie);
		if (rc < 0)
			goto out;

		printf("Client Sending RDMA message from %s to %s on port %d\n",
		       localaddr, remoteaddr, sin.sin_port);
	} else {
		rc = build_rds_packet(&msg, buf);
		if (rc < 0)
			goto out;

		printf("client sending %d byte message %s from %s to %s on port %d\n",
		       (uint32_t) msg.msg_iov->iov_len, buf, localaddr, remoteaddr, sin.sin_port);
	}

	rc = sendmsg(sock, &msg, 0);
	if (rc < 0) {
		printf("%s: Error sending message: %d %d\n", __func__, rc, errno);
		goto out1;
	}

	if (rdma) {
		printf("Cookie = %lx\n", cookie);

		/* reuse the same msg, as it should no longer be necessary and this incoming msg should be empty */
		rc = recvmsg(sock, &msg, 0);
		if (rc < 0) {
			printf("%s: Error receiving message: %d %d\n", __func__, rc, errno);
		}
	}

out1:
	if (msg.msg_control)
		free(msg.msg_control);
	if (msg.msg_iov)
		free(msg.msg_iov);
out:
	free(buf);
}

int main(int argc, char **argv)
{
	char *serveraddr = NULL, *clientaddr = NULL;
	int i, rdma = 0;

	if (argc < 3) {
		printf("not enough args\n");
		return -1;
	}

	for (i = 1; i < argc; i++) {
		if (!strcmp("-s", argv[i])) {
			serveraddr = argv[i+1];
			i++;
		} else if (!strcmp("-c", argv[i])) {
			clientaddr = argv[i+1];
			i++;
		} else if (!strcmp("--rdma", argv[i])) {
			rdma = 1;
		} else
			printf("Invalid param\n");
	}

	if (serveraddr && !clientaddr) {
		server(serveraddr);
	} else if (serveraddr && clientaddr) {
		client(clientaddr, serveraddr, rdma);
	}

	return 0;
}



More information about the rds-devel mailing list