[rds-devel] [PATCH] RDS/IB: Fix memory leak on the HCA driver unload.

[Vladimir Sokolovsky]

RDS/IB: Fix memory leak on the HCA driver unload.

While removing HCA driver ib_dealloc_pd can be invoked before all existing
connection (QPs) closed and then it will fail on reference counter.

Added list of connections per HCA device. All existing connections
will be shutted down on device removal before deallocating the PD.

Signed-off-by: Vladimir Sokolovsky <vlad at mellanox.co.il>
---
diff --git a/net/rds/ib.c b/net/rds/ib.c
index fbe0e65..a3e1ef7 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -112,6 +112,7 @@ void rds_ib_add_one(struct ib_device *device)
         }

         INIT_LIST_HEAD(&rds_ibdev->ipaddr_list);
+       INIT_LIST_HEAD(&rds_ibdev->conn_list);
         list_add_tail(&rds_ibdev->list, &rds_ib_devices);

         ib_set_client_data(device, &rds_ib_client, rds_ibdev);
@@ -131,20 +132,33 @@ free_attr:
  void rds_ib_remove_one(struct ib_device *device)
  {
         struct rds_ib_device *rds_ibdev;
-       struct rds_ib_ipaddr *i_ipaddr, *next;
+       struct rds_ib_ipaddr *i_ipaddr, *i_next;
+       struct rds_ib_devconn *i_conn, *c_next;

         rds_ibdev = ib_get_client_data(device, &rds_ib_client);

-       list_for_each_entry_safe(i_ipaddr, next, &rds_ibdev->ipaddr_list, list) {
+       list_for_each_entry_safe(i_ipaddr, i_next, &rds_ibdev->ipaddr_list, list) {
                 list_del(&i_ipaddr->list);
                 kfree(i_ipaddr);
         }

+       list_for_each_entry_safe(i_conn, c_next, &rds_ibdev->conn_list, list) {
+               if (rds_conn_up(i_conn->conn)) {
+                       rds_conn_drop(i_conn->conn);
+               }
+               list_del(&i_conn->list);
+               kfree(i_conn);
+       }
+
         if (rds_ibdev->fmr_pool)
                 ib_destroy_fmr_pool(rds_ibdev->fmr_pool);

         ib_dereg_mr(rds_ibdev->mr);
-       ib_dealloc_pd(rds_ibdev->pd);
+
+       while (ib_dealloc_pd(rds_ibdev->pd)) {
+               rdsdebug("%s-%d Failed to dealloc pd %p\n", __func__, __LINE__, rds_ibdev->pd);
+               msleep(1);
+       }

         list_del(&rds_ibdev->list);
         kfree(rds_ibdev);
diff --git a/net/rds/ib.h b/net/rds/ib.h
index e73a1e2..a7528cb 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -113,9 +113,15 @@ struct rds_ib_ipaddr {
         __be32                  ipaddr;
  };

+struct rds_ib_devconn {
+       struct list_head        list;
+       struct rds_connection   *conn;
+};
+
  struct rds_ib_device {
         struct list_head        list;
         struct list_head        ipaddr_list;
+       struct list_head        conn_list;
         struct ib_device        *dev;
         struct ib_pd            *pd;
         struct ib_mr            *mr;
@@ -174,6 +180,7 @@ void __rds_ib_conn_error(struct rds_connection *conn, const char *, ...);

  /* ib_rdma.c */
  int ib_update_ipaddr_for_device(struct rds_ib_device *rds_ibdev, __be32 ipaddr);
+int ib_update_conn_for_device(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
  void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
                     __be32 ip_addr, u64 *key_ret, u64 *phyaddr);
  void rds_ib_free_mr(void *trans_private, int invalidate,
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 5e62f3a..0fe8e0e 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -66,6 +66,7 @@ static void rds_ib_connect_complete(struct rds_connection *conn)
         /* update ib_device with this local ipaddr */
         rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
         ib_update_ipaddr_for_device(rds_ibdev, conn->c_laddr);
+       ib_update_conn_for_device(rds_ibdev, conn);

         rds_connect_complete(conn);
  }
@@ -320,6 +321,7 @@ static int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
         /* update ib_device with this local ipaddr */
         rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
         ib_update_ipaddr_for_device(rds_ibdev, dp->dp_saddr);
+       ib_update_conn_for_device(rds_ibdev, conn);

         return 0;

diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index 0adb686..cad3470 100644
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -74,6 +74,32 @@ int ib_update_ipaddr_for_device(struct rds_ib_device *rds_ibdev, __be32 ipaddr)
         return 0;
  }

+int ib_update_conn_for_device(struct rds_ib_device *rds_ibdev, struct rds_connection *conn)
+{
+       struct rds_ib_devconn *i_conn;
+
+       spin_lock_irq(&rds_ibdev->spinlock);
+       list_for_each_entry(i_conn, &rds_ibdev->conn_list, list) {
+               if (i_conn->conn == conn) {
+                       spin_unlock_irq(&rds_ibdev->spinlock);
+                       return 0;
+               }
+       }
+       spin_unlock_irq(&rds_ibdev->spinlock);
+
+       i_conn = kmalloc(sizeof *i_conn, GFP_KERNEL);
+       if (!i_conn)
+               return -ENOMEM;
+
+       i_conn->conn = conn;
+
+       spin_lock_irq(&rds_ibdev->spinlock);
+       list_add_tail(&i_conn->list, &rds_ibdev->conn_list);
+       spin_unlock_irq(&rds_ibdev->spinlock);
+
+       return 0;
+}
+
  struct rds_ib_device* ib_get_device(__be32 ipaddr)
  {
         struct rds_ib_device *rds_ibdev;