[DTrace-devel] [PATCH v2 2/7] libdtrace: share vmlinux BTF/CTF globally to support faster startup

[Alan Maguire]

In a multiple DTrace session per process environment, it makes sense
to avoid doing expensive operations like reading in vmlinux BTF and
converting it to CTF more than once. Shared (vmlinux) BTF and CTF
generated from it are shared and reference-counted globally here
to cut down on cost.  With this change, we can launch ~1 DTrace
session per second.

Signed-off-by: Alan Maguire <alan.maguire at oracle.com>
---
 libdtrace/dt_btf.c  | 57 ++++++++++++++++++++++++++++++++++++++++++---
 libdtrace/dt_btf.h  |  1 +
 libdtrace/dt_open.c |  5 +---
 3 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/libdtrace/dt_btf.c b/libdtrace/dt_btf.c
index 1f793397..0730a531 100644
--- a/libdtrace/dt_btf.c
+++ b/libdtrace/dt_btf.c
@@ -693,6 +693,9 @@ dt_btf_to_ctf(dtrace_hdl_t *dtp, dt_module_t *dmp, dt_btf_t *btf)
 	ctf_dict_t	*ctf;
 	ctf_encoding_t	enc = { CTF_INT_SIGNED, 0, 0 };
 
+	if (dmp == NULL && dtp->dt_shared_ctf)
+		return dtp->dt_shared_ctf;
+
 	ctf = ctf_create(&dtp->dt_ctferr);
 	if (ctf == NULL)
 		return NULL;
@@ -764,6 +767,52 @@ out:
 }
 #endif
 
+static dt_btf_t *dt_shared_btf = NULL;
+static ctf_dict_t *dt_shared_ctf = NULL;
+static pthread_mutex_t dt_shared_mutex = PTHREAD_MUTEX_INITIALIZER;
+static int dt_shared_refcnt = 0;
+
+static int
+dt_shared_get(dtrace_hdl_t *dtp, dt_module_t *dmp)
+{
+	pthread_mutex_lock(&dt_shared_mutex);
+	if (!dt_shared_btf)
+		dt_shared_btf = dt_btf_load_file(dtp, "/sys/kernel/btf/vmlinux");
+	if (dt_shared_btf) {
+		dt_shared_refcnt++;
+		dtp->dt_shared_btf = dmp->dm_btf = dt_shared_btf;
+		if (!dt_shared_ctf) {
+			ctf_dict_t *ctf = dt_btf_to_ctf(dtp, NULL, dt_shared_btf);
+
+			if (ctf)
+				dtp->dt_shared_ctf = dt_shared_ctf = dmp->dm_ctfp = ctf;
+		} else {
+			dtp->dt_shared_ctf = dmp->dm_ctfp = dt_shared_ctf;
+		}
+	}
+	pthread_mutex_unlock(&dt_shared_mutex);
+	if (!dt_shared_btf)
+		return -1;
+	return 0;
+}
+
+void
+dt_shared_put(dtrace_hdl_t *dtp)
+{
+	pthread_mutex_lock(&dt_shared_mutex);
+	if (dtp->dt_shared_btf && dtp->dt_shared_btf == dt_shared_btf) {
+		if (--dt_shared_refcnt == 0) {
+			dt_btf_destroy(dtp, dtp->dt_shared_btf);
+			dtp->dt_shared_btf = dt_shared_btf = NULL;
+		}
+	}
+	if (dt_shared_refcnt == 0) {
+		ctf_close(dtp->dt_shared_ctf);
+		dtp->dt_shared_ctf = dt_shared_ctf = NULL;
+	}
+	pthread_mutex_unlock(&dt_shared_mutex);
+}
+
 dt_btf_t *
 dt_btf_load_module(dtrace_hdl_t *dtp, dt_module_t *dmp)
 {
@@ -774,6 +823,11 @@ dt_btf_load_module(dtrace_hdl_t *dtp, dt_module_t *dmp)
 	if (dmp->dm_btf)
 		return dmp->dm_btf;
 
+	if (strcmp(dmp->dm_name, "vmlinux") == 0) {
+		if (dt_shared_get(dtp, dmp) == 0)
+			return dmp->dm_btf;
+	}
+
 	/*
 	 * Default: /sys/kernel/btf/<module>
 	 * If "none", disable BTF.
@@ -792,9 +846,6 @@ dt_btf_load_module(dtrace_hdl_t *dtp, dt_module_t *dmp)
 	btf = dt_btf_load_file(dtp, fn);
 	free(fn);
 
-	if (btf && !dtp->dt_shared_btf && strcmp(dmp->dm_name, "vmlinux") == 0)
-		dtp->dt_shared_btf = btf;
-
 	return dmp->dm_btf = btf;
 }
 
diff --git a/libdtrace/dt_btf.h b/libdtrace/dt_btf.h
index 2c921a03..b1ed2b65 100644
--- a/libdtrace/dt_btf.h
+++ b/libdtrace/dt_btf.h
@@ -18,6 +18,7 @@ typedef struct dt_btf		dt_btf_t;
 typedef struct bpf_btf_info	btf_info_t;
 
 extern void dt_btf_destroy(dtrace_hdl_t *, dt_btf_t *);
+extern void dt_shared_put(dtrace_hdl_t *);
 extern dt_btf_t *dt_btf_load_module(dtrace_hdl_t *, dt_module_t *);
 extern ctf_dict_t *dt_btf_module_ctf(dtrace_hdl_t *, dt_module_t *);
 extern const char *dt_btf_get_string(dtrace_hdl_t *, const dt_btf_t *,
diff --git a/libdtrace/dt_open.c b/libdtrace/dt_open.c
index 30b8758d..352cec49 100644
--- a/libdtrace/dt_open.c
+++ b/libdtrace/dt_open.c
@@ -1299,10 +1299,7 @@ dtrace_close(dtrace_hdl_t *dtp)
 	dt_htab_destroy(dtp->dt_mods);
 	dt_htab_destroy(dtp->dt_kernpaths);
 
-	if (dtp->dt_shared_btf != NULL)
-		dt_btf_destroy(dtp, dtp->dt_shared_btf);
-	if (dtp->dt_shared_ctf != NULL)
-		ctf_close(dtp->dt_shared_ctf);
+	dt_shared_put(dtp);
 	if (dtp->dt_ctfa != NULL)
 		ctf_arc_close(dtp->dt_ctfa);
 
-- 
2.43.5