[DTrace-devel] [PATCH v2 10/61] Locked-memory limit
eugene.loh at oracle.com
eugene.loh at oracle.com
Thu Jul 28 19:35:23 UTC 2022
From: Eugene Loh <eugene.loh at oracle.com>
Locked memory is needed for BPF operations such as creating maps and
loading programs. Therefore, dt_vopen() sets RLIMIT_MEMLOCK to 32 Mbytes,
an amount that seems sufficient for many tests in our suite. On the other
hand, the value is somewhat arbitrary, excessive for many D scripts yet
insufficient for others, including some of our tests. Further, it is
silently modifying a resource limit.
Remove dt_vopen()'s silent modification of the locked-memory limit.
Explicitly set "ulimit -l" in runtest.sh to accommodate the tests in our
suite.
While users can similarly set "ulimit -l" explicitly, it would be more
convenient if there were alternative mechanisms, especially when using
"sudo dtrace". Therefore, add a D option lockmem. Notice that ulimit
allows the value "unlimited" for users who simply want to ignore any limit.
Add some more verbose error messages that explain that encountering
EPERM during BPF map creation or BPF program load may be solved by
adjusting the locked-memory limit.
With UEKR7, the locked-memory limit seems to be ignored for root.
Signed-off-by: Eugene Loh <eugene.loh at oracle.com>
---
include/dtrace/options_defines.h | 3 +-
libdtrace/dt_bpf.c | 35 ++++++++++++++++++-----
libdtrace/dt_open.c | 13 ---------
libdtrace/dt_options.c | 15 ++++++++++
libdtrace/dt_work.c | 10 +++++++
runtest.sh | 4 +++
test/unittest/misc/tst.lockmem-cmdline.r | 12 ++++++++
test/unittest/misc/tst.lockmem-cmdline.sh | 18 ++++++++++++
test/unittest/misc/tst.lockmem-cmdline.x | 16 +++++++++++
test/unittest/misc/tst.lockmem-envvar.r | 20 +++++++++++++
test/unittest/misc/tst.lockmem-envvar.sh | 18 ++++++++++++
test/unittest/misc/tst.lockmem-envvar.x | 16 +++++++++++
test/unittest/misc/tst.lockmem-pragma.r | 20 +++++++++++++
test/unittest/misc/tst.lockmem-pragma.sh | 26 +++++++++++++++++
test/unittest/misc/tst.lockmem-pragma.x | 16 +++++++++++
test/unittest/misc/tst.lockmem-x.r | 20 +++++++++++++
test/unittest/misc/tst.lockmem-x.sh | 18 ++++++++++++
test/unittest/misc/tst.lockmem-x.x | 16 +++++++++++
18 files changed, 275 insertions(+), 21 deletions(-)
create mode 100644 test/unittest/misc/tst.lockmem-cmdline.r
create mode 100755 test/unittest/misc/tst.lockmem-cmdline.sh
create mode 100755 test/unittest/misc/tst.lockmem-cmdline.x
create mode 100644 test/unittest/misc/tst.lockmem-envvar.r
create mode 100755 test/unittest/misc/tst.lockmem-envvar.sh
create mode 100755 test/unittest/misc/tst.lockmem-envvar.x
create mode 100644 test/unittest/misc/tst.lockmem-pragma.r
create mode 100755 test/unittest/misc/tst.lockmem-pragma.sh
create mode 100755 test/unittest/misc/tst.lockmem-pragma.x
create mode 100644 test/unittest/misc/tst.lockmem-x.r
create mode 100755 test/unittest/misc/tst.lockmem-x.sh
create mode 100755 test/unittest/misc/tst.lockmem-x.x
diff --git a/include/dtrace/options_defines.h b/include/dtrace/options_defines.h
index 5ecd6285..61a23f42 100644
--- a/include/dtrace/options_defines.h
+++ b/include/dtrace/options_defines.h
@@ -61,7 +61,8 @@
#define DTRACEOPT_MAXFRAMES 31 /* maximum number of stack frames */
#define DTRACEOPT_BPFLOG 32 /* always output BPF verifier log */
#define DTRACEOPT_SCRATCHSIZE 33 /* max scratch size permitted */
-#define DTRACEOPT_MAX 34 /* number of options */
+#define DTRACEOPT_LOCKMEM 34 /* max locked memory */
+#define DTRACEOPT_MAX 35 /* number of options */
#define DTRACEOPT_UNSET (dtrace_optval_t)-2 /* unset option */
diff --git a/libdtrace/dt_bpf.c b/libdtrace/dt_bpf.c
index e68bf561..957a6261 100644
--- a/libdtrace/dt_bpf.c
+++ b/libdtrace/dt_bpf.c
@@ -58,6 +58,16 @@ dt_bpf_error(dtrace_hdl_t *dtp, const char *fmt, ...)
return dt_set_errno(dtp, EDT_BPF);
}
+static int
+dt_bpf_lockmem_error(dtrace_hdl_t *dtp, const char *msg)
+{
+ return dt_bpf_error(dtp, "%s:\n"
+ "\tThe kernel locked-memory limit is possibly too low. Set a\n"
+ "\thigher limit with the DTrace option '-xlockmem=N'. Or, use\n"
+ "\t'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.\n"
+ , msg);
+}
+
/*
* Load the value for the given key in the map referenced by the given fd.
*/
@@ -113,9 +123,15 @@ create_gmap(dtrace_hdl_t *dtp, const char *name, enum bpf_map_type type,
dt_dprintf("Creating BPF map '%s' (ksz %u, vsz %u, sz %d)\n",
name, ksz, vsz, size);
fd = bpf_create_map_name(type, name, ksz, vsz, size, 0);
- if (fd < 0)
- return dt_bpf_error(dtp, "failed to create BPF map '%s': %s\n",
- name, strerror(errno));
+ if (fd < 0) {
+ char msg[64];
+
+ snprintf(msg, sizeof(msg),
+ "failed to create BPF map '%s'", name);
+ if (errno == EPERM)
+ return dt_bpf_lockmem_error(dtp, msg);
+ return dt_bpf_error(dtp, "%s: %s\n", msg, strerror(errno));
+ }
dt_dprintf("BPF map '%s' is FD %d (ksz %u, vsz %u, sz %d)\n",
name, fd, ksz, vsz, size);
@@ -464,10 +480,15 @@ dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
assert(log != NULL);
rc = bpf_load_program_xattr(&attr, log, logsz);
if (rc < 0) {
- dt_bpf_error(dtp,
- "BPF program load for '%s:%s:%s:%s' failed: %s\n",
- pdp->prv, pdp->mod, pdp->fun, pdp->prb,
- strerror(origerrno ? origerrno : errno));
+ char msg[64];
+
+ snprintf(msg, sizeof(msg),
+ "BPF program load for '%s:%s:%s:%s' failed",
+ pdp->prv, pdp->mod, pdp->fun, pdp->prb);
+ if (errno == EPERM)
+ return dt_bpf_lockmem_error(dtp, msg);
+ dt_bpf_error(dtp, "%s: %s\n", msg,
+ strerror(origerrno ? origerrno : errno));
/* check whether we have an incomplete BPF log */
if (errno == ENOSPC) {
diff --git a/libdtrace/dt_open.c b/libdtrace/dt_open.c
index 316ef4d0..4ba44ab3 100644
--- a/libdtrace/dt_open.c
+++ b/libdtrace/dt_open.c
@@ -707,19 +707,6 @@ dt_vopen(int version, int flags, int *errp,
setrlimit(RLIMIT_NOFILE, &rl);
}
- /*
- * Also, raise the limit on size that can be locked into memory,
- * which is needed for BPF operations.
- */
- if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) {
- rlim_t lim = 32 * 1024 * 1024;
-
- if (rl.rlim_cur < lim) {
- rl.rlim_cur = rl.rlim_max = lim;
- setrlimit(RLIMIT_MEMLOCK, &rl);
- }
- }
-
if ((dtp = malloc(sizeof(dtrace_hdl_t))) == NULL)
return set_open_errno(dtp, errp, EDT_NOMEM);
diff --git a/libdtrace/dt_options.c b/libdtrace/dt_options.c
index 5d3ff2ae..1c32dc3d 100644
--- a/libdtrace/dt_options.c
+++ b/libdtrace/dt_options.c
@@ -750,6 +750,20 @@ dt_opt_size(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
return 0;
}
+static int
+dt_opt_lockmem(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
+{
+ if (arg == NULL)
+ return dt_set_errno(dtp, EDT_BADOPTVAL);
+
+ if (strcmp(arg, "unlimited") == 0)
+ dtp->dt_options[option] = RLIM_INFINITY;
+ else
+ dt_opt_size(dtp, arg, option);
+
+ return 0;
+}
+
static int
dt_opt_scratchsize(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
{
@@ -1120,6 +1134,7 @@ static const dt_option_t _dtrace_rtoptions[] = {
{ "grabanon", dt_opt_runtime, DTRACEOPT_GRABANON },
{ "jstackframes", dt_opt_runtime, DTRACEOPT_JSTACKFRAMES },
{ "jstackstrsize", dt_opt_size, DTRACEOPT_JSTACKSTRSIZE },
+ { "lockmem", dt_opt_lockmem, DTRACEOPT_LOCKMEM },
{ "maxframes", dt_opt_runtime, DTRACEOPT_MAXFRAMES },
{ "nspec", dt_opt_runtime, DTRACEOPT_NSPEC },
{ "pcapsize", dt_opt_pcapsize, DTRACEOPT_PCAPSIZE },
diff --git a/libdtrace/dt_work.c b/libdtrace/dt_work.c
index 8936b52f..da08d9c5 100644
--- a/libdtrace/dt_work.c
+++ b/libdtrace/dt_work.c
@@ -55,10 +55,20 @@ dtrace_go(dtrace_hdl_t *dtp, uint_t cflags)
size_t size;
int err;
struct epoll_event ev;
+ dtrace_optval_t lockmem = dtp->dt_options[DTRACEOPT_LOCKMEM];
+ struct rlimit rl;
if (dtp->dt_active)
return dt_set_errno(dtp, EINVAL);
+ /*
+ * Set the locked-memory limit if so directed by the user.
+ */
+ if (lockmem != DTRACEOPT_UNSET) {
+ rl.rlim_cur = rl.rlim_max = lockmem;
+ setrlimit(RLIMIT_MEMLOCK, &rl);
+ }
+
/*
* Create the global BPF maps. This is done only once regardless of
* how many programs there are.
diff --git a/runtest.sh b/runtest.sh
index 1495285d..f97c1169 100755
--- a/runtest.sh
+++ b/runtest.sh
@@ -312,6 +312,10 @@ logdir=$(find_next_numeric_dir test/log)
LOGFILE=$logdir/runtest.log
SUMFILE=$logdir/runtest.sum
+# Set a locked-memory limit that should be big enough for the test suite.
+
+ulimit -l $((256 * 1024 * 1024))
+
# If running as root, remember and turn off core_pattern, and set the
# coredumpsize to a biggish value.
diff --git a/test/unittest/misc/tst.lockmem-cmdline.r b/test/unittest/misc/tst.lockmem-cmdline.r
new file mode 100644
index 00000000..537d655d
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-cmdline.r
@@ -0,0 +1,12 @@
+1
+
+ 1234
+0
+
+ 1234
+0
+-- @@stderr --
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
diff --git a/test/unittest/misc/tst.lockmem-cmdline.sh b/test/unittest/misc/tst.lockmem-cmdline.sh
new file mode 100755
index 00000000..f7303ec1
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-cmdline.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+for val in 1 16384 unlimited; do
+ ulimit -l $val
+
+ $dtrace -qn 'BEGIN { @ = avg(1234); exit(0); }'
+ echo $?
+done
+
+exit 0
diff --git a/test/unittest/misc/tst.lockmem-cmdline.x b/test/unittest/misc/tst.lockmem-cmdline.x
new file mode 100755
index 00000000..5f27acb6
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-cmdline.x
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
+
+if [ $MAJOR -lt 5 ]; then
+ exit 0
+fi
+if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
+ exit 0
+fi
+
+# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
+# but UEKR7 (5.15.0) does not
+
+echo "no locked-memory limit on newer kernels?"
+exit 1
diff --git a/test/unittest/misc/tst.lockmem-envvar.r b/test/unittest/misc/tst.lockmem-envvar.r
new file mode 100644
index 00000000..1e6918e7
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-envvar.r
@@ -0,0 +1,20 @@
+1
+1
+
+ 1234
+0
+
+ 1234
+0
+
+ 1234
+0
+-- @@stderr --
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
diff --git a/test/unittest/misc/tst.lockmem-envvar.sh b/test/unittest/misc/tst.lockmem-envvar.sh
new file mode 100755
index 00000000..8b1946d7
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-envvar.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+ulimit -l 1
+
+for val in 16 1K 16384K 16M unlimited; do
+ DTRACE_OPT_LOCKMEM=$val $dtrace -qn 'BEGIN { @ = avg(1234); exit(0); }'
+ echo $?
+done
+
+exit 0
diff --git a/test/unittest/misc/tst.lockmem-envvar.x b/test/unittest/misc/tst.lockmem-envvar.x
new file mode 100755
index 00000000..5f27acb6
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-envvar.x
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
+
+if [ $MAJOR -lt 5 ]; then
+ exit 0
+fi
+if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
+ exit 0
+fi
+
+# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
+# but UEKR7 (5.15.0) does not
+
+echo "no locked-memory limit on newer kernels?"
+exit 1
diff --git a/test/unittest/misc/tst.lockmem-pragma.r b/test/unittest/misc/tst.lockmem-pragma.r
new file mode 100644
index 00000000..1e6918e7
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-pragma.r
@@ -0,0 +1,20 @@
+1
+1
+
+ 1234
+0
+
+ 1234
+0
+
+ 1234
+0
+-- @@stderr --
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
diff --git a/test/unittest/misc/tst.lockmem-pragma.sh b/test/unittest/misc/tst.lockmem-pragma.sh
new file mode 100755
index 00000000..05963f00
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-pragma.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+ulimit -l 1
+
+for val in 16 1K 16384K 16M unlimited; do
+ $dtrace -qs /dev/stdin << EOF
+ #pragma D option lockmem=$val
+
+ BEGIN
+ {
+ @ = avg(1234);
+ exit(0);
+ }
+EOF
+ echo $?
+done
+
+exit 0
diff --git a/test/unittest/misc/tst.lockmem-pragma.x b/test/unittest/misc/tst.lockmem-pragma.x
new file mode 100755
index 00000000..5f27acb6
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-pragma.x
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
+
+if [ $MAJOR -lt 5 ]; then
+ exit 0
+fi
+if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
+ exit 0
+fi
+
+# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
+# but UEKR7 (5.15.0) does not
+
+echo "no locked-memory limit on newer kernels?"
+exit 1
diff --git a/test/unittest/misc/tst.lockmem-x.r b/test/unittest/misc/tst.lockmem-x.r
new file mode 100644
index 00000000..1e6918e7
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-x.r
@@ -0,0 +1,20 @@
+1
+1
+
+ 1234
+0
+
+ 1234
+0
+
+ 1234
+0
+-- @@stderr --
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
+dtrace: could not enable tracing: failed to create BPF map 'state':
+ The kernel locked-memory limit is possibly too low. Set a
+ higher limit with the DTrace option '-xlockmem=N'. Or, use
+ 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
diff --git a/test/unittest/misc/tst.lockmem-x.sh b/test/unittest/misc/tst.lockmem-x.sh
new file mode 100755
index 00000000..de5e583f
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-x.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Oracle Linux DTrace.
+# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at
+# http://oss.oracle.com/licenses/upl.
+#
+
+dtrace=$1
+
+ulimit -l 1
+
+for val in 16 1K 16384K 16M unlimited; do
+ $dtrace -xlockmem=$val -qn 'BEGIN { @ = avg(1234); exit(0); }'
+ echo $?
+done
+
+exit 0
diff --git a/test/unittest/misc/tst.lockmem-x.x b/test/unittest/misc/tst.lockmem-x.x
new file mode 100755
index 00000000..5f27acb6
--- /dev/null
+++ b/test/unittest/misc/tst.lockmem-x.x
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
+
+if [ $MAJOR -lt 5 ]; then
+ exit 0
+fi
+if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
+ exit 0
+fi
+
+# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
+# but UEKR7 (5.15.0) does not
+
+echo "no locked-memory limit on newer kernels?"
+exit 1
--
2.18.4
More information about the DTrace-devel
mailing list