[DTrace-devel] [PATCH v2 10/61] Locked-memory limit
Kris Van Hees
kris.van.hees at oracle.com
Fri Jul 29 15:51:04 UTC 2022
On Thu, Jul 28, 2022 at 03:35:23PM -0400, eugene.loh--- via DTrace-devel wrote:
> From: Eugene Loh <eugene.loh at oracle.com>
>
> Locked memory is needed for BPF operations such as creating maps and
> loading programs. Therefore, dt_vopen() sets RLIMIT_MEMLOCK to 32 Mbytes,
> an amount that seems sufficient for many tests in our suite. On the other
> hand, the value is somewhat arbitrary, excessive for many D scripts yet
> insufficient for others, including some of our tests. Further, it is
> silently modifying a resource limit.
>
> Remove dt_vopen()'s silent modification of the locked-memory limit.
>
> Explicitly set "ulimit -l" in runtest.sh to accommodate the tests in our
> suite.
>
> While users can similarly set "ulimit -l" explicitly, it would be more
> convenient if there were alternative mechanisms, especially when using
> "sudo dtrace". Therefore, add a D option lockmem. Notice that ulimit
> allows the value "unlimited" for users who simply want to ignore any limit.
>
> Add some more verbose error messages that explain that encountering
> EPERM during BPF map creation or BPF program load may be solved by
> adjusting the locked-memory limit.
>
> With UEKR7, the locked-memory limit seems to be ignored for root.
>
> Signed-off-by: Eugene Loh <eugene.loh at oracle.com>
Reviewed-by: Kris Van Hees <kris.van.hees at oracle.com>
... with minor modificataions (see below for the .x files)
... queued for dev
> ---
> include/dtrace/options_defines.h | 3 +-
> libdtrace/dt_bpf.c | 35 ++++++++++++++++++-----
> libdtrace/dt_open.c | 13 ---------
> libdtrace/dt_options.c | 15 ++++++++++
> libdtrace/dt_work.c | 10 +++++++
> runtest.sh | 4 +++
> test/unittest/misc/tst.lockmem-cmdline.r | 12 ++++++++
> test/unittest/misc/tst.lockmem-cmdline.sh | 18 ++++++++++++
> test/unittest/misc/tst.lockmem-cmdline.x | 16 +++++++++++
> test/unittest/misc/tst.lockmem-envvar.r | 20 +++++++++++++
> test/unittest/misc/tst.lockmem-envvar.sh | 18 ++++++++++++
> test/unittest/misc/tst.lockmem-envvar.x | 16 +++++++++++
> test/unittest/misc/tst.lockmem-pragma.r | 20 +++++++++++++
> test/unittest/misc/tst.lockmem-pragma.sh | 26 +++++++++++++++++
> test/unittest/misc/tst.lockmem-pragma.x | 16 +++++++++++
> test/unittest/misc/tst.lockmem-x.r | 20 +++++++++++++
> test/unittest/misc/tst.lockmem-x.sh | 18 ++++++++++++
> test/unittest/misc/tst.lockmem-x.x | 16 +++++++++++
> 18 files changed, 275 insertions(+), 21 deletions(-)
> create mode 100644 test/unittest/misc/tst.lockmem-cmdline.r
> create mode 100755 test/unittest/misc/tst.lockmem-cmdline.sh
> create mode 100755 test/unittest/misc/tst.lockmem-cmdline.x
> create mode 100644 test/unittest/misc/tst.lockmem-envvar.r
> create mode 100755 test/unittest/misc/tst.lockmem-envvar.sh
> create mode 100755 test/unittest/misc/tst.lockmem-envvar.x
> create mode 100644 test/unittest/misc/tst.lockmem-pragma.r
> create mode 100755 test/unittest/misc/tst.lockmem-pragma.sh
> create mode 100755 test/unittest/misc/tst.lockmem-pragma.x
> create mode 100644 test/unittest/misc/tst.lockmem-x.r
> create mode 100755 test/unittest/misc/tst.lockmem-x.sh
> create mode 100755 test/unittest/misc/tst.lockmem-x.x
>
> diff --git a/include/dtrace/options_defines.h b/include/dtrace/options_defines.h
> index 5ecd6285..61a23f42 100644
> --- a/include/dtrace/options_defines.h
> +++ b/include/dtrace/options_defines.h
> @@ -61,7 +61,8 @@
> #define DTRACEOPT_MAXFRAMES 31 /* maximum number of stack frames */
> #define DTRACEOPT_BPFLOG 32 /* always output BPF verifier log */
> #define DTRACEOPT_SCRATCHSIZE 33 /* max scratch size permitted */
> -#define DTRACEOPT_MAX 34 /* number of options */
> +#define DTRACEOPT_LOCKMEM 34 /* max locked memory */
> +#define DTRACEOPT_MAX 35 /* number of options */
>
> #define DTRACEOPT_UNSET (dtrace_optval_t)-2 /* unset option */
>
> diff --git a/libdtrace/dt_bpf.c b/libdtrace/dt_bpf.c
> index e68bf561..957a6261 100644
> --- a/libdtrace/dt_bpf.c
> +++ b/libdtrace/dt_bpf.c
> @@ -58,6 +58,16 @@ dt_bpf_error(dtrace_hdl_t *dtp, const char *fmt, ...)
> return dt_set_errno(dtp, EDT_BPF);
> }
>
> +static int
> +dt_bpf_lockmem_error(dtrace_hdl_t *dtp, const char *msg)
> +{
> + return dt_bpf_error(dtp, "%s:\n"
> + "\tThe kernel locked-memory limit is possibly too low. Set a\n"
> + "\thigher limit with the DTrace option '-xlockmem=N'. Or, use\n"
> + "\t'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.\n"
> + , msg);
> +}
> +
> /*
> * Load the value for the given key in the map referenced by the given fd.
> */
> @@ -113,9 +123,15 @@ create_gmap(dtrace_hdl_t *dtp, const char *name, enum bpf_map_type type,
> dt_dprintf("Creating BPF map '%s' (ksz %u, vsz %u, sz %d)\n",
> name, ksz, vsz, size);
> fd = bpf_create_map_name(type, name, ksz, vsz, size, 0);
> - if (fd < 0)
> - return dt_bpf_error(dtp, "failed to create BPF map '%s': %s\n",
> - name, strerror(errno));
> + if (fd < 0) {
> + char msg[64];
> +
> + snprintf(msg, sizeof(msg),
> + "failed to create BPF map '%s'", name);
> + if (errno == EPERM)
> + return dt_bpf_lockmem_error(dtp, msg);
> + return dt_bpf_error(dtp, "%s: %s\n", msg, strerror(errno));
> + }
>
> dt_dprintf("BPF map '%s' is FD %d (ksz %u, vsz %u, sz %d)\n",
> name, fd, ksz, vsz, size);
> @@ -464,10 +480,15 @@ dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
> assert(log != NULL);
> rc = bpf_load_program_xattr(&attr, log, logsz);
> if (rc < 0) {
> - dt_bpf_error(dtp,
> - "BPF program load for '%s:%s:%s:%s' failed: %s\n",
> - pdp->prv, pdp->mod, pdp->fun, pdp->prb,
> - strerror(origerrno ? origerrno : errno));
> + char msg[64];
> +
> + snprintf(msg, sizeof(msg),
> + "BPF program load for '%s:%s:%s:%s' failed",
> + pdp->prv, pdp->mod, pdp->fun, pdp->prb);
> + if (errno == EPERM)
> + return dt_bpf_lockmem_error(dtp, msg);
> + dt_bpf_error(dtp, "%s: %s\n", msg,
> + strerror(origerrno ? origerrno : errno));
>
> /* check whether we have an incomplete BPF log */
> if (errno == ENOSPC) {
> diff --git a/libdtrace/dt_open.c b/libdtrace/dt_open.c
> index 316ef4d0..4ba44ab3 100644
> --- a/libdtrace/dt_open.c
> +++ b/libdtrace/dt_open.c
> @@ -707,19 +707,6 @@ dt_vopen(int version, int flags, int *errp,
> setrlimit(RLIMIT_NOFILE, &rl);
> }
>
> - /*
> - * Also, raise the limit on size that can be locked into memory,
> - * which is needed for BPF operations.
> - */
> - if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) {
> - rlim_t lim = 32 * 1024 * 1024;
> -
> - if (rl.rlim_cur < lim) {
> - rl.rlim_cur = rl.rlim_max = lim;
> - setrlimit(RLIMIT_MEMLOCK, &rl);
> - }
> - }
> -
> if ((dtp = malloc(sizeof(dtrace_hdl_t))) == NULL)
> return set_open_errno(dtp, errp, EDT_NOMEM);
>
> diff --git a/libdtrace/dt_options.c b/libdtrace/dt_options.c
> index 5d3ff2ae..1c32dc3d 100644
> --- a/libdtrace/dt_options.c
> +++ b/libdtrace/dt_options.c
> @@ -750,6 +750,20 @@ dt_opt_size(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
> return 0;
> }
>
> +static int
> +dt_opt_lockmem(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
> +{
> + if (arg == NULL)
> + return dt_set_errno(dtp, EDT_BADOPTVAL);
> +
> + if (strcmp(arg, "unlimited") == 0)
> + dtp->dt_options[option] = RLIM_INFINITY;
> + else
> + dt_opt_size(dtp, arg, option);
> +
> + return 0;
> +}
> +
> static int
> dt_opt_scratchsize(dtrace_hdl_t *dtp, const char *arg, uintptr_t option)
> {
> @@ -1120,6 +1134,7 @@ static const dt_option_t _dtrace_rtoptions[] = {
> { "grabanon", dt_opt_runtime, DTRACEOPT_GRABANON },
> { "jstackframes", dt_opt_runtime, DTRACEOPT_JSTACKFRAMES },
> { "jstackstrsize", dt_opt_size, DTRACEOPT_JSTACKSTRSIZE },
> + { "lockmem", dt_opt_lockmem, DTRACEOPT_LOCKMEM },
> { "maxframes", dt_opt_runtime, DTRACEOPT_MAXFRAMES },
> { "nspec", dt_opt_runtime, DTRACEOPT_NSPEC },
> { "pcapsize", dt_opt_pcapsize, DTRACEOPT_PCAPSIZE },
> diff --git a/libdtrace/dt_work.c b/libdtrace/dt_work.c
> index 8936b52f..da08d9c5 100644
> --- a/libdtrace/dt_work.c
> +++ b/libdtrace/dt_work.c
> @@ -55,10 +55,20 @@ dtrace_go(dtrace_hdl_t *dtp, uint_t cflags)
> size_t size;
> int err;
> struct epoll_event ev;
> + dtrace_optval_t lockmem = dtp->dt_options[DTRACEOPT_LOCKMEM];
> + struct rlimit rl;
>
> if (dtp->dt_active)
> return dt_set_errno(dtp, EINVAL);
>
> + /*
> + * Set the locked-memory limit if so directed by the user.
> + */
> + if (lockmem != DTRACEOPT_UNSET) {
> + rl.rlim_cur = rl.rlim_max = lockmem;
> + setrlimit(RLIMIT_MEMLOCK, &rl);
> + }
> +
> /*
> * Create the global BPF maps. This is done only once regardless of
> * how many programs there are.
> diff --git a/runtest.sh b/runtest.sh
> index 1495285d..f97c1169 100755
> --- a/runtest.sh
> +++ b/runtest.sh
> @@ -312,6 +312,10 @@ logdir=$(find_next_numeric_dir test/log)
> LOGFILE=$logdir/runtest.log
> SUMFILE=$logdir/runtest.sum
>
> +# Set a locked-memory limit that should be big enough for the test suite.
> +
> +ulimit -l $((256 * 1024 * 1024))
> +
> # If running as root, remember and turn off core_pattern, and set the
> # coredumpsize to a biggish value.
>
> diff --git a/test/unittest/misc/tst.lockmem-cmdline.r b/test/unittest/misc/tst.lockmem-cmdline.r
> new file mode 100644
> index 00000000..537d655d
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-cmdline.r
> @@ -0,0 +1,12 @@
> +1
> +
> + 1234
> +0
> +
> + 1234
> +0
> +-- @@stderr --
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> diff --git a/test/unittest/misc/tst.lockmem-cmdline.sh b/test/unittest/misc/tst.lockmem-cmdline.sh
> new file mode 100755
> index 00000000..f7303ec1
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-cmdline.sh
> @@ -0,0 +1,18 @@
> +#!/bin/bash
> +#
> +# Oracle Linux DTrace.
> +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
> +# Licensed under the Universal Permissive License v 1.0 as shown at
> +# http://oss.oracle.com/licenses/upl.
> +#
> +
> +dtrace=$1
> +
> +for val in 1 16384 unlimited; do
> + ulimit -l $val
> +
> + $dtrace -qn 'BEGIN { @ = avg(1234); exit(0); }'
> + echo $?
> +done
> +
> +exit 0
> diff --git a/test/unittest/misc/tst.lockmem-cmdline.x b/test/unittest/misc/tst.lockmem-cmdline.x
> new file mode 100755
> index 00000000..5f27acb6
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-cmdline.x
> @@ -0,0 +1,16 @@
> +#!/bin/sh
> +
> +read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
You are using a construct that may not be available in /bin/sh (it isn't on
Debian's default /bin/sh). So, make these scripts #!/bin/bash.
> +
> +if [ $MAJOR -lt 5 ]; then
> + exit 0
> +fi
> +if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
> + exit 0
> +fi
> +
> +# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
> +# but UEKR7 (5.15.0) does not
> +
> +echo "no locked-memory limit on newer kernels?"
> +exit 1
> diff --git a/test/unittest/misc/tst.lockmem-envvar.r b/test/unittest/misc/tst.lockmem-envvar.r
> new file mode 100644
> index 00000000..1e6918e7
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-envvar.r
> @@ -0,0 +1,20 @@
> +1
> +1
> +
> + 1234
> +0
> +
> + 1234
> +0
> +
> + 1234
> +0
> +-- @@stderr --
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> diff --git a/test/unittest/misc/tst.lockmem-envvar.sh b/test/unittest/misc/tst.lockmem-envvar.sh
> new file mode 100755
> index 00000000..8b1946d7
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-envvar.sh
> @@ -0,0 +1,18 @@
> +#!/bin/bash
> +#
> +# Oracle Linux DTrace.
> +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
> +# Licensed under the Universal Permissive License v 1.0 as shown at
> +# http://oss.oracle.com/licenses/upl.
> +#
> +
> +dtrace=$1
> +
> +ulimit -l 1
> +
> +for val in 16 1K 16384K 16M unlimited; do
> + DTRACE_OPT_LOCKMEM=$val $dtrace -qn 'BEGIN { @ = avg(1234); exit(0); }'
> + echo $?
> +done
> +
> +exit 0
> diff --git a/test/unittest/misc/tst.lockmem-envvar.x b/test/unittest/misc/tst.lockmem-envvar.x
> new file mode 100755
> index 00000000..5f27acb6
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-envvar.x
> @@ -0,0 +1,16 @@
> +#!/bin/sh
> +
> +read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
> +
> +if [ $MAJOR -lt 5 ]; then
> + exit 0
> +fi
> +if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
> + exit 0
> +fi
> +
> +# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
> +# but UEKR7 (5.15.0) does not
> +
> +echo "no locked-memory limit on newer kernels?"
> +exit 1
> diff --git a/test/unittest/misc/tst.lockmem-pragma.r b/test/unittest/misc/tst.lockmem-pragma.r
> new file mode 100644
> index 00000000..1e6918e7
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-pragma.r
> @@ -0,0 +1,20 @@
> +1
> +1
> +
> + 1234
> +0
> +
> + 1234
> +0
> +
> + 1234
> +0
> +-- @@stderr --
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> diff --git a/test/unittest/misc/tst.lockmem-pragma.sh b/test/unittest/misc/tst.lockmem-pragma.sh
> new file mode 100755
> index 00000000..05963f00
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-pragma.sh
> @@ -0,0 +1,26 @@
> +#!/bin/bash
> +#
> +# Oracle Linux DTrace.
> +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
> +# Licensed under the Universal Permissive License v 1.0 as shown at
> +# http://oss.oracle.com/licenses/upl.
> +#
> +
> +dtrace=$1
> +
> +ulimit -l 1
> +
> +for val in 16 1K 16384K 16M unlimited; do
> + $dtrace -qs /dev/stdin << EOF
> + #pragma D option lockmem=$val
> +
> + BEGIN
> + {
> + @ = avg(1234);
> + exit(0);
> + }
> +EOF
> + echo $?
> +done
> +
> +exit 0
> diff --git a/test/unittest/misc/tst.lockmem-pragma.x b/test/unittest/misc/tst.lockmem-pragma.x
> new file mode 100755
> index 00000000..5f27acb6
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-pragma.x
> @@ -0,0 +1,16 @@
> +#!/bin/sh
> +
> +read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
> +
> +if [ $MAJOR -lt 5 ]; then
> + exit 0
> +fi
> +if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
> + exit 0
> +fi
> +
> +# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
> +# but UEKR7 (5.15.0) does not
> +
> +echo "no locked-memory limit on newer kernels?"
> +exit 1
> diff --git a/test/unittest/misc/tst.lockmem-x.r b/test/unittest/misc/tst.lockmem-x.r
> new file mode 100644
> index 00000000..1e6918e7
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-x.r
> @@ -0,0 +1,20 @@
> +1
> +1
> +
> + 1234
> +0
> +
> + 1234
> +0
> +
> + 1234
> +0
> +-- @@stderr --
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> +dtrace: could not enable tracing: failed to create BPF map 'state':
> + The kernel locked-memory limit is possibly too low. Set a
> + higher limit with the DTrace option '-xlockmem=N'. Or, use
> + 'ulimit -l N' (Kbytes). Or, make N the string 'unlimited'.
> diff --git a/test/unittest/misc/tst.lockmem-x.sh b/test/unittest/misc/tst.lockmem-x.sh
> new file mode 100755
> index 00000000..de5e583f
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-x.sh
> @@ -0,0 +1,18 @@
> +#!/bin/bash
> +#
> +# Oracle Linux DTrace.
> +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
> +# Licensed under the Universal Permissive License v 1.0 as shown at
> +# http://oss.oracle.com/licenses/upl.
> +#
> +
> +dtrace=$1
> +
> +ulimit -l 1
> +
> +for val in 16 1K 16384K 16M unlimited; do
> + $dtrace -xlockmem=$val -qn 'BEGIN { @ = avg(1234); exit(0); }'
> + echo $?
> +done
> +
> +exit 0
> diff --git a/test/unittest/misc/tst.lockmem-x.x b/test/unittest/misc/tst.lockmem-x.x
> new file mode 100755
> index 00000000..5f27acb6
> --- /dev/null
> +++ b/test/unittest/misc/tst.lockmem-x.x
> @@ -0,0 +1,16 @@
> +#!/bin/sh
> +
> +read MAJOR MINOR <<< `uname -r | grep -Eo '^[0-9]+\.[0-9]+' | tr '.' ' '`
> +
> +if [ $MAJOR -lt 5 ]; then
> + exit 0
> +fi
> +if [ $MAJOR -eq 5 -a $MINOR -lt 15 ]; then
> + exit 0
> +fi
> +
> +# Somehow, UEKR6 (5.4.17) has problems with the the locked-memory limit,
> +# but UEKR7 (5.15.0) does not
> +
> +echo "no locked-memory limit on newer kernels?"
> +exit 1
> --
> 2.18.4
>
>
> _______________________________________________
> DTrace-devel mailing list
> DTrace-devel at oss.oracle.com
> https://oss.oracle.com/mailman/listinfo/dtrace-devel
More information about the DTrace-devel
mailing list