[DTrace-devel] [PATCH] dtrace: BPF program load for '...' failed: No space left on device
eugene.loh at oracle.com
eugene.loh at oracle.com
Tue Dec 15 19:13:01 PST 2020
From: Eugene Loh <eugene.loh at oracle.com>
A D script that produces BPF code with many code paths can result
in 16 Mbytes of BPF log file, ending with the above error message.
The log file really says nothing about why the BPF load failed.
The actual problem is that we supplied a 16-Mbyte log buffer that
is too small.
If no log buffer is supplied, this problem is not encountered.
Change DTrace's BPF program load to use no log buffer at first.
If the load fails, then retry with a log buffer. The load should
again fail, but if the failure is not ENOSPC, we can simply report
the log and be done. If the failure becomes ENOSPC, inform the
user of the problem and what action can be taken to increase the
buffer size.
One question is whether, in the case of ENOSPC, the partial buffer
should be reported. By default, do not, since the output is very
large and likely does not indicate what the problem is.
Provide new DTrace options to control the log buffer size and
whether a partial buffer should be reported.
Add tests for this fix. Specifically, the aggregation function
quantize() can be used, since it must quantize a value into one
of 127 different bins. The algorithm used has many code paths
and thereby exercises the BPF verifier well.
Orabug: 32290849
Signed-off-by: Eugene Loh <eugene.loh at oracle.com>
---
include/dtrace/options_defines.h | 4 +-
libdtrace/dt_bpf.c | 57 ++++++++++++----
libdtrace/dt_options.c | 2 +
test/unittest/misc/err.BPFlog.d | 114 +++++++++++++++++++++++++++++++
test/unittest/misc/err.BPFlog.r | 7 ++
test/unittest/misc/tst.BPFlog.d | 39 +++++++++++
test/unittest/misc/tst.BPFlog.r | 7 ++
7 files changed, 214 insertions(+), 16 deletions(-)
create mode 100644 test/unittest/misc/err.BPFlog.d
create mode 100644 test/unittest/misc/err.BPFlog.r
create mode 100644 test/unittest/misc/tst.BPFlog.d
create mode 100644 test/unittest/misc/tst.BPFlog.r
diff --git a/include/dtrace/options_defines.h b/include/dtrace/options_defines.h
index ed4a7c15..9711f582 100644
--- a/include/dtrace/options_defines.h
+++ b/include/dtrace/options_defines.h
@@ -57,7 +57,9 @@
#define DTRACEOPT_QUIETRESIZE 27 /* quieten buffer-resize messages */
#define DTRACEOPT_NORESOLVE 28 /* prevent resolution of symbols */
#define DTRACEOPT_PCAPSIZE 29 /* number of bytes to be captured */
-#define DTRACEOPT_MAX 30 /* number of options */
+#define DTRACEOPT_BPFLOGPART 30 /* BPF verifier log, report partial log */
+#define DTRACEOPT_BPFLOGSIZE 31 /* BPF verifier log, max # bytes */
+#define DTRACEOPT_MAX 32 /* number of options */
#define DTRACEOPT_UNSET (dtrace_optval_t)-2 /* unset option */
diff --git a/libdtrace/dt_bpf.c b/libdtrace/dt_bpf.c
index c6102f15..bdb6180b 100644
--- a/libdtrace/dt_bpf.c
+++ b/libdtrace/dt_bpf.c
@@ -323,9 +323,11 @@ dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
const dtrace_difo_t *dp)
{
struct bpf_load_program_attr attr;
- int logsz = BPF_LOG_BUF_SIZE;
+ size_t logsz;
char *log;
- int rc;
+ int rc, orig_errno;
+ const dtrace_probedesc_t *pdp = prp->desc;
+ char *p, *q;
/*
* Check whether there are any probe-specific relocations to be
@@ -340,26 +342,51 @@ dt_bpf_load_prog(dtrace_hdl_t *dtp, const dt_probe_t *prp,
memset(&attr, 0, sizeof(struct bpf_load_program_attr));
- log = dt_zalloc(dtp, logsz);
- assert(log != NULL);
-
attr.prog_type = prp->prov->impl->prog_type;
attr.name = NULL;
attr.insns = dp->dtdo_buf;
attr.insns_cnt = dp->dtdo_len;
attr.license = BPF_CG_LICENSE;
- attr.log_level = 4 | 2 | 1;
+ rc = bpf_load_program_xattr(&attr, NULL, 0);
+ if (rc >= 0)
+ return rc;
+
+ /* if failure, note errno and then rerun with logging */
+ orig_errno = errno;
+ if (dtp->dt_options[DTRACEOPT_BPFLOGSIZE] != DTRACEOPT_UNSET)
+ logsz = dtp->dt_options[DTRACEOPT_BPFLOGSIZE];
+ else
+ logsz = BPF_LOG_BUF_SIZE;
+ attr.log_level = 4 | 2 | 1;
+ log = dt_zalloc(dtp, logsz);
+ assert(log != NULL);
rc = bpf_load_program_xattr(&attr, log, logsz);
- if (rc < 0) {
- const dtrace_probedesc_t *pdp = prp->desc;
- char *p, *q;
-
- rc = dt_bpf_error(dtp,
- "BPF program load for '%s:%s:%s:%s' failed: "
- "%s\n",
- pdp->prv, pdp->mod, pdp->fun, pdp->prb,
- strerror(errno));
+
+ /* since it failed once, it should fail again */
+ assert(rc < 0);
+
+ /* report original error */
+ rc = dt_bpf_error(dtp,
+ "BPF program load for '%s:%s:%s:%s' failed: %s\n",
+ pdp->prv, pdp->mod, pdp->fun, pdp->prb,
+ strerror(orig_errno));
+
+ /* check whether to skip reporting a (partial/incomplete) BPF log */
+ if (errno == ENOSPC &&
+ dtp->dt_options[DTRACEOPT_BPFLOGPART] == DTRACEOPT_UNSET) {
+ fprintf(stderr,
+ "BPF verifier log is incomplete and is not reported.\n"
+ "Set DTrace option:\n"
+ " - 'bpflogpart' (no value) for partial log output, or\n"
+ " - 'bpflogsize' to some greater size for more output.\n"
+ " (Current size is %ld.)\n", logsz);
+ } else {
+ if (errno == ENOSPC)
+ fprintf(stderr,
+ "BPF verifier log is incomplete; partial log is reported.\n"
+ "Set DTrace option 'bpflogsize' to some greater size\n"
+ "for more output. Current size is %ld.\n", logsz);
/*
* If there is BPF verifier output, print it with a "BPF: "
diff --git a/libdtrace/dt_options.c b/libdtrace/dt_options.c
index 53c9507a..3d461916 100644
--- a/libdtrace/dt_options.c
+++ b/libdtrace/dt_options.c
@@ -1093,6 +1093,8 @@ static const dt_option_t _dtrace_ctoptions[] = {
*/
static const dt_option_t _dtrace_rtoptions[] = {
{ "aggsize", dt_opt_size, DTRACEOPT_AGGSIZE },
+ { "bpflogpart", dt_opt_runtime, DTRACEOPT_BPFLOGPART },
+ { "bpflogsize", dt_opt_size, DTRACEOPT_BPFLOGSIZE },
{ "bufsize", dt_opt_size, DTRACEOPT_BUFSIZE },
{ "bufpolicy", dt_opt_bufpolicy, DTRACEOPT_BUFPOLICY },
{ "bufresize", dt_opt_bufresize, DTRACEOPT_BUFRESIZE },
diff --git a/test/unittest/misc/err.BPFlog.d b/test/unittest/misc/err.BPFlog.d
new file mode 100644
index 00000000..e63ec3c3
--- /dev/null
+++ b/test/unittest/misc/err.BPFlog.d
@@ -0,0 +1,114 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+#pragma D option quiet
+
+BEGIN
+{
+ x = 1;
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ exit(0);
+}
diff --git a/test/unittest/misc/err.BPFlog.r b/test/unittest/misc/err.BPFlog.r
new file mode 100644
index 00000000..e8c7be04
--- /dev/null
+++ b/test/unittest/misc/err.BPFlog.r
@@ -0,0 +1,7 @@
+-- @@stderr --
+BPF verifier log is incomplete and is not reported.
+Set DTrace option:
+ - 'bpflogpart' (no value) for partial log output, or
+ - 'bpflogsize' to some greater size for more output.
+ (Current size is 16777215.)
+dtrace: could not enable tracing: BPF program load for 'dtrace:::BEGIN' failed: Argument list too long
diff --git a/test/unittest/misc/tst.BPFlog.d b/test/unittest/misc/tst.BPFlog.d
new file mode 100644
index 00000000..09b42d65
--- /dev/null
+++ b/test/unittest/misc/tst.BPFlog.d
@@ -0,0 +1,39 @@
+/*
+ * Oracle Linux DTrace.
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+#pragma D option quiet
+
+BEGIN
+{
+ x = 1;
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ @ = quantize(x); @ = quantize(x); @ = quantize(x); @ = quantize(x);
+ exit(0);
+}
diff --git a/test/unittest/misc/tst.BPFlog.r b/test/unittest/misc/tst.BPFlog.r
new file mode 100644
index 00000000..3b880015
--- /dev/null
+++ b/test/unittest/misc/tst.BPFlog.r
@@ -0,0 +1,7 @@
+
+
+ value ------------- Distribution ------------- count
+ 0 | 0
+ 1 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 100
+ 2 | 0
+
--
2.18.4
More information about the DTrace-devel
mailing list