[DTrace-devel] [PATCH 1/2] ctf: rejig with better builtin object detection
Nick Alcock
nick.alcock at oracle.com
Tue Oct 1 17:12:51 UTC 2024
Drop reliance on modules.builtin.objs: instead, compute it by consulting the
KBUILD_MODNAME arguments recorded in the .cmd files associated with each
object file that ends up in vmlinux.a.
Every input compile line has a KBUILD_MODNAME, including those that aren't
tristate: so filter it down using the module list in modules.builtin.
Constructing this list ourselves lets us simplify its layout in ways that
makes life easier for the iterator in modules_builtin.c (now renamed to
module_objnames.c), reducing its length by about forty lines.
Doing things this way also lets us avoid relinking the kernel when doing a
make ctf. It will go wrong if people start deleting .*.cmd files from the
kernel build tree while leaving the object files behind, but I'm fairly sure
this will break other things too, so we can probably rely on this not
happening.
(POC: still being tested.)
Signed-off-by: Nick Alcock <nick.alcock at oracle.com>
---
scripts/Makefile.ctfa | 2 +-
scripts/Makefile.ctfa-toplevel | 41 ++--
scripts/ctf-module-objnames.sh | 35 +++
scripts/ctf/Makefile | 2 +-
scripts/ctf/ctfarchive.c | 14 +-
.../{modules_builtin.c => module_objnames.c} | 2 +-
.../{modules_builtin.h => module_objnames.h} | 2 +-
scripts/module_objnames.c | 160 ++++++++++++++
.../{modules_builtin.h => module_objnames.h} | 20 +-
scripts/modules_builtin.c | 200 ------------------
10 files changed, 235 insertions(+), 243 deletions(-)
create mode 100755 scripts/ctf-module-objnames.sh
rename scripts/ctf/{modules_builtin.c => module_objnames.c} (54%)
rename scripts/ctf/{modules_builtin.h => module_objnames.h} (54%)
create mode 100644 scripts/module_objnames.c
rename scripts/{modules_builtin.h => module_objnames.h} (64%)
delete mode 100644 scripts/modules_builtin.c
diff --git a/scripts/Makefile.ctfa b/scripts/Makefile.ctfa
index 2b10de139dce..1bcdeda375dd 100644
--- a/scripts/Makefile.ctfa
+++ b/scripts/Makefile.ctfa
@@ -35,7 +35,7 @@ ifdef CONFIG_CTF
ifeq ($(KBUILD_EXTMOD),)
ctf-modules := $(shell find . -name '*.ko.ctf' -print)
quiet_cmd_ctfa_raw = CTFARAW
- cmd_ctfa_raw = scripts/ctf/ctfarchive $@ .tmp_objects.builtin modules.builtin.objs $(ctf-filelist)
+ cmd_ctfa_raw = scripts/ctf/ctfarchive $@ .tmp_objects.builtin .tmp_module.objnames $(ctf-filelist)
ctf-builtins := .tmp_objects.builtin
ctf-filelist := .tmp_ctf.filelist
ctf-filelist-raw := .tmp_ctf.filelist.raw
diff --git a/scripts/Makefile.ctfa-toplevel b/scripts/Makefile.ctfa-toplevel
index 210bef3854e9..29416c9c07ce 100644
--- a/scripts/Makefile.ctfa-toplevel
+++ b/scripts/Makefile.ctfa-toplevel
@@ -12,27 +12,24 @@ ifeq ($(KBUILD_EXTMOD),)
# This contains all the object files that are built directly into the
# kernel (including built-in modules), for consumption by ctfarchive in
-# Makefile.modfinal.
-# This is made doubly annoying by the presence of '.o' files which are actually
-# thin ar archives, and the need to support file(1) versions too old to
-# recognize them as archives at all. (So we assume that everything that is notr
-# an ELF object is an archive.)
-ifeq ($(SRCARCH),x86)
-.tmp_objects.builtin: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),bzImage) FORCE
-else
-ifeq ($(SRCARCH),arm64)
-.tmp_objects.builtin: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),Image) FORCE
-else
-.tmp_objects.builtin: $(vmlinux-dirs) $(if $(KBUILD_BUILTIN),vmlinux) FORCE
-endif
-endif
- @echo $(KBUILD_VMLINUX_OBJS) | \
- tr " " "\n" | grep "\.o$$" | xargs -r file | \
- grep ELF | cut -d: -f1 > .tmp_objects.builtin
- @for archive in $$(echo $(KBUILD_VMLINUX_OBJS) |\
- tr " " "\n" | xargs -r file | grep -v ELF | cut -d: -f1); do \
- $(AR) t "$$archive" >> .tmp_objects.builtin; \
- done
+# Makefile.modfinal. Make sure that all lines that are not absolute paths
+# start with a ./ (for consistency with .tmp_ctf.filelist).
+# We sort it after the fact to eliminate duplicates (there are quite a lot).
+.tmp_objects.builtin: vmlinux.a $(KBUILD_VMLINUX_LIBS)
+ $(Q)for archive in $^; do \
+ $(AR) t "$$archive" >> $@.unsorted; \
+ done; \
+ sed -i '/^[^/]/s,^\([^\.]\),./\1,' $@.unsorted; \
+ sort -u < $@.unsorted > $@; \
+ rm -f $@.unsorted
+
+# This contains a mapping from module name to object file name for all
+# objects named in .tmp_objects.builtin.
+# Extract possible module names from the command-lines used to
+# compile the modules, filter them by the set of actual built-in modules
+# in modules.builtin
+.tmp_module.objnames: .tmp_objects.builtin modules.builtin
+ $(Q)$(srctree)/scripts/ctf-module-objnames.sh $@
ctf: vmlinux.ctfa
PHONY += ctf ctf_install
@@ -41,7 +38,7 @@ PHONY += ctf ctf_install
# built if not already done, since we need the .o files for the machinery
# above to work.
vmlinux.ctfa: KBUILD_BUILTIN := 1
-vmlinux.ctfa: modules.builtin.objs .tmp_objects.builtin
+vmlinux.ctfa: .tmp_objects.builtin .tmp_module.objnames
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal vmlinux.ctfa
ctf_install:
diff --git a/scripts/ctf-module-objnames.sh b/scripts/ctf-module-objnames.sh
new file mode 100755
index 000000000000..4fc8f24b8888
--- /dev/null
+++ b/scripts/ctf-module-objnames.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0+
+# Produce a file mapping module names to object file names for all built-in
+# modules in the kernel. Used by ctfarchive.
+
+set -e
+
+# Translate e.g. ./lib/zstd/common/entropy_common.o into
+# ./lib/zstd/common/.entropy_common.o.cmd, and extract the module name from
+# that.
+
+sed 's,^\(.*/\)\(.*\)$,\1.\2.cmd,g' .tmp_objects.builtin |\
+ xargs grep -o -- '-DKBUILD_MODNAME=[^ ]*' | \
+ # Translate e.g. ./lib/zstd/common/.entropy_common.o.cmd:
+ # -KDBUILD_MODNAME='"zstd_common"' into zstd_common
+ # ./lib/zstd/common/entropy_common.o, and sort the result so the same module
+ # always occupies consecutive lines.
+ sed 's,^\([^:]*/\)\.\([^:]*\).cmd.*-DKBUILD_MODNAME=."\([^"]*\)".*$,\3 \1\2,' | sort -k1 | \
+ # Accumulate filename portions for the same module into one line.
+ awk -F ' ' 'BEGIN { mod=""; objs="";}
+ $1 != mod { printf ("%s %s\n", mod, objs); mod=$1; objs=""; }
+ { objs=objs $2 " "; }
+ END { printf ("%s %s\n", mod, objs); }' > .tmp_possible_modobjs
+
+# Filter out the maybe-module names from this list, and sort them.
+sed 's, .*,,' < .tmp_possible_modobjs | sort -u -k 1b,1 > .tmp_possible_modules
+
+# Filter the list of possible modules by the list of modules actually in the
+# kernel, then use that to exclude non-modules from the list we computed
+# earlier. Trim off trailing spaces, to help the iterator in
+# modules_builtin.c. Complicated a bit by the need to trim off the leading
+# kernel/ from modules.builtin.
+sed 's,^.*/\([^/]*\)\.ko$,\1,' modules.builtin | sort -u -k 1b,1 | \
+ comm - .tmp_possible_modules -12 | join -j 1 - .tmp_possible_modobjs | \
+ sed 's, *$,,'> $1
diff --git a/scripts/ctf/Makefile b/scripts/ctf/Makefile
index 3b83f93bb9f9..cddf9710ef55 100644
--- a/scripts/ctf/Makefile
+++ b/scripts/ctf/Makefile
@@ -1,5 +1,5 @@
ifdef CONFIG_CTF
hostprogs-always-y := ctfarchive
-ctfarchive-objs := ctfarchive.o modules_builtin.o
+ctfarchive-objs := ctfarchive.o module_objnames.o
HOSTLDLIBS_ctfarchive := -lctf
endif
diff --git a/scripts/ctf/ctfarchive.c b/scripts/ctf/ctfarchive.c
index b91e48e5d46f..eac15b5b707b 100644
--- a/scripts/ctf/ctfarchive.c
+++ b/scripts/ctf/ctfarchive.c
@@ -5,7 +5,7 @@
* deduplicated CTF derived from those object files, split up by kernel
* module.
*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
#include <stdlib.h>
#include <string.h>
#include <ctf-api.h>
-#include "modules_builtin.h"
+#include "module_objnames.h"
static ctf_file_t *output;
@@ -242,19 +242,19 @@ static void suck_in_lines(const char *filename, void (*func)(const char *line))
/*
* Pull in modules.builtin.objs and turn it into CU mappings.
*/
-static void suck_in_modules(const char *modules_builtin_name)
+static void suck_in_modules(const char *module_objnames_name)
{
- struct modules_builtin_iter *i;
+ struct module_objnames_iter *i;
char *module_name = NULL;
char **paths;
- i = modules_builtin_iter_new(modules_builtin_name);
+ i = module_objnames_iter_new(module_objnames_name);
if (i == NULL) {
fprintf(stderr, "Cannot iterate over builtin module file.\n");
exit(1);
}
- while ((paths = modules_builtin_iter_next(i, &module_name)) != NULL) {
+ while ((paths = module_objnames_iter_next(i, &module_name)) != NULL) {
size_t j;
for (j = 0; paths[j] != NULL; j++) {
@@ -268,7 +268,7 @@ static void suck_in_modules(const char *modules_builtin_name)
free(paths);
}
free(module_name);
- modules_builtin_iter_free(i);
+ module_objnames_iter_free(i);
}
/*
diff --git a/scripts/ctf/modules_builtin.c b/scripts/ctf/module_objnames.c
similarity index 54%
rename from scripts/ctf/modules_builtin.c
rename to scripts/ctf/module_objnames.c
index 10af2bbc80e0..4e3d1d6c6389 100644
--- a/scripts/ctf/modules_builtin.c
+++ b/scripts/ctf/module_objnames.c
@@ -1,2 +1,2 @@
/* SPDX-License-Identifier: GPL-2.0 */
-#include "../modules_builtin.c"
+#include "../module_objnames.c"
diff --git a/scripts/ctf/modules_builtin.h b/scripts/ctf/module_objnames.h
similarity index 54%
rename from scripts/ctf/modules_builtin.h
rename to scripts/ctf/module_objnames.h
index 5e0299e5600c..b22906bff02f 100644
--- a/scripts/ctf/modules_builtin.h
+++ b/scripts/ctf/module_objnames.h
@@ -1,2 +1,2 @@
/* SPDX-License-Identifier: GPL-2.0 */
-#include "../modules_builtin.h"
+#include "../module_objnames.h"
diff --git a/scripts/module_objnames.c b/scripts/module_objnames.c
new file mode 100644
index 000000000000..bd7267710c8e
--- /dev/null
+++ b/scripts/module_objnames.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * A simple .tmp_module.objnames reader.
+ *
+ * (C) 2014, 2024 Oracle, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "module_objnames.h"
+
+/*
+ * Read a .tmp_module.objnames.objs file and translate it into a stream of
+ * (module name, object file names).
+ */
+
+/*
+ * Construct a module_objnames iterator.
+ */
+struct module_objnames_iter *
+module_objnames_iter_new(const char *module_objnames_file)
+{
+ struct module_objnames_iter *i;
+
+ i = calloc(1, sizeof(struct module_objnames_iter));
+ if (i == NULL)
+ return NULL;
+
+ i->f = fopen(module_objnames_file, "r");
+
+ if (i->f == NULL) {
+ fprintf(stderr, "Cannot open builtin module file %s: %s\n",
+ module_objnames_file, strerror(errno));
+ return NULL;
+ }
+
+ return i;
+}
+
+/*
+ * Iterate, returning a new null-terminated array of object file names, and a
+ * new dynamically-allocated module name. (The module name passed in is freed.)
+ *
+ * The array of object file names should be freed by the caller: the strings it
+ * points to are owned by the iterator, and should not be freed.
+ */
+
+char ** __attribute__((__nonnull__))
+module_objnames_iter_next(struct module_objnames_iter *i, char **module_name)
+{
+ size_t npaths = 1;
+ char **module_paths;
+ char *trailing_linefeed;
+ char *object_name;
+ char *p;
+ char *one_object;
+ size_t j = 0;
+
+ /*
+ * Read in all module entries, building the next arrayful of object
+ * file names for return.
+ *
+ * The first word of an entry is the module name: the second and
+ * subsequent words are object file names (there must be at least
+ * one, and may be more than one).
+ */
+
+ /*
+ * Reinvocation of exhausted iterator. Return NULL, once.
+ */
+retry:
+ if (getline(&i->line, &i->line_size, i->f) < 0) {
+ if (ferror(i->f)) {
+ fprintf(stderr, "Error reading from .tmp_module.objnames file:"
+ " %s\n", strerror(errno));
+ exit(1);
+ }
+ rewind(i->f);
+ return NULL;
+ }
+
+ if (i->line[0] == '\0')
+ goto retry;
+
+ trailing_linefeed = strchr(i->line, '\n');
+ if (trailing_linefeed != NULL)
+ *trailing_linefeed = '\0';
+
+ /*
+ * Slice the line in two at the first space: the elements past it are the
+ * object file names.
+ */
+ if (strchr(i->line, ' ') == NULL) {
+ fprintf(stderr, "Invalid line in .tmp_module.objnames: %s\n",
+ i->line);
+ exit(1);
+ }
+
+ p = strchr(i->line, ' ');
+ *p = '\0';
+ p++;
+ object_name = p;
+ free(*module_name);
+ *module_name = strdup(i->line);
+
+ /*
+ * The array size may be an overestimate if any object file names
+ * start or end with spaces (very unlikely) but cannot be an
+ * underestimate. (Check for it anyway.)
+ */
+
+ for (npaths = 0, one_object = object_name;
+ one_object != NULL;
+ npaths++, one_object = strchr(one_object + 1, ' '));
+
+ module_paths = malloc((npaths + 1) * sizeof(char *));
+ if (!module_paths) {
+ fprintf(stderr, "%s: out of memory on module %s\n", __func__,
+ *module_name);
+ exit(1);
+ }
+
+
+ while ((one_object = strsep(&object_name, " ")) != NULL) {
+ if (j >= npaths) {
+ fprintf(stderr, "%s: num_objs overflow on module "
+ "%s: this is a bug.\n", __func__,
+ *module_name);
+ exit(1);
+ }
+
+ module_paths[j++] = one_object;
+ }
+
+ module_paths[npaths] = NULL;
+
+ return module_paths;
+}
+
+/*
+ * Free an iterator. Can be called while iteration is underway, so even
+ * state that is freed at the end of iteration must be freed here too.
+ */
+void
+module_objnames_iter_free(struct module_objnames_iter *i)
+{
+ if (i == NULL)
+ return;
+ fclose(i->f);
+ free(i->line);
+ free(i);
+}
diff --git a/scripts/modules_builtin.h b/scripts/module_objnames.h
similarity index 64%
rename from scripts/modules_builtin.h
rename to scripts/module_objnames.h
index 5138792b42ef..94fa48498313 100644
--- a/scripts/modules_builtin.h
+++ b/scripts/module_objnames.h
@@ -1,8 +1,8 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
- * A simple modules.builtin.objs reader.
+ * A simple .tmp_module.objnames reader.
*
- * (C) 2014, 2022 Oracle, Inc. All rights reserved.
+ * (C) 2014, 2024 Oracle, Inc. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -10,8 +10,8 @@
* (at your option) any later version.
*/
-#ifndef _LINUX_MODULES_BUILTIN_H
-#define _LINUX_MODULES_BUILTIN_H
+#ifndef _LINUX_MODULE_OBJNAMES_H
+#define _LINUX_MODULE_OBJNAMES_H
#include <stdio.h>
#include <stddef.h>
@@ -19,17 +19,17 @@
/*
* modules.builtin.objs iteration state.
*/
-struct modules_builtin_iter {
+struct module_objnames_iter {
FILE *f;
char *line;
size_t line_size;
};
/*
- * Construct a modules_builtin.objs iterator.
+ * Construct a module_objnames.objs iterator.
*/
-struct modules_builtin_iter *
-modules_builtin_iter_new(const char *modules_builtin_file);
+struct module_objnames_iter *
+module_objnames_iter_new(const char *module_objnames_file);
/*
* Iterate, returning a new null-terminated array of object file names, and a
@@ -40,9 +40,9 @@ modules_builtin_iter_new(const char *modules_builtin_file);
*/
char ** __attribute__((__nonnull__))
-modules_builtin_iter_next(struct modules_builtin_iter *i, char **module_name);
+module_objnames_iter_next(struct module_objnames_iter *i, char **module_name);
void
-modules_builtin_iter_free(struct modules_builtin_iter *i);
+module_objnames_iter_free(struct module_objnames_iter *i);
#endif
diff --git a/scripts/modules_builtin.c b/scripts/modules_builtin.c
deleted file mode 100644
index df52932a4417..000000000000
--- a/scripts/modules_builtin.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A simple modules_builtin reader.
- *
- * (C) 2014, 2022 Oracle, Inc. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "modules_builtin.h"
-
-/*
- * Read a modules.builtin.objs file and translate it into a stream of
- * name / module-name pairs.
- */
-
-/*
- * Construct a modules.builtin.objs iterator.
- */
-struct modules_builtin_iter *
-modules_builtin_iter_new(const char *modules_builtin_file)
-{
- struct modules_builtin_iter *i;
-
- i = calloc(1, sizeof(struct modules_builtin_iter));
- if (i == NULL)
- return NULL;
-
- i->f = fopen(modules_builtin_file, "r");
-
- if (i->f == NULL) {
- fprintf(stderr, "Cannot open builtin module file %s: %s\n",
- modules_builtin_file, strerror(errno));
- return NULL;
- }
-
- return i;
-}
-
-/*
- * Iterate, returning a new null-terminated array of object file names, and a
- * new dynamically-allocated module name. (The module name passed in is freed.)
- *
- * The array of object file names should be freed by the caller: the strings it
- * points to are owned by the iterator, and should not be freed.
- */
-
-char ** __attribute__((__nonnull__))
-modules_builtin_iter_next(struct modules_builtin_iter *i, char **module_name)
-{
- size_t npaths = 1;
- char **module_paths;
- char *last_slash;
- char *last_dot;
- char *trailing_linefeed;
- char *object_name = i->line;
- char *dash;
- int composite = 0;
-
- /*
- * Read in all module entries, computing the suffixless, pathless name
- * of the module and building the next arrayful of object file names for
- * return.
- *
- * Modules can consist of multiple files: in this case, the portion
- * before the colon is the path to the module (as before): the portion
- * after the colon is a space-separated list of files that should be
- * considered part of this module. In this case, the portion before the
- * name is an "object file" that does not actually exist: it is merged
- * into built-in.a without ever being written out.
- *
- * All module names have - translated to _, to match what is done to the
- * names of the same things when built as modules.
- */
-
- /*
- * Reinvocation of exhausted iterator. Return NULL, once.
- */
-retry:
- if (getline(&i->line, &i->line_size, i->f) < 0) {
- if (ferror(i->f)) {
- fprintf(stderr, "Error reading from modules_builtin file:"
- " %s\n", strerror(errno));
- exit(1);
- }
- rewind(i->f);
- return NULL;
- }
-
- if (i->line[0] == '\0')
- goto retry;
-
- trailing_linefeed = strchr(i->line, '\n');
- if (trailing_linefeed != NULL)
- *trailing_linefeed = '\0';
-
- /*
- * Slice the line in two at the colon, if any. If there is anything
- * past the ': ', this is a composite module. (We allow for no colon
- * for robustness, even though one should always be present.)
- */
- if (strchr(i->line, ':') != NULL) {
- char *name_start;
-
- object_name = strchr(i->line, ':');
- *object_name = '\0';
- object_name++;
- name_start = object_name + strspn(object_name, " \n");
- if (*name_start != '\0') {
- composite = 1;
- object_name = name_start;
- }
- }
-
- /*
- * Figure out the module name.
- */
- last_slash = strrchr(i->line, '/');
- last_slash = (!last_slash) ? i->line :
- last_slash + 1;
- free(*module_name);
- *module_name = strdup(last_slash);
- dash = *module_name;
-
- while (dash != NULL) {
- dash = strchr(dash, '-');
- if (dash != NULL)
- *dash = '_';
- }
-
- last_dot = strrchr(*module_name, '.');
- if (last_dot != NULL)
- *last_dot = '\0';
-
- /*
- * Multifile separator? Object file names explicitly stated:
- * slice them up and shuffle them in.
- *
- * The array size may be an overestimate if any object file
- * names start or end with spaces (very unlikely) but cannot be
- * an underestimate. (Check for it anyway.)
- */
- if (composite) {
- char *one_object;
-
- for (npaths = 0, one_object = object_name;
- one_object != NULL;
- npaths++, one_object = strchr(one_object + 1, ' '));
- }
-
- module_paths = malloc((npaths + 1) * sizeof(char *));
- if (!module_paths) {
- fprintf(stderr, "%s: out of memory on module %s\n", __func__,
- *module_name);
- exit(1);
- }
-
- if (composite) {
- char *one_object;
- size_t i = 0;
-
- while ((one_object = strsep(&object_name, " ")) != NULL) {
- if (i >= npaths) {
- fprintf(stderr, "%s: num_objs overflow on module "
- "%s: this is a bug.\n", __func__,
- *module_name);
- exit(1);
- }
-
- module_paths[i++] = one_object;
- }
- } else
- module_paths[0] = i->line; /* untransformed module name */
-
- module_paths[npaths] = NULL;
-
- return module_paths;
-}
-
-/*
- * Free an iterator. Can be called while iteration is underway, so even
- * state that is freed at the end of iteration must be freed here too.
- */
-void
-modules_builtin_iter_free(struct modules_builtin_iter *i)
-{
- if (i == NULL)
- return;
- fclose(i->f);
- free(i->line);
- free(i);
-}
--
2.46.0.278.g36e3a12567
More information about the DTrace-devel
mailing list