diff options
Diffstat (limited to 'scripts/lib')
| -rwxr-xr-x | scripts/lib/jobserver.py | 149 | ||||
| -rw-r--r-- | scripts/lib/kdoc/kdoc_files.py | 5 | ||||
| -rw-r--r-- | scripts/lib/kdoc/kdoc_item.py | 3 | ||||
| -rw-r--r-- | scripts/lib/kdoc/kdoc_output.py | 85 | ||||
| -rw-r--r-- | scripts/lib/kdoc/kdoc_parser.py | 15 |
5 files changed, 246 insertions, 11 deletions
diff --git a/scripts/lib/jobserver.py b/scripts/lib/jobserver.py new file mode 100755 index 000000000000..a24f30ef4fa8 --- /dev/null +++ b/scripts/lib/jobserver.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# pylint: disable=C0103,C0209 +# +# + +""" +Interacts with the POSIX jobserver during the Kernel build time. + +A "normal" jobserver task, like the one initiated by a make subrocess would do: + + - open read/write file descriptors to communicate with the job server; + - ask for one slot by calling: + claim = os.read(reader, 1) + - when the job finshes, call: + os.write(writer, b"+") # os.write(writer, claim) + +Here, the goal is different: This script aims to get the remaining number +of slots available, using all of them to run a command which handle tasks in +parallel. To to that, it has a loop that ends only after there are no +slots left. It then increments the number by one, in order to allow a +call equivalent to make -j$((claim+1)), e.g. having a parent make creating +$claim child to do the actual work. + +The end goal here is to keep the total number of build tasks under the +limit established by the initial make -j$n_proc call. + +See: + https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +""" + +import errno +import os +import subprocess +import sys + +class JobserverExec: + """ + Claim all slots from make using POSIX Jobserver. + + The main methods here are: + - open(): reserves all slots; + - close(): method returns all used slots back to make; + - run(): executes a command setting PARALLELISM=<available slots jobs + 1> + """ + + def __init__(self): + """Initialize internal vars""" + self.claim = 0 + self.jobs = b"" + self.reader = None + self.writer = None + self.is_open = False + + def open(self): + """Reserve all available slots to be claimed later on""" + + if self.is_open: + return + + try: + # Fetch the make environment options. + flags = os.environ["MAKEFLAGS"] + # Look for "--jobserver=R,W" + # Note that GNU Make has used --jobserver-fds and --jobserver-auth + # so this handles all of them. + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + # If the MAKEFLAGS variable contains multiple instances of the + # --jobserver-auth= option, the last one is relevant. + fds = opts[-1].split("=", 1)[1] + + # Starting with GNU Make 4.4, named pipes are used for reader + # and writer. + # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134 + _, _, path = fds.partition("fifo:") + + if path: + self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + self.writer = os.open(path, os.O_WRONLY) + else: + self.reader, self.writer = [int(x) for x in fds.split(",", 1)] + # Open a private copy of reader to avoid setting nonblocking + # on an unexpecting process with the same reader fd. + self.reader = os.open("/proc/self/fd/%d" % (self.reader), + os.O_RDONLY | os.O_NONBLOCK) + + # Read out as many jobserver slots as possible + while True: + try: + slot = os.read(self.reader, 8) + self.jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop at the end of the jobserver queue. + break + # If something went wrong, give back the jobs. + if self.jobs: + os.write(self.writer, self.jobs) + raise e + + # Add a bump for our caller's reserveration, since we're just going + # to sit here blocked on our child. + self.claim = len(self.jobs) + 1 + + except (KeyError, IndexError, ValueError, OSError, IOError): + # Any missing environment strings or bad fds should result in just + # not being parallel. + self.claim = None + + self.is_open = True + + def close(self): + """Return all reserved slots to Jobserver""" + + if not self.is_open: + return + + # Return all the reserved slots. + if len(self.jobs): + os.write(self.writer, self.jobs) + + self.is_open = False + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + + def run(self, cmd, *args, **pwargs): + """ + Run a command setting PARALLELISM env variable to the number of + available job slots (claim) + 1, e.g. it will reserve claim slots + to do the actual build work, plus one to monitor its children. + """ + self.open() # Ensure that self.claim is set + + # We can only claim parallelism if there was a jobserver (i.e. a + # top-level "-jN" argument) and there were no other failures. Otherwise + # leave out the environment variable and let the child figure out what + # is best. + if self.claim: + os.environ["PARALLELISM"] = str(self.claim) + + return subprocess.call(cmd, *args, **pwargs) diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 9e09b45b02fa..061c033f32da 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -275,7 +275,10 @@ class KernelFiles(): self.config.log.warning("No kernel-doc for file %s", fname) continue - for arg in self.results[fname]: + symbols = self.results[fname] + self.out_style.set_symbols(symbols) + + for arg in symbols: m = self.out_msg(fname, arg.name, arg) if m is None: diff --git a/scripts/lib/kdoc/kdoc_item.py b/scripts/lib/kdoc/kdoc_item.py index b3b225764550..19805301cb2c 100644 --- a/scripts/lib/kdoc/kdoc_item.py +++ b/scripts/lib/kdoc/kdoc_item.py @@ -5,8 +5,9 @@ # class KdocItem: - def __init__(self, name, type, start_line, **other_stuff): + def __init__(self, name, fname, type, start_line, **other_stuff): self.name = name + self.fname = fname self.type = type self.declaration_start_line = start_line self.sections = {} diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index ea8914537ba0..58f115059e93 100644 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -215,6 +215,9 @@ class OutputFormat: # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. + def set_symbols(self, symbols): + """Get a list of all symbols from kernel_doc""" + def out_doc(self, fname, name, args): """Outputs a DOC block""" @@ -577,6 +580,7 @@ class ManFormat(OutputFormat): super().__init__() self.modulename = modulename + self.symbols = [] dt = None tstamp = os.environ.get("KBUILD_BUILD_TIMESTAMP") @@ -593,6 +597,69 @@ class ManFormat(OutputFormat): self.man_date = dt.strftime("%B %Y") + def arg_name(self, args, name): + """ + Return the name that will be used for the man page. + + As we may have the same name on different namespaces, + prepend the data type for all types except functions and typedefs. + + The doc section is special: it uses the modulename. + """ + + dtype = args.type + + if dtype == "doc": + return self.modulename + + if dtype in ["function", "typedef"]: + return name + + return f"{dtype} {name}" + + def set_symbols(self, symbols): + """ + Get a list of all symbols from kernel_doc. + + Man pages will uses it to add a SEE ALSO section with other + symbols at the same file. + """ + self.symbols = symbols + + def out_tail(self, fname, name, args): + """Adds a tail for all man pages""" + + # SEE ALSO section + self.data += f'.SH "SEE ALSO"' + "\n.PP\n" + self.data += (f"Kernel file \\fB{args.fname}\\fR\n") + if len(self.symbols) >= 2: + cur_name = self.arg_name(args, name) + + related = [] + for arg in self.symbols: + out_name = self.arg_name(arg, arg.name) + + if cur_name == out_name: + continue + + related.append(f"\\fB{out_name}\\fR(9)") + + self.data += ",\n".join(related) + "\n" + + # TODO: does it make sense to add other sections? Maybe + # REPORTING ISSUES? LICENSE? + + def msg(self, fname, name, args): + """ + Handles a single entry from kernel-doc parser. + + Add a tail at the end of man pages output. + """ + super().msg(fname, name, args) + self.out_tail(fname, name, args) + + return self.data + def output_highlight(self, block): """ Outputs a C symbol that may require being highlighted with @@ -618,7 +685,9 @@ class ManFormat(OutputFormat): if not self.check_doc(name, args): return - self.data += f'.TH "{self.modulename}" 9 "{self.modulename}" "{self.man_date}" "API Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" for section, text in args.sections.items(): self.data += f'.SH "{section}"' + "\n" @@ -627,7 +696,9 @@ class ManFormat(OutputFormat): def out_function(self, fname, name, args): """output function in man""" - self.data += f'.TH "{name}" 9 "{name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{name}" 9 "{out_name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -671,7 +742,9 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_enum(self, fname, name, args): - self.data += f'.TH "{self.modulename}" 9 "enum {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + out_name = self.arg_name(args, name) + + self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"enum {name} \\- {args['purpose']}\n" @@ -703,8 +776,9 @@ class ManFormat(OutputFormat): def out_typedef(self, fname, name, args): module = self.modulename purpose = args.get('purpose') + out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"typedef {name} \\- {purpose}\n" @@ -717,8 +791,9 @@ class ManFormat(OutputFormat): module = self.modulename purpose = args.get('purpose') definition = args.get('definition') + out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{args.type} {name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"{args.type} {name} \\- {purpose}\n" diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 2376f180b1fa..6e5c115cbdf3 100644 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -254,8 +254,9 @@ SECTION_DEFAULT = "Description" # default section class KernelEntry: - def __init__(self, config, ln): + def __init__(self, config, fname, ln): self.config = config + self.fname = fname self._contents = [] self.prototype = "" @@ -350,6 +351,7 @@ class KernelEntry: self.section = SECTION_DEFAULT self._contents = [] +python_warning = False class KernelDoc: """ @@ -383,9 +385,13 @@ class KernelDoc: # We need Python 3.7 for its "dicts remember the insertion # order" guarantee # - if sys.version_info.major == 3 and sys.version_info.minor < 7: + global python_warning + if (not python_warning and + sys.version_info.major == 3 and sys.version_info.minor < 7): + self.emit_msg(0, 'Python 3.7 or later is required for correct results') + python_warning = True def emit_msg(self, ln, msg, warning=True): """Emit a message""" @@ -417,7 +423,8 @@ class KernelDoc: The actual output and output filters will be handled elsewhere """ - item = KdocItem(name, dtype, self.entry.declaration_start_line, **args) + item = KdocItem(name, self.fname, dtype, + self.entry.declaration_start_line, **args) item.warnings = self.entry.warnings # Drop empty sections @@ -440,7 +447,7 @@ class KernelDoc: variables used by the state machine. """ - self.entry = KernelEntry(self.config, ln) + self.entry = KernelEntry(self.config, self.fname, ln) # State flags self.state = state.NORMAL |