From bb979965008793635a630ecdc7a3869c2fccd283 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:22 +0200 Subject: docs: kernel_include.py: Update its coding style With the help of tools like black, pylint, autopep8 and flake, improve the code style in preparation for further changes. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/f64c3af47fdfd632bb5f8eb88e3c7d94b0b84a66.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 100 ++++++++++++++++----------------- 1 file changed, 47 insertions(+), 53 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 1e566e87ebcd..1212786ac516 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -# -*- coding: utf-8; mode: python -*- # SPDX-License-Identifier: GPL-2.0 -# pylint: disable=R0903, C0330, R0914, R0912, E0401 +# pylint: disable=R0903, R0912, R0914, R0915, C0209,W0707 """ kernel-include @@ -40,41 +39,38 @@ from docutils.parsers.rst import directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines from docutils.parsers.rst.directives.misc import Include -__version__ = '1.0' +__version__ = "1.0" + # ============================================================================== def setup(app): -# ============================================================================== - + """Setup Sphinx exension""" app.add_directive("kernel-include", KernelInclude) - return dict( - version = __version__, - parallel_read_safe = True, - parallel_write_safe = True - ) + return { + "version": __version__, + "parallel_read_safe": True, + "parallel_write_safe": True, + } + # ============================================================================== class KernelInclude(Include): -# ============================================================================== - """KernelInclude (``kernel-include``) directive""" def run(self): env = self.state.document.settings.env - path = os.path.realpath( - os.path.expandvars(self.arguments[0])) + path = os.path.realpath(os.path.expandvars(self.arguments[0])) # to get a bit security back, prohibit /etc: if path.startswith(os.sep + "etc"): - raise self.severe( - 'Problems with "%s" directive, prohibited path: %s' - % (self.name, path)) + raise self.severe('Problems with "%s" directive, prohibited path: %s' % + (self.name, path)) self.arguments[0] = path env.note_dependency(os.path.abspath(path)) - #return super(KernelInclude, self).run() # won't work, see HINTs in _run() + # return super(KernelInclude, self).run() # won't work, see HINTs in _run() return self._run() def _run(self): @@ -87,41 +83,39 @@ class KernelInclude(Include): if not self.state.document.settings.file_insertion_enabled: raise self.warning('"%s" directive disabled.' % self.name) - source = self.state_machine.input_lines.source( - self.lineno - self.state_machine.input_offset - 1) + source = self.state_machine.input_lines.source(self.lineno - + self.state_machine.input_offset - 1) source_dir = os.path.dirname(os.path.abspath(source)) path = directives.path(self.arguments[0]) - if path.startswith('<') and path.endswith('>'): + if path.startswith("<") and path.endswith(">"): path = os.path.join(self.standard_include_path, path[1:-1]) path = os.path.normpath(os.path.join(source_dir, path)) # HINT: this is the only line I had to change / commented out: - #path = utils.relative_path(None, path) + # path = utils.relative_path(None, path) - encoding = self.options.get( - 'encoding', self.state.document.settings.input_encoding) - e_handler=self.state.document.settings.input_encoding_error_handler - tab_width = self.options.get( - 'tab-width', self.state.document.settings.tab_width) + encoding = self.options.get("encoding", + self.state.document.settings.input_encoding) + e_handler = self.state.document.settings.input_encoding_error_handler + tab_width = self.options.get("tab-width", + self.state.document.settings.tab_width) try: self.state.document.settings.record_dependencies.add(path) - include_file = io.FileInput(source_path=path, - encoding=encoding, + include_file = io.FileInput(source_path=path, encoding=encoding, error_handler=e_handler) - except UnicodeEncodeError as error: + except UnicodeEncodeError: raise self.severe('Problems with "%s" directive path:\n' 'Cannot encode input file path "%s" ' - '(wrong locale?).' % - (self.name, SafeString(path))) + "(wrong locale?)." % (self.name, SafeString(path))) except IOError as error: - raise self.severe('Problems with "%s" directive path:\n%s.' % - (self.name, ErrorString(error))) - startline = self.options.get('start-line', None) - endline = self.options.get('end-line', None) + raise self.severe('Problems with "%s" directive path:\n%s.' + % (self.name, ErrorString(error))) + startline = self.options.get("start-line", None) + endline = self.options.get("end-line", None) try: if startline or (endline is not None): lines = include_file.readlines() - rawtext = ''.join(lines[startline:endline]) + rawtext = "".join(lines[startline:endline]) else: rawtext = include_file.read() except UnicodeError as error: @@ -129,43 +123,43 @@ class KernelInclude(Include): (self.name, ErrorString(error))) # start-after/end-before: no restrictions on newlines in match-text, # and no restrictions on matching inside lines vs. line boundaries - after_text = self.options.get('start-after', None) + after_text = self.options.get("start-after", None) if after_text: # skip content in rawtext before *and incl.* a matching text after_index = rawtext.find(after_text) if after_index < 0: raise self.severe('Problem with "start-after" option of "%s" ' - 'directive:\nText not found.' % self.name) - rawtext = rawtext[after_index + len(after_text):] - before_text = self.options.get('end-before', None) + "directive:\nText not found." % self.name) + rawtext = rawtext[after_index + len(after_text) :] + before_text = self.options.get("end-before", None) if before_text: # skip content in rawtext after *and incl.* a matching text before_index = rawtext.find(before_text) if before_index < 0: raise self.severe('Problem with "end-before" option of "%s" ' - 'directive:\nText not found.' % self.name) + "directive:\nText not found." % self.name) rawtext = rawtext[:before_index] include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) - if 'literal' in self.options: + if "literal" in self.options: # Convert tabs to spaces, if `tab_width` is positive. if tab_width >= 0: text = rawtext.expandtabs(tab_width) else: text = rawtext literal_block = nodes.literal_block(rawtext, source=path, - classes=self.options.get('class', [])) + classes=self.options.get("class", []) + ) literal_block.line = 1 self.add_name(literal_block) - if 'number-lines' in self.options: + if "number-lines" in self.options: try: - startline = int(self.options['number-lines'] or 1) + startline = int(self.options["number-lines"] or 1) except ValueError: - raise self.error(':number-lines: with non-integer ' - 'start value') + raise self.error(":number-lines: with non-integer start value") endline = startline + len(include_lines) - if text.endswith('\n'): + if text.endswith("\n"): text = text[:-1] tokens = NumberLines([([], text)], startline, endline) for classes, value in tokens: @@ -177,12 +171,12 @@ class KernelInclude(Include): else: literal_block += nodes.Text(text, text) return [literal_block] - if 'code' in self.options: - self.options['source'] = path + if "code" in self.options: + self.options["source"] = path codeblock = CodeBlock(self.name, - [self.options.pop('code')], # arguments + [self.options.pop("code")], # arguments self.options, - include_lines, # content + include_lines, # content self.lineno, self.content_offset, self.block_text, -- cgit v1.2.3 From 0cb6aee3584604ceecdbc3bcc00d017f9a827873 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:23 +0200 Subject: docs: kernel_include.py: allow cross-reference generation kernel_include extension was originally designed to be used by the media comprehensive uAPI documentation, where, instead of simpler kernel-doc markups, the uAPI documentation is enriched with a larger text, with images, complex tables, graphs, etc. There, we wanted to include the much simpler yet documented .h file. This extension is needed to include files from other parts of the Kernel tree outside Documentation, because the original Sphinx include tag doesn't allow going outside of the directory passed via sphinx-build command line. Yet, the cross-references themselves to the full documentation were using a perl script to create cross-references against the comprehensive documentation. As the perl script is now converted to Phython and there is a Python class producing an include-compatible output with cross references, add two optional arguments to kernel_include.py: 1. :generate-cross-refs: If present, instead of reading the file, it calls ParseDataStructs() class, which converts C data structures into cross-references to be linked to ReST files containing a more comprehensive documentation; Don't use it together with :start-line: and/or :end-line:, as filtering input file line range is currently not supported. 2. :exception-file: Used together with :generate-cross-refs:. Points to a file containing rules to ignore C data structs or to use a different reference name, optionally using a different reference type. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/efc39c8e54a2056ae2fdb94d5006fcb19e227198.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 94 ++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 20 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 1212786ac516..fc37e6fa9d96 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -25,6 +25,24 @@ Substrings of the form $name or ${name} are replaced by the value of environment variable name. Malformed variable names and references to non-existing variables are left unchanged. + + This extension overrides Sphinx include directory, adding two extra + arguments: + + 1. :generate-cross-refs: + + If present, instead of reading the file, it calls ParseDataStructs() + class, which converts C data structures into cross-references to + be linked to ReST files containing a more comprehensive documentation; + + Don't use it together with :start-line: and/or :end-line:, as + filtering input file line range is currently not supported. + + 2. :exception-file: + + Used together with :generate-cross-refs:. Points to a file containing + rules to ignore C data structs or to use a different reference name, + optionally using a different reference type. """ # ============================================================================== @@ -32,6 +50,7 @@ # ============================================================================== import os.path +import sys from docutils import io, nodes, statemachine from docutils.utils.error_reporting import SafeString, ErrorString @@ -39,6 +58,11 @@ from docutils.parsers.rst import directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines from docutils.parsers.rst.directives.misc import Include +srctree = os.path.abspath(os.environ["srctree"]) +sys.path.insert(0, os.path.join(srctree, "tools/docs/lib")) + +from parse_data_structs import ParseDataStructs + __version__ = "1.0" @@ -57,6 +81,14 @@ def setup(app): class KernelInclude(Include): """KernelInclude (``kernel-include``) directive""" + # Add extra options + option_spec = Include.option_spec.copy() + + option_spec.update({ + 'generate-cross-refs': directives.flag, + 'exception-file': directives.unchanged, + }) + def run(self): env = self.state.document.settings.env path = os.path.realpath(os.path.expandvars(self.arguments[0])) @@ -99,28 +131,49 @@ class KernelInclude(Include): e_handler = self.state.document.settings.input_encoding_error_handler tab_width = self.options.get("tab-width", self.state.document.settings.tab_width) - try: - self.state.document.settings.record_dependencies.add(path) - include_file = io.FileInput(source_path=path, encoding=encoding, - error_handler=e_handler) - except UnicodeEncodeError: - raise self.severe('Problems with "%s" directive path:\n' - 'Cannot encode input file path "%s" ' - "(wrong locale?)." % (self.name, SafeString(path))) - except IOError as error: - raise self.severe('Problems with "%s" directive path:\n%s.' - % (self.name, ErrorString(error))) startline = self.options.get("start-line", None) endline = self.options.get("end-line", None) - try: - if startline or (endline is not None): - lines = include_file.readlines() - rawtext = "".join(lines[startline:endline]) - else: - rawtext = include_file.read() - except UnicodeError as error: - raise self.severe('Problem with "%s" directive:\n%s' % - (self.name, ErrorString(error))) + + # Get optional arguments to related to cross-references generation + if 'generate-cross-refs' in self.options: + parser = ParseDataStructs() + parser.parse_file(path) + + exceptions_file = self.options.get('exception-file') + if exceptions_file: + exceptions_file = os.path.join(source_dir, exceptions_file) + parser.process_exceptions(exceptions_file) + + title = os.path.basename(path) + rawtext = parser.gen_output() + if startline or endline: + raise self.severe('generate-cross-refs can\'t be used together with "start-line" or "end-line"') + + if "code" not in self.options: + rawtext = ".. parsed-literal::\n\n" + rawtext + else: + try: + self.state.document.settings.record_dependencies.add(path) + include_file = io.FileInput(source_path=path, encoding=encoding, + error_handler=e_handler) + except UnicodeEncodeError: + raise self.severe('Problems with "%s" directive path:\n' + 'Cannot encode input file path "%s" ' + "(wrong locale?)." % (self.name, SafeString(path))) + except IOError as error: + raise self.severe('Problems with "%s" directive path:\n%s.' + % (self.name, ErrorString(error))) + + try: + if startline or (endline is not None): + lines = include_file.readlines() + rawtext = "".join(lines[startline:endline]) + else: + rawtext = include_file.read() + except UnicodeError as error: + raise self.severe('Problem with "%s" directive:\n%s' % + (self.name, ErrorString(error))) + # start-after/end-before: no restrictions on newlines in match-text, # and no restrictions on matching inside lines vs. line boundaries after_text = self.options.get("start-after", None) @@ -171,6 +224,7 @@ class KernelInclude(Include): else: literal_block += nodes.Text(text, text) return [literal_block] + if "code" in self.options: self.options["source"] = path codeblock = CodeBlock(self.name, -- cgit v1.2.3 From 39f5f2fa8c959eb897416e928d12f6aec1e7d5e4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:24 +0200 Subject: docs: kernel_include.py: generate warnings for broken refs In the past, Sphinx used to warn about broken references. That's basically the rationale for adding media uAPI files: to get warnings about missed symbols. This is not true anymore. So, we need to explicitly check them after doctree-resolved event. While here, move setup() to the end, to make it closer to what we do on other extensions. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/73be9a198746421687e2eee916ccf8bf67980b7d.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 108 +++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 19 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index fc37e6fa9d96..0a3e5377dd1e 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -26,7 +26,7 @@ environment variable name. Malformed variable names and references to non-existing variables are left unchanged. - This extension overrides Sphinx include directory, adding two extra + This extension overrides Sphinx include directory, adding some extra arguments: 1. :generate-cross-refs: @@ -35,14 +35,20 @@ class, which converts C data structures into cross-references to be linked to ReST files containing a more comprehensive documentation; - Don't use it together with :start-line: and/or :end-line:, as - filtering input file line range is currently not supported. - 2. :exception-file: - Used together with :generate-cross-refs:. Points to a file containing - rules to ignore C data structs or to use a different reference name, - optionally using a different reference type. + Used together with :generate-cross-refs + + Points to a file containing rules to ignore C data structs or to + use a different reference name, optionally using a different + reference type. + + 3. :warn-broken: + + Used together with :generate-cross-refs: + + Detect if the auto-generated cross references doesn't exist. + """ # ============================================================================== @@ -50,6 +56,7 @@ # ============================================================================== import os.path +import re import sys from docutils import io, nodes, statemachine @@ -58,23 +65,18 @@ from docutils.parsers.rst import directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines from docutils.parsers.rst.directives.misc import Include +from sphinx.util import logging + srctree = os.path.abspath(os.environ["srctree"]) sys.path.insert(0, os.path.join(srctree, "tools/docs/lib")) from parse_data_structs import ParseDataStructs __version__ = "1.0" +logger = logging.getLogger(__name__) - -# ============================================================================== -def setup(app): - """Setup Sphinx exension""" - app.add_directive("kernel-include", KernelInclude) - return { - "version": __version__, - "parallel_read_safe": True, - "parallel_write_safe": True, - } +RE_DOMAIN_REF = re.compile(r'\\ :(ref|c:type|c:func):`([^<`]+)(?:<([^>]+)>)?`\\') +RE_SIMPLE_REF = re.compile(r'`([^`]+)`') # ============================================================================== @@ -86,6 +88,7 @@ class KernelInclude(Include): option_spec.update({ 'generate-cross-refs': directives.flag, + 'warn-broken': directives.flag, 'exception-file': directives.unchanged, }) @@ -103,9 +106,9 @@ class KernelInclude(Include): env.note_dependency(os.path.abspath(path)) # return super(KernelInclude, self).run() # won't work, see HINTs in _run() - return self._run() + return self._run(env) - def _run(self): + def _run(self, env): """Include a file as part of the content of this reST file.""" # HINT: I had to copy&paste the whole Include.run method. I'am not happy @@ -151,6 +154,10 @@ class KernelInclude(Include): if "code" not in self.options: rawtext = ".. parsed-literal::\n\n" + rawtext + + # Store references on a symbol dict to be used at check time + if 'warn-broken' in self.options: + env._xref_files.add(path) else: try: self.state.document.settings.record_dependencies.add(path) @@ -239,3 +246,66 @@ class KernelInclude(Include): return codeblock.run() self.state_machine.insert_input(include_lines, path) return [] + +# ============================================================================== + +reported = set() + +def check_missing_refs(app, env, node, contnode): + """Check broken refs for the files it creates xrefs""" + if not node.source: + return None + + try: + xref_files = env._xref_files + except AttributeError: + logger.critical("FATAL: _xref_files not initialized!") + raise + + # Only show missing references for kernel-include reference-parsed files + if node.source not in xref_files: + return None + + target = node.get('reftarget', '') + domain = node.get('refdomain', 'std') + reftype = node.get('reftype', '') + + msg = f"can't link to: {domain}:{reftype}:: {target}" + + # Don't duplicate warnings + data = (node.source, msg) + if data in reported: + return None + reported.add(data) + + logger.warning(msg, location=node, type='ref', subtype='missing') + + return None + +def merge_xref_info(app, env, docnames, other): + """ + As each process modify env._xref_files, we need to merge them back. + """ + if not hasattr(other, "_xref_files"): + return + env._xref_files.update(getattr(other, "_xref_files", set())) + +def init_xref_docs(app, env, docnames): + """Initialize a list of files that we're generating cross references¨""" + app.env._xref_files = set() + +# ============================================================================== + +def setup(app): + """Setup Sphinx exension""" + + app.connect("env-before-read-docs", init_xref_docs) + app.connect("env-merge-info", merge_xref_info) + app.add_directive("kernel-include", KernelInclude) + app.connect("missing-reference", check_missing_refs) + + return { + "version": __version__, + "parallel_read_safe": True, + "parallel_write_safe": True, + } -- cgit v1.2.3 From 012e00dda347e72a6079c6d21f0c8f97333cc477 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:25 +0200 Subject: docs: kernel_include.py: move rawtext logic to separate functions The run function is too complex. merge run() and _run() into a single function and move the read logic to separate functions. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/04776a94c85b6c931c198a149f08b299c9f571a3.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 82 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 39 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 0a3e5377dd1e..ef86ee9e79d6 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -92,7 +92,47 @@ class KernelInclude(Include): 'exception-file': directives.unchanged, }) + def read_rawtext(self, path, encoding): + """Read and process file content with error handling""" + try: + self.state.document.settings.record_dependencies.add(path) + include_file = io.FileInput(source_path=path, + encoding=encoding, + error_handler=self.state.document.settings.input_encoding_error_handler) + except UnicodeEncodeError: + raise self.severe('Problems with directive path:\n' + 'Cannot encode input file path "%s" ' + '(wrong locale?).' % SafeString(path)) + except IOError as error: + raise self.severe('Problems with directive path:\n%s.' % ErrorString(error)) + + try: + return include_file.read() + except UnicodeError as error: + raise self.severe('Problem with directive:\n%s' % ErrorString(error)) + + def read_rawtext_with_xrefs(self, env, path): + parser = ParseDataStructs() + parser.parse_file(path) + + if 'exception-file' in self.options: + source_dir = os.path.dirname(os.path.abspath( + self.state_machine.input_lines.source( + self.lineno - self.state_machine.input_offset - 1))) + exceptions_file = os.path.join(source_dir, self.options['exception-file']) + parser.process_exceptions(exceptions_file) + + if self.options.get("start-line") or self.options.get("end-line"): + raise self.severe('generate-cross-refs can\'t be used with "start-line" or "end-line"') + + # Store references on a symbol dict to be used at check time + if 'warn-broken' in self.options: + env._xref_files.add(path) + + return parser.gen_output() + def run(self): + """Include a file as part of the content of this reST file.""" env = self.state.document.settings.env path = os.path.realpath(os.path.expandvars(self.arguments[0])) @@ -105,12 +145,6 @@ class KernelInclude(Include): env.note_dependency(os.path.abspath(path)) - # return super(KernelInclude, self).run() # won't work, see HINTs in _run() - return self._run(env) - - def _run(self, env): - """Include a file as part of the content of this reST file.""" - # HINT: I had to copy&paste the whole Include.run method. I'am not happy # with this, but due to security reasons, the Include.run method does # not allow absolute or relative pathnames pointing to locations *above* @@ -139,47 +173,17 @@ class KernelInclude(Include): # Get optional arguments to related to cross-references generation if 'generate-cross-refs' in self.options: - parser = ParseDataStructs() - parser.parse_file(path) - - exceptions_file = self.options.get('exception-file') - if exceptions_file: - exceptions_file = os.path.join(source_dir, exceptions_file) - parser.process_exceptions(exceptions_file) + rawtext = self.read_rawtext_with_xrefs(env, path) title = os.path.basename(path) - rawtext = parser.gen_output() + if startline or endline: raise self.severe('generate-cross-refs can\'t be used together with "start-line" or "end-line"') if "code" not in self.options: rawtext = ".. parsed-literal::\n\n" + rawtext - - # Store references on a symbol dict to be used at check time - if 'warn-broken' in self.options: - env._xref_files.add(path) else: - try: - self.state.document.settings.record_dependencies.add(path) - include_file = io.FileInput(source_path=path, encoding=encoding, - error_handler=e_handler) - except UnicodeEncodeError: - raise self.severe('Problems with "%s" directive path:\n' - 'Cannot encode input file path "%s" ' - "(wrong locale?)." % (self.name, SafeString(path))) - except IOError as error: - raise self.severe('Problems with "%s" directive path:\n%s.' - % (self.name, ErrorString(error))) - - try: - if startline or (endline is not None): - lines = include_file.readlines() - rawtext = "".join(lines[startline:endline]) - else: - rawtext = include_file.read() - except UnicodeError as error: - raise self.severe('Problem with "%s" directive:\n%s' % - (self.name, ErrorString(error))) + rawtext = self.read_rawtext(path, encoding) # start-after/end-before: no restrictions on newlines in match-text, # and no restrictions on matching inside lines vs. line boundaries -- cgit v1.2.3 From 3f7f3d494119170dc6636228e8425048effa2931 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:26 +0200 Subject: docs: kernel_include.py: move range logic to a separate function Cleanup run() function by moving the range logic to a separate function. Here, I ended checking the current Sphinx implementation, as it has some extra logic for the range check. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/12fa2204a9e7e309ae4b8694a37ebad9327ca634.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 51 ++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 18 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index ef86ee9e79d6..c5f4f34e22cb 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -131,6 +131,38 @@ class KernelInclude(Include): return parser.gen_output() + def apply_range(self, rawtext): + # Get to-be-included content + startline = self.options.get('start-line', None) + endline = self.options.get('end-line', None) + try: + if startline or (endline is not None): + lines = rawtext.splitlines() + rawtext = '\n'.join(lines[startline:endline]) + except UnicodeError as error: + raise self.severe(f'Problem with "{self.name}" directive:\n' + + io.error_string(error)) + # start-after/end-before: no restrictions on newlines in match-text, + # and no restrictions on matching inside lines vs. line boundaries + after_text = self.options.get("start-after", None) + if after_text: + # skip content in rawtext before *and incl.* a matching text + after_index = rawtext.find(after_text) + if after_index < 0: + raise self.severe('Problem with "start-after" option of "%s" ' + "directive:\nText not found." % self.name) + rawtext = rawtext[after_index + len(after_text) :] + before_text = self.options.get("end-before", None) + if before_text: + # skip content in rawtext after *and incl.* a matching text + before_index = rawtext.find(before_text) + if before_index < 0: + raise self.severe('Problem with "end-before" option of "%s" ' + "directive:\nText not found." % self.name) + rawtext = rawtext[:before_index] + + return rawtext + def run(self): """Include a file as part of the content of this reST file.""" env = self.state.document.settings.env @@ -185,24 +217,7 @@ class KernelInclude(Include): else: rawtext = self.read_rawtext(path, encoding) - # start-after/end-before: no restrictions on newlines in match-text, - # and no restrictions on matching inside lines vs. line boundaries - after_text = self.options.get("start-after", None) - if after_text: - # skip content in rawtext before *and incl.* a matching text - after_index = rawtext.find(after_text) - if after_index < 0: - raise self.severe('Problem with "start-after" option of "%s" ' - "directive:\nText not found." % self.name) - rawtext = rawtext[after_index + len(after_text) :] - before_text = self.options.get("end-before", None) - if before_text: - # skip content in rawtext after *and incl.* a matching text - before_index = rawtext.find(before_text) - if before_index < 0: - raise self.severe('Problem with "end-before" option of "%s" ' - "directive:\nText not found." % self.name) - rawtext = rawtext[:before_index] + rawtext = self.apply_range(rawtext) include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) -- cgit v1.2.3 From 67faed5d213d87f3b5bdfc78a12d1d9d3e7aac46 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:27 +0200 Subject: docs: kernel_include.py: remove range restriction for gen docs Originally, parse-readers were generating an output where the first two lines were setting a literal block. The script now gets only the actual parsed data without that, so it is now safe to allow start-line and end-line parameters to be handled. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5dff693860a6a3faade15c24abdc380f09db468d.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index c5f4f34e22cb..4cdd1c77982e 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -122,9 +122,6 @@ class KernelInclude(Include): exceptions_file = os.path.join(source_dir, self.options['exception-file']) parser.process_exceptions(exceptions_file) - if self.options.get("start-line") or self.options.get("end-line"): - raise self.severe('generate-cross-refs can\'t be used with "start-line" or "end-line"') - # Store references on a symbol dict to be used at check time if 'warn-broken' in self.options: env._xref_files.add(path) @@ -209,9 +206,6 @@ class KernelInclude(Include): title = os.path.basename(path) - if startline or endline: - raise self.severe('generate-cross-refs can\'t be used together with "start-line" or "end-line"') - if "code" not in self.options: rawtext = ".. parsed-literal::\n\n" + rawtext else: -- cgit v1.2.3 From 9be2a5c3c8b7aad78341677002c428c996cc9570 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:28 +0200 Subject: docs: kernel_include.py: move code and literal functions Simplify run() even more by moving the code which handles with code and literal blocks to their own functions. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/78d08dfa3f08adabc30bf93b8a1cde4e19b7bd41.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 102 +++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 42 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 4cdd1c77982e..0909eb3a07ea 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -160,6 +160,52 @@ class KernelInclude(Include): return rawtext + def literal(self, path, tab_width, rawtext): + """Output a literal block""" + + # Convert tabs to spaces, if `tab_width` is positive. + if tab_width >= 0: + text = rawtext.expandtabs(tab_width) + else: + text = rawtext + literal_block = nodes.literal_block(rawtext, source=path, + classes=self.options.get("class", [])) + literal_block.line = 1 + self.add_name(literal_block) + if "number-lines" in self.options: + try: + startline = int(self.options["number-lines"] or 1) + except ValueError: + raise self.error(":number-lines: with non-integer start value") + endline = startline + len(include_lines) + if text.endswith("\n"): + text = text[:-1] + tokens = NumberLines([([], text)], startline, endline) + for classes, value in tokens: + if classes: + literal_block += nodes.inline(value, value, + classes=classes) + else: + literal_block += nodes.Text(value, value) + else: + literal_block += nodes.Text(text, text) + return [literal_block] + + def code(self, path, include_lines): + """Output a code block""" + + self.options["source"] = path + codeblock = CodeBlock(self.name, + [self.options.pop("code")], # arguments + self.options, + include_lines, + self.lineno, + self.content_offset, + self.block_text, + self.state, + self.state_machine) + return codeblock.run() + def run(self): """Include a file as part of the content of this reST file.""" env = self.state.document.settings.env @@ -200,6 +246,13 @@ class KernelInclude(Include): startline = self.options.get("start-line", None) endline = self.options.get("end-line", None) + if "literal" in self.options: + ouptut_type = "literal" + elif "code" in self.options: + ouptut_type = "code" + else: + ouptut_type = "normal" + # Get optional arguments to related to cross-references generation if 'generate-cross-refs' in self.options: rawtext = self.read_rawtext_with_xrefs(env, path) @@ -213,50 +266,15 @@ class KernelInclude(Include): rawtext = self.apply_range(rawtext) + if ouptut_type == "literal": + return self.literal(path, tab_width, rawtext) + include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) - if "literal" in self.options: - # Convert tabs to spaces, if `tab_width` is positive. - if tab_width >= 0: - text = rawtext.expandtabs(tab_width) - else: - text = rawtext - literal_block = nodes.literal_block(rawtext, source=path, - classes=self.options.get("class", []) - ) - literal_block.line = 1 - self.add_name(literal_block) - if "number-lines" in self.options: - try: - startline = int(self.options["number-lines"] or 1) - except ValueError: - raise self.error(":number-lines: with non-integer start value") - endline = startline + len(include_lines) - if text.endswith("\n"): - text = text[:-1] - tokens = NumberLines([([], text)], startline, endline) - for classes, value in tokens: - if classes: - literal_block += nodes.inline(value, value, - classes=classes) - else: - literal_block += nodes.Text(value, value) - else: - literal_block += nodes.Text(text, text) - return [literal_block] - - if "code" in self.options: - self.options["source"] = path - codeblock = CodeBlock(self.name, - [self.options.pop("code")], # arguments - self.options, - include_lines, # content - self.lineno, - self.content_offset, - self.block_text, - self.state, - self.state_machine) - return codeblock.run() + + if ouptut_type == "code": + return self.code(path, include_lines) + self.state_machine.insert_input(include_lines, path) return [] -- cgit v1.2.3 From e4d91787deffffa64abd397315d4f7e4a3cf02f3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:29 +0200 Subject: docs: kernel_include.py: add support to generate a TOC table When generate-cross-refs is used, instead of just implementing the default of generating a literal block, we can also generate a ReST file as a TOC. The advantage is that, by being a ReST file, missing references will point to the place inside the header file that has the broken link. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/c0d32cd1ef94017e05984b0a38bd2516f7db21e2.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 0909eb3a07ea..79682408105e 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -89,6 +89,7 @@ class KernelInclude(Include): option_spec.update({ 'generate-cross-refs': directives.flag, 'warn-broken': directives.flag, + 'toc': directives.flag, 'exception-file': directives.unchanged, }) @@ -111,7 +112,7 @@ class KernelInclude(Include): except UnicodeError as error: raise self.severe('Problem with directive:\n%s' % ErrorString(error)) - def read_rawtext_with_xrefs(self, env, path): + def read_rawtext_with_xrefs(self, env, path, output_type): parser = ParseDataStructs() parser.parse_file(path) @@ -126,7 +127,10 @@ class KernelInclude(Include): if 'warn-broken' in self.options: env._xref_files.add(path) - return parser.gen_output() + if output_type == "toc": + return parser.gen_toc() + + return ".. parsed-literal::\n\n" + parser.gen_output() def apply_range(self, rawtext): # Get to-be-included content @@ -243,39 +247,43 @@ class KernelInclude(Include): e_handler = self.state.document.settings.input_encoding_error_handler tab_width = self.options.get("tab-width", self.state.document.settings.tab_width) - startline = self.options.get("start-line", None) - endline = self.options.get("end-line", None) if "literal" in self.options: - ouptut_type = "literal" + output_type = "literal" elif "code" in self.options: - ouptut_type = "code" + output_type = "code" else: - ouptut_type = "normal" + output_type = "rst" # Get optional arguments to related to cross-references generation - if 'generate-cross-refs' in self.options: - rawtext = self.read_rawtext_with_xrefs(env, path) + if "generate-cross-refs" in self.options: + if "toc" in self.options: + output_type = "toc" - title = os.path.basename(path) + rawtext = self.read_rawtext_with_xrefs(env, path, output_type) + + # When :generate-cross-refs: is used, the input is always a C + # file, so it has to be handled as a parsed-literal + if output_type == "rst": + output_type = "literal" - if "code" not in self.options: - rawtext = ".. parsed-literal::\n\n" + rawtext + title = os.path.basename(path) else: rawtext = self.read_rawtext(path, encoding) rawtext = self.apply_range(rawtext) - if ouptut_type == "literal": + if output_type == "literal": return self.literal(path, tab_width, rawtext) include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) - if ouptut_type == "code": + if output_type == "code": return self.code(path, include_lines) self.state_machine.insert_input(include_lines, path) + return [] # ============================================================================== -- cgit v1.2.3 From 4ad9cabc34d1b45ef77fa9c70e446658f6f5934b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:30 +0200 Subject: docs: kernel_include.py: append line numbers to better report errors It is best to point to the original line of code that generated an error than to point to the beginning of a directive. Add support for it. It should be noticed that this won't work for literal or code blocks, as Sphinx will ignore it, pointing to the beginning of the directive. Yet, when the output is known to be in ReST format, like on TOC, this makes the error a lot more easier to be handled. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a0953af8b71e64aaf2e0ba4593ad39e19587d50a.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 81 ++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 37 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 79682408105e..90ed8428f776 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -60,6 +60,7 @@ import re import sys from docutils import io, nodes, statemachine +from docutils.statemachine import ViewList from docutils.utils.error_reporting import SafeString, ErrorString from docutils.parsers.rst import directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines @@ -112,7 +113,14 @@ class KernelInclude(Include): except UnicodeError as error: raise self.severe('Problem with directive:\n%s' % ErrorString(error)) - def read_rawtext_with_xrefs(self, env, path, output_type): + def xref_text(self, env, path, tab_width): + """ + Read and add contents from a C file parsed to have cross references. + + There are two types of supported output here: + - A C source code with cross-references; + - a TOC table containing cross references. + """ parser = ParseDataStructs() parser.parse_file(path) @@ -127,10 +135,33 @@ class KernelInclude(Include): if 'warn-broken' in self.options: env._xref_files.add(path) - if output_type == "toc": - return parser.gen_toc() + if "toc" in self.options: + rawtext = parser.gen_toc() + else: + rawtext = ".. parsed-literal::\n\n" + parser.gen_output() + self.apply_range(rawtext) + + title = os.path.basename(path) + + include_lines = statemachine.string2lines(rawtext, tab_width, + convert_whitespace=True) - return ".. parsed-literal::\n\n" + parser.gen_output() + # Append line numbers data + + startline = self.options.get('start-line', None) + + result = ViewList() + if startline and startline > 0: + offset = startline - 1 + else: + offset = 0 + + for ln, line in enumerate(include_lines, start=offset): + result.append(line, path, ln) + + self.state_machine.insert_input(result, path) + + return [] def apply_range(self, rawtext): # Get to-be-included content @@ -195,9 +226,12 @@ class KernelInclude(Include): literal_block += nodes.Text(text, text) return [literal_block] - def code(self, path, include_lines): + def code(self, path, tab_width): """Output a code block""" + include_lines = statemachine.string2lines(rawtext, tab_width, + convert_whitespace=True) + self.options["source"] = path codeblock = CodeBlock(self.name, [self.options.pop("code")], # arguments @@ -244,47 +278,20 @@ class KernelInclude(Include): encoding = self.options.get("encoding", self.state.document.settings.input_encoding) - e_handler = self.state.document.settings.input_encoding_error_handler tab_width = self.options.get("tab-width", self.state.document.settings.tab_width) - if "literal" in self.options: - output_type = "literal" - elif "code" in self.options: - output_type = "code" - else: - output_type = "rst" - # Get optional arguments to related to cross-references generation if "generate-cross-refs" in self.options: - if "toc" in self.options: - output_type = "toc" - - rawtext = self.read_rawtext_with_xrefs(env, path, output_type) - - # When :generate-cross-refs: is used, the input is always a C - # file, so it has to be handled as a parsed-literal - if output_type == "rst": - output_type = "literal" - - title = os.path.basename(path) - else: - rawtext = self.read_rawtext(path, encoding) + return self.xref_text(env, path, tab_width) + rawtext = self.read_rawtext(path, encoding) rawtext = self.apply_range(rawtext) - if output_type == "literal": - return self.literal(path, tab_width, rawtext) - - include_lines = statemachine.string2lines(rawtext, tab_width, - convert_whitespace=True) + if "code" in self.options: + return self.code(path, tab_width, rawtext) - if output_type == "code": - return self.code(path, include_lines) - - self.state_machine.insert_input(include_lines, path) - - return [] + return self.literal(path, tab_width, rawtext) # ============================================================================== -- cgit v1.2.3 From 01dba1680cb4047d4f6e057276f805f93b7eea00 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:31 +0200 Subject: docs: kernel_include.py: move apply_range() and add a docstring While not required, better to have caller functions at the end. As apply_range() is now called by xref_text(), move it to be before the latter. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a6ce0fd7c03a01338753fd81ed0c4631f78311d6.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 68 ++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 32 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 90ed8428f776..fd4887f80577 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -113,6 +113,42 @@ class KernelInclude(Include): except UnicodeError as error: raise self.severe('Problem with directive:\n%s' % ErrorString(error)) + def apply_range(self, rawtext): + """ + Handles start-line, end-line, start-after and end-before parameters + """ + + # Get to-be-included content + startline = self.options.get('start-line', None) + endline = self.options.get('end-line', None) + try: + if startline or (endline is not None): + lines = rawtext.splitlines() + rawtext = '\n'.join(lines[startline:endline]) + except UnicodeError as error: + raise self.severe(f'Problem with "{self.name}" directive:\n' + + io.error_string(error)) + # start-after/end-before: no restrictions on newlines in match-text, + # and no restrictions on matching inside lines vs. line boundaries + after_text = self.options.get("start-after", None) + if after_text: + # skip content in rawtext before *and incl.* a matching text + after_index = rawtext.find(after_text) + if after_index < 0: + raise self.severe('Problem with "start-after" option of "%s" ' + "directive:\nText not found." % self.name) + rawtext = rawtext[after_index + len(after_text) :] + before_text = self.options.get("end-before", None) + if before_text: + # skip content in rawtext after *and incl.* a matching text + before_index = rawtext.find(before_text) + if before_index < 0: + raise self.severe('Problem with "end-before" option of "%s" ' + "directive:\nText not found." % self.name) + rawtext = rawtext[:before_index] + + return rawtext + def xref_text(self, env, path, tab_width): """ Read and add contents from a C file parsed to have cross references. @@ -163,38 +199,6 @@ class KernelInclude(Include): return [] - def apply_range(self, rawtext): - # Get to-be-included content - startline = self.options.get('start-line', None) - endline = self.options.get('end-line', None) - try: - if startline or (endline is not None): - lines = rawtext.splitlines() - rawtext = '\n'.join(lines[startline:endline]) - except UnicodeError as error: - raise self.severe(f'Problem with "{self.name}" directive:\n' - + io.error_string(error)) - # start-after/end-before: no restrictions on newlines in match-text, - # and no restrictions on matching inside lines vs. line boundaries - after_text = self.options.get("start-after", None) - if after_text: - # skip content in rawtext before *and incl.* a matching text - after_index = rawtext.find(after_text) - if after_index < 0: - raise self.severe('Problem with "start-after" option of "%s" ' - "directive:\nText not found." % self.name) - rawtext = rawtext[after_index + len(after_text) :] - before_text = self.options.get("end-before", None) - if before_text: - # skip content in rawtext after *and incl.* a matching text - before_index = rawtext.find(before_text) - if before_index < 0: - raise self.severe('Problem with "end-before" option of "%s" ' - "directive:\nText not found." % self.name) - rawtext = rawtext[:before_index] - - return rawtext - def literal(self, path, tab_width, rawtext): """Output a literal block""" -- cgit v1.2.3 From 4aa578f9c087d58d841e3dfbde1bf57483d9e696 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:32 +0200 Subject: docs: kernel_include.py: remove line numbers from parsed-literal When parsed-literal directive is added to rawtext, while cross references will be properly displayed, Sphinx will ignore line numbers. So, it is not worth adding them. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/b484fe5fcbf6e5217f112f205fbf54f0bbc3dcca.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index fd4887f80577..3a1753486319 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -171,13 +171,24 @@ class KernelInclude(Include): if 'warn-broken' in self.options: env._xref_files.add(path) - if "toc" in self.options: - rawtext = parser.gen_toc() - else: + if "toc" not in self.options: + rawtext = ".. parsed-literal::\n\n" + parser.gen_output() self.apply_range(rawtext) - title = os.path.basename(path) + include_lines = statemachine.string2lines(rawtext, tab_width, + convert_whitespace=True) + + # Sphinx always blame the ".. ", so placing + # line numbers here won't make any difference + + self.state_machine.insert_input(include_lines, path) + return [] + + # TOC output is a ReST file, not a literal. So, we can add line + # numbers + + rawtext = parser.gen_toc() include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) -- cgit v1.2.3 From 428c1d35118fb755e12a0e5d2745632d4cf3a76e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:33 +0200 Subject: docs: kernel_include.py: remove Include class inheritance While the original code came from the Sphinx Include class, such class is monolithic: it has only one function that does everything, and 3 variables that are used: - required_arguments - optional_arguments - option_spec So, basically those are the only members that remain from the original class, but hey! Those are the same vars that every other Sphinx directive extension has to define! In summary, keeping inheritance here doesn't make much sense. Worse than that, kernel-include doesn't support the current set of options that the original Include class has, but it also has its own set of options. So, let's fill in the argument vars with what it does support, dropping the rest. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a9f2eebf11c6b0c3a2e3bf42e71392cdfd2835d1.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 40 +++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 3a1753486319..e6f734476ab3 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -62,9 +62,8 @@ import sys from docutils import io, nodes, statemachine from docutils.statemachine import ViewList from docutils.utils.error_reporting import SafeString, ErrorString -from docutils.parsers.rst import directives +from docutils.parsers.rst import Directive, directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines -from docutils.parsers.rst.directives.misc import Include from sphinx.util import logging @@ -81,18 +80,43 @@ RE_SIMPLE_REF = re.compile(r'`([^`]+)`') # ============================================================================== -class KernelInclude(Include): - """KernelInclude (``kernel-include``) directive""" +class KernelInclude(Directive): + """ + KernelInclude (``kernel-include``) directive + + Most of the stuff here came from Include directive defined at: + docutils/parsers/rst/directives/misc.py - # Add extra options - option_spec = Include.option_spec.copy() + Yet, overriding the class don't has any benefits: the original class + only have run() and argument list. Not all of them are implemented, + when checked against latest Sphinx version, as with time more arguments + were added. - option_spec.update({ + So, keep its own list of supported arguments + """ + + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + option_spec = { + 'literal': directives.flag, + 'code': directives.unchanged, + 'encoding': directives.encoding, + 'tab-width': int, + 'start-line': int, + 'end-line': int, + 'start-after': directives.unchanged_required, + 'end-before': directives.unchanged_required, + # ignored except for 'literal' or 'code': + 'number-lines': directives.unchanged, # integer or None + 'class': directives.class_option, + + # Arguments that aren't from Sphinx Include directive 'generate-cross-refs': directives.flag, 'warn-broken': directives.flag, 'toc': directives.flag, 'exception-file': directives.unchanged, - }) + } def read_rawtext(self, path, encoding): """Read and process file content with error handling""" -- cgit v1.2.3 From a49adfab496f7720fcd588d454b36bfb7922051e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 22 Aug 2025 16:19:34 +0200 Subject: docs: kernel_include.py: document all supported parameters As we're actually a fork of Sphinx Include, update its docstring to contain the documentation for the actual implemented parameters. Let's use :param: for parameters, as defined at: https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/f193160889a2dc296b4df2cc7ebc9934d717ccef.1755872208.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 88 ++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 30 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index e6f734476ab3..23566ab74866 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -2,53 +2,81 @@ # SPDX-License-Identifier: GPL-2.0 # pylint: disable=R0903, R0912, R0914, R0915, C0209,W0707 + """ - kernel-include - ~~~~~~~~~~~~~~ +Implementation of the ``kernel-include`` reST-directive. + +:copyright: Copyright (C) 2016 Markus Heiser +:license: GPL Version 2, June 1991 see linux/COPYING for details. + +The ``kernel-include`` reST-directive is a replacement for the ``include`` +directive. The ``kernel-include`` directive expand environment variables in +the path name and allows to include files from arbitrary locations. + +.. hint:: + + Including files from arbitrary locations (e.g. from ``/etc``) is a + security risk for builders. This is why the ``include`` directive from + docutils *prohibit* pathnames pointing to locations *above* the filesystem + tree where the reST document with the include directive is placed. + +Substrings of the form $name or ${name} are replaced by the value of +environment variable name. Malformed variable names and references to +non-existing variables are left unchanged. + +**Supported Sphinx Include Options**: - Implementation of the ``kernel-include`` reST-directive. +:param literal: + If present, the included file is inserted as a literal block. - :copyright: Copyright (C) 2016 Markus Heiser - :license: GPL Version 2, June 1991 see linux/COPYING for details. +:param code: + Specify the language for syntax highlighting (e.g., 'c', 'python'). - The ``kernel-include`` reST-directive is a replacement for the ``include`` - directive. The ``kernel-include`` directive expand environment variables in - the path name and allows to include files from arbitrary locations. +:param encoding: + Specify the encoding of the included file (default: 'utf-8'). - .. hint:: +:param tab-width: + Specify the number of spaces that a tab represents. - Including files from arbitrary locations (e.g. from ``/etc``) is a - security risk for builders. This is why the ``include`` directive from - docutils *prohibit* pathnames pointing to locations *above* the filesystem - tree where the reST document with the include directive is placed. +:param start-line: + Line number at which to start including the file (1-based). - Substrings of the form $name or ${name} are replaced by the value of - environment variable name. Malformed variable names and references to - non-existing variables are left unchanged. +:param end-line: + Line number at which to stop including the file (inclusive). - This extension overrides Sphinx include directory, adding some extra - arguments: +:param start-after: + Include lines after the first line matching this text. - 1. :generate-cross-refs: +:param end-before: + Include lines before the first line matching this text. - If present, instead of reading the file, it calls ParseDataStructs() - class, which converts C data structures into cross-references to - be linked to ReST files containing a more comprehensive documentation; +:param number-lines: + Number the included lines (integer specifies start number). + Only effective with 'literal' or 'code' options. - 2. :exception-file: +:param class: + Specify HTML class attribute for the included content. - Used together with :generate-cross-refs +**Kernel-specific Extensions**: - Points to a file containing rules to ignore C data structs or to - use a different reference name, optionally using a different - reference type. +:param generate-cross-refs: + If present, instead of directly including the file, it calls + ParseDataStructs() to convert C data structures into cross-references + that link to comprehensive documentation in other ReST files. - 3. :warn-broken: +:param exception-file: + (Used with generate-cross-refs) - Used together with :generate-cross-refs: + Path to a file containing rules for handling special cases: + - Ignore specific C data structures + - Use alternative reference names + - Specify different reference types - Detect if the auto-generated cross references doesn't exist. +:param warn-broken: + (Used with generate-cross-refs) + Enables warnings when auto-generated cross-references don't point to + existing documentation targets. """ # ============================================================================== -- cgit v1.2.3 From 8dbb1779ae22e5cd84d807b7023d29791f892a02 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 1 Sep 2025 15:21:21 +0200 Subject: docs: kernel_include.py: fix an issue when O= is used As reported by Stephen, building docs with O= is now broken. Fix it by ensuring that it will seek files under Kernel source tree. The original logic was defined to accept including files under Documentation/output. The new logic doesn't need it anymore for media, but it might still be useful to preserve the previous behavior. So, I ended preserving it. Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/all/20250901142639.4de35a11@canb.auug.org.au/ Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/da91980ce42f31730dc982920167b2757b9d2769.1756732363.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 44 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 23566ab74866..661ed978bed8 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -314,15 +314,49 @@ class KernelInclude(Directive): def run(self): """Include a file as part of the content of this reST file.""" env = self.state.document.settings.env - path = os.path.realpath(os.path.expandvars(self.arguments[0])) - # to get a bit security back, prohibit /etc: - if path.startswith(os.sep + "etc"): - raise self.severe('Problems with "%s" directive, prohibited path: %s' % - (self.name, path)) + # + # The include logic accepts only patches relative to: + # - Kernel source tree + # - Documentation output directory + # + # The logic does check it to prevent directory traverse + # + + srctree = os.path.abspath(os.environ["srctree"]) + + path = os.path.expandvars(self.arguments[0]) + src_path = os.path.join(srctree, path) + + if os.path.isfile(src_path): + base = srctree + path = src_path + elif os.path.exists(arg): + # Allow patches from output dir + base = os.getcwd() + path = os.path.abspath(path) + else: + raise self.warning(f'File "%s" doesn\'t exist', path) + + abs_base = os.path.abspath(base) + abs_full_path = os.path.abspath(os.path.join(base, path)) + + try: + if os.path.commonpath([abs_full_path, abs_base]) != abs_base: + raise self.severe('Problems with "%s" directive, prohibited path: %s' % + (self.name, path)) + except ValueError: + # Paths don't have the same drive (Windows) or other incompatibility + raise self.severe('Problems with "%s" directive, invalid path: %s' % + (self.name, path)) self.arguments[0] = path + # + # Add path location to Sphinx dependencies to ensure proper cache + # invalidation check. + # + env.note_dependency(os.path.abspath(path)) # HINT: I had to copy&paste the whole Include.run method. I'am not happy -- cgit v1.2.3 From 118e54633ca894748f300c0f582f4ae1b6254f8d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 1 Sep 2025 15:21:22 +0200 Subject: docs: kernel_include.py: drop some old behavior The old behavior is not using anymore, so let's drop it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/00cdf3cbe2481aac875c543ded14b5eacfe071ec.1756732363.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_include.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 661ed978bed8..2c4bb8215b4c 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -316,11 +316,9 @@ class KernelInclude(Directive): env = self.state.document.settings.env # - # The include logic accepts only patches relative to: - # - Kernel source tree - # - Documentation output directory - # - # The logic does check it to prevent directory traverse + # The include logic accepts only patches relative to the + # Kernel source tree. The logic does check it to prevent + # directory traverse issues. # srctree = os.path.abspath(os.environ["srctree"]) @@ -331,10 +329,6 @@ class KernelInclude(Directive): if os.path.isfile(src_path): base = srctree path = src_path - elif os.path.exists(arg): - # Allow patches from output dir - base = os.getcwd() - path = os.path.abspath(path) else: raise self.warning(f'File "%s" doesn\'t exist', path) @@ -359,11 +353,6 @@ class KernelInclude(Directive): env.note_dependency(os.path.abspath(path)) - # HINT: I had to copy&paste the whole Include.run method. I'am not happy - # with this, but due to security reasons, the Include.run method does - # not allow absolute or relative pathnames pointing to locations *above* - # the filesystem tree where the reST document is placed. - if not self.state.document.settings.file_insertion_enabled: raise self.warning('"%s" directive disabled.' % self.name) source = self.state_machine.input_lines.source(self.lineno - -- cgit v1.2.3 From 00d95fcc4dee66dfb6980de6f2973b32f973a1eb Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 9 Sep 2025 13:35:37 -0600 Subject: docs: kdoc: handle the obsolescensce of docutils.ErrorString() The ErrorString() and SafeString() docutils functions were helpers meant to ease the handling of encodings during the Python 3 transition. There is no real need for them after Python 3.6, and docutils 0.22 removes them, breaking the docs build Handle this by just injecting our own one-liner version of ErrorString(), and removing the sole SafeString() call entirely. Reported-by: Zhixu Liu Signed-off-by: Jonathan Corbet Message-ID: <87ldmnv2pi.fsf@trenco.lwn.net> --- Documentation/sphinx/kernel_feat.py | 4 +++- Documentation/sphinx/kernel_include.py | 6 ++++-- Documentation/sphinx/maintainers_include.py | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) (limited to 'Documentation/sphinx/kernel_include.py') diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index e3a51867f27b..aaac76892ceb 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -40,9 +40,11 @@ import sys from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive -from docutils.utils.error_reporting import ErrorString from sphinx.util.docutils import switch_source_input +def ErrorString(exc): # Shamelessly stolen from docutils + return f'{exc.__class__.__name}: {exc}' + __version__ = '1.0' def setup(app): diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index 2c4bb8215b4c..f94412cd17c9 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -89,7 +89,6 @@ import sys from docutils import io, nodes, statemachine from docutils.statemachine import ViewList -from docutils.utils.error_reporting import SafeString, ErrorString from docutils.parsers.rst import Directive, directives from docutils.parsers.rst.directives.body import CodeBlock, NumberLines @@ -106,6 +105,9 @@ logger = logging.getLogger(__name__) RE_DOMAIN_REF = re.compile(r'\\ :(ref|c:type|c:func):`([^<`]+)(?:<([^>]+)>)?`\\') RE_SIMPLE_REF = re.compile(r'`([^`]+)`') +def ErrorString(exc): # Shamelessly stolen from docutils + return f'{exc.__class__.__name}: {exc}' + # ============================================================================== class KernelInclude(Directive): @@ -156,7 +158,7 @@ class KernelInclude(Directive): except UnicodeEncodeError: raise self.severe('Problems with directive path:\n' 'Cannot encode input file path "%s" ' - '(wrong locale?).' % SafeString(path)) + '(wrong locale?).' % path) except IOError as error: raise self.severe('Problems with directive path:\n%s.' % ErrorString(error)) diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py index d31cff867436..519ad18685b2 100755 --- a/Documentation/sphinx/maintainers_include.py +++ b/Documentation/sphinx/maintainers_include.py @@ -22,10 +22,12 @@ import re import os.path from docutils import statemachine -from docutils.utils.error_reporting import ErrorString from docutils.parsers.rst import Directive from docutils.parsers.rst.directives.misc import Include +def ErrorString(exc): # Shamelessly stolen from docutils + return f'{exc.__class__.__name}: {exc}' + __version__ = '1.0' def setup(app): -- cgit v1.2.3