Documentation/sphinx/automarkup.py

0001 # SPDX-License-Identifier: GPL-2.0
0002 # Copyright 2019 Jonathan Corbet <corbet@lwn.net>
0003 #
0004 # Apply kernel-specific tweaks after the initial document processing
0005 # has been done.
0006 #
0007 from docutils import nodes
0008 import sphinx
0009 from sphinx import addnodes
0010 if sphinx.version_info[0] < 2 or \
0011    sphinx.version_info[0] == 2 and sphinx.version_info[1] < 1:
0012     from sphinx.environment import NoUri
0013 else:
0014     from sphinx.errors import NoUri
0015 import re
0016 from itertools import chain
0017
0018 #
0019 # Python 2 lacks re.ASCII...
0020 #
0021 try:
0022     ascii_p3 = re.ASCII
0023 except AttributeError:
0024     ascii_p3 = 0
0025
0026 #
0027 # Regex nastiness.  Of course.
0028 # Try to identify "function()" that's not already marked up some
0029 # other way.  Sphinx doesn't like a lot of stuff right after a
0030 # :c:func: block (i.e. ":c:func:`mmap()`s" flakes out), so the last
0031 # bit tries to restrict matches to things that won't create trouble.
0032 #
0033 RE_function = re.compile(r'\b(([a-zA-Z_]\w+)\(\))', flags=ascii_p3)
0034
0035 #
0036 # Sphinx 2 uses the same :c:type role for struct, union, enum and typedef
0037 #
0038 RE_generic_type = re.compile(r'\b(struct|union|enum|typedef)\s+([a-zA-Z_]\w+)',
0039                              flags=ascii_p3)
0040
0041 #
0042 # Sphinx 3 uses a different C role for each one of struct, union, enum and
0043 # typedef
0044 #
0045 RE_struct = re.compile(r'\b(struct)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
0046 RE_union = re.compile(r'\b(union)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
0047 RE_enum = re.compile(r'\b(enum)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
0048 RE_typedef = re.compile(r'\b(typedef)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
0049
0050 #
0051 # Detects a reference to a documentation page of the form Documentation/... with
0052 # an optional extension
0053 #
0054 RE_doc = re.compile(r'(\bDocumentation/)?((\.\./)*[\w\-/]+)\.(rst|txt)')
0055
0056 RE_namespace = re.compile(r'^\s*..\s*c:namespace::\s*(\S+)\s*$')
0057
0058 #
0059 # Reserved C words that we should skip when cross-referencing
0060 #
0061 Skipnames = [ 'for', 'if', 'register', 'sizeof', 'struct', 'unsigned' ]
0062
0063
0064 #
0065 # Many places in the docs refer to common system calls.  It is
0066 # pointless to try to cross-reference them and, as has been known
0067 # to happen, somebody defining a function by these names can lead
0068 # to the creation of incorrect and confusing cross references.  So
0069 # just don't even try with these names.
0070 #
0071 Skipfuncs = [ 'open', 'close', 'read', 'write', 'fcntl', 'mmap',
0072               'select', 'poll', 'fork', 'execve', 'clone', 'ioctl',
0073               'socket' ]
0074
0075 c_namespace = ''
0076
0077 def markup_refs(docname, app, node):
0078     t = node.astext()
0079     done = 0
0080     repl = [ ]
0081     #
0082     # Associate each regex with the function that will markup its matches
0083     #
0084     markup_func_sphinx2 = {RE_doc: markup_doc_ref,
0085                            RE_function: markup_c_ref,
0086                            RE_generic_type: markup_c_ref}
0087
0088     markup_func_sphinx3 = {RE_doc: markup_doc_ref,
0089                            RE_function: markup_func_ref_sphinx3,
0090                            RE_struct: markup_c_ref,
0091                            RE_union: markup_c_ref,
0092                            RE_enum: markup_c_ref,
0093                            RE_typedef: markup_c_ref}
0094
0095     if sphinx.version_info[0] >= 3:
0096         markup_func = markup_func_sphinx3
0097     else:
0098         markup_func = markup_func_sphinx2
0099
0100     match_iterators = [regex.finditer(t) for regex in markup_func]
0101     #
0102     # Sort all references by the starting position in text
0103     #
0104     sorted_matches = sorted(chain(*match_iterators), key=lambda m: m.start())
0105     for m in sorted_matches:
0106         #
0107         # Include any text prior to match as a normal text node.
0108         #
0109         if m.start() > done:
0110             repl.append(nodes.Text(t[done:m.start()]))
0111
0112         #
0113         # Call the function associated with the regex that matched this text and
0114         # append its return to the text
0115         #
0116         repl.append(markup_func[m.re](docname, app, m))
0117
0118         done = m.end()
0119     if done < len(t):
0120         repl.append(nodes.Text(t[done:]))
0121     return repl
0122
0123 #
0124 # Keep track of cross-reference lookups that failed so we don't have to
0125 # do them again.
0126 #
0127 failed_lookups = { }
0128 def failure_seen(target):
0129     return (target) in failed_lookups
0130 def note_failure(target):
0131     failed_lookups[target] = True
0132
0133 #
0134 # In sphinx3 we can cross-reference to C macro and function, each one with its
0135 # own C role, but both match the same regex, so we try both.
0136 #
0137 def markup_func_ref_sphinx3(docname, app, match):
0138     cdom = app.env.domains['c']
0139     #
0140     # Go through the dance of getting an xref out of the C domain
0141     #
0142     base_target = match.group(2)
0143     target_text = nodes.Text(match.group(0))
0144     xref = None
0145     possible_targets = [base_target]
0146     # Check if this document has a namespace, and if so, try
0147     # cross-referencing inside it first.
0148     if c_namespace:
0149         possible_targets.insert(0, c_namespace + "." + base_target)
0150
0151     if base_target not in Skipnames:
0152         for target in possible_targets:
0153             if (target not in Skipfuncs) and not failure_seen(target):
0154                 lit_text = nodes.literal(classes=['xref', 'c', 'c-func'])
0155                 lit_text += target_text
0156                 pxref = addnodes.pending_xref('', refdomain = 'c',
0157                                               reftype = 'function',
0158                                               reftarget = target,
0159                                               modname = None,
0160                                               classname = None)
0161                 #
0162                 # XXX The Latex builder will throw NoUri exceptions here,
0163                 # work around that by ignoring them.
0164                 #
0165                 try:
0166                     xref = cdom.resolve_xref(app.env, docname, app.builder,
0167                                              'function', target, pxref,
0168                                              lit_text)
0169                 except NoUri:
0170                     xref = None
0171
0172                 if xref:
0173                     return xref
0174                 note_failure(target)
0175
0176     return target_text
0177
0178 def markup_c_ref(docname, app, match):
0179     class_str = {# Sphinx 2 only
0180                  RE_function: 'c-func',
0181                  RE_generic_type: 'c-type',
0182                  # Sphinx 3+ only
0183                  RE_struct: 'c-struct',
0184                  RE_union: 'c-union',
0185                  RE_enum: 'c-enum',
0186                  RE_typedef: 'c-type',
0187                  }
0188     reftype_str = {# Sphinx 2 only
0189                    RE_function: 'function',
0190                    RE_generic_type: 'type',
0191                    # Sphinx 3+ only
0192                    RE_struct: 'struct',
0193                    RE_union: 'union',
0194                    RE_enum: 'enum',
0195                    RE_typedef: 'type',
0196                    }
0197
0198     cdom = app.env.domains['c']
0199     #
0200     # Go through the dance of getting an xref out of the C domain
0201     #
0202     base_target = match.group(2)
0203     target_text = nodes.Text(match.group(0))
0204     xref = None
0205     possible_targets = [base_target]
0206     # Check if this document has a namespace, and if so, try
0207     # cross-referencing inside it first.
0208     if c_namespace:
0209         possible_targets.insert(0, c_namespace + "." + base_target)
0210
0211     if base_target not in Skipnames:
0212         for target in possible_targets:
0213             if not (match.re == RE_function and target in Skipfuncs):
0214                 lit_text = nodes.literal(classes=['xref', 'c', class_str[match.re]])
0215                 lit_text += target_text
0216                 pxref = addnodes.pending_xref('', refdomain = 'c',
0217                                               reftype = reftype_str[match.re],
0218                                               reftarget = target, modname = None,
0219                                               classname = None)
0220                 #
0221                 # XXX The Latex builder will throw NoUri exceptions here,
0222                 # work around that by ignoring them.
0223                 #
0224                 try:
0225                     xref = cdom.resolve_xref(app.env, docname, app.builder,
0226                                              reftype_str[match.re], target, pxref,
0227                                              lit_text)
0228                 except NoUri:
0229                     xref = None
0230
0231                 if xref:
0232                     return xref
0233
0234     return target_text
0235
0236 #
0237 # Try to replace a documentation reference of the form Documentation/... with a
0238 # cross reference to that page
0239 #
0240 def markup_doc_ref(docname, app, match):
0241     stddom = app.env.domains['std']
0242     #
0243     # Go through the dance of getting an xref out of the std domain
0244     #
0245     absolute = match.group(1)
0246     target = match.group(2)
0247     if absolute:
0248        target = "/" + target
0249     xref = None
0250     pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'doc',
0251                                   reftarget = target, modname = None,
0252                                   classname = None, refexplicit = False)
0253     #
0254     # XXX The Latex builder will throw NoUri exceptions here,
0255     # work around that by ignoring them.
0256     #
0257     try:
0258         xref = stddom.resolve_xref(app.env, docname, app.builder, 'doc',
0259                                    target, pxref, None)
0260     except NoUri:
0261         xref = None
0262     #
0263     # Return the xref if we got it; otherwise just return the plain text.
0264     #
0265     if xref:
0266         return xref
0267     else:
0268         return nodes.Text(match.group(0))
0269
0270 def get_c_namespace(app, docname):
0271     source = app.env.doc2path(docname)
0272     with open(source) as f:
0273         for l in f:
0274             match = RE_namespace.search(l)
0275             if match:
0276                 return match.group(1)
0277     return ''
0278
0279 def auto_markup(app, doctree, name):
0280     global c_namespace
0281     c_namespace = get_c_namespace(app, name)
0282     def text_but_not_a_reference(node):
0283         # The nodes.literal test catches ``literal text``, its purpose is to
0284         # avoid adding cross-references to functions that have been explicitly
0285         # marked with cc:func:.
0286         if not isinstance(node, nodes.Text) or isinstance(node.parent, nodes.literal):
0287             return False
0288
0289         child_of_reference = False
0290         parent = node.parent
0291         while parent:
0292             if isinstance(parent, nodes.Referential):
0293                 child_of_reference = True
0294                 break
0295             parent = parent.parent
0296         return not child_of_reference
0297
0298     #
0299     # This loop could eventually be improved on.  Someday maybe we
0300     # want a proper tree traversal with a lot of awareness of which
0301     # kinds of nodes to prune.  But this works well for now.
0302     #
0303     for para in doctree.traverse(nodes.paragraph):
0304         for node in para.traverse(condition=text_but_not_a_reference):
0305             node.parent.replace(node, markup_refs(name, app, node))
0306
0307 def setup(app):
0308     app.connect('doctree-resolved', auto_markup)
0309     return {
0310         'parallel_read_safe': True,
0311         'parallel_write_safe': True,
0312         }