[PATCH 1/5] Documentation: gen_commands.py: escape special characters

[PATCH 1/5] Documentation: gen_commands.py: escape special characters

[Ahmad Fatoum]

commands/global.c has following line:

  BAREBOX_CMD_HELP_TEXT("Without options, print all global and env
                        (prefixed with *) variables.")

which triggers a warning as the * has a special meaning in ReST and it
is never terminated.

On the other hand, we have escapes for double quotes, which should be
stripped:

  BAREBOX_CMD_HELP_TEXT("- \"bootchooser\": boot with barebox bootchooser")

Fix these and other issues by processing escapes for all strings we extract,
except for those in literal blocks.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 Documentation/gen_commands.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/Documentation/gen_commands.py b/Documentation/gen_commands.py
index 4fcb4dda6e74..53fd59d8f692 100755
--- a/Documentation/gen_commands.py
+++ b/Documentation/gen_commands.py
@@ -26,12 +26,19 @@ CMD_END    = re.compile(r"""^BAREBOX_CMD_END\s*$""")
 
 CONT = re.compile(r"""\s*"(.*?)"\s*\)?\s*$""")
 
+ESCAPE = re.compile(r"""([*_`|<>\\])""")
+UNESCAPE = re.compile(r'''\\(["\\])''')
+
 CMDS = {}
 
+
+def string_escape_literal(s):
+    return re.sub(UNESCAPE, r'\1', s.replace(r'\t', '').replace(r'\n', ''))
+
+
 def string_escape(s):
-  # This used to do s.decode("string_escape") which isn't available on Python 3.
-  # Actually we only need to drop '\t' and '\n', so do this here.
-  return s.replace(r'\t', '').replace(r'\n', '')
+    return re.sub(ESCAPE, r'\\\1', string_escape_literal(s))
+
 
 def parse_c(name):
   cmd = None
@@ -78,7 +85,7 @@ def parse_c(name):
     x = CMD_OPTS.match(line)
     if x:
       last = cmd['c_opts']
-      last.append(string_escape(x.group(1)))
+      last.append(string_escape_literal(x.group(1)))
       continue
     x = CMD_GROUP.match(line)
     if x:
-- 
2.47.3

[PATCH 2/5] Documentation: gen_commands.py: align whitespace with coding style

[Ahmad Fatoum]

We use four spaces for indentation in python files and an extra empty
line before subroutine definition. This is the default enforced by pylsp
for example, so let's do this whitespace only change in preparation for
further work.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 Documentation/gen_commands.py | 297 +++++++++++++++++-----------------
 1 file changed, 149 insertions(+), 148 deletions(-)

diff --git a/Documentation/gen_commands.py b/Documentation/gen_commands.py
index 53fd59d8f692..8e129d6b253f 100755
--- a/Documentation/gen_commands.py
+++ b/Documentation/gen_commands.py
@@ -41,163 +41,164 @@ def string_escape(s):
 
 
 def parse_c(name):
-  cmd = None
-  last = None
-  for line in open(name, 'r'):
-    x = HELP_START.match(line)
-    if x:
-      cmd = CMDS.setdefault(x.group(1), defaultdict(list))
-      cmd.setdefault("files", set()).add(name)
-      continue
-    x = CMD_START.match(line)
-    if x:
-      cmd = CMDS.setdefault(x.group(1), defaultdict(list))
-      cmd.setdefault("files", set()).add(name)
-      continue
-    if cmd is None:
-      continue
-    x = HELP_TEXT.match(line)
-    if x:
-      if 'h_opts' not in cmd:
-        last = cmd['h_pre']
-      else:
-        last = cmd['h_post']
-      last.append(string_escape(x.group(1)).strip())
-      continue
-    x = HELP_OPT.match(line)
-    if x:
-      last = cmd['h_opts']
-      last.append([
-        string_escape(x.group(1)),
-        string_escape(x.group(2)),
-      ])
-      continue
-    x = CMD_FUNC.match(line)
-    if x:
-      last = cmd['c_func']
-      last.append(x.group(1))
-      continue
-    x = CMD_DESC.match(line)
-    if x:
-      last = cmd['c_desc']
-      last.append(string_escape(x.group(1)))
-      continue
-    x = CMD_OPTS.match(line)
-    if x:
-      last = cmd['c_opts']
-      last.append(string_escape_literal(x.group(1)))
-      continue
-    x = CMD_GROUP.match(line)
-    if x:
-      last = cmd['c_group']
-      last.append(x.group(1).split('_')[-1].lower())
-      continue
-    x = CONT.match(line)
-    if x:
-      if last is None:
-        raise Exception("Parse error in %s: %r" % (name, line))
-      if isinstance(last[-1], str):
-        last[-1] += string_escape(x.group(1))
-      elif isinstance(last[-1], list):
-        last[-1][1] += string_escape(x.group(1))
-      continue
-    x = HELP_END.match(line)
-    if x:
-      cmd = last = None
-    x = CMD_END.match(line)
-    if x:
-      cmd = last = None
+    cmd = None
+    last = None
+    for line in open(name, 'r'):
+        x = HELP_START.match(line)
+        if x:
+            cmd = CMDS.setdefault(x.group(1), defaultdict(list))
+            cmd.setdefault("files", set()).add(name)
+            continue
+        x = CMD_START.match(line)
+        if x:
+            cmd = CMDS.setdefault(x.group(1), defaultdict(list))
+            cmd.setdefault("files", set()).add(name)
+            continue
+        if cmd is None:
+            continue
+        x = HELP_TEXT.match(line)
+        if x:
+            if 'h_opts' not in cmd:
+                last = cmd['h_pre']
+            else:
+                last = cmd['h_post']
+            last.append(string_escape(x.group(1)).strip())
+            continue
+        x = HELP_OPT.match(line)
+        if x:
+            last = cmd['h_opts']
+            last.append([
+                string_escape(x.group(1)),
+                string_escape(x.group(2)),
+            ])
+            continue
+        x = CMD_FUNC.match(line)
+        if x:
+            last = cmd['c_func']
+            last.append(x.group(1))
+            continue
+        x = CMD_DESC.match(line)
+        if x:
+            last = cmd['c_desc']
+            last.append(string_escape(x.group(1)))
+            continue
+        x = CMD_OPTS.match(line)
+        if x:
+            last = cmd['c_opts']
+            last.append(string_escape_literal(x.group(1)))
+            continue
+        x = CMD_GROUP.match(line)
+        if x:
+            last = cmd['c_group']
+            last.append(x.group(1).split('_')[-1].lower())
+            continue
+        x = CONT.match(line)
+        if x:
+            if last is None:
+                raise Exception("Parse error in %s: %r" % (name, line))
+            if isinstance(last[-1], str):
+                last[-1] += string_escape(x.group(1))
+            elif isinstance(last[-1], list):
+                last[-1][1] += string_escape(x.group(1))
+            continue
+        x = HELP_END.match(line)
+        if x:
+            cmd = last = None
+        x = CMD_END.match(line)
+        if x:
+            cmd = last = None
+
 
 def gen_rst(name, cmd):
-  out = []
-  out.append('.. index:: %s (command)' % name)
-  out.append('')
-  out.append('.. _command_%s:' % name)
-  out.append('')
-  if 'c_desc' in cmd:
-    out.append("%s - %s" % (name, ''.join(cmd['c_desc']).strip()))
-  else:
-    out.append("%s" % (name,))
-  out.append('='*len(out[-1]))
-  out.append('')
-  if 'c_opts' in cmd:
-    out.append('Usage')
-    out.append('^'*len(out[-1]))
-    out.append('``%s %s``' % (name, ''.join(cmd['c_opts']).strip()))
+    out = []
+    out.append('.. index:: %s (command)' % name)
     out.append('')
-  if 'h_pre' in cmd:
-    pre = cmd['h_pre']
-    if pre and pre[-1] == "Options:":
-      del pre[-1]
-    if pre and pre[-1] == "":
-      del pre[-1]
-    if pre:
-      out.append('Synopsis')
-      out.append('^'*len(out[-1]))
-      out.append('\n'.join(cmd['h_pre']).strip())
-      out.append('')
-  if 'h_opts' in cmd:
-    out.append('Options')
-    out.append('^'*len(out[-1]))
-    for o, d in cmd['h_opts']:
-      o = o.strip()
-      d = d.strip()
-      if o:
-        out.append('%s\n %s' % (o, d))
-      else:
-        out.append(' %s' % (d,))
-      out.append('')
+    out.append('.. _command_%s:' % name)
     out.append('')
-  if 'h_post' in cmd:
-    post = cmd['h_post']
-    if post and post[0] == "":
-      del post[0]
-    if post:
-      out.append('Description')
-      out.append('^'*len(out[-1]))
-      out.append('\n'.join(cmd['h_post']).strip())
-      out.append('')
-  out.append('.. generated from: %s' % ', '.join(cmd['files']))
-  if 'c_func' in cmd:
-    out.append('.. command function: %s' % ', '.join(cmd['c_func']))
-  return '\n'.join(out)
+    if 'c_desc' in cmd:
+        out.append("%s - %s" % (name, ''.join(cmd['c_desc']).strip()))
+    else:
+        out.append("%s" % (name,))
+    out.append('=' * len(out[-1]))
+    out.append('')
+    if 'c_opts' in cmd:
+        out.append('Usage')
+        out.append('^' * len(out[-1]))
+        out.append('``%s %s``' % (name, ''.join(cmd['c_opts']).strip()))
+        out.append('')
+    if 'h_pre' in cmd:
+        pre = cmd['h_pre']
+        if pre and pre[-1] == "Options:":
+            del pre[-1]
+        if pre and pre[-1] == "":
+            del pre[-1]
+        if pre:
+            out.append('Synopsis')
+            out.append('^' * len(out[-1]))
+            out.append('\n'.join(cmd['h_pre']).strip())
+            out.append('')
+    if 'h_opts' in cmd:
+        out.append('Options')
+        out.append('^' * len(out[-1]))
+        for o, d in cmd['h_opts']:
+            o = o.strip()
+            d = d.strip()
+            if o:
+                out.append('%s\n %s' % (o, d))
+            else:
+                out.append(' %s' % (d,))
+            out.append('')
+        out.append('')
+    if 'h_post' in cmd:
+        post = cmd['h_post']
+        if post and post[0] == "":
+            del post[0]
+        if post:
+            out.append('Description')
+            out.append('^' * len(out[-1]))
+            out.append('\n'.join(cmd['h_post']).strip())
+            out.append('')
+    out.append('.. generated from: %s' % ', '.join(cmd['files']))
+    if 'c_func' in cmd:
+        out.append('.. command function: %s' % ', '.join(cmd['c_func']))
+    return '\n'.join(out)
 
 for root, dirs, files in os.walk(sys.argv[1]):
-  for name in files:
-    if name.endswith('.c'):
-      source = os.path.join(root, name)
-      parse_c(source)
+    for name in files:
+        if name.endswith('.c'):
+            source = os.path.join(root, name)
+            parse_c(source)
 
 for name in CMDS.keys():
-  CMDS[name] = dict(CMDS[name])
+    CMDS[name] = dict(CMDS[name])
 
 for name, cmd in CMDS.items():
-  #pprint({name: cmd})
-  rst = gen_rst(name, cmd)
-  group = cmd.get('c_group')
-  if group is None:
-    print("gen_commands: warning: using default group 'misc' for command '%s'" % name, file=sys.stderr)
-    group = ['misc']
-  subdir = os.path.join(sys.argv[2], group[0])
-  try:
-    os.makedirs(subdir)
-  except OSError as e:
-    if e.errno == errno.EEXIST and os.path.isdir(subdir):
-      pass
-    else:
-      raise
-  target = os.path.join(subdir, name+'.rst')
+    #pprint({name: cmd})
+    rst = gen_rst(name, cmd)
+    group = cmd.get('c_group')
+    if group is None:
+        print("gen_commands: warning: using default group 'misc' for command '%s'" % name, file=sys.stderr)
+        group = ['misc']
+    subdir = os.path.join(sys.argv[2], group[0])
+    try:
+        os.makedirs(subdir)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(subdir):
+            pass
+        else:
+            raise
+    target = os.path.join(subdir, name + '.rst')
 
-  # Only write the new rst if it differs from the old one. Wroto
-  hash_old = hashlib.sha1()
-  try:
-    f = open(target, 'rb')
-    hash_old.update(f.read())
-  except:
-    pass
-  hash_new = hashlib.sha1()
-  hash_new.update(rst.encode('utf-8'))
-  if hash_old.hexdigest() == hash_new.hexdigest():
-    continue
+    # Only write the new rst if it differs from the old one. Wroto
+    hash_old = hashlib.sha1()
+    try:
+        f = open(target, 'rb')
+        hash_old.update(f.read())
+    except:
+        pass
+    hash_new = hashlib.sha1()
+    hash_new.update(rst.encode('utf-8'))
+    if hash_old.hexdigest() == hash_new.hexdigest():
+        continue
 
-  open(target, 'w').write(rst)
+    open(target, 'w').write(rst)
-- 
2.47.3

[PATCH 3/5] Documentation: gen_commands.py: rework if statements for compactness

[Ahmad Fatoum]

The code can get a bit easier to follow by removing some clutter:

- use elif instead continue
- combine assignments and checks into one line

No functional change. This was verified by regenerating the docs and
checking the diff.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 Documentation/gen_commands.py | 66 ++++++++++++++---------------------
 1 file changed, 27 insertions(+), 39 deletions(-)

diff --git a/Documentation/gen_commands.py b/Documentation/gen_commands.py
index 8e129d6b253f..2f15c8b40540 100755
--- a/Documentation/gen_commands.py
+++ b/Documentation/gen_commands.py
@@ -44,69 +44,57 @@ def parse_c(name):
     cmd = None
     last = None
     for line in open(name, 'r'):
-        x = HELP_START.match(line)
-        if x:
+        if x := HELP_START.match(line):
             cmd = CMDS.setdefault(x.group(1), defaultdict(list))
             cmd.setdefault("files", set()).add(name)
-            continue
-        x = CMD_START.match(line)
-        if x:
+
+        elif x := CMD_START.match(line):
             cmd = CMDS.setdefault(x.group(1), defaultdict(list))
             cmd.setdefault("files", set()).add(name)
-            continue
-        if cmd is None:
-            continue
-        x = HELP_TEXT.match(line)
-        if x:
-            if 'h_opts' not in cmd:
-                last = cmd['h_pre']
-            else:
-                last = cmd['h_post']
+
+        elif cmd is None:
+            pass
+
+        elif x := HELP_TEXT.match(line):
+            last = cmd['h_post' if 'h_opts' in cmd else 'h_pre']
             last.append(string_escape(x.group(1)).strip())
-            continue
-        x = HELP_OPT.match(line)
-        if x:
+
+        elif x := HELP_OPT.match(line):
             last = cmd['h_opts']
             last.append([
                 string_escape(x.group(1)),
                 string_escape(x.group(2)),
             ])
-            continue
-        x = CMD_FUNC.match(line)
-        if x:
+
+        elif x := CMD_FUNC.match(line):
             last = cmd['c_func']
             last.append(x.group(1))
-            continue
-        x = CMD_DESC.match(line)
-        if x:
+
+        elif x := CMD_DESC.match(line):
             last = cmd['c_desc']
             last.append(string_escape(x.group(1)))
-            continue
-        x = CMD_OPTS.match(line)
-        if x:
+
+        elif x := CMD_OPTS.match(line):
             last = cmd['c_opts']
             last.append(string_escape_literal(x.group(1)))
-            continue
-        x = CMD_GROUP.match(line)
-        if x:
+
+        elif x := CMD_GROUP.match(line):
             last = cmd['c_group']
             last.append(x.group(1).split('_')[-1].lower())
-            continue
-        x = CONT.match(line)
-        if x:
+
+        elif x := CONT.match(line):
             if last is None:
                 raise Exception("Parse error in %s: %r" % (name, line))
             if isinstance(last[-1], str):
                 last[-1] += string_escape(x.group(1))
             elif isinstance(last[-1], list):
                 last[-1][1] += string_escape(x.group(1))
-            continue
-        x = HELP_END.match(line)
-        if x:
-            cmd = last = None
-        x = CMD_END.match(line)
-        if x:
-            cmd = last = None
+
+        else:
+            if x := HELP_END.match(line):
+                cmd = last = None
+            if x := CMD_END.match(line):
+                cmd = last = None
 
 
 def gen_rst(name, cmd):
-- 
2.47.3

[PATCH 4/5] Documentation: gen_commands.py: use literal block for help text

[Ahmad Fatoum]

The help text can have lists, like in the boot command[1] and printing
this as normal text messes up the formatting.
Therefore, let's just print it as a code block.

This doesn't solve interleaved output like fiptool's, but it makes e.g.
the boot help text or menu actually readable.

[1]: https://www.barebox.org/doc/latest/commands/boot/boot.html

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 Documentation/gen_commands.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Documentation/gen_commands.py b/Documentation/gen_commands.py
index 2f15c8b40540..f8a7ed7b879f 100755
--- a/Documentation/gen_commands.py
+++ b/Documentation/gen_commands.py
@@ -57,7 +57,7 @@ def parse_c(name):
 
         elif x := HELP_TEXT.match(line):
             last = cmd['h_post' if 'h_opts' in cmd else 'h_pre']
-            last.append(string_escape(x.group(1)).strip())
+            last.append(string_escape_literal(x.group(1)).strip())
 
         elif x := HELP_OPT.match(line):
             last = cmd['h_opts']
@@ -123,7 +123,8 @@ def gen_rst(name, cmd):
         if pre:
             out.append('Synopsis')
             out.append('^' * len(out[-1]))
-            out.append('\n'.join(cmd['h_pre']).strip())
+            out.append('\n::\n')
+            out.append('\n'.join([f'  {s}' for s in cmd['h_pre']]))
             out.append('')
     if 'h_opts' in cmd:
         out.append('Options')
@@ -144,7 +145,8 @@ def gen_rst(name, cmd):
         if post:
             out.append('Description')
             out.append('^' * len(out[-1]))
-            out.append('\n'.join(cmd['h_post']).strip())
+            out.append('\n::\n')
+            out.append('\n'.join([f'  {s}' for s in cmd['h_post']]))
             out.append('')
     out.append('.. generated from: %s' % ', '.join(cmd['files']))
     if 'c_func' in cmd:
-- 
2.47.3

[PATCH 5/5] Documentation: gen_commands.py: escape name in title

[Ahmad Fatoum]

Underscores in headings seem to be ignored, but let's play it safe and
escape the name as well.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 Documentation/gen_commands.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Documentation/gen_commands.py b/Documentation/gen_commands.py
index f8a7ed7b879f..1a7353a854cd 100755
--- a/Documentation/gen_commands.py
+++ b/Documentation/gen_commands.py
@@ -104,9 +104,9 @@ def gen_rst(name, cmd):
     out.append('.. _command_%s:' % name)
     out.append('')
     if 'c_desc' in cmd:
-        out.append("%s - %s" % (name, ''.join(cmd['c_desc']).strip()))
+        out.append("%s - %s" % (string_escape(name), ''.join(cmd['c_desc']).strip()))
     else:
-        out.append("%s" % (name,))
+        out.append("%s" % (string_escape(name),))
     out.append('=' * len(out[-1]))
     out.append('')
     if 'c_opts' in cmd:
-- 
2.47.3