[PATCH master] Kbuild: update gen_compile_commands.py

[PATCH master] Kbuild: update gen_compile_commands.py

[Ahmad Fatoum]

With python 3.12, we now get following warning when running the script:

    SyntaxWarning: invalid escape sequence '\#'

This and other minor issues have been fixed upstream since we first
imported the script, so let's sync and import the content of following
Linux v6.11-rc6 commits:

  074075aea2ff scripts/clang-tools: switch explicitly to Python 3
  585d32f9b053 gen_compile_commands: prune some directories
  265264b814c2 gen_compile_commands: extract compiler command from a series of commands
  ec783c7cb249 gen_compile_commands: fix missing 'sys' package
  a4ab14e1d8fe gen_compile_commands: handle multiple lines per .mod file
  f65a486821cf kbuild: change module.order to list *.o instead of *.ko
  92215e7a801d kbuild: rename cmd_$@ to savedcmd_$@ in *.cmd files
  1c67921444bf gen_compile_commands: add assembly files to compilation database
  52c15e7e7928 gen_compile_commands: Allow the line prefix to still be cmd_
  9e56d3be4bfd gen_compile_commands: Sort output compile commands by file name
  880946158b01 gen_compile_commands.py: fix path resolve with symlinks in it
  dae4a0171e25 gen_compile_commands: fix invalid escape sequence warning

There is one commit that we don't take over though as that doesn't apply
to barebox yet:

  321648455061 kbuild: use obj-y instead extra-y for objects placed at the head

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 scripts/clang-tools/gen_compile_commands.py | 50 ++++++++++++---------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py
index 7ed3919f453a..b8ec35e65ae6 100755
--- a/scripts/clang-tools/gen_compile_commands.py
+++ b/scripts/clang-tools/gen_compile_commands.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (C) Google LLC, 2018
@@ -11,17 +11,19 @@ import argparse
 import json
 import logging
 import os
-import sys
 import re
 import subprocess
+import sys
 
 _DEFAULT_OUTPUT = 'compile_commands.json'
 _DEFAULT_LOG_LEVEL = 'WARNING'
 
 _FILENAME_PATTERN = r'^\..*\.cmd$'
-_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
+_LINE_PATTERN = r'^(saved)?cmd_[^ ]*\.o := (?P<command_prefix>.* )(?P<file_path>[^ ]*\.[cS]) *(;|$)'
 _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
-
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
 
 def parse_arguments():
     """Sets up and parses command-line arguments.
@@ -62,7 +64,7 @@ def parse_arguments():
     args = parser.parse_args()
 
     return (args.log_level,
-            os.path.abspath(args.directory),
+            os.path.realpath(args.directory),
             args.output,
             args.ar,
             args.paths if len(args.paths) > 0 else [args.directory])
@@ -81,8 +83,14 @@ def cmdfiles_in_dir(directory):
     """
 
     filename_matcher = re.compile(_FILENAME_PATTERN)
+    exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+        # Prune unwanted directories.
+        if dirpath in exclude_dirs:
+            dirnames[:] = []
+            continue
 
-    for dirpath, _, filenames in os.walk(directory):
         for filename in filenames:
             if filename_matcher.match(filename):
                 yield os.path.join(dirpath, filename)
@@ -144,15 +152,15 @@ def cmdfiles_for_modorder(modorder):
     """
     with open(modorder) as f:
         for line in f:
-            ko = line.rstrip()
-            base, ext = os.path.splitext(ko)
-            if ext != '.ko':
-                sys.exit('{}: module path must end with .ko'.format(ko))
+            obj = line.rstrip()
+            base, ext = os.path.splitext(obj)
+            if ext != '.o':
+                sys.exit('{}: module path must end with .o'.format(obj))
             mod = base + '.mod'
-	    # The first line of *.mod lists the objects that compose the module.
+            # Read from *.mod, to get a list of objects that compose the module.
             with open(mod) as m:
-                for obj in m.readline().split():
-                    yield to_cmdfile(obj)
+                for mod_line in m:
+                    yield to_cmdfile(mod_line.rstrip())
 
 
 def process_line(root_directory, command_prefix, file_path):
@@ -176,10 +184,10 @@ def process_line(root_directory, command_prefix, file_path):
     # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
     # kernel version). The compile_commands.json file is not interepreted
     # by Make, so this code replaces the escaped version with '#'.
-    prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
+    prefix = command_prefix.replace(r'\#', '#').replace('$(pound)', '#')
 
-    # Use os.path.abspath() to normalize the path resolving '.' and '..' .
-    abs_path = os.path.abspath(os.path.join(root_directory, file_path))
+    # Return the canonical path, eliminating any symbolic links encountered in the path.
+    abs_path = os.path.realpath(os.path.join(root_directory, file_path))
     if not os.path.exists(abs_path):
         raise ValueError('File %s not found' % abs_path)
     return {
@@ -208,10 +216,10 @@ def main():
         # Modules are listed in modules.order.
         if os.path.isdir(path):
             cmdfiles = cmdfiles_in_dir(path)
-        elif path.endswith('.o'):
-            cmdfiles = cmdfiles_for_o(path)
         elif path.endswith('.a'):
             cmdfiles = cmdfiles_for_a(path, ar)
+        elif path.endswith('.o'):
+            cmdfiles = cmdfiles_for_o(path)
         elif path.endswith('modules.order'):
             cmdfiles = cmdfiles_for_modorder(path)
         else:
@@ -222,15 +230,15 @@ def main():
                 result = line_matcher.match(f.readline())
                 if result:
                     try:
-                        entry = process_line(directory, result.group(1),
-                                             result.group(2))
+                        entry = process_line(directory, result.group('command_prefix'),
+                                             result.group('file_path'))
                         compile_commands.append(entry)
                     except ValueError as err:
                         logging.info('Could not add line from %s: %s',
                                      cmdfile, err)
 
     with open(output, 'wt') as f:
-        json.dump(compile_commands, f, indent=2, sort_keys=True)
+        json.dump(sorted(compile_commands, key=lambda x: x["file"]), f, indent=2, sort_keys=True)
 
 
 if __name__ == '__main__':
-- 
2.39.2

Re: [PATCH master] Kbuild: update gen_compile_commands.py

[Sascha Hauer]

On Tue, 10 Sep 2024 10:51:46 +0200, Ahmad Fatoum wrote:
> With python 3.12, we now get following warning when running the script:
> 
>     SyntaxWarning: invalid escape sequence '\#'
> 
> This and other minor issues have been fixed upstream since we first
> imported the script, so let's sync and import the content of following
> Linux v6.11-rc6 commits:
> 
> [...]

Applied, thanks!

[1/1] Kbuild: update gen_compile_commands.py
      https://git.pengutronix.de/cgit/barebox/commit/?id=898776bbecd1 (link may not be stable)

Best regards,
-- 
Sascha Hauer <s.hauer@pengutronix.de>