build/ios/update_bundle_filelist.py

#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Updates .filelist files using data from corresponding .globlist files (or
checks whether they are up to date).

bundle_data targets require an explicit source list, but maintaining these large
lists can be cumbersome. This script aims to simplify the process of updating
these lists by either expanding globs to update file lists or check that an
existing file list matches such an expansion (i.e., checking during presubmit).

The .globlist file contains a list of globs that will be expanded to either
compare or replace a corresponding .filelist. It is possible to exclude items
from the file list with globs as well. These lines are prefixed with '-' and are
processed in order, so be sure that exclusions succeed inclusions in the list of
globs. Comments and empty lines are permitted in .globfiles; comments are
prefixed with '#'.

By convention, the base name of the .globlist and .filelist files matches the
label of their corresponding bundle_data from the .gn file. In order to ensure
that these filelists don't get stale, there should also be a PRESUBMIT.py
which uses this script to check that list is up to date.

By default, the script will update the file list to match the expanded globs.
"""

import argparse
import datetime
import difflib
import glob
import os.path
import re
import sys

# Character to set colors in terminal. Taken, along with the printing routine
# below, from update_deps.py.
TERMINAL_ERROR_COLOR = '\033[91m'
TERMINAL_RESET_COLOR = '\033[0m'

_HEADER = """# Copyright %d The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
#       If it requires updating, you should get a presubmit error with
#       instructions on how to regenerate. Otherwise, do not edit.
""" % (datetime.datetime.now().year)

_HEADER_PATTERN = re.compile(r"""# Copyright [0-9]+ The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
#       If it requires updating, you should get a presubmit error with
#       instructions on how to regenerate. Otherwise, do not edit.
""")

_HEADER_HEIGHT = 6


def parse_filelist(filelist_name):
  try:
    with open(filelist_name) as filelist:
      unfiltered = [l for l in filelist]
      header = ''.join(unfiltered[:_HEADER_HEIGHT])
      files = sorted(l.strip() for l in unfiltered[_HEADER_HEIGHT:])
      return (files, header)
  except Exception as e:
    print_error(f'Could not read file list: {filelist_name}', f'{type(e)}: {e}')
    return []


def parse_and_expand_globlist(globlist_name, glob_root):
  try:
    # The following expects glob_root not to end in a trailing slash.
    if glob_root.endswith('/'):
      glob_root = glob_root[:-1]

    with open(globlist_name) as globlist:
      files = []
      for g in globlist:
        g = g.strip()

        # Ignore blank lines and comments.
        if not g or g.startswith('#'):
          continue

        # Exclusions are prefixed with '-'.
        is_exclusion = g.startswith('-')
        if is_exclusion:
          g = g[1:]

        prefix_size = len(glob_root)
        full_glob = ''

        if g.startswith('//'):
          # If globlist is relative to the repository root, os.path
          # will consider it absolute and os.path.join will fail.
          # In this case, we can simply concatenate the paths.
          full_glob = glob_root + g
        else:
          full_glob = os.path.join(glob_root, g)
          # We need to account for the separator.
          prefix_size += 1

        expansion = glob.glob(full_glob, recursive=True)

        # Filter out directories.
        expansion = [f for f in expansion if os.path.isfile(f)]

        # Make relative to |glob_root|.
        expansion = [f[prefix_size:] for f in expansion]

        if is_exclusion:
          files = [f for f in files if f.replace('\\', '/') not in expansion]
        else:
          files += expansion

      # Handle Windows backslashes
      files = [f.replace('\\', '/') for f in files]
      files.sort()
      return files

  except Exception as e:
    print_error(f'Could not read glob list: {globlist_name}', f'{type(e)}: {e}')
    return []


def compare_lists(a, b, verbose):
  differ = difflib.Differ()
  full_diff = differ.compare(a, b)
  diff = '\n'.join([d for d in full_diff if not d.startswith('  ')])
  if diff:
    if verbose:
      print_error('File list does not match glob expansion', f'{diff}')
    return False
  return True


def write_filelist(filelist_name, files, header):
  try:
    with open(filelist_name, 'w', encoding='utf-8', newline='') as filelist:
      if not _HEADER_PATTERN.search(header):
        header = _HEADER
      filelist.write(header)
      for file in files:
        filelist.write(f'{file}\n')
  except Exception as e:
    print_error(f'Could not write file list: {filelist_name}',
                f'{type(e)}: {e}')
    return []


def process_filelist(filelist, globlist, globroot, check=False, verbose=False):
  files_from_globlist = parse_and_expand_globlist(globlist, globroot)
  (files, header) = parse_filelist(filelist)
  if check:
    if not _HEADER_PATTERN.search(header):
      if verbose:
        print_error(f'Unexpected header for {filelist}', f'{header}')
      return 1
    if compare_lists(files, files_from_globlist, verbose):
      return 0
    return 1
  else:
    write_filelist(filelist, files_from_globlist, header)
    return 0


def main(args):
  parser = argparse.ArgumentParser(
      description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument('filelist', help='Contains one file per line')
  parser.add_argument('globlist',
                      help='Contains globs that, when expanded, '
                      'should match the filelist. Use '
                      '--help for details on syntax')
  parser.add_argument('globroot',
                      help='Directory from which globs are relative')
  parser.add_argument('-c',
                      '--check',
                      action='store_true',
                      help='Prevents modifying the file list')
  parser.add_argument('-v',
                      '--verbose',
                      action='store_true',
                      help='Use this to print details on differences')
  args = parser.parse_args()
  return process_filelist(args.filelist,
                          args.globlist,
                          args.globroot,
                          check=args.check,
                          verbose=args.verbose)


def print_error(error_message, error_info):
  """ Print the `error_message` with additional `error_info` """
  color_start, color_end = adapted_color_for_output(TERMINAL_ERROR_COLOR,
                                                    TERMINAL_RESET_COLOR)

  error_message = color_start + 'ERROR: ' + error_message + color_end
  if len(error_info) > 0:
    error_message = error_message + '\n' + error_info
  print(error_message, file=sys.stderr)


def adapted_color_for_output(color_start, color_end):
  """ Returns a the `color_start`, `color_end` tuple if the output is a
    terminal, or empty strings otherwise """
  if not sys.stdout.isatty():
    return '', ''
  return color_start, color_end


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))