1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2013 The Chromium OS Authors.
3#
4
5import multiprocessing
6import os
7import shutil
8import subprocess
9import sys
10
11from buildman import board
12from buildman import bsettings
13from buildman import toolchain
14from buildman.builder import Builder
15from patman import command
16from patman import gitutil
17from patman import patchstream
18from patman import terminal
19from patman.terminal import Print
20
21def GetPlural(count):
22    """Returns a plural 's' if count is not 1"""
23    return 's' if count != 1 else ''
24
25def GetActionSummary(is_summary, commits, selected, options):
26    """Return a string summarising the intended action.
27
28    Returns:
29        Summary string.
30    """
31    if commits:
32        count = len(commits)
33        count = (count + options.step - 1) // options.step
34        commit_str = '%d commit%s' % (count, GetPlural(count))
35    else:
36        commit_str = 'current source'
37    str = '%s %s for %d boards' % (
38        'Summary of' if is_summary else 'Building', commit_str,
39        len(selected))
40    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
42    return str
43
44def ShowActions(series, why_selected, boards_selected, builder, options,
45                board_warnings):
46    """Display a list of actions that we would take, if not a dry run.
47
48    Args:
49        series: Series object
50        why_selected: Dictionary where each key is a buildman argument
51                provided by the user, and the value is the list of boards
52                brought in by that argument. For example, 'arm' might bring
53                in 400 boards, so in this case the key would be 'arm' and
54                the value would be a list of board names.
55        boards_selected: Dict of selected boards, key is target name,
56                value is Board object
57        builder: The builder that will be used to build the commits
58        options: Command line options object
59        board_warnings: List of warnings obtained from board selected
60    """
61    col = terminal.Color()
62    print('Dry run, so not doing much. But I would do this:')
63    print()
64    if series:
65        commits = series.commits
66    else:
67        commits = None
68    print(GetActionSummary(False, commits, boards_selected,
69            options))
70    print('Build directory: %s' % builder.base_dir)
71    if commits:
72        for upto in range(0, len(series.commits), options.step):
73            commit = series.commits[upto]
74            print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
75            print(commit.subject)
76    print()
77    for arg in why_selected:
78        if arg != 'all':
79            print(arg, ': %d boards' % len(why_selected[arg]))
80            if options.verbose:
81                print('   %s' % ' '.join(why_selected[arg]))
82    print(('Total boards to build for each commit: %d\n' %
83            len(why_selected['all'])))
84    if board_warnings:
85        for warning in board_warnings:
86            print(col.Color(col.YELLOW, warning))
87
88def ShowToolchainPrefix(boards, toolchains):
89    """Show information about a the tool chain used by one or more boards
90
91    The function checks that all boards use the same toolchain, then prints
92    the correct value for CROSS_COMPILE.
93
94    Args:
95        boards: Boards object containing selected boards
96        toolchains: Toolchains object containing available toolchains
97
98    Return:
99        None on success, string error message otherwise
100    """
101    boards = boards.GetSelectedDict()
102    tc_set = set()
103    for brd in boards.values():
104        tc_set.add(toolchains.Select(brd.arch))
105    if len(tc_set) != 1:
106        return 'Supplied boards must share one toolchain'
107        return False
108    tc = tc_set.pop()
109    print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
110    return None
111
112def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
113               clean_dir=False):
114    """The main control code for buildman
115
116    Args:
117        options: Command line options object
118        args: Command line arguments (list of strings)
119        toolchains: Toolchains to use - this should be a Toolchains()
120                object. If None, then it will be created and scanned
121        make_func: Make function to use for the builder. This is called
122                to execute 'make'. If this is None, the normal function
123                will be used, which calls the 'make' tool with suitable
124                arguments. This setting is useful for tests.
125        board: Boards() object to use, containing a list of available
126                boards. If this is None it will be created and scanned.
127    """
128    global builder
129
130    if options.full_help:
131        pager = os.getenv('PAGER')
132        if not pager:
133            pager = 'more'
134        fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
135                             'README')
136        command.Run(pager, fname)
137        return 0
138
139    gitutil.Setup()
140    col = terminal.Color()
141
142    options.git_dir = os.path.join(options.git, '.git')
143
144    no_toolchains = toolchains is None
145    if no_toolchains:
146        toolchains = toolchain.Toolchains(options.override_toolchain)
147
148    if options.fetch_arch:
149        if options.fetch_arch == 'list':
150            sorted_list = toolchains.ListArchs()
151            print(col.Color(col.BLUE, 'Available architectures: %s\n' %
152                            ' '.join(sorted_list)))
153            return 0
154        else:
155            fetch_arch = options.fetch_arch
156            if fetch_arch == 'all':
157                fetch_arch = ','.join(toolchains.ListArchs())
158                print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
159                                fetch_arch))
160            for arch in fetch_arch.split(','):
161                print()
162                ret = toolchains.FetchAndInstall(arch)
163                if ret:
164                    return ret
165            return 0
166
167    if no_toolchains:
168        toolchains.GetSettings()
169        toolchains.Scan(options.list_tool_chains and options.verbose)
170    if options.list_tool_chains:
171        toolchains.List()
172        print()
173        return 0
174
175    if options.incremental:
176        print(col.Color(col.RED,
177                        'Warning: -I has been removed. See documentation'))
178    if not options.output_dir:
179        if options.work_in_output:
180            sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
181        options.output_dir = '..'
182
183    # Work out what subset of the boards we are building
184    if not boards:
185        if not os.path.exists(options.output_dir):
186            os.makedirs(options.output_dir)
187        board_file = os.path.join(options.output_dir, 'boards.cfg')
188        our_path = os.path.dirname(os.path.realpath(__file__))
189        genboardscfg = os.path.join(our_path, '../genboardscfg.py')
190        if not os.path.exists(genboardscfg):
191            genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
192        status = subprocess.call([genboardscfg, '-q', '-o', board_file])
193        if status != 0:
194            # Older versions don't support -q
195            status = subprocess.call([genboardscfg, '-o', board_file])
196            if status != 0:
197                sys.exit("Failed to generate boards.cfg")
198
199        boards = board.Boards()
200        boards.ReadBoards(board_file)
201
202    exclude = []
203    if options.exclude:
204        for arg in options.exclude:
205            exclude += arg.split(',')
206
207    if options.boards:
208        requested_boards = []
209        for b in options.boards:
210            requested_boards += b.split(',')
211    else:
212        requested_boards = None
213    why_selected, board_warnings = boards.SelectBoards(args, exclude,
214                                                       requested_boards)
215    selected = boards.GetSelected()
216    if not len(selected):
217        sys.exit(col.Color(col.RED, 'No matching boards found'))
218
219    if options.print_prefix:
220        err = ShowToolchainPrefix(boards, toolchains)
221        if err:
222            sys.exit(col.Color(col.RED, err))
223        return 0
224
225    # Work out how many commits to build. We want to build everything on the
226    # branch. We also build the upstream commit as a control so we can see
227    # problems introduced by the first commit on the branch.
228    count = options.count
229    has_range = options.branch and '..' in options.branch
230    if count == -1:
231        if not options.branch:
232            count = 1
233        else:
234            if has_range:
235                count, msg = gitutil.CountCommitsInRange(options.git_dir,
236                                                         options.branch)
237            else:
238                count, msg = gitutil.CountCommitsInBranch(options.git_dir,
239                                                          options.branch)
240            if count is None:
241                sys.exit(col.Color(col.RED, msg))
242            elif count == 0:
243                sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
244                                   options.branch))
245            if msg:
246                print(col.Color(col.YELLOW, msg))
247            count += 1   # Build upstream commit also
248
249    if not count:
250        str = ("No commits found to process in branch '%s': "
251               "set branch's upstream or use -c flag" % options.branch)
252        sys.exit(col.Color(col.RED, str))
253    if options.work_in_output:
254        if len(selected) != 1:
255            sys.exit(col.Color(col.RED,
256                               '-w can only be used with a single board'))
257        if count != 1:
258            sys.exit(col.Color(col.RED,
259                               '-w can only be used with a single commit'))
260
261    # Read the metadata from the commits. First look at the upstream commit,
262    # then the ones in the branch. We would like to do something like
263    # upstream/master~..branch but that isn't possible if upstream/master is
264    # a merge commit (it will list all the commits that form part of the
265    # merge)
266    # Conflicting tags are not a problem for buildman, since it does not use
267    # them. For example, Series-version is not useful for buildman. On the
268    # other hand conflicting tags will cause an error. So allow later tags
269    # to overwrite earlier ones by setting allow_overwrite=True
270    if options.branch:
271        if count == -1:
272            if has_range:
273                range_expr = options.branch
274            else:
275                range_expr = gitutil.GetRangeInBranch(options.git_dir,
276                                                      options.branch)
277            upstream_commit = gitutil.GetUpstream(options.git_dir,
278                                                  options.branch)
279            series = patchstream.get_metadata_for_list(upstream_commit,
280                options.git_dir, 1, series=None, allow_overwrite=True)
281
282            series = patchstream.get_metadata_for_list(range_expr,
283                    options.git_dir, None, series, allow_overwrite=True)
284        else:
285            # Honour the count
286            series = patchstream.get_metadata_for_list(options.branch,
287                    options.git_dir, count, series=None, allow_overwrite=True)
288    else:
289        series = None
290        if not options.dry_run:
291            options.verbose = True
292            if not options.summary:
293                options.show_errors = True
294
295    # By default we have one thread per CPU. But if there are not enough jobs
296    # we can have fewer threads and use a high '-j' value for make.
297    if options.threads is None:
298        options.threads = min(multiprocessing.cpu_count(), len(selected))
299    if not options.jobs:
300        options.jobs = max(1, (multiprocessing.cpu_count() +
301                len(selected) - 1) // len(selected))
302
303    if not options.step:
304        options.step = len(series.commits) - 1
305
306    gnu_make = command.Output(os.path.join(options.git,
307            'scripts/show-gnu-make'), raise_on_error=False).rstrip()
308    if not gnu_make:
309        sys.exit('GNU Make not found')
310
311    # Create a new builder with the selected options.
312    output_dir = options.output_dir
313    if options.branch:
314        dirname = options.branch.replace('/', '_')
315        # As a special case allow the board directory to be placed in the
316        # output directory itself rather than any subdirectory.
317        if not options.no_subdirs:
318            output_dir = os.path.join(options.output_dir, dirname)
319        if clean_dir and os.path.exists(output_dir):
320            shutil.rmtree(output_dir)
321    builder = Builder(toolchains, output_dir, options.git_dir,
322            options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
323            show_unknown=options.show_unknown, step=options.step,
324            no_subdirs=options.no_subdirs, full_path=options.full_path,
325            verbose_build=options.verbose_build,
326            mrproper=options.mrproper,
327            per_board_out_dir=options.per_board_out_dir,
328            config_only=options.config_only,
329            squash_config_y=not options.preserve_config_y,
330            warnings_as_errors=options.warnings_as_errors,
331            work_in_output=options.work_in_output)
332    builder.force_config_on_failure = not options.quick
333    if make_func:
334        builder.do_make = make_func
335
336    # For a dry run, just show our actions as a sanity check
337    if options.dry_run:
338        ShowActions(series, why_selected, selected, builder, options,
339                    board_warnings)
340    else:
341        builder.force_build = options.force_build
342        builder.force_build_failures = options.force_build_failures
343        builder.force_reconfig = options.force_reconfig
344        builder.in_tree = options.in_tree
345
346        # Work out which boards to build
347        board_selected = boards.GetSelectedDict()
348
349        if series:
350            commits = series.commits
351            # Number the commits for test purposes
352            for commit in range(len(commits)):
353                commits[commit].sequence = commit
354        else:
355            commits = None
356
357        Print(GetActionSummary(options.summary, commits, board_selected,
358                               options))
359
360        # We can't show function sizes without board details at present
361        if options.show_bloat:
362            options.show_detail = True
363        builder.SetDisplayOptions(
364            options.show_errors, options.show_sizes, options.show_detail,
365            options.show_bloat, options.list_error_boards, options.show_config,
366            options.show_environment, options.filter_dtb_warnings,
367            options.filter_migration_warnings)
368        if options.summary:
369            builder.ShowSummary(commits, board_selected)
370        else:
371            fail, warned = builder.BuildBoards(commits, board_selected,
372                                options.keep_outputs, options.verbose)
373            if fail:
374                return 100
375            elif warned and not options.ignore_warnings:
376                return 101
377    return 0
378