1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2014 Google, Inc
3#
4
5import os
6import shutil
7import sys
8import tempfile
9import unittest
10
11from buildman import board
12from buildman import bsettings
13from buildman import cmdline
14from buildman import control
15from buildman import toolchain
16from patman import command
17from patman import gitutil
18from patman import terminal
19from patman import tools
20
21settings_data = '''
22# Buildman settings file
23
24[toolchain]
25
26[toolchain-alias]
27
28[make-flags]
29src=/home/sjg/c/src
30chroot=/home/sjg/c/chroot
31vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
32chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
33chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
34chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
35'''
36
37boards = [
38    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
39    ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
40    ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
41    ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
42]
43
44commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4539403bb patman: Use --no-pager' to stop git from forking a pager
46db6e6f2 patman: Remove the -a option
47f2ccf03 patman: Correct unit tests to run correctly
481d097f9 patman: Fix indentation in terminal.py
49d073747 patman: Support the 'reverse' option for 'git log
50"""
51
52commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
53Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
54Date:   Fri Aug 22 19:12:41 2014 +0900
55
56    buildman: refactor help message
57
58    "buildman [options]" is displayed by default.
59
60    Append the rest of help messages to parser.usage
61    instead of replacing it.
62
63    Besides, "-b <branch>" is not mandatory since commit fea5858e.
64    Drop it from the usage.
65
66    Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
67""",
68"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
69Author: Simon Glass <sjg@chromium.org>
70Date:   Thu Aug 14 16:48:25 2014 -0600
71
72    patman: Support the 'reverse' option for 'git log'
73
74    This option is currently not supported, but needs to be, for buildman to
75    operate as expected.
76
77    Series-changes: 7
78    - Add new patch to fix the 'reverse' bug
79
80    Series-version: 8
81
82    Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
83    Reported-by: York Sun <yorksun@freescale.com>
84    Signed-off-by: Simon Glass <sjg@chromium.org>
85
86""",
87"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
88Author: Simon Glass <sjg@chromium.org>
89Date:   Sat Aug 9 11:44:32 2014 -0600
90
91    patman: Fix indentation in terminal.py
92
93    This code came from a different project with 2-character indentation. Fix
94    it for U-Boot.
95
96    Series-changes: 6
97    - Add new patch to fix indentation in teminal.py
98
99    Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
100    Signed-off-by: Simon Glass <sjg@chromium.org>
101
102""",
103"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
104Author: Simon Glass <sjg@chromium.org>
105Date:   Sat Aug 9 11:08:24 2014 -0600
106
107    patman: Correct unit tests to run correctly
108
109    It seems that doctest behaves differently now, and some of the unit tests
110    do not run. Adjust the tests to work correctly.
111
112     ./tools/patman/patman --test
113    <unittest.result.TestResult run=10 errors=0 failures=0>
114
115    Series-changes: 6
116    - Add new patch to fix patman unit tests
117
118    Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
119
120""",
121"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
122Author: Simon Glass <sjg@chromium.org>
123Date:   Sat Aug 9 12:06:02 2014 -0600
124
125    patman: Remove the -a option
126
127    It seems that this is no longer needed, since checkpatch.pl will catch
128    whitespace problems in patches. Also the option is not widely used, so
129    it seems safe to just remove it.
130
131    Series-changes: 6
132    - Add new patch to remove patman's -a option
133
134    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
135    Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
136
137""",
138"""commit 39403bb4f838153028a6f21ca30bf100f3791133
139Author: Simon Glass <sjg@chromium.org>
140Date:   Thu Aug 14 21:50:52 2014 -0600
141
142    patman: Use --no-pager' to stop git from forking a pager
143
144""",
145"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
146Author: Simon Glass <sjg@chromium.org>
147Date:   Fri Aug 22 15:57:39 2014 -0600
148
149    patman: Avoid changing the order of tags
150
151    patman collects tags that it sees in the commit and places them nicely
152    sorted at the end of the patch. However, this is not really necessary and
153    in fact is apparently not desirable.
154
155    Series-changes: 9
156    - Add new patch to avoid changing the order of tags
157
158    Series-version: 9
159
160    Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
161    Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
162"""]
163
164TEST_BRANCH = '__testbranch'
165
166class TestFunctional(unittest.TestCase):
167    """Functional test for buildman.
168
169    This aims to test from just below the invocation of buildman (parsing
170    of arguments) to 'make' and 'git' invocation. It is not a true
171    emd-to-end test, as it mocks git, make and the tool chain. But this
172    makes it easier to detect when the builder is doing the wrong thing,
173    since in many cases this test code will fail. For example, only a
174    very limited subset of 'git' arguments is supported - anything
175    unexpected will fail.
176    """
177    def setUp(self):
178        self._base_dir = tempfile.mkdtemp()
179        self._output_dir = tempfile.mkdtemp()
180        self._git_dir = os.path.join(self._base_dir, 'src')
181        self._buildman_pathname = sys.argv[0]
182        self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
183        command.test_result = self._HandleCommand
184        self.setupToolchains()
185        self._toolchains.Add('arm-gcc', test=False)
186        self._toolchains.Add('powerpc-gcc', test=False)
187        bsettings.Setup(None)
188        bsettings.AddFile(settings_data)
189        self._boards = board.Boards()
190        for brd in boards:
191            self._boards.AddBoard(board.Board(*brd))
192
193        # Directories where the source been cloned
194        self._clone_dirs = []
195        self._commits = len(commit_shortlog.splitlines()) + 1
196        self._total_builds = self._commits * len(boards)
197
198        # Number of calls to make
199        self._make_calls = 0
200
201        # Map of [board, commit] to error messages
202        self._error = {}
203
204        self._test_branch = TEST_BRANCH
205
206        # Avoid sending any output and clear all terminal output
207        terminal.SetPrintTestMode()
208        terminal.GetPrintTestLines()
209
210    def tearDown(self):
211        shutil.rmtree(self._base_dir)
212        #shutil.rmtree(self._output_dir)
213
214    def setupToolchains(self):
215        self._toolchains = toolchain.Toolchains()
216        self._toolchains.Add('gcc', test=False)
217
218    def _RunBuildman(self, *args):
219        return command.RunPipe([[self._buildman_pathname] + list(args)],
220                capture=True, capture_stderr=True)
221
222    def _RunControl(self, *args, clean_dir=False, boards=None):
223        sys.argv = [sys.argv[0]] + list(args)
224        options, args = cmdline.ParseArgs()
225        result = control.DoBuildman(options, args, toolchains=self._toolchains,
226                make_func=self._HandleMake, boards=boards or self._boards,
227                clean_dir=clean_dir)
228        self._builder = control.builder
229        return result
230
231    def testFullHelp(self):
232        command.test_result = None
233        result = self._RunBuildman('-H')
234        help_file = os.path.join(self._buildman_dir, 'README')
235        # Remove possible extraneous strings
236        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
237        gothelp = result.stdout.replace(extra, '')
238        self.assertEqual(len(gothelp), os.path.getsize(help_file))
239        self.assertEqual(0, len(result.stderr))
240        self.assertEqual(0, result.return_code)
241
242    def testHelp(self):
243        command.test_result = None
244        result = self._RunBuildman('-h')
245        help_file = os.path.join(self._buildman_dir, 'README')
246        self.assertTrue(len(result.stdout) > 1000)
247        self.assertEqual(0, len(result.stderr))
248        self.assertEqual(0, result.return_code)
249
250    def testGitSetup(self):
251        """Test gitutils.Setup(), from outside the module itself"""
252        command.test_result = command.CommandResult(return_code=1)
253        gitutil.Setup()
254        self.assertEqual(gitutil.use_no_decorate, False)
255
256        command.test_result = command.CommandResult(return_code=0)
257        gitutil.Setup()
258        self.assertEqual(gitutil.use_no_decorate, True)
259
260    def _HandleCommandGitLog(self, args):
261        if args[-1] == '--':
262            args = args[:-1]
263        if '-n0' in args:
264            return command.CommandResult(return_code=0)
265        elif args[-1] == 'upstream/master..%s' % self._test_branch:
266            return command.CommandResult(return_code=0, stdout=commit_shortlog)
267        elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
268            if args[-1] == self._test_branch:
269                count = int(args[3][2:])
270                return command.CommandResult(return_code=0,
271                                            stdout=''.join(commit_log[:count]))
272
273        # Not handled, so abort
274        print('git log', args)
275        sys.exit(1)
276
277    def _HandleCommandGitConfig(self, args):
278        config = args[0]
279        if config == 'sendemail.aliasesfile':
280            return command.CommandResult(return_code=0)
281        elif config.startswith('branch.badbranch'):
282            return command.CommandResult(return_code=1)
283        elif config == 'branch.%s.remote' % self._test_branch:
284            return command.CommandResult(return_code=0, stdout='upstream\n')
285        elif config == 'branch.%s.merge' % self._test_branch:
286            return command.CommandResult(return_code=0,
287                                         stdout='refs/heads/master\n')
288
289        # Not handled, so abort
290        print('git config', args)
291        sys.exit(1)
292
293    def _HandleCommandGit(self, in_args):
294        """Handle execution of a git command
295
296        This uses a hacked-up parser.
297
298        Args:
299            in_args: Arguments after 'git' from the command line
300        """
301        git_args = []           # Top-level arguments to git itself
302        sub_cmd = None          # Git sub-command selected
303        args = []               # Arguments to the git sub-command
304        for arg in in_args:
305            if sub_cmd:
306                args.append(arg)
307            elif arg[0] == '-':
308                git_args.append(arg)
309            else:
310                if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
311                    git_args.append(arg)
312                else:
313                    sub_cmd = arg
314        if sub_cmd == 'config':
315            return self._HandleCommandGitConfig(args)
316        elif sub_cmd == 'log':
317            return self._HandleCommandGitLog(args)
318        elif sub_cmd == 'clone':
319            return command.CommandResult(return_code=0)
320        elif sub_cmd == 'checkout':
321            return command.CommandResult(return_code=0)
322        elif sub_cmd == 'worktree':
323            return command.CommandResult(return_code=0)
324
325        # Not handled, so abort
326        print('git', git_args, sub_cmd, args)
327        sys.exit(1)
328
329    def _HandleCommandNm(self, args):
330        return command.CommandResult(return_code=0)
331
332    def _HandleCommandObjdump(self, args):
333        return command.CommandResult(return_code=0)
334
335    def _HandleCommandObjcopy(self, args):
336        return command.CommandResult(return_code=0)
337
338    def _HandleCommandSize(self, args):
339        return command.CommandResult(return_code=0)
340
341    def _HandleCommand(self, **kwargs):
342        """Handle a command execution.
343
344        The command is in kwargs['pipe-list'], as a list of pipes, each a
345        list of commands. The command should be emulated as required for
346        testing purposes.
347
348        Returns:
349            A CommandResult object
350        """
351        pipe_list = kwargs['pipe_list']
352        wc = False
353        if len(pipe_list) != 1:
354            if pipe_list[1] == ['wc', '-l']:
355                wc = True
356            else:
357                print('invalid pipe', kwargs)
358                sys.exit(1)
359        cmd = pipe_list[0][0]
360        args = pipe_list[0][1:]
361        result = None
362        if cmd == 'git':
363            result = self._HandleCommandGit(args)
364        elif cmd == './scripts/show-gnu-make':
365            return command.CommandResult(return_code=0, stdout='make')
366        elif cmd.endswith('nm'):
367            return self._HandleCommandNm(args)
368        elif cmd.endswith('objdump'):
369            return self._HandleCommandObjdump(args)
370        elif cmd.endswith('objcopy'):
371            return self._HandleCommandObjcopy(args)
372        elif cmd.endswith( 'size'):
373            return self._HandleCommandSize(args)
374
375        if not result:
376            # Not handled, so abort
377            print('unknown command', kwargs)
378            sys.exit(1)
379
380        if wc:
381            result.stdout = len(result.stdout.splitlines())
382        return result
383
384    def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
385        """Handle execution of 'make'
386
387        Args:
388            commit: Commit object that is being built
389            brd: Board object that is being built
390            stage: Stage that we are at (mrproper, config, build)
391            cwd: Directory where make should be run
392            args: Arguments to pass to make
393            kwargs: Arguments to pass to command.RunPipe()
394        """
395        self._make_calls += 1
396        if stage == 'mrproper':
397            return command.CommandResult(return_code=0)
398        elif stage == 'config':
399            return command.CommandResult(return_code=0,
400                    combined='Test configuration complete')
401        elif stage == 'build':
402            stderr = ''
403            out_dir = ''
404            for arg in args:
405                if arg.startswith('O='):
406                    out_dir = arg[2:]
407            fname = os.path.join(cwd or '', out_dir, 'u-boot')
408            tools.WriteFile(fname, b'U-Boot')
409            if type(commit) is not str:
410                stderr = self._error.get((brd.target, commit.sequence))
411            if stderr:
412                return command.CommandResult(return_code=1, stderr=stderr)
413            return command.CommandResult(return_code=0)
414
415        # Not handled, so abort
416        print('make', stage)
417        sys.exit(1)
418
419    # Example function to print output lines
420    def print_lines(self, lines):
421        print(len(lines))
422        for line in lines:
423            print(line)
424        #self.print_lines(terminal.GetPrintTestLines())
425
426    def testNoBoards(self):
427        """Test that buildman aborts when there are no boards"""
428        self._boards = board.Boards()
429        with self.assertRaises(SystemExit):
430            self._RunControl()
431
432    def testCurrentSource(self):
433        """Very simple test to invoke buildman on the current source"""
434        self.setupToolchains();
435        self._RunControl('-o', self._output_dir)
436        lines = terminal.GetPrintTestLines()
437        self.assertIn('Building current source for %d boards' % len(boards),
438                      lines[0].text)
439
440    def testBadBranch(self):
441        """Test that we can detect an invalid branch"""
442        with self.assertRaises(ValueError):
443            self._RunControl('-b', 'badbranch')
444
445    def testBadToolchain(self):
446        """Test that missing toolchains are detected"""
447        self.setupToolchains();
448        ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
449        lines = terminal.GetPrintTestLines()
450
451        # Buildman always builds the upstream commit as well
452        self.assertIn('Building %d commits for %d boards' %
453                (self._commits, len(boards)), lines[0].text)
454        self.assertEqual(self._builder.count, self._total_builds)
455
456        # Only sandbox should succeed, the others don't have toolchains
457        self.assertEqual(self._builder.fail,
458                         self._total_builds - self._commits)
459        self.assertEqual(ret_code, 100)
460
461        for commit in range(self._commits):
462            for board in self._boards.GetList():
463                if board.arch != 'sandbox':
464                  errfile = self._builder.GetErrFile(commit, board.target)
465                  fd = open(errfile)
466                  self.assertEqual(fd.readlines(),
467                          ['No tool chain for %s\n' % board.arch])
468                  fd.close()
469
470    def testBranch(self):
471        """Test building a branch with all toolchains present"""
472        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
473        self.assertEqual(self._builder.count, self._total_builds)
474        self.assertEqual(self._builder.fail, 0)
475
476    def testCount(self):
477        """Test building a specific number of commitst"""
478        self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir)
479        self.assertEqual(self._builder.count, 2 * len(boards))
480        self.assertEqual(self._builder.fail, 0)
481        # Each board has a config, and then one make per commit
482        self.assertEqual(self._make_calls, len(boards) * (1 + 2))
483
484    def testIncremental(self):
485        """Test building a branch twice - the second time should do nothing"""
486        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
487
488        # Each board has a mrproper, config, and then one make per commit
489        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
490        self._make_calls = 0
491        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
492        self.assertEqual(self._make_calls, 0)
493        self.assertEqual(self._builder.count, self._total_builds)
494        self.assertEqual(self._builder.fail, 0)
495
496    def testForceBuild(self):
497        """The -f flag should force a rebuild"""
498        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
499        self._make_calls = 0
500        self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False)
501        # Each board has a config and one make per commit
502        self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
503
504    def testForceReconfigure(self):
505        """The -f flag should force a rebuild"""
506        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
507        # Each commit has a config and make
508        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
509
510    def testForceReconfigure(self):
511        """The -f flag should force a rebuild"""
512        self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
513        # Each commit has a config and make
514        self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
515
516    def testMrproper(self):
517        """The -f flag should force a rebuild"""
518        self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir)
519        # Each board has a mkproper, config and then one make per commit
520        self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
521
522    def testErrors(self):
523        """Test handling of build errors"""
524        self._error['board2', 1] = 'fred\n'
525        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
526        self.assertEqual(self._builder.count, self._total_builds)
527        self.assertEqual(self._builder.fail, 1)
528
529        # Remove the error. This should have no effect since the commit will
530        # not be rebuilt
531        del self._error['board2', 1]
532        self._make_calls = 0
533        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
534        self.assertEqual(self._builder.count, self._total_builds)
535        self.assertEqual(self._make_calls, 0)
536        self.assertEqual(self._builder.fail, 1)
537
538        # Now use the -F flag to force rebuild of the bad commit
539        self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False)
540        self.assertEqual(self._builder.count, self._total_builds)
541        self.assertEqual(self._builder.fail, 0)
542        self.assertEqual(self._make_calls, 2)
543
544    def testBranchWithSlash(self):
545        """Test building a branch with a '/' in the name"""
546        self._test_branch = '/__dev/__testbranch'
547        self._RunControl('-b', self._test_branch, clean_dir=False)
548        self.assertEqual(self._builder.count, self._total_builds)
549        self.assertEqual(self._builder.fail, 0)
550
551    def testEnvironment(self):
552        """Test that the done and environment files are written to out-env"""
553        self._RunControl('-o', self._output_dir)
554        board0_dir = os.path.join(self._output_dir, 'current', 'board0')
555        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
556        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'out-env')))
557
558    def testWorkInOutput(self):
559        """Test the -w option which should write directly to the output dir"""
560        board_list = board.Boards()
561        board_list.AddBoard(board.Board(*boards[0]))
562        self._RunControl('-o', self._output_dir, '-w', clean_dir=False,
563                         boards=board_list)
564        self.assertTrue(
565            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
566        self.assertTrue(
567            os.path.exists(os.path.join(self._output_dir, 'done')))
568        self.assertTrue(
569            os.path.exists(os.path.join(self._output_dir, 'out-env')))
570
571    def testWorkInOutputFail(self):
572        """Test the -w option failures"""
573        with self.assertRaises(SystemExit) as e:
574            self._RunControl('-o', self._output_dir, '-w', clean_dir=False)
575        self.assertIn("single board", str(e.exception))
576        self.assertFalse(
577            os.path.exists(os.path.join(self._output_dir, 'u-boot')))
578
579        board_list = board.Boards()
580        board_list.AddBoard(board.Board(*boards[0]))
581        with self.assertRaises(SystemExit) as e:
582            self._RunControl('-b', self._test_branch, '-o', self._output_dir,
583                             '-w', clean_dir=False, boards=board_list)
584        self.assertIn("single commit", str(e.exception))
585
586        board_list = board.Boards()
587        board_list.AddBoard(board.Board(*boards[0]))
588        with self.assertRaises(SystemExit) as e:
589            self._RunControl('-w', clean_dir=False)
590        self.assertIn("specify -o", str(e.exception))
591