1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5# 6 7""" 8Move config options from headers to defconfig files. 9 10Since Kconfig was introduced to U-Boot, we have worked on moving 11config options from headers to Kconfig (defconfig). 12 13This tool intends to help this tremendous work. 14 15 16Usage 17----- 18 19First, you must edit the Kconfig to add the menu entries for the configs 20you are moving. 21 22And then run this tool giving CONFIG names you want to move. 23For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 24simply type as follows: 25 26 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 27 28The tool walks through all the defconfig files and move the given CONFIGs. 29 30The log is also displayed on the terminal. 31 32The log is printed for each defconfig as follows: 33 34<defconfig_name> 35 <action1> 36 <action2> 37 <action3> 38 ... 39 40<defconfig_name> is the name of the defconfig. 41 42<action*> shows what the tool did for that defconfig. 43It looks like one of the following: 44 45 - Move 'CONFIG_... ' 46 This config option was moved to the defconfig 47 48 - CONFIG_... is not defined in Kconfig. Do nothing. 49 The entry for this CONFIG was not found in Kconfig. The option is not 50 defined in the config header, either. So, this case can be just skipped. 51 52 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 53 This option is defined in the config header, but its entry was not found 54 in Kconfig. 55 There are two common cases: 56 - You forgot to create an entry for the CONFIG before running 57 this tool, or made a typo in a CONFIG passed to this tool. 58 - The entry was hidden due to unmet 'depends on'. 59 The tool does not know if the result is reasonable, so please check it 60 manually. 61 62 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 63 The define in the config header matched the one in Kconfig. 64 We do not need to touch it. 65 66 - Compiler is missing. Do nothing. 67 The compiler specified for this architecture was not found 68 in your PATH environment. 69 (If -e option is passed, the tool exits immediately.) 70 71 - Failed to process. 72 An error occurred during processing this defconfig. Skipped. 73 (If -e option is passed, the tool exits immediately on error.) 74 75Finally, you will be asked, Clean up headers? [y/n]: 76 77If you say 'y' here, the unnecessary config defines are removed 78from the config headers (include/configs/*.h). 79It just uses the regex method, so you should not rely on it. 80Just in case, please do 'git diff' to see what happened. 81 82 83How does it work? 84----------------- 85 86This tool runs configuration and builds include/autoconf.mk for every 87defconfig. The config options defined in Kconfig appear in the .config 88file (unless they are hidden because of unmet dependency.) 89On the other hand, the config options defined by board headers are seen 90in include/autoconf.mk. The tool looks for the specified options in both 91of them to decide the appropriate action for the options. If the given 92config option is found in the .config, but its value does not match the 93one from the board header, the config option in the .config is replaced 94with the define in the board header. Then, the .config is synced by 95"make savedefconfig" and the defconfig is updated with it. 96 97For faster processing, this tool handles multi-threading. It creates 98separate build directories where the out-of-tree build is run. The 99temporary build directories are automatically created and deleted as 100needed. The number of threads are chosen based on the number of the CPU 101cores of your system although you can change it via -j (--jobs) option. 102 103 104Toolchains 105---------- 106 107Appropriate toolchain are necessary to generate include/autoconf.mk 108for all the architectures supported by U-Boot. Most of them are available 109at the kernel.org site, some are not provided by kernel.org. This tool uses 110the same tools as buildman, so see that tool for setup (e.g. --fetch-arch). 111 112 113Tips and trips 114-------------- 115 116To sync only X86 defconfigs: 117 118 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 119 120or: 121 122 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 123 124To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 125 126 ls configs/{hrcon*,iocon*,strider*} | \ 127 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 128 129 130Finding implied CONFIGs 131----------------------- 132 133Some CONFIG options can be implied by others and this can help to reduce 134the size of the defconfig files. For example, CONFIG_X86 implies 135CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 136all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 137each of the x86 defconfig files. 138 139This tool can help find such configs. To use it, first build a database: 140 141 ./tools/moveconfig.py -b 142 143Then try to query it: 144 145 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 146 CONFIG_CMD_IRQ found in 311/2384 defconfigs 147 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 148 41 : CONFIG_SYS_FSL_ERRATUM_A007075 149 31 : CONFIG_SYS_FSL_DDR_VER_44 150 28 : CONFIG_ARCH_P1010 151 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 152 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 153 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 154 25 : CONFIG_SYS_FSL_ERRATUM_A008044 155 22 : CONFIG_ARCH_P1020 156 21 : CONFIG_SYS_FSL_DDR_VER_46 157 20 : CONFIG_MAX_PIRQ_LINKS 158 20 : CONFIG_HPET_ADDRESS 159 20 : CONFIG_X86 160 20 : CONFIG_PCIE_ECAM_SIZE 161 20 : CONFIG_IRQ_SLOT_COUNT 162 20 : CONFIG_I8259_PIC 163 20 : CONFIG_CPU_ADDR_BITS 164 20 : CONFIG_RAMBASE 165 20 : CONFIG_SYS_FSL_ERRATUM_A005871 166 20 : CONFIG_PCIE_ECAM_BASE 167 20 : CONFIG_X86_TSC_TIMER 168 20 : CONFIG_I8254_TIMER 169 20 : CONFIG_CMD_GETTIME 170 19 : CONFIG_SYS_FSL_ERRATUM_A005812 171 18 : CONFIG_X86_RUN_32BIT 172 17 : CONFIG_CMD_CHIP_CONFIG 173 ... 174 175This shows a list of config options which might imply CONFIG_CMD_EEPROM along 176with how many defconfigs they cover. From this you can see that CONFIG_X86 177implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 178the defconfig of every x86 board, you could add a single imply line to the 179Kconfig file: 180 181 config X86 182 bool "x86 architecture" 183 ... 184 imply CMD_EEPROM 185 186That will cover 20 defconfigs. Many of the options listed are not suitable as 187they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 188CMD_EEPROM. 189 190Using this search you can reduce the size of moveconfig patches. 191 192You can automatically add 'imply' statements in the Kconfig with the -a 193option: 194 195 ./tools/moveconfig.py -s -i CONFIG_SCSI \ 196 -a CONFIG_ARCH_LS1021A,CONFIG_ARCH_LS1043A 197 198This will add 'imply SCSI' to the two CONFIG options mentioned, assuming that 199the database indicates that they do actually imply CONFIG_SCSI and do not 200already have an 'imply SCSI'. 201 202The output shows where the imply is added: 203 204 18 : CONFIG_ARCH_LS1021A arch/arm/cpu/armv7/ls102xa/Kconfig:1 205 13 : CONFIG_ARCH_LS1043A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:11 206 12 : CONFIG_ARCH_LS1046A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:31 207 208The first number is the number of boards which can avoid having a special 209CONFIG_SCSI option in their defconfig file if this 'imply' is added. 210The location at the right is the Kconfig file and line number where the config 211appears. For example, adding 'imply CONFIG_SCSI' to the 'config ARCH_LS1021A' 212in arch/arm/cpu/armv7/ls102xa/Kconfig at line 1 will help 18 boards to reduce 213the size of their defconfig files. 214 215If you want to add an 'imply' to every imply config in the list, you can use 216 217 ./tools/moveconfig.py -s -i CONFIG_SCSI -a all 218 219To control which ones are displayed, use -I <list> where list is a list of 220options (use '-I help' to see possible options and their meaning). 221 222To skip showing you options that already have an 'imply' attached, use -A. 223 224When you have finished adding 'imply' options you can regenerate the 225defconfig files for affected boards with something like: 226 227 git show --stat | ./tools/moveconfig.py -s -d - 228 229This will regenerate only those defconfigs changed in the current commit. 230If you start with (say) 100 defconfigs being changed in the commit, and add 231a few 'imply' options as above, then regenerate, hopefully you can reduce the 232number of defconfigs changed in the commit. 233 234 235Available options 236----------------- 237 238 -c, --color 239 Surround each portion of the log with escape sequences to display it 240 in color on the terminal. 241 242 -C, --commit 243 Create a git commit with the changes when the operation is complete. A 244 standard commit message is used which may need to be edited. 245 246 -d, --defconfigs 247 Specify a file containing a list of defconfigs to move. The defconfig 248 files can be given with shell-style wildcards. Use '-' to read from stdin. 249 250 -n, --dry-run 251 Perform a trial run that does not make any changes. It is useful to 252 see what is going to happen before one actually runs it. 253 254 -e, --exit-on-error 255 Exit immediately if Make exits with a non-zero status while processing 256 a defconfig file. 257 258 -s, --force-sync 259 Do "make savedefconfig" forcibly for all the defconfig files. 260 If not specified, "make savedefconfig" only occurs for cases 261 where at least one CONFIG was moved. 262 263 -S, --spl 264 Look for moved config options in spl/include/autoconf.mk instead of 265 include/autoconf.mk. This is useful for moving options for SPL build 266 because SPL related options (mostly prefixed with CONFIG_SPL_) are 267 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 268 269 -H, --headers-only 270 Only cleanup the headers; skip the defconfig processing 271 272 -j, --jobs 273 Specify the number of threads to run simultaneously. If not specified, 274 the number of threads is the same as the number of CPU cores. 275 276 -r, --git-ref 277 Specify the git ref to clone for building the autoconf.mk. If unspecified 278 use the CWD. This is useful for when changes to the Kconfig affect the 279 default values and you want to capture the state of the defconfig from 280 before that change was in effect. If in doubt, specify a ref pre-Kconfig 281 changes (use HEAD if Kconfig changes are not committed). Worst case it will 282 take a bit longer to run, but will always do the right thing. 283 284 -v, --verbose 285 Show any build errors as boards are built 286 287 -y, --yes 288 Instead of prompting, automatically go ahead with all operations. This 289 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 290 and the README. 291 292To see the complete list of supported options, run 293 294 $ tools/moveconfig.py -h 295 296""" 297 298import asteval 299import collections 300import copy 301import difflib 302import filecmp 303import fnmatch 304import glob 305import multiprocessing 306import optparse 307import os 308import queue 309import re 310import shutil 311import subprocess 312import sys 313import tempfile 314import threading 315import time 316 317from buildman import bsettings 318from buildman import kconfiglib 319from buildman import toolchain 320 321SHOW_GNU_MAKE = 'scripts/show-gnu-make' 322SLEEP_TIME=0.03 323 324STATE_IDLE = 0 325STATE_DEFCONFIG = 1 326STATE_AUTOCONF = 2 327STATE_SAVEDEFCONFIG = 3 328 329ACTION_MOVE = 0 330ACTION_NO_ENTRY = 1 331ACTION_NO_ENTRY_WARN = 2 332ACTION_NO_CHANGE = 3 333 334COLOR_BLACK = '0;30' 335COLOR_RED = '0;31' 336COLOR_GREEN = '0;32' 337COLOR_BROWN = '0;33' 338COLOR_BLUE = '0;34' 339COLOR_PURPLE = '0;35' 340COLOR_CYAN = '0;36' 341COLOR_LIGHT_GRAY = '0;37' 342COLOR_DARK_GRAY = '1;30' 343COLOR_LIGHT_RED = '1;31' 344COLOR_LIGHT_GREEN = '1;32' 345COLOR_YELLOW = '1;33' 346COLOR_LIGHT_BLUE = '1;34' 347COLOR_LIGHT_PURPLE = '1;35' 348COLOR_LIGHT_CYAN = '1;36' 349COLOR_WHITE = '1;37' 350 351AUTO_CONF_PATH = 'include/config/auto.conf' 352CONFIG_DATABASE = 'moveconfig.db' 353 354CONFIG_LEN = len('CONFIG_') 355 356SIZES = { 357 "SZ_1": 0x00000001, "SZ_2": 0x00000002, 358 "SZ_4": 0x00000004, "SZ_8": 0x00000008, 359 "SZ_16": 0x00000010, "SZ_32": 0x00000020, 360 "SZ_64": 0x00000040, "SZ_128": 0x00000080, 361 "SZ_256": 0x00000100, "SZ_512": 0x00000200, 362 "SZ_1K": 0x00000400, "SZ_2K": 0x00000800, 363 "SZ_4K": 0x00001000, "SZ_8K": 0x00002000, 364 "SZ_16K": 0x00004000, "SZ_32K": 0x00008000, 365 "SZ_64K": 0x00010000, "SZ_128K": 0x00020000, 366 "SZ_256K": 0x00040000, "SZ_512K": 0x00080000, 367 "SZ_1M": 0x00100000, "SZ_2M": 0x00200000, 368 "SZ_4M": 0x00400000, "SZ_8M": 0x00800000, 369 "SZ_16M": 0x01000000, "SZ_32M": 0x02000000, 370 "SZ_64M": 0x04000000, "SZ_128M": 0x08000000, 371 "SZ_256M": 0x10000000, "SZ_512M": 0x20000000, 372 "SZ_1G": 0x40000000, "SZ_2G": 0x80000000, 373 "SZ_4G": 0x100000000 374} 375 376### helper functions ### 377def get_devnull(): 378 """Get the file object of '/dev/null' device.""" 379 try: 380 devnull = subprocess.DEVNULL # py3k 381 except AttributeError: 382 devnull = open(os.devnull, 'wb') 383 return devnull 384 385def check_top_directory(): 386 """Exit if we are not at the top of source directory.""" 387 for f in ('README', 'Licenses'): 388 if not os.path.exists(f): 389 sys.exit('Please run at the top of source directory.') 390 391def check_clean_directory(): 392 """Exit if the source tree is not clean.""" 393 for f in ('.config', 'include/config'): 394 if os.path.exists(f): 395 sys.exit("source tree is not clean, please run 'make mrproper'") 396 397def get_make_cmd(): 398 """Get the command name of GNU Make. 399 400 U-Boot needs GNU Make for building, but the command name is not 401 necessarily "make". (for example, "gmake" on FreeBSD). 402 Returns the most appropriate command name on your system. 403 """ 404 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 405 ret = process.communicate() 406 if process.returncode: 407 sys.exit('GNU Make not found') 408 return ret[0].rstrip() 409 410def get_matched_defconfig(line): 411 """Get the defconfig files that match a pattern 412 413 Args: 414 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 415 'k2*_defconfig'. If no directory is provided, 'configs/' is 416 prepended 417 418 Returns: 419 a list of matching defconfig files 420 """ 421 dirname = os.path.dirname(line) 422 if dirname: 423 pattern = line 424 else: 425 pattern = os.path.join('configs', line) 426 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 427 428def get_matched_defconfigs(defconfigs_file): 429 """Get all the defconfig files that match the patterns in a file. 430 431 Args: 432 defconfigs_file: File containing a list of defconfigs to process, or 433 '-' to read the list from stdin 434 435 Returns: 436 A list of paths to defconfig files, with no duplicates 437 """ 438 defconfigs = [] 439 if defconfigs_file == '-': 440 fd = sys.stdin 441 defconfigs_file = 'stdin' 442 else: 443 fd = open(defconfigs_file) 444 for i, line in enumerate(fd): 445 line = line.strip() 446 if not line: 447 continue # skip blank lines silently 448 if ' ' in line: 449 line = line.split(' ')[0] # handle 'git log' input 450 matched = get_matched_defconfig(line) 451 if not matched: 452 print("warning: %s:%d: no defconfig matched '%s'" % \ 453 (defconfigs_file, i + 1, line), file=sys.stderr) 454 455 defconfigs += matched 456 457 # use set() to drop multiple matching 458 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 459 460def get_all_defconfigs(): 461 """Get all the defconfig files under the configs/ directory.""" 462 defconfigs = [] 463 for (dirpath, dirnames, filenames) in os.walk('configs'): 464 dirpath = dirpath[len('configs') + 1:] 465 for filename in fnmatch.filter(filenames, '*_defconfig'): 466 defconfigs.append(os.path.join(dirpath, filename)) 467 468 return defconfigs 469 470def color_text(color_enabled, color, string): 471 """Return colored string.""" 472 if color_enabled: 473 # LF should not be surrounded by the escape sequence. 474 # Otherwise, additional whitespace or line-feed might be printed. 475 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 476 for s in string.split('\n') ]) 477 else: 478 return string 479 480def show_diff(a, b, file_path, color_enabled): 481 """Show unidified diff. 482 483 Arguments: 484 a: A list of lines (before) 485 b: A list of lines (after) 486 file_path: Path to the file 487 color_enabled: Display the diff in color 488 """ 489 490 diff = difflib.unified_diff(a, b, 491 fromfile=os.path.join('a', file_path), 492 tofile=os.path.join('b', file_path)) 493 494 for line in diff: 495 if line[0] == '-' and line[1] != '-': 496 print(color_text(color_enabled, COLOR_RED, line), end=' ') 497 elif line[0] == '+' and line[1] != '+': 498 print(color_text(color_enabled, COLOR_GREEN, line), end=' ') 499 else: 500 print(line, end=' ') 501 502def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 503 extend_post): 504 """Extend matched lines if desired patterns are found before/after already 505 matched lines. 506 507 Arguments: 508 lines: A list of lines handled. 509 matched: A list of line numbers that have been already matched. 510 (will be updated by this function) 511 pre_patterns: A list of regular expression that should be matched as 512 preamble. 513 post_patterns: A list of regular expression that should be matched as 514 postamble. 515 extend_pre: Add the line number of matched preamble to the matched list. 516 extend_post: Add the line number of matched postamble to the matched list. 517 """ 518 extended_matched = [] 519 520 j = matched[0] 521 522 for i in matched: 523 if i == 0 or i < j: 524 continue 525 j = i 526 while j in matched: 527 j += 1 528 if j >= len(lines): 529 break 530 531 for p in pre_patterns: 532 if p.search(lines[i - 1]): 533 break 534 else: 535 # not matched 536 continue 537 538 for p in post_patterns: 539 if p.search(lines[j]): 540 break 541 else: 542 # not matched 543 continue 544 545 if extend_pre: 546 extended_matched.append(i - 1) 547 if extend_post: 548 extended_matched.append(j) 549 550 matched += extended_matched 551 matched.sort() 552 553def confirm(options, prompt): 554 if not options.yes: 555 while True: 556 choice = input('{} [y/n]: '.format(prompt)) 557 choice = choice.lower() 558 print(choice) 559 if choice == 'y' or choice == 'n': 560 break 561 562 if choice == 'n': 563 return False 564 565 return True 566 567def cleanup_empty_blocks(header_path, options): 568 """Clean up empty conditional blocks 569 570 Arguments: 571 header_path: path to the cleaned file. 572 options: option flags. 573 """ 574 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M) 575 with open(header_path) as f: 576 data = f.read() 577 578 new_data = pattern.sub('\n', data) 579 580 show_diff(data.splitlines(True), new_data.splitlines(True), header_path, 581 options.color) 582 583 if options.dry_run: 584 return 585 586 with open(header_path, 'w') as f: 587 f.write(new_data) 588 589def cleanup_one_header(header_path, patterns, options): 590 """Clean regex-matched lines away from a file. 591 592 Arguments: 593 header_path: path to the cleaned file. 594 patterns: list of regex patterns. Any lines matching to these 595 patterns are deleted. 596 options: option flags. 597 """ 598 with open(header_path) as f: 599 lines = f.readlines() 600 601 matched = [] 602 for i, line in enumerate(lines): 603 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 604 matched.append(i) 605 continue 606 for pattern in patterns: 607 if pattern.search(line): 608 matched.append(i) 609 break 610 611 if not matched: 612 return 613 614 # remove empty #ifdef ... #endif, successive blank lines 615 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 616 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 617 pattern_endif = re.compile(r'#\s*endif\W') # #endif 618 pattern_blank = re.compile(r'^\s*$') # empty line 619 620 while True: 621 old_matched = copy.copy(matched) 622 extend_matched_lines(lines, matched, [pattern_if], 623 [pattern_endif], True, True) 624 extend_matched_lines(lines, matched, [pattern_elif], 625 [pattern_elif, pattern_endif], True, False) 626 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 627 [pattern_blank], False, True) 628 extend_matched_lines(lines, matched, [pattern_blank], 629 [pattern_elif, pattern_endif], True, False) 630 extend_matched_lines(lines, matched, [pattern_blank], 631 [pattern_blank], True, False) 632 if matched == old_matched: 633 break 634 635 tolines = copy.copy(lines) 636 637 for i in reversed(matched): 638 tolines.pop(i) 639 640 show_diff(lines, tolines, header_path, options.color) 641 642 if options.dry_run: 643 return 644 645 with open(header_path, 'w') as f: 646 for line in tolines: 647 f.write(line) 648 649def cleanup_headers(configs, options): 650 """Delete config defines from board headers. 651 652 Arguments: 653 configs: A list of CONFIGs to remove. 654 options: option flags. 655 """ 656 if not confirm(options, 'Clean up headers?'): 657 return 658 659 patterns = [] 660 for config in configs: 661 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 662 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 663 664 for dir in 'include', 'arch', 'board': 665 for (dirpath, dirnames, filenames) in os.walk(dir): 666 if dirpath == os.path.join('include', 'generated'): 667 continue 668 for filename in filenames: 669 if not filename.endswith(('~', '.dts', '.dtsi', '.bin', 670 '.elf')): 671 header_path = os.path.join(dirpath, filename) 672 # This file contains UTF-16 data and no CONFIG symbols 673 if header_path == 'include/video_font_data.h': 674 continue 675 cleanup_one_header(header_path, patterns, options) 676 cleanup_empty_blocks(header_path, options) 677 678def cleanup_one_extra_option(defconfig_path, configs, options): 679 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 680 681 Arguments: 682 defconfig_path: path to the cleaned defconfig file. 683 configs: A list of CONFIGs to remove. 684 options: option flags. 685 """ 686 687 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 688 end = '"\n' 689 690 with open(defconfig_path) as f: 691 lines = f.readlines() 692 693 for i, line in enumerate(lines): 694 if line.startswith(start) and line.endswith(end): 695 break 696 else: 697 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 698 return 699 700 old_tokens = line[len(start):-len(end)].split(',') 701 new_tokens = [] 702 703 for token in old_tokens: 704 pos = token.find('=') 705 if not (token[:pos] if pos >= 0 else token) in configs: 706 new_tokens.append(token) 707 708 if new_tokens == old_tokens: 709 return 710 711 tolines = copy.copy(lines) 712 713 if new_tokens: 714 tolines[i] = start + ','.join(new_tokens) + end 715 else: 716 tolines.pop(i) 717 718 show_diff(lines, tolines, defconfig_path, options.color) 719 720 if options.dry_run: 721 return 722 723 with open(defconfig_path, 'w') as f: 724 for line in tolines: 725 f.write(line) 726 727def cleanup_extra_options(configs, options): 728 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 729 730 Arguments: 731 configs: A list of CONFIGs to remove. 732 options: option flags. 733 """ 734 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 735 return 736 737 configs = [ config[len('CONFIG_'):] for config in configs ] 738 739 defconfigs = get_all_defconfigs() 740 741 for defconfig in defconfigs: 742 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 743 options) 744 745def cleanup_whitelist(configs, options): 746 """Delete config whitelist entries 747 748 Arguments: 749 configs: A list of CONFIGs to remove. 750 options: option flags. 751 """ 752 if not confirm(options, 'Clean up whitelist entries?'): 753 return 754 755 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 756 lines = f.readlines() 757 758 lines = [x for x in lines if x.strip() not in configs] 759 760 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 761 f.write(''.join(lines)) 762 763def find_matching(patterns, line): 764 for pat in patterns: 765 if pat.search(line): 766 return True 767 return False 768 769def cleanup_readme(configs, options): 770 """Delete config description in README 771 772 Arguments: 773 configs: A list of CONFIGs to remove. 774 options: option flags. 775 """ 776 if not confirm(options, 'Clean up README?'): 777 return 778 779 patterns = [] 780 for config in configs: 781 patterns.append(re.compile(r'^\s+%s' % config)) 782 783 with open('README') as f: 784 lines = f.readlines() 785 786 found = False 787 newlines = [] 788 for line in lines: 789 if not found: 790 found = find_matching(patterns, line) 791 if found: 792 continue 793 794 if found and re.search(r'^\s+CONFIG', line): 795 found = False 796 797 if not found: 798 newlines.append(line) 799 800 with open('README', 'w') as f: 801 f.write(''.join(newlines)) 802 803def try_expand(line): 804 """If value looks like an expression, try expanding it 805 Otherwise just return the existing value 806 """ 807 if line.find('=') == -1: 808 return line 809 810 try: 811 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) 812 cfg, val = re.split("=", line) 813 val= val.strip('\"') 814 if re.search("[*+-/]|<<|SZ_+|\(([^\)]+)\)", val): 815 newval = hex(aeval(val)) 816 print("\tExpanded expression %s to %s" % (val, newval)) 817 return cfg+'='+newval 818 except: 819 print("\tFailed to expand expression in %s" % line) 820 821 return line 822 823 824### classes ### 825class Progress: 826 827 """Progress Indicator""" 828 829 def __init__(self, total): 830 """Create a new progress indicator. 831 832 Arguments: 833 total: A number of defconfig files to process. 834 """ 835 self.current = 0 836 self.total = total 837 838 def inc(self): 839 """Increment the number of processed defconfig files.""" 840 841 self.current += 1 842 843 def show(self): 844 """Display the progress.""" 845 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ') 846 sys.stdout.flush() 847 848 849class KconfigScanner: 850 """Kconfig scanner.""" 851 852 def __init__(self): 853 """Scan all the Kconfig files and create a Config object.""" 854 # Define environment variables referenced from Kconfig 855 os.environ['srctree'] = os.getcwd() 856 os.environ['UBOOTVERSION'] = 'dummy' 857 os.environ['KCONFIG_OBJDIR'] = '' 858 self.conf = kconfiglib.Kconfig() 859 860 861class KconfigParser: 862 863 """A parser of .config and include/autoconf.mk.""" 864 865 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 866 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 867 868 def __init__(self, configs, options, build_dir): 869 """Create a new parser. 870 871 Arguments: 872 configs: A list of CONFIGs to move. 873 options: option flags. 874 build_dir: Build directory. 875 """ 876 self.configs = configs 877 self.options = options 878 self.dotconfig = os.path.join(build_dir, '.config') 879 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 880 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 881 'autoconf.mk') 882 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 883 self.defconfig = os.path.join(build_dir, 'defconfig') 884 885 def get_arch(self): 886 """Parse .config file and return the architecture. 887 888 Returns: 889 Architecture name (e.g. 'arm'). 890 """ 891 arch = '' 892 cpu = '' 893 for line in open(self.dotconfig): 894 m = self.re_arch.match(line) 895 if m: 896 arch = m.group(1) 897 continue 898 m = self.re_cpu.match(line) 899 if m: 900 cpu = m.group(1) 901 902 if not arch: 903 return None 904 905 # fix-up for aarch64 906 if arch == 'arm' and cpu == 'armv8': 907 arch = 'aarch64' 908 909 return arch 910 911 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 912 """Parse .config, defconfig, include/autoconf.mk for one config. 913 914 This function looks for the config options in the lines from 915 defconfig, .config, and include/autoconf.mk in order to decide 916 which action should be taken for this defconfig. 917 918 Arguments: 919 config: CONFIG name to parse. 920 dotconfig_lines: lines from the .config file. 921 autoconf_lines: lines from the include/autoconf.mk file. 922 923 Returns: 924 A tupple of the action for this defconfig and the line 925 matched for the config. 926 """ 927 not_set = '# %s is not set' % config 928 929 for line in autoconf_lines: 930 line = line.rstrip() 931 if line.startswith(config + '='): 932 new_val = line 933 break 934 else: 935 new_val = not_set 936 937 new_val = try_expand(new_val) 938 939 for line in dotconfig_lines: 940 line = line.rstrip() 941 if line.startswith(config + '=') or line == not_set: 942 old_val = line 943 break 944 else: 945 if new_val == not_set: 946 return (ACTION_NO_ENTRY, config) 947 else: 948 return (ACTION_NO_ENTRY_WARN, config) 949 950 # If this CONFIG is neither bool nor trisate 951 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 952 # tools/scripts/define2mk.sed changes '1' to 'y'. 953 # This is a problem if the CONFIG is int type. 954 # Check the type in Kconfig and handle it correctly. 955 if new_val[-2:] == '=y': 956 new_val = new_val[:-1] + '1' 957 958 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 959 new_val) 960 961 def update_dotconfig(self): 962 """Parse files for the config options and update the .config. 963 964 This function parses the generated .config and include/autoconf.mk 965 searching the target options. 966 Move the config option(s) to the .config as needed. 967 968 Arguments: 969 defconfig: defconfig name. 970 971 Returns: 972 Return a tuple of (updated flag, log string). 973 The "updated flag" is True if the .config was updated, False 974 otherwise. The "log string" shows what happend to the .config. 975 """ 976 977 results = [] 978 updated = False 979 suspicious = False 980 rm_files = [self.config_autoconf, self.autoconf] 981 982 if self.options.spl: 983 if os.path.exists(self.spl_autoconf): 984 autoconf_path = self.spl_autoconf 985 rm_files.append(self.spl_autoconf) 986 else: 987 for f in rm_files: 988 os.remove(f) 989 return (updated, suspicious, 990 color_text(self.options.color, COLOR_BROWN, 991 "SPL is not enabled. Skipped.") + '\n') 992 else: 993 autoconf_path = self.autoconf 994 995 with open(self.dotconfig) as f: 996 dotconfig_lines = f.readlines() 997 998 with open(autoconf_path) as f: 999 autoconf_lines = f.readlines() 1000 1001 for config in self.configs: 1002 result = self.parse_one_config(config, dotconfig_lines, 1003 autoconf_lines) 1004 results.append(result) 1005 1006 log = '' 1007 1008 for (action, value) in results: 1009 if action == ACTION_MOVE: 1010 actlog = "Move '%s'" % value 1011 log_color = COLOR_LIGHT_GREEN 1012 elif action == ACTION_NO_ENTRY: 1013 actlog = "%s is not defined in Kconfig. Do nothing." % value 1014 log_color = COLOR_LIGHT_BLUE 1015 elif action == ACTION_NO_ENTRY_WARN: 1016 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 1017 log_color = COLOR_YELLOW 1018 suspicious = True 1019 elif action == ACTION_NO_CHANGE: 1020 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 1021 % value 1022 log_color = COLOR_LIGHT_PURPLE 1023 elif action == ACTION_SPL_NOT_EXIST: 1024 actlog = "SPL is not enabled for this defconfig. Skip." 1025 log_color = COLOR_PURPLE 1026 else: 1027 sys.exit("Internal Error. This should not happen.") 1028 1029 log += color_text(self.options.color, log_color, actlog) + '\n' 1030 1031 with open(self.dotconfig, 'a') as f: 1032 for (action, value) in results: 1033 if action == ACTION_MOVE: 1034 f.write(value + '\n') 1035 updated = True 1036 1037 self.results = results 1038 for f in rm_files: 1039 os.remove(f) 1040 1041 return (updated, suspicious, log) 1042 1043 def check_defconfig(self): 1044 """Check the defconfig after savedefconfig 1045 1046 Returns: 1047 Return additional log if moved CONFIGs were removed again by 1048 'make savedefconfig'. 1049 """ 1050 1051 log = '' 1052 1053 with open(self.defconfig) as f: 1054 defconfig_lines = f.readlines() 1055 1056 for (action, value) in self.results: 1057 if action != ACTION_MOVE: 1058 continue 1059 if not value + '\n' in defconfig_lines: 1060 log += color_text(self.options.color, COLOR_YELLOW, 1061 "'%s' was removed by savedefconfig.\n" % 1062 value) 1063 1064 return log 1065 1066 1067class DatabaseThread(threading.Thread): 1068 """This thread processes results from Slot threads. 1069 1070 It collects the data in the master config directary. There is only one 1071 result thread, and this helps to serialise the build output. 1072 """ 1073 def __init__(self, config_db, db_queue): 1074 """Set up a new result thread 1075 1076 Args: 1077 builder: Builder which will be sent each result 1078 """ 1079 threading.Thread.__init__(self) 1080 self.config_db = config_db 1081 self.db_queue= db_queue 1082 1083 def run(self): 1084 """Called to start up the result thread. 1085 1086 We collect the next result job and pass it on to the build. 1087 """ 1088 while True: 1089 defconfig, configs = self.db_queue.get() 1090 self.config_db[defconfig] = configs 1091 self.db_queue.task_done() 1092 1093 1094class Slot: 1095 1096 """A slot to store a subprocess. 1097 1098 Each instance of this class handles one subprocess. 1099 This class is useful to control multiple threads 1100 for faster processing. 1101 """ 1102 1103 def __init__(self, toolchains, configs, options, progress, devnull, 1104 make_cmd, reference_src_dir, db_queue): 1105 """Create a new process slot. 1106 1107 Arguments: 1108 toolchains: Toolchains object containing toolchains. 1109 configs: A list of CONFIGs to move. 1110 options: option flags. 1111 progress: A progress indicator. 1112 devnull: A file object of '/dev/null'. 1113 make_cmd: command name of GNU Make. 1114 reference_src_dir: Determine the true starting config state from this 1115 source tree. 1116 db_queue: output queue to write config info for the database 1117 """ 1118 self.toolchains = toolchains 1119 self.options = options 1120 self.progress = progress 1121 self.build_dir = tempfile.mkdtemp() 1122 self.devnull = devnull 1123 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1124 self.reference_src_dir = reference_src_dir 1125 self.db_queue = db_queue 1126 self.parser = KconfigParser(configs, options, self.build_dir) 1127 self.state = STATE_IDLE 1128 self.failed_boards = set() 1129 self.suspicious_boards = set() 1130 1131 def __del__(self): 1132 """Delete the working directory 1133 1134 This function makes sure the temporary directory is cleaned away 1135 even if Python suddenly dies due to error. It should be done in here 1136 because it is guaranteed the destructor is always invoked when the 1137 instance of the class gets unreferenced. 1138 1139 If the subprocess is still running, wait until it finishes. 1140 """ 1141 if self.state != STATE_IDLE: 1142 while self.ps.poll() == None: 1143 pass 1144 shutil.rmtree(self.build_dir) 1145 1146 def add(self, defconfig): 1147 """Assign a new subprocess for defconfig and add it to the slot. 1148 1149 If the slot is vacant, create a new subprocess for processing the 1150 given defconfig and add it to the slot. Just returns False if 1151 the slot is occupied (i.e. the current subprocess is still running). 1152 1153 Arguments: 1154 defconfig: defconfig name. 1155 1156 Returns: 1157 Return True on success or False on failure 1158 """ 1159 if self.state != STATE_IDLE: 1160 return False 1161 1162 self.defconfig = defconfig 1163 self.log = '' 1164 self.current_src_dir = self.reference_src_dir 1165 self.do_defconfig() 1166 return True 1167 1168 def poll(self): 1169 """Check the status of the subprocess and handle it as needed. 1170 1171 Returns True if the slot is vacant (i.e. in idle state). 1172 If the configuration is successfully finished, assign a new 1173 subprocess to build include/autoconf.mk. 1174 If include/autoconf.mk is generated, invoke the parser to 1175 parse the .config and the include/autoconf.mk, moving 1176 config options to the .config as needed. 1177 If the .config was updated, run "make savedefconfig" to sync 1178 it, update the original defconfig, and then set the slot back 1179 to the idle state. 1180 1181 Returns: 1182 Return True if the subprocess is terminated, False otherwise 1183 """ 1184 if self.state == STATE_IDLE: 1185 return True 1186 1187 if self.ps.poll() == None: 1188 return False 1189 1190 if self.ps.poll() != 0: 1191 self.handle_error() 1192 elif self.state == STATE_DEFCONFIG: 1193 if self.reference_src_dir and not self.current_src_dir: 1194 self.do_savedefconfig() 1195 else: 1196 self.do_autoconf() 1197 elif self.state == STATE_AUTOCONF: 1198 if self.current_src_dir: 1199 self.current_src_dir = None 1200 self.do_defconfig() 1201 elif self.options.build_db: 1202 self.do_build_db() 1203 else: 1204 self.do_savedefconfig() 1205 elif self.state == STATE_SAVEDEFCONFIG: 1206 self.update_defconfig() 1207 else: 1208 sys.exit("Internal Error. This should not happen.") 1209 1210 return True if self.state == STATE_IDLE else False 1211 1212 def handle_error(self): 1213 """Handle error cases.""" 1214 1215 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1216 "Failed to process.\n") 1217 if self.options.verbose: 1218 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1219 self.ps.stderr.read().decode()) 1220 self.finish(False) 1221 1222 def do_defconfig(self): 1223 """Run 'make <board>_defconfig' to create the .config file.""" 1224 1225 cmd = list(self.make_cmd) 1226 cmd.append(self.defconfig) 1227 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1228 stderr=subprocess.PIPE, 1229 cwd=self.current_src_dir) 1230 self.state = STATE_DEFCONFIG 1231 1232 def do_autoconf(self): 1233 """Run 'make AUTO_CONF_PATH'.""" 1234 1235 arch = self.parser.get_arch() 1236 try: 1237 toolchain = self.toolchains.Select(arch) 1238 except ValueError: 1239 self.log += color_text(self.options.color, COLOR_YELLOW, 1240 "Tool chain for '%s' is missing. Do nothing.\n" % arch) 1241 self.finish(False) 1242 return 1243 env = toolchain.MakeEnvironment(False) 1244 1245 cmd = list(self.make_cmd) 1246 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1247 cmd.append(AUTO_CONF_PATH) 1248 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env, 1249 stderr=subprocess.PIPE, 1250 cwd=self.current_src_dir) 1251 self.state = STATE_AUTOCONF 1252 1253 def do_build_db(self): 1254 """Add the board to the database""" 1255 configs = {} 1256 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1257 for line in fd.readlines(): 1258 if line.startswith('CONFIG'): 1259 config, value = line.split('=', 1) 1260 configs[config] = value.rstrip() 1261 self.db_queue.put([self.defconfig, configs]) 1262 self.finish(True) 1263 1264 def do_savedefconfig(self): 1265 """Update the .config and run 'make savedefconfig'.""" 1266 1267 (updated, suspicious, log) = self.parser.update_dotconfig() 1268 if suspicious: 1269 self.suspicious_boards.add(self.defconfig) 1270 self.log += log 1271 1272 if not self.options.force_sync and not updated: 1273 self.finish(True) 1274 return 1275 if updated: 1276 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1277 "Syncing by savedefconfig...\n") 1278 else: 1279 self.log += "Syncing by savedefconfig (forced by option)...\n" 1280 1281 cmd = list(self.make_cmd) 1282 cmd.append('savedefconfig') 1283 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1284 stderr=subprocess.PIPE) 1285 self.state = STATE_SAVEDEFCONFIG 1286 1287 def update_defconfig(self): 1288 """Update the input defconfig and go back to the idle state.""" 1289 1290 log = self.parser.check_defconfig() 1291 if log: 1292 self.suspicious_boards.add(self.defconfig) 1293 self.log += log 1294 orig_defconfig = os.path.join('configs', self.defconfig) 1295 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1296 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1297 1298 if updated: 1299 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1300 "defconfig was updated.\n") 1301 1302 if not self.options.dry_run and updated: 1303 shutil.move(new_defconfig, orig_defconfig) 1304 self.finish(True) 1305 1306 def finish(self, success): 1307 """Display log along with progress and go to the idle state. 1308 1309 Arguments: 1310 success: Should be True when the defconfig was processed 1311 successfully, or False when it fails. 1312 """ 1313 # output at least 30 characters to hide the "* defconfigs out of *". 1314 log = self.defconfig.ljust(30) + '\n' 1315 1316 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1317 # Some threads are running in parallel. 1318 # Print log atomically to not mix up logs from different threads. 1319 print(log, file=(sys.stdout if success else sys.stderr)) 1320 1321 if not success: 1322 if self.options.exit_on_error: 1323 sys.exit("Exit on error.") 1324 # If --exit-on-error flag is not set, skip this board and continue. 1325 # Record the failed board. 1326 self.failed_boards.add(self.defconfig) 1327 1328 self.progress.inc() 1329 self.progress.show() 1330 self.state = STATE_IDLE 1331 1332 def get_failed_boards(self): 1333 """Returns a set of failed boards (defconfigs) in this slot. 1334 """ 1335 return self.failed_boards 1336 1337 def get_suspicious_boards(self): 1338 """Returns a set of boards (defconfigs) with possible misconversion. 1339 """ 1340 return self.suspicious_boards - self.failed_boards 1341 1342class Slots: 1343 1344 """Controller of the array of subprocess slots.""" 1345 1346 def __init__(self, toolchains, configs, options, progress, 1347 reference_src_dir, db_queue): 1348 """Create a new slots controller. 1349 1350 Arguments: 1351 toolchains: Toolchains object containing toolchains. 1352 configs: A list of CONFIGs to move. 1353 options: option flags. 1354 progress: A progress indicator. 1355 reference_src_dir: Determine the true starting config state from this 1356 source tree. 1357 db_queue: output queue to write config info for the database 1358 """ 1359 self.options = options 1360 self.slots = [] 1361 devnull = get_devnull() 1362 make_cmd = get_make_cmd() 1363 for i in range(options.jobs): 1364 self.slots.append(Slot(toolchains, configs, options, progress, 1365 devnull, make_cmd, reference_src_dir, 1366 db_queue)) 1367 1368 def add(self, defconfig): 1369 """Add a new subprocess if a vacant slot is found. 1370 1371 Arguments: 1372 defconfig: defconfig name to be put into. 1373 1374 Returns: 1375 Return True on success or False on failure 1376 """ 1377 for slot in self.slots: 1378 if slot.add(defconfig): 1379 return True 1380 return False 1381 1382 def available(self): 1383 """Check if there is a vacant slot. 1384 1385 Returns: 1386 Return True if at lease one vacant slot is found, False otherwise. 1387 """ 1388 for slot in self.slots: 1389 if slot.poll(): 1390 return True 1391 return False 1392 1393 def empty(self): 1394 """Check if all slots are vacant. 1395 1396 Returns: 1397 Return True if all the slots are vacant, False otherwise. 1398 """ 1399 ret = True 1400 for slot in self.slots: 1401 if not slot.poll(): 1402 ret = False 1403 return ret 1404 1405 def show_failed_boards(self): 1406 """Display all of the failed boards (defconfigs).""" 1407 boards = set() 1408 output_file = 'moveconfig.failed' 1409 1410 for slot in self.slots: 1411 boards |= slot.get_failed_boards() 1412 1413 if boards: 1414 boards = '\n'.join(boards) + '\n' 1415 msg = "The following boards were not processed due to error:\n" 1416 msg += boards 1417 msg += "(the list has been saved in %s)\n" % output_file 1418 print(color_text(self.options.color, COLOR_LIGHT_RED, 1419 msg), file=sys.stderr) 1420 1421 with open(output_file, 'w') as f: 1422 f.write(boards) 1423 1424 def show_suspicious_boards(self): 1425 """Display all boards (defconfigs) with possible misconversion.""" 1426 boards = set() 1427 output_file = 'moveconfig.suspicious' 1428 1429 for slot in self.slots: 1430 boards |= slot.get_suspicious_boards() 1431 1432 if boards: 1433 boards = '\n'.join(boards) + '\n' 1434 msg = "The following boards might have been converted incorrectly.\n" 1435 msg += "It is highly recommended to check them manually:\n" 1436 msg += boards 1437 msg += "(the list has been saved in %s)\n" % output_file 1438 print(color_text(self.options.color, COLOR_YELLOW, 1439 msg), file=sys.stderr) 1440 1441 with open(output_file, 'w') as f: 1442 f.write(boards) 1443 1444class ReferenceSource: 1445 1446 """Reference source against which original configs should be parsed.""" 1447 1448 def __init__(self, commit): 1449 """Create a reference source directory based on a specified commit. 1450 1451 Arguments: 1452 commit: commit to git-clone 1453 """ 1454 self.src_dir = tempfile.mkdtemp() 1455 print("Cloning git repo to a separate work directory...") 1456 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1457 cwd=self.src_dir) 1458 print("Checkout '%s' to build the original autoconf.mk." % \ 1459 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()) 1460 subprocess.check_output(['git', 'checkout', commit], 1461 stderr=subprocess.STDOUT, cwd=self.src_dir) 1462 1463 def __del__(self): 1464 """Delete the reference source directory 1465 1466 This function makes sure the temporary directory is cleaned away 1467 even if Python suddenly dies due to error. It should be done in here 1468 because it is guaranteed the destructor is always invoked when the 1469 instance of the class gets unreferenced. 1470 """ 1471 shutil.rmtree(self.src_dir) 1472 1473 def get_dir(self): 1474 """Return the absolute path to the reference source directory.""" 1475 1476 return self.src_dir 1477 1478def move_config(toolchains, configs, options, db_queue): 1479 """Move config options to defconfig files. 1480 1481 Arguments: 1482 configs: A list of CONFIGs to move. 1483 options: option flags 1484 """ 1485 if len(configs) == 0: 1486 if options.force_sync: 1487 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ') 1488 elif options.build_db: 1489 print('Building %s database' % CONFIG_DATABASE) 1490 else: 1491 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ') 1492 else: 1493 print('Move ' + ', '.join(configs), end=' ') 1494 print('(jobs: %d)\n' % options.jobs) 1495 1496 if options.git_ref: 1497 reference_src = ReferenceSource(options.git_ref) 1498 reference_src_dir = reference_src.get_dir() 1499 else: 1500 reference_src_dir = None 1501 1502 if options.defconfigs: 1503 defconfigs = get_matched_defconfigs(options.defconfigs) 1504 else: 1505 defconfigs = get_all_defconfigs() 1506 1507 progress = Progress(len(defconfigs)) 1508 slots = Slots(toolchains, configs, options, progress, reference_src_dir, 1509 db_queue) 1510 1511 # Main loop to process defconfig files: 1512 # Add a new subprocess into a vacant slot. 1513 # Sleep if there is no available slot. 1514 for defconfig in defconfigs: 1515 while not slots.add(defconfig): 1516 while not slots.available(): 1517 # No available slot: sleep for a while 1518 time.sleep(SLEEP_TIME) 1519 1520 # wait until all the subprocesses finish 1521 while not slots.empty(): 1522 time.sleep(SLEEP_TIME) 1523 1524 print('') 1525 slots.show_failed_boards() 1526 slots.show_suspicious_boards() 1527 1528def find_kconfig_rules(kconf, config, imply_config): 1529 """Check whether a config has a 'select' or 'imply' keyword 1530 1531 Args: 1532 kconf: Kconfiglib.Kconfig object 1533 config: Name of config to check (without CONFIG_ prefix) 1534 imply_config: Implying config (without CONFIG_ prefix) which may or 1535 may not have an 'imply' for 'config') 1536 1537 Returns: 1538 Symbol object for 'config' if found, else None 1539 """ 1540 sym = kconf.syms.get(imply_config) 1541 if sym: 1542 for sel in sym.get_selected_symbols() | sym.get_implied_symbols(): 1543 if sel.get_name() == config: 1544 return sym 1545 return None 1546 1547def check_imply_rule(kconf, config, imply_config): 1548 """Check if we can add an 'imply' option 1549 1550 This finds imply_config in the Kconfig and looks to see if it is possible 1551 to add an 'imply' for 'config' to that part of the Kconfig. 1552 1553 Args: 1554 kconf: Kconfiglib.Kconfig object 1555 config: Name of config to check (without CONFIG_ prefix) 1556 imply_config: Implying config (without CONFIG_ prefix) which may or 1557 may not have an 'imply' for 'config') 1558 1559 Returns: 1560 tuple: 1561 filename of Kconfig file containing imply_config, or None if none 1562 line number within the Kconfig file, or 0 if none 1563 message indicating the result 1564 """ 1565 sym = kconf.syms.get(imply_config) 1566 if not sym: 1567 return 'cannot find sym' 1568 locs = sym.get_def_locations() 1569 if len(locs) != 1: 1570 return '%d locations' % len(locs) 1571 fname, linenum = locs[0] 1572 cwd = os.getcwd() 1573 if cwd and fname.startswith(cwd): 1574 fname = fname[len(cwd) + 1:] 1575 file_line = ' at %s:%d' % (fname, linenum) 1576 with open(fname) as fd: 1577 data = fd.read().splitlines() 1578 if data[linenum - 1] != 'config %s' % imply_config: 1579 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line) 1580 return fname, linenum, 'adding%s' % file_line 1581 1582def add_imply_rule(config, fname, linenum): 1583 """Add a new 'imply' option to a Kconfig 1584 1585 Args: 1586 config: config option to add an imply for (without CONFIG_ prefix) 1587 fname: Kconfig filename to update 1588 linenum: Line number to place the 'imply' before 1589 1590 Returns: 1591 Message indicating the result 1592 """ 1593 file_line = ' at %s:%d' % (fname, linenum) 1594 data = open(fname).read().splitlines() 1595 linenum -= 1 1596 1597 for offset, line in enumerate(data[linenum:]): 1598 if line.strip().startswith('help') or not line: 1599 data.insert(linenum + offset, '\timply %s' % config) 1600 with open(fname, 'w') as fd: 1601 fd.write('\n'.join(data) + '\n') 1602 return 'added%s' % file_line 1603 1604 return 'could not insert%s' 1605 1606(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 1607 1, 2, 4, 8) 1608 1609IMPLY_FLAGS = { 1610 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 1611 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 1612 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 1613 'non-arch-board': [ 1614 IMPLY_NON_ARCH_BOARD, 1615 'Allow Kconfig options outside arch/ and /board/ to imply'], 1616}; 1617 1618def do_imply_config(config_list, add_imply, imply_flags, skip_added, 1619 check_kconfig=True, find_superset=False): 1620 """Find CONFIG options which imply those in the list 1621 1622 Some CONFIG options can be implied by others and this can help to reduce 1623 the size of the defconfig files. For example, CONFIG_X86 implies 1624 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1625 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1626 each of the x86 defconfig files. 1627 1628 This function uses the moveconfig database to find such options. It 1629 displays a list of things that could possibly imply those in the list. 1630 The algorithm ignores any that start with CONFIG_TARGET since these 1631 typically refer to only a few defconfigs (often one). It also does not 1632 display a config with less than 5 defconfigs. 1633 1634 The algorithm works using sets. For each target config in config_list: 1635 - Get the set 'defconfigs' which use that target config 1636 - For each config (from a list of all configs): 1637 - Get the set 'imply_defconfig' of defconfigs which use that config 1638 - 1639 - If imply_defconfigs contains anything not in defconfigs then 1640 this config does not imply the target config 1641 1642 Params: 1643 config_list: List of CONFIG options to check (each a string) 1644 add_imply: Automatically add an 'imply' for each config. 1645 imply_flags: Flags which control which implying configs are allowed 1646 (IMPLY_...) 1647 skip_added: Don't show options which already have an imply added. 1648 check_kconfig: Check if implied symbols already have an 'imply' or 1649 'select' for the target config, and show this information if so. 1650 find_superset: True to look for configs which are a superset of those 1651 already found. So for example if CONFIG_EXYNOS5 implies an option, 1652 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1653 implies that option, this will drop the former in favour of the 1654 latter. In practice this option has not proved very used. 1655 1656 Note the terminoloy: 1657 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1658 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1659 """ 1660 kconf = KconfigScanner().conf if check_kconfig else None 1661 if add_imply and add_imply != 'all': 1662 add_imply = add_imply.split() 1663 1664 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1665 config_db = {} 1666 1667 # Holds a dict containing the set of defconfigs that contain each config 1668 # key is config, value is set of defconfigs using that config 1669 defconfig_db = collections.defaultdict(set) 1670 1671 # Set of all config options we have seen 1672 all_configs = set() 1673 1674 # Set of all defconfigs we have seen 1675 all_defconfigs = set() 1676 1677 # Read in the database 1678 configs = {} 1679 with open(CONFIG_DATABASE) as fd: 1680 for line in fd.readlines(): 1681 line = line.rstrip() 1682 if not line: # Separator between defconfigs 1683 config_db[defconfig] = configs 1684 all_defconfigs.add(defconfig) 1685 configs = {} 1686 elif line[0] == ' ': # CONFIG line 1687 config, value = line.strip().split('=', 1) 1688 configs[config] = value 1689 defconfig_db[config].add(defconfig) 1690 all_configs.add(config) 1691 else: # New defconfig 1692 defconfig = line 1693 1694 # Work through each target config option in tern, independently 1695 for config in config_list: 1696 defconfigs = defconfig_db.get(config) 1697 if not defconfigs: 1698 print('%s not found in any defconfig' % config) 1699 continue 1700 1701 # Get the set of defconfigs without this one (since a config cannot 1702 # imply itself) 1703 non_defconfigs = all_defconfigs - defconfigs 1704 num_defconfigs = len(defconfigs) 1705 print('%s found in %d/%d defconfigs' % (config, num_defconfigs, 1706 len(all_configs))) 1707 1708 # This will hold the results: key=config, value=defconfigs containing it 1709 imply_configs = {} 1710 rest_configs = all_configs - set([config]) 1711 1712 # Look at every possible config, except the target one 1713 for imply_config in rest_configs: 1714 if 'ERRATUM' in imply_config: 1715 continue 1716 if not (imply_flags & IMPLY_CMD): 1717 if 'CONFIG_CMD' in imply_config: 1718 continue 1719 if not (imply_flags & IMPLY_TARGET): 1720 if 'CONFIG_TARGET' in imply_config: 1721 continue 1722 1723 # Find set of defconfigs that have this config 1724 imply_defconfig = defconfig_db[imply_config] 1725 1726 # Get the intersection of this with defconfigs containing the 1727 # target config 1728 common_defconfigs = imply_defconfig & defconfigs 1729 1730 # Get the set of defconfigs containing this config which DO NOT 1731 # also contain the taret config. If this set is non-empty it means 1732 # that this config affects other defconfigs as well as (possibly) 1733 # the ones affected by the target config. This means it implies 1734 # things we don't want to imply. 1735 not_common_defconfigs = imply_defconfig & non_defconfigs 1736 if not_common_defconfigs: 1737 continue 1738 1739 # If there are common defconfigs, imply_config may be useful 1740 if common_defconfigs: 1741 skip = False 1742 if find_superset: 1743 for prev in list(imply_configs.keys()): 1744 prev_count = len(imply_configs[prev]) 1745 count = len(common_defconfigs) 1746 if (prev_count > count and 1747 (imply_configs[prev] & common_defconfigs == 1748 common_defconfigs)): 1749 # skip imply_config because prev is a superset 1750 skip = True 1751 break 1752 elif count > prev_count: 1753 # delete prev because imply_config is a superset 1754 del imply_configs[prev] 1755 if not skip: 1756 imply_configs[imply_config] = common_defconfigs 1757 1758 # Now we have a dict imply_configs of configs which imply each config 1759 # The value of each dict item is the set of defconfigs containing that 1760 # config. Rank them so that we print the configs that imply the largest 1761 # number of defconfigs first. 1762 ranked_iconfigs = sorted(imply_configs, 1763 key=lambda k: len(imply_configs[k]), reverse=True) 1764 kconfig_info = '' 1765 cwd = os.getcwd() 1766 add_list = collections.defaultdict(list) 1767 for iconfig in ranked_iconfigs: 1768 num_common = len(imply_configs[iconfig]) 1769 1770 # Don't bother if there are less than 5 defconfigs affected. 1771 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1772 continue 1773 missing = defconfigs - imply_configs[iconfig] 1774 missing_str = ', '.join(missing) if missing else 'all' 1775 missing_str = '' 1776 show = True 1777 if kconf: 1778 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1779 iconfig[CONFIG_LEN:]) 1780 kconfig_info = '' 1781 if sym: 1782 locs = sym.get_def_locations() 1783 if len(locs) == 1: 1784 fname, linenum = locs[0] 1785 if cwd and fname.startswith(cwd): 1786 fname = fname[len(cwd) + 1:] 1787 kconfig_info = '%s:%d' % (fname, linenum) 1788 if skip_added: 1789 show = False 1790 else: 1791 sym = kconf.syms.get(iconfig[CONFIG_LEN:]) 1792 fname = '' 1793 if sym: 1794 locs = sym.get_def_locations() 1795 if len(locs) == 1: 1796 fname, linenum = locs[0] 1797 if cwd and fname.startswith(cwd): 1798 fname = fname[len(cwd) + 1:] 1799 in_arch_board = not sym or (fname.startswith('arch') or 1800 fname.startswith('board')) 1801 if (not in_arch_board and 1802 not (imply_flags & IMPLY_NON_ARCH_BOARD)): 1803 continue 1804 1805 if add_imply and (add_imply == 'all' or 1806 iconfig in add_imply): 1807 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1808 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1809 if fname: 1810 add_list[fname].append(linenum) 1811 1812 if show and kconfig_info != 'skip': 1813 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30), 1814 kconfig_info, missing_str)) 1815 1816 # Having collected a list of things to add, now we add them. We process 1817 # each file from the largest line number to the smallest so that 1818 # earlier additions do not affect our line numbers. E.g. if we added an 1819 # imply at line 20 it would change the position of each line after 1820 # that. 1821 for fname, linenums in add_list.items(): 1822 for linenum in sorted(linenums, reverse=True): 1823 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1824 1825 1826def main(): 1827 try: 1828 cpu_count = multiprocessing.cpu_count() 1829 except NotImplementedError: 1830 cpu_count = 1 1831 1832 parser = optparse.OptionParser() 1833 # Add options here 1834 parser.add_option('-a', '--add-imply', type='string', default='', 1835 help='comma-separated list of CONFIG options to add ' 1836 "an 'imply' statement to for the CONFIG in -i") 1837 parser.add_option('-A', '--skip-added', action='store_true', default=False, 1838 help="don't show options which are already marked as " 1839 'implying others') 1840 parser.add_option('-b', '--build-db', action='store_true', default=False, 1841 help='build a CONFIG database') 1842 parser.add_option('-c', '--color', action='store_true', default=False, 1843 help='display the log in color') 1844 parser.add_option('-C', '--commit', action='store_true', default=False, 1845 help='Create a git commit for the operation') 1846 parser.add_option('-d', '--defconfigs', type='string', 1847 help='a file containing a list of defconfigs to move, ' 1848 "one per line (for example 'snow_defconfig') " 1849 "or '-' to read from stdin") 1850 parser.add_option('-i', '--imply', action='store_true', default=False, 1851 help='find options which imply others') 1852 parser.add_option('-I', '--imply-flags', type='string', default='', 1853 help="control the -i option ('help' for help") 1854 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1855 help='perform a trial run (show log with no changes)') 1856 parser.add_option('-e', '--exit-on-error', action='store_true', 1857 default=False, 1858 help='exit immediately on any error') 1859 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1860 help='force sync by savedefconfig') 1861 parser.add_option('-S', '--spl', action='store_true', default=False, 1862 help='parse config options defined for SPL build') 1863 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1864 action='store_true', default=False, 1865 help='only cleanup the headers') 1866 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1867 help='the number of jobs to run simultaneously') 1868 parser.add_option('-r', '--git-ref', type='string', 1869 help='the git ref to clone for building the autoconf.mk') 1870 parser.add_option('-y', '--yes', action='store_true', default=False, 1871 help="respond 'yes' to any prompts") 1872 parser.add_option('-v', '--verbose', action='store_true', default=False, 1873 help='show any build errors as boards are built') 1874 parser.usage += ' CONFIG ...' 1875 1876 (options, configs) = parser.parse_args() 1877 1878 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1879 options.imply)): 1880 parser.print_usage() 1881 sys.exit(1) 1882 1883 # prefix the option name with CONFIG_ if missing 1884 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1885 for config in configs ] 1886 1887 check_top_directory() 1888 1889 if options.imply: 1890 imply_flags = 0 1891 if options.imply_flags == 'all': 1892 imply_flags = -1 1893 1894 elif options.imply_flags: 1895 for flag in options.imply_flags.split(','): 1896 bad = flag not in IMPLY_FLAGS 1897 if bad: 1898 print("Invalid flag '%s'" % flag) 1899 if flag == 'help' or bad: 1900 print("Imply flags: (separate with ',')") 1901 for name, info in IMPLY_FLAGS.items(): 1902 print(' %-15s: %s' % (name, info[1])) 1903 parser.print_usage() 1904 sys.exit(1) 1905 imply_flags |= IMPLY_FLAGS[flag][0] 1906 1907 do_imply_config(configs, options.add_imply, imply_flags, 1908 options.skip_added) 1909 return 1910 1911 config_db = {} 1912 db_queue = queue.Queue() 1913 t = DatabaseThread(config_db, db_queue) 1914 t.setDaemon(True) 1915 t.start() 1916 1917 if not options.cleanup_headers_only: 1918 check_clean_directory() 1919 bsettings.Setup('') 1920 toolchains = toolchain.Toolchains() 1921 toolchains.GetSettings() 1922 toolchains.Scan(verbose=False) 1923 move_config(toolchains, configs, options, db_queue) 1924 db_queue.join() 1925 1926 if configs: 1927 cleanup_headers(configs, options) 1928 cleanup_extra_options(configs, options) 1929 cleanup_whitelist(configs, options) 1930 cleanup_readme(configs, options) 1931 1932 if options.commit: 1933 subprocess.call(['git', 'add', '-u']) 1934 if configs: 1935 msg = 'Convert %s %sto Kconfig' % (configs[0], 1936 'et al ' if len(configs) > 1 else '') 1937 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1938 '\n '.join(configs)) 1939 else: 1940 msg = 'configs: Resync with savedefconfig' 1941 msg += '\n\nRsync all defconfig files using moveconfig.py' 1942 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1943 1944 if options.build_db: 1945 with open(CONFIG_DATABASE, 'w') as fd: 1946 for defconfig, configs in config_db.items(): 1947 fd.write('%s\n' % defconfig) 1948 for config in sorted(configs.keys()): 1949 fd.write(' %s=%s\n' % (config, configs[config])) 1950 fd.write('\n') 1951 1952if __name__ == '__main__': 1953 main() 1954