#!/usr/bin/python3 # Build many configurations of glibc. # Copyright (C) 2016 Free Software Foundation, Inc. # This file is part of the GNU C Library. # # The GNU C Library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # The GNU C Library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with the GNU C Library; if not, see # . """Build many configurations of glibc. This script takes as arguments a directory name (containing a src subdirectory with sources of the relevant toolchain components) and a description of what to do: 'checkout', to check out sources into that directory, 'host-libraries', to build libraries required by the toolchain, 'compilers', to build cross-compilers for various configurations, or 'glibcs', to build glibc for various configurations and run the compilation parts of the testsuite. Subsequent arguments name the versions of components to check out (-. no_isolate = ('-fno-isolate-erroneous-paths-dereference' ' -fno-isolate-erroneous-paths-attribute') self.add_config(arch='sh3', os_name='linux-gnu', glibcs=[{'ccopts': no_isolate}]) self.add_config(arch='sh3eb', os_name='linux-gnu', glibcs=[{'ccopts': no_isolate}]) self.add_config(arch='sh4', os_name='linux-gnu', glibcs=[{'ccopts': no_isolate}]) self.add_config(arch='sh4eb', os_name='linux-gnu', glibcs=[{'ccopts': no_isolate}]) self.add_config(arch='sh4', os_name='linux-gnu', variant='soft', gcc_cfg=['--without-fp'], glibcs=[{'variant': 'soft', 'cfg': ['--without-fp'], 'ccopts': no_isolate}]) self.add_config(arch='sh4eb', os_name='linux-gnu', variant='soft', gcc_cfg=['--without-fp'], glibcs=[{'variant': 'soft', 'cfg': ['--without-fp'], 'ccopts': no_isolate}]) self.add_config(arch='sparc64', os_name='linux-gnu', glibcs=[{}, {'arch': 'sparcv9', 'ccopts': '-m32 -mlong-double-128'}]) self.add_config(arch='tilegx', os_name='linux-gnu', glibcs=[{}, {'variant': '32', 'ccopts': '-m32'}]) self.add_config(arch='tilegxbe', os_name='linux-gnu', glibcs=[{}, {'variant': '32', 'ccopts': '-m32'}]) self.add_config(arch='tilepro', os_name='linux-gnu') self.add_config(arch='x86_64', os_name='linux-gnu', gcc_cfg=['--with-multilib-list=m64,m32,mx32'], glibcs=[{}, {'variant': 'x32', 'ccopts': '-mx32'}, {'arch': 'i686', 'ccopts': '-m32 -march=i686'}], extra_glibcs=[{'variant': 'disable-multi-arch', 'cfg': ['--disable-multi-arch']}, {'variant': 'disable-multi-arch', 'arch': 'i686', 'ccopts': '-m32 -march=i686', 'cfg': ['--disable-multi-arch']}, {'arch': 'i486', 'ccopts': '-m32 -march=i486'}, {'arch': 'i586', 'ccopts': '-m32 -march=i586'}]) def add_config(self, **args): """Add an individual build configuration.""" cfg = Config(self, **args) if cfg.name in self.configs: print('error: duplicate config %s' % cfg.name) exit(1) self.configs[cfg.name] = cfg for c in cfg.all_glibcs: if c.name in self.glibc_configs: print('error: duplicate glibc config %s' % c.name) exit(1) self.glibc_configs[c.name] = c def component_srcdir(self, component): """Return the source directory for a given component, e.g. gcc.""" return os.path.join(self.srcdir, component) def component_builddir(self, action, config, component, subconfig=None): """Return the directory to use for a build.""" if config is None: # Host libraries. assert subconfig is None return os.path.join(self.builddir, action, component) if subconfig is None: return os.path.join(self.builddir, action, config, component) else: # glibc build as part of compiler build. return os.path.join(self.builddir, action, config, component, subconfig) def compiler_installdir(self, config): """Return the directory in which to install a compiler.""" return os.path.join(self.installdir, 'compilers', config) def compiler_bindir(self, config): """Return the directory in which to find compiler binaries.""" return os.path.join(self.compiler_installdir(config), 'bin') def compiler_sysroot(self, config): """Return the sysroot directory for a compiler.""" return os.path.join(self.compiler_installdir(config), 'sysroot') def glibc_installdir(self, config): """Return the directory in which to install glibc.""" return os.path.join(self.installdir, 'glibcs', config) def run_builds(self, action, configs): """Run the requested builds.""" if action == 'checkout': self.checkout(configs) return elif action == 'host-libraries': if configs: print('error: configurations specified for host-libraries') exit(1) self.build_host_libraries() elif action == 'compilers': self.build_compilers(configs) else: self.build_glibcs(configs) self.write_files() self.do_build() @staticmethod def remove_dirs(*args): """Remove directories and their contents if they exist.""" for dir in args: shutil.rmtree(dir, ignore_errors=True) @staticmethod def remove_recreate_dirs(*args): """Remove directories if they exist, and create them as empty.""" Context.remove_dirs(*args) for dir in args: os.makedirs(dir, exist_ok=True) def add_makefile_cmdlist(self, target, cmdlist, logsdir): """Add makefile text for a list of commands.""" commands = cmdlist.makefile_commands(self.wrapper, logsdir) self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' % (target, target, target, commands)) def write_files(self): """Write out the Makefile and wrapper script.""" mftext = ''.join(self.makefile_pieces) with open(self.makefile, 'w') as f: f.write(mftext) wrapper_text = ( '#!/bin/sh\n' 'prev_base=$1\n' 'this_base=$2\n' 'desc=$3\n' 'dir=$4\n' 'path=$5\n' 'shift 5\n' 'prev_status=$prev_base-status.txt\n' 'this_status=$this_base-status.txt\n' 'this_log=$this_base-log.txt\n' 'date > "$this_log"\n' 'echo >> "$this_log"\n' 'echo "Description: $desc" >> "$this_log"\n' 'echo "Command: $*" >> "$this_log"\n' 'echo "Directory: $dir" >> "$this_log"\n' 'echo "Path addition: $path" >> "$this_log"\n' 'echo >> "$this_log"\n' 'record_status ()\n' '{\n' ' echo >> "$this_log"\n' ' echo "$1: $desc" > "$this_status"\n' ' echo "$1: $desc" >> "$this_log"\n' ' echo >> "$this_log"\n' ' date >> "$this_log"\n' ' echo "$1: $desc"\n' ' exit 0\n' '}\n' 'check_error ()\n' '{\n' ' if [ "$1" != "0" ]; then\n' ' record_status FAIL\n' ' fi\n' '}\n' 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n' ' record_status UNRESOLVED\n' 'fi\n' 'if [ "$dir" ]; then\n' ' cd "$dir"\n' ' check_error "$?"\n' 'fi\n' 'if [ "$path" ]; then\n' ' PATH=$path:$PATH\n' 'fi\n' '"$@" < /dev/null >> "$this_log" 2>&1\n' 'check_error "$?"\n' 'record_status PASS\n') with open(self.wrapper, 'w') as f: f.write(wrapper_text) os.chmod(self.wrapper, (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP| stat.S_IROTH|stat.S_IXOTH)) save_logs_text = ( '#!/bin/sh\n' 'if ! [ -f tests.sum ]; then\n' ' echo "No test summary available."\n' ' exit 0\n' 'fi\n' 'save_file ()\n' '{\n' ' echo "Contents of $1:"\n' ' echo\n' ' cat "$1"\n' ' echo\n' ' echo "End of contents of $1."\n' ' echo\n' '}\n' 'save_file tests.sum\n' 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n' 'for t in $non_pass_tests; do\n' ' if [ -f "$t.out" ]; then\n' ' save_file "$t.out"\n' ' fi\n' 'done\n') with open(self.save_logs, 'w') as f: f.write(save_logs_text) os.chmod(self.save_logs, (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP| stat.S_IROTH|stat.S_IXOTH)) def do_build(self): """Do the actual build.""" cmd = ['make', '-j%d' % self.parallelism] subprocess.run(cmd, cwd=self.builddir, check=True) def build_host_libraries(self): """Build the host libraries.""" installdir = self.host_libraries_installdir builddir = os.path.join(self.builddir, 'host-libraries') logsdir = os.path.join(self.logsdir, 'host-libraries') self.remove_recreate_dirs(installdir, builddir, logsdir) cmdlist = CommandList('host-libraries', self.keep) self.build_host_library(cmdlist, 'gmp') self.build_host_library(cmdlist, 'mpfr', ['--with-gmp=%s' % installdir]) self.build_host_library(cmdlist, 'mpc', ['--with-gmp=%s' % installdir, '--with-mpfr=%s' % installdir]) cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')]) self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir) def build_host_library(self, cmdlist, lib, extra_opts=None): """Build one host library.""" srcdir = self.component_srcdir(lib) builddir = self.component_builddir('host-libraries', None, lib) installdir = self.host_libraries_installdir cmdlist.push_subdesc(lib) cmdlist.create_use_dir(builddir) cfg_cmd = [os.path.join(srcdir, 'configure'), '--prefix=%s' % installdir, '--disable-shared'] if extra_opts: cfg_cmd.extend (extra_opts) cmdlist.add_command('configure', cfg_cmd) cmdlist.add_command('build', ['make']) cmdlist.add_command('check', ['make', 'check']) cmdlist.add_command('install', ['make', 'install']) cmdlist.cleanup_dir() cmdlist.pop_subdesc() def build_compilers(self, configs): """Build the compilers.""" if not configs: self.remove_dirs(os.path.join(self.builddir, 'compilers')) self.remove_dirs(os.path.join(self.installdir, 'compilers')) self.remove_dirs(os.path.join(self.logsdir, 'compilers')) configs = sorted(self.configs.keys()) for c in configs: self.configs[c].build() def build_glibcs(self, configs): """Build the glibcs.""" if not configs: self.remove_dirs(os.path.join(self.builddir, 'glibcs')) self.remove_dirs(os.path.join(self.installdir, 'glibcs')) self.remove_dirs(os.path.join(self.logsdir, 'glibcs')) configs = sorted(self.glibc_configs.keys()) for c in configs: self.glibc_configs[c].build() def checkout(self, versions): """Check out the desired component versions.""" default_versions = {'binutils': 'vcs-2.27', 'gcc': 'vcs-6', 'glibc': 'vcs-mainline', 'gmp': '6.1.1', 'linux': '4.8.6', 'mpc': '1.0.3', 'mpfr': '3.1.5'} use_versions = {} for v in versions: found_v = False for k in default_versions.keys(): kx = k + '-' if v.startswith(kx): vx = v[len(kx):] if k in use_versions: print('error: multiple versions for %s' % k) exit(1) use_versions[k] = vx found_v = True break if not found_v: print('error: unknown component in %s' % v) exit(1) for k in default_versions.keys(): if k not in use_versions: use_versions[k] = default_versions[k] os.makedirs(self.srcdir, exist_ok=True) for k in sorted(default_versions.keys()): update = os.access(self.component_srcdir(k), os.F_OK) v = use_versions[k] if v.startswith('vcs-'): self.checkout_vcs(k, v[4:], update) else: self.checkout_tar(k, v, update) def checkout_vcs(self, component, version, update): """Check out the given version of the given component from version control.""" if component == 'binutils': git_url = 'git://sourceware.org/git/binutils-gdb.git' if version == 'mainline': git_branch = 'master' else: trans = str.maketrans({'.': '_'}) git_branch = 'binutils-%s-branch' % version.translate(trans) self.git_checkout(component, git_url, git_branch, update) elif component == 'gcc': if version == 'mainline': branch = 'trunk' else: trans = str.maketrans({'.': '_'}) branch = 'branches/gcc-%s-branch' % version.translate(trans) svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch self.gcc_checkout(svn_url, update) elif component == 'glibc': git_url = 'git://sourceware.org/git/glibc.git' if version == 'mainline': git_branch = 'master' else: git_branch = 'release/%s/master' % version self.git_checkout(component, git_url, git_branch, update) self.fix_glibc_timestamps() else: print('error: component %s coming from VCS' % component) exit(1) def git_checkout(self, component, git_url, git_branch, update): """Check out a component from git.""" if update: subprocess.run(['git', 'remote', 'prune', 'origin'], cwd=self.component_srcdir(component), check=True) subprocess.run(['git', 'pull', '-q'], cwd=self.component_srcdir(component), check=True) else: subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url, self.component_srcdir(component)], check=True) def fix_glibc_timestamps(self): """Fix timestamps in a glibc checkout.""" # Ensure that builds do not try to regenerate generated files # in the source tree. srcdir = self.component_srcdir('glibc') for dirpath, dirnames, filenames in os.walk(srcdir): for f in filenames: if (f == 'configure' or f == 'preconfigure' or f.endswith('-kw.h')): to_touch = os.path.join(dirpath, f) subprocess.run(['touch', to_touch], check=True) def gcc_checkout(self, svn_url, update): """Check out GCC from SVN.""" if not update: subprocess.run(['svn', 'co', '-q', svn_url, self.component_srcdir('gcc')], check=True) subprocess.run(['contrib/gcc_update', '--silent'], cwd=self.component_srcdir('gcc'), check=True) def checkout_tar(self, component, version, update): """Check out the given version of the given component from a tarball.""" if update: return url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2', 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2', 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz', 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz', 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz', 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'} if component not in url_map: print('error: component %s coming from tarball' % component) exit(1) url = url_map[component] % {'version': version} filename = os.path.join(self.srcdir, url.split('/')[-1]) response = urllib.request.urlopen(url) data = response.read() with open(filename, 'wb') as f: f.write(data) subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename], check=True) os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)), self.component_srcdir(component)) os.remove(filename) class Config: """A configuration for building a compiler and associated libraries.""" def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None, first_gcc_cfg=None, glibcs=None, extra_glibcs=None): """Initialize a Config object.""" self.ctx = ctx self.arch = arch self.os = os_name self.variant = variant if variant is None: self.name = '%s-%s' % (arch, os_name) else: self.name = '%s-%s-%s' % (arch, os_name, variant) self.triplet = '%s-glibc-%s' % (arch, os_name) if gcc_cfg is None: self.gcc_cfg = [] else: self.gcc_cfg = gcc_cfg if first_gcc_cfg is None: self.first_gcc_cfg = [] else: self.first_gcc_cfg = first_gcc_cfg if glibcs is None: glibcs = [{'variant': variant}] if extra_glibcs is None: extra_glibcs = [] glibcs = [Glibc(self, **g) for g in glibcs] extra_glibcs = [Glibc(self, **g) for g in extra_glibcs] self.all_glibcs = glibcs + extra_glibcs self.compiler_glibcs = glibcs self.installdir = ctx.compiler_installdir(self.name) self.bindir = ctx.compiler_bindir(self.name) self.sysroot = ctx.compiler_sysroot(self.name) self.builddir = os.path.join(ctx.builddir, 'compilers', self.name) self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name) def component_builddir(self, component): """Return the directory to use for a (non-glibc) build.""" return self.ctx.component_builddir('compilers', self.name, component) def build(self): """Generate commands to build this compiler.""" self.ctx.remove_recreate_dirs(self.installdir, self.builddir, self.logsdir) cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep) cmdlist.add_command('check-host-libraries', ['test', '-f', os.path.join(self.ctx.host_libraries_installdir, 'ok')]) cmdlist.use_path(self.bindir) self.build_cross_tool(cmdlist, 'binutils', 'binutils', ['--disable-gdb', '--disable-libdecnumber', '--disable-readline', '--disable-sim']) if self.os.startswith('linux'): self.install_linux_headers(cmdlist) self.build_gcc(cmdlist, True) for g in self.compiler_glibcs: cmdlist.push_subdesc('glibc') cmdlist.push_subdesc(g.name) g.build_glibc(cmdlist, True) cmdlist.pop_subdesc() cmdlist.pop_subdesc() self.build_gcc(cmdlist, False) cmdlist.add_command('done', ['touch', os.path.join(self.installdir, 'ok')]) self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist, self.logsdir) def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None): """Build one cross tool.""" srcdir = self.ctx.component_srcdir(tool_src) builddir = self.component_builddir(tool_build) cmdlist.push_subdesc(tool_build) cmdlist.create_use_dir(builddir) cfg_cmd = [os.path.join(srcdir, 'configure'), '--prefix=%s' % self.installdir, '--build=%s' % self.ctx.build_triplet, '--host=%s' % self.ctx.build_triplet, '--target=%s' % self.triplet, '--with-sysroot=%s' % self.sysroot] if extra_opts: cfg_cmd.extend(extra_opts) cmdlist.add_command('configure', cfg_cmd) cmdlist.add_command('build', ['make']) cmdlist.add_command('install', ['make', 'install']) cmdlist.cleanup_dir() cmdlist.pop_subdesc() def install_linux_headers(self, cmdlist): """Install Linux kernel headers.""" arch_map = {'aarch64': 'arm64', 'alpha': 'alpha', 'arm': 'arm', 'hppa': 'parisc', 'i486': 'x86', 'i586': 'x86', 'i686': 'x86', 'i786': 'x86', 'ia64': 'ia64', 'm68k': 'm68k', 'microblaze': 'microblaze', 'mips': 'mips', 'nios2': 'nios2', 'powerpc': 'powerpc', 's390': 's390', 'sh': 'sh', 'sparc': 'sparc', 'tile': 'tile', 'x86_64': 'x86'} linux_arch = None for k in arch_map: if self.arch.startswith(k): linux_arch = arch_map[k] break assert linux_arch is not None srcdir = self.ctx.component_srcdir('linux') builddir = self.component_builddir('linux') headers_dir = os.path.join(self.sysroot, 'usr') cmdlist.push_subdesc('linux') cmdlist.create_use_dir(builddir) cmdlist.add_command('install-headers', ['make', '-C', srcdir, 'O=%s' % builddir, 'ARCH=%s' % linux_arch, 'INSTALL_HDR_PATH=%s' % headers_dir, 'headers_install']) cmdlist.cleanup_dir() cmdlist.pop_subdesc() def build_gcc(self, cmdlist, bootstrap): """Build GCC.""" # libsanitizer commonly breaks because of glibc header # changes, or on unusual targets. libssp is of little # relevance with glibc's own stack checking support. cfg_opts = list(self.gcc_cfg) cfg_opts += ['--disable-libsanitizer', '--disable-libssp'] if bootstrap: tool_build = 'gcc-first' # Building a static-only, C-only compiler that is # sufficient to build glibc. Various libraries and # features that may require libc headers must be disabled. # When configuring with a sysroot, --with-newlib is # required to define inhibit_libc (to stop some parts of # libgcc including libc headers); --without-headers is not # sufficient. cfg_opts += ['--enable-languages=c', '--disable-shared', '--disable-threads', '--disable-libatomic', '--disable-decimal-float', '--disable-libffi', '--disable-libgomp', '--disable-libitm', '--disable-libmpx', '--disable-libquadmath', '--without-headers', '--with-newlib', '--with-glibc-version=%s' % self.ctx.glibc_version ] cfg_opts += self.first_gcc_cfg else: tool_build = 'gcc' cfg_opts += ['--enable-languages=c,c++', '--enable-shared', '--enable-threads'] self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts) class Glibc: """A configuration for building glibc.""" def __init__(self, compiler, arch=None, os_name=None, variant=None, cfg=None, ccopts=None): """Initialize a Glibc object.""" self.ctx = compiler.ctx self.compiler = compiler if arch is None: self.arch = compiler.arch else: self.arch = arch if os_name is None: self.os = compiler.os else: self.os = os_name self.variant = variant if variant is None: self.name = '%s-%s' % (self.arch, self.os) else: self.name = '%s-%s-%s' % (self.arch, self.os, variant) self.triplet = '%s-glibc-%s' % (self.arch, self.os) if cfg is None: self.cfg = [] else: self.cfg = cfg self.ccopts = ccopts def tool_name(self, tool): """Return the name of a cross-compilation tool.""" ctool = '%s-%s' % (self.compiler.triplet, tool) if self.ccopts and (tool == 'gcc' or tool == 'g++'): ctool = '%s %s' % (ctool, self.ccopts) return ctool def build(self): """Generate commands to build this glibc.""" builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc') installdir = self.ctx.glibc_installdir(self.name) logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name) self.ctx.remove_recreate_dirs(installdir, builddir, logsdir) cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep) cmdlist.add_command('check-compilers', ['test', '-f', os.path.join(self.compiler.installdir, 'ok')]) cmdlist.use_path(self.compiler.bindir) self.build_glibc(cmdlist, False) self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist, logsdir) def build_glibc(self, cmdlist, for_compiler): """Generate commands to build this glibc, either as part of a compiler build or with the bootstrapped compiler (and in the latter case, run tests as well).""" srcdir = self.ctx.component_srcdir('glibc') if for_compiler: builddir = self.ctx.component_builddir('compilers', self.compiler.name, 'glibc', self.name) installdir = self.compiler.sysroot srcdir_copy = self.ctx.component_builddir('compilers', self.compiler.name, 'glibc-src', self.name) else: builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc') installdir = self.ctx.glibc_installdir(self.name) srcdir_copy = self.ctx.component_builddir('glibcs', self.name, 'glibc-src') cmdlist.create_use_dir(builddir) # glibc builds write into the source directory, and even if # not intentionally there is a risk of bugs that involve # writing into the working directory. To avoid possible # concurrency issues, copy the source directory. cmdlist.create_copy_dir(srcdir, srcdir_copy) cfg_cmd = [os.path.join(srcdir_copy, 'configure'), '--prefix=/usr', '--enable-add-ons', '--build=%s' % self.ctx.build_triplet, '--host=%s' % self.triplet, 'CC=%s' % self.tool_name('gcc'), 'CXX=%s' % self.tool_name('g++'), 'AR=%s' % self.tool_name('ar'), 'AS=%s' % self.tool_name('as'), 'LD=%s' % self.tool_name('ld'), 'NM=%s' % self.tool_name('nm'), 'OBJCOPY=%s' % self.tool_name('objcopy'), 'OBJDUMP=%s' % self.tool_name('objdump'), 'RANLIB=%s' % self.tool_name('ranlib'), 'READELF=%s' % self.tool_name('readelf'), 'STRIP=%s' % self.tool_name('strip')] cfg_cmd += self.cfg cmdlist.add_command('configure', cfg_cmd) cmdlist.add_command('build', ['make']) cmdlist.add_command('install', ['make', 'install', 'install_root=%s' % installdir]) # GCC uses paths such as lib/../lib64, so make sure lib # directories always exist. cmdlist.add_command('mkdir-lib', ['mkdir', '-p', os.path.join(installdir, 'lib'), os.path.join(installdir, 'usr', 'lib')]) if not for_compiler: cmdlist.add_command('check', ['make', 'check']) cmdlist.add_command('save-logs', [self.ctx.save_logs], always_run=True) cmdlist.cleanup_dir('cleanup-src', srcdir_copy) cmdlist.cleanup_dir() class Command: """A command run in the build process.""" def __init__(self, desc, num, dir, path, command, always_run=False): """Initialize a Command object.""" self.dir = dir self.path = path self.desc = desc trans = str.maketrans({' ': '-'}) self.logbase = '%03d-%s' % (num, desc.translate(trans)) self.command = command self.always_run = always_run @staticmethod def shell_make_quote_string(s): """Given a string not containing a newline, quote it for use by the shell and make.""" assert '\n' not in s if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s): return s strans = str.maketrans({"'": "'\\''"}) s = "'%s'" % s.translate(strans) mtrans = str.maketrans({'$': '$$'}) return s.translate(mtrans) @staticmethod def shell_make_quote_list(l, translate_make): """Given a list of strings not containing newlines, quote them for use by the shell and make, returning a single string. If translate_make is true and the first string is 'make', change it to $(MAKE).""" l = [Command.shell_make_quote_string(s) for s in l] if translate_make and l[0] == 'make': l[0] = '$(MAKE)' return ' '.join(l) def shell_make_quote(self): """Return this command quoted for the shell and make.""" return self.shell_make_quote_list(self.command, True) class CommandList: """A list of commands run in the build process.""" def __init__(self, desc, keep): """Initialize a CommandList object.""" self.cmdlist = [] self.dir = None self.path = None self.desc = [desc] self.keep = keep def desc_txt(self, desc): """Return the description to use for a command.""" return '%s %s' % (' '.join(self.desc), desc) def use_dir(self, dir): """Set the default directory for subsequent commands.""" self.dir = dir def use_path(self, path): """Set a directory to be prepended to the PATH for subsequent commands.""" self.path = path def push_subdesc(self, subdesc): """Set the default subdescription for subsequent commands (e.g., the name of a component being built, within the series of commands building it).""" self.desc.append(subdesc) def pop_subdesc(self): """Pop a subdescription from the list of descriptions.""" self.desc.pop() def create_use_dir(self, dir): """Remove and recreate a directory and use it for subsequent commands.""" self.add_command_dir('rm', None, ['rm', '-rf', dir]) self.add_command_dir('mkdir', None, ['mkdir', '-p', dir]) self.use_dir(dir) def create_copy_dir(self, src, dest): """Remove a directory and recreate it as a copy from the given source.""" self.add_command_dir('copy-rm', None, ['rm', '-rf', dest]) parent = os.path.dirname(dest) self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent]) self.add_command_dir('copy', None, ['cp', '-a', src, dest]) def add_command_dir(self, desc, dir, command, always_run=False): """Add a command to run in a given directory.""" cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path, command, always_run) self.cmdlist.append(cmd) def add_command(self, desc, command, always_run=False): """Add a command to run in the default directory.""" cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir, self.path, command, always_run) self.cmdlist.append(cmd) def cleanup_dir(self, desc='cleanup', dir=None): """Clean up a build directory. If no directory is specified, the default directory is cleaned up and ceases to be the default directory.""" if dir is None: dir = self.dir self.use_dir(None) if self.keep != 'all': self.add_command_dir(desc, None, ['rm', '-rf', dir], always_run=(self.keep == 'none')) def makefile_commands(self, wrapper, logsdir): """Return the sequence of commands in the form of text for a Makefile. The given wrapper script takes arguments: base of logs for previous command, or empty; base of logs for this command; description; directory; PATH addition; the command itself.""" # prev_base is the base of the name for logs of the previous # command that is not always-run (that is, a build command, # whose failure should stop subsequent build commands from # being run, as opposed to a cleanup command, which is run # even if previous commands failed). prev_base = '' cmds = [] for c in self.cmdlist: ctxt = c.shell_make_quote() if prev_base and not c.always_run: prev_log = os.path.join(logsdir, prev_base) else: prev_log = '' this_log = os.path.join(logsdir, c.logbase) if not c.always_run: prev_base = c.logbase if c.dir is None: dir = '' else: dir = c.dir if c.path is None: path = '' else: path = c.path prelims = [wrapper, prev_log, this_log, c.desc, dir, path] prelim_txt = Command.shell_make_quote_list(prelims, False) cmds.append('\t@%s %s' % (prelim_txt, ctxt)) return '\n'.join(cmds) def get_parser(): """Return an argument parser for this module.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-j', dest='parallelism', help='Run this number of jobs in parallel', type=int, default=os.cpu_count()) parser.add_argument('--keep', dest='keep', help='Whether to keep all build directories, ' 'none or only those from failed builds', default='none', choices=('none', 'all', 'failed')) parser.add_argument('topdir', help='Toplevel working directory') parser.add_argument('action', help='What to do', choices=('checkout', 'host-libraries', 'compilers', 'glibcs')) parser.add_argument('configs', help='Versions to check out or configurations to build', nargs='*') return parser def main(argv): """The main entry point.""" parser = get_parser() opts = parser.parse_args(argv) topdir = os.path.abspath(opts.topdir) ctx = Context(topdir, opts.parallelism, opts.keep, opts.action) ctx.run_builds(opts.action, opts.configs) if __name__ == '__main__': main(sys.argv[1:])