[PATCH 05/11] Move internal scripts to the "helper" subdirectory
Artem Bityutskiy
dedekind1 at gmail.com
Fri Feb 7 07:28:18 PST 2014
From: Artem Bityutskiy <artem.bityutskiy at linux.intel.com>
This patch implements the following TODO entry:
Re-arrange the source code structure so that the internal scripts would all sit
in a separate sub-directory. Internal scripts are those which users are not
supposed to run directly. For example, 'aiaiai-locker' is one of them.
Signed-off-by: Artem Bityutskiy <artem.bityutskiy at linux.intel.com>
---
.gitignore | 2 +-
Makefile | 4 -
aiaiai-checker | 220 ---------------------------
aiaiai-decode-rfc-2047 | 18 ---
aiaiai-diff-log | 193 ------------------------
aiaiai-diff-log-helper | 146 ------------------
aiaiai-locker.c | 229 ----------------------------
aiaiai-make-kernel | 285 -----------------------------------
aiaiai-match-keywords | 93 ------------
aiaiai-sh-functions | 229 ----------------------------
aiaiai-test-bisectability | 273 ---------------------------------
doc/TODO.txt | 4 -
email/aiaiai-email-dispatcher | 3 +-
email/aiaiai-email-dispatcher-helper | 2 +-
email/aiaiai-email-lda | 2 +-
email/aiaiai-email-test-patchset | 2 +-
gerrit/aiaiai-jenkins-test-patchset | 4 +-
helpers/Makefile | 5 +-
helpers/aiaiai-checker | 220 +++++++++++++++++++++++++++
helpers/aiaiai-decode-rfc-2047 | 18 +++
helpers/aiaiai-diff-log | 193 ++++++++++++++++++++++++
helpers/aiaiai-diff-log-helper | 146 ++++++++++++++++++
helpers/aiaiai-locker.c | 229 ++++++++++++++++++++++++++++
helpers/aiaiai-make-kernel | 285 +++++++++++++++++++++++++++++++++++
helpers/aiaiai-match-keywords | 93 ++++++++++++
helpers/aiaiai-sh-functions | 229 ++++++++++++++++++++++++++++
helpers/aiaiai-test-bisectability | 273 +++++++++++++++++++++++++++++++++
27 files changed, 1696 insertions(+), 1704 deletions(-)
delete mode 100755 aiaiai-checker
delete mode 100755 aiaiai-decode-rfc-2047
delete mode 100755 aiaiai-diff-log
delete mode 100755 aiaiai-diff-log-helper
delete mode 100644 aiaiai-locker.c
delete mode 100755 aiaiai-make-kernel
delete mode 100755 aiaiai-match-keywords
delete mode 100644 aiaiai-sh-functions
delete mode 100755 aiaiai-test-bisectability
create mode 100755 helpers/aiaiai-checker
create mode 100755 helpers/aiaiai-decode-rfc-2047
create mode 100755 helpers/aiaiai-diff-log
create mode 100755 helpers/aiaiai-diff-log-helper
create mode 100644 helpers/aiaiai-locker.c
create mode 100755 helpers/aiaiai-make-kernel
create mode 100755 helpers/aiaiai-match-keywords
create mode 100644 helpers/aiaiai-sh-functions
create mode 100755 helpers/aiaiai-test-bisectability
diff --git a/.gitignore b/.gitignore
index 4361bc4..d707290 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,2 @@
-aiaiai-locker
+helpers/aiaiai-locker
helpers/remap-log
diff --git a/Makefile b/Makefile
index a6c722c..00039af 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,5 @@
-CFLAGS = -O2 -g -Wextra
-
all:
- $(CC) $(CFLAGS) -o aiaiai-locker aiaiai-locker.c
make -C helpers
clean:
- $(RM) aiaiai-locker
make -C helpers clean
diff --git a/aiaiai-checker b/aiaiai-checker
deleted file mode 100755
index 1c68efc..0000000
--- a/aiaiai-checker
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/bin/sh -efu
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Artem Bityutskiy
-# License: GPLv2
-
-srcdir="$(readlink -ev -- ${0%/*})"
-PATH="$srcdir:$srcdir/helpers/libshell:$srcdir/helpers:$PATH"
-
-. shell-error
-. shell-args
-. shell-signal
-. shell-quote
-. aiaiai-sh-functions
-
-PROG="${0##*/}"
-
-# This is a small trick to make sure the script is portable - check if 'dash'
-# is present, and if yes - use it.
-if can_switch_to_dash; then
- exec dash -euf -- "$srcdir/$PROG" "$@"
- exit $?
-fi
-
-show_usage()
-{
- cat <<-EOF
-Usage: $PROG [options] -- ...
-
-This is an internal aiaiai helper program which runs various source code
-analysis tools when building the kernel. Options for $PROG has to go first,
-then there has to be a "--" delimiter, and then any number of options which are
-not interpreted by this script but passed further to the code analysis tool
-(sparse or smatch).
-
-Options:
- --sparse check with sparse;
- --smatch check with smatch;
- --cppcheck check with cppcheck;
- --coccinelle check with coccinelle (spatch);
- --check-only=FILE check only files listed in FILE;
- -h, --help show this text and exit.
-EOF
-}
-
-fail_usage()
-{
- [ -z "$1" ] || printf "%s\n" "$1"
- show_usage
- exit 1
-}
-
-# Fetch the file name to check from the input arguments.
-get_file_to_check()
-{
- # The file name is the last argument
- for file; do true; done
- printf "%s" "$file"
-}
-
-# Check the file against all the coccinelle scripts we have
-run_coccinelle()
-{
- local spatch spatches flags pid
- local pids=
-
- spatches="$(find $srcdir -name '*.cocci')"
- for spatch in $spatches; do
- # Coccinelle is not stable enough so far and dies because of
- # internal issues sometimes or just never stops. So we specify
- # a timeout as well as ignore its error code.
- flags="-D report -no_show_diff -very_quiet -no_includes -include_headers -timeout 60"
-
- # Run coccinelle for each semantic patch in parallel. This may load the
- # system too heavily, though. We use aiaiai-locker to make sure
- # we have non-scrambled output.
- aiaiai-locker -s -l "$tmpdir/lockfile" -c \
- "spatch $flags -sp_file $spatch $file_to_check" ||: &
- pids="$pids $!"
- done
-
- for pid in $pids; do
- wait "$pid"
- done
-}
-
-tmpdir=
-cleanup_handler()
-{
- rm -rf $verbose -- "$tmpdir"
-
- # Just in case we were interrupted, kill all our children
- local child
- for child in $(ps -o pid --no-headers --ppid "$$"); do
- kill "$child" > /dev/null 2>&1 ||:
- done
-}
-set_cleanup_handler cleanup_handler
-
-TEMP=`getopt -n $PROG -o h --long sparse,smatch,cppcheck,coccinelle,check-only:,help -- "$@"` ||
- fail_usage ""
-eval set -- "$TEMP"
-
-[ "$#" -gt "0" ] || exit 0
-
-run_sparse=
-run_smatch=
-run_cppcheck=
-run_coccinelle=
-check_only=
-
-while true; do
- case "$1" in
- --sparse)
- run_sparse=1
- program_required "sparse" "See section 'sparse' in doc/README"
- ;;
- --smatch)
- run_smatch=1
- program_required "smatch" "See section 'smatch' in doc/README"
- ;;
- --cppcheck)
- run_cppcheck=1
- program_required "cppcheck" "Usually Linux distribution provide a cppcheck package"
- ;;
- --coccinelle)
- run_coccinelle=1
- program_required "spatch" "Usually Linux distribution provide a 'spatch' or 'coccinelle' package"
- ;;
- --check-only)
- check_only="$(opt_check_read "$1" "$2")"
- shift
- ;;
- -h|--help)
- show_usage
- exit 0
- ;;
- --) shift; break
- ;;
- *) fail_usage "Unrecognized option: $1"
- ;;
- esac
- shift
-done
-
-program_required "aiaiai-locker" \
- "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
-
-tmpdir="$(mktemp -dt "$PROG.XXXX")"
-file_to_check="$(get_file_to_check "$@")"
-
-# Exit immediately if there is nothing to check
-if [ -z "$run_sparse" ] && [ -z "$run_smatch" ] && [ -z "$run_cppcheck" ] &&
- [ -z "$run_coccinelle" ]; then
- exit 0
-fi
-
-if [ -n "$check_only" ]; then
- match=
- files="$(cat "$check_only")"
- for file in $files; do
- match="$(printf "%s" "$@" | sed -n "/$(quote_sed_regexp "$file")/p")"
- [ -z "$match" ] || break
- done
-
- [ -n "$match" ] || exit 0
-fi
-
-# Run all the tools in background
-
-if [ -n "$run_sparse" ]; then
- # Sparse uses stderr for check results, not sure about stdout
- sparse -Wsparse-all "$@" > "$tmpdir/sparse" 2>&1 &
- pid_sparse="$!"
-fi
-if [ -n "$run_smatch" ]; then
- # Smatch uses stderr for reporting about internal issues and stdout for
- # check results
- smatch --project=kernel "$@" > "$tmpdir/smatch" 2>/dev/null &
- pid_smatch="$!"
-fi
-if [ -n "$run_cppcheck" ]; then
- # Cppcheck uses stderr for reporting about internal issues and stdout for
- # check results
- cppcheck -f -q --template=gcc "$file_to_check" > "$tmpdir/cppcheck" 2>/dev/null &
- pid_cppcheck="$!"
-fi
-if [ -n "$run_coccinelle" ]; then
- # Coccinelle uses stderr for reporting about internal issues and stdout for
- # check results
- run_coccinelle > "$tmpdir/coccinelle" 2>/dev/null &
- pid_coccinelle="$!"
-fi
-
-# Wait for the tools
-
-if [ -n "$run_sparse" ]; then
- wait "$pid_sparse" ||:
- if [ -s "$tmpdir/sparse" ]; then
- cat "$tmpdir/sparse" | sed -e "s/$/ [sparse]/" 1>&2
- fi
-fi
-if [ -n "$run_smatch" ]; then
- wait "$pid_smatch" ||:
- if [ -s "$tmpdir/smatch" ]; then
- cat "$tmpdir/smatch" | sed -e "s/$/ [smatch]/" 1>&2
- fi
-fi
-if [ -n "$run_cppcheck" ]; then
- wait "$pid_cppcheck" ||:
- if [ -s "$tmpdir/cppcheck" ]; then
- cat "$tmpdir/cppcheck" | sed -e "s/$/ [cppcheck]/" 1>&2
- fi
-fi
-if [ -n "$run_coccinelle" ]; then
- wait "$pid_coccinelle" ||:
- if [ -s "$tmpdir/coccinelle" ]; then
- cat "$tmpdir/coccinelle" | sed -e "s/$/ [coccinelle]/" 1>&2
- fi
-fi
diff --git a/aiaiai-decode-rfc-2047 b/aiaiai-decode-rfc-2047
deleted file mode 100755
index 24ece5b..0000000
--- a/aiaiai-decode-rfc-2047
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/perl
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Artem Bityutskiy
-# License: GPLv2
-#
-# This is a small helper script which decodes headers if they are encoded using
-# the MIME format defined in RFC2047.
-
-use Encode qw/encode decode/;
-
-while (my $line = <>) {
- chomp($line);
- $line = decode('UTF-8', $line);
- $line = decode('MIME-Header', $line);
- $line = encode('UTF-8', $line);
- print("$line\n");
-}
diff --git a/aiaiai-diff-log b/aiaiai-diff-log
deleted file mode 100755
index 54228d9..0000000
--- a/aiaiai-diff-log
+++ /dev/null
@@ -1,193 +0,0 @@
-#!/bin/sh -euf
-
-# Copyright 2011-2012 Intel Corporation
-# Authors: Artem Bityutskiy
-# Kirill Shutemov
-# License: GPLv2
-
-srcdir="$(readlink -ev -- ${0%/*})"
-PATH="$srcdir:$srcdir/helpers/libshell:$srcdir/helpers:$PATH"
-
-. shell-error
-. shell-args
-. shell-signal
-. shell-quote
-. aiaiai-sh-functions
-
-PROG="${0##*/}"
-message_time="yes"
-
-# This is a small trick to make sure the script is portable - check if 'dash'
-# is present, and if yes - use it.
-if can_switch_to_dash; then
- exec dash -euf -- "$srcdir/$PROG" "$@"
- exit $?
-fi
-
-show_usage()
-{
- cat <<-EOF
-Usage: $PROG [options] <patch> <log_before> <log_after>
-
-Compare 2 kernel build logs - before and after <patch> was applied.
-
-<patch> - the patch which was applied to the "before" kernel
-<log_before> - build log before <patch> was applied
-<log_after> - build log after <patch> was applied
-
-Options:
- -w, --workdir=WDIR path to the work directory (default: a temporary
- directory is created using mktemp);
- -p, --preserve preserve all the temporary files - do not clean up;
- -v, --verbose be verbose;
- -h, --help show this text and exit.
-EOF
-}
-
-fail_usage()
-{
- [ -z "$1" ] || printf "%s\n" "$1"
- show_usage
- exit 1
-}
-
-# After joining the "before" and "after" build logs, then sorting and splitting
-# them back, there may be identical lines but just shifted a bit, and this is
-# visible in the diff, which we do not want. E.g.:
-#
-# @@ @@
-#-cc1: warning: drivers/media/dvb/dvb-core: No such file or directory [enabled by default]
-#-cc1: warning: drivers/media/dvb/frontends: No such file or directory [enabled by default]
-#@@ @@
-#+cc1: warning: drivers/media/dvb/dvb-core: No such file or directory [enabled by default]
-#+cc1: warning: drivers/media/dvb/frontends: No such file or directory [enabled by default]
-#@@ @@
-#
-# This function tries to fix this up. Yes, this is ugly and we need to write a
-# good tool which compares parallel builds in C and invent a smarter algorithm
-# of comparison.
-remove_same_lines()
-{
- local diff="$1"
- local line
-
- while IFS= read -r line; do
- local l
-
- l="$(printf "%s" "$line" | sed -n -e 's/^[+-]\{1\}\([^+-]\{1\}.*\)$/\1/p')"
- if [ -z "$l" ]; then
- printf "%s\n" "$line"
- continue
- fi
-
- local p c1 c2
- quote_sed_regexp_variable p "$l"
- c1="$(grep -c -- "^-$p$" < "$diff" ||:)"
- c2="$(grep -c -- "^+$p$" < "$diff" ||:)"
-
- [ "$c1" -eq "$c2" ] || printf "%s\n" "$line"
- done < "$diff" | sed -n -e 'p; /^@@ @@$/ {:a n; /@@ @@/ba; p}' # Remove multiple "@@ @@"
-}
-
-# The remap-log utility handles renames for lines which start with the file
-# name, but this does not cover all the case. This function checks if the patch
-# renames any file and if yes, substitutes old names with new names.
-rename_log()
-{
- local file="$1"
- local renames pattern from to
-
- # Look for pairs of lines like this in the diff file:
- #
- # rename from drivers/a/xxx.c
- # rename to drivers/a/yyy.c
- #
- # and transform them to lines like this:
- #
- # drivers/a/xxx.c drivers/a/yyy.c
- #
- # and then pipe to the while loop
- sed -n -e '/^rename from / {
- s/^rename from \(.*\)/\1/
- N
- /\nrename to / s/\nrename to//p
- }' "$patch" | \
- while read -r from to; do
- local q1 q2
-
- # Quote special symbols and substitute old names with new names
- quote_sed_regexp_variable q1 "$from"
- quote_sed_regexp_variable q2 "$to"
- sed -i -e "s/$q1/$q2/g" "$file"
- done
-
-}
-
-tmpdir=
-preserve=
-cleanup_handler()
-{
- if [ -n "$preserve" ]; then
- message "Preserved tmpdir: $tmpdir"
- else
- rm $verbose -rf -- "$tmpdir" >&2
- fi
-}
-set_cleanup_handler cleanup_handler
-
-TEMP=`getopt -n $PROG -o w:,p,v,h --long workdir:,preserve,verbose,help -- "$@"` ||
- fail_usage ""
-eval set -- "$TEMP"
-
-verbose=
-
-while true; do
- case "$1" in
- -w|--workdir)
- mkdir $verbose -p -- "$2" >&2
- tmpdir="$(mktemp --tmpdir="$(readlink -fv -- "$2")" -dt "$PROG.XXXX")"
- shift
- ;;
- -p|--preserve)
- preserve="--preserve"
- ;;
- -v|--verbose) verbose=-v
- ;;
- -h|--help)
- show_usage
- exit 0
- ;;
- --) shift; break
- ;;
- *) fail_usage "Unrecognized option: $1"
- ;;
- esac
- shift
-done
-
-[ "$#" = 3 ] || die "Insufficient or too many arguments"
-
-program_required "remap-log" \
- "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
-
-patch="$(readlink -ev -- "$1")"; shift
-input_log1="$(readlink -ev -- "$1")"; shift
-input_log2="$(readlink -ev -- "$1")"; shift
-
-[ -n "$tmpdir" ] || tmpdir="$(mktemp -dt "$PROG.XXXX")"
-
-log1="$tmpdir/${input_log1##*/}.before"
-log2="$tmpdir/${input_log2##*/}.after"
-
-verbose "Comparing $input_log1 and $input_log2"
-
-remap-log < "$patch" > "$tmpdir/map"
-remap-log -o "" -p "line " "$tmpdir/map" < "$input_log1" > "$log1.remapped"
-
-rename_log "$log1.remapped"
-
-aiaiai-diff-log-helper "$log1.remapped" "$input_log2" "$tmpdir/diff"
-
-cat "$tmpdir/diff"
-
-verbose "Successfully compared $input_log1 and $input_log2"
diff --git a/aiaiai-diff-log-helper b/aiaiai-diff-log-helper
deleted file mode 100755
index 2372391..0000000
--- a/aiaiai-diff-log-helper
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-Sort an compare 2 build logs.
-
-Author: Ed Bartosh <eduard.bartosh at intel.com>
-Licence: GPLv2
-"""
-
-# Sorts 2 build logs and compares them. Blocks can be as follows.
-#
-# 1a. All consequitive lines starting with the same file prefix belong to one
-# block, e.g.:
-#
-# drivers/s.c:472:22: warning: incorrect type in assignment (different address spaces) [sparse]
-# drivers/s.c:472:22: expected struct table_header *mapped_table [sparse]
-# drivers/s.c:472:22: got void [noderef] <asn:2>* [sparse]
-#
-# (the prefix is "drivers/s.c:472")
-#
-# 1b. All consequitive lines starting with prefix "make:", "make[1]:", etc.
-# These usually belong to build failures.
-#
-# 2. GCC 'In file included from' blocks look like this:
-#
-# In file included from include/linux/kernel.h:17:0,
-# from include/linux/sched.h:55,
-# from arch/arm/kernel/asm-offsets.c:13
-# include/linux/bitops.h: In function 'hweight_long':
-# include/linux/bitops.h:55:26: warning: signed and unsigned type in expression
-#
-# or
-#
-# In file included from arch/x86/include/asm/uaccess.h:570:0,
-# from include/linux/uaccess.h:5,
-# from include/linux/highmem.h:7,
-# from include/linux/pagemap.h:10,
-# from fs/binfmt_misc.c:26:
-# In function ‘copy_from_user’,
-# inlined from ‘parse_command.part.0’ at fs/binfmt_misc.c:422:20:
-# arch/x86/include/asm/uaccess_32.h:211:26: warning: call to ‘copy_from_user’
-#
-# 3. GCC 'In function' blocks look like this:
-#
-# drivers/i.c: In function ‘gluebi_erase’:
-# drivers/i.c:177:2: warning: comparison of unsigned expression < 0 is always false [-Wtype-limits]
-#
-# 4. Any other line comprises an independent block
-
-import sys
-import os
-import re
-import itertools
-
-def gen_blocks(stream):
- """Parses input stream. Yields found blocks."""
- btype, prefix, block = "", "", []
- for line in stream:
- # append line to the current block if prefix matches
- if prefix and line.startswith(prefix):
- block.append(line)
-
- # Define prefix for cases 2 and 3 for further processing
- elif not prefix and btype in ("ifi", "infunc"):
- block.append(line)
- if re.match("^[^\s]+/[^\s]+:\d+:", line):
- prefix = ":".join(line.split(':')[:2])
- # 'In file inculded' block (case 2.)
- elif re.match("^In file included from .+", line):
- yield block
- btype = "ifi"
- prefix = ""
- block = [line]
- # 'In function' block (case 3.)
- elif re.match("^[^\s]+/[^\s]+: In function .+", line):
- yield block
- btype = "infunc"
- prefix = ""
- block = [line]
- # file prefixed block (case 1a.)
- elif re.match("^[^\s]+/[^\s]+:\d+:", line):
- yield block
- prefix = ":".join(line.split(':')[:2])
- btype = "prefix"
- block = [line]
- # file prefixed block (case 1b.)
- elif re.match("^make(\[\d+\]){0,1}:", line):
- yield block
- prefix = line.split(':')[0]
- btype = "prefix"
- block = [line]
- # the rest (case 4.)
- else:
- yield block
- btype = "plain"
- prefix = ""
- block = [line]
- yield block
-
-def main(argv):
- """Script entry point."""
-
- infile1, infile2 = open(argv[1]), open(argv[2])
- outfile = sys.stdout
- if len(argv) > 3:
- outfile = open(argv[3], "w")
-
- with open(argv[1]) as infile1, \
- open(argv[2]) as infile2:
-
- result = {}
- for blk1, blk2 in itertools.izip_longest(gen_blocks(infile1),
- gen_blocks(infile2), fillvalue=[]):
- if blk1 == blk2:
- continue
- for block, sign in [(tuple(blk1), "-"), (tuple(blk2), "+")]:
- if block:
- if block in result:
- del result[block]
- else:
- result[block] = sign
-
- result = sorted(result.items())
- if result:
- print "--- before_patching.log"
- print "+++ after_patching.log"
- prefix = ""
- for block, sign in result:
- if not prefix or not block[0].startswith(prefix):
- print "@@ @@"
- for line in block:
- print "%s%s" % (sign, line),
- if re.match("^[^\s]+/[^\s]+:", block[0]):
- prefix = block[0].split(':')[0]
-
- outfile.close()
-
-if __name__ == "__main__":
- if len(sys.argv) == 1 or sys.argv[1] == "--help":
- print "Usage: %s <input log 1> <input log 2> [<diff file>]" % \
- os.path.basename(sys.argv[0])
- sys.exit(0)
- sys.exit(main(sys.argv))
-
-# vim: ts=4 et sw=4 sts=4 ai sta:
diff --git a/aiaiai-locker.c b/aiaiai-locker.c
deleted file mode 100644
index e76d284..0000000
--- a/aiaiai-locker.c
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Copyright 2012 Intel Corporation
- * Author: Artem Bityutskiy <dedekind1 at gmail.com>
- * License: GPLv2
- *
- * Credits to Eric Melski for the idea and reference implementation:
- * http://www.cmcrossroads.com/ask-mr-make/12909-descrambling-parallel-build-logs
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <getopt.h>
-#include <stdbool.h>
-#include <sys/wait.h>
-#include <sys/file.h>
-#include <sys/stat.h>
-
-#define PROGRAM_NAME "aiaiai-locker"
-#define TMPFILE_NAME "/tmp/" PROGRAM_NAME ".XXXXXX"
-#define BUF_SIZE 8192
-
-static char *lockfile;
-static char *commands;
-static bool split = false;
-
-static void usage(FILE *dest, int status)
-{
- fprintf(dest,
-"Usage: " PROGRAM_NAME " -l <lockfile> [options]\n"
-"\n"
-"Capture entire stdout and stderr output of shell <commands> and print it\n"
-"when the commands finish. The idea is that commands should also use\n"
-PROGRAM_NAME " to serizlize (descrabmle) the output. This can be used\n"
-"with parallell make (e.g., -j10) to descramble the build log and make it\n"
-"readable. For example:\n"
-"\n"
-PROGRAM_NAME " -l lock -c \"make SHELL='" PROGRAM_NAME " -l lock' -j20\"\n"
-"\n"
-"The lockfile is a mandatory parameter which is used to make sure than only\n"
-"one process prints to stdout/stderr at a time.\n"
-"\n"
-"Options:\n"
-" -l, --lockfile=FILE path to the file to use for serializing the output;\n"
-" -c COMMANDS the commands to execute in a shell;\n"
-" -s, --split split stderr and stdout;\n"
-" -h, --help print help message and exit.\n"
-);
- exit(status);
-}
-
-static const struct option long_options[] = {
- { .name = "lock-file", .has_arg = 1, .flag = NULL, .val = 'l' },
- { .name = "split", .has_arg = 1, .flag = NULL, .val = 's' },
- { .name = "", .has_arg = 1, .flag = NULL, .val = 'c' },
- { .name = "help", .has_arg = 0, .flag = NULL, .val = 'h' },
- { NULL, 0, NULL, 0},
-};
-
-static void parse_options(int argc, char *argv[])
-{
- if (argc < 2) {
- fprintf(stderr, "Error: please, specify the lock file\n");
- usage(stderr, EXIT_FAILURE);
- }
-
- while (1) {
- int key;
-
- key = getopt_long(argc, argv, "l:c:sh", long_options, NULL);
- if (key == -1)
- break;
-
- switch (key) {
- case 'l':
- lockfile = optarg;
- break;
- case 'c':
- commands = optarg;
- break;
- case 's':
- split = true;
- break;
- case 'h':
- usage(stdout, EXIT_SUCCESS);
- case ':':
- fprintf(stderr, "Error: parameter is missing\n");
- default:
- fprintf(stderr, "Use -h for help\n");
- exit(EXIT_FAILURE);
- }
- }
-
- if (!lockfile) {
- fprintf(stderr, "Error: please, specify the lockfile\n");
- exit(EXIT_FAILURE);
- }
-
- if (!commands)
- exit(EXIT_SUCCESS);
-}
-
-int main(int argc, char *argv[])
-{
- int fstdout, fstderr = -1, lockfd;
- char fstdout_name[sizeof(TMPFILE_NAME)];
- char fstderr_name[sizeof(TMPFILE_NAME)];
- char buf[BUF_SIZE];
- pid_t child;
- int status = 0;
- ssize_t rd;
-
- parse_options(argc, argv);
-
- /* Create temporary files for stdout and stderr of the child program */
- memcpy(fstdout_name, TMPFILE_NAME, sizeof(TMPFILE_NAME));
- memcpy(fstderr_name, TMPFILE_NAME, sizeof(TMPFILE_NAME));
- fstdout = mkstemp(fstdout_name);
- if (fstdout == -1) {
- perror("mkstemp");
- exit(EXIT_FAILURE);
- }
- if (split) {
- fstderr = mkstemp(fstderr_name);
- if (fstderr == -1) {
- perror("mkstemp");
- goto out_fstdout;
- }
- }
-
- child = fork();
- if (child == -1) {
- perror("fork");
- goto out_fstderr;
- }
-
- if (child == 0) {
- char * const sh_argv[4] = { "/bin/sh", "-c", commands, 0 };
-
- close(1);
- close(2);
-
- if (dup2(fstdout, 1) == -1)
- exit(EXIT_FAILURE);
- if (dup2(split ? fstderr : fstdout, 2) == -1)
- exit(EXIT_FAILURE);
- if (execvp("/bin/sh", sh_argv) == -1) {
- perror("execvp");
- exit(EXIT_FAILURE);
- }
-
- exit(EXIT_FAILURE);
- }
-
- if (waitpid(child, &status, 0) == -1) {
- perror("waitpid");
- goto out_fstderr;
- }
-
- if (lseek(fstdout, 0, SEEK_SET) == -1) {
- perror("lseek");
- goto out_fstderr;
- }
- if (split) {
- if (lseek(fstderr, 0, SEEK_SET) == -1) {
- perror("lseek");
- goto out_fstderr;
- }
- }
-
- lockfd = open(lockfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
- if (lockfd == -1) {
- perror("open");
- goto out_fstderr;
- }
-
- if (flock(lockfd, LOCK_EX) == -1) {
- perror("flock");
- goto out_fstderr;
- }
-
- while ((rd = read(fstdout, buf, BUF_SIZE)) > 0) {
- if (write(1, buf, rd) != rd) {
- perror("write");
- goto out_unlock;
- }
- }
- if (rd == -1) {
- perror("read");
- goto out_unlock;
- }
-
- if (split) {
- while ((rd = read(fstderr, buf, BUF_SIZE)) > 0) {
- if (write(2, buf, rd) != rd) {
- perror("write");
- goto out_unlock;
- }
- }
- if (rd == -1) {
- perror("read");
- goto out_unlock;
- }
- }
-
- flock(lockfd, LOCK_UN);
- close(lockfd);
- if (split) {
- close(fstderr);
- unlink(fstderr_name);
- }
- close(fstdout);
- unlink(fstdout_name);
- exit(WEXITSTATUS(status));
-
-out_unlock:
- flock(lockfd, LOCK_UN);
- close(lockfd);
-out_fstderr:
- if (split) {
- close(fstderr);
- unlink(fstderr_name);
- }
-out_fstdout:
- close(fstdout);
- unlink(fstdout_name);
- return EXIT_FAILURE;
-}
diff --git a/aiaiai-make-kernel b/aiaiai-make-kernel
deleted file mode 100755
index 3bb2f02..0000000
--- a/aiaiai-make-kernel
+++ /dev/null
@@ -1,285 +0,0 @@
-#!/bin/sh -efu
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Kirill Shutemov, Artem Bityutskiy
-# License: GPLv2
-
-srcdir="$(readlink -ev -- ${0%/*})"
-export PATH="$srcdir:$srcdir/helpers/libshell:$srcdir/helpers:$PATH"
-
-. shell-error
-. shell-args
-. shell-signal
-. shell-quote
-. aiaiai-sh-functions
-
-PROG="${0##*/}"
-message_time="yes"
-
-# This is a small trick to make sure the script is portable - check if 'dash'
-# is present, and if yes - use it.
-if can_switch_to_dash; then
- exec dash -euf -- "$srcdir/$PROG" "$@"
- exit $?
-fi
-
-show_usage()
-{
- cat <<-EOF
-Usage: $PROG [options] <kernel-tree> <targets...>
-
-<kernel-tree> - directory with kernel sources
-<targets...> - list of targets to make.
-
-Options:
- -o, --objdir=OBJDIR locate all output files in OBJDIR, including .config
- (O=OBJDIR Kbuild feature); if undefined - use the
- source directory (<kernel-tree>);
- -D, --defconfig=CONF name of the defconfig to use for each target; if the
- name contains a "/" character, it is interpreted as a
- stand-alone defconfig file, otherwise it is assumed
- that CONF is part of the <kernel-tree>;
- -j, --jobs=N allow to run N jobs simultaneously (default is 1);
- -O, --stdout=FILE redirect stdout of the build to FILE;
- -E, --stderr=FILE redirect stderr of the build to FILE (note, if neither
- -O nor -E are specified, everything is redirected to
- stdout);
- -k, --keep-going continue as much as possible after an error (see
- -k GNU make option);
- -a, --arch=A,[C] test for specified architecture (ARCH=) and the
- cross-compiler prefix (CROSS_COMPILE=); Examples:
- -a i386, -a arm,arm-eabi-, etc;
- --sparse check with sparse while building;
- --smatch check with smatch while building;
- --cppcheck check with cppcheck while building;
- --coccinelle check with coccinelle (spatch) while building;
- --check-only=FILE check only files listed in FILE;
- -M, --kmake-opts additional options to append to the final kernel
- compilation 'make' command
- (e.g., W=2 KALLSYMS_EXTRA_PASS=1)
- -v, --verbose be verbose;
- -h, --help show this text and exit.
-EOF
-}
-
-fail_usage()
-{
- [ -z "$1" ] || printf "%s\n" "$1"
- show_usage
- exit 1
-}
-
-# Strip useless stuff from the log and make it possible to compare it to
-# another build log.
-strip_log()
-{
- local logfile="$1"
-
- sed -i -e "# Strip useless stuff
- /^CRC [[:xdigit:]]\+$/d
- /^System is [[:digit:]]\+ kB$/d
- /^Setup is [[:digit:]]\+ bytes (padded to [[:digit:]]\+ bytes).$/d
- /^Root device is ([[:digit:]]\+, [[:digit:]]\+)$/d
-
- # Strip not very useful messages from modpost
- /^To see full details build your kernel with:$/d
- /^'make CONFIG_DEBUG_SECTION_MISMATCH=y'$/d
-
- # Make all source paths relative to the kernel tree root
- s/$(quote_sed_regexp "$kernel_tree/")//g" "$logfile"
-
- # Strip objdir paths as well
- [ -z "$objdir" ] || sed -i -e "s/$(quote_sed_regexp "$objdir/")//g" "$logfile"
-}
-
-make_target()
-{
- local target="$1"
-
- if [ -n "$defconfig" ]; then
- if printf "%s" "$defconfig" | grep -q '/'; then
- # The defconfig is a path to a stand-alone defconfig file
- # FIXME: ugly hack!
- make -C $kernel_tree ${arch:+ARCH="$arch"} \
- ${cross:+CROSS_COMPILE="$cross"} \
- ${objdir:+O="$objdir"} \
- KCONFIG_ALLCONFIG="$defconfig" -- alldefconfig
- else
- # The defconfig is part of the git tree
- make -C $kernel_tree ${arch:+ARCH="$arch"} \
- ${cross:+CROSS_COMPILE="$cross"} \
- ${objdir:+O="$objdir"} -- "$defconfig"
- fi
- fi
-
- aiaiai-locker $split -l "$lockfile" -c \
- "make $keep_going -j $jobs -C $kernel_tree ${arch:+ARCH="$arch"} \
- ${cross:+CROSS_COMPILE="$cross"} ${objdir:+O="$objdir"} \
- CHECK=\"aiaiai-checker $sparse $smatch $cppcheck $coccinelle $check_only --\" \
- KCFLAGS='-Wno-missing-field-initializers -Wno-sign-compare' \
- C=$check ${check:+CF="-D__CHECK_ENDIAN__"} W=1 SHELL=\"aiaiai-locker $split -l $lockfile\" \
- $kmake_opts $target"
-}
-
-lockfile=
-build_stdout=
-build_stdout_given=
-build_stderr=
-build_stderr_given=
-build_output=
-cleanup_handler()
-{
- rm $verbose -f -- "$lockfile" >&2
- [ -n "$build_stdout_given" ] || rm $verbose -f -- "$build_stdout" >&2
- [ -n "$build_stderr_given" ] || rm $verbose -f -- "$build_stderr" >&2
- rm $verbose -f -- "$build_output" >&2
-}
-set_cleanup_handler cleanup_handler
-
-TEMP=`getopt -n $PROG -o o:,D:,j:,O:,E:,k:,a:,M:,v,h --long objdir:,defconfig:,jobs:,stdout:,stderr:,keep-going,arch:,sparse,smatch,cppcheck,coccinelle,check-only:,kmake-opts:,verbose,help -- "$@"` ||
- fail_usage ""
-eval set -- "$TEMP"
-
-objdir=
-defconfig=
-jobs=1
-keep_going=
-arch=
-cross=
-sparse=
-smatch=
-cppcheck=
-coccinelle=
-check_only=
-check=0
-kmake_opts=
-verbose=
-
-while true; do
- case "$1" in
- -o|--objdir)
- mkdir $verbose -p -- "$2" >&2
- objdir="$(readlink -fv -- "$2")"
- shift
- ;;
- -D|--defconfig)
- defconfig="$2"
- shift
- ;;
- -j|--jobs)
- jobs=$(opt_check_number "$1" "$2")
- shift
- ;;
- -O|--stdout)
- touch "$2"
- build_stdout="$(readlink -fv -- "$2")"
- build_stdout_given="y"
- shift
- ;;
- -E|--stderr)
- touch "$2"
- build_stderr="$(readlink -fv -- "$2")"
- build_stderr_given="y"
- shift
- ;;
- -k|--keep-going)
- keep_going="--keep-going"
- ;;
- -a|--arch)
- arch="$(leave_first "$2")"
- cross="$(leave_second "$2")"
- shift
- ;;
- --sparse)
- sparse="--sparse"
- check=1
- ;;
- --smatch)
- smatch="--smatch"
- check=1
- ;;
- --cppcheck)
- cppcheck="--cppcheck"
- check=1
- ;;
- --coccinelle)
- coccinelle="--coccinelle"
- check=1
- ;;
- --check-only)
- check_only="--check-only $(opt_check_read "$1" "$2")"
- shift
- ;;
- -M|--kmake-opts)
- kmake_opts="$2"
- shift
- ;;
- -v|--verbose) verbose=-v
- ;;
- -h|--help)
- show_usage
- exit 0
- ;;
- --) shift; break
- ;;
- *) fail_usage "Unrecognized option: $1"
- ;;
- esac
- shift
-done
-
-[ "$#" -ge 2 ] || die "Insufficient arguments"
-
-program_required "aiaiai-locker" \
- "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
-program_required "${cross}gcc" ""
-program_required "make" ""
-
-kernel_tree="$(opt_check_dir "kernel-tree" "$1")"; shift
-build_info="$kernel_tree${objdir:+ (O="$objdir")}"
-
-if [ -n "$build_stdout_given" ] || [ -n "$build_stderr_given" ]; then
- split="-s"
- [ -n "$build_stdout_given" ] || build_stdout="$(mktemp -t "$PROG.stdout.XXXX")"
- [ -n "$build_stderr_given" ] || build_stderr="$(mktemp -t "$PROG.stderr.XXXX")"
-else
- split=
- build_output="$(mktemp -t "$PROG.output.XXXX")"
-fi
-
-lockfile="$(mktemp -t "$PROG.lock.XXXX")"
-
-failed=
-for target in $@; do
- verbose "Making target \"$target\" for \"$build_info\""
-
- if [ -z "$split" ]; then
- make_target "$target" > "$build_output" || failed=1
- [ -z "$failed" ] || echo "FAILURE" >> "$build_output"
- else
- make_target "$target" > "$build_stdout" 2> "$build_stderr" || failed=1
- [ -z "$failed" ] || echo "FAILURE" >> "$build_stderr"
- fi
-
- if [ -n "$failed" ]; then
- verbose "Failed to make target \"$target\" for \"$build_info\""
- break
- fi
-done
-
-[ -z "$build_stdout" ] || strip_log "$build_stdout"
-[ -z "$build_stderr" ] || strip_log "$build_stderr"
-[ -z "$build_output" ] || strip_log "$build_output"
-
-if [ -n "$build_stdout_given" ] || [ -n "$build_stderr_given" ]; then
- [ -n "$build_stdout_given" ] || cat "$build_stdout"
- [ -n "$build_stderr_given" ] || cat "$build_stderr"
-else
- [ -z "$build_output" ] || cat "$build_output"
-fi
-
-if [ -z "$failed" ]; then
- verbose "Successfully made all targets for \"$build_info\""
-else
- verbose "Failed to make a target, exiting"
-fi
diff --git a/aiaiai-match-keywords b/aiaiai-match-keywords
deleted file mode 100755
index bab7902..0000000
--- a/aiaiai-match-keywords
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/sh -euf
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Alexander Shishkin, Artem Bityutskiy
-# License: GPLv2
-
-srcdir="$(readlink -ev -- ${0%/*})"
-PATH="$srcdir:$srcdir/helpers/libshell:$PATH"
-
-. shell-error
-. shell-args
-. shell-quote
-. aiaiai-sh-functions
-
-PROG="${0##*/}"
-message_time="yes"
-
-# This is a small trick to make sure the script is portable - check if 'dash'
-# is present, and if yes - use it.
-if can_switch_to_dash; then
- exec dash -euf -- "$srcdir/$PROG" "$@"
- exit $?
-fi
-
-show_usage()
-{
- cat <<-EOF
-Usage: $PROG [options] <file>
-
-Match various keywords - e.g., companies often prohibit some for whatever
-reasons. The keywords are read from stdin (one keyword per line) and matched
-against <file>.
-
-Options:
- -v, --verbose be verbose;
- -h, --help show this text and exit.
-EOF
-}
-
-fail_usage()
-{
- [ -z "$1" ] || printf "%s\n" "$1"
- show_usage
- exit 1
-}
-
-first_match=
-look_for_keyword()
-{
- local kw="$1"
- local match
-
- verbose "look for keyword \"$kw\""
-
- match="$(grep -i "$(quote_sed_regexp "$kw")" "$file" ||:)"
- [ -n "$match" ] || return 0
-
- [ -z "$first_match" ] || echo ""
-
- first_match="no"
- printf "%s\n\n" "Matched keyword \"$kw\""
- printf "%s\n" "$match" | head -n8
-}
-
-TEMP=`getopt -n $PROG -o v,h --long verbose,help -- "$@"` || fail_usage ""
-eval set -- "$TEMP"
-
-mbox=
-verbose=
-
-while true; do
- case "$1" in
- -v|--verbose)
- verbose=-v
- ;;
- -h|--help)
- show_usage
- exit 0
- ;;
- --) shift; break
- ;;
- *) fail_usage "Unrecognized option: $1"
- ;;
- esac
- shift
-done
-
-[ "$#" -eq 1 ] || die "Please, specify exactly one argument - the file with keywords"
-file="$(readlink -ev -- "$1")"
-
-while IFS= read -r kw; do
- look_for_keyword "$kw"
-done
diff --git a/aiaiai-sh-functions b/aiaiai-sh-functions
deleted file mode 100644
index abbd0f4..0000000
--- a/aiaiai-sh-functions
+++ /dev/null
@@ -1,229 +0,0 @@
-#!/bin/sh -efu
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Artem Bityutskiy
-# License: GPLv2
-
-. shell-error
-
-if [ -z "${__included_aiaiai_sh_functions-}" ]; then
-__included_aiaiai_sh_functions=1
-
-# Print an error message and exit
-# Usage: die <message>
-die()
-{
- fatal "Fatal error: $1"
-}
-
-# Print a separator line to stdout
-print_separator()
-{
- local i=0
-
- while [ $i -lt 80 ]; do
- i=$(($i+1));
- printf "-";
- done
- echo
-}
-
-# Check if dash is available and we are not running in dash
-can_switch_to_dash()
-{
- if command -v "dash" >/dev/null 2>&1; then
- if [ -n "${BASH_VERSION:-}" ]; then
- return 0
- fi
- fi
-
- return 1
-}
-
-# Die if a program is not in PATH
-# Usage: program_required <program_name>
-program_required()
-{
- local prg="$1"; shift
- local msg="$1"; shift
-
- if ! command -v "$prg" >/dev/null 2>&1; then
- message "Program \"$prg\" is required but not found in PATH"
- if [ -n "$msg" ]; then
- die "$msg"
- else
- exit 1
- fi
- fi
-}
-
-# Fetch the first occurrence of an mbox header from a file
-# Usage: fetch_header <header_name> < <mbox_file>
-fetch_header()
-{
- local hdr="$1"
-
- program_required "formail" ""
-
- # Take only the first occurrence of the header
- formail -z -c -x "$hdr:" | head -n1 | aiaiai-decode-rfc-2047
-}
-
-# Similar to fetch_header, but exits with code 1 if the header hasn't been
-# found, and has a little bit different interface (the result is stored in
-# "<var>").
-#
-# Usage: fetch_header_or_die <var> <header_name> < <mbox_file>
-fetch_header_or_die()
-{
- local var="$1"; shift
- local hdr="$1"; shift
- local res="$(fetch_header "$hdr")"
-
- [ -n "$res" ] || die "Cannot find the \"$hdr:\" header"
-
- eval "$var=\"\$res\""
-}
-
-fetch_header_per_patch()
-{
- local hdr="$1"
-
- program_required "formail" ""
-
- # Take only the first occurrence of the header per message
- formail -s sh -c "formail -z -c -x \"$hdr:\" | head -n1" | aiaiai-decode-rfc-2047
-}
-
-git_dir()
-{
- local path="$1"
- local dotgit="$path/.git"
-
- if [ -d "$dotgit" ]; then
- printf "%s" "$dotgit"
- elif [ -d "$path" ]; then
- printf "%s" "$path"
- else
- die "not a git repository: $path"
- fi
-}
-
-# Apply a patch. In case of error, print user-friendly diagnostic messages to
-# stdin.
-# Usage: apply_patch < <mbox_file>
-apply_patch()
-{
- local am cmt
-
- program_required "patch" ""
-
- cmt="$(git rev-parse "HEAD^{commit}")"
-
- am="$(git am 2>&1)" || {
- cat <<EOF
-Failed to apply patch(es) with git am on top of:
-$(git log -1 --oneline "$cmt")
-
-$am
-
-Results of "patch --merge=diff3 -p1 < .git/rebase-apply/patch":
-
-$(patch --merge=diff3 -p1 < .git/rebase-apply/patch 2>&1)
-
-$(print_separator)
-
-$(git diff --no-color)
-EOF
- return 1
- }
-}
-
-# A helper function for 'build_failure()'. This function expects the properly
-# formatted build log a stdin and outputs user-readable failure report to
-# stdout.
-__print_build_log()
-{
- local config="$(leave_first "$1")";
- local arch="$(leave_second "$1")"; shift
- local commit_id="$1"; shift
- local commit_info="$(git log -1 --oneline "$commit_id")"
- local commit_nickname="${1:-""}"
-
- cat <<EOF
-Failed to build $commit_nickname: $commit_info
-Configuration: "$config${arch:+", architecture $arch"}".
-
-$(cat)
-EOF
-}
-
-# Format a build failure report.
-# Usage: build_failure <defconfig> <commit_id> <commit_nickname> < <build_log>
-build_failure()
-{
- # The build log might have been generated with multiple jobs which
- # means it is probably messy and the error message is probably not at
- # the very end. To make it more probable that we actually print the
- # build error message within 24 lines we do the following:
- # * filter sparse/smatch/cppcheck/coccinelle output
- # * filter out 'CHECK drivers/blah.c' Kbuild lines
- # * print 24 lines preceding the 'make[]: *** [] blah' pattern which
- # make generates after an error
-
- sed -n '# Filter out useless stuff
- /\[sparse\]$/d
- /\[smatch\]$/d
- /\[cppcheck\]$/d
- /\[coccinelle\]$/d
- /^ CHECK /d
- # Add the line to the hold buffer
- H
- # If the line is the error marker, print out the entire hold
- # buffer and quit
- /^make\[.*\]: \*\*\* \[.*\]/ { g; p; q; }
- # Do the same if the last line is reached
- $ { g; p; q; }' | tail -n24 |
- __print_build_log "$@"
-}
-
-# Check if the build failed.
-# Usage: build_failed <build_log>
-build_failed()
-{
- local build_log="$1"
- local failed
-
- failed="$(tail -n1 -- "$build_log")"
- test "$failed" = "FAILURE"
-}
-
-# Filter out the first element from a comma-separated list of elements.
-# Usage: strip_first <list>
-strip_first()
-{
- printf "%s" "$1" | sed -e 's/^[^,]*,\{0,1\}//g'
-}
-
-# Filter out all but the first element from a comma-separated list of elements.
-# Usage: leave_first <list>
-leave_first()
-{
- printf "%s" "$1" | sed -e 's/,.*$//g'
-}
-
-# Filter out all but the second element from a comma-separated list of elements.
-# Usage: leave_second <list>
-leave_second()
-{
- leave_first "$(strip_first "$1")"
-}
-
-# Filter out all but the third element from a comma-separated list of elements.
-# Usage: leave_third <list>
-leave_third()
-{
- leave_second "$(strip_first "$1")"
-}
-
-fi #__included_aiaiai_sh_functions
diff --git a/aiaiai-test-bisectability b/aiaiai-test-bisectability
deleted file mode 100755
index ffc0f55..0000000
--- a/aiaiai-test-bisectability
+++ /dev/null
@@ -1,273 +0,0 @@
-#!/bin/sh -efu
-
-# Copyright 2011-2012 Intel Corporation
-# Author: Artem Bityutskiy
-# License: GPLv2
-
-srcdir="$(readlink -ev -- ${0%/*})"
-PATH="$srcdir:$srcdir/helpers/libshell:$srcdir/helpers:$PATH"
-
-. shell-error
-. shell-args
-. shell-signal
-. aiaiai-sh-functions
-
-PROG="${0##*/}"
-message_time="yes"
-
-# This is a small trick to make sure the script is portable - check if 'dash'
-# is present, and if yes - use it.
-if can_switch_to_dash; then
- exec dash -euf -- "$srcdir/$PROG" "$@"
- exit $?
-fi
-
-show_usage()
-{
- cat <<-EOF
-Usage: $PROG [options] <kernel-tree> <defconfig[,arch[,cross]]>
-
-<kernel-tree> - directory with kernel sources
-<defconfig[,arch[,cross]]> - the configuration to test.
-
-The mbox file containing the patches to test is expected to come from stdin or
--i option can be used instead.
-
-The configuration is specified as follows:
-
- defconfig[,arch[,cross]]
-
-where
-
- * "defconfig" is the name of the kernel defconfig file to test
- * "arch" is the architecture (translates to Kbuild's ARCH variable value)
- * "cross" is the cross-compiler prefix (translates to Kbuild's CROSS_COMPILE
- variable value).
-
-For example:
-
- omap2plus_defconfig,arm,arm-eabi-
-
-means testing with defconfig "omap2plus_defconfig", ARCH=arm and
-CROSS_COMPILE=arm-eabi-.
-
-By default, $PROG assumes that the defconfig file is part of
-the kernel tree, unless --confdir option is specified, in which case the
-defconfig file will be searched for only in the specified directory.
-
-Options:
- -j, --jobs=N allow to run N jobs simultaneously (default is 1);
- -c, --commit-id=ID the commit id to test against (default is the head of
- the git tree (HEAD));
- -d, --dont-clone work directly in <kernel-tree>, do not clone a
- temporary copy of <kernel-tree>;
- -o, --objdir=OBJDIR locate all output files in OBJDIR, including .config
- (O=OBJDIR Kbuild feature); if undefined - a temporary
- directory is created using mktemp);
- -C, --confdir=CDIR path to the directory containing the defconfig file
- by default the defconfig file is assumed to be part of
- the <kernel-tree>; this option makes it possible to use
- a stand-alone defconfig file instead;
- -i, --input=MBOX use the MBOX file instead of stdin;
- -w, --workdir=WDIR path to the work directory where the kernel will
- be built (default: a temporary directory is created
- using mktemp);
- -M, --kmake-opts additional options to append to the final kernel
- compilation 'make' command
- (e.g., W=2 KALLSYMS_EXTRA_PASS=1)
- -p, --preserve preserve all the temporary files - do not clean up;
- -v, --verbose be verbose;
- -h, --help show this text and exit.
-EOF
-}
-
-fail_usage()
-{
- [ -z "$1" ] || printf "%s\n" "$1"
- show_usage
- exit 1
-}
-
-# Apply patch <mbox>, compile the kernel and save build logs at <stdout_log>
-# and <stderr_log>.
-# Usage: compile_patch <mbox> <stdout_log> <stderr_log>
-compile_patch()
-{
- local file="$1"; shift
- local stdout_log="$1"; shift
- local stderr_log="$1"; shift
- local subj
-
- fetch_header_or_die subj "Subject" < "$file"
-
- verbose "Compiling \"$subj\""
-
- # Apply the patch
- apply_patch < "$file" || exit 0
-
- aiaiai-make-kernel $verbose -j "$jobs" -o "$objdir" $arch_opt -D "$dconf" \
- -O "$stdout_log" -E "$stderr_log" ${kmake_opts:+-M "$kmake_opts"} \
- -- "$cloned_kernel" all
-}
-
-tmpdir=
-preserve=
-cleanup_handler()
-{
- if [ -n "$preserve" ]; then
- message "Preserved tmpdir: $tmpdir"
- else
- [ -z "$tmpdir" ] || verbose "Removing $tmpdir";
- rm -rf -- "$tmpdir" >&2
- fi
-}
-set_cleanup_handler cleanup_handler
-
-TEMP=`getopt -n $PROG -o j:,c:,d,o:,C:,i:,w:,M:,p,v,h --long jobs:,commit-id:,dont-clone,objdir:,confdir:,input:,workdir:,kmake-opts:,preserve,verbose,help -- "$@"` ||
- fail_usage ""
-eval set -- "$TEMP"
-
-jobs=1
-commit_id=HEAD
-dont_clone=
-objdir=
-confdir=
-mbox=
-kmake_opts=
-verbose=
-quiet="-q"
-
-while true; do
- case "$1" in
- -j|--jobs)
- jobs=$(opt_check_number "$1" "$2")
- shift
- ;;
- -c|--commit-id)
- commit_id="$2"
- shift
- ;;
- -d|--dont-clone)
- dont_clone="--dont-clone"
- ;;
- -o|--objdir)
- objdir="$(readlink -fv -- "$2")"
- shift
- ;;
- -C|--confdir)
- confdir="$(opt_check_dir "$1" "$2")"
- shift
- ;;
- -i|--input)
- mbox="$(opt_check_read "$1" "$2")"
- shift
- ;;
- -w|--workdir)
- mkdir $verbose -p -- "$2" >&2
- tmpdir="$(mktemp --tmpdir="$(readlink -fv -- "$2")" -dt "$PROG.XXXX")"
- shift
- ;;
- -M|--kmake-opts)
- kmake_opts="$2"
- shift
- ;;
- -p|--preserve)
- preserve="--preserve"
- ;;
- -v|--verbose)
- verbose=-v
- quiet=
- ;;
- -h|--help)
- show_usage
- exit 0
- ;;
- --) shift; break
- ;;
- *) fail_usage "Unrecognized option: $1"
- ;;
- esac
- shift
-done
-
-[ "$#" -eq 2 ] || die "Please, specify exactly 2 arguments - the kernel tree and the configuration"
-
-kernel_tree="$(readlink -ev -- "$1")"; shift
-defconfig_orig="$1"
-defconfig="$(leave_first "$1")";
-arch="$(leave_second "$1")";
-cross="$(leave_third "$1")";
-arch_opt=
-
-[ -z "$arch" ] || arch_opt="-a $arch${cross:+",$cross"}"
-
-[ -n "$tmpdir" ] || tmpdir="$(mktemp -dt "$PROG.XXXX")"
-tmp_mbox="$tmpdir/tmp_mbox"
-[ -n "$objdir" ] || objdir="$tmpdir/obj.$defconfig"
-
-# Save the mbox to a temporary file if it comes from stdin
-if [ -z "$mbox" ]; then
- mbox="$tmpdir/mbox"
- cat > "$mbox"
-fi
-
-patch_cnt="$(formail -s echo 1 < "$mbox" | wc -l)"
-if [ "$patch_cnt" -le 1 ]; then
- verbose "Only one patch, nothing to test"
- exit 0;
-fi
-
-verbose "Bisecting $patch_cnt patches for configuration \"$defconfig_orig\""
-
-if [ -z "$dont_clone" ]; then
- cloned_kernel="$tmpdir/src"
- # Use a shared clone since it is local and we won't mess up the tree
- git clone --shared "${verbose:--q}" "$kernel_tree" "$cloned_kernel" >&2
-else
- cloned_kernel="$kernel_tree"
-fi
-
-cd "$cloned_kernel"
-commit_id="$(git --git-dir="$(git_dir "$kernel_tree")" rev-parse "$commit_id^{commit}")"
-git reset $quiet --hard "$commit_id" >&2
-
-dconf="$defconfig"
-[ -z "$confdir" ] || dconf="$confdir/$dconf"
-
-base_stdout_log="$tmpdir/$defconfig.base.stdout.log"
-base_stderr_log="$tmpdir/$defconfig.base.stderr.log"
-stdout_log="$tmpdir/$defconfig.stdout.log"
-stderr_log="$tmpdir/$defconfig.stderr.log"
-
-verbose "Building the base tree"
-base_commit_failed=
-aiaiai-make-kernel $verbose -j "$jobs" -o "$objdir" $arch_opt -D "$dconf" \
- -O "$base_stdout_log" -E "$base_stderr_log" \
- ${kmake_opts:+-M "$kmake_opts"} -- "$cloned_kernel" all
-if build_failed "$base_stderr_log"; then
- # Failed to build the base commit, report, but don't exit in case the
- # first patch fixes the issue.
- base_commit_failed="yes"
-fi
-
-n=0
-while [ "$n" -lt "$patch_cnt" ]; do
- formail +$n -1 -s < "$mbox" > "$tmp_mbox"
- compile_patch "$tmp_mbox" "$stdout_log" "$stderr_log"
- n="$(($n+1))"
-
- if build_failed "$stderr_log"; then
- if [ "$n" = 1 ] && [ -n "$base_commit_failed" ]; then
- # Base commit failed too, report about this
- build_failure "$defconfig_orig" "$commit_id" \
- "the base commit" < "$base_stderr_log"
- printf "\n\n"
- fi
- build_failure "$defconfig_orig" "HEAD" "patch #$n"< "$stderr_log"
- exit 0
- fi
-done
-
-if [ -n "$base_commit_failed" ]; then
- printf "%s\n" "Base commit build failed, but the first patch fixed the build failure."
-fi
diff --git a/doc/TODO.txt b/doc/TODO.txt
index 7652d16..c5eed38 100644
--- a/doc/TODO.txt
+++ b/doc/TODO.txt
@@ -39,10 +39,6 @@ implementing them.
instead of just failing. This is probably more user-friendly.
* Describe some review policy, e.g., how many days I promise to wait
for review/reply before applying patches.
- * Re-arrange the source code structure so that the internal scripts
- would all sit in a separate sub-directory. Internal scripts are
- those which users are not supposed to run directly. For example,
- 'aiaiai-locker' is one of them.
* Functional improvements
* Gcc 4.8 change the format of warnings and now they look like this:
diff --git a/email/aiaiai-email-dispatcher b/email/aiaiai-email-dispatcher
index 7487cd9..5920968 100755
--- a/email/aiaiai-email-dispatcher
+++ b/email/aiaiai-email-dispatcher
@@ -5,8 +5,9 @@
# License: GPLv2
srcdir="$(readlink -ev -- ${0%/*}/..)"
-PATH="$srcdir:$srcdir/email:$srcdir/helpers/libshell:$PATH"
+PATH="$srcdir/helpers:$srcdir/email:$srcdir/helpers/libshell:$PATH"
+echo $PATH
. shell-error
. shell-args
. shell-signal
diff --git a/email/aiaiai-email-dispatcher-helper b/email/aiaiai-email-dispatcher-helper
index 7d0365d..9dfe115 100755
--- a/email/aiaiai-email-dispatcher-helper
+++ b/email/aiaiai-email-dispatcher-helper
@@ -5,7 +5,7 @@
# License: GPLv2
srcdir="$(readlink -ev -- ${0%/*}/..)"
-PATH="$srcdir:$srcdir/email:$srcdir/helpers/libshell:$PATH"
+PATH="$srcdir/helpers:$srcdir/email:$srcdir/libshell:$PATH"
. shell-error
. shell-signal
diff --git a/email/aiaiai-email-lda b/email/aiaiai-email-lda
index b9f2854..770f96e 100755
--- a/email/aiaiai-email-lda
+++ b/email/aiaiai-email-lda
@@ -5,7 +5,7 @@
# License: GPLv2
srcdir="$(readlink -ev -- ${0%/*}/..)"
-PATH="$srcdir:$srcdir/email:$srcdir/helpers/libshell:$PATH"
+PATH="$srcdir/helpers:$srcdir/email:$srcdir/helpers/libshell:$PATH"
. shell-error
. shell-args
diff --git a/email/aiaiai-email-test-patchset b/email/aiaiai-email-test-patchset
index aa76b71..1988a71 100755
--- a/email/aiaiai-email-test-patchset
+++ b/email/aiaiai-email-test-patchset
@@ -5,7 +5,7 @@
# License: GPLv2
srcdir="$(readlink -ev -- ${0%/*}/..)"
-PATH="$srcdir:$srcdir/email:$srcdir/helpers/libshell:$PATH"
+PATH="$srcdir:$srcdir/helpers:$srcdir/email:$srcdir/helpers/libshell:$PATH"
. shell-error
. shell-args
diff --git a/gerrit/aiaiai-jenkins-test-patchset b/gerrit/aiaiai-jenkins-test-patchset
index 326268b..86294fc 100755
--- a/gerrit/aiaiai-jenkins-test-patchset
+++ b/gerrit/aiaiai-jenkins-test-patchset
@@ -4,8 +4,8 @@
# Author: Artem Bityutskiy
# License: GPLv2
-srcdir="$(readlink -ev -- ${0%/*})"
-PATH="$srcdir/..:$srcdir/../helpers/libshell:$PATH"
+srcdir="$(readlink -ev -- ${0%/*}/..)"
+PATH="$srcdir:$srcdir/helpers/libshell:$PATH"
. shell-error
. shell-args
diff --git a/helpers/Makefile b/helpers/Makefile
index cbc9493..4a06f73 100644
--- a/helpers/Makefile
+++ b/helpers/Makefile
@@ -1,7 +1,6 @@
-CFLAGS = -O2 -g -Wextra
-
all:
$(CC) $(CFLAGS) -o remap-log remap-log.c
+ $(CC) $(CFLAGS) -o aiaiai-locker aiaiai-locker.c
clean:
- $(RM) remap-log
+ $(RM) remap-log aiaiai-locker
diff --git a/helpers/aiaiai-checker b/helpers/aiaiai-checker
new file mode 100755
index 0000000..0487670
--- /dev/null
+++ b/helpers/aiaiai-checker
@@ -0,0 +1,220 @@
+#!/bin/sh -efu
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Artem Bityutskiy
+# License: GPLv2
+
+srcdir="$(readlink -ev -- ${0%/*})"
+PATH="$srcdir:$srcdir/libshell:$PATH"
+
+. shell-error
+. shell-args
+. shell-signal
+. shell-quote
+. aiaiai-sh-functions
+
+PROG="${0##*/}"
+
+# This is a small trick to make sure the script is portable - check if 'dash'
+# is present, and if yes - use it.
+if can_switch_to_dash; then
+ exec dash -euf -- "$srcdir/$PROG" "$@"
+ exit $?
+fi
+
+show_usage()
+{
+ cat <<-EOF
+Usage: $PROG [options] -- ...
+
+This is an internal aiaiai helper program which runs various source code
+analysis tools when building the kernel. Options for $PROG has to go first,
+then there has to be a "--" delimiter, and then any number of options which are
+not interpreted by this script but passed further to the code analysis tool
+(sparse or smatch).
+
+Options:
+ --sparse check with sparse;
+ --smatch check with smatch;
+ --cppcheck check with cppcheck;
+ --coccinelle check with coccinelle (spatch);
+ --check-only=FILE check only files listed in FILE;
+ -h, --help show this text and exit.
+EOF
+}
+
+fail_usage()
+{
+ [ -z "$1" ] || printf "%s\n" "$1"
+ show_usage
+ exit 1
+}
+
+# Fetch the file name to check from the input arguments.
+get_file_to_check()
+{
+ # The file name is the last argument
+ for file; do true; done
+ printf "%s" "$file"
+}
+
+# Check the file against all the coccinelle scripts we have
+run_coccinelle()
+{
+ local spatch spatches flags pid
+ local pids=
+
+ spatches="$(find $srcdir -name '*.cocci')"
+ for spatch in $spatches; do
+ # Coccinelle is not stable enough so far and dies because of
+ # internal issues sometimes or just never stops. So we specify
+ # a timeout as well as ignore its error code.
+ flags="-D report -no_show_diff -very_quiet -no_includes -include_headers -timeout 60"
+
+ # Run coccinelle for each semantic patch in parallel. This may load the
+ # system too heavily, though. We use aiaiai-locker to make sure
+ # we have non-scrambled output.
+ aiaiai-locker -s -l "$tmpdir/lockfile" -c \
+ "spatch $flags -sp_file $spatch $file_to_check" ||: &
+ pids="$pids $!"
+ done
+
+ for pid in $pids; do
+ wait "$pid"
+ done
+}
+
+tmpdir=
+cleanup_handler()
+{
+ rm -rf $verbose -- "$tmpdir"
+
+ # Just in case we were interrupted, kill all our children
+ local child
+ for child in $(ps -o pid --no-headers --ppid "$$"); do
+ kill "$child" > /dev/null 2>&1 ||:
+ done
+}
+set_cleanup_handler cleanup_handler
+
+TEMP=`getopt -n $PROG -o h --long sparse,smatch,cppcheck,coccinelle,check-only:,help -- "$@"` ||
+ fail_usage ""
+eval set -- "$TEMP"
+
+[ "$#" -gt "0" ] || exit 0
+
+run_sparse=
+run_smatch=
+run_cppcheck=
+run_coccinelle=
+check_only=
+
+while true; do
+ case "$1" in
+ --sparse)
+ run_sparse=1
+ program_required "sparse" "See section 'sparse' in doc/README"
+ ;;
+ --smatch)
+ run_smatch=1
+ program_required "smatch" "See section 'smatch' in doc/README"
+ ;;
+ --cppcheck)
+ run_cppcheck=1
+ program_required "cppcheck" "Usually Linux distribution provide a cppcheck package"
+ ;;
+ --coccinelle)
+ run_coccinelle=1
+ program_required "spatch" "Usually Linux distribution provide a 'spatch' or 'coccinelle' package"
+ ;;
+ --check-only)
+ check_only="$(opt_check_read "$1" "$2")"
+ shift
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ --) shift; break
+ ;;
+ *) fail_usage "Unrecognized option: $1"
+ ;;
+ esac
+ shift
+done
+
+program_required "aiaiai-locker" \
+ "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
+
+tmpdir="$(mktemp -dt "$PROG.XXXX")"
+file_to_check="$(get_file_to_check "$@")"
+
+# Exit immediately if there is nothing to check
+if [ -z "$run_sparse" ] && [ -z "$run_smatch" ] && [ -z "$run_cppcheck" ] &&
+ [ -z "$run_coccinelle" ]; then
+ exit 0
+fi
+
+if [ -n "$check_only" ]; then
+ match=
+ files="$(cat "$check_only")"
+ for file in $files; do
+ match="$(printf "%s" "$@" | sed -n "/$(quote_sed_regexp "$file")/p")"
+ [ -z "$match" ] || break
+ done
+
+ [ -n "$match" ] || exit 0
+fi
+
+# Run all the tools in background
+
+if [ -n "$run_sparse" ]; then
+ # Sparse uses stderr for check results, not sure about stdout
+ sparse -Wsparse-all "$@" > "$tmpdir/sparse" 2>&1 &
+ pid_sparse="$!"
+fi
+if [ -n "$run_smatch" ]; then
+ # Smatch uses stderr for reporting about internal issues and stdout for
+ # check results
+ smatch --project=kernel "$@" > "$tmpdir/smatch" 2>/dev/null &
+ pid_smatch="$!"
+fi
+if [ -n "$run_cppcheck" ]; then
+ # Cppcheck uses stderr for reporting about internal issues and stdout for
+ # check results
+ cppcheck -f -q --template=gcc "$file_to_check" > "$tmpdir/cppcheck" 2>/dev/null &
+ pid_cppcheck="$!"
+fi
+if [ -n "$run_coccinelle" ]; then
+ # Coccinelle uses stderr for reporting about internal issues and stdout for
+ # check results
+ run_coccinelle > "$tmpdir/coccinelle" 2>/dev/null &
+ pid_coccinelle="$!"
+fi
+
+# Wait for the tools
+
+if [ -n "$run_sparse" ]; then
+ wait "$pid_sparse" ||:
+ if [ -s "$tmpdir/sparse" ]; then
+ cat "$tmpdir/sparse" | sed -e "s/$/ [sparse]/" 1>&2
+ fi
+fi
+if [ -n "$run_smatch" ]; then
+ wait "$pid_smatch" ||:
+ if [ -s "$tmpdir/smatch" ]; then
+ cat "$tmpdir/smatch" | sed -e "s/$/ [smatch]/" 1>&2
+ fi
+fi
+if [ -n "$run_cppcheck" ]; then
+ wait "$pid_cppcheck" ||:
+ if [ -s "$tmpdir/cppcheck" ]; then
+ cat "$tmpdir/cppcheck" | sed -e "s/$/ [cppcheck]/" 1>&2
+ fi
+fi
+if [ -n "$run_coccinelle" ]; then
+ wait "$pid_coccinelle" ||:
+ if [ -s "$tmpdir/coccinelle" ]; then
+ cat "$tmpdir/coccinelle" | sed -e "s/$/ [coccinelle]/" 1>&2
+ fi
+fi
diff --git a/helpers/aiaiai-decode-rfc-2047 b/helpers/aiaiai-decode-rfc-2047
new file mode 100755
index 0000000..24ece5b
--- /dev/null
+++ b/helpers/aiaiai-decode-rfc-2047
@@ -0,0 +1,18 @@
+#!/usr/bin/perl
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Artem Bityutskiy
+# License: GPLv2
+#
+# This is a small helper script which decodes headers if they are encoded using
+# the MIME format defined in RFC2047.
+
+use Encode qw/encode decode/;
+
+while (my $line = <>) {
+ chomp($line);
+ $line = decode('UTF-8', $line);
+ $line = decode('MIME-Header', $line);
+ $line = encode('UTF-8', $line);
+ print("$line\n");
+}
diff --git a/helpers/aiaiai-diff-log b/helpers/aiaiai-diff-log
new file mode 100755
index 0000000..e053edc
--- /dev/null
+++ b/helpers/aiaiai-diff-log
@@ -0,0 +1,193 @@
+#!/bin/sh -euf
+
+# Copyright 2011-2012 Intel Corporation
+# Authors: Artem Bityutskiy
+# Kirill Shutemov
+# License: GPLv2
+
+srcdir="$(readlink -ev -- ${0%/*})"
+PATH="$srcdir:$srcdir/libshell:$PATH"
+
+. shell-error
+. shell-args
+. shell-signal
+. shell-quote
+. aiaiai-sh-functions
+
+PROG="${0##*/}"
+message_time="yes"
+
+# This is a small trick to make sure the script is portable - check if 'dash'
+# is present, and if yes - use it.
+if can_switch_to_dash; then
+ exec dash -euf -- "$srcdir/$PROG" "$@"
+ exit $?
+fi
+
+show_usage()
+{
+ cat <<-EOF
+Usage: $PROG [options] <patch> <log_before> <log_after>
+
+Compare 2 kernel build logs - before and after <patch> was applied.
+
+<patch> - the patch which was applied to the "before" kernel
+<log_before> - build log before <patch> was applied
+<log_after> - build log after <patch> was applied
+
+Options:
+ -w, --workdir=WDIR path to the work directory (default: a temporary
+ directory is created using mktemp);
+ -p, --preserve preserve all the temporary files - do not clean up;
+ -v, --verbose be verbose;
+ -h, --help show this text and exit.
+EOF
+}
+
+fail_usage()
+{
+ [ -z "$1" ] || printf "%s\n" "$1"
+ show_usage
+ exit 1
+}
+
+# After joining the "before" and "after" build logs, then sorting and splitting
+# them back, there may be identical lines but just shifted a bit, and this is
+# visible in the diff, which we do not want. E.g.:
+#
+# @@ @@
+#-cc1: warning: drivers/media/dvb/dvb-core: No such file or directory [enabled by default]
+#-cc1: warning: drivers/media/dvb/frontends: No such file or directory [enabled by default]
+#@@ @@
+#+cc1: warning: drivers/media/dvb/dvb-core: No such file or directory [enabled by default]
+#+cc1: warning: drivers/media/dvb/frontends: No such file or directory [enabled by default]
+#@@ @@
+#
+# This function tries to fix this up. Yes, this is ugly and we need to write a
+# good tool which compares parallel builds in C and invent a smarter algorithm
+# of comparison.
+remove_same_lines()
+{
+ local diff="$1"
+ local line
+
+ while IFS= read -r line; do
+ local l
+
+ l="$(printf "%s" "$line" | sed -n -e 's/^[+-]\{1\}\([^+-]\{1\}.*\)$/\1/p')"
+ if [ -z "$l" ]; then
+ printf "%s\n" "$line"
+ continue
+ fi
+
+ local p c1 c2
+ quote_sed_regexp_variable p "$l"
+ c1="$(grep -c -- "^-$p$" < "$diff" ||:)"
+ c2="$(grep -c -- "^+$p$" < "$diff" ||:)"
+
+ [ "$c1" -eq "$c2" ] || printf "%s\n" "$line"
+ done < "$diff" | sed -n -e 'p; /^@@ @@$/ {:a n; /@@ @@/ba; p}' # Remove multiple "@@ @@"
+}
+
+# The remap-log utility handles renames for lines which start with the file
+# name, but this does not cover all the case. This function checks if the patch
+# renames any file and if yes, substitutes old names with new names.
+rename_log()
+{
+ local file="$1"
+ local renames pattern from to
+
+ # Look for pairs of lines like this in the diff file:
+ #
+ # rename from drivers/a/xxx.c
+ # rename to drivers/a/yyy.c
+ #
+ # and transform them to lines like this:
+ #
+ # drivers/a/xxx.c drivers/a/yyy.c
+ #
+ # and then pipe to the while loop
+ sed -n -e '/^rename from / {
+ s/^rename from \(.*\)/\1/
+ N
+ /\nrename to / s/\nrename to//p
+ }' "$patch" | \
+ while read -r from to; do
+ local q1 q2
+
+ # Quote special symbols and substitute old names with new names
+ quote_sed_regexp_variable q1 "$from"
+ quote_sed_regexp_variable q2 "$to"
+ sed -i -e "s/$q1/$q2/g" "$file"
+ done
+
+}
+
+tmpdir=
+preserve=
+cleanup_handler()
+{
+ if [ -n "$preserve" ]; then
+ message "Preserved tmpdir: $tmpdir"
+ else
+ rm $verbose -rf -- "$tmpdir" >&2
+ fi
+}
+set_cleanup_handler cleanup_handler
+
+TEMP=`getopt -n $PROG -o w:,p,v,h --long workdir:,preserve,verbose,help -- "$@"` ||
+ fail_usage ""
+eval set -- "$TEMP"
+
+verbose=
+
+while true; do
+ case "$1" in
+ -w|--workdir)
+ mkdir $verbose -p -- "$2" >&2
+ tmpdir="$(mktemp --tmpdir="$(readlink -fv -- "$2")" -dt "$PROG.XXXX")"
+ shift
+ ;;
+ -p|--preserve)
+ preserve="--preserve"
+ ;;
+ -v|--verbose) verbose=-v
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ --) shift; break
+ ;;
+ *) fail_usage "Unrecognized option: $1"
+ ;;
+ esac
+ shift
+done
+
+[ "$#" = 3 ] || die "Insufficient or too many arguments"
+
+program_required "remap-log" \
+ "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
+
+patch="$(readlink -ev -- "$1")"; shift
+input_log1="$(readlink -ev -- "$1")"; shift
+input_log2="$(readlink -ev -- "$1")"; shift
+
+[ -n "$tmpdir" ] || tmpdir="$(mktemp -dt "$PROG.XXXX")"
+
+log1="$tmpdir/${input_log1##*/}.before"
+log2="$tmpdir/${input_log2##*/}.after"
+
+verbose "Comparing $input_log1 and $input_log2"
+
+remap-log < "$patch" > "$tmpdir/map"
+remap-log -o "" -p "line " "$tmpdir/map" < "$input_log1" > "$log1.remapped"
+
+rename_log "$log1.remapped"
+
+aiaiai-diff-log-helper "$log1.remapped" "$input_log2" "$tmpdir/diff"
+
+cat "$tmpdir/diff"
+
+verbose "Successfully compared $input_log1 and $input_log2"
diff --git a/helpers/aiaiai-diff-log-helper b/helpers/aiaiai-diff-log-helper
new file mode 100755
index 0000000..2372391
--- /dev/null
+++ b/helpers/aiaiai-diff-log-helper
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Sort an compare 2 build logs.
+
+Author: Ed Bartosh <eduard.bartosh at intel.com>
+Licence: GPLv2
+"""
+
+# Sorts 2 build logs and compares them. Blocks can be as follows.
+#
+# 1a. All consequitive lines starting with the same file prefix belong to one
+# block, e.g.:
+#
+# drivers/s.c:472:22: warning: incorrect type in assignment (different address spaces) [sparse]
+# drivers/s.c:472:22: expected struct table_header *mapped_table [sparse]
+# drivers/s.c:472:22: got void [noderef] <asn:2>* [sparse]
+#
+# (the prefix is "drivers/s.c:472")
+#
+# 1b. All consequitive lines starting with prefix "make:", "make[1]:", etc.
+# These usually belong to build failures.
+#
+# 2. GCC 'In file included from' blocks look like this:
+#
+# In file included from include/linux/kernel.h:17:0,
+# from include/linux/sched.h:55,
+# from arch/arm/kernel/asm-offsets.c:13
+# include/linux/bitops.h: In function 'hweight_long':
+# include/linux/bitops.h:55:26: warning: signed and unsigned type in expression
+#
+# or
+#
+# In file included from arch/x86/include/asm/uaccess.h:570:0,
+# from include/linux/uaccess.h:5,
+# from include/linux/highmem.h:7,
+# from include/linux/pagemap.h:10,
+# from fs/binfmt_misc.c:26:
+# In function ‘copy_from_user’,
+# inlined from ‘parse_command.part.0’ at fs/binfmt_misc.c:422:20:
+# arch/x86/include/asm/uaccess_32.h:211:26: warning: call to ‘copy_from_user’
+#
+# 3. GCC 'In function' blocks look like this:
+#
+# drivers/i.c: In function ‘gluebi_erase’:
+# drivers/i.c:177:2: warning: comparison of unsigned expression < 0 is always false [-Wtype-limits]
+#
+# 4. Any other line comprises an independent block
+
+import sys
+import os
+import re
+import itertools
+
+def gen_blocks(stream):
+ """Parses input stream. Yields found blocks."""
+ btype, prefix, block = "", "", []
+ for line in stream:
+ # append line to the current block if prefix matches
+ if prefix and line.startswith(prefix):
+ block.append(line)
+
+ # Define prefix for cases 2 and 3 for further processing
+ elif not prefix and btype in ("ifi", "infunc"):
+ block.append(line)
+ if re.match("^[^\s]+/[^\s]+:\d+:", line):
+ prefix = ":".join(line.split(':')[:2])
+ # 'In file inculded' block (case 2.)
+ elif re.match("^In file included from .+", line):
+ yield block
+ btype = "ifi"
+ prefix = ""
+ block = [line]
+ # 'In function' block (case 3.)
+ elif re.match("^[^\s]+/[^\s]+: In function .+", line):
+ yield block
+ btype = "infunc"
+ prefix = ""
+ block = [line]
+ # file prefixed block (case 1a.)
+ elif re.match("^[^\s]+/[^\s]+:\d+:", line):
+ yield block
+ prefix = ":".join(line.split(':')[:2])
+ btype = "prefix"
+ block = [line]
+ # file prefixed block (case 1b.)
+ elif re.match("^make(\[\d+\]){0,1}:", line):
+ yield block
+ prefix = line.split(':')[0]
+ btype = "prefix"
+ block = [line]
+ # the rest (case 4.)
+ else:
+ yield block
+ btype = "plain"
+ prefix = ""
+ block = [line]
+ yield block
+
+def main(argv):
+ """Script entry point."""
+
+ infile1, infile2 = open(argv[1]), open(argv[2])
+ outfile = sys.stdout
+ if len(argv) > 3:
+ outfile = open(argv[3], "w")
+
+ with open(argv[1]) as infile1, \
+ open(argv[2]) as infile2:
+
+ result = {}
+ for blk1, blk2 in itertools.izip_longest(gen_blocks(infile1),
+ gen_blocks(infile2), fillvalue=[]):
+ if blk1 == blk2:
+ continue
+ for block, sign in [(tuple(blk1), "-"), (tuple(blk2), "+")]:
+ if block:
+ if block in result:
+ del result[block]
+ else:
+ result[block] = sign
+
+ result = sorted(result.items())
+ if result:
+ print "--- before_patching.log"
+ print "+++ after_patching.log"
+ prefix = ""
+ for block, sign in result:
+ if not prefix or not block[0].startswith(prefix):
+ print "@@ @@"
+ for line in block:
+ print "%s%s" % (sign, line),
+ if re.match("^[^\s]+/[^\s]+:", block[0]):
+ prefix = block[0].split(':')[0]
+
+ outfile.close()
+
+if __name__ == "__main__":
+ if len(sys.argv) == 1 or sys.argv[1] == "--help":
+ print "Usage: %s <input log 1> <input log 2> [<diff file>]" % \
+ os.path.basename(sys.argv[0])
+ sys.exit(0)
+ sys.exit(main(sys.argv))
+
+# vim: ts=4 et sw=4 sts=4 ai sta:
diff --git a/helpers/aiaiai-locker.c b/helpers/aiaiai-locker.c
new file mode 100644
index 0000000..e76d284
--- /dev/null
+++ b/helpers/aiaiai-locker.c
@@ -0,0 +1,229 @@
+/*
+ * Copyright 2012 Intel Corporation
+ * Author: Artem Bityutskiy <dedekind1 at gmail.com>
+ * License: GPLv2
+ *
+ * Credits to Eric Melski for the idea and reference implementation:
+ * http://www.cmcrossroads.com/ask-mr-make/12909-descrambling-parallel-build-logs
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <getopt.h>
+#include <stdbool.h>
+#include <sys/wait.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+
+#define PROGRAM_NAME "aiaiai-locker"
+#define TMPFILE_NAME "/tmp/" PROGRAM_NAME ".XXXXXX"
+#define BUF_SIZE 8192
+
+static char *lockfile;
+static char *commands;
+static bool split = false;
+
+static void usage(FILE *dest, int status)
+{
+ fprintf(dest,
+"Usage: " PROGRAM_NAME " -l <lockfile> [options]\n"
+"\n"
+"Capture entire stdout and stderr output of shell <commands> and print it\n"
+"when the commands finish. The idea is that commands should also use\n"
+PROGRAM_NAME " to serizlize (descrabmle) the output. This can be used\n"
+"with parallell make (e.g., -j10) to descramble the build log and make it\n"
+"readable. For example:\n"
+"\n"
+PROGRAM_NAME " -l lock -c \"make SHELL='" PROGRAM_NAME " -l lock' -j20\"\n"
+"\n"
+"The lockfile is a mandatory parameter which is used to make sure than only\n"
+"one process prints to stdout/stderr at a time.\n"
+"\n"
+"Options:\n"
+" -l, --lockfile=FILE path to the file to use for serializing the output;\n"
+" -c COMMANDS the commands to execute in a shell;\n"
+" -s, --split split stderr and stdout;\n"
+" -h, --help print help message and exit.\n"
+);
+ exit(status);
+}
+
+static const struct option long_options[] = {
+ { .name = "lock-file", .has_arg = 1, .flag = NULL, .val = 'l' },
+ { .name = "split", .has_arg = 1, .flag = NULL, .val = 's' },
+ { .name = "", .has_arg = 1, .flag = NULL, .val = 'c' },
+ { .name = "help", .has_arg = 0, .flag = NULL, .val = 'h' },
+ { NULL, 0, NULL, 0},
+};
+
+static void parse_options(int argc, char *argv[])
+{
+ if (argc < 2) {
+ fprintf(stderr, "Error: please, specify the lock file\n");
+ usage(stderr, EXIT_FAILURE);
+ }
+
+ while (1) {
+ int key;
+
+ key = getopt_long(argc, argv, "l:c:sh", long_options, NULL);
+ if (key == -1)
+ break;
+
+ switch (key) {
+ case 'l':
+ lockfile = optarg;
+ break;
+ case 'c':
+ commands = optarg;
+ break;
+ case 's':
+ split = true;
+ break;
+ case 'h':
+ usage(stdout, EXIT_SUCCESS);
+ case ':':
+ fprintf(stderr, "Error: parameter is missing\n");
+ default:
+ fprintf(stderr, "Use -h for help\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ if (!lockfile) {
+ fprintf(stderr, "Error: please, specify the lockfile\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (!commands)
+ exit(EXIT_SUCCESS);
+}
+
+int main(int argc, char *argv[])
+{
+ int fstdout, fstderr = -1, lockfd;
+ char fstdout_name[sizeof(TMPFILE_NAME)];
+ char fstderr_name[sizeof(TMPFILE_NAME)];
+ char buf[BUF_SIZE];
+ pid_t child;
+ int status = 0;
+ ssize_t rd;
+
+ parse_options(argc, argv);
+
+ /* Create temporary files for stdout and stderr of the child program */
+ memcpy(fstdout_name, TMPFILE_NAME, sizeof(TMPFILE_NAME));
+ memcpy(fstderr_name, TMPFILE_NAME, sizeof(TMPFILE_NAME));
+ fstdout = mkstemp(fstdout_name);
+ if (fstdout == -1) {
+ perror("mkstemp");
+ exit(EXIT_FAILURE);
+ }
+ if (split) {
+ fstderr = mkstemp(fstderr_name);
+ if (fstderr == -1) {
+ perror("mkstemp");
+ goto out_fstdout;
+ }
+ }
+
+ child = fork();
+ if (child == -1) {
+ perror("fork");
+ goto out_fstderr;
+ }
+
+ if (child == 0) {
+ char * const sh_argv[4] = { "/bin/sh", "-c", commands, 0 };
+
+ close(1);
+ close(2);
+
+ if (dup2(fstdout, 1) == -1)
+ exit(EXIT_FAILURE);
+ if (dup2(split ? fstderr : fstdout, 2) == -1)
+ exit(EXIT_FAILURE);
+ if (execvp("/bin/sh", sh_argv) == -1) {
+ perror("execvp");
+ exit(EXIT_FAILURE);
+ }
+
+ exit(EXIT_FAILURE);
+ }
+
+ if (waitpid(child, &status, 0) == -1) {
+ perror("waitpid");
+ goto out_fstderr;
+ }
+
+ if (lseek(fstdout, 0, SEEK_SET) == -1) {
+ perror("lseek");
+ goto out_fstderr;
+ }
+ if (split) {
+ if (lseek(fstderr, 0, SEEK_SET) == -1) {
+ perror("lseek");
+ goto out_fstderr;
+ }
+ }
+
+ lockfd = open(lockfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+ if (lockfd == -1) {
+ perror("open");
+ goto out_fstderr;
+ }
+
+ if (flock(lockfd, LOCK_EX) == -1) {
+ perror("flock");
+ goto out_fstderr;
+ }
+
+ while ((rd = read(fstdout, buf, BUF_SIZE)) > 0) {
+ if (write(1, buf, rd) != rd) {
+ perror("write");
+ goto out_unlock;
+ }
+ }
+ if (rd == -1) {
+ perror("read");
+ goto out_unlock;
+ }
+
+ if (split) {
+ while ((rd = read(fstderr, buf, BUF_SIZE)) > 0) {
+ if (write(2, buf, rd) != rd) {
+ perror("write");
+ goto out_unlock;
+ }
+ }
+ if (rd == -1) {
+ perror("read");
+ goto out_unlock;
+ }
+ }
+
+ flock(lockfd, LOCK_UN);
+ close(lockfd);
+ if (split) {
+ close(fstderr);
+ unlink(fstderr_name);
+ }
+ close(fstdout);
+ unlink(fstdout_name);
+ exit(WEXITSTATUS(status));
+
+out_unlock:
+ flock(lockfd, LOCK_UN);
+ close(lockfd);
+out_fstderr:
+ if (split) {
+ close(fstderr);
+ unlink(fstderr_name);
+ }
+out_fstdout:
+ close(fstdout);
+ unlink(fstdout_name);
+ return EXIT_FAILURE;
+}
diff --git a/helpers/aiaiai-make-kernel b/helpers/aiaiai-make-kernel
new file mode 100755
index 0000000..7153629
--- /dev/null
+++ b/helpers/aiaiai-make-kernel
@@ -0,0 +1,285 @@
+#!/bin/sh -efu
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Kirill Shutemov, Artem Bityutskiy
+# License: GPLv2
+
+srcdir="$(readlink -ev -- ${0%/*})"
+export PATH="$srcdir:$srcdir/libshell:$PATH"
+
+. shell-error
+. shell-args
+. shell-signal
+. shell-quote
+. aiaiai-sh-functions
+
+PROG="${0##*/}"
+message_time="yes"
+
+# This is a small trick to make sure the script is portable - check if 'dash'
+# is present, and if yes - use it.
+if can_switch_to_dash; then
+ exec dash -euf -- "$srcdir/$PROG" "$@"
+ exit $?
+fi
+
+show_usage()
+{
+ cat <<-EOF
+Usage: $PROG [options] <kernel-tree> <targets...>
+
+<kernel-tree> - directory with kernel sources
+<targets...> - list of targets to make.
+
+Options:
+ -o, --objdir=OBJDIR locate all output files in OBJDIR, including .config
+ (O=OBJDIR Kbuild feature); if undefined - use the
+ source directory (<kernel-tree>);
+ -D, --defconfig=CONF name of the defconfig to use for each target; if the
+ name contains a "/" character, it is interpreted as a
+ stand-alone defconfig file, otherwise it is assumed
+ that CONF is part of the <kernel-tree>;
+ -j, --jobs=N allow to run N jobs simultaneously (default is 1);
+ -O, --stdout=FILE redirect stdout of the build to FILE;
+ -E, --stderr=FILE redirect stderr of the build to FILE (note, if neither
+ -O nor -E are specified, everything is redirected to
+ stdout);
+ -k, --keep-going continue as much as possible after an error (see
+ -k GNU make option);
+ -a, --arch=A,[C] test for specified architecture (ARCH=) and the
+ cross-compiler prefix (CROSS_COMPILE=); Examples:
+ -a i386, -a arm,arm-eabi-, etc;
+ --sparse check with sparse while building;
+ --smatch check with smatch while building;
+ --cppcheck check with cppcheck while building;
+ --coccinelle check with coccinelle (spatch) while building;
+ --check-only=FILE check only files listed in FILE;
+ -M, --kmake-opts additional options to append to the final kernel
+ compilation 'make' command
+ (e.g., W=2 KALLSYMS_EXTRA_PASS=1)
+ -v, --verbose be verbose;
+ -h, --help show this text and exit.
+EOF
+}
+
+fail_usage()
+{
+ [ -z "$1" ] || printf "%s\n" "$1"
+ show_usage
+ exit 1
+}
+
+# Strip useless stuff from the log and make it possible to compare it to
+# another build log.
+strip_log()
+{
+ local logfile="$1"
+
+ sed -i -e "# Strip useless stuff
+ /^CRC [[:xdigit:]]\+$/d
+ /^System is [[:digit:]]\+ kB$/d
+ /^Setup is [[:digit:]]\+ bytes (padded to [[:digit:]]\+ bytes).$/d
+ /^Root device is ([[:digit:]]\+, [[:digit:]]\+)$/d
+
+ # Strip not very useful messages from modpost
+ /^To see full details build your kernel with:$/d
+ /^'make CONFIG_DEBUG_SECTION_MISMATCH=y'$/d
+
+ # Make all source paths relative to the kernel tree root
+ s/$(quote_sed_regexp "$kernel_tree/")//g" "$logfile"
+
+ # Strip objdir paths as well
+ [ -z "$objdir" ] || sed -i -e "s/$(quote_sed_regexp "$objdir/")//g" "$logfile"
+}
+
+make_target()
+{
+ local target="$1"
+
+ if [ -n "$defconfig" ]; then
+ if printf "%s" "$defconfig" | grep -q '/'; then
+ # The defconfig is a path to a stand-alone defconfig file
+ # FIXME: ugly hack!
+ make -C $kernel_tree ${arch:+ARCH="$arch"} \
+ ${cross:+CROSS_COMPILE="$cross"} \
+ ${objdir:+O="$objdir"} \
+ KCONFIG_ALLCONFIG="$defconfig" -- alldefconfig
+ else
+ # The defconfig is part of the git tree
+ make -C $kernel_tree ${arch:+ARCH="$arch"} \
+ ${cross:+CROSS_COMPILE="$cross"} \
+ ${objdir:+O="$objdir"} -- "$defconfig"
+ fi
+ fi
+
+ aiaiai-locker $split -l "$lockfile" -c \
+ "make $keep_going -j $jobs -C $kernel_tree ${arch:+ARCH="$arch"} \
+ ${cross:+CROSS_COMPILE="$cross"} ${objdir:+O="$objdir"} \
+ CHECK=\"aiaiai-checker $sparse $smatch $cppcheck $coccinelle $check_only --\" \
+ KCFLAGS='-Wno-missing-field-initializers -Wno-sign-compare' \
+ C=$check ${check:+CF="-D__CHECK_ENDIAN__"} W=1 SHELL=\"aiaiai-locker $split -l $lockfile\" \
+ $kmake_opts $target"
+}
+
+lockfile=
+build_stdout=
+build_stdout_given=
+build_stderr=
+build_stderr_given=
+build_output=
+cleanup_handler()
+{
+ rm $verbose -f -- "$lockfile" >&2
+ [ -n "$build_stdout_given" ] || rm $verbose -f -- "$build_stdout" >&2
+ [ -n "$build_stderr_given" ] || rm $verbose -f -- "$build_stderr" >&2
+ rm $verbose -f -- "$build_output" >&2
+}
+set_cleanup_handler cleanup_handler
+
+TEMP=`getopt -n $PROG -o o:,D:,j:,O:,E:,k:,a:,M:,v,h --long objdir:,defconfig:,jobs:,stdout:,stderr:,keep-going,arch:,sparse,smatch,cppcheck,coccinelle,check-only:,kmake-opts:,verbose,help -- "$@"` ||
+ fail_usage ""
+eval set -- "$TEMP"
+
+objdir=
+defconfig=
+jobs=1
+keep_going=
+arch=
+cross=
+sparse=
+smatch=
+cppcheck=
+coccinelle=
+check_only=
+check=0
+kmake_opts=
+verbose=
+
+while true; do
+ case "$1" in
+ -o|--objdir)
+ mkdir $verbose -p -- "$2" >&2
+ objdir="$(readlink -fv -- "$2")"
+ shift
+ ;;
+ -D|--defconfig)
+ defconfig="$2"
+ shift
+ ;;
+ -j|--jobs)
+ jobs=$(opt_check_number "$1" "$2")
+ shift
+ ;;
+ -O|--stdout)
+ touch "$2"
+ build_stdout="$(readlink -fv -- "$2")"
+ build_stdout_given="y"
+ shift
+ ;;
+ -E|--stderr)
+ touch "$2"
+ build_stderr="$(readlink -fv -- "$2")"
+ build_stderr_given="y"
+ shift
+ ;;
+ -k|--keep-going)
+ keep_going="--keep-going"
+ ;;
+ -a|--arch)
+ arch="$(leave_first "$2")"
+ cross="$(leave_second "$2")"
+ shift
+ ;;
+ --sparse)
+ sparse="--sparse"
+ check=1
+ ;;
+ --smatch)
+ smatch="--smatch"
+ check=1
+ ;;
+ --cppcheck)
+ cppcheck="--cppcheck"
+ check=1
+ ;;
+ --coccinelle)
+ coccinelle="--coccinelle"
+ check=1
+ ;;
+ --check-only)
+ check_only="--check-only $(opt_check_read "$1" "$2")"
+ shift
+ ;;
+ -M|--kmake-opts)
+ kmake_opts="$2"
+ shift
+ ;;
+ -v|--verbose) verbose=-v
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ --) shift; break
+ ;;
+ *) fail_usage "Unrecognized option: $1"
+ ;;
+ esac
+ shift
+done
+
+[ "$#" -ge 2 ] || die "Insufficient arguments"
+
+program_required "aiaiai-locker" \
+ "You probably did not compile it, run 'make' in the topmost aiaiai source code directory"
+program_required "${cross}gcc" ""
+program_required "make" ""
+
+kernel_tree="$(opt_check_dir "kernel-tree" "$1")"; shift
+build_info="$kernel_tree${objdir:+ (O="$objdir")}"
+
+if [ -n "$build_stdout_given" ] || [ -n "$build_stderr_given" ]; then
+ split="-s"
+ [ -n "$build_stdout_given" ] || build_stdout="$(mktemp -t "$PROG.stdout.XXXX")"
+ [ -n "$build_stderr_given" ] || build_stderr="$(mktemp -t "$PROG.stderr.XXXX")"
+else
+ split=
+ build_output="$(mktemp -t "$PROG.output.XXXX")"
+fi
+
+lockfile="$(mktemp -t "$PROG.lock.XXXX")"
+
+failed=
+for target in $@; do
+ verbose "Making target \"$target\" for \"$build_info\""
+
+ if [ -z "$split" ]; then
+ make_target "$target" > "$build_output" || failed=1
+ [ -z "$failed" ] || echo "FAILURE" >> "$build_output"
+ else
+ make_target "$target" > "$build_stdout" 2> "$build_stderr" || failed=1
+ [ -z "$failed" ] || echo "FAILURE" >> "$build_stderr"
+ fi
+
+ if [ -n "$failed" ]; then
+ verbose "Failed to make target \"$target\" for \"$build_info\""
+ break
+ fi
+done
+
+[ -z "$build_stdout" ] || strip_log "$build_stdout"
+[ -z "$build_stderr" ] || strip_log "$build_stderr"
+[ -z "$build_output" ] || strip_log "$build_output"
+
+if [ -n "$build_stdout_given" ] || [ -n "$build_stderr_given" ]; then
+ [ -n "$build_stdout_given" ] || cat "$build_stdout"
+ [ -n "$build_stderr_given" ] || cat "$build_stderr"
+else
+ [ -z "$build_output" ] || cat "$build_output"
+fi
+
+if [ -z "$failed" ]; then
+ verbose "Successfully made all targets for \"$build_info\""
+else
+ verbose "Failed to make a target, exiting"
+fi
diff --git a/helpers/aiaiai-match-keywords b/helpers/aiaiai-match-keywords
new file mode 100755
index 0000000..f316792
--- /dev/null
+++ b/helpers/aiaiai-match-keywords
@@ -0,0 +1,93 @@
+#!/bin/sh -euf
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Alexander Shishkin, Artem Bityutskiy
+# License: GPLv2
+
+srcdir="$(readlink -ev -- ${0%/*})"
+PATH="$srcdir:$srcdir/libshell:$PATH"
+
+. shell-error
+. shell-args
+. shell-quote
+. aiaiai-sh-functions
+
+PROG="${0##*/}"
+message_time="yes"
+
+# This is a small trick to make sure the script is portable - check if 'dash'
+# is present, and if yes - use it.
+if can_switch_to_dash; then
+ exec dash -euf -- "$srcdir/$PROG" "$@"
+ exit $?
+fi
+
+show_usage()
+{
+ cat <<-EOF
+Usage: $PROG [options] <file>
+
+Match various keywords - e.g., companies often prohibit some for whatever
+reasons. The keywords are read from stdin (one keyword per line) and matched
+against <file>.
+
+Options:
+ -v, --verbose be verbose;
+ -h, --help show this text and exit.
+EOF
+}
+
+fail_usage()
+{
+ [ -z "$1" ] || printf "%s\n" "$1"
+ show_usage
+ exit 1
+}
+
+first_match=
+look_for_keyword()
+{
+ local kw="$1"
+ local match
+
+ verbose "look for keyword \"$kw\""
+
+ match="$(grep -i "$(quote_sed_regexp "$kw")" "$file" ||:)"
+ [ -n "$match" ] || return 0
+
+ [ -z "$first_match" ] || echo ""
+
+ first_match="no"
+ printf "%s\n\n" "Matched keyword \"$kw\""
+ printf "%s\n" "$match" | head -n8
+}
+
+TEMP=`getopt -n $PROG -o v,h --long verbose,help -- "$@"` || fail_usage ""
+eval set -- "$TEMP"
+
+mbox=
+verbose=
+
+while true; do
+ case "$1" in
+ -v|--verbose)
+ verbose=-v
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ --) shift; break
+ ;;
+ *) fail_usage "Unrecognized option: $1"
+ ;;
+ esac
+ shift
+done
+
+[ "$#" -eq 1 ] || die "Please, specify exactly one argument - the file with keywords"
+file="$(readlink -ev -- "$1")"
+
+while IFS= read -r kw; do
+ look_for_keyword "$kw"
+done
diff --git a/helpers/aiaiai-sh-functions b/helpers/aiaiai-sh-functions
new file mode 100644
index 0000000..abbd0f4
--- /dev/null
+++ b/helpers/aiaiai-sh-functions
@@ -0,0 +1,229 @@
+#!/bin/sh -efu
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Artem Bityutskiy
+# License: GPLv2
+
+. shell-error
+
+if [ -z "${__included_aiaiai_sh_functions-}" ]; then
+__included_aiaiai_sh_functions=1
+
+# Print an error message and exit
+# Usage: die <message>
+die()
+{
+ fatal "Fatal error: $1"
+}
+
+# Print a separator line to stdout
+print_separator()
+{
+ local i=0
+
+ while [ $i -lt 80 ]; do
+ i=$(($i+1));
+ printf "-";
+ done
+ echo
+}
+
+# Check if dash is available and we are not running in dash
+can_switch_to_dash()
+{
+ if command -v "dash" >/dev/null 2>&1; then
+ if [ -n "${BASH_VERSION:-}" ]; then
+ return 0
+ fi
+ fi
+
+ return 1
+}
+
+# Die if a program is not in PATH
+# Usage: program_required <program_name>
+program_required()
+{
+ local prg="$1"; shift
+ local msg="$1"; shift
+
+ if ! command -v "$prg" >/dev/null 2>&1; then
+ message "Program \"$prg\" is required but not found in PATH"
+ if [ -n "$msg" ]; then
+ die "$msg"
+ else
+ exit 1
+ fi
+ fi
+}
+
+# Fetch the first occurrence of an mbox header from a file
+# Usage: fetch_header <header_name> < <mbox_file>
+fetch_header()
+{
+ local hdr="$1"
+
+ program_required "formail" ""
+
+ # Take only the first occurrence of the header
+ formail -z -c -x "$hdr:" | head -n1 | aiaiai-decode-rfc-2047
+}
+
+# Similar to fetch_header, but exits with code 1 if the header hasn't been
+# found, and has a little bit different interface (the result is stored in
+# "<var>").
+#
+# Usage: fetch_header_or_die <var> <header_name> < <mbox_file>
+fetch_header_or_die()
+{
+ local var="$1"; shift
+ local hdr="$1"; shift
+ local res="$(fetch_header "$hdr")"
+
+ [ -n "$res" ] || die "Cannot find the \"$hdr:\" header"
+
+ eval "$var=\"\$res\""
+}
+
+fetch_header_per_patch()
+{
+ local hdr="$1"
+
+ program_required "formail" ""
+
+ # Take only the first occurrence of the header per message
+ formail -s sh -c "formail -z -c -x \"$hdr:\" | head -n1" | aiaiai-decode-rfc-2047
+}
+
+git_dir()
+{
+ local path="$1"
+ local dotgit="$path/.git"
+
+ if [ -d "$dotgit" ]; then
+ printf "%s" "$dotgit"
+ elif [ -d "$path" ]; then
+ printf "%s" "$path"
+ else
+ die "not a git repository: $path"
+ fi
+}
+
+# Apply a patch. In case of error, print user-friendly diagnostic messages to
+# stdin.
+# Usage: apply_patch < <mbox_file>
+apply_patch()
+{
+ local am cmt
+
+ program_required "patch" ""
+
+ cmt="$(git rev-parse "HEAD^{commit}")"
+
+ am="$(git am 2>&1)" || {
+ cat <<EOF
+Failed to apply patch(es) with git am on top of:
+$(git log -1 --oneline "$cmt")
+
+$am
+
+Results of "patch --merge=diff3 -p1 < .git/rebase-apply/patch":
+
+$(patch --merge=diff3 -p1 < .git/rebase-apply/patch 2>&1)
+
+$(print_separator)
+
+$(git diff --no-color)
+EOF
+ return 1
+ }
+}
+
+# A helper function for 'build_failure()'. This function expects the properly
+# formatted build log a stdin and outputs user-readable failure report to
+# stdout.
+__print_build_log()
+{
+ local config="$(leave_first "$1")";
+ local arch="$(leave_second "$1")"; shift
+ local commit_id="$1"; shift
+ local commit_info="$(git log -1 --oneline "$commit_id")"
+ local commit_nickname="${1:-""}"
+
+ cat <<EOF
+Failed to build $commit_nickname: $commit_info
+Configuration: "$config${arch:+", architecture $arch"}".
+
+$(cat)
+EOF
+}
+
+# Format a build failure report.
+# Usage: build_failure <defconfig> <commit_id> <commit_nickname> < <build_log>
+build_failure()
+{
+ # The build log might have been generated with multiple jobs which
+ # means it is probably messy and the error message is probably not at
+ # the very end. To make it more probable that we actually print the
+ # build error message within 24 lines we do the following:
+ # * filter sparse/smatch/cppcheck/coccinelle output
+ # * filter out 'CHECK drivers/blah.c' Kbuild lines
+ # * print 24 lines preceding the 'make[]: *** [] blah' pattern which
+ # make generates after an error
+
+ sed -n '# Filter out useless stuff
+ /\[sparse\]$/d
+ /\[smatch\]$/d
+ /\[cppcheck\]$/d
+ /\[coccinelle\]$/d
+ /^ CHECK /d
+ # Add the line to the hold buffer
+ H
+ # If the line is the error marker, print out the entire hold
+ # buffer and quit
+ /^make\[.*\]: \*\*\* \[.*\]/ { g; p; q; }
+ # Do the same if the last line is reached
+ $ { g; p; q; }' | tail -n24 |
+ __print_build_log "$@"
+}
+
+# Check if the build failed.
+# Usage: build_failed <build_log>
+build_failed()
+{
+ local build_log="$1"
+ local failed
+
+ failed="$(tail -n1 -- "$build_log")"
+ test "$failed" = "FAILURE"
+}
+
+# Filter out the first element from a comma-separated list of elements.
+# Usage: strip_first <list>
+strip_first()
+{
+ printf "%s" "$1" | sed -e 's/^[^,]*,\{0,1\}//g'
+}
+
+# Filter out all but the first element from a comma-separated list of elements.
+# Usage: leave_first <list>
+leave_first()
+{
+ printf "%s" "$1" | sed -e 's/,.*$//g'
+}
+
+# Filter out all but the second element from a comma-separated list of elements.
+# Usage: leave_second <list>
+leave_second()
+{
+ leave_first "$(strip_first "$1")"
+}
+
+# Filter out all but the third element from a comma-separated list of elements.
+# Usage: leave_third <list>
+leave_third()
+{
+ leave_second "$(strip_first "$1")"
+}
+
+fi #__included_aiaiai_sh_functions
diff --git a/helpers/aiaiai-test-bisectability b/helpers/aiaiai-test-bisectability
new file mode 100755
index 0000000..913c0a6
--- /dev/null
+++ b/helpers/aiaiai-test-bisectability
@@ -0,0 +1,273 @@
+#!/bin/sh -efu
+
+# Copyright 2011-2012 Intel Corporation
+# Author: Artem Bityutskiy
+# License: GPLv2
+
+srcdir="$(readlink -ev -- ${0%/*})"
+PATH="$srcdir:$srcdir/libshell:$PATH"
+
+. shell-error
+. shell-args
+. shell-signal
+. aiaiai-sh-functions
+
+PROG="${0##*/}"
+message_time="yes"
+
+# This is a small trick to make sure the script is portable - check if 'dash'
+# is present, and if yes - use it.
+if can_switch_to_dash; then
+ exec dash -euf -- "$srcdir/$PROG" "$@"
+ exit $?
+fi
+
+show_usage()
+{
+ cat <<-EOF
+Usage: $PROG [options] <kernel-tree> <defconfig[,arch[,cross]]>
+
+<kernel-tree> - directory with kernel sources
+<defconfig[,arch[,cross]]> - the configuration to test.
+
+The mbox file containing the patches to test is expected to come from stdin or
+-i option can be used instead.
+
+The configuration is specified as follows:
+
+ defconfig[,arch[,cross]]
+
+where
+
+ * "defconfig" is the name of the kernel defconfig file to test
+ * "arch" is the architecture (translates to Kbuild's ARCH variable value)
+ * "cross" is the cross-compiler prefix (translates to Kbuild's CROSS_COMPILE
+ variable value).
+
+For example:
+
+ omap2plus_defconfig,arm,arm-eabi-
+
+means testing with defconfig "omap2plus_defconfig", ARCH=arm and
+CROSS_COMPILE=arm-eabi-.
+
+By default, $PROG assumes that the defconfig file is part of
+the kernel tree, unless --confdir option is specified, in which case the
+defconfig file will be searched for only in the specified directory.
+
+Options:
+ -j, --jobs=N allow to run N jobs simultaneously (default is 1);
+ -c, --commit-id=ID the commit id to test against (default is the head of
+ the git tree (HEAD));
+ -d, --dont-clone work directly in <kernel-tree>, do not clone a
+ temporary copy of <kernel-tree>;
+ -o, --objdir=OBJDIR locate all output files in OBJDIR, including .config
+ (O=OBJDIR Kbuild feature); if undefined - a temporary
+ directory is created using mktemp);
+ -C, --confdir=CDIR path to the directory containing the defconfig file
+ by default the defconfig file is assumed to be part of
+ the <kernel-tree>; this option makes it possible to use
+ a stand-alone defconfig file instead;
+ -i, --input=MBOX use the MBOX file instead of stdin;
+ -w, --workdir=WDIR path to the work directory where the kernel will
+ be built (default: a temporary directory is created
+ using mktemp);
+ -M, --kmake-opts additional options to append to the final kernel
+ compilation 'make' command
+ (e.g., W=2 KALLSYMS_EXTRA_PASS=1)
+ -p, --preserve preserve all the temporary files - do not clean up;
+ -v, --verbose be verbose;
+ -h, --help show this text and exit.
+EOF
+}
+
+fail_usage()
+{
+ [ -z "$1" ] || printf "%s\n" "$1"
+ show_usage
+ exit 1
+}
+
+# Apply patch <mbox>, compile the kernel and save build logs at <stdout_log>
+# and <stderr_log>.
+# Usage: compile_patch <mbox> <stdout_log> <stderr_log>
+compile_patch()
+{
+ local file="$1"; shift
+ local stdout_log="$1"; shift
+ local stderr_log="$1"; shift
+ local subj
+
+ fetch_header_or_die subj "Subject" < "$file"
+
+ verbose "Compiling \"$subj\""
+
+ # Apply the patch
+ apply_patch < "$file" || exit 0
+
+ aiaiai-make-kernel $verbose -j "$jobs" -o "$objdir" $arch_opt -D "$dconf" \
+ -O "$stdout_log" -E "$stderr_log" ${kmake_opts:+-M "$kmake_opts"} \
+ -- "$cloned_kernel" all
+}
+
+tmpdir=
+preserve=
+cleanup_handler()
+{
+ if [ -n "$preserve" ]; then
+ message "Preserved tmpdir: $tmpdir"
+ else
+ [ -z "$tmpdir" ] || verbose "Removing $tmpdir";
+ rm -rf -- "$tmpdir" >&2
+ fi
+}
+set_cleanup_handler cleanup_handler
+
+TEMP=`getopt -n $PROG -o j:,c:,d,o:,C:,i:,w:,M:,p,v,h --long jobs:,commit-id:,dont-clone,objdir:,confdir:,input:,workdir:,kmake-opts:,preserve,verbose,help -- "$@"` ||
+ fail_usage ""
+eval set -- "$TEMP"
+
+jobs=1
+commit_id=HEAD
+dont_clone=
+objdir=
+confdir=
+mbox=
+kmake_opts=
+verbose=
+quiet="-q"
+
+while true; do
+ case "$1" in
+ -j|--jobs)
+ jobs=$(opt_check_number "$1" "$2")
+ shift
+ ;;
+ -c|--commit-id)
+ commit_id="$2"
+ shift
+ ;;
+ -d|--dont-clone)
+ dont_clone="--dont-clone"
+ ;;
+ -o|--objdir)
+ objdir="$(readlink -fv -- "$2")"
+ shift
+ ;;
+ -C|--confdir)
+ confdir="$(opt_check_dir "$1" "$2")"
+ shift
+ ;;
+ -i|--input)
+ mbox="$(opt_check_read "$1" "$2")"
+ shift
+ ;;
+ -w|--workdir)
+ mkdir $verbose -p -- "$2" >&2
+ tmpdir="$(mktemp --tmpdir="$(readlink -fv -- "$2")" -dt "$PROG.XXXX")"
+ shift
+ ;;
+ -M|--kmake-opts)
+ kmake_opts="$2"
+ shift
+ ;;
+ -p|--preserve)
+ preserve="--preserve"
+ ;;
+ -v|--verbose)
+ verbose=-v
+ quiet=
+ ;;
+ -h|--help)
+ show_usage
+ exit 0
+ ;;
+ --) shift; break
+ ;;
+ *) fail_usage "Unrecognized option: $1"
+ ;;
+ esac
+ shift
+done
+
+[ "$#" -eq 2 ] || die "Please, specify exactly 2 arguments - the kernel tree and the configuration"
+
+kernel_tree="$(readlink -ev -- "$1")"; shift
+defconfig_orig="$1"
+defconfig="$(leave_first "$1")";
+arch="$(leave_second "$1")";
+cross="$(leave_third "$1")";
+arch_opt=
+
+[ -z "$arch" ] || arch_opt="-a $arch${cross:+",$cross"}"
+
+[ -n "$tmpdir" ] || tmpdir="$(mktemp -dt "$PROG.XXXX")"
+tmp_mbox="$tmpdir/tmp_mbox"
+[ -n "$objdir" ] || objdir="$tmpdir/obj.$defconfig"
+
+# Save the mbox to a temporary file if it comes from stdin
+if [ -z "$mbox" ]; then
+ mbox="$tmpdir/mbox"
+ cat > "$mbox"
+fi
+
+patch_cnt="$(formail -s echo 1 < "$mbox" | wc -l)"
+if [ "$patch_cnt" -le 1 ]; then
+ verbose "Only one patch, nothing to test"
+ exit 0;
+fi
+
+verbose "Bisecting $patch_cnt patches for configuration \"$defconfig_orig\""
+
+if [ -z "$dont_clone" ]; then
+ cloned_kernel="$tmpdir/src"
+ # Use a shared clone since it is local and we won't mess up the tree
+ git clone --shared "${verbose:--q}" "$kernel_tree" "$cloned_kernel" >&2
+else
+ cloned_kernel="$kernel_tree"
+fi
+
+cd "$cloned_kernel"
+commit_id="$(git --git-dir="$(git_dir "$kernel_tree")" rev-parse "$commit_id^{commit}")"
+git reset $quiet --hard "$commit_id" >&2
+
+dconf="$defconfig"
+[ -z "$confdir" ] || dconf="$confdir/$dconf"
+
+base_stdout_log="$tmpdir/$defconfig.base.stdout.log"
+base_stderr_log="$tmpdir/$defconfig.base.stderr.log"
+stdout_log="$tmpdir/$defconfig.stdout.log"
+stderr_log="$tmpdir/$defconfig.stderr.log"
+
+verbose "Building the base tree"
+base_commit_failed=
+aiaiai-make-kernel $verbose -j "$jobs" -o "$objdir" $arch_opt -D "$dconf" \
+ -O "$base_stdout_log" -E "$base_stderr_log" \
+ ${kmake_opts:+-M "$kmake_opts"} -- "$cloned_kernel" all
+if build_failed "$base_stderr_log"; then
+ # Failed to build the base commit, report, but don't exit in case the
+ # first patch fixes the issue.
+ base_commit_failed="yes"
+fi
+
+n=0
+while [ "$n" -lt "$patch_cnt" ]; do
+ formail +$n -1 -s < "$mbox" > "$tmp_mbox"
+ compile_patch "$tmp_mbox" "$stdout_log" "$stderr_log"
+ n="$(($n+1))"
+
+ if build_failed "$stderr_log"; then
+ if [ "$n" = 1 ] && [ -n "$base_commit_failed" ]; then
+ # Base commit failed too, report about this
+ build_failure "$defconfig_orig" "$commit_id" \
+ "the base commit" < "$base_stderr_log"
+ printf "\n\n"
+ fi
+ build_failure "$defconfig_orig" "HEAD" "patch #$n"< "$stderr_log"
+ exit 0
+ fi
+done
+
+if [ -n "$base_commit_failed" ]; then
+ printf "%s\n" "Base commit build failed, but the first patch fixed the build failure."
+fi
--
1.8.5.2
More information about the aiaiai
mailing list