diff options
Diffstat (limited to 'scripts')
30 files changed, 1905 insertions, 244 deletions
diff --git a/scripts/Android.bp b/scripts/Android.bp new file mode 100644 index 000000000..1f5503051 --- /dev/null +++ b/scripts/Android.bp @@ -0,0 +1,156 @@ +python_binary_host { + name: "manifest_fixer", + main: "manifest_fixer.py", + srcs: [ + "manifest_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "manifest_fixer_test", + main: "manifest_fixer_test.py", + srcs: [ + "manifest_fixer_test.py", + "manifest_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_library_host { + name: "manifest_utils", + srcs: [ + "manifest.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, +} + +python_binary_host { + name: "manifest_check", + main: "manifest_check.py", + srcs: [ + "manifest_check.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "manifest_check_test", + main: "manifest_check_test.py", + srcs: [ + "manifest_check_test.py", + "manifest_check.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_binary_host { + name: "jsonmodify", + main: "jsonmodify.py", + srcs: [ + "jsonmodify.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + } +} + +python_binary_host { + name: "test_config_fixer", + main: "test_config_fixer.py", + srcs: [ + "test_config_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], +} + +python_test_host { + name: "test_config_fixer_test", + main: "test_config_fixer_test.py", + srcs: [ + "test_config_fixer_test.py", + "test_config_fixer.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, + libs: [ + "manifest_utils", + ], + test_suites: ["general-tests"], +} + +python_binary_host { + name: "lint-project-xml", + main: "lint-project-xml.py", + srcs: ["lint-project-xml.py"], +} diff --git a/scripts/OWNERS b/scripts/OWNERS index 076b3f5c1..9e97a6011 100644 --- a/scripts/OWNERS +++ b/scripts/OWNERS @@ -1 +1,2 @@ per-file system-clang-format,system-clang-format-2 = enh@google.com,smoreland@google.com +per-file build-mainline-modules.sh = ngeoffray@google.com,paulduffin@google.com,mast@google.com diff --git a/scripts/TEST_MAPPING b/scripts/TEST_MAPPING new file mode 100644 index 000000000..1b0a2298f --- /dev/null +++ b/scripts/TEST_MAPPING @@ -0,0 +1,12 @@ +{ + "presubmit" : [ + { + "name": "manifest_check_test", + "host": true + }, + { + "name": "manifest_fixer_test", + "host": true + } + ] +} diff --git a/scripts/archive_repack.sh b/scripts/archive_repack.sh new file mode 100755 index 000000000..f09372dd9 --- /dev/null +++ b/scripts/archive_repack.sh @@ -0,0 +1,87 @@ +#!/bin/bash -e + +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to extract and repack an archive with specified object files. +# Inputs: +# Environment: +# CLANG_BIN: path to the clang bin directory +# Arguments: +# -i ${file}: input file +# -o ${file}: output file +# -d ${file}: deps file + +set -o pipefail + +OPTSTRING=d:i:o: + +usage() { + cat <<EOF +Usage: archive_repack.sh [options] <objects to repack> + +OPTIONS: + -i <file>: input file + -o <file>: output file + -d <file>: deps file +EOF + exit 1 +} + +while getopts $OPTSTRING opt; do + case "$opt" in + d) depsfile="${OPTARG}" ;; + i) infile="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + ?) usage ;; + esac +done +shift "$(($OPTIND -1))" + +if [ -z "${infile}" ]; then + echo "-i argument is required" + usage +fi + +if [ -z "${outfile}" ]; then + echo "-o argument is required" + usage +fi + +# Produce deps file +if [ ! -z "${depsfile}" ]; then + cat <<EOF > "${depsfile}" +${outfile}: ${infile} ${CLANG_BIN}/llvm-ar +EOF +fi + +# Get absolute path for outfile and llvm-ar. +LLVM_AR="${PWD}/${CLANG_BIN}/llvm-ar" +if [[ "$outfile" != /* ]]; then + outfile="${PWD}/${outfile}" +fi + +tempdir="${outfile}.tmp" + +# Clean up any previous temporary files. +rm -f "${outfile}" +rm -rf "${tempdir}" + +# Do repack +# We have to change working directory since ar only allows extracting to CWD. +mkdir "${tempdir}" +cp "${infile}" "${tempdir}/archive" +cd "${tempdir}" +"${LLVM_AR}" x "archive" +"${LLVM_AR}" --format=gnu qc "${outfile}" "$@" diff --git a/scripts/build-aml-prebuilts.sh b/scripts/build-aml-prebuilts.sh new file mode 100755 index 000000000..c60eaa1f6 --- /dev/null +++ b/scripts/build-aml-prebuilts.sh @@ -0,0 +1,104 @@ +#!/bin/bash -e + +# This is a wrapper around "m" that builds the given modules in multi-arch mode +# for all architectures supported by Mainline modules. The make (kati) stage is +# skipped, so the build targets in the arguments can only be Soong modules or +# intermediate output files - make targets and normal installed paths are not +# supported. +# +# This script is typically used with "sdk" or "module_export" modules, which +# Soong will install in $OUT_DIR/soong/mainline-sdks (cf +# PathForMainlineSdksInstall in android/paths.go). + +export OUT_DIR=${OUT_DIR:-out} + +if [ -e ${OUT_DIR}/soong/.soong.in_make ]; then + # If ${OUT_DIR} has been created without --skip-make, Soong will create an + # ${OUT_DIR}/soong/build.ninja that leaves out many targets which are + # expected to be supplied by the .mk files, and that might cause errors in + # "m --skip-make" below. We therefore default to a different out dir + # location in that case. + AML_OUT_DIR=out/aml + echo "Avoiding in-make OUT_DIR '${OUT_DIR}' - building in '${AML_OUT_DIR}' instead" + OUT_DIR=${AML_OUT_DIR} +fi + +if [ ! -e "build/envsetup.sh" ]; then + echo "$0 must be run from the top of the tree" + exit 1 +fi + +source build/envsetup.sh + +my_get_build_var() { + # get_build_var will run Soong in normal in-make mode where it creates + # .soong.in_make. That would clobber our real out directory, so we need to + # run it in a different one. + OUT_DIR=${OUT_DIR}/get_build_var get_build_var "$@" +} + +readonly PLATFORM_SDK_VERSION="$(my_get_build_var PLATFORM_SDK_VERSION)" +readonly PLATFORM_VERSION="$(my_get_build_var PLATFORM_VERSION)" +PLATFORM_VERSION_ALL_CODENAMES="$(my_get_build_var PLATFORM_VERSION_ALL_CODENAMES)" + +# PLATFORM_VERSION_ALL_CODENAMES is a comma separated list like O,P. We need to +# turn this into ["O","P"]. +PLATFORM_VERSION_ALL_CODENAMES="${PLATFORM_VERSION_ALL_CODENAMES/,/'","'}" +PLATFORM_VERSION_ALL_CODENAMES="[\"${PLATFORM_VERSION_ALL_CODENAMES}\"]" + +# Logic from build/make/core/goma.mk +if [ "${USE_GOMA}" = true ]; then + if [ -n "${GOMA_DIR}" ]; then + goma_dir="${GOMA_DIR}" + else + goma_dir="${HOME}/goma" + fi + GOMA_CC="${goma_dir}/gomacc" + export CC_WRAPPER="${CC_WRAPPER}${CC_WRAPPER:+ }${GOMA_CC}" + export CXX_WRAPPER="${CXX_WRAPPER}${CXX_WRAPPER:+ }${GOMA_CC}" + export JAVAC_WRAPPER="${JAVAC_WRAPPER}${JAVAC_WRAPPER:+ }${GOMA_CC}" +else + USE_GOMA=false +fi + +readonly SOONG_OUT=${OUT_DIR}/soong +mkdir -p ${SOONG_OUT} +readonly SOONG_VARS=${SOONG_OUT}/soong.variables + +# Aml_abis: true +# - This flag configures Soong to compile for all architectures required for +# Mainline modules. +# CrossHost: linux_bionic +# CrossHostArch: x86_64 +# - Enable Bionic on host as ART needs prebuilts for it. +cat > ${SOONG_VARS}.new << EOF +{ + "Platform_sdk_version": ${PLATFORM_SDK_VERSION}, + "Platform_sdk_codename": "${PLATFORM_VERSION}", + "Platform_version_active_codenames": ${PLATFORM_VERSION_ALL_CODENAMES}, + + "DeviceName": "generic_arm64", + "HostArch": "x86_64", + "HostSecondaryArch": "x86", + "CrossHost": "linux_bionic", + "CrossHostArch": "x86_64", + "Aml_abis": true, + + "UseGoma": ${USE_GOMA} +} +EOF + +if [ -f ${SOONG_VARS} ] && cmp -s ${SOONG_VARS} ${SOONG_VARS}.new; then + # Don't touch soong.variables if we don't have to, to avoid Soong rebuilding + # the ninja file when it isn't necessary. + rm ${SOONG_VARS}.new +else + mv ${SOONG_VARS}.new ${SOONG_VARS} +fi + +# We use force building LLVM components flag (even though we actually don't +# compile them) because we don't have bionic host prebuilts +# for them. +export FORCE_BUILD_LLVM_COMPONENTS=true + +m --skip-make "$@" diff --git a/scripts/build-mainline-modules.sh b/scripts/build-mainline-modules.sh new file mode 100755 index 000000000..f836ea9fa --- /dev/null +++ b/scripts/build-mainline-modules.sh @@ -0,0 +1,68 @@ +#!/bin/bash -e + +# Non exhaustive list of modules where we want prebuilts. More can be added as +# needed. +MAINLINE_MODULES=( + com.android.art.debug + com.android.art.release + com.android.art.testing + com.android.conscrypt + com.android.runtime + com.android.tzdata + com.android.i18n +) + +# List of SDKs and module exports we know of. +MODULES_SDK_AND_EXPORTS=( + art-module-sdk + art-module-test-exports + conscrypt-module-sdk + conscrypt-module-test-exports + conscrypt-module-host-exports + runtime-module-sdk +) + +# We want to create apex modules for all supported architectures. +PRODUCTS=( + aosp_arm + aosp_arm64 + aosp_x86 + aosp_x86_64 +) + +if [ ! -e "build/make/core/Makefile" ]; then + echo "$0 must be run from the top of the tree" + exit 1 +fi + +echo_and_run() { + echo "$*" + "$@" +} + +OUT_DIR=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT= get_build_var OUT_DIR) +DIST_DIR=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT= get_build_var DIST_DIR) + +for product in "${PRODUCTS[@]}"; do + echo_and_run build/soong/soong_ui.bash --make-mode $@ \ + TARGET_PRODUCT=${product} \ + ${MAINLINE_MODULES[@]} + + PRODUCT_OUT=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT=${product} get_build_var PRODUCT_OUT) + TARGET_ARCH=$(source build/envsetup.sh > /dev/null; TARGET_PRODUCT=${product} get_build_var TARGET_ARCH) + rm -rf ${DIST_DIR}/${TARGET_ARCH}/ + mkdir -p ${DIST_DIR}/${TARGET_ARCH}/ + for module in "${MAINLINE_MODULES[@]}"; do + echo_and_run cp ${PWD}/${PRODUCT_OUT}/system/apex/${module}.apex ${DIST_DIR}/${TARGET_ARCH}/ + done +done + + +# Create multi-archs SDKs in a different out directory. The multi-arch script +# uses Soong in --skip-make mode which cannot use the same directory as normal +# mode with make. +export OUT_DIR=${OUT_DIR}/aml +echo_and_run build/soong/scripts/build-aml-prebuilts.sh ${MODULES_SDK_AND_EXPORTS[@]} + +rm -rf ${DIST_DIR}/mainline-sdks +echo_and_run cp -R ${OUT_DIR}/soong/mainline-sdks ${DIST_DIR} diff --git a/scripts/build-ndk-prebuilts.sh b/scripts/build-ndk-prebuilts.sh index 947458a62..b6ed65940 100755 --- a/scripts/build-ndk-prebuilts.sh +++ b/scripts/build-ndk-prebuilts.sh @@ -25,7 +25,7 @@ source build/envsetup.sh PLATFORM_SDK_VERSION=$(get_build_var PLATFORM_SDK_VERSION) PLATFORM_VERSION_ALL_CODENAMES=$(get_build_var PLATFORM_VERSION_ALL_CODENAMES) -# PLATFORM_VERSION_ALL_CODESNAMES is a comma separated list like O,P. We need to +# PLATFORM_VERSION_ALL_CODENAMES is a comma separated list like O,P. We need to # turn this into ["O","P"]. PLATFORM_VERSION_ALL_CODENAMES=${PLATFORM_VERSION_ALL_CODENAMES/,/'","'} PLATFORM_VERSION_ALL_CODENAMES="[\"${PLATFORM_VERSION_ALL_CODENAMES}\"]" diff --git a/scripts/build_broken_logs.go b/scripts/build_broken_logs.go index f081f262d..8021e55d0 100644 --- a/scripts/build_broken_logs.go +++ b/scripts/build_broken_logs.go @@ -60,37 +60,11 @@ var buildBrokenSettings = []struct { warnings []string }{ { - name: "BUILD_BROKEN_DUP_COPY_HEADERS", - behavior: DefaultDeprecated, - warnings: []string{"Duplicate header copy:"}, - }, - { name: "BUILD_BROKEN_DUP_RULES", behavior: DefaultFalse, warnings: []string{"overriding commands for target"}, }, { - name: "BUILD_BROKEN_ANDROIDMK_EXPORTS", - behavior: DefaultFalse, - warnings: []string{"export_keyword"}, - }, - { - name: "BUILD_BROKEN_PHONY_TARGETS", - behavior: DefaultFalse, - warnings: []string{ - "depends on PHONY target", - "looks like a real file", - "writing to readonly directory", - }, - }, - { - name: "BUILD_BROKEN_ENG_DEBUG_TAGS", - behavior: DefaultTrue, - warnings: []string{ - "Changes.md#LOCAL_MODULE_TAGS", - }, - }, - { name: "BUILD_BROKEN_USES_NETWORK", behavior: DefaultDeprecated, }, diff --git a/scripts/clang-tidy.sh b/scripts/clang-tidy.sh deleted file mode 100755 index 04d0bdd9a..000000000 --- a/scripts/clang-tidy.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e - -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Wrapper script to remove clang compiler flags rejected by clang-tidy. -# Inputs: -# Environment: -# CLANG_TIDY: path to the real clang-tidy program - -# clang-tidy doesn't recognize every flag that clang compiler does. -# It gives clang-diagnostic-unused-command-line-argument warnings -# to -Wa,* flags. -# The -flto flags caused clang-tidy to ignore the -I flags, -# see https://bugs.llvm.org/show_bug.cgi?id=38332. -# -fsanitize and -fwhole-program-vtables need -flto. -args=("${@}") -n=${#args[@]} -for ((i=0; i<$n; ++i)); do - case ${args[i]} in - -Wa,*|-flto|-flto=*|-fsanitize=*|-fsanitize-*|-fwhole-program-vtables) - unset args[i] - ;; - esac -done -${CLANG_TIDY} "${args[@]}" diff --git a/scripts/freeze-sysprop-api-files.sh b/scripts/freeze-sysprop-api-files.sh new file mode 100755 index 000000000..1b2ff7c72 --- /dev/null +++ b/scripts/freeze-sysprop-api-files.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script freezes APIs of a sysprop_library after checking compatibility +# between latest API and current API. +# +# Usage: freeze-sysprop-api-files.sh <modulePath> <moduleName> +# +# <modulePath>: the directory, either relative or absolute, which holds the +# Android.bp file defining sysprop_library. +# +# <moduleName>: the name of sysprop_library to freeze API. +# +# Example: +# $ . build/envsetup.sh && lunch aosp_arm64-user +# $ . build/soong/scripts/freeze-sysprop-api-files.sh \ +# system/libsysprop/srcs PlatformProperties + +if [[ -z "$1" || -z "$2" ]]; then + echo "usage: $0 <modulePath> <moduleName>" >&2 + exit 1 +fi + +api_dir=$1/api + +m "$2-check-api" && cp -f "${api_dir}/$2-current.txt" "${api_dir}/$2-latest.txt" diff --git a/scripts/gen-java-current-api-files.sh b/scripts/gen-java-current-api-files.sh index 517d3916e..547387a46 100755 --- a/scripts/gen-java-current-api-files.sh +++ b/scripts/gen-java-current-api-files.sh @@ -15,15 +15,16 @@ # limitations under the License. if [[ -z "$1" ]]; then - echo "usage: $0 <modulePath>" >&2 + echo "usage: $0 <modulePath> scopes..." >&2 exit 1 fi -api_dir=$1/api +api_dir=$1 +shift mkdir -p "$api_dir" -scopes=("" system- test-) +scopes=("" "$@") apis=(current removed) for scope in "${scopes[@]}"; do @@ -31,3 +32,4 @@ for scope in "${scopes[@]}"; do touch "${api_dir}/${scope}${api}.txt" done done + diff --git a/scripts/gen-kotlin-build-file.sh b/scripts/gen-kotlin-build-file.sh index 1e03f72e0..177ca1b04 100755 --- a/scripts/gen-kotlin-build-file.sh +++ b/scripts/gen-kotlin-build-file.sh @@ -17,7 +17,7 @@ # Generates kotlinc module xml file to standard output based on rsp files if [[ -z "$1" ]]; then - echo "usage: $0 <classpath> <outDir> <rspFiles>..." >&2 + echo "usage: $0 <classpath> <name> <outDir> <rspFiles>..." >&2 exit 1 fi @@ -27,8 +27,9 @@ if [[ $1 == "-classpath" ]]; then fi; classpath=$1 -out_dir=$2 -shift 2 +name=$2 +out_dir=$3 +shift 3 # Path in the build file may be relative to the build file, we need to make them # absolute @@ -44,7 +45,7 @@ get_abs_path () { } # Print preamble -echo "<modules><module name=\"name\" type=\"java-production\" outputDir=\"${out_dir}\">" +echo "<modules><module name=\"${name}\" type=\"java-production\" outputDir=\"${out_dir}\">" # Print classpath entries for file in $(echo "$classpath" | tr ":" "\n"); do diff --git a/scripts/gen-sysprop-api-files.sh b/scripts/gen-sysprop-api-files.sh new file mode 100755 index 000000000..a4cb50649 --- /dev/null +++ b/scripts/gen-sysprop-api-files.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "$1" || -z "$2" ]]; then + echo "usage: $0 <modulePath> <moduleName>" >&2 + exit 1 +fi + +api_dir=$1/api + +mkdir -p "$api_dir" +touch "${api_dir}/$2-current.txt" +touch "${api_dir}/$2-latest.txt" diff --git a/scripts/gen_sorted_bss_symbols.sh b/scripts/gen_sorted_bss_symbols.sh new file mode 100755 index 000000000..244ed0dea --- /dev/null +++ b/scripts/gen_sorted_bss_symbols.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e + +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script to generate a symbol ordering file that sorts bss section symbols by +# their sizes. +# Inputs: +# Environment: +# CROSS_COMPILE: prefix added to nm tools +# Arguments: +# $1: Input ELF file +# $2: Output symbol ordering file + +set -o pipefail + +${CROSS_COMPILE}nm --size-sort $1 | awk '{if ($2 == "b" || $2 == "B") print $3}' > $2 diff --git a/scripts/jar-wrapper.sh b/scripts/jar-wrapper.sh index 71c1d9067..b46804157 100644 --- a/scripts/jar-wrapper.sh +++ b/scripts/jar-wrapper.sh @@ -48,11 +48,11 @@ if [ ! -r "${jardir}/${jarfile}" ]; then exit 1 fi -javaOpts="" +declare -a javaOpts=() while expr "x$1" : 'x-J' >/dev/null; do - opt=`expr "$1" : '-J\(.*\)'` - javaOpts="${javaOpts} -${opt}" + opt=`expr "$1" : '-J-\{0,1\}\(.*\)'` + javaOpts+=("-${opt}") shift done -exec java ${javaOpts} -jar ${jardir}/${jarfile} "$@" +exec java "${javaOpts[@]}" -jar ${jardir}/${jarfile} "$@" diff --git a/scripts/jsonmodify.py b/scripts/jsonmodify.py new file mode 100755 index 000000000..4b2c3c250 --- /dev/null +++ b/scripts/jsonmodify.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import collections +import json +import sys + +def follow_path(obj, path): + cur = obj + last_key = None + for key in path.split('.'): + if last_key: + if last_key not in cur: + return None,None + cur = cur[last_key] + last_key = key + if last_key not in cur: + return None,None + return cur, last_key + + +def ensure_path(obj, path): + cur = obj + last_key = None + for key in path.split('.'): + if last_key: + if last_key not in cur: + cur[last_key] = dict() + cur = cur[last_key] + last_key = key + return cur, last_key + + +class SetValue(str): + def apply(self, obj, val): + cur, key = ensure_path(obj, self) + cur[key] = val + + +class Replace(str): + def apply(self, obj, val): + cur, key = follow_path(obj, self) + if cur: + cur[key] = val + + +class Remove(str): + def apply(self, obj): + cur, key = follow_path(obj, self) + if cur: + del cur[key] + + +class AppendList(str): + def apply(self, obj, *args): + cur, key = ensure_path(obj, self) + if key not in cur: + cur[key] = list() + if not isinstance(cur[key], list): + raise ValueError(self + " should be a array.") + cur[key].extend(args) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--out', + help='write result to a file. If omitted, print to stdout', + metavar='output', + action='store') + parser.add_argument('input', nargs='?', help='JSON file') + parser.add_argument("-v", "--value", type=SetValue, + help='set value of the key specified by path. If path doesn\'t exist, creates new one.', + metavar=('path', 'value'), + nargs=2, dest='patch', default=[], action='append') + parser.add_argument("-s", "--replace", type=Replace, + help='replace value of the key specified by path. If path doesn\'t exist, no op.', + metavar=('path', 'value'), + nargs=2, dest='patch', action='append') + parser.add_argument("-r", "--remove", type=Remove, + help='remove the key specified by path. If path doesn\'t exist, no op.', + metavar='path', + nargs=1, dest='patch', action='append') + parser.add_argument("-a", "--append_list", type=AppendList, + help='append values to the list specified by path. If path doesn\'t exist, creates new list for it.', + metavar=('path', 'value'), + nargs='+', dest='patch', default=[], action='append') + args = parser.parse_args() + + if args.input: + with open(args.input) as f: + obj = json.load(f, object_pairs_hook=collections.OrderedDict) + else: + obj = json.load(sys.stdin, object_pairs_hook=collections.OrderedDict) + + for p in args.patch: + p[0].apply(obj, *p[1:]) + + if args.out: + with open(args.out, "w") as f: + json.dump(obj, f, indent=2) + else: + print(json.dumps(obj, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/lint-project-xml.py b/scripts/lint-project-xml.py new file mode 100755 index 000000000..38c57cadf --- /dev/null +++ b/scripts/lint-project-xml.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This file generates project.xml and lint.xml files used to drive the Android Lint CLI tool.""" + +import argparse + + +def check_action(check_type): + """ + Returns an action that appends a tuple of check_type and the argument to the dest. + """ + class CheckAction(argparse.Action): + def __init__(self, option_strings, dest, nargs=None, **kwargs): + if nargs is not None: + raise ValueError("nargs must be None, was %s" % nargs) + super(CheckAction, self).__init__(option_strings, dest, **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + checks = getattr(namespace, self.dest, []) + checks.append((check_type, values)) + setattr(namespace, self.dest, checks) + return CheckAction + + +def parse_args(): + """Parse commandline arguments.""" + + def convert_arg_line_to_args(arg_line): + for arg in arg_line.split(): + if arg.startswith('#'): + return + if not arg.strip(): + continue + yield arg + + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.convert_arg_line_to_args = convert_arg_line_to_args + parser.add_argument('--project_out', dest='project_out', + help='file to which the project.xml contents will be written.') + parser.add_argument('--config_out', dest='config_out', + help='file to which the lint.xml contents will be written.') + parser.add_argument('--name', dest='name', + help='name of the module.') + parser.add_argument('--srcs', dest='srcs', action='append', default=[], + help='file containing whitespace separated list of source files.') + parser.add_argument('--generated_srcs', dest='generated_srcs', action='append', default=[], + help='file containing whitespace separated list of generated source files.') + parser.add_argument('--resources', dest='resources', action='append', default=[], + help='file containing whitespace separated list of resource files.') + parser.add_argument('--classes', dest='classes', action='append', default=[], + help='file containing the module\'s classes.') + parser.add_argument('--classpath', dest='classpath', action='append', default=[], + help='file containing classes from dependencies.') + parser.add_argument('--extra_checks_jar', dest='extra_checks_jars', action='append', default=[], + help='file containing extra lint checks.') + parser.add_argument('--manifest', dest='manifest', + help='file containing the module\'s manifest.') + parser.add_argument('--merged_manifest', dest='merged_manifest', + help='file containing merged manifest for the module and its dependencies.') + parser.add_argument('--library', dest='library', action='store_true', + help='mark the module as a library.') + parser.add_argument('--test', dest='test', action='store_true', + help='mark the module as a test.') + parser.add_argument('--cache_dir', dest='cache_dir', + help='directory to use for cached file.') + parser.add_argument('--root_dir', dest='root_dir', + help='directory to use for root dir.') + group = parser.add_argument_group('check arguments', 'later arguments override earlier ones.') + group.add_argument('--fatal_check', dest='checks', action=check_action('fatal'), default=[], + help='treat a lint issue as a fatal error.') + group.add_argument('--error_check', dest='checks', action=check_action('error'), default=[], + help='treat a lint issue as an error.') + group.add_argument('--warning_check', dest='checks', action=check_action('warning'), default=[], + help='treat a lint issue as a warning.') + group.add_argument('--disable_check', dest='checks', action=check_action('ignore'), default=[], + help='disable a lint issue.') + return parser.parse_args() + + +class NinjaRspFileReader: + """ + Reads entries from a Ninja rsp file. Ninja escapes any entries in the file that contain a + non-standard character by surrounding the whole entry with single quotes, and then replacing + any single quotes in the entry with the escape sequence '\''. + """ + + def __init__(self, filename): + self.f = open(filename, 'r') + self.r = self.character_reader(self.f) + + def __iter__(self): + return self + + def character_reader(self, f): + """Turns a file into a generator that returns one character at a time.""" + while True: + c = f.read(1) + if c: + yield c + else: + return + + def __next__(self): + entry = self.read_entry() + if entry: + return entry + else: + raise StopIteration + + def read_entry(self): + c = next(self.r, "") + if not c: + return "" + elif c == "'": + return self.read_quoted_entry() + else: + entry = c + for c in self.r: + if c == " " or c == "\n": + break + entry += c + return entry + + def read_quoted_entry(self): + entry = "" + for c in self.r: + if c == "'": + # Either the end of the quoted entry, or the beginning of an escape sequence, read the next + # character to find out. + c = next(self.r) + if not c or c == " " or c == "\n": + # End of the item + return entry + elif c == "\\": + # Escape sequence, expect a ' + c = next(self.r) + if c != "'": + # Malformed escape sequence + raise "malformed escape sequence %s'\\%s" % (entry, c) + entry += "'" + else: + raise "malformed escape sequence %s'%s" % (entry, c) + else: + entry += c + raise "unterminated quoted entry %s" % entry + + +def write_project_xml(f, args): + test_attr = "test='true' " if args.test else "" + + f.write("<?xml version='1.0' encoding='utf-8'?>\n") + f.write("<project>\n") + if args.root_dir: + f.write(" <root dir='%s' />\n" % args.root_dir) + f.write(" <module name='%s' android='true' %sdesugar='full' >\n" % (args.name, "library='true' " if args.library else "")) + if args.manifest: + f.write(" <manifest file='%s' %s/>\n" % (args.manifest, test_attr)) + if args.merged_manifest: + f.write(" <merged-manifest file='%s' %s/>\n" % (args.merged_manifest, test_attr)) + for src_file in args.srcs: + for src in NinjaRspFileReader(src_file): + f.write(" <src file='%s' %s/>\n" % (src, test_attr)) + for src_file in args.generated_srcs: + for src in NinjaRspFileReader(src_file): + f.write(" <src file='%s' generated='true' %s/>\n" % (src, test_attr)) + for res_file in args.resources: + for res in NinjaRspFileReader(res_file): + f.write(" <resource file='%s' %s/>\n" % (res, test_attr)) + for classes in args.classes: + f.write(" <classes jar='%s' />\n" % classes) + for classpath in args.classpath: + f.write(" <classpath jar='%s' />\n" % classpath) + for extra in args.extra_checks_jars: + f.write(" <lint-checks jar='%s' />\n" % extra) + f.write(" </module>\n") + if args.cache_dir: + f.write(" <cache dir='%s'/>\n" % args.cache_dir) + f.write("</project>\n") + + +def write_config_xml(f, args): + f.write("<?xml version='1.0' encoding='utf-8'?>\n") + f.write("<lint>\n") + for check in args.checks: + f.write(" <issue id='%s' severity='%s' />\n" % (check[1], check[0])) + f.write("</lint>\n") + + +def main(): + """Program entry point.""" + args = parse_args() + + if args.project_out: + with open(args.project_out, 'w') as f: + write_project_xml(f, args) + + if args.config_out: + with open(args.config_out, 'w') as f: + write_config_xml(f, args) + + +if __name__ == '__main__': + main() diff --git a/scripts/manifest.py b/scripts/manifest.py new file mode 100755 index 000000000..04f7405df --- /dev/null +++ b/scripts/manifest.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for inserting values from the build system into a manifest or a test config.""" + +from __future__ import print_function +from xml.dom import minidom + + +android_ns = 'http://schemas.android.com/apk/res/android' + + +def get_children_with_tag(parent, tag_name): + children = [] + for child in parent.childNodes: + if child.nodeType == minidom.Node.ELEMENT_NODE and \ + child.tagName == tag_name: + children.append(child) + return children + + +def find_child_with_attribute(element, tag_name, namespace_uri, + attr_name, value): + for child in get_children_with_tag(element, tag_name): + attr = child.getAttributeNodeNS(namespace_uri, attr_name) + if attr is not None and attr.value == value: + return child + return None + + +def parse_manifest(doc): + """Get the manifest element.""" + + manifest = doc.documentElement + if manifest.tagName != 'manifest': + raise RuntimeError('expected manifest tag at root') + return manifest + + +def ensure_manifest_android_ns(doc): + """Make sure the manifest tag defines the android namespace.""" + + manifest = parse_manifest(doc) + + ns = manifest.getAttributeNodeNS(minidom.XMLNS_NAMESPACE, 'android') + if ns is None: + attr = doc.createAttributeNS(minidom.XMLNS_NAMESPACE, 'xmlns:android') + attr.value = android_ns + manifest.setAttributeNode(attr) + elif ns.value != android_ns: + raise RuntimeError('manifest tag has incorrect android namespace ' + + ns.value) + + +def parse_test_config(doc): + """ Get the configuration element. """ + + test_config = doc.documentElement + if test_config.tagName != 'configuration': + raise RuntimeError('expected configuration tag at root') + return test_config + + +def as_int(s): + try: + i = int(s) + except ValueError: + return s, False + return i, True + + +def compare_version_gt(a, b): + """Compare two SDK versions. + + Compares a and b, treating codenames like 'Q' as higher + than numerical versions like '28'. + + Returns True if a > b + + Args: + a: value to compare + b: value to compare + Returns: + True if a is a higher version than b + """ + + a, a_is_int = as_int(a.upper()) + b, b_is_int = as_int(b.upper()) + + if a_is_int == b_is_int: + # Both are codenames or both are versions, compare directly + return a > b + else: + # One is a codename, the other is not. Return true if + # b is an integer version + return b_is_int + + +def get_indent(element, default_level): + indent = '' + if element is not None and element.nodeType == minidom.Node.TEXT_NODE: + text = element.nodeValue + indent = text[:len(text)-len(text.lstrip())] + if not indent or indent == '\n': + # 1 indent = 4 space + indent = '\n' + (' ' * default_level * 4) + return indent + + +def write_xml(f, doc): + f.write('<?xml version="1.0" encoding="utf-8"?>\n') + for node in doc.childNodes: + f.write(node.toxml(encoding='utf-8') + '\n') diff --git a/scripts/manifest_check.py b/scripts/manifest_check.py new file mode 100755 index 000000000..9122da1fb --- /dev/null +++ b/scripts/manifest_check.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for checking that a manifest agrees with the build system.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +from manifest import android_ns +from manifest import get_children_with_tag +from manifest import parse_manifest +from manifest import write_xml + + +class ManifestMismatchError(Exception): + pass + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('--uses-library', dest='uses_libraries', + action='append', + help='specify uses-library entries known to the build system') + parser.add_argument('--optional-uses-library', + dest='optional_uses_libraries', + action='append', + help='specify uses-library entries known to the build system with required:false') + parser.add_argument('--enforce-uses-libraries', + dest='enforce_uses_libraries', + action='store_true', + help='check the uses-library entries known to the build system against the manifest') + parser.add_argument('--extract-target-sdk-version', + dest='extract_target_sdk_version', + action='store_true', + help='print the targetSdkVersion from the manifest') + parser.add_argument('--output', '-o', dest='output', help='output AndroidManifest.xml file') + parser.add_argument('input', help='input AndroidManifest.xml file') + return parser.parse_args() + + +def enforce_uses_libraries(doc, uses_libraries, optional_uses_libraries): + """Verify that the <uses-library> tags in the manifest match those provided by the build system. + + Args: + doc: The XML document. + uses_libraries: The names of <uses-library> tags known to the build system + optional_uses_libraries: The names of <uses-library> tags with required:fals + known to the build system + Raises: + RuntimeError: Invalid manifest + ManifestMismatchError: Manifest does not match + """ + + manifest = parse_manifest(doc) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + if uses_libraries or optional_uses_libraries: + raise ManifestMismatchError('no <application> tag found') + return + + verify_uses_library(application, uses_libraries, optional_uses_libraries) + + +def verify_uses_library(application, uses_libraries, optional_uses_libraries): + """Verify that the uses-library values known to the build system match the manifest. + + Args: + application: the <application> tag in the manifest. + uses_libraries: the names of expected <uses-library> tags. + optional_uses_libraries: the names of expected <uses-library> tags with required="false". + Raises: + ManifestMismatchError: Manifest does not match + """ + + if uses_libraries is None: + uses_libraries = [] + + if optional_uses_libraries is None: + optional_uses_libraries = [] + + manifest_uses_libraries, manifest_optional_uses_libraries = parse_uses_library(application) + + err = [] + if manifest_uses_libraries != uses_libraries: + err.append('Expected required <uses-library> tags "%s", got "%s"' % + (', '.join(uses_libraries), ', '.join(manifest_uses_libraries))) + + if manifest_optional_uses_libraries != optional_uses_libraries: + err.append('Expected optional <uses-library> tags "%s", got "%s"' % + (', '.join(optional_uses_libraries), ', '.join(manifest_optional_uses_libraries))) + + if err: + raise ManifestMismatchError('\n'.join(err)) + + +def parse_uses_library(application): + """Extract uses-library tags from the manifest. + + Args: + application: the <application> tag in the manifest. + """ + + libs = get_children_with_tag(application, 'uses-library') + + uses_libraries = [uses_library_name(x) for x in libs if uses_library_required(x)] + optional_uses_libraries = [uses_library_name(x) for x in libs if not uses_library_required(x)] + + return first_unique_elements(uses_libraries), first_unique_elements(optional_uses_libraries) + + +def first_unique_elements(l): + result = [] + [result.append(x) for x in l if x not in result] + return result + + +def uses_library_name(lib): + """Extract the name attribute of a uses-library tag. + + Args: + lib: a <uses-library> tag. + """ + name = lib.getAttributeNodeNS(android_ns, 'name') + return name.value if name is not None else "" + + +def uses_library_required(lib): + """Extract the required attribute of a uses-library tag. + + Args: + lib: a <uses-library> tag. + """ + required = lib.getAttributeNodeNS(android_ns, 'required') + return (required.value == 'true') if required is not None else True + + +def extract_target_sdk_version(doc): + """Returns the targetSdkVersion from the manifest. + + Args: + doc: The XML document. + Raises: + RuntimeError: invalid manifest + """ + + manifest = parse_manifest(doc) + + # Get or insert the uses-sdk element + uses_sdk = get_children_with_tag(manifest, 'uses-sdk') + if len(uses_sdk) > 1: + raise RuntimeError('found multiple uses-sdk elements') + elif len(uses_sdk) == 0: + raise RuntimeError('missing uses-sdk element') + + uses_sdk = uses_sdk[0] + + min_attr = uses_sdk.getAttributeNodeNS(android_ns, 'minSdkVersion') + if min_attr is None: + raise RuntimeError('minSdkVersion is not specified') + + target_attr = uses_sdk.getAttributeNodeNS(android_ns, 'targetSdkVersion') + if target_attr is None: + target_attr = min_attr + + return target_attr.value + + +def main(): + """Program entry point.""" + try: + args = parse_args() + + doc = minidom.parse(args.input) + + if args.enforce_uses_libraries: + enforce_uses_libraries(doc, + args.uses_libraries, + args.optional_uses_libraries) + + if args.extract_target_sdk_version: + print(extract_target_sdk_version(doc)) + + if args.output: + with open(args.output, 'wb') as f: + write_xml(f, doc) + + # pylint: disable=broad-except + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + +if __name__ == '__main__': + main() diff --git a/scripts/manifest_check_test.py b/scripts/manifest_check_test.py new file mode 100755 index 000000000..7baad5d38 --- /dev/null +++ b/scripts/manifest_check_test.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for manifest_fixer.py.""" + +import sys +import unittest +from xml.dom import minidom + +import manifest_check + +sys.dont_write_bytecode = True + + +def uses_library(name, attr=''): + return '<uses-library android:name="%s"%s />' % (name, attr) + + +def required(value): + return ' android:required="%s"' % ('true' if value else 'false') + + +class EnforceUsesLibrariesTest(unittest.TestCase): + """Unit tests for add_extract_native_libs function.""" + + def run_test(self, input_manifest, uses_libraries=None, optional_uses_libraries=None): + doc = minidom.parseString(input_manifest) + try: + manifest_check.enforce_uses_libraries(doc, uses_libraries, optional_uses_libraries) + return True + except manifest_check.ManifestMismatchError: + return False + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <application>\n' + ' %s\n' + ' </application>\n' + '</manifest>\n') + + def test_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_uses_library_required(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(True))) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_expected_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_expected_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_uses_library(self): + manifest_input = self.manifest_tmpl % ('') + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertFalse(matches) + + def test_missing_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('') + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertFalse(matches) + + def test_extra_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo')) + matches = self.run_test(manifest_input) + self.assertFalse(matches) + + def test_extra_optional_uses_library(self): + manifest_input = self.manifest_tmpl % (uses_library('foo', required(False))) + matches = self.run_test(manifest_input) + self.assertFalse(matches) + + def test_multiple_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar')])) + matches = self.run_test(manifest_input, uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_multiple_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo', 'bar']) + self.assertTrue(matches) + + def test_order_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar')])) + matches = self.run_test(manifest_input, uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_order_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['bar', 'foo']) + self.assertFalse(matches) + + def test_duplicate_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('foo')])) + matches = self.run_test(manifest_input, uses_libraries=['foo']) + self.assertTrue(matches) + + def test_duplicate_optional_uses_library(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo', required(False)), + uses_library('foo', required(False))])) + matches = self.run_test(manifest_input, optional_uses_libraries=['foo']) + self.assertTrue(matches) + + def test_mixed(self): + manifest_input = self.manifest_tmpl % ('\n'.join([uses_library('foo'), + uses_library('bar', required(False))])) + matches = self.run_test(manifest_input, uses_libraries=['foo'], + optional_uses_libraries=['bar']) + self.assertTrue(matches) + + +class ExtractTargetSdkVersionTest(unittest.TestCase): + def test_target_sdk_version(self): + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <uses-sdk android:minSdkVersion="28" android:targetSdkVersion="29" />\n' + '</manifest>\n') + doc = minidom.parseString(manifest) + target_sdk_version = manifest_check.extract_target_sdk_version(doc) + self.assertEqual(target_sdk_version, '29') + + def test_min_sdk_version(self): + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <uses-sdk android:minSdkVersion="28" />\n' + '</manifest>\n') + doc = minidom.parseString(manifest) + target_sdk_version = manifest_check.extract_target_sdk_version(doc) + self.assertEqual(target_sdk_version, '28') + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/manifest_fixer.py b/scripts/manifest_fixer.py index 83868e603..c59732bb9 100755 --- a/scripts/manifest_fixer.py +++ b/scripts/manifest_fixer.py @@ -17,30 +17,20 @@ """A tool for inserting values from the build system into a manifest.""" from __future__ import print_function + import argparse import sys from xml.dom import minidom -android_ns = 'http://schemas.android.com/apk/res/android' - - -def get_children_with_tag(parent, tag_name): - children = [] - for child in parent.childNodes: - if child.nodeType == minidom.Node.ELEMENT_NODE and \ - child.tagName == tag_name: - children.append(child) - return children - - -def find_child_with_attribute(element, tag_name, namespace_uri, - attr_name, value): - for child in get_children_with_tag(element, tag_name): - attr = child.getAttributeNodeNS(namespace_uri, attr_name) - if attr is not None and attr.value == value: - return child - return None +from manifest import android_ns +from manifest import compare_version_gt +from manifest import ensure_manifest_android_ns +from manifest import find_child_with_attribute +from manifest import get_children_with_tag +from manifest import get_indent +from manifest import parse_manifest +from manifest import write_xml def parse_args(): @@ -61,6 +51,9 @@ def parse_args(): help='specify additional <uses-library> tag to add. android:requred is set to false') parser.add_argument('--uses-non-sdk-api', dest='uses_non_sdk_api', action='store_true', help='manifest is for a package built against the platform') + parser.add_argument('--logging-parent', dest='logging_parent', default='', + help=('specify logging parent as an additional <meta-data> tag. ' + 'This value is ignored if the logging_parent meta-data tag is present.')) parser.add_argument('--use-embedded-dex', dest='use_embedded_dex', action='store_true', help=('specify if the app wants to use embedded dex and avoid extracted,' 'locally compiled code. Must not conflict if already declared ' @@ -69,81 +62,14 @@ def parse_args(): default=None, type=lambda x: (str(x).lower() == 'true'), help=('specify if the app wants to use embedded native libraries. Must not conflict ' 'if already declared in the manifest.')) + parser.add_argument('--has-no-code', dest='has_no_code', action='store_true', + help=('adds hasCode="false" attribute to application. Ignored if application elem ' + 'already has a hasCode attribute.')) parser.add_argument('input', help='input AndroidManifest.xml file') parser.add_argument('output', help='output AndroidManifest.xml file') return parser.parse_args() -def parse_manifest(doc): - """Get the manifest element.""" - - manifest = doc.documentElement - if manifest.tagName != 'manifest': - raise RuntimeError('expected manifest tag at root') - return manifest - - -def ensure_manifest_android_ns(doc): - """Make sure the manifest tag defines the android namespace.""" - - manifest = parse_manifest(doc) - - ns = manifest.getAttributeNodeNS(minidom.XMLNS_NAMESPACE, 'android') - if ns is None: - attr = doc.createAttributeNS(minidom.XMLNS_NAMESPACE, 'xmlns:android') - attr.value = android_ns - manifest.setAttributeNode(attr) - elif ns.value != android_ns: - raise RuntimeError('manifest tag has incorrect android namespace ' + - ns.value) - - -def as_int(s): - try: - i = int(s) - except ValueError: - return s, False - return i, True - - -def compare_version_gt(a, b): - """Compare two SDK versions. - - Compares a and b, treating codenames like 'Q' as higher - than numerical versions like '28'. - - Returns True if a > b - - Args: - a: value to compare - b: value to compare - Returns: - True if a is a higher version than b - """ - - a, a_is_int = as_int(a.upper()) - b, b_is_int = as_int(b.upper()) - - if a_is_int == b_is_int: - # Both are codenames or both are versions, compare directly - return a > b - else: - # One is a codename, the other is not. Return true if - # b is an integer version - return b_is_int - - -def get_indent(element, default_level): - indent = '' - if element is not None and element.nodeType == minidom.Node.TEXT_NODE: - text = element.nodeValue - indent = text[:len(text)-len(text.lstrip())] - if not indent or indent == '\n': - # 1 indent = 4 space - indent = '\n' + (' ' * default_level * 4) - return indent - - def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): """Ensure the manifest contains a <uses-sdk> tag with a minSdkVersion. @@ -151,6 +77,7 @@ def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): doc: The XML document. May be modified by this function. min_sdk_version: The requested minSdkVersion attribute. target_sdk_version: The requested targetSdkVersion attribute. + library: True if the manifest is for a library. Raises: RuntimeError: invalid manifest """ @@ -200,6 +127,52 @@ def raise_min_sdk_version(doc, min_sdk_version, target_sdk_version, library): element.setAttributeNode(target_attr) +def add_logging_parent(doc, logging_parent_value): + """Add logging parent as an additional <meta-data> tag. + + Args: + doc: The XML document. May be modified by this function. + logging_parent_value: A string representing the logging + parent value. + Raises: + RuntimeError: Invalid manifest + """ + manifest = parse_manifest(doc) + + logging_parent_key = 'android.content.pm.LOGGING_PARENT' + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + application = doc.createElement('application') + indent = get_indent(manifest.firstChild, 1) + first = manifest.firstChild + manifest.insertBefore(doc.createTextNode(indent), first) + manifest.insertBefore(application, first) + + indent = get_indent(application.firstChild, 2) + + last = application.lastChild + if last is not None and last.nodeType != minidom.Node.TEXT_NODE: + last = None + + if not find_child_with_attribute(application, 'meta-data', android_ns, + 'name', logging_parent_key): + ul = doc.createElement('meta-data') + ul.setAttributeNS(android_ns, 'android:name', logging_parent_key) + ul.setAttributeNS(android_ns, 'android:value', logging_parent_value) + application.insertBefore(doc.createTextNode(indent), last) + application.insertBefore(ul, last) + last = application.lastChild + + # align the closing tag with the opening tag if it's not + # indented + if last and last.nodeType != minidom.Node.TEXT_NODE: + indent = get_indent(application.previousSibling, 1) + application.appendChild(doc.createTextNode(indent)) + + def add_uses_libraries(doc, new_uses_libraries, required): """Add additional <uses-library> tags @@ -249,6 +222,7 @@ def add_uses_libraries(doc, new_uses_libraries, required): indent = get_indent(application.previousSibling, 1) application.appendChild(doc.createTextNode(indent)) + def add_uses_non_sdk_api(doc): """Add android:usesNonSdkApi=true attribute to <application>. @@ -323,10 +297,26 @@ def add_extract_native_libs(doc, extract_native_libs): (attr.value, value)) -def write_xml(f, doc): - f.write('<?xml version="1.0" encoding="utf-8"?>\n') - for node in doc.childNodes: - f.write(node.toxml(encoding='utf-8') + '\n') +def set_has_code_to_false(doc): + manifest = parse_manifest(doc) + elems = get_children_with_tag(manifest, 'application') + application = elems[0] if len(elems) == 1 else None + if len(elems) > 1: + raise RuntimeError('found multiple <application> tags') + elif not elems: + application = doc.createElement('application') + indent = get_indent(manifest.firstChild, 1) + first = manifest.firstChild + manifest.insertBefore(doc.createTextNode(indent), first) + manifest.insertBefore(application, first) + + attr = application.getAttributeNodeNS(android_ns, 'hasCode') + if attr is not None: + # Do nothing if the application already has a hasCode attribute. + return + attr = doc.createAttributeNS(android_ns, 'android:hasCode') + attr.value = 'false' + application.setAttributeNode(attr) def main(): @@ -350,9 +340,15 @@ def main(): if args.uses_non_sdk_api: add_uses_non_sdk_api(doc) + if args.logging_parent: + add_logging_parent(doc, args.logging_parent) + if args.use_embedded_dex: add_use_embedded_dex(doc) + if args.has_no_code: + set_has_code_to_false(doc) + if args.extract_native_libs is not None: add_extract_native_libs(doc, args.extract_native_libs) diff --git a/scripts/manifest_fixer_test.py b/scripts/manifest_fixer_test.py index 4ad9afaf3..d6e7f2674 100755 --- a/scripts/manifest_fixer_test.py +++ b/scripts/manifest_fixer_test.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Unit tests for manifest_fixer_test.py.""" +"""Unit tests for manifest_fixer.py.""" import StringIO import sys @@ -226,12 +226,53 @@ class RaiseMinSdkVersionTest(unittest.TestCase): self.assertEqual(output, expected) +class AddLoggingParentTest(unittest.TestCase): + """Unit tests for add_logging_parent function.""" + + def add_logging_parent_test(self, input_manifest, logging_parent=None): + doc = minidom.parseString(input_manifest) + if logging_parent: + manifest_fixer.add_logging_parent(doc, logging_parent) + output = StringIO.StringIO() + manifest_fixer.write_xml(output, doc) + return output.getvalue() + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + '%s' + '</manifest>\n') + + def uses_logging_parent(self, logging_parent=None): + attrs = '' + if logging_parent: + meta_text = ('<meta-data android:name="android.content.pm.LOGGING_PARENT" ' + 'android:value="%s"/>\n') % (logging_parent) + attrs += ' <application>\n %s </application>\n' % (meta_text) + + return attrs + + def test_no_logging_parent(self): + """Tests manifest_fixer with no logging_parent.""" + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % self.uses_logging_parent() + output = self.add_logging_parent_test(manifest_input) + self.assertEqual(output, expected) + + def test_logging_parent(self): + """Tests manifest_fixer with no logging_parent.""" + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % self.uses_logging_parent('FOO') + output = self.add_logging_parent_test(manifest_input, 'FOO') + self.assertEqual(output, expected) + + class AddUsesLibrariesTest(unittest.TestCase): """Unit tests for add_uses_libraries function.""" def run_test(self, input_manifest, new_uses_libraries): doc = minidom.parseString(input_manifest) - manifest_fixer.add_uses_libraries(doc, new_uses_libraries) + manifest_fixer.add_uses_libraries(doc, new_uses_libraries, True) output = StringIO.StringIO() manifest_fixer.write_xml(output, doc) return output.getvalue() @@ -393,10 +434,10 @@ class AddExtractNativeLibsTest(unittest.TestCase): return output.getvalue() manifest_tmpl = ( - '<?xml version="1.0" encoding="utf-8"?>\n' - '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' - ' <application%s/>\n' - '</manifest>\n') + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + ' <application%s/>\n' + '</manifest>\n') def extract_native_libs(self, value): return ' android:extractNativeLibs="%s"' % value @@ -424,5 +465,47 @@ class AddExtractNativeLibsTest(unittest.TestCase): self.assertRaises(RuntimeError, self.run_test, manifest_input, False) +class AddNoCodeApplicationTest(unittest.TestCase): + """Unit tests for set_has_code_to_false function.""" + + def run_test(self, input_manifest): + doc = minidom.parseString(input_manifest) + manifest_fixer.set_has_code_to_false(doc) + output = StringIO.StringIO() + manifest_fixer.write_xml(output, doc) + return output.getvalue() + + manifest_tmpl = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android">\n' + '%s' + '</manifest>\n') + + def test_no_application(self): + manifest_input = self.manifest_tmpl % '' + expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, expected) + + def test_has_application_no_has_code(self): + manifest_input = self.manifest_tmpl % ' <application/>\n' + expected = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, expected) + + def test_has_application_has_code_false(self): + """ Do nothing if there's already an application elemeent. """ + manifest_input = self.manifest_tmpl % ' <application android:hasCode="false"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, manifest_input) + + def test_has_application_has_code_true(self): + """ Do nothing if there's already an application elemeent even if its + hasCode attribute is true. """ + manifest_input = self.manifest_tmpl % ' <application android:hasCode="true"/>\n' + output = self.run_test(manifest_input) + self.assertEqual(output, manifest_input) + + if __name__ == '__main__': - unittest.main() + unittest.main(verbosity=2) diff --git a/scripts/package-check.sh b/scripts/package-check.sh index f982e8244..d7e602f31 100755 --- a/scripts/package-check.sh +++ b/scripts/package-check.sh @@ -52,6 +52,7 @@ zip_contents=`zipinfo -1 $jar_file` # Check all class file names against the expected prefixes. old_ifs=${IFS} IFS=$'\n' +failed=false for zip_entry in ${zip_contents}; do # Check the suffix. if [[ "${zip_entry}" = *.class ]]; then @@ -65,8 +66,11 @@ for zip_entry in ${zip_contents}; do done if [[ "${found}" == "false" ]]; then echo "Class file ${zip_entry} is outside specified packages." - exit 1 + failed=true fi fi done +if [[ "${failed}" == "true" ]]; then + exit 1 +fi IFS=${old_ifs} diff --git a/scripts/setup-android-build.sh b/scripts/setup-android-build.sh new file mode 100755 index 000000000..dbb66c3d9 --- /dev/null +++ b/scripts/setup-android-build.sh @@ -0,0 +1,93 @@ +#! /bin/bash +# +# Sets the current directory as Android build output directory for a +# given target by writing the "prefix script" to it. Commands prefixed +# by this script are executed in the Android build environment. E.g., +# running +# ./run <command> +# runs <command> as if we issued +# cd <source> +# mount --bind <build dir> out +# . build/envsetup.sh +# lunch <config> +# <command> +# exit +# +# This arrangement eliminates the need to issue envsetup/lunch commands +# manually, and allows to run multiple builds from the same shell. +# Thus, if your source tree is in ~/aosp and you are building for +# 'blueline' and 'cuttlefish', issuing +# cd /sdx/blueline && \ +# ~/aosp/build/soong/scripts/setup-android-build.sh aosp_blueline-userdebug +# cd /sdx/cuttlefish && \ +# ~/aosp/build/soong/scripts/setup-android-build.sh aosp_cf_arm64_phone-userdebug +# sets up build directories in /sdx/blueline and /sdx/cuttlefish respectively. +# After that, issue +# /sdx/blueline/run m +# to build blueline image, and issue +# /sdx/cuttlefish atest CtsSecurityBulletinHostTestCases +# to run CTS tests. Notice there is no need to change to a specific directory for that. +# +# Argument: +# * configuration (one of those shown by `lunch` command). +# +set -e +function die() { printf "$@"; exit 1; } + +# Find out where the source tree using the fact that we are in its +# build/ subdirectory. +[[ "$(uname)" == Linux ]] || die "This setup runs only on Linux\n" +declare -r mydir="${0%/*}" +declare -r source="${mydir%/build/soong/scripts}" +[[ "/${mydir}/" =~ '/build/soong/scripts/' ]] || \ + die "$0 should be in build/soong/scripts/ subdirectory of the source tree\n" +[[ ! -e .repo && ! -e .git ]] || \ + die "Current directory looks like source. You should be in the _target_ directory.\n" +# Do not override old run script. +if [[ -x ./run ]]; then + # Set variables from config=xxx and source=xxx comments in the existing script. + . <(sed -nr 's/^# *source=(.*)/oldsource=\1/p;s/^# *config=(.*)/oldconfig=\1/p' run) + die "This directory has been already set up to build Android for %s from %s.\n\ +Remove 'run' file if you want to set it up afresh\n" "$oldconfig" "$oldsource" +fi + +(($#<2)) || die "usage: %s [<config>]\n" $0 + +if (($#==1)); then + # Configuration is provided, emit run script. + declare -r config="$1" + declare -r target="$PWD" + cat >./run <<EOF +#! /bin/bash +# source=$source +# config=$config +declare -r cmd=\$(printf ' %q' "\$@") +"$source/prebuilts/build-tools/linux-x86/bin/nsjail"\ + -Mo -q -e -t 0\ + -EANDROID_QUIET_BUILD=true \ + -B / -B "$target:$source/out"\ + --cwd "$source"\ + --skip_setsid \ + --keep_caps\ + --disable_clone_newcgroup\ + --disable_clone_newnet\ + --rlimit_as soft\ + --rlimit_core soft\ + --rlimit_cpu soft\ + --rlimit_fsize soft\ + --rlimit_nofile soft\ + --proc_rw\ + --hostname $(hostname) \ + --\ + /bin/bash -i -c ". build/envsetup.sh && lunch "$config" &&\$cmd" +EOF + chmod +x ./run +else + # No configuration, show available ones. + printf "Please specify build target. Common values:\n" + (cd "$source" + . build/envsetup.sh + get_build_var COMMON_LUNCH_CHOICES | tr ' ' '\n' | pr -c4 -tT -W"$(tput cols)" + ) + exit 1 +fi diff --git a/scripts/setup_go_workspace_for_soong.sh b/scripts/setup_go_workspace_for_soong.sh index 6374aae7f..479d09c0f 100755 --- a/scripts/setup_go_workspace_for_soong.sh +++ b/scripts/setup_go_workspace_for_soong.sh @@ -349,6 +349,7 @@ readonly BIND_PATHS=( "${ANDROID_PATH}/external/golang-protobuf|${OUTPUT_PATH}/src/github.com/golang/protobuf" "${ANDROID_PATH}/external/llvm/soong|${OUTPUT_PATH}/src/android/soong/llvm" "${ANDROID_PATH}/external/clang/soong|${OUTPUT_PATH}/src/android/soong/clang" + "${ANDROID_PATH}/external/robolectric-shadows/soong|${OUTPUT_PATH}/src/android/soong/robolectric" ) main "$@" diff --git a/scripts/strip.sh b/scripts/strip.sh index 0f77da8a9..40f018425 100755 --- a/scripts/strip.sh +++ b/scripts/strip.sh @@ -28,7 +28,7 @@ # --add-gnu-debuglink # --keep-mini-debug-info # --keep-symbols -# --use-gnu-strip +# --keep-symbols-and-debug-frame # --remove-build-id set -o pipefail @@ -39,80 +39,59 @@ usage() { cat <<EOF Usage: strip.sh [options] -k symbols -i in-file -o out-file -d deps-file Options: - --add-gnu-debuglink Add a gnu-debuglink section to out-file - --keep-mini-debug-info Keep compressed debug info in out-file - --keep-symbols Keep symbols in out-file - --use-gnu-strip Use strip/objcopy instead of llvm-{strip,objcopy} - --remove-build-id Remove the gnu build-id section in out-file + --add-gnu-debuglink Add a gnu-debuglink section to out-file + --keep-mini-debug-info Keep compressed debug info in out-file + --keep-symbols Keep symbols in out-file + --keep-symbols-and-debug-frame Keep symbols and .debug_frame in out-file + --remove-build-id Remove the gnu build-id section in out-file EOF exit 1 } -# Without --use-gnu-strip, GNU strip is replaced with llvm-strip to work around -# old GNU strip bug on lld output files, b/80093681. -# Similary, calls to objcopy are replaced with llvm-objcopy, -# with some exceptions. - do_strip() { - # ${CROSS_COMPILE}strip --strip-all does not strip .ARM.attributes, + # GNU strip --strip-all does not strip .ARM.attributes, # so we tell llvm-strip to keep it too. - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" --strip-all -keep-section=.ARM.attributes "${infile}" -o "${outfile}.tmp" - else - "${CROSS_COMPILE}strip" --strip-all "${infile}" -o "${outfile}.tmp" - fi + "${CLANG_BIN}/llvm-strip" --strip-all --keep-section=.ARM.attributes "${infile}" -o "${outfile}.tmp" +} + +do_strip_keep_symbols_and_debug_frame() { + REMOVE_SECTIONS=`"${CLANG_BIN}/llvm-readelf" -S "${infile}" | awk '/.debug_/ {if ($2 != ".debug_frame") {print "--remove-section " $2}}' | xargs` + "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} } do_strip_keep_symbols() { - REMOVE_SECTIONS=`"${CROSS_COMPILE}readelf" -S "${infile}" | awk '/.debug_/ {print "--remove-section " $2}' | xargs` - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} - else - "${CROSS_COMPILE}objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} - fi + REMOVE_SECTIONS=`"${CLANG_BIN}/llvm-readelf" -S "${infile}" | awk '/.debug_/ {print "--remove-section " $2}' | xargs` + "${CLANG_BIN}/llvm-objcopy" "${infile}" "${outfile}.tmp" ${REMOVE_SECTIONS} } do_strip_keep_symbol_list() { - if [ -z "${use_gnu_strip}" ]; then - echo "do_strip_keep_symbol_list does not work with llvm-objcopy" - echo "http://b/131631155" - usage - fi - echo "${symbols_to_keep}" | tr ',' '\n' > "${outfile}.symbolList" - KEEP_SYMBOLS="-w --strip-unneeded-symbol=* --keep-symbols=" - KEEP_SYMBOLS+="${outfile}.symbolList" - "${CROSS_COMPILE}objcopy" "${infile}" "${outfile}.tmp" ${KEEP_SYMBOLS} + KEEP_SYMBOLS="--strip-unneeded-symbol=* --keep-symbols=" + KEEP_SYMBOLS+="${outfile}.symbolList" + "${CROSS_COMPILE}objcopy" -w "${infile}" "${outfile}.tmp" ${KEEP_SYMBOLS} } do_strip_keep_mini_debug_info() { rm -f "${outfile}.dynsyms" "${outfile}.funcsyms" "${outfile}.keep_symbols" "${outfile}.debug" "${outfile}.mini_debuginfo" "${outfile}.mini_debuginfo.xz" local fail= - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" --strip-all -keep-section=.ARM.attributes -remove-section=.comment "${infile}" -o "${outfile}.tmp" || fail=true - else - "${CROSS_COMPILE}strip" --strip-all -R .comment "${infile}" -o "${outfile}.tmp" || fail=true - fi + "${CLANG_BIN}/llvm-strip" --strip-all --keep-section=.ARM.attributes --remove-section=.comment "${infile}" -o "${outfile}.tmp" || fail=true + if [ -z $fail ]; then - # Current prebult llvm-objcopy does not support the following flags: - # --only-keep-debug --rename-section --keep-symbols - # For the following use cases, ${CROSS_COMPILE}objcopy does fine with lld linked files, - # except the --add-section flag. + # Current prebult llvm-objcopy does not support --only-keep-debug flag, + # and cannot process object files that are produced with the flag. Use + # GNU objcopy instead for now. (b/141010852) "${CROSS_COMPILE}objcopy" --only-keep-debug "${infile}" "${outfile}.debug" - "${CROSS_COMPILE}nm" -D "${infile}" --format=posix --defined-only 2> /dev/null | awk '{ print $1 }' | sort >"${outfile}.dynsyms" - "${CROSS_COMPILE}nm" "${infile}" --format=posix --defined-only | awk '{ if ($2 == "T" || $2 == "t" || $2 == "D") print $1 }' | sort > "${outfile}.funcsyms" + "${CLANG_BIN}/llvm-nm" -D "${infile}" --format=posix --defined-only 2> /dev/null | awk '{ print $1 }' | sort >"${outfile}.dynsyms" + "${CLANG_BIN}/llvm-nm" "${infile}" --format=posix --defined-only | awk '{ if ($2 == "T" || $2 == "t" || $2 == "D") print $1 }' | sort > "${outfile}.funcsyms" comm -13 "${outfile}.dynsyms" "${outfile}.funcsyms" > "${outfile}.keep_symbols" echo >> "${outfile}.keep_symbols" # Ensure that the keep_symbols file is not empty. "${CROSS_COMPILE}objcopy" --rename-section .debug_frame=saved_debug_frame "${outfile}.debug" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" -S --remove-section .gdb_index --remove-section .comment --keep-symbols="${outfile}.keep_symbols" "${outfile}.mini_debuginfo" "${CROSS_COMPILE}objcopy" --rename-section saved_debug_frame=.debug_frame "${outfile}.mini_debuginfo" "${XZ}" "${outfile}.mini_debuginfo" - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" - else - "${CROSS_COMPILE}objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" - fi + + "${CLANG_BIN}/llvm-objcopy" --add-section .gnu_debugdata="${outfile}.mini_debuginfo.xz" "${outfile}.tmp" rm -f "${outfile}.dynsyms" "${outfile}.funcsyms" "${outfile}.keep_symbols" "${outfile}.debug" "${outfile}.mini_debuginfo" "${outfile}.mini_debuginfo.xz" else cp -f "${infile}" "${outfile}.tmp" @@ -120,19 +99,11 @@ do_strip_keep_mini_debug_info() { } do_add_gnu_debuglink() { - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" - else - "${CROSS_COMPILE}objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" - fi + "${CLANG_BIN}/llvm-objcopy" --add-gnu-debuglink="${infile}" "${outfile}.tmp" } do_remove_build_id() { - if [ -z "${use_gnu_strip}" ]; then - "${CLANG_BIN}/llvm-strip" -remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" - else - "${CROSS_COMPILE}strip" --remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" - fi + "${CLANG_BIN}/llvm-strip" --remove-section=.note.gnu.build-id "${outfile}.tmp" -o "${outfile}.tmp.no-build-id" rm -f "${outfile}.tmp" mv "${outfile}.tmp.no-build-id" "${outfile}.tmp" } @@ -148,8 +119,8 @@ while getopts $OPTSTRING opt; do add-gnu-debuglink) add_gnu_debuglink=true ;; keep-mini-debug-info) keep_mini_debug_info=true ;; keep-symbols) keep_symbols=true ;; + keep-symbols-and-debug-frame) keep_symbols_and_debug_frame=true ;; remove-build-id) remove_build_id=true ;; - use-gnu-strip) use_gnu_strip=true ;; *) echo "Unknown option --${OPTARG}"; usage ;; esac;; ?) usage ;; @@ -177,6 +148,16 @@ if [ ! -z "${keep_symbols}" -a ! -z "${keep_mini_debug_info}" ]; then usage fi +if [ ! -z "${keep_symbols}" -a ! -z "${keep_symbols_and_debug_frame}" ]; then + echo "--keep-symbols and --keep-symbols-and-debug-frame cannot be used together" + usage +fi + +if [ ! -z "${keep_mini_debug_info}" -a ! -z "${keep_symbols_and_debug_frame}" ]; then + echo "--keep-symbols-mini-debug-info and --keep-symbols-and-debug-frame cannot be used together" + usage +fi + if [ ! -z "${symbols_to_keep}" -a ! -z "${keep_symbols}" ]; then echo "--keep-symbols and -k cannot be used together" usage @@ -195,6 +176,8 @@ elif [ ! -z "${symbols_to_keep}" ]; then do_strip_keep_symbol_list elif [ ! -z "${keep_mini_debug_info}" ]; then do_strip_keep_mini_debug_info +elif [ ! -z "${keep_symbols_and_debug_frame}" ]; then + do_strip_keep_symbols_and_debug_frame else do_strip fi @@ -210,18 +193,13 @@ fi rm -f "${outfile}" mv "${outfile}.tmp" "${outfile}" -if [ -z "${use_gnu_strip}" ]; then - USED_STRIP_OBJCOPY="${CLANG_BIN}/llvm-strip ${CLANG_BIN}/llvm-objcopy" -else - USED_STRIP_OBJCOPY="${CROSS_COMPILE}strip" -fi - cat <<EOF > "${depsfile}" ${outfile}: \ ${infile} \ - ${CROSS_COMPILE}nm \ ${CROSS_COMPILE}objcopy \ - ${CROSS_COMPILE}readelf \ - ${USED_STRIP_OBJCOPY} + ${CLANG_BIN}/llvm-nm \ + ${CLANG_BIN}/llvm-objcopy \ + ${CLANG_BIN}/llvm-readelf \ + ${CLANG_BIN}/llvm-strip EOF diff --git a/scripts/system-clang-format b/scripts/system-clang-format index 55773a29f..a7614d29f 100644 --- a/scripts/system-clang-format +++ b/scripts/system-clang-format @@ -1,9 +1,11 @@ BasedOnStyle: Google +Standard: Cpp11 AccessModifierOffset: -2 AllowShortFunctionsOnASingleLine: Inline ColumnLimit: 100 CommentPragmas: NOLINT:.* DerivePointerAlignment: false +IncludeBlocks: Preserve IndentWidth: 4 ContinuationIndentWidth: 8 PointerAlignment: Left diff --git a/scripts/system-clang-format-2 b/scripts/system-clang-format-2 index ede5d7e18..a4e23f860 100644 --- a/scripts/system-clang-format-2 +++ b/scripts/system-clang-format-2 @@ -1,8 +1,10 @@ BasedOnStyle: Google +Standard: Cpp11 AllowShortFunctionsOnASingleLine: Inline ColumnLimit: 100 CommentPragmas: NOLINT:.* DerivePointerAlignment: false +IncludeBlocks: Preserve IndentWidth: 2 PointerAlignment: Left TabWidth: 2 diff --git a/scripts/test_config_fixer.py b/scripts/test_config_fixer.py new file mode 100644 index 000000000..32d5b175e --- /dev/null +++ b/scripts/test_config_fixer.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A tool for modifying values in a test config.""" + +from __future__ import print_function + +import argparse +import sys +from xml.dom import minidom + + +from manifest import get_children_with_tag +from manifest import parse_manifest +from manifest import parse_test_config +from manifest import write_xml + + +def parse_args(): + """Parse commandline arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument('--manifest', default='', dest='manifest', + help=('AndroidManifest.xml that contains the original package name')) + parser.add_argument('--package-name', default='', dest='package_name', + help=('overwrite package fields in the test config')) + parser.add_argument('--test-file-name', default='', dest='test_file_name', + help=('overwrite test file name in the test config')) + parser.add_argument('input', help='input test config file') + parser.add_argument('output', help='output test config file') + return parser.parse_args() + + +def overwrite_package_name(test_config_doc, manifest_doc, package_name): + + manifest = parse_manifest(manifest_doc) + original_package = manifest.getAttribute('package') + + test_config = parse_test_config(test_config_doc) + tests = get_children_with_tag(test_config, 'test') + + for test in tests: + options = get_children_with_tag(test, 'option') + for option in options: + if option.getAttribute('name') == "package" and option.getAttribute('value') == original_package: + option.setAttribute('value', package_name) + +def overwrite_test_file_name(test_config_doc, test_file_name): + + test_config = parse_test_config(test_config_doc) + tests = get_children_with_tag(test_config, 'target_preparer') + + for test in tests: + if test.getAttribute('class') == "com.android.tradefed.targetprep.TestAppInstallSetup": + options = get_children_with_tag(test, 'option') + for option in options: + if option.getAttribute('name') == "test-file-name": + option.setAttribute('value', test_file_name) + +def main(): + """Program entry point.""" + try: + args = parse_args() + + doc = minidom.parse(args.input) + + if args.package_name: + if not args.manifest: + raise RuntimeError('--manifest flag required for --package-name') + manifest_doc = minidom.parse(args.manifest) + overwrite_package_name(doc, manifest_doc, args.package_name) + + if args.test_file_name: + overwrite_test_file_name(doc, args.test_file_name) + + with open(args.output, 'wb') as f: + write_xml(f, doc) + + # pylint: disable=broad-except + except Exception as err: + print('error: ' + str(err), file=sys.stderr) + sys.exit(-1) + +if __name__ == '__main__': + main() diff --git a/scripts/test_config_fixer_test.py b/scripts/test_config_fixer_test.py new file mode 100644 index 000000000..1272c6b33 --- /dev/null +++ b/scripts/test_config_fixer_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for test_config_fixer.py.""" + +import StringIO +import sys +import unittest +from xml.dom import minidom + +import test_config_fixer + +sys.dont_write_bytecode = True + + +class OverwritePackageNameTest(unittest.TestCase): + """ Unit tests for overwrite_package_name function """ + + manifest = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<manifest xmlns:android="http://schemas.android.com/apk/res/android"\n' + ' package="com.android.foo">\n' + ' <application>\n' + ' </application>\n' + '</manifest>\n') + + test_config = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <option name="test-suite-tag" value="apct"/>\n' + ' <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">\n' + ' <option name="package" value="%s"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="%s"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="%s"/>\n' + ' <option name="runtime-hint" value="15s"/>\n' + ' </test>\n' + '</configuration>\n') + + def test_all(self): + doc = minidom.parseString(self.test_config % ("com.android.foo", "com.android.foo", "com.android.bar")) + manifest = minidom.parseString(self.manifest) + + test_config_fixer.overwrite_package_name(doc, manifest, "com.soong.foo") + output = StringIO.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the matching package name in a test node should be updated. + expected = self.test_config % ("com.android.foo", "com.soong.foo", "com.android.bar") + self.assertEqual(expected, output.getvalue()) + + +class OverwriteTestFileNameTest(unittest.TestCase): + """ Unit tests for overwrite_test_file_name function """ + + test_config = ( + '<?xml version="1.0" encoding="utf-8"?>\n' + '<configuration description="Runs some tests.">\n' + ' <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup">\n' + ' <option name="test-file-name" value="%s"/>\n' + ' </target_preparer>\n' + ' <test class="com.android.tradefed.testtype.AndroidJUnitTest">\n' + ' <option name="package" value="com.android.foo"/>\n' + ' <option name="runtime-hint" value="20s"/>\n' + ' </test>\n' + '</configuration>\n') + + def test_all(self): + doc = minidom.parseString(self.test_config % ("foo.apk")) + + test_config_fixer.overwrite_test_file_name(doc, "bar.apk") + output = StringIO.StringIO() + test_config_fixer.write_xml(output, doc) + + # Only the matching package name in a test node should be updated. + expected = self.test_config % ("bar.apk") + self.assertEqual(expected, output.getvalue()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) |