cm: Add generic tools for extract-files / setup-makefiles scripts
* Instead of having to enumerate packages and do a bunch of tedious
shit, we now have a way to do it automatically based only on
the blob lists.
* Devices still need to implement a short script, but this library
handles all the core functions.
* Yes, we should probably be doing this in Python- Bash is
absolutely horrible at dealing with arrays and lists.
Change-Id: I73b30ff4531c4d1b33ac53db33abf62555f500b4
diff --git a/extract_utils.sh b/extract_utils.sh
new file mode 100644
index 0000000..a898a78
--- /dev/null
+++ b/extract_utils.sh
@@ -0,0 +1,626 @@
+#!/bin/bash
+#
+# Copyright (C) 2016 The CyanogenMod Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+PRODUCT_COPY_FILES_LIST=()
+PRODUCT_PACKAGES_LIST=()
+PACKAGE_LIST=()
+VENDOR_STATE=-1
+COMMON=-1
+
+#
+# setup_vendor
+#
+# $1: device name
+# $2: vendor name
+# $3: CM root directory
+# $4: is common device - optional, default to false
+# $5: cleanup - optional, default to true
+#
+# Must be called before any other functions can be used. This
+# sets up the internal state for a new vendor configuration.
+#
+function setup_vendor() {
+ local DEVICE="$1"
+ if [ -z "$DEVICE" ]; then
+ echo "\$DEVICE must be set before including this script!"
+ exit 1
+ fi
+
+ export VENDOR="$2"
+ if [ -z "$VENDOR" ]; then
+ echo "\$VENDOR must be set before including this script!"
+ exit 1
+ fi
+
+ export CM_ROOT="$3"
+ if [ ! -d "$CM_ROOT" ]; then
+ echo "\$CM_ROOT must be set and valid before including this script!"
+ exit 1
+ fi
+
+ export OUTDIR=vendor/"$VENDOR"/"$DEVICE"
+ if [ ! -d "$CM_ROOT/$OUTDIR" ]; then
+ mkdir -p "$CM_ROOT/$OUTDIR"
+ fi
+
+ export PRODUCTMK="$CM_ROOT"/"$OUTDIR"/"$DEVICE"-vendor.mk
+ export ANDROIDMK="$CM_ROOT"/"$OUTDIR"/Android.mk
+ export BOARDMK="$CM_ROOT"/"$OUTDIR"/BoardConfigVendor.mk
+
+ if [ "$4" == "true" ] || [ "$4" == "1" ]; then
+ COMMON=1
+ else
+ COMMON=0
+ fi
+
+ if [ "$5" == "true" ] || [ "$5" == "1" ]; then
+ VENDOR_STATE=1
+ else
+ VENDOR_STATE=0
+ fi
+}
+
+#
+# target_file:
+#
+# $1: colon delimited list
+#
+# Returns destination filename without args
+#
+function target_file() {
+ local LINE="$1"
+ local SPLIT=(${LINE//:/ })
+ local COUNT=${#SPLIT[@]}
+ if [ "$COUNT" -gt "1" ]; then
+ if [[ "${SPLIT[1]}" =~ .*/.* ]]; then
+ printf '%s\n' "${SPLIT[1]}"
+ return 0
+ fi
+ fi
+ printf '%s\n' "${SPLIT[0]}"
+}
+
+#
+# target_args:
+#
+# $1: colon delimited list
+#
+# Returns optional arguments (last value) for given target
+#
+function target_args() {
+ local LINE="$1"
+ local SPLIT=(${LINE//:/ })
+ local COUNT=${#SPLIT[@]}
+ if [ "$COUNT" -gt "1" ]; then
+ if [[ ! "${SPLIT[$COUNT-1]}" =~ .*/.* ]]; then
+ printf '%s\n' "${SPLIT[$COUNT-1]}"
+ fi
+ fi
+}
+
+#
+# prefix_match:
+#
+# $1: the prefix to match on
+#
+# Internal function which loops thru the packages list and returns a new
+# list containing the matched files with the prefix stripped away.
+#
+function prefix_match() {
+ local PREFIX="$1"
+ for FILE in "${PRODUCT_PACKAGES_LIST[@]}"; do
+ if [[ "$FILE" =~ ^"$PREFIX" ]]; then
+ printf '%s\n' "${FILE#$PREFIX}"
+ fi
+ done
+}
+
+#
+# write_product_copy_files:
+#
+# Creates the PRODUCT_COPY_FILES section in the product makefile for all
+# items in the list which do not start with a dash (-).
+#
+function write_product_copy_files() {
+ local COUNT=${#PRODUCT_COPY_FILES_LIST[@]}
+ local TARGET=
+ local FILE=
+ local LINEEND=
+
+ if [ "$COUNT" -eq "0" ]; then
+ return 0
+ fi
+
+ printf '%s\n' "PRODUCT_COPY_FILES += \\" >> "$PRODUCTMK"
+ for (( i=1; i<COUNT+1; i++ )); do
+ FILE="${PRODUCT_COPY_FILES_LIST[$i-1]}"
+ LINEEND=" \\"
+ if [ "$i" -eq "$COUNT" ]; then
+ LINEEND=""
+ fi
+
+ TARGET=$(target_file "$FILE")
+ printf ' %s/proprietary/%s:system/%s%s\n' \
+ "$OUTDIR" "$TARGET" "$TARGET" "$LINEEND" >> "$PRODUCTMK"
+ done
+ return 0
+}
+
+#
+# write_packages:
+#
+# $1: The LOCAL_MODULE_CLASS for the given module list
+# $2: "true" if this package is part of the vendor/ path
+# $3: "true" if this is a privileged module (only valid for APPS)
+# $4: The multilib mode, "32", "64", "both", or "none"
+# $5: Name of the array holding the target list
+#
+# Internal function which writes out the BUILD_PREBUILT stanzas
+# for all modules in the list. This is called by write_product_packages
+# after the modules are categorized.
+#
+function write_packages() {
+
+ local CLASS="$1"
+ local VENDOR_PKG="$2"
+ local PRIVILEGED="$3"
+ local MULTILIB="$4"
+
+ # Yes, this is a horrible hack - we create a new array using indirection
+ local ARR_NAME="$5[@]"
+ local FILELIST=("${!ARR_NAME}")
+
+ local FILE=
+ local ARGS=
+ local BASENAME=
+ local EXTENSION=
+ local PKGNAME=
+ local SRC=
+
+ for P in "${FILELIST[@]}"; do
+ FILE=$(target_file "$P")
+ ARGS=$(target_args "$P")
+
+ BASENAME=$(basename "$FILE")
+ EXTENSION=${BASENAME##*.}
+ PKGNAME=${BASENAME%.*}
+
+ # Add to final package list
+ PACKAGE_LIST+=("$PKGNAME")
+
+ SRC="proprietary"
+ if [ "$VENDOR_PKG" = "true" ]; then
+ SRC+="/vendor"
+ fi
+
+ printf 'include $(CLEAR_VARS)\n'
+ printf 'LOCAL_MODULE := %s\n' "$PKGNAME"
+ printf 'LOCAL_MODULE_OWNER := %s\n' "$VENDOR"
+ if [ "$CLASS" = "SHARED_LIBRARIES" ]; then
+ if [ "$MULTILIB" = "both" ]; then
+ printf 'LOCAL_SRC_FILES_64 := %s/lib64/%s\n' "$SRC" "$FILE"
+ printf 'LOCAL_SRC_FILES_32 := %s/lib/%s\n' "$SRC" "$FILE"
+ #if [ "$VENDOR_PKG" = "true" ]; then
+ # echo "LOCAL_MODULE_PATH_64 := \$(TARGET_OUT_VENDOR_SHARED_LIBRARIES)"
+ # echo "LOCAL_MODULE_PATH_32 := \$(2ND_TARGET_OUT_VENDOR_SHARED_LIBRARIES)"
+ #else
+ # echo "LOCAL_MODULE_PATH_64 := \$(TARGET_OUT_SHARED_LIBRARIES)"
+ # echo "LOCAL_MODULE_PATH_32 := \$(2ND_TARGET_OUT_SHARED_LIBRARIES)"
+ #fi
+ elif [ "$MULTILIB" = "64" ]; then
+ printf 'LOCAL_SRC_FILES := %s/lib64/%s\n' "$SRC" "$FILE"
+ else
+ printf 'LOCAL_SRC_FILES := %s/lib/%s\n' "$SRC" "$FILE"
+ fi
+ if [ "$MULTILIB" != "none" ]; then
+ printf 'LOCAL_MULTILIB := %s\n' "$MULTILIB"
+ fi
+ elif [ "$CLASS" = "APPS" ]; then
+ if [ -z "$ARGS" ]; then
+ if [ "$PRIVILEGED" = "true" ]; then
+ SRC="$SRC/priv-app"
+ else
+ SRC="$SRC/app"
+ fi
+ fi
+ printf 'LOCAL_SRC_FILES := %s/%s\n' "$SRC" "$FILE"
+ local CERT=platform
+ if [ ! -z "$ARGS" ]; then
+ CERT="$ARGS"
+ fi
+ printf 'LOCAL_CERTIFICATE := %s\n' "$CERT"
+ elif [ "$CLASS" = "JAVA_LIBRARIES" ]; then
+ printf 'LOCAL_SRC_FILES := %s/framework/%s\n' "$SRC" "$FILE"
+ elif [ "$CLASS" = "ETC" ]; then
+ printf 'LOCAL_SRC_FILES := %s/etc/%s\n' "$SRC" "$FILE"
+ elif [ "$CLASS" = "EXECUTABLES" ]; then
+ printf 'LOCAL_SRC_FILES := %s/bin/%s\n' "$SRC" "$FILE"
+ else
+ printf 'LOCAL_SRC_FILES := %s/%s' "$SRC" "$FILE"
+ fi
+ printf 'LOCAL_MODULE_TAGS := optional\n'
+ printf 'LOCAL_MODULE_CLASS := %s\n' "$CLASS"
+ printf 'LOCAL_MODULE_SUFFIX := .%s\n' "$EXTENSION"
+ if [ "$PRIVILEGED" = "true" ]; then
+ printf 'LOCAL_PRIVILEGED_MODULE := true\n'
+ fi
+ if [ "$VENDOR_PKG" = "true" ]; then
+ printf 'LOCAL_PROPRIETARY_MODULE := true\n'
+ fi
+ printf 'include $(BUILD_PREBUILT)\n\n'
+ done
+}
+
+#
+# write_product_packages:
+#
+# This function will create BUILD_PREBUILT entries in the
+# Android.mk and associated PRODUCT_PACKAGES list in the
+# product makefile for all files in the blob list which
+# start with a single dash (-) character.
+#
+function write_product_packages() {
+ PACKAGE_LIST=()
+
+ local COUNT=${#PRODUCT_PACKAGES_LIST[@]}
+
+ if [ "$COUNT" = "0" ]; then
+ return 0
+ fi
+
+ # Figure out what's 32-bit, what's 64-bit, and what's multilib
+ # I really should not be doing this in bash due to shitty array passing :(
+ local T_LIB32=( $(prefix_match "lib/") )
+ local T_LIB64=( $(prefix_match "lib64/") )
+ local MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${T_LIB64[@]}")) )
+ local LIB32=( $(comm -23 <(printf '%s\n' "${T_LIB32[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) )
+ local LIB64=( $(comm -13 <(printf '%s\n' "${T_LIB64[@]}") <(printf '%s\n' "${MULTILIBS[@]}")) )
+
+ if [ "${#MULTILIBS[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "false" "false" "both" "MULTILIBS" >> "$ANDROIDMK"
+ fi
+ if [ "${#LIB32[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "false" "false" "32" "LIB32" >> "$ANDROIDMK"
+ fi
+ if [ ! "${#LIB64[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "false" "false" "64" "LIB64" >> "$ANDROIDMK"
+ fi
+
+ local T_V_LIB32=( $(prefix_match "vendor/lib/") )
+ local T_V_LIB64=( $(prefix_match "vendor/lib64/") )
+ local V_MULTILIBS=( $(comm -12 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${T_V_LIB64[@]}")) )
+ local V_LIB32=( $(comm -23 <(printf '%s\n' "${T_V_LIB32[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) )
+ local V_LIB64=( $(comm -13 <(printf '%s\n' "${T_V_LIB64[@]}") <(printf '%s\n' "${V_MULTILIBS[@]}")) )
+
+ if [ "${#V_MULTILIBS[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "true" "false" "both" "V_MULTILIBS" >> "$ANDROIDMK"
+ fi
+ if [ "${#V_LIB32[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "true" "false" "32" "V_LIB32" >> "$ANDROIDMK"
+ fi
+ if [ "${#V_LIB64[@]}" -gt "0" ]; then
+ write_packages "SHARED_LIBRARIES" "true" "false" "64" "V_LIB64" >> "$ANDROIDMK"
+ fi
+
+ # Apps
+ local APPS=( $(prefix_match "app/") )
+ if [ "${#APPS[@]}" -gt "0" ]; then
+ write_packages "APPS" "false" "false" "none" "APPS" >> "$ANDROIDMK"
+ fi
+ local PRIV_APPS=( $(prefix_match "priv-app/") )
+ if [ "${#PRIV_APPS[@]}" -gt "0" ]; then
+ write_packages "APPS" "false" "true" "none" "PRIV_APPS" >> "$ANDROIDMK"
+ fi
+ local V_APPS=( $(prefix_match "vendor/app/") )
+ if [ "${#V_APPS[@]}" -gt "0" ]; then
+ write_packages "APPS" "true" "false" "none" "V_APPS" >> "$ANDROIDMK"
+ fi
+ local V_PRIV_APPS=( $(prefix_match "vendor/priv-app/") )
+ if [ "${#V_PRIV_APPS[@]}" -gt "0" ]; then
+ write_packages "APPS" "true" "true" "none" "V_PRIV_APPS" >> "$ANDROIDMK"
+ fi
+
+ # Framework
+ local FRAMEWORK=( $(prefix_match "framework/") )
+ if [ "${#FRAMEWORK[@]}" -gt "0" ]; then
+ write_packages "JAVA_LIBRARIES" "false" "false" "none" "FRAMEWORK" >> "$ANDROIDMK"
+ fi
+
+ # Etc
+ local ETC=( $(prefix_match "etc/") )
+ if [ "${#ETC[@]}" -gt "0" ]; then
+ write_packages "ETC" "false" "false" "none" "ETC" >> "$ANDROIDMK"
+ fi
+ local V_ETC=( $(prefix_match "vendor/etc/") )
+ if [ "${#V_ETC[@]}" -gt "0" ]; then
+ write_packages "ETC" "true" "false" "none" "V_ETC" >> "$ANDROIDMK"
+ fi
+
+ # Executables
+ local BIN=( $(prefix_match "bin/") )
+ if [ "${#BIN[@]}" -gt "0" ]; then
+ write_packages "EXECUTABLES" "false" "false" "none" "BIN" >> "$ANDROIDMK"
+ fi
+ local V_BIN=( $(prefix_match "vendor/bin/") )
+ if [ "${#V_BIN[@]}" -gt "0" ]; then
+ write_packages "EXECUTABLES" "true" "false" "none" "V_BIN" >> "$ANDROIDMK"
+ fi
+
+ # Actually write out the final PRODUCT_PACKAGES list
+ local PACKAGE_COUNT=${#PACKAGE_LIST[@]}
+
+ if [ "$PACKAGE_COUNT" -eq "0" ]; then
+ return 0
+ fi
+
+ printf '\n%s\n' "PRODUCT_PACKAGES += \\" >> "$PRODUCTMK"
+ for (( i=1; i<PACKAGE_COUNT+1; i++ )); do
+ local LINEEND=" \\"
+ if [ "$i" -eq "$PACKAGE_COUNT" ]; then
+ LINEEND=""
+ fi
+ printf ' %s%s\n' "${PACKAGE_LIST[$i-1]}" "$LINEEND" >> "$PRODUCTMK"
+ done
+}
+
+#
+# write_header:
+#
+# $1: file which will be written to
+#
+# writes out the copyright header with the current year.
+# note that this is not an append operation, and should
+# be executed first!
+#
+function write_header() {
+ YEAR=$(date +"%Y")
+
+ [ "$COMMON" -eq 1 ] && local DEVICE="$DEVICE_COMMON"
+
+ cat << EOF > $1
+# Copyright (C) $YEAR The CyanogenMod Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is generated by device/$VENDOR/$DEVICE/setup-makefiles.sh
+
+EOF
+}
+
+#
+# write_headers:
+#
+# $1: devices falling under common to be added to guard - optional
+#
+# Calls write_header for each of the makefiles and creates
+# the initial path declaration and device guard for the
+# Android.mk
+#
+function write_headers() {
+ write_header "$ANDROIDMK"
+ cat << EOF >> "$ANDROIDMK"
+LOCAL_PATH := \$(call my-dir)
+
+EOF
+ if [ "$COMMON" -ne 1 ]; then
+ cat << EOF >> "$ANDROIDMK"
+ifeq (\$(TARGET_DEVICE),$DEVICE)
+
+EOF
+ else
+ if [ -z "$1" ]; then
+ echo "Argument with devices to be added to guard must be set!"
+ exit 1
+ fi
+ cat << EOF >> "$ANDROIDMK"
+ifneq (\$(filter $1,\$(TARGET_DEVICE)),)
+
+EOF
+ fi
+
+ write_header "$BOARDMK"
+ write_header "$PRODUCTMK"
+}
+
+#
+# write_footers:
+#
+# Closes the inital guard and any other finalization tasks. Must
+# be called as the final step.
+#
+function write_footers() {
+ cat << EOF >> "$ANDROIDMK"
+endif
+EOF
+}
+
+# Return success if adb is up and not in recovery
+function _adb_connected {
+ {
+ if [[ "$(adb get-state)" == device &&
+ "$(adb shell test -e /sbin/recovery; echo $?)" == 0 ]]
+ then
+ return 0
+ fi
+ } 2>/dev/null
+
+ return 1
+};
+
+#
+# parse_file_list
+#
+function parse_file_list() {
+ if [ ! -e "$1" ]; then
+ echo "$1 does not exist!"
+ exit 1
+ fi
+
+ PRODUCT_PACKAGES_LIST=()
+ PRODUCT_COPY_FILES_LIST=()
+
+ while read -r line; do
+ if [ -z "$line" ]; then continue; fi
+
+ # if line starts with a dash, it needs to be packaged
+ if [[ "$line" =~ ^- ]]; then
+ PRODUCT_PACKAGES_LIST+=("${line#-}")
+ else
+ PRODUCT_COPY_FILES_LIST+=("$line")
+ fi
+
+ done < <(egrep -v '(^#|^[[:space:]]*$)' "$1" | sort | uniq)
+}
+
+#
+# write_makefiles:
+#
+# $1: file containing the list of items to extract
+#
+# Calls write_product_copy_files and write_product_packages on
+# the given file and appends to the Android.mk as well as
+# the product makefile.
+#
+function write_makefiles() {
+ if [ ! -e "$1" ]; then
+ echo "$1 does not exist!"
+ exit 1
+ fi
+ parse_file_list "$1"
+ write_product_copy_files
+ write_product_packages
+}
+
+#
+# init_adb_connection:
+#
+# Starts adb server and waits for the device
+#
+function init_adb_connection() {
+ adb start-server # Prevent unexpected starting server message from adb get-state in the next line
+ if ! _adb_connected; then
+ echo "No device is online. Waiting for one..."
+ echo "Please connect USB and/or enable USB debugging"
+ until _adb_connected; do
+ sleep 1
+ done
+ echo "Device Found."
+ fi
+
+ # Retrieve IP and PORT info if we're using a TCP connection
+ TCPIPPORT=$(adb devices | egrep '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+[^0-9]+' \
+ | head -1 | awk '{print $1}')
+ adb root &> /dev/null
+ sleep 0.3
+ if [ -n "$TCPIPPORT" ]; then
+ # adb root just killed our connection
+ # so reconnect...
+ adb connect "$TCPIPPORT"
+ fi
+ adb wait-for-device &> /dev/null
+ sleep 0.3
+}
+
+#
+# extract:
+#
+# $1: file containing the list of items to extract
+# $2: path to extracted system folder, or "adb" to extract from device
+#
+function extract() {
+ if [ ! -e "$1" ]; then
+ echo "$1 does not exist!"
+ exit 1
+ fi
+
+ if [ -z "$OUTDIR" ]; then
+ echo "Output dir not set!"
+ exit 1
+ fi
+
+ parse_file_list "$1"
+
+ # Allow failing, so we can try $DEST and/or $FILE
+ set +e
+
+ local FILELIST=( ${PRODUCT_COPY_FILES_LIST[@]} ${PRODUCT_PACKAGES_LIST[@]} )
+ local COUNT=${#FILELIST[@]}
+ local FILE=
+ local DEST=
+ local SRC="$2"
+ local OUTPUT_DIR="$CM_ROOT"/"$OUTDIR"/proprietary
+ local DIR=
+
+ if [ "$SRC" = "adb" ]; then
+ init_adb_connection
+ fi
+
+ if [ "$VENDOR_STATE" -eq "0" ]; then
+ echo "Cleaning output directory ($OUTPUT_DIR).."
+ rm -rf "${OUTPUT_DIR:?}/"*
+ VENDOR_STATE=1
+ fi
+
+ echo "Extracting $COUNT files in $1 from $SRC:"
+
+ for (( i=1; i<COUNT+1; i++ )); do
+ local SPLIT=(${FILELIST[$i-1]//:/ })
+ local FILE="${SPLIT[0]#-}"
+ local DEST="${SPLIT[1]}"
+ if [ -z "$DEST" ]; then
+ DEST="$FILE"
+ fi
+ if [ "$SRC" = "adb" ]; then
+ printf ' - %s .. ' "/system/$FILE"
+ else
+ printf ' - %s \n' "/system/$FILE"
+ fi
+ DIR=$(dirname "$DEST")
+ if [ ! -d "$OUTPUT_DIR/$DIR" ]; then
+ mkdir -p "$OUTPUT_DIR/$DIR"
+ fi
+ if [ "$SRC" = "adb" ]; then
+ # Try CM target first
+ adb pull "/system/$DEST" "$OUTPUT_DIR/$DEST"
+ # if file does not exist try OEM target
+ if [ "$?" != "0" ]; then
+ adb pull "/system/$FILE" "$OUTPUT_DIR/$DEST"
+ fi
+ else
+ # Try OEM target first
+ cp "$SRC/system/$FILE" "$OUTPUT_DIR/$DEST"
+ # if file does not exist try CM target
+ if [ "$?" != "0" ]; then
+ cp "$SRC/system/$DEST" "$OUTPUT_DIR/$DEST"
+ fi
+ fi
+ chmod 644 "$OUTPUT_DIR/$DEST"
+ done
+
+ # Don't allow failing
+ set -e
+}