diff options
author | 2025-01-09 13:20:42 -0800 | |
---|---|---|
committer | 2025-01-14 10:26:57 -0800 | |
commit | 6434c0301d320233bdc7da77eb800f5e87de07f3 (patch) | |
tree | 49f78664c09070e6bfe38d0da1be4b896994e49d | |
parent | 816c7b4c30dc1da4ee74610fbef934aee3c216fe (diff) |
[ravenwood] Add a tool to convert text policy to annotations
- Add a new command "ravenhelper", which performs different functions
depending on the first argument ("subcommand") just like the git command.
- For now, it has one subcommand "pta" -- policy-to-annotation, which
reads the policy file and add corresponding annotations to the java
source files.
- This will also add classes to ravenwood-annotation-allowed-classes.txt
as needed.
- Use the f/b/r/scripts/pta-framework.sh script to run it.
- For safety and easier testing/debugging, the command actually won't
directly update any files. Instead, it'll generate a shell script which
does so, using sed(1).
Here's an example script: https://paste.googleplex.com/6561708886458368?raw
Bug: 388607679
Flag: EXEMPT host test change only
Test: ./frameworks/base/ravenwood/scripts/pta-framework.sh, and
manually check /tmp/pta.sh
Change-Id: I5dcf078ca47eb373bcaf435ef133335f44bce6b3
13 files changed, 1598 insertions, 56 deletions
diff --git a/ravenwood/scripts/pta-framework.sh b/ravenwood/scripts/pta-framework.sh new file mode 100755 index 000000000000..224ab59e2e09 --- /dev/null +++ b/ravenwood/scripts/pta-framework.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Use "ravehleper pta" to create a shell script which: +# - Reads the text "policy" files +# - Convert to java annotations (using sed) +# + +set -e + + +# Uncomment it to always build ravenhelper (slow) +# ${BUILD_CMD:-m} ravenhelper + +# Get the target directory. Default to $ANDROID_BUILD_TOP. +TARGET_DIR="${TARGET_DIR:-${ANDROID_BUILD_TOP?\$ANDROID_BUILD_TOP must be set}}" + +echo "Target dir=$TARGET_DIR" + +cd "$TARGET_DIR" + +# Add -v or -d as needed. +extra_args="$@" + +OUT_SCRIPT="${OUT_SCRIPT:-/tmp/pta.sh}" + +rm -f "$OUT_SCRIPT" + +# If you want to run on other files, run this script with the following +# env vars predefined. + +POLICIES="${POLICIES:- +frameworks/base/ravenwood/texts/ravenwood-common-policies.txt +frameworks/base/ravenwood/texts/ravenwood-framework-policies.txt +}" + +SOURCES="${SOURCES:- +frameworks/base/core/java/ +frameworks/base/graphics/java/ +}" + +AAC="${AAC:-frameworks/base/ravenwood/texts/ravenwood-annotation-allowed-classes.txt}" + +with_flag() { + local flag="$1" + shift + + for arg in "$@"; do + echo "$flag $arg" + done +} + +run() { + echo "Running: $*" + "$@" +} + +run_pta() { + local extra_args="$@" + + run ${RAVENHELPER_CMD:-ravenhelper pta} \ + --output-script $OUT_SCRIPT \ + --annotation-allowed-classes-file $AAC \ + $(with_flag --policy-override-file $POLICIES) \ + $(with_flag --src $SOURCES) \ + $extra_args + + if ! [[ -f $OUT_SCRIPT ]] ; then + # no operations generated. + exit 0 + fi + + echo + echo "Created script at $OUT_SCRIPT. Run it with: sh $OUT_SCRIPT" + return 0 +} + +run_pta "$extra_args"
\ No newline at end of file diff --git a/ravenwood/texts/ravenwood-common-policies.txt b/ravenwood/texts/ravenwood-common-policies.txt index 83c31512eb70..fd4ea6cf40c2 100644 --- a/ravenwood/texts/ravenwood-common-policies.txt +++ b/ravenwood/texts/ravenwood-common-policies.txt @@ -1,5 +1,8 @@ # Ravenwood "policy" that should apply to all code. +# The "no-pta" marker is used to exclude the lines from "ravenhelper pta", +# which tries to convert policies to annotations. + # Keep all AIDL interfaces class :aidl keepclass @@ -13,8 +16,8 @@ class :sysprops keepclass class :r keepclass # Support APIs not available in standard JRE -class java.io.FileDescriptor keep +class java.io.FileDescriptor # no-pta method getInt$ @com.android.ravenwood.RavenwoodJdkPatch.getInt$ method setInt$ @com.android.ravenwood.RavenwoodJdkPatch.setInt$ -class java.util.LinkedHashMap keep +class java.util.LinkedHashMap # no-pta method eldest @com.android.ravenwood.RavenwoodJdkPatch.eldest diff --git a/ravenwood/texts/ravenwood-framework-policies.txt b/ravenwood/texts/ravenwood-framework-policies.txt index 26b6fe3d82ad..4033782c607e 100644 --- a/ravenwood/texts/ravenwood-framework-policies.txt +++ b/ravenwood/texts/ravenwood-framework-policies.txt @@ -1,62 +1,65 @@ # Ravenwood "policy" file for framework-minus-apex. +# The "no-pta" marker is used to exclude the lines from "ravenhelper pta", +# which tries to convert policies to annotations. + # To avoid VerifyError on nano proto files (b/324063814), we rename nano proto classes. # Note: The "rename" directive must use slashes (/) as a package name separator. rename com/.*/nano/ devicenano/ rename android/.*/nano/ devicenano/ # StatsD auto-generated -class com.android.internal.util.FrameworkStatsLog keepclass +class com.android.internal.util.FrameworkStatsLog keepclass # no-pta # Exported to Mainline modules; cannot use annotations -class com.android.internal.util.FastXmlSerializer keepclass -class com.android.internal.util.FileRotator keepclass -class com.android.internal.util.HexDump keepclass -class com.android.internal.util.IndentingPrintWriter keepclass -class com.android.internal.util.LocalLog keepclass -class com.android.internal.util.MessageUtils keepclass -class com.android.internal.util.TokenBucket keepclass -class android.os.HandlerExecutor keepclass -class android.util.BackupUtils keepclass -class android.util.IndentingPrintWriter keepclass -class android.util.LocalLog keepclass -class android.util.Pair keepclass -class android.util.Rational keepclass +class com.android.internal.util.FastXmlSerializer keepclass # no-pta +class com.android.internal.util.FileRotator keepclass # no-pta +class com.android.internal.util.HexDump keepclass # no-pta +class com.android.internal.util.IndentingPrintWriter keepclass # no-pta +class com.android.internal.util.LocalLog keepclass # no-pta +class com.android.internal.util.MessageUtils keepclass # no-pta +class com.android.internal.util.TokenBucket keepclass # no-pta +class android.os.HandlerExecutor keepclass # no-pta +class android.util.BackupUtils keepclass # no-pta +class android.util.IndentingPrintWriter keepclass # no-pta +class android.util.LocalLog keepclass # no-pta +class android.util.Pair keepclass # no-pta +class android.util.Rational keepclass # no-pta # From modules-utils; cannot use annotations -class com.android.internal.util.Preconditions keepclass -class com.android.internal.logging.InstanceId keepclass -class com.android.internal.logging.InstanceIdSequence keepclass -class com.android.internal.logging.UiEvent keepclass -class com.android.internal.logging.UiEventLogger keepclass +class com.android.internal.util.Preconditions keepclass # no-pta +class com.android.internal.logging.InstanceId keepclass # no-pta +class com.android.internal.logging.InstanceIdSequence keepclass # no-pta +class com.android.internal.logging.UiEvent keepclass # no-pta +class com.android.internal.logging.UiEventLogger keepclass # no-pta # From modules-utils; cannot use annotations -class com.android.modules.utils.BinaryXmlPullParser keepclass -class com.android.modules.utils.BinaryXmlSerializer keepclass -class com.android.modules.utils.FastDataInput keepclass -class com.android.modules.utils.FastDataOutput keepclass -class com.android.modules.utils.ModifiedUtf8 keepclass -class com.android.modules.utils.TypedXmlPullParser keepclass -class com.android.modules.utils.TypedXmlSerializer keepclass +class com.android.modules.utils.BinaryXmlPullParser keepclass # no-pta +class com.android.modules.utils.BinaryXmlSerializer keepclass # no-pta +class com.android.modules.utils.FastDataInput keepclass # no-pta +class com.android.modules.utils.FastDataOutput keepclass # no-pta +class com.android.modules.utils.ModifiedUtf8 keepclass # no-pta +class com.android.modules.utils.TypedXmlPullParser keepclass # no-pta +class com.android.modules.utils.TypedXmlSerializer keepclass # no-pta # Uri -class android.net.Uri keepclass -class android.net.UriCodec keepclass +class android.net.Uri keepclass # no-pta +class android.net.UriCodec keepclass # no-pta # Telephony -class android.telephony.PinResult keepclass +class android.telephony.PinResult keepclass # no-pta # Just enough to support mocking, no further functionality -class android.content.BroadcastReceiver keep +class android.content.BroadcastReceiver keep # no-pta method <init> ()V keep -class android.content.Context keep +class android.content.Context keep # no-pta method <init> ()V keep - method getSystemService (Ljava/lang/Class;)Ljava/lang/Object; keep -class android.content.pm.PackageManager + method getSystemService (Ljava/lang/Class;)Ljava/lang/Object; keep # no-pta +class android.content.pm.PackageManager # no-pta method <init> ()V keep -class android.text.ClipboardManager keep +class android.text.ClipboardManager keep # no-pta method <init> ()V keep # Just enough to allow ResourcesManager to run -class android.hardware.display.DisplayManagerGlobal keep +class android.hardware.display.DisplayManagerGlobal keep # no-pta method getInstance ()Landroid/hardware/display/DisplayManagerGlobal; ignore diff --git a/ravenwood/texts/ravenwood-services-policies.txt b/ravenwood/texts/ravenwood-services-policies.txt index 530e5c8f5986..e3be9afdba5c 100644 --- a/ravenwood/texts/ravenwood-services-policies.txt +++ b/ravenwood/texts/ravenwood-services-policies.txt @@ -1,12 +1,15 @@ # Ravenwood "policy" file for services.core. +# The "no-pta" marker is used to exclude the lines from "ravenhelper pta", +# which tries to convert policies to annotations. + # Auto-generated from XSD -class com.android.server.compat.config.Change keepclass -class com.android.server.compat.config.Config keepclass -class com.android.server.compat.config.XmlParser keepclass -class com.android.server.compat.overrides.ChangeOverrides keepclass -class com.android.server.compat.overrides.OverrideValue keepclass -class com.android.server.compat.overrides.Overrides keepclass -class com.android.server.compat.overrides.RawOverrideValue keepclass -class com.android.server.compat.overrides.XmlParser keepclass -class com.android.server.compat.overrides.XmlWriter keepclass
\ No newline at end of file +class com.android.server.compat.config.Change keepclass # no-pta +class com.android.server.compat.config.Config keepclass # no-pta +class com.android.server.compat.config.XmlParser keepclass # no-pta +class com.android.server.compat.overrides.ChangeOverrides keepclass # no-pta +class com.android.server.compat.overrides.OverrideValue keepclass # no-pta +class com.android.server.compat.overrides.Overrides keepclass # no-pta +class com.android.server.compat.overrides.RawOverrideValue keepclass # no-pta +class com.android.server.compat.overrides.XmlParser keepclass # no-pta +class com.android.server.compat.overrides.XmlWriter keepclass # no-pta
\ No newline at end of file diff --git a/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt b/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt index c5500831e21a..9782f3d0f591 100644 --- a/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt +++ b/ravenwood/tools/hoststubgen/src/com/android/hoststubgen/filters/TextFileFilterPolicyParser.kt @@ -69,9 +69,9 @@ interface PolicyFileProcessor { fun onRename(pattern: Pattern, prefix: String) /** "class" directive. */ - fun onSimpleClassStart(className: String) + fun onClassStart(className: String) fun onSimpleClassPolicy(className: String, policy: FilterPolicyWithReason) - fun onSimpleClassEnd(className: String) + fun onClassEnd(className: String) fun onSubClassPolicy(superClassName: String, policy: FilterPolicyWithReason) fun onRedirectionClass(fromClassName: String, toClassName: String) @@ -162,10 +162,10 @@ class TextFileFilterPolicyBuilder( ) } - override fun onSimpleClassStart(className: String) { + override fun onClassStart(className: String) { } - override fun onSimpleClassEnd(className: String) { + override fun onClassEnd(className: String) { } override fun onSimpleClassPolicy(className: String, policy: FilterPolicyWithReason) { @@ -273,20 +273,23 @@ class TextFileFilterPolicyParser { private var rFilePolicy: FilterPolicyWithReason? = null /** Name of the file that's currently being processed. */ - var filename: String? = null + var filename: String = "" private set /** 1-based line number in the current file */ var lineNumber = -1 private set + /** Current line */ + var currentLineText = "" + private set + /** * Parse a given "policy" file. */ fun parse(reader: Reader, inputName: String, processor: PolicyFileProcessor) { filename = inputName - log.i("Parsing text policy file $inputName ...") this.processor = processor BufferedReader(reader).use { rd -> lineNumber = 0 @@ -297,6 +300,7 @@ class TextFileFilterPolicyParser { break } lineNumber++ + currentLineText = line line = normalizeTextLine(line) // Remove comment and trim. if (line.isEmpty()) { continue @@ -312,7 +316,7 @@ class TextFileFilterPolicyParser { private fun finishLastClass() { currentClassName?.let { className -> - processor.onSimpleClassEnd(className) + processor.onClassEnd(className) currentClassName = null } } @@ -416,7 +420,7 @@ class TextFileFilterPolicyParser { if (fields.size <= 1) { throw ParseException("Class ('c') expects 1 or 2 fields.") } - val className = fields[1] + val className = fields[1].toHumanReadableClassName() // superClass is set when the class name starts with a "*". val superClass = resolveExtendingClass(className) @@ -436,6 +440,8 @@ class TextFileFilterPolicyParser { // It's a redirection class. val toClass = policyStr.substring(1) + currentClassName = className + processor.onClassStart(className) processor.onRedirectionClass(className, toClass) } else if (policyStr.startsWith("~")) { if (classType != SpecialClass.NotSpecial) { @@ -447,6 +453,8 @@ class TextFileFilterPolicyParser { // It's a class-load hook val callback = policyStr.substring(1) + currentClassName = className + processor.onClassStart(className) processor.onClassLoadHook(className, callback) } else { // Special case: if it's a class directive with no policy, then it encloses @@ -455,7 +463,6 @@ class TextFileFilterPolicyParser { if (policyStr == "") { if (classType == SpecialClass.NotSpecial && superClass == null) { currentClassName = className - processor.onSimpleClassStart(className) return } throw ParseException("Special class or subclass directive must have a policy") @@ -471,7 +478,7 @@ class TextFileFilterPolicyParser { // TODO: Duplicate check, etc if (superClass == null) { currentClassName = className - processor.onSimpleClassStart(className) + processor.onClassStart(className) processor.onSimpleClassPolicy(className, policy.withReason(FILTER_REASON)) } else { processor.onSubClassPolicy( diff --git a/ravenwood/tools/ravenhelper/Android.bp b/ravenwood/tools/ravenhelper/Android.bp new file mode 100644 index 000000000000..a7ee4684506e --- /dev/null +++ b/ravenwood/tools/ravenhelper/Android.bp @@ -0,0 +1,26 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "frameworks_base_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["frameworks_base_license"], +} + +java_binary_host { + name: "ravenhelper", + main_class: "com.android.platform.test.ravenwood.ravenhelper.RavenHelperMain", + srcs: ["src/**/*.kt"], + static_libs: [ + "guava", + "hoststubgen-lib", + "junit", + "metalava-gradle-plugin-deps", // Get lint/PSI related classes from here. + "ow2-asm", + "ow2-asm-analysis", + "ow2-asm-commons", + "ow2-asm-tree", + "ow2-asm-util", + ], + visibility: ["//visibility:public"], +} diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/RavenHelperMain.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/RavenHelperMain.kt new file mode 100644 index 000000000000..e6efbf6c5223 --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/RavenHelperMain.kt @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JvmName("RavenHelperMain") +package com.android.platform.test.ravenwood.ravenhelper + +/* + * This file contains the main entry point for the "ravenhelper" command, which + * contains subcommands to help various tasks. + */ + +import com.android.hoststubgen.GeneralUserErrorException +import com.android.hoststubgen.LogLevel +import com.android.hoststubgen.executableName +import com.android.hoststubgen.log +import com.android.hoststubgen.runMainWithBoilerplate +import com.android.platform.test.ravenwood.ravenhelper.policytoannot.PtaProcessor + +interface SubcommandHandler { + fun handle(args: List<String>) +} + +fun usage() { + System.out.println(""" + Usage: + ravenhelper SUBCOMMAND options... + + Subcommands: + pta: "policy-to-annotations" Convert policy file to annotations. + (See the pta-framework.sh script for usage.) 1 + + """.trimIndent()) +} + +fun main(args: Array<String>) { + executableName = "RavenHelper" + log.setConsoleLogLevel(LogLevel.Info) + + runMainWithBoilerplate { + log.i("$executableName started") + + if (args.size == 0) { + usage() + return + } + + // Find the subcommand handler. + val subcommand = args[0] + val handler: SubcommandHandler = when (subcommand) { + "pta" -> PtaProcessor() + else -> { + usage() + throw GeneralUserErrorException("Unknown subcommand '$subcommand'") + } + } + + // Run the subcommand. + handler.handle(args.copyOfRange(1, args.size).toList()) + } +} diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Annotations.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Annotations.kt new file mode 100644 index 000000000000..4a11259a8ef7 --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Annotations.kt @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.platform.test.ravenwood.ravenhelper.policytoannot + +import com.android.hoststubgen.filters.FilterPolicy + + +/** + * This class knows about the Ravenwood annotations. + */ +class Annotations { + enum class Target { + Class, + Field, + Method, + } + + fun get(policy: FilterPolicy, target: Target): String? { + return when (policy) { + FilterPolicy.Keep -> + if (target == Target.Class) { + "@android.ravenwood.annotation.RavenwoodKeepPartialClass" + } else { + "@android.ravenwood.annotation.RavenwoodKeep" + } + FilterPolicy.KeepClass -> + "@android.ravenwood.annotation.RavenwoodKeepWholeClass" + FilterPolicy.Substitute -> + "@android.ravenwood.annotation.RavenwoodReplace" + FilterPolicy.Redirect -> + "@android.ravenwood.annotation.RavenwoodRedirect" + FilterPolicy.Throw -> + "@android.ravenwood.annotation.RavenwoodThrow" + FilterPolicy.Ignore -> null // Ignore has no annotation. (because it's not very safe.) + FilterPolicy.Remove -> + "@android.ravenwood.annotation.RavenwoodRemove" + } + } + + private fun withArg(annot: String, arg: String): String { + return "@$annot(\"$arg\")" + } + + fun getClassLoadHookAnnotation(arg: String): String { + return withArg("android.ravenwood.annotation.RavenwoodClassLoadHook", arg) + } + + fun getRedirectionClassAnnotation(arg: String): String { + return withArg("android.ravenwood.annotation.RavenwoodRedirectionClass", arg) + } +} + diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Operations.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Operations.kt new file mode 100644 index 000000000000..3531ba951b1c --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/Operations.kt @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenhelper.policytoannot + +/* + * This file contains classes and functions about file edit operations, such as + * "insert a line", "delete a line". + */ + + +import com.android.hoststubgen.log +import java.io.BufferedWriter +import java.io.File + +enum class SourceOperationType { + /** Insert a line */ + Insert, + + /** delete a line */ + Delete, + + /** Insert a text at the beginning of a line */ + Prepend, +} + +data class SourceOperation( + /** Target file to edit. */ + val sourceFile: String, + + /** 1-based line number. Use -1 to add at the end of the file. */ + val lineNumber: Int, + + /** Operation type.*/ + val type: SourceOperationType, + + /** Operand -- text to insert or prepend. Ignored for delete. */ + val text: String = "", + + /** Human-readable description of why this operation was created */ + val description: String, +) { + override fun toString(): String { + return "SourceOperation(sourceFile='$sourceFile', " + + "lineNumber=$lineNumber, type=$type, text='$text' desc='$description')" + } +} + +/** + * Stores list of [SourceOperation]s for each file. + */ +class SourceOperations { + var size: Int = 0 + private set + private val fileOperations = mutableMapOf<String, MutableList<SourceOperation>>() + + fun add(op: SourceOperation) { + log.forVerbose { + log.v("Adding operation: $op") + } + size++ + fileOperations[op.sourceFile]?.let { ops -> + ops.add(op) + return + } + fileOperations[op.sourceFile] = mutableListOf(op) + } + + /** + * Get the collected [SourceOperation]s for each file. + */ + fun getOperations(): MutableMap<String, MutableList<SourceOperation>> { + return fileOperations + } +} + +/** + * Create a shell script to apply all the operations (using sed). + */ +fun createShellScript(ops: SourceOperations, writer: BufferedWriter) { + // File header. + // Note ${'$'} is an ugly way to put a dollar sign ($) in a multi-line string. + writer.write( + """ + #!/bin/bash + + set -e # Finish when any command fails. + + function apply() { + local file="${'$'}1" + + # The script is given via stdin. Write it to file. + local sed="/tmp/pta-script.sed.tmp" + cat > "${'$'}sed" + + echo "Running: sed -i -f \"${'$'}sed\" \"${'$'}file\"" + + if ! sed -i -f "${'$'}sed" "${'$'}file" ; then + echo 'Failed!' 1>&2 + return 1 + fi + } + + """.trimIndent() + ) + + ops.getOperations().toSortedMap().forEach { (origFile, ops) -> + val file = File(origFile).absolutePath + + writer.write("\n") + + writer.write("#") + writer.write("=".repeat(78)) + writer.write("\n") + + writer.write("\n") + + writer.write("apply \"$file\" <<'__END_OF_SCRIPT__'\n") + toSedScript(ops, writer) + writer.write("__END_OF_SCRIPT__\n") + } + + writer.write("\n") + + writer.write("echo \"All files updated successfully!\"\n") + writer.flush() +} + +/** + * Create a sed script to apply a list of operations. + */ +private fun toSedScript(ops: List<SourceOperation>, writer: BufferedWriter) { + ops.sortedBy { it.lineNumber }.forEach { op -> + if (op.text.contains('\n')) { + throw RuntimeException("Operation $op may not contain newlines.") + } + + // Convert each operation to a sed operation. Examples: + // + // - Insert "abc" to line 2 + // 2i\ + // abc + // + // - Insert "abc" to the end of the file + // $a\ + // abc + // + // - Delete line 2 + // 2d + // + // - Prepend abc to line 2 + // 2s/^/abc/ + // + // The line numbers are all the line numbers in the original file. Even though + // the script itself will change them because of inserts and deletes, we don't need to + // change the line numbers in the script. + + // Write the target line number. + writer.write("\n") + writer.write("# ${op.description}\n") + if (op.lineNumber >= 0) { + writer.write(op.lineNumber.toString()) + } else { + writer.write("$") + } + + when (op.type) { + SourceOperationType.Insert -> { + if (op.lineNumber >= 0) { + writer.write("i\\\n") // "Insert" + } else { + // If it's the end of the file, we need to use "a" (append) + writer.write("a\\\n") + } + writer.write(op.text) + writer.write("\n") + } + SourceOperationType.Delete -> { + writer.write("d\n") + } + SourceOperationType.Prepend -> { + if (op.text.contains('/')) { + TODO("Operation $op contains character(s) that needs to be escaped.") + } + writer.write("s/^/${op.text}/\n") + } + } + } +}
\ No newline at end of file diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt new file mode 100644 index 000000000000..08bd95fd532b --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaOptions.kt @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenhelper.policytoannot + +import com.android.hoststubgen.ArgIterator +import com.android.hoststubgen.ArgumentsException +import com.android.hoststubgen.SetOnce +import com.android.hoststubgen.ensureFileExists +import com.android.hoststubgen.log + +/** + * Options for the "ravenhelper pta" subcommand. + */ +class PtaOptions( + /** Text policy files */ + var policyOverrideFiles: MutableList<String> = mutableListOf(), + + /** Annotation allowed list file. */ + var annotationAllowedClassesFile: SetOnce<String?> = SetOnce(null), + + /** Source files or directories. */ + var sourceFilesOrDirectories: MutableList<String> = mutableListOf(), + + /** Output script file. */ + var outputScriptFile: SetOnce<String?> = SetOnce(null), + + /** Dump the operations (for debugging) */ + var dumpOperations: SetOnce<Boolean> = SetOnce(false), +) { + companion object { + fun parseArgs(args: List<String>): PtaOptions { + val ret = PtaOptions() + val ai = ArgIterator.withAtFiles(args.toTypedArray()) + + while (true) { + val arg = ai.nextArgOptional() ?: break + + fun nextArg(): String = ai.nextArgRequired(arg) + + if (log.maybeHandleCommandLineArg(arg) { nextArg() }) { + continue + } + try { + when (arg) { + // TODO: Write help + "-h", "--help" -> TODO("Help is not implemented yet") + + "-p", "--policy-override-file" -> + ret.policyOverrideFiles.add(nextArg().ensureFileExists()) + + "-a", "--annotation-allowed-classes-file" -> + ret.annotationAllowedClassesFile.set(nextArg().ensureFileExists()) + + "-s", "--src" -> + ret.sourceFilesOrDirectories.add(nextArg().ensureFileExists()) + + "--dump" -> + ret.dumpOperations.set(true) + + "-o", "--output-script" -> + ret.outputScriptFile.set(nextArg()) + + else -> throw ArgumentsException("Unknown option: $arg") + } + } catch (e: SetOnce.SetMoreThanOnceException) { + throw ArgumentsException("Duplicate or conflicting argument found: $arg") + } + } + + if (ret.policyOverrideFiles.size == 0) { + throw ArgumentsException("Must specify at least one policy file") + } + + if (ret.sourceFilesOrDirectories.size == 0) { + throw ArgumentsException("Must specify at least one source path") + } + + return ret + } + } + + override fun toString(): String { + return """ + PtaOptions{ + policyOverrideFiles=$policyOverrideFiles + annotationAllowedClassesFile=$annotationAllowedClassesFile + sourceFilesOrDirectories=$sourceFilesOrDirectories + outputScriptFile=$outputScriptFile + dumpOperations=$dumpOperations + } + """.trimIndent() + } +}
\ No newline at end of file diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaProcessor.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaProcessor.kt new file mode 100644 index 000000000000..5984e4fc8f9f --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/policytoannot/PtaProcessor.kt @@ -0,0 +1,479 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenhelper.policytoannot + +import com.android.hoststubgen.LogLevel +import com.android.hoststubgen.asm.CLASS_INITIALIZER_NAME +import com.android.hoststubgen.asm.toJvmClassName +import com.android.hoststubgen.filters.FilterPolicyWithReason +import com.android.hoststubgen.filters.PolicyFileProcessor +import com.android.hoststubgen.filters.SpecialClass +import com.android.hoststubgen.filters.TextFileFilterPolicyParser +import com.android.hoststubgen.filters.TextFilePolicyMethodReplaceFilter +import com.android.hoststubgen.log +import com.android.hoststubgen.utils.ClassFilter +import com.android.platform.test.ravenwood.ravenhelper.SubcommandHandler +import com.android.platform.test.ravenwood.ravenhelper.psi.createUastEnvironment +import com.android.platform.test.ravenwood.ravenhelper.sourcemap.AllClassInfo +import com.android.platform.test.ravenwood.ravenhelper.sourcemap.ClassInfo +import com.android.platform.test.ravenwood.ravenhelper.sourcemap.MethodInfo +import com.android.platform.test.ravenwood.ravenhelper.sourcemap.SourceLoader +import java.io.BufferedWriter +import java.io.FileOutputStream +import java.io.FileReader +import java.io.OutputStreamWriter +import java.util.regex.Pattern + +/** + * This is the main routine of the "pta" -- policy-to-annotation -- subcommands. + */ +class PtaProcessor : SubcommandHandler { + override fun handle(args: List<String>) { + val options = PtaOptions.parseArgs(args) + + log.v("Options: $options") + + val converter = TextPolicyToAnnotationConverter( + options.policyOverrideFiles, + options.sourceFilesOrDirectories, + options.annotationAllowedClassesFile.get, + Annotations(), + options.dumpOperations.get || log.isEnabled(LogLevel.Debug), + ) + converter.process() + + val ops = converter.resultOperations + + if (ops.size == 0) { + log.i("No files need to be updated.") + return + } + + val scriptWriter = BufferedWriter(OutputStreamWriter( + options.outputScriptFile.get?.let { file -> + FileOutputStream(file) + } ?: System.out + )) + + scriptWriter.use { writer -> + options.outputScriptFile.get?.let { + log.i("Creating script file at $it ...") + } + createShellScript(ops, writer) + } + } +} + +/** + * This class implements the actual logic. + */ +private class TextPolicyToAnnotationConverter( + val policyFiles: List<String>, + val sourceFilesOrDirectories: List<String>, + val annotationAllowedClassesFile: String?, + val annotations: Annotations, + val dumpOperations: Boolean, +) { + private val annotationAllowedClasses: ClassFilter = annotationAllowedClassesFile.let { file -> + if (file == null) { + ClassFilter.newNullFilter(true) // Allow all classes + } else { + ClassFilter.loadFromFile(file, false) + } + } + + val resultOperations = SourceOperations() + private val classes = AllClassInfo() + private val policyParser = TextFileFilterPolicyParser() + private val annotationNeedingClasses = mutableSetOf<String>() + + /** + * Entry point. + */ + fun process() { + // First, load + val env = createUastEnvironment() + try { + loadSources() + + processPolicies() + + addToAnnotationsAllowedListFile() + + if (dumpOperations) { + log.withIndent { + resultOperations.getOperations().toSortedMap().forEach { (file, ops) -> + log.i("ops: $file") + ops.forEach { op -> + log.i(" line: ${op.lineNumber}: ${op.type}: \"${op.text}\" " + + "(${op.description})") + } + } + } + } + } finally { + env.dispose() + } + } + + /** + * Load all the java source files into [classes]. + */ + private fun loadSources() { + val env = createUastEnvironment() + try { + val loader = SourceLoader(env) + loader.load(sourceFilesOrDirectories, classes) + } finally { + env.dispose() + } + } + + private fun addToAnnotationsAllowedListFile() { + log.i("Generating operations to update annotation allowlist file...") + log.withIndent { + annotationNeedingClasses.sorted().forEach { className -> + if (!annotationAllowedClasses.matches(className.toJvmClassName())) { + resultOperations.add( + SourceOperation( + annotationAllowedClassesFile!!, + -1, // add to the end + SourceOperationType.Insert, + className, + "add to annotation allowlist" + )) + } + } + } + } + + /** + * Process the policy files with [Processor]. + */ + private fun processPolicies() { + log.i("Loading the policy files and generating operations...") + log.withIndent { + policyFiles.forEach { policyFile -> + log.i("Parsing $policyFile ...") + log.withIndent { + policyParser.parse(FileReader(policyFile), policyFile, Processor()) + } + } + } + } + + private inner class Processor : PolicyFileProcessor { + + var classPolicyText = "" + var classPolicyLine = -1 + + // Whether the current class has a skip marker, in which case we ignore all members. + // Applicable only within a "simple class" + var classSkipping = false + + var classLineConverted = false + var classHasMember = false + + private fun currentLineHasSkipMarker(): Boolean { + val ret = policyParser.currentLineText.contains("no-pta") + + if (ret) { + log.forVerbose { + log.v("Current line has a skip marker: ${policyParser.currentLineText}") + } + } + + return ret + } + + private fun shouldSkipCurrentLine(): Boolean { + // If a line contains a special marker "no-pta", we'll skip it. + return classSkipping || currentLineHasSkipMarker() + } + + /** Print a warning about an unsupported policy directive. */ + private fun warnOnPolicy(message: String, policyLine: String, lineNumber: Int) { + log.w("Warning: $message") + log.w(" policy: \"$policyLine\"") + log.w(" at ${policyParser.filename}:$lineNumber") + } + + /** Print a warning about an unsupported policy directive. */ + private fun warnOnCurrentPolicy(message: String) { + warnOnPolicy(message, policyParser.currentLineText, policyParser.lineNumber) + } + + /** Print a warning about an unsupported policy directive on the class line. */ + private fun warnOnClassPolicy(message: String) { + warnOnPolicy(message, classPolicyText, classPolicyLine) + } + + override fun onPackage(name: String, policy: FilterPolicyWithReason) { + warnOnCurrentPolicy("'package' directive isn't supported (yet).") + } + + override fun onRename(pattern: Pattern, prefix: String) { + // Rename will never be supported, so don't show a warning. + } + + private fun addOperation(op: SourceOperation) { + resultOperations.add(op) + } + + private fun commentOutPolicy(lineNumber: Int, description: String) { + addOperation( + SourceOperation( + policyParser.filename, + lineNumber, + SourceOperationType.Prepend, + "#[PTA]: ", // comment out. + description, + ) + ) + } + + override fun onClassStart(className: String) { + classSkipping = currentLineHasSkipMarker() + classLineConverted = false + classHasMember = false + classPolicyLine = policyParser.lineNumber + classPolicyText = policyParser.currentLineText + } + + override fun onClassEnd(className: String) { + if (classSkipping) { + classSkipping = false + return + } + if (!classLineConverted) { + // Class line is still needed in the policy file. + // (Because the source file wasn't found.) + return + } + if (!classHasMember) { + commentOutPolicy(classPolicyLine, "remove class policy on $className") + } else { + warnOnClassPolicy( + "Class policy on $className can't be removed because it still has members.") + } + } + + private fun findClass(className: String): ClassInfo? { + val ci = classes.findClass(className) + if (ci == null) { + warnOnCurrentPolicy("Class not found: $className") + } + return ci + } + + private fun addClassAnnotation( + className: String, + annotation: String, + ): Boolean { + val ci = findClass(className) ?: return false + + // Add the annotation to the source file. + addOperation( + SourceOperation( + ci.location.file, + ci.location.line, + SourceOperationType.Insert, + ci.location.getIndent() + annotation, + "add class annotation to $className" + ) + ) + annotationNeedingClasses.add(className) + return true + } + + override fun onSimpleClassPolicy(className: String, policy: FilterPolicyWithReason) { + if (shouldSkipCurrentLine()) { + return + } + log.v("Found simple class policy: $className - ${policy.policy}") + + val annot = annotations.get(policy.policy, Annotations.Target.Class)!! + if (addClassAnnotation(className, annot)) { + classLineConverted = true + } + } + + override fun onSubClassPolicy(superClassName: String, policy: FilterPolicyWithReason) { + warnOnCurrentPolicy("Subclass policies isn't supported (yet).") + } + + override fun onRedirectionClass(fromClassName: String, toClassName: String) { + if (shouldSkipCurrentLine()) { + return + } + + log.v("Found class redirection: $fromClassName - $toClassName") + + if (addClassAnnotation( + fromClassName, + annotations.getRedirectionClassAnnotation(toClassName), + )) { + commentOutPolicy(policyParser.lineNumber, + "remove class redirection policy on $fromClassName") + } + } + + override fun onClassLoadHook(className: String, callback: String) { + if (shouldSkipCurrentLine()) { + return + } + + log.v("Found class load hook: $className - $callback") + + if (addClassAnnotation( + className, + annotations.getClassLoadHookAnnotation(callback), + )) { + commentOutPolicy(policyParser.lineNumber, + "remove class load hook policy on $className") + } + } + + override fun onSpecialClassPolicy(type: SpecialClass, policy: FilterPolicyWithReason) { + // This can't be converted to an annotation, so don't show a warning. + } + + override fun onField(className: String, fieldName: String, policy: FilterPolicyWithReason) { + if (shouldSkipCurrentLine()) { + return + } + + log.v("Found field policy: $className.$fieldName - ${policy.policy}") + + val ci = findClass(className) ?: return + + ci.findField(fieldName)?.let { fi -> + val annot = annotations.get(policy.policy, Annotations.Target.Field)!! + + addOperation( + SourceOperation( + fi.location.file, + fi.location.line, + SourceOperationType.Insert, + fi.location.getIndent() + annot, + "add annotation to field $className.$fieldName", + ) + ) + commentOutPolicy(policyParser.lineNumber, + "remove field policy $className.$fieldName") + + annotationNeedingClasses.add(className) + } ?: { + warnOnCurrentPolicy("Field not found: $className.$fieldName") + } + } + + override fun onSimpleMethodPolicy( + className: String, + methodName: String, + methodDesc: String, + policy: FilterPolicyWithReason + ) { + if (shouldSkipCurrentLine()) { + return + } + val readableName = "$className.$methodName$methodDesc" + log.v("Found simple method policy: $readableName - ${policy.policy}") + + + // Inner method to get the matching methods for this policy. + // + // If this policy can't be converted for any reason, it'll return null. + // Otherwise, it'll return a pair of method list and the annotation string. + fun getMethods(): Pair<List<MethodInfo>, String>? { + if (methodName == CLASS_INITIALIZER_NAME) { + warnOnClassPolicy("Policy for class initializers not supported.") + return null + } + val ci = findClass(className) ?: return null + val methods = ci.findMethods(methodName, methodDesc) + if (methods == null) { + warnOnCurrentPolicy("Method not found: $readableName") + return null + } + + // If the policy is "ignore", we can't convert it to an annotation, in which case + // annotations.get() will return null. + val annot = annotations.get(policy.policy, Annotations.Target.Method) + if (annot == null) { + warnOnCurrentPolicy("Annotation for policy '${policy.policy}' isn't available") + return null + } + return Pair(methods, annot) + } + + val methodsAndAnnot = getMethods() + + if (methodsAndAnnot == null) { + classHasMember = true + return // This policy can't converted. + } + val methods = methodsAndAnnot.first + val annot = methodsAndAnnot.second + + var found = false + methods.forEach { mi -> + found = true + addOperation( + SourceOperation( + mi.location.file, + mi.location.line, + SourceOperationType.Insert, + mi.location.getIndent() + annot, + "add annotation to method $readableName", + ) + ) + } + if (found) { + commentOutPolicy( + policyParser.lineNumber, + "remove method policy $readableName" + ) + + annotationNeedingClasses.add(className) + } else { + warnOnCurrentPolicy("Method not found: $readableName") + } + } + + override fun onMethodInClassReplace( + className: String, + methodName: String, + methodDesc: String, + targetName: String, + policy: FilterPolicyWithReason + ) { + warnOnCurrentPolicy("Found method replace but it's not supported yet: " + + "$className.$methodName$methodDesc - $targetName") + } + + override fun onMethodOutClassReplace( + className: String, + methodName: String, + methodDesc: String, + replaceSpec: TextFilePolicyMethodReplaceFilter.MethodCallReplaceSpec, + policy: FilterPolicyWithReason + ) { + // This can't be converted to an annotation. + classHasMember = true + } + } +}
\ No newline at end of file diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/psi/PsiUtil.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/psi/PsiUtil.kt new file mode 100644 index 000000000000..6775135e1ac5 --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/psi/PsiUtil.kt @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenhelper.psi + +import com.android.tools.lint.UastEnvironment + +// PSI is a library to parse Java/Kotlin source files, which is part of JetBrains' IntelliJ/ +// Android Studio, and other IDEs. +// +// PSI is normally used by IntelliJ's plugins, and as such, there isn't really a good documentation +// on how to use it from a standalone program. However, fortunately, Android Studio's Lint +// and Metalava both use PSI. Metalava reuses some of the APIs exposed by Lint. We also use the +// same Lint APIs used by Metalava here. +// +// Some code pointers around the relevant projects: +// +// - We stole code from Metalava, but the recent version of Metalava is too complicated, +// and hard to understand. Older Metalava, such as this one: +// https://android.git.corp.google.com/platform/tools/metalava/+/refs/heads/android13-dev +// is easier to understand. +// +// - PSI is source code is available in IntelliJ's code base: +// https://github.com/JetBrains/intellij-community.git +// +// - Lint is in Android studio. +// https://android.googlesource.com/platform/tools/base/+/studio-master-dev/source.md + + +/** + * Create [UastEnvironment] enough to parse Java source files. + */ +fun createUastEnvironment(): UastEnvironment { + val config = UastEnvironment.Configuration.create( + enableKotlinScripting = false, + useFirUast = false, + ) + + config.javaLanguageLevel = com.intellij.pom.java.LanguageLevel.JDK_21 + + // The following code exists in Metalava, but we don't seem to need it. + // We may need to when we need to support kotlin. +// config.kotlinLanguageLevel = kotlinLanguageLevel +// config.addSourceRoots(listOf(File(root))) +// config.addClasspathRoots(classpath.map { it.absoluteFile }) +// options.jdkHome?.let { +// if (options.isJdkModular(it)) { +// config.kotlinCompilerConfig.put(JVMConfigurationKeys.JDK_HOME, it) +// config.kotlinCompilerConfig.put(JVMConfigurationKeys.NO_JDK, false) +// } +// } + + return UastEnvironment.create(config) +} diff --git a/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/SourceMapGenerator.kt b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/SourceMapGenerator.kt new file mode 100644 index 000000000000..58e4497f9f9c --- /dev/null +++ b/ravenwood/tools/ravenhelper/src/com/android/platform/test/ravenwood/ravenhelper/sourcemap/SourceMapGenerator.kt @@ -0,0 +1,419 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.platform.test.ravenwood.ravenhelper.sourcemap + +/* + * This file contains classes used to parse Java source files to build "source map" which + * basically tells you what classes/methods/fields are declared in what line of what file. + */ + +import com.android.hoststubgen.GeneralUserErrorException +import com.android.hoststubgen.log +import com.android.tools.lint.UastEnvironment +import com.intellij.openapi.editor.Document +import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.psi.PsiClass +import com.intellij.psi.PsiClassOwner +import com.intellij.psi.PsiElement +import com.intellij.psi.PsiFile +import com.intellij.psi.PsiManager +import com.intellij.psi.PsiMethod +import com.intellij.psi.PsiNameIdentifierOwner +import com.intellij.psi.SyntheticElement +import java.io.File + + +/** + * Represents the location of an item. (class, field or method) + */ +data class Location ( + /** Full path filename. */ + val file: String, + + /** 1-based line number */ + val line: Int, + + /** Indent of the line */ + val indent: Int, +) { + + fun getIndent(): String { + return " ".repeat(indent) + } + + fun dump() { + log.i("Location: $file:$line (indent: $indent)") + } +} + +/** + * Represents the type of item. + */ +enum class ItemType { + Class, + Field, + Method, +} + +/** Holds a field's location. */ +data class FieldInfo ( + val name: String, + val location: Location, +) { + fun dump() { + log.i("Field: $name") + log.withIndent { + location.dump() + } + } +} + +/** Holds a method's location. */ +data class MethodInfo ( + val name: String, + /** "Simplified" description. */ + val simpleDesc: String, + val location: Location, +) { + fun dump() { + log.i("Method: $name$simpleDesc") + log.withIndent { + location.dump() + } + } +} + +/** Holds a class's location and members. */ +data class ClassInfo ( + val fullName: String, + val location: Location, + val fields: MutableMap<String, FieldInfo> = mutableMapOf(), + val methods: MutableMap<String, MutableList<MethodInfo>> = mutableMapOf(), +) { + fun add(fi: FieldInfo) { + fields.put(fi.name, fi) + } + + fun add(mi: MethodInfo) { + val list = methods.get(mi.name) + if (list != null) { + list.add(mi) + } else { + methods.put(mi.name, mutableListOf(mi)) + } + } + + fun dump() { + log.i("Class: $fullName") + log.withIndent { + location.dump() + + // Sort and print fields and methods. + methods.toSortedMap().forEach { entry -> + entry.value.sortedBy { method -> method.simpleDesc }.forEach { + it.dump() + } + } + } + } + + /** Find a field by name */ + fun findField(fieldName: String): FieldInfo? { + return fields[fieldName] + } + + /** + * Find a field by name and descriptor. + * + * If [descriptor] is "*", then all methods with the name will be returned. + */ + fun findMethods(methodName: String, methodDesc: String): List<MethodInfo>? { + val list = methods[methodName] ?: return null + + // Wildcard method policy. + if (methodDesc == "*") { + return list + } + + val simpleDesc = simplifyMethodDesc(methodDesc) + list.forEach { mi -> + if (simpleDesc == mi.simpleDesc) { + return listOf(mi) + } + } + log.w("Method $fullName.$methodName found, but none match description '$methodDesc'") + return null + } +} + +/** + * Stores all classes + */ +data class AllClassInfo ( + val classes: MutableMap<String, ClassInfo> = mutableMapOf(), +) { + fun add(ci: ClassInfo) { + classes.put(ci.fullName, ci) + } + + fun dump() { + classes.toSortedMap { a, b -> a.compareTo(b) }.forEach { + it.value.dump() + } + } + + fun findClass(name: String): ClassInfo? { + return classes.get(name) + } +} + +fun typeToSimpleDesc(origType: String): String { + var type = origType + + // Detect arrays. + var arrayPrefix = "" + while (type.endsWith("[]")) { + arrayPrefix += "[" + type = type.substring(0, type.length - 2) + } + + // Delete generic parameters. (delete everything after '<') + type.indexOf('<').let { pos -> + if (pos >= 0) { + type = type.substring(0, pos) + } + } + + // Handle builtins. + val builtinType = when (type) { + "byte" -> "B" + "short" -> "S" + "int" -> "I" + "long" -> "J" + "float" -> "F" + "double" -> "D" + "boolean" -> "Z" + "char" -> "C" + "void" -> "V" + else -> null + } + + builtinType?.let { + return arrayPrefix + builtinType + } + + return arrayPrefix + "L" + type + ";" +} + +/** + * Get a "simple" description of a method. + * + * "Simple" descriptions are similar to "real" ones, except: + * - No return type. + * - No package names in type names. + */ +fun getSimpleDesc(method: PsiMethod): String { + val sb = StringBuilder() + + sb.append("(") + + val params = method.parameterList + for (i in 0..<params.parametersCount) { + val param = params.getParameter(i) + + val type = param?.type?.presentableText + + if (type == null) { + throw RuntimeException( + "Unable to decode parameter list from method from ${params.parent}") + } + + sb.append(typeToSimpleDesc(type)) + } + + sb.append(")") + + return sb.toString() +} + +private val reTypeFinder = "L.*/".toRegex() + +private fun simplifyMethodDesc(origMethodDesc: String): String { + // We don't need the return type, so remove everything after the ')'. + val pos = origMethodDesc.indexOf(')') + var desc = if (pos < 0) { origMethodDesc } else { origMethodDesc.substring(0, pos + 1) } + + // Then we remove the package names from all the class names. + // i.e. convert "Ljava/lang/String" to "LString". + + return desc.replace(reTypeFinder, "L") +} + +/** + * Class that reads and parses java source files using PSI and populate [AllClassInfo]. + */ +class SourceLoader( + val environment: UastEnvironment, +) { + private val fileSystem = StandardFileSystems.local() + private val manager = PsiManager.getInstance(environment.ideaProject) + + /** Classes that were parsed */ + private var numParsedClasses = 0 + + /** + * Main entry point. + */ + fun load(filesOrDirectories: List<String>, classes: AllClassInfo) { + val psiFiles = mutableListOf<PsiFile>() + log.i("Loading source files...") + log.iTime("Discovering source files") { + load(filesOrDirectories.map { File(it) }, psiFiles) + } + + log.i("${psiFiles.size} file(s) found.") + + if (psiFiles.size == 0) { + throw GeneralUserErrorException("No source files found.") + } + + log.iTime("Parsing source files") { + log.withIndent { + for (file in psiFiles.asSequence().distinct()) { + val classesInFile = (file as? PsiClassOwner)?.classes?.toList() + classesInFile?.forEach { clazz -> + loadClass(clazz)?.let { classes.add(it) } + + clazz.innerClasses.forEach { inner -> + loadClass(inner)?.let { classes.add(it) } + } + } + } + } + } + log.i("$numParsedClasses class(es) found.") + } + + private fun load(filesOrDirectories: List<File>, result: MutableList<PsiFile>) { + filesOrDirectories.forEach { + load(it, result) + } + } + + private fun load(file: File, result: MutableList<PsiFile>) { + if (file.isDirectory) { + file.listFiles()?.forEach { child -> + load(child, result) + } + return + } + + // It's a file + when (file.extension) { + "java" -> { + // Load it. + } + "kt" -> { + log.w("Kotlin not supported, not loading ${file.path}") + return + } + else -> return // Silently skip + } + fileSystem.findFileByPath(file.path)?.let { virtualFile -> + manager.findFile(virtualFile)?.let { psiFile -> + result.add(psiFile) + } + } + } + + private fun loadClass(clazz: PsiClass): ClassInfo? { + if (clazz is SyntheticElement) { + return null + } + log.forVerbose { + log.v("Class found: ${clazz.qualifiedName}") + } + numParsedClasses++ + + log.withIndent { + val ci = ClassInfo( + clazz.qualifiedName!!, + getLocation(clazz) ?: return null, + ) + + // Load fields. + clazz.fields.filter { it !is SyntheticElement }.forEach { + val name = it.name + log.forDebug { log.d("Field found: $name") } + val loc = getLocation(it) ?: return@forEach + ci.add(FieldInfo(name, loc)) + } + + // Load methods. + clazz.methods.filter { it !is SyntheticElement }.forEach { + val name = resolveMethodName(it) + val simpleDesc = getSimpleDesc(it) + log.forDebug { log.d("Method found: $name$simpleDesc") } + val loc = getLocation(it) ?: return@forEach + ci.add(MethodInfo(name, simpleDesc, loc)) + } + return ci + } + } + + private fun resolveMethodName(method: PsiMethod): String { + val clazz = method.containingClass!! + if (clazz.name == method.name) { + return "<init>" // It's a constructor. + } + return method.name + } + + private fun getLocation(elem: PsiElement): Location? { + val lineAndIndent = getLineNumberAndIndent(elem) + if (lineAndIndent == null) { + log.w("Unable to determine location of $elem") + return null + } + return Location( + elem.containingFile.originalFile.virtualFile.path, + lineAndIndent.first, + lineAndIndent.second, + ) + } + + private fun getLineNumberAndIndent(element: PsiElement): Pair<Int, Int>? { + val psiFile: PsiFile = element.containingFile ?: return null + val document: Document = psiFile.viewProvider.document ?: return null + + // Actual elements such as PsiClass, PsiMethod and PsiField contains the leading + // javadoc, etc, so use the "identifier"'s element, if available. + // For synthesized elements, this may return null. + val targetRange = ( + (element as PsiNameIdentifierOwner).nameIdentifier?.textRange ?: element.textRange + ) ?: return null + val lineNumber = document.getLineNumber(targetRange.startOffset) + val lineStartOffset = document.getLineStartOffset(lineNumber) + + val lineLeadingText = document.getText( + com.intellij.openapi.util.TextRange(lineStartOffset, targetRange.startOffset)) + + val indent = lineLeadingText.takeWhile { it.isWhitespace() }.length + + // Line numbers are 0-based, add 1 for human-readable format + return Pair(lineNumber + 1, indent) + } +}
\ No newline at end of file |