diff options
249 files changed, 14999 insertions, 7083 deletions
diff --git a/.gitignore b/.gitignore index a09c56df5..45884c46f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /.idea +*.iml @@ -26,6 +26,6 @@ yudiliu@google.com jingwen@google.com # EMEA -hansson@google.com +hansson@google.com #{LAST_RESORT_SUGGESTION} lberki@google.com -paulduffin@google.com +paulduffin@google.com #{LAST_RESORT_SUGGESTION} diff --git a/android/Android.bp b/android/Android.bp index d58370391..cbd345945 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -49,6 +49,7 @@ bootstrap_go_package { "expand.go", "filegroup.go", "fixture.go", + "gen_notice.go", "hooks.go", "image.go", "license.go", @@ -97,6 +98,7 @@ bootstrap_go_package { "apex_test.go", "arch_test.go", "bazel_handler_test.go", + "bazel_paths_test.go", "bazel_test.go", "config_test.go", "config_bp2build_test.go", @@ -106,6 +108,7 @@ bootstrap_go_package { "deptag_test.go", "expand_test.go", "fixture_test.go", + "gen_notice_test.go", "license_kind_test.go", "license_test.go", "licenses_test.go", diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go index 26cfd5725..a5cea1720 100644 --- a/android/allowlists/allowlists.go +++ b/android/allowlists/allowlists.go @@ -37,16 +37,22 @@ const ( var ( Bp2buildDefaultConfig = Bp2BuildConfig{ - "art/libartpalette": Bp2BuildDefaultTrueRecursively, - "art/libdexfile": Bp2BuildDefaultTrueRecursively, - "art/runtime": Bp2BuildDefaultTrueRecursively, - "art/tools": Bp2BuildDefaultTrue, - "bionic": Bp2BuildDefaultTrueRecursively, - "bootable/recovery/tools/recovery_l10n": Bp2BuildDefaultTrue, - "build/bazel/examples/soong_config_variables": Bp2BuildDefaultTrueRecursively, + "prebuilts/runtime/mainline/platform/sdk": Bp2BuildDefaultTrueRecursively, + "art/libartpalette": Bp2BuildDefaultTrueRecursively, + "art/libartbase": Bp2BuildDefaultTrueRecursively, + "art/libdexfile": Bp2BuildDefaultTrueRecursively, + "art/libnativebridge": Bp2BuildDefaultTrueRecursively, + "art/runtime": Bp2BuildDefaultTrueRecursively, + "art/tools": Bp2BuildDefaultTrue, + "bionic": Bp2BuildDefaultTrueRecursively, + "bootable/recovery/tools/recovery_l10n": Bp2BuildDefaultTrue, "build/bazel/examples/apex/minimal": Bp2BuildDefaultTrueRecursively, - "build/make/tools/signapk": Bp2BuildDefaultTrue, + "build/bazel/examples/soong_config_variables": Bp2BuildDefaultTrueRecursively, + "build/bazel/examples/python": Bp2BuildDefaultTrueRecursively, + "build/bazel/examples/gensrcs": Bp2BuildDefaultTrueRecursively, "build/make/target/product/security": Bp2BuildDefaultTrue, + "build/make/tools/signapk": Bp2BuildDefaultTrue, + "build/make/tools/zipalign": Bp2BuildDefaultTrueRecursively, "build/soong": Bp2BuildDefaultTrue, "build/soong/cc/libbuildversion": Bp2BuildDefaultTrue, // Skip tests subdir "build/soong/cc/ndkstubgen": Bp2BuildDefaultTrue, @@ -90,6 +96,7 @@ var ( "development/samples/VoicemailProviderDemo": Bp2BuildDefaultTrue, "development/samples/WiFiDirectDemo": Bp2BuildDefaultTrue, "development/sdk": Bp2BuildDefaultTrueRecursively, + "external/aac": Bp2BuildDefaultTrueRecursively, "external/arm-optimized-routines": Bp2BuildDefaultTrueRecursively, "external/auto/android-annotation-stubs": Bp2BuildDefaultTrueRecursively, "external/auto/common": Bp2BuildDefaultTrueRecursively, @@ -99,21 +106,33 @@ var ( "external/brotli": Bp2BuildDefaultTrue, "external/conscrypt": Bp2BuildDefaultTrue, "external/e2fsprogs": Bp2BuildDefaultTrueRecursively, + "external/eigen": Bp2BuildDefaultTrueRecursively, + "external/erofs-utils": Bp2BuildDefaultTrueRecursively, "external/error_prone": Bp2BuildDefaultTrueRecursively, + "external/expat": Bp2BuildDefaultTrueRecursively, + "external/f2fs-tools": Bp2BuildDefaultTrue, + "external/flac": Bp2BuildDefaultTrueRecursively, "external/fmtlib": Bp2BuildDefaultTrueRecursively, "external/google-benchmark": Bp2BuildDefaultTrueRecursively, "external/googletest": Bp2BuildDefaultTrueRecursively, "external/gwp_asan": Bp2BuildDefaultTrueRecursively, + "external/hamcrest": Bp2BuildDefaultTrueRecursively, "external/icu": Bp2BuildDefaultTrueRecursively, "external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete "external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete + "external/jarjar": Bp2BuildDefaultTrueRecursively, "external/javapoet": Bp2BuildDefaultTrueRecursively, "external/jemalloc_new": Bp2BuildDefaultTrueRecursively, "external/jsoncpp": Bp2BuildDefaultTrueRecursively, + "external/junit": Bp2BuildDefaultTrueRecursively, + "external/libavc": Bp2BuildDefaultTrueRecursively, "external/libcap": Bp2BuildDefaultTrueRecursively, "external/libcxx": Bp2BuildDefaultTrueRecursively, "external/libcxxabi": Bp2BuildDefaultTrueRecursively, "external/libevent": Bp2BuildDefaultTrueRecursively, + "external/libgav1": Bp2BuildDefaultTrueRecursively, + "external/libhevc": Bp2BuildDefaultTrueRecursively, + "external/libmpeg2": Bp2BuildDefaultTrueRecursively, "external/libpng": Bp2BuildDefaultTrueRecursively, "external/lz4/lib": Bp2BuildDefaultTrue, "external/lzma/C": Bp2BuildDefaultTrueRecursively, @@ -122,15 +141,22 @@ var ( "external/pcre": Bp2BuildDefaultTrueRecursively, "external/protobuf": Bp2BuildDefaultTrueRecursively, "external/python/six": Bp2BuildDefaultTrueRecursively, + "external/rappor": Bp2BuildDefaultTrueRecursively, "external/scudo": Bp2BuildDefaultTrueRecursively, "external/selinux/libselinux": Bp2BuildDefaultTrueRecursively, "external/selinux/libsepol": Bp2BuildDefaultTrueRecursively, "external/zlib": Bp2BuildDefaultTrueRecursively, + "external/zopfli": Bp2BuildDefaultTrueRecursively, "external/zstd": Bp2BuildDefaultTrueRecursively, + "frameworks/av/media/codecs": Bp2BuildDefaultTrueRecursively, + "frameworks/av/services/minijail": Bp2BuildDefaultTrueRecursively, "frameworks/base/media/tests/MediaDump": Bp2BuildDefaultTrue, "frameworks/base/startop/apps/test": Bp2BuildDefaultTrue, "frameworks/base/tests/appwidgets/AppWidgetHostTest": Bp2BuildDefaultTrueRecursively, "frameworks/native/libs/adbd_auth": Bp2BuildDefaultTrueRecursively, + "frameworks/native/libs/arect": Bp2BuildDefaultTrueRecursively, + "frameworks/native/libs/math": Bp2BuildDefaultTrueRecursively, + "frameworks/native/libs/nativebase": Bp2BuildDefaultTrueRecursively, "frameworks/native/opengl/tests/gl2_cameraeye": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/gl2_java": Bp2BuildDefaultTrue, "frameworks/native/opengl/tests/testLatency": Bp2BuildDefaultTrue, @@ -156,8 +182,10 @@ var ( "prebuilts/clang/host/linux-x86": Bp2BuildDefaultTrueRecursively, "prebuilts/tools/common/m2": Bp2BuildDefaultTrue, "system/apex": Bp2BuildDefaultFalse, // TODO(b/207466993): flaky failures - "system/apex/proto": Bp2BuildDefaultTrueRecursively, + "system/apex/apexer": Bp2BuildDefaultTrue, "system/apex/libs": Bp2BuildDefaultTrueRecursively, + "system/apex/proto": Bp2BuildDefaultTrueRecursively, + "system/apex/tools": Bp2BuildDefaultTrueRecursively, "system/core/debuggerd": Bp2BuildDefaultTrueRecursively, "system/core/diagnose_usb": Bp2BuildDefaultTrueRecursively, "system/core/libasyncio": Bp2BuildDefaultTrue, @@ -171,14 +199,19 @@ var ( "system/core/libutils": Bp2BuildDefaultTrueRecursively, "system/core/libvndksupport": Bp2BuildDefaultTrueRecursively, "system/core/property_service/libpropertyinfoparser": Bp2BuildDefaultTrueRecursively, + "system/libartpalette": Bp2BuildDefaultTrueRecursively, "system/libbase": Bp2BuildDefaultTrueRecursively, + "system/libfmq": Bp2BuildDefaultTrue, + "system/libhwbinder": Bp2BuildDefaultTrueRecursively, "system/libprocinfo": Bp2BuildDefaultTrue, "system/libziparchive": Bp2BuildDefaultTrueRecursively, "system/logging/liblog": Bp2BuildDefaultTrueRecursively, + "system/media/audio": Bp2BuildDefaultTrueRecursively, + "system/memory/libion": Bp2BuildDefaultTrueRecursively, + "system/memory/libmemunreachable": Bp2BuildDefaultTrueRecursively, "system/sepolicy/apex": Bp2BuildDefaultTrueRecursively, "system/timezone/apex": Bp2BuildDefaultTrueRecursively, "system/timezone/output_data": Bp2BuildDefaultTrueRecursively, - "system/unwinding/libbacktrace": Bp2BuildDefaultTrueRecursively, "system/unwinding/libunwindstack": Bp2BuildDefaultTrueRecursively, "tools/apksig": Bp2BuildDefaultTrue, "tools/platform-compat/java/android/compat": Bp2BuildDefaultTrueRecursively, @@ -195,12 +228,14 @@ var ( "build/bazel/ci/dist":/* recursive = */ false, "build/bazel/examples/android_app":/* recursive = */ true, "build/bazel/examples/java":/* recursive = */ true, + "build/bazel/examples/partitions":/* recursive = */ true, "build/bazel/bazel_skylib":/* recursive = */ true, "build/bazel/rules":/* recursive = */ true, "build/bazel/rules_cc":/* recursive = */ true, "build/bazel/scripts":/* recursive = */ true, "build/bazel/tests":/* recursive = */ true, "build/bazel/platforms":/* recursive = */ true, + "build/bazel/product_config":/* recursive = */ true, "build/bazel/product_variables":/* recursive = */ true, "build/bazel/vendor/google":/* recursive = */ true, "build/bazel_common_rules":/* recursive = */ true, @@ -222,7 +257,7 @@ var ( "prebuilts/bundletool":/* recursive = */ true, "prebuilts/gcc":/* recursive = */ true, - "prebuilts/build-tools":/* recursive = */ false, + "prebuilts/build-tools":/* recursive = */ true, "prebuilts/jdk/jdk11":/* recursive = */ false, "prebuilts/sdk":/* recursive = */ false, "prebuilts/sdk/current/extras/app-toolkit":/* recursive = */ false, @@ -232,6 +267,68 @@ var ( } Bp2buildModuleAlwaysConvertList = []string{ + // cc mainline modules + "code_coverage.policy", + "code_coverage.policy.other", + "codec2_soft_exports", + "com.android.media.swcodec-androidManifest", + "com.android.media.swcodec-ld.config.txt", + "com.android.media.swcodec-mediaswcodec.rc", + "com.android.media.swcodec.certificate", + "com.android.media.swcodec.key", + "com.android.neuralnetworks-androidManifest", + "com.android.neuralnetworks.certificate", + "com.android.neuralnetworks.key", + "flatbuffer_headers", + "gemmlowp_headers", + "gl_headers", + "libandroid_runtime_lazy", + "libandroid_runtime_vm_headers", + "libaudioclient_aidl_conversion_util", + "libaudioutils_fixedfft", + "libbinder_headers", + "libbinder_headers_platform_shared", + "libbluetooth-types-header", + "libbufferhub_headers", + "libcodec2", + "libcodec2_headers", + "libcodec2_internal", + "libdmabufheap", + "libdvr_headers", + "libgsm", + "libgui_bufferqueue_sources", + "libhardware", + "libhardware_headers", + "libincfs_headers", + "libnativeloader-headers", + "libnativewindow_headers", + "libneuralnetworks_headers", + "libopus", + "libpdx_headers", + "libprocpartition", + "libruy_static", + "libserviceutils", + "libstagefright_enc_common", + "libstagefright_foundation_headers", + "libstagefright_headers", + "libsurfaceflinger_headers", + "libsync", + "libtextclassifier_hash_headers", + "libtextclassifier_hash_static", + "libtflite_kernel_utils", + "libtinyxml2", + "libui-types", + "libui_headers", + "libvorbisidec", + "media_ndk_headers", + "media_plugin_headers", + "mediaswcodec.policy", + "mediaswcodec.xml", + "philox_random", + "philox_random_headers", + "server_configurable_flags", + "tensorflow_headers", + //external/avb "avbtool", "libavb", @@ -245,6 +342,7 @@ var ( //system/extras/ext4_utils "libext4_utils", + "mke2fs_conf", //system/extras/libfec "libfec", @@ -271,22 +369,25 @@ var ( Bp2buildModuleDoNotConvertList = []string{ // cc bugs - "libsepol", // TODO(b/207408632): Unsupported case of .l sources in cc library rules "libactivitymanager_aidl", // TODO(b/207426160): Unsupported use of aidl sources (via Dactivity_manager_procstate_aidl) in a cc_library "gen-kotlin-build-file.py", // TODO(b/198619163) module has same name as source "libgtest_ndk_c++", "libgtest_main_ndk_c++", // TODO(b/201816222): Requires sdk_version support. "linkerconfig", "mdnsd", // TODO(b/202876379): has arch-variant static_executable - "linker", // TODO(b/228316882): cc_binary uses link_crt - "libdebuggerd", // TODO(b/228314770): support product variable-specific header_libs - "versioner", // TODO(b/228313961): depends on prebuilt shared library libclang-cpp_host as a shared library, which does not supply expected providers for a shared library + "linker", // TODO(b/228316882): cc_binary uses link_crt + "libdebuggerd", // TODO(b/228314770): support product variable-specific header_libs + "versioner", // TODO(b/228313961): depends on prebuilt shared library libclang-cpp_host as a shared library, which does not supply expected providers for a shared library + "libspeexresampler", // TODO(b/231995978): Filter out unknown cflags + "libjpeg", "libvpx", // TODO(b/233948256): Convert .asm files + "art_libartbase_headers", // TODO(b/236268577): Header libraries do not support export_shared_libs_headers + "apexer_test", // Requires aapt2 + "apexer_test_host_tools", + "host_apex_verifier", // java bugs "libbase_ndk", // TODO(b/186826477): fails to link libctscamera2_jni for device (required for CtsCameraTestCases) // python protos - "libprotobuf-python", // TODO(b/196084681): contains .proto sources - "apex_build_info_proto", "apex_manifest_proto", // TODO(b/196084681): a python lib with proto sources - "linker_config_proto", // TODO(b/196084681): contains .proto sources + "libprotobuf-python", // Has a handcrafted alternative // genrule incompatibilities "brotli-fuzzer-corpus", // TODO(b/202015218): outputs are in location incompatible with bazel genrule handling. @@ -297,6 +398,9 @@ var ( "prebuilt_platform-robolectric-4.4-prebuilt", // aosp/1999250, needs .aar support in Jars "prebuilt_platform-robolectric-4.5.1-prebuilt", // aosp/1999250, needs .aar support in Jars + // proto support + "libstats_proto_host", // TODO(b/236055697): handle protos from other packages + // path property for filegroups "conscrypt", // TODO(b/210751803), we don't handle path property for filegroups "conscrypt-for-host", // TODO(b/210751803), we don't handle path property for filegroups @@ -305,6 +409,7 @@ var ( "libprotobuf-internal-python-srcs", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-full", // TODO(b/210751803), we don't handle path property for filegroups "libprotobuf-java-util-full", // TODO(b/210751803), we don't handle path property for filegroups + "auto_value_plugin_resources", // TODO(b/210751803), we don't handle path property for filegroups // go deps: "analyze_bcpf", // depends on bpmodify a blueprint_go_binary. @@ -323,15 +428,14 @@ var ( "libtombstoned_client_rust_bridge_code", "libtombstoned_client_wrapper", // rust conversions are not supported // unconverted deps - "CarHTMLViewer", // depends on unconverted modules android.car-stubs, car-ui-lib - "abb", // depends on unconverted modules: libcmd, libbinder - "adb", // depends on unconverted modules: AdbWinApi, libandroidfw, libopenscreen-discovery, libopenscreen-platform-impl, libusb, bin2c_fastdeployagent, AdbWinUsbApi - "android_icu4j_srcgen", // depends on unconverted modules: currysrc - "android_icu4j_srcgen_binary", // depends on unconverted modules: android_icu4j_srcgen, currysrc - "apex_manifest_proto_java", // b/210751803, depends on libprotobuf-java-full - "art-script", // depends on unconverted modules: dalvikvm, dex2oat - "bin2c_fastdeployagent", // depends on unconverted modules: deployagent - "chkcon", "sefcontext_compile", // depends on unconverted modules: libsepol + "CarHTMLViewer", // depends on unconverted modules android.car-stubs, car-ui-lib + "abb", // depends on unconverted modules: libcmd, libbinder + "adb", // depends on unconverted modules: AdbWinApi, libandroidfw, libopenscreen-discovery, libopenscreen-platform-impl, libusb, bin2c_fastdeployagent, AdbWinUsbApi + "android_icu4j_srcgen", // depends on unconverted modules: currysrc + "android_icu4j_srcgen_binary", // depends on unconverted modules: android_icu4j_srcgen, currysrc + "apex_manifest_proto_java", // b/210751803, depends on libprotobuf-java-full + "art-script", // depends on unconverted modules: dalvikvm, dex2oat + "bin2c_fastdeployagent", // depends on unconverted modules: deployagent "com.android.runtime", // depends on unconverted modules: bionic-linker-config, linkerconfig "conv_linker_config", // depends on unconverted modules: linker_config_proto "currysrc", // depends on unconverted modules: currysrc_org.eclipse, guavalib, jopt-simple-4.9 @@ -365,9 +469,11 @@ var ( "stats-log-api-gen", // depends on unconverted modules: libstats_proto_host "statslog.cpp", "statslog.h", "statslog.rs", // depends on unconverted modules: stats-log-api-gen "statslog_art.cpp", "statslog_art.h", "statslog_header.rs", // depends on unconverted modules: stats-log-api-gen - "timezone-host", // depends on unconverted modules: art.module.api.annotations - "truth-host-prebuilt", // depends on unconverted modules: truth-prebuilt - "truth-prebuilt", // depends on unconverted modules: asm-7.0, guava + "timezone-host", // depends on unconverted modules: art.module.api.annotations + "truth-host-prebuilt", // depends on unconverted modules: truth-prebuilt + "truth-prebuilt", // depends on unconverted modules: asm-7.0, guava + "libartbase-art-gtest", // depends on unconverted modules: libgtest_isolated, libart, libart-compiler, libdexfile, libprofile + "libartbased-art-gtest", // depends on unconverted modules: libgtest_isolated, libartd, libartd-compiler, libdexfiled, libprofiled // b/215723302; awaiting tz{data,_version} to then rename targets conflicting with srcs "tzdata", @@ -377,6 +483,8 @@ var ( Bp2buildCcLibraryStaticOnlyList = []string{} MixedBuildsDisabledList = []string{ + "libruy_static", "libtflite_kernel_utils", // TODO(b/237315968); Depend on prebuilt stl, not from source + "art_libdexfile_dex_instruction_list_header", // breaks libart_mterp.armng, header not found "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy diff --git a/android/androidmk.go b/android/androidmk.go index 5c715b473..832c7dfe3 100644 --- a/android/androidmk.go +++ b/android/androidmk.go @@ -288,6 +288,8 @@ func (a *AndroidMkEntries) AddCompatibilityTestSuites(suites ...string) { // The contributions to the dist. type distContributions struct { + // Path to license metadata file. + licenseMetadataFile Path // List of goals and the dist copy instructions. copiesForGoals []*copiesForGoals } @@ -364,6 +366,8 @@ func (a *AndroidMkEntries) getDistContributions(mod blueprint.Module) *distContr // Collate the contributions this module makes to the dist. distContributions := &distContributions{} + distContributions.licenseMetadataFile = amod.licenseMetadataFile + // Iterate over this module's dist structs, merged from the dist and dists properties. for _, dist := range amod.Dists() { // Get the list of goals this dist should be enabled for. e.g. sdk, droidcore @@ -456,6 +460,10 @@ func generateDistContributionsForMake(distContributions *distContributions) []st for _, c := range d.copies { ret = append( ret, + fmt.Sprintf("$(if $(strip $(ALL_TARGETS.%s.META_LIC)),,$(eval ALL_TARGETS.%s.META_LIC := %s))\n", + c.from.String(), c.from.String(), distContributions.licenseMetadataFile.String())) + ret = append( + ret, fmt.Sprintf("$(call dist-for-goals,%s,%s:%s)\n", d.goals, c.from.String(), c.dest)) } } @@ -937,7 +945,10 @@ func shouldSkipAndroidMkProcessing(module *ModuleBase) bool { return !module.Enabled() || module.commonProperties.HideFromMake || // Make does not understand LinuxBionic - module.Os() == LinuxBionic + module.Os() == LinuxBionic || + // Make does not understand LinuxMusl, except when we are building with USE_HOST_MUSL=true + // and all host binaries are LinuxMusl + (module.Os() == LinuxMusl && module.Target().HostCross) } // A utility func to format LOCAL_TEST_DATA outputs. See the comments on DataPath to understand how diff --git a/android/androidmk_test.go b/android/androidmk_test.go index caf11f10f..ae2187f48 100644 --- a/android/androidmk_test.go +++ b/android/androidmk_test.go @@ -50,6 +50,8 @@ const ( func (m *customModule) GenerateAndroidBuildActions(ctx ModuleContext) { + m.base().licenseMetadataFile = PathForOutput(ctx, "meta_lic") + // If the dist_output_file: true then create an output file that is stored in // the OutputFile property of the AndroidMkEntry. if proptools.BoolDefault(m.properties.Dist_output_file, true) { @@ -198,10 +200,13 @@ func TestGenerateDistContributionsForMake(t *testing.T) { }, } + dc.licenseMetadataFile = PathForTesting("meta_lic") makeOutput := generateDistContributionsForMake(dc) assertStringEquals(t, `.PHONY: my_goal +$(if $(strip $(ALL_TARGETS.one.out.META_LIC)),,$(eval ALL_TARGETS.one.out.META_LIC := meta_lic)) $(call dist-for-goals,my_goal,one.out:one.out) +$(if $(strip $(ALL_TARGETS.two.out.META_LIC)),,$(eval ALL_TARGETS.two.out.META_LIC := meta_lic)) $(call dist-for-goals,my_goal,two.out:other.out) `, strings.Join(makeOutput, "")) } @@ -243,18 +248,26 @@ func TestGetDistForGoals(t *testing.T) { expectedAndroidMkLines := []string{ ".PHONY: my_second_goal\n", + "$(if $(strip $(ALL_TARGETS.two.out.META_LIC)),,$(eval ALL_TARGETS.two.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_second_goal,two.out:two.out)\n", + "$(if $(strip $(ALL_TARGETS.three/four.out.META_LIC)),,$(eval ALL_TARGETS.three/four.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_second_goal,three/four.out:four.out)\n", ".PHONY: my_third_goal\n", + "$(if $(strip $(ALL_TARGETS.one.out.META_LIC)),,$(eval ALL_TARGETS.one.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_third_goal,one.out:test/dir/one.out)\n", ".PHONY: my_fourth_goal\n", + "$(if $(strip $(ALL_TARGETS.one.out.META_LIC)),,$(eval ALL_TARGETS.one.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_fourth_goal,one.out:one.suffix.out)\n", ".PHONY: my_fifth_goal\n", + "$(if $(strip $(ALL_TARGETS.one.out.META_LIC)),,$(eval ALL_TARGETS.one.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_fifth_goal,one.out:new-name)\n", ".PHONY: my_sixth_goal\n", + "$(if $(strip $(ALL_TARGETS.one.out.META_LIC)),,$(eval ALL_TARGETS.one.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_sixth_goal,one.out:some/dir/new-name.suffix)\n", ".PHONY: my_goal my_other_goal\n", + "$(if $(strip $(ALL_TARGETS.two.out.META_LIC)),,$(eval ALL_TARGETS.two.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_goal my_other_goal,two.out:two.out)\n", + "$(if $(strip $(ALL_TARGETS.three/four.out.META_LIC)),,$(eval ALL_TARGETS.three/four.out.META_LIC := meta_lic))\n", "$(call dist-for-goals,my_goal my_other_goal,three/four.out:four.out)\n", } @@ -274,7 +287,7 @@ func TestGetDistForGoals(t *testing.T) { ) } for idx, line := range androidMkLines { - expectedLine := expectedAndroidMkLines[idx] + expectedLine := strings.ReplaceAll(expectedAndroidMkLines[idx], "meta_lic", module.base().licenseMetadataFile.String()) if line != expectedLine { t.Errorf( "Expected AndroidMk line to be '%s', got '%s'", diff --git a/android/apex.go b/android/apex.go index b127f7410..019efdd96 100644 --- a/android/apex.go +++ b/android/apex.go @@ -58,9 +58,6 @@ type ApexInfo struct { // to true. UsePlatformApis bool - // The list of SDK modules that the containing apexBundle depends on. - RequiredSdks SdkRefs - // List of Apex variant names that this module is associated with. This initially is the // same as the `ApexVariationName` field. Then when multiple apex variants are merged in // mergeApexVariations, ApexInfo struct of the merged variant holds the list of apexBundles @@ -110,9 +107,6 @@ func (i ApexInfo) AddJSONData(d *map[string]interface{}) { // thus wouldn't be merged. func (i ApexInfo) mergedName(ctx PathContext) string { name := "apex" + strconv.Itoa(i.MinSdkVersion.FinalOrFutureInt()) - for _, sdk := range i.RequiredSdks { - name += "_" + sdk.Name + "_" + sdk.Version - } return name } @@ -850,52 +844,28 @@ var minSdkVersionAllowlist = func(apiMap map[string]int) map[string]ApiLevel { } return list }(map[string]int{ - "android.net.ipsec.ike": 30, - "androidx.annotation_annotation-nodeps": 29, - "androidx.arch.core_core-common-nodeps": 29, - "androidx.collection_collection-nodeps": 29, - "androidx.collection_collection-ktx-nodeps": 30, - "androidx.concurrent_concurrent-futures-nodeps": 30, - "androidx.lifecycle_lifecycle-common-java8-nodeps": 30, - "androidx.lifecycle_lifecycle-common-nodeps": 29, - "androidx.room_room-common-nodeps": 30, "androidx-constraintlayout_constraintlayout-solver-nodeps": 29, "apache-commons-compress": 29, "bouncycastle_ike_digests": 30, "brotli-java": 29, - "captiveportal-lib": 28, - "error_prone_annotations": 30, "flatbuffer_headers": 30, - "framework-permission": 30, "gemmlowp_headers": 30, - "guava-listenablefuture-prebuilt-jar": 30, "ike-internals": 30, - "kotlinx-coroutines-android": 28, - "kotlinx-coroutines-android-nodeps": 30, - "kotlinx-coroutines-core": 28, - "kotlinx-coroutines-core-nodeps": 30, "libbrotli": 30, "libcrypto_static": 30, "libeigen": 30, "liblz4": 30, "libmdnssd": 30, - "libneuralnetworks_common": 30, - "libneuralnetworks_headers": 30, - "libneuralnetworks": 30, "libprocpartition": 30, "libprotobuf-java-lite": 30, "libprotoutil": 30, "libtextclassifier_hash_headers": 30, "libtextclassifier_hash_static": 30, "libtflite_kernel_utils": 30, - "libwatchdog": 29, "libzstd": 30, - "metrics-constants-protos": 28, "net-utils-framework-common": 29, - "permissioncontroller-statsd": 28, "philox_random_headers": 30, "philox_random": 30, - "service-permission": 30, "tensorflow_headers": 30, "xz-java": 29, }) diff --git a/android/apex_test.go b/android/apex_test.go index 1e2f3bde4..0bf4c9c36 100644 --- a/android/apex_test.go +++ b/android/apex_test.go @@ -33,10 +33,10 @@ func Test_mergeApexVariations(t *testing.T) { { name: "single", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"foo", "apex10000"}, @@ -45,25 +45,25 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000_baz_1", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false}}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, false}}, wantAliases: [][2]string{ - {"bar", "apex10000_baz_1"}, - {"foo", "apex10000_baz_1"}, + {"bar", "apex10000"}, + {"foo", "apex10000"}, }, }, { name: "don't merge version", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", uncheckedFinalApiLevel(30), false, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex30", uncheckedFinalApiLevel(30), false, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - {"apex10000", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"apex30", uncheckedFinalApiLevel(30), false, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex30"}, @@ -73,11 +73,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge updatable", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -85,32 +85,17 @@ func Test_mergeApexVariations(t *testing.T) { }, }, { - name: "don't merge sdks", - in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, false, SdkRefs{{"baz", "2"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - }, - wantMerged: []ApexInfo{ - {"apex10000_baz_2", FutureApiLevel, false, false, SdkRefs{{"baz", "2"}}, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, - {"apex10000_baz_1", FutureApiLevel, false, false, SdkRefs{{"baz", "1"}}, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - }, - wantAliases: [][2]string{ - {"bar", "apex10000_baz_2"}, - {"foo", "apex10000_baz_1"}, - }, - }, - { name: "don't merge when for prebuilt_apex", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, true, false, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, true, false, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, // This one should not be merged in with the others because it is for // a prebuilt_apex. - {"baz", FutureApiLevel, true, false, nil, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, true, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, - {"baz", FutureApiLevel, true, false, nil, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, + {"apex10000", FutureApiLevel, true, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"baz", FutureApiLevel, true, false, []string{"baz"}, []string{"baz"}, nil, ForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -120,11 +105,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge different UsePlatformApis but don't allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, false, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, false, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, false, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, false, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, @@ -134,11 +119,11 @@ func Test_mergeApexVariations(t *testing.T) { { name: "merge same UsePlatformApis and allow using platform api", in: []ApexInfo{ - {"foo", FutureApiLevel, false, true, nil, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, - {"bar", FutureApiLevel, false, true, nil, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, + {"foo", FutureApiLevel, false, true, []string{"foo"}, []string{"foo"}, nil, NotForPrebuiltApex}, + {"bar", FutureApiLevel, false, true, []string{"bar"}, []string{"bar"}, nil, NotForPrebuiltApex}, }, wantMerged: []ApexInfo{ - {"apex10000", FutureApiLevel, false, true, nil, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, + {"apex10000", FutureApiLevel, false, true, []string{"bar", "foo"}, []string{"bar", "foo"}, nil, NotForPrebuiltApex}, }, wantAliases: [][2]string{ {"bar", "apex10000"}, diff --git a/android/api_levels.go b/android/api_levels.go index aa55aa149..da50b1927 100644 --- a/android/api_levels.go +++ b/android/api_levels.go @@ -192,17 +192,9 @@ var FirstNonLibAndroidSupportVersion = uncheckedFinalApiLevel(21) // a core-for-system-modules.jar for the module-lib API scope. var LastWithoutModuleLibCoreSystemModules = uncheckedFinalApiLevel(31) -// If the `raw` input is the codename of an API level has been finalized, this -// function returns the API level number associated with that API level. If the -// input is *not* a finalized codename, the input is returned unmodified. -// -// For example, at the time of writing, R has been finalized as API level 30, -// but S is in development so it has no number assigned. For the following -// inputs: -// -// * "30" -> "30" -// * "R" -> "30" -// * "S" -> "S" +// ReplaceFinalizedCodenames returns the API level number associated with that API level +// if the `raw` input is the codename of an API level has been finalized. +// If the input is *not* a finalized codename, the input is returned unmodified. func ReplaceFinalizedCodenames(config Config, raw string) string { num, ok := getFinalCodenamesMap(config)[raw] if !ok { diff --git a/android/arch.go b/android/arch.go index f732a7dd0..e72614cfa 100644 --- a/android/arch.go +++ b/android/arch.go @@ -307,7 +307,7 @@ var ( // Linux is the OS for the Linux kernel plus the glibc runtime. Linux = newOsType("linux_glibc", Host, false, X86, X86_64) // LinuxMusl is the OS for the Linux kernel plus the musl runtime. - LinuxMusl = newOsType("linux_musl", Host, false, X86, X86_64) + LinuxMusl = newOsType("linux_musl", Host, false, X86, X86_64, Arm64, Arm) // Darwin is the OS for MacOS/Darwin host machines. Darwin = newOsType("darwin", Host, false, Arm64, X86_64) // LinuxBionic is the OS for the Linux kernel plus the Bionic libc runtime, but without the @@ -1825,17 +1825,19 @@ func getCommonTargets(targets []Target) []Target { for _, t := range targets { if _, found := set[t.Os.String()]; !found { set[t.Os.String()] = true - ret = append(ret, commonTargetMap[t.Os.String()]) + common := commonTargetMap[t.Os.String()] + common.HostCross = t.HostCross + ret = append(ret, common) } } return ret } -// firstTarget takes a list of Targets and a list of multilib values and returns a list of Targets +// FirstTarget takes a list of Targets and a list of multilib values and returns a list of Targets // that contains zero or one Target for each OsType, selecting the one that matches the earliest // filter. -func firstTarget(targets []Target, filters ...string) []Target { +func FirstTarget(targets []Target, filters ...string) []Target { // find the first target from each OS var ret []Target hasHost := false @@ -1865,9 +1867,9 @@ func decodeMultilibTargets(multilib string, targets []Target, prefer32 bool) ([] case "common_first": buildTargets = getCommonTargets(targets) if prefer32 { - buildTargets = append(buildTargets, firstTarget(targets, "lib32", "lib64")...) + buildTargets = append(buildTargets, FirstTarget(targets, "lib32", "lib64")...) } else { - buildTargets = append(buildTargets, firstTarget(targets, "lib64", "lib32")...) + buildTargets = append(buildTargets, FirstTarget(targets, "lib64", "lib32")...) } case "both": if prefer32 { @@ -1883,12 +1885,12 @@ func decodeMultilibTargets(multilib string, targets []Target, prefer32 bool) ([] buildTargets = filterMultilibTargets(targets, "lib64") case "first": if prefer32 { - buildTargets = firstTarget(targets, "lib32", "lib64") + buildTargets = FirstTarget(targets, "lib32", "lib64") } else { - buildTargets = firstTarget(targets, "lib64", "lib32") + buildTargets = FirstTarget(targets, "lib64", "lib32") } case "first_prefer32": - buildTargets = firstTarget(targets, "lib32", "lib64") + buildTargets = FirstTarget(targets, "lib32", "lib64") case "prefer32": buildTargets = filterMultilibTargets(targets, "lib32") if len(buildTargets) == 0 { diff --git a/android/bazel.go b/android/bazel.go index 4ef8d7881..40f2917db 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -115,6 +115,27 @@ type Bazelable interface { SetBaseModuleType(baseModuleType string) } +// MixedBuildBuildable is an interface that module types should implement in order +// to be "handled by Bazel" in a mixed build. +type MixedBuildBuildable interface { + // IsMixedBuildSupported returns true if and only if this module should be + // "handled by Bazel" in a mixed build. + // This "escape hatch" allows modules with corner-case scenarios to opt out + // of being built with Bazel. + IsMixedBuildSupported(ctx BaseModuleContext) bool + + // QueueBazelCall invokes request-queueing functions on the BazelContext + // so that these requests are handled when Bazel's cquery is invoked. + QueueBazelCall(ctx BaseModuleContext) + + // ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext) + // to set module fields and providers to propagate this module's metadata upstream. + // This effectively "bridges the gap" between Bazel and Soong in a mixed build. + // Soong modules depending on this module should be oblivious to the fact that + // this module was handled by Bazel. + ProcessBazelQueryResponse(ctx ModuleContext) +} + // BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules. type BazelModule interface { Module @@ -300,25 +321,31 @@ func (a bp2BuildConversionAllowlist) SetMixedBuildsDisabledList(mixedBuildsDisab return a } -var bp2buildAllowlist = NewBp2BuildAllowlist(). - SetDefaultConfig(allowlists.Bp2buildDefaultConfig). - SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile). - SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList). - SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList). - SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList). - SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList). - SetMixedBuildsDisabledList(allowlists.MixedBuildsDisabledList) +var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist") +var bp2buildAllowlist OncePer + +func getBp2BuildAllowList() bp2BuildConversionAllowlist { + return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} { + return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig). + SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile). + SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList). + SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList). + SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList). + SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList). + SetMixedBuildsDisabledList(allowlists.MixedBuildsDisabledList) + }).(bp2BuildConversionAllowlist) +} // GenerateCcLibraryStaticOnly returns whether a cc_library module should only // generate a static version of itself based on the current global configuration. func GenerateCcLibraryStaticOnly(moduleName string) bool { - return bp2buildAllowlist.ccLibraryStaticOnly[moduleName] + return getBp2BuildAllowList().ccLibraryStaticOnly[moduleName] } // ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be // added to the build symlink forest based on the current global configuration. func ShouldKeepExistingBuildFileForDir(dir string) bool { - return shouldKeepExistingBuildFileForDir(bp2buildAllowlist, dir) + return shouldKeepExistingBuildFileForDir(getBp2BuildAllowList(), dir) } func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, dir string) bool { @@ -338,9 +365,19 @@ func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, di return false } -// MixedBuildsEnabled checks that a module is ready to be replaced by a +// MixedBuildsEnabled returns true if a module is ready to be replaced by a +// converted or handcrafted Bazel target. As a side effect, calling this +// method will also log whether this module is mixed build enabled for +// metrics reporting. +func MixedBuildsEnabled(ctx BaseModuleContext) bool { + mixedBuildEnabled := mixedBuildPossible(ctx) + ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled) + return mixedBuildEnabled +} + +// mixedBuildPossible returns true if a module is ready to be replaced by a // converted or handcrafted Bazel target. -func (b *BazelModuleBase) MixedBuildsEnabled(ctx ModuleContext) bool { +func mixedBuildPossible(ctx BaseModuleContext) bool { if ctx.Os() == Windows { // Windows toolchains are not currently supported. return false @@ -361,7 +398,7 @@ func (b *BazelModuleBase) MixedBuildsEnabled(ctx ModuleContext) bool { // variants of a cc_library. return false } - return !bp2buildAllowlist.mixedBuildsDisabled[ctx.Module().Name()] + return !getBp2BuildAllowList().mixedBuildsDisabled[ctx.Module().Name()] } // ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel. diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 6b2be693b..f1ec55eb9 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -28,10 +29,31 @@ import ( "android/soong/bazel/cquery" "android/soong/shared" + "github.com/google/blueprint" "android/soong/bazel" ) +func init() { + RegisterMixedBuildsMutator(InitRegistrationContext) +} + +func RegisterMixedBuildsMutator(ctx RegistrationContext) { + ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel() + }) +} + +func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) { + if m := ctx.Module(); m.Enabled() { + if mixedBuildMod, ok := m.(MixedBuildBuildable); ok { + if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) { + mixedBuildMod.QueueBazelCall(ctx) + } + } + } +} + type cqueryRequest interface { // Name returns a string name for this request type. Such request type names must be unique, // and must only consist of alphanumeric characters. @@ -61,37 +83,36 @@ type cqueryKey struct { configKey configKey } -// bazelHandler is the interface for a helper object related to deferring to Bazel for -// processing a module (during Bazel mixed builds). Individual module types should define -// their own bazel handler if they support deferring to Bazel. -type BazelHandler interface { - // Issue query to Bazel to retrieve information about Bazel's view of the current module. - // If Bazel returns this information, set module properties on the current module to reflect - // the returned information. - // Returns true if information was available from Bazel, false if bazel invocation still needs to occur. - GenerateBazelBuildActions(ctx ModuleContext, label string) bool -} - +// BazelContext is a context object useful for interacting with Bazel during +// the course of a build. Use of Bazel to evaluate part of the build graph +// is referred to as a "mixed build". (Some modules are managed by Soong, +// some are managed by Bazel). To facilitate interop between these build +// subgraphs, Soong may make requests to Bazel and evaluate their responses +// so that Soong modules may accurately depend on Bazel targets. type BazelContext interface { - // The methods below involve queuing cquery requests to be later invoked - // by bazel. If any of these methods return (_, false), then the request - // has been queued to be run later. + // Add a cquery request to the bazel request queue. All queued requests + // will be sent to Bazel on a subsequent invocation of InvokeBazel. + QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) + + // ** Cquery Results Retrieval Functions + // The below functions pertain to retrieving cquery results from a prior + // InvokeBazel function call and parsing the results. // Returns result files built by building the given bazel target label. - GetOutputFiles(label string, cfgKey configKey) ([]string, bool) + GetOutputFiles(label string, cfgKey configKey) ([]string, error) - // TODO(cparsons): Other cquery-related methods should be added here. // Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order). - GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) + GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) // Returns the executable binary resultant from building together the python sources - GetPythonBinary(label string, cfgKey configKey) (string, bool) + // TODO(b/232976601): Remove. + GetPythonBinary(label string, cfgKey configKey) (string, error) - // ** End cquery methods + // ** end Cquery Results Retrieval Functions // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. - InvokeBazel() error + InvokeBazel(config Config) error // Returns true if bazel is enabled for the given configuration. BazelEnabled() bool @@ -101,6 +122,9 @@ type BazelContext interface { // Returns build statements which should get registered to reflect Bazel's outputs. BuildStatementsToRegister() []bazel.BuildStatement + + // Returns the depsets defined in Bazel's aquery response. + AqueryDepsets() []bazel.AqueryDepset } type bazelRunner interface { @@ -128,6 +152,9 @@ type bazelContext struct { // Build statements which should get registered to reflect Bazel's outputs. buildStatements []bazel.BuildStatement + + // Depsets which should be used for Bazel's build statements. + depsets []bazel.AqueryDepset } var _ BazelContext = &bazelContext{} @@ -146,22 +173,26 @@ type MockBazelContext struct { LabelToPythonBinary map[string]string } -func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { - result, ok := m.LabelToOutputFiles[label] - return result, ok +func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { + panic("unimplemented") } -func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { - result, ok := m.LabelToCcInfo[label] - return result, ok, nil +func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { + result, _ := m.LabelToOutputFiles[label] + return result, nil } -func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { - result, ok := m.LabelToPythonBinary[label] - return result, ok +func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { + result, _ := m.LabelToCcInfo[label] + return result, nil } -func (m MockBazelContext) InvokeBazel() error { +func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { + result, _ := m.LabelToPythonBinary[label] + return result, nil +} + +func (m MockBazelContext) InvokeBazel(config Config) error { panic("unimplemented") } @@ -175,52 +206,63 @@ func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { return []bazel.BuildStatement{} } +func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset { + return []bazel.AqueryDepset{} +} + var _ BazelContext = MockBazelContext{} -func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { - rawString, ok := bazelCtx.cquery(label, cquery.GetOutputFiles, cfgKey) - var ret []string - if ok { +func (bazelCtx *bazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { + key := cqueryKey{label, requestType, cfgKey} + bazelCtx.requestMutex.Lock() + defer bazelCtx.requestMutex.Unlock() + bazelCtx.requests[key] = true +} + +func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { + key := cqueryKey{label, cquery.GetOutputFiles, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { bazelOutput := strings.TrimSpace(rawString) - ret = cquery.GetOutputFiles.ParseResult(bazelOutput) + return cquery.GetOutputFiles.ParseResult(bazelOutput), nil } - return ret, ok + return nil, fmt.Errorf("no bazel response found for %v", key) } -func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { - result, ok := bazelCtx.cquery(label, cquery.GetCcInfo, cfgKey) - if !ok { - return cquery.CcInfo{}, ok, nil +func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { + key := cqueryKey{label, cquery.GetCcInfo, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { + bazelOutput := strings.TrimSpace(rawString) + return cquery.GetCcInfo.ParseResult(bazelOutput) } - - bazelOutput := strings.TrimSpace(result) - ret, err := cquery.GetCcInfo.ParseResult(bazelOutput) - return ret, ok, err + return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key) } -func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { - rawString, ok := bazelCtx.cquery(label, cquery.GetPythonBinary, cfgKey) - var ret string - if ok { +func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { + key := cqueryKey{label, cquery.GetPythonBinary, cfgKey} + if rawString, ok := bazelCtx.results[key]; ok { bazelOutput := strings.TrimSpace(rawString) - ret = cquery.GetPythonBinary.ParseResult(bazelOutput) + return cquery.GetPythonBinary.ParseResult(bazelOutput), nil } - return ret, ok + return "", fmt.Errorf("no bazel response found for %v", key) } -func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) { +func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) { panic("unimplemented") } -func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) { +func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) { panic("unimplemented") } -func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) { +func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) { panic("unimplemented") } -func (n noopBazelContext) InvokeBazel() error { +func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) { + panic("unimplemented") +} + +func (n noopBazelContext) InvokeBazel(config Config) error { panic("unimplemented") } @@ -236,6 +278,10 @@ func (m noopBazelContext) BuildStatementsToRegister() []bazel.BuildStatement { return []bazel.BuildStatement{} } +func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset { + return []bazel.AqueryDepset{} +} + func NewBazelContext(c *config) (BazelContext, error) { // TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds" // are production ready. @@ -299,24 +345,6 @@ func (context *bazelContext) BazelEnabled() bool { return true } -// Adds a cquery request to the Bazel request queue, to be later invoked, or -// returns the result of the given request if the request was already made. -// If the given request was already made (and the results are available), then -// returns (result, true). If the request is queued but no results are available, -// then returns ("", false). -func (context *bazelContext) cquery(label string, requestType cqueryRequest, - cfgKey configKey) (string, bool) { - key := cqueryKey{label, requestType, cfgKey} - if result, ok := context.results[key]; ok { - return result, true - } else { - context.requestMutex.Lock() - defer context.requestMutex.Unlock() - context.requests[key] = true - return "", false - } -} - func pwdPrefix() string { // Darwin doesn't have /proc if runtime.GOOS != "darwin" { @@ -334,6 +362,7 @@ type bazelCommand struct { type mockBazelRunner struct { bazelCommandResults map[bazelCommand]string commands []bazelCommand + extraFlags []string } func (r *mockBazelRunner) issueBazelCommand(paths *bazelPaths, @@ -341,6 +370,7 @@ func (r *mockBazelRunner) issueBazelCommand(paths *bazelPaths, command bazelCommand, extraFlags ...string) (string, string, error) { r.commands = append(r.commands, command) + r.extraFlags = append(r.extraFlags, strings.Join(extraFlags, " ")) if ret, ok := r.bazelCommandResults[command]; ok { return ret, "", nil } @@ -499,7 +529,7 @@ config_node(name = "%s", configNodesSection := "" labelsByConfig := map[string][]string{} - for val, _ := range context.requests { + for val := range context.requests { labelString := fmt.Sprintf("\"@%s\"", val.label) configString := getConfigString(val) labelsByConfig[configString] = append(labelsByConfig[configString], labelString) @@ -537,7 +567,7 @@ func indent(original string) string { // request type. func (context *bazelContext) cqueryStarlarkFileContents() []byte { requestTypeToCqueryIdEntries := map[cqueryRequest][]string{} - for val, _ := range context.requests { + for val := range context.requests { cqueryId := getCqueryId(val) mapEntryString := fmt.Sprintf("%q : True", cqueryId) requestTypeToCqueryIdEntries[val.requestType] = @@ -649,7 +679,7 @@ func (p *bazelPaths) outDir() string { // Issues commands to Bazel to receive results for all cquery requests // queued in the BazelContext. -func (context *bazelContext) InvokeBazel() error { +func (context *bazelContext) InvokeBazel(config Config) error { context.results = make(map[cqueryKey]string) var cqueryOutput string @@ -732,21 +762,37 @@ func (context *bazelContext) InvokeBazel() error { // Issue an aquery command to retrieve action information about the bazel build tree. // - // TODO(cparsons): Use --target_pattern_file to avoid command line limits. var aqueryOutput string + var coverageFlags []string + if Bool(config.productVariables.ClangCoverage) { + coverageFlags = append(coverageFlags, "--collect_code_coverage") + if len(config.productVariables.NativeCoveragePaths) > 0 || + len(config.productVariables.NativeCoverageExcludePaths) > 0 { + includePaths := JoinWithPrefixAndSeparator(config.productVariables.NativeCoveragePaths, "+", ",") + excludePaths := JoinWithPrefixAndSeparator(config.productVariables.NativeCoverageExcludePaths, "-", ",") + if len(includePaths) > 0 && len(excludePaths) > 0 { + includePaths += "," + } + coverageFlags = append(coverageFlags, fmt.Sprintf(`--instrumentation_filter=%s`, + includePaths+excludePaths)) + } + } + + extraFlags := append([]string{"--output=jsonproto"}, coverageFlags...) + aqueryOutput, _, err = context.issueBazelCommand( context.paths, bazel.AqueryBuildRootRunName, bazelCommand{"aquery", fmt.Sprintf("deps(%s)", buildrootLabel)}, // Use jsonproto instead of proto; actual proto parsing would require a dependency on Bazel's // proto sources, which would add a number of unnecessary dependencies. - "--output=jsonproto") + extraFlags...) if err != nil { return err } - context.buildStatements, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) + context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput)) if err != nil { return err } @@ -772,6 +818,10 @@ func (context *bazelContext) BuildStatementsToRegister() []bazel.BuildStatement return context.buildStatements } +func (context *bazelContext) AqueryDepsets() []bazel.AqueryDepset { + return context.depsets +} + func (context *bazelContext) OutputBase() string { return context.paths.outputBase } @@ -804,57 +854,85 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { ctx.AddNinjaFileDeps(file) } - // Register bazel-owned build statements (obtained from the aquery invocation). + for _, depset := range ctx.Config().BazelContext.AqueryDepsets() { + var outputs []Path + for _, depsetDepHash := range depset.TransitiveDepSetHashes { + otherDepsetName := bazelDepsetName(depsetDepHash) + outputs = append(outputs, PathForPhony(ctx, otherDepsetName)) + } + for _, artifactPath := range depset.DirectArtifacts { + outputs = append(outputs, PathForBazelOut(ctx, artifactPath)) + } + thisDepsetName := bazelDepsetName(depset.ContentHash) + ctx.Build(pctx, BuildParams{ + Rule: blueprint.Phony, + Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)}, + Implicits: outputs, + }) + } + + executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__") + bazelOutDir := path.Join(executionRoot, "bazel-out") for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() { if len(buildStatement.Command) < 1 { panic(fmt.Sprintf("unhandled build statement: %v", buildStatement)) } rule := NewRuleBuilder(pctx, ctx) - cmd := rule.Command() - - // cd into Bazel's execution root, which is the action cwd. - cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ &&", ctx.Config().BazelContext.OutputBase())) - - // Remove old outputs, as some actions might not rerun if the outputs are detected. - if len(buildStatement.OutputPaths) > 0 { - cmd.Text("rm -f") - for _, outputPath := range buildStatement.OutputPaths { - cmd.Text(outputPath) - } - cmd.Text("&&") - } - - for _, pair := range buildStatement.Env { - // Set per-action env variables, if any. - cmd.Flag(pair.Key + "=" + pair.Value) - } + createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx) + desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) + rule.Build(fmt.Sprintf("bazel %d", index), desc) + } +} - // The actual Bazel action. - cmd.Text(" " + buildStatement.Command) +// Register bazel-owned build statements (obtained from the aquery invocation). +func createCommand(cmd *RuleBuilderCommand, buildStatement bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx PathContext) { + // executionRoot is the action cwd. + cmd.Text(fmt.Sprintf("cd '%s' &&", executionRoot)) + // Remove old outputs, as some actions might not rerun if the outputs are detected. + if len(buildStatement.OutputPaths) > 0 { + cmd.Text("rm -f") for _, outputPath := range buildStatement.OutputPaths { - cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) - } - for _, inputPath := range buildStatement.InputPaths { - cmd.Implicit(PathForBazelOut(ctx, inputPath)) + cmd.Text(fmt.Sprintf("'%s'", outputPath)) } + cmd.Text("&&") + } - if depfile := buildStatement.Depfile; depfile != nil { - cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) - } + for _, pair := range buildStatement.Env { + // Set per-action env variables, if any. + cmd.Flag(pair.Key + "=" + pair.Value) + } - for _, symlinkPath := range buildStatement.SymlinkPaths { - cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath)) - } + // The actual Bazel action. + cmd.Text(buildStatement.Command) + + for _, outputPath := range buildStatement.OutputPaths { + cmd.ImplicitOutput(PathForBazelOut(ctx, outputPath)) + } + for _, inputPath := range buildStatement.InputPaths { + cmd.Implicit(PathForBazelOut(ctx, inputPath)) + } + for _, inputDepsetHash := range buildStatement.InputDepsetHashes { + otherDepsetName := bazelDepsetName(inputDepsetHash) + cmd.Implicit(PathForPhony(ctx, otherDepsetName)) + } - // This is required to silence warnings pertaining to unexpected timestamps. Particularly, - // some Bazel builtins (such as files in the bazel_tools directory) have far-future - // timestamps. Without restat, Ninja would emit warnings that the input files of a - // build statement have later timestamps than the outputs. - rule.Restat() + if depfile := buildStatement.Depfile; depfile != nil { + // The paths in depfile are relative to `executionRoot`. + // Hence, they need to be corrected by replacing "bazel-out" + // with the full `bazelOutDir`. + // Otherwise, implicit outputs and implicit inputs under "bazel-out/" + // would be deemed missing. + // (Note: The regexp uses a capture group because the version of sed + // does not support a look-behind pattern.) + replacement := fmt.Sprintf(`&& sed -i'' -E 's@(^|\s|")bazel-out/@\1%s/@g' '%s'`, + bazelOutDir, *depfile) + cmd.Text(replacement) + cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile)) + } - desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths) - rule.Build(fmt.Sprintf("bazel %d", index), desc) + for _, symlinkPath := range buildStatement.SymlinkPaths { + cmd.ImplicitSymlinkOutput(PathForBazelOut(ctx, symlinkPath)) } } @@ -865,21 +943,32 @@ func getCqueryId(key cqueryKey) string { func getConfigString(key cqueryKey) string { arch := key.configKey.arch if len(arch) == 0 || arch == "common" { - // Use host platform, which is currently hardcoded to be x86_64. - arch = "x86_64" + if key.configKey.osType.Class == Device { + // For the generic Android, the expected result is "target|android", which + // corresponds to the product_variable_config named "android_target" in + // build/bazel/platforms/BUILD.bazel. + arch = "target" + } else { + // Use host platform, which is currently hardcoded to be x86_64. + arch = "x86_64" + } } - os := key.configKey.osType.Name - if len(os) == 0 || os == "common_os" || os == "linux_glibc" { + osName := key.configKey.osType.Name + if len(osName) == 0 || osName == "common_os" || osName == "linux_glibc" { // Use host OS, which is currently hardcoded to be linux. - os = "linux" + osName = "linux" } - return arch + "|" + os + return arch + "|" + osName } -func GetConfigKey(ctx ModuleContext) configKey { +func GetConfigKey(ctx BaseModuleContext) configKey { return configKey{ // use string because Arch is not a valid key in go arch: ctx.Arch().String(), osType: ctx.Os(), } } + +func bazelDepsetName(contentHash string) string { + return fmt.Sprintf("bazel_depset_%s", contentHash) +} diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go index e5cff9031..ec2541b3d 100644 --- a/android/bazel_handler_test.go +++ b/android/bazel_handler_test.go @@ -4,26 +4,28 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" + + "android/soong/bazel/cquery" ) +var testConfig = TestConfig("out", nil, "", nil) + func TestRequestResultsAfterInvokeBazel(t *testing.T) { label := "//foo:bar" cfg := configKey{"arm64_armv8-a", Android} bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`, }) - g, ok := bazelContext.GetOutputFiles(label, cfg) - if ok { - t.Errorf("Did not expect cquery results prior to running InvokeBazel(), but got %s", g) - } - err := bazelContext.InvokeBazel() + bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg) + err := bazelContext.InvokeBazel(testConfig) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } - g, ok = bazelContext.GetOutputFiles(label, cfg) - if !ok { - t.Errorf("Expected cquery results after running InvokeBazel(), but got none") + g, err := bazelContext.GetOutputFiles(label, cfg) + if err != nil { + t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err) } else if w := []string{"out/foo/bar.txt"}; !reflect.DeepEqual(w, g) { t.Errorf("Expected output %s, got %s", w, g) } @@ -31,7 +33,7 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) { func TestInvokeBazelWritesBazelFiles(t *testing.T) { bazelContext, baseDir := testBazelContext(t, map[bazelCommand]string{}) - err := bazelContext.InvokeBazel() + err := bazelContext.InvokeBazel(testConfig) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } @@ -55,16 +57,17 @@ func TestInvokeBazelWritesBazelFiles(t *testing.T) { } func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { - bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ - bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: ` + type testCase struct { + input string + command string + } + + var testCases = []testCase{ + {` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -74,28 +77,103 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) { "outputIds": [1], "primaryOutputId": 1 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }] +}`, + "cd 'test/exec_root' && rm -f 'one' && touch foo", + }, {` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 10 }, + { "id": 2, "pathFragmentId": 20 }], + "actions": [{ + "targetId": 100, + "actionKey": "x", + "mnemonic": "x", + "arguments": ["bogus", "command"], + "outputIds": [1, 2], + "primaryOutputId": 1 }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }] + "pathFragments": [ + { "id": 10, "label": "one", "parentId": 30 }, + { "id": 20, "label": "one.d", "parentId": 30 }, + { "id": 30, "label": "parent" }] }`, - }) - err := bazelContext.InvokeBazel() + `cd 'test/exec_root' && rm -f 'parent/one' && bogus command && sed -i'' -E 's@(^|\s|")bazel-out/@\1test/bazel_out/@g' 'parent/one.d'`, + }, + } + + for i, testCase := range testCases { + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{ + bazelCommand{command: "aquery", expression: "deps(@soong_injection//mixed_builds:buildroot)"}: testCase.input}) + + err := bazelContext.InvokeBazel(testConfig) + if err != nil { + t.Fatalf("testCase #%d: did not expect error invoking Bazel, but got %s", i+1, err) + } + + got := bazelContext.BuildStatementsToRegister() + if want := 1; len(got) != want { + t.Fatalf("expected %d registered build statements, but got %#v", want, got) + } + + cmd := RuleBuilderCommand{} + createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", PathContextForTesting(TestConfig("out", nil, "", nil))) + if actual, expected := cmd.buf.String(), testCase.command; expected != actual { + t.Errorf("expected: [%s], actual: [%s]", expected, actual) + } + } +} + +func TestCoverageFlagsAfterInvokeBazel(t *testing.T) { + testConfig.productVariables.ClangCoverage = boolPtr(true) + + testConfig.productVariables.NativeCoveragePaths = []string{"foo1", "foo2"} + testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1", "bar2"} + verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1,+foo2,-bar1,-bar2`) + + testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} + testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} + verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1,-bar1`) + + testConfig.productVariables.NativeCoveragePaths = []string{"foo1"} + testConfig.productVariables.NativeCoverageExcludePaths = nil + verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=+foo1`) + + testConfig.productVariables.NativeCoveragePaths = nil + testConfig.productVariables.NativeCoverageExcludePaths = []string{"bar1"} + verifyExtraFlags(t, testConfig, `--collect_code_coverage --instrumentation_filter=-bar1`) + + testConfig.productVariables.ClangCoverage = boolPtr(false) + actual := verifyExtraFlags(t, testConfig, ``) + if strings.Contains(actual, "--collect_code_coverage") || + strings.Contains(actual, "--instrumentation_filter=") { + t.Errorf("Expected code coverage disabled, but got %#v", actual) + } +} + +func verifyExtraFlags(t *testing.T, config Config, expected string) string { + bazelContext, _ := testBazelContext(t, map[bazelCommand]string{}) + + err := bazelContext.InvokeBazel(config) if err != nil { t.Fatalf("Did not expect error invoking Bazel, but got %s", err) } - got := bazelContext.BuildStatementsToRegister() - if want := 1; len(got) != want { - t.Errorf("Expected %d registered build statements, got %#v", want, got) + flags := bazelContext.bazelRunner.(*mockBazelRunner).extraFlags + if expected := 3; len(flags) != expected { + t.Errorf("Expected %d extra flags got %#v", expected, flags) + } + + actual := flags[1] + if !strings.Contains(actual, expected) { + t.Errorf("Expected %#v got %#v", expected, actual) } + + return actual } func testBazelContext(t *testing.T, bazelCommandResults map[bazelCommand]string) (*bazelContext, string) { diff --git a/android/bazel_paths.go b/android/bazel_paths.go index fa10f62b4..1d0a6d541 100644 --- a/android/bazel_paths.go +++ b/android/bazel_paths.go @@ -449,25 +449,51 @@ func (p BazelOutPath) objPathWithExt(ctx ModuleOutPathContext, subdir, ext strin return PathForModuleObj(ctx, subdir, pathtools.ReplaceExtension(p.path, ext)) } -// PathForBazelOut returns a Path representing the paths... under an output directory dedicated to -// bazel-owned outputs. -func PathForBazelOut(ctx PathContext, paths ...string) BazelOutPath { - execRootPathComponents := append([]string{"execroot", "__main__"}, paths...) - execRootPath := filepath.Join(execRootPathComponents...) - validatedExecRootPath, err := validatePath(execRootPath) +// PathForBazelOutRelative returns a BazelOutPath representing the path under an output directory dedicated to +// bazel-owned outputs. Calling .Rel() on the result will give the input path as relative to the given +// relativeRoot. +func PathForBazelOutRelative(ctx PathContext, relativeRoot string, path string) BazelOutPath { + validatedPath, err := validatePath(filepath.Join("execroot", "__main__", path)) if err != nil { reportPathError(ctx, err) } + relativeRootPath := filepath.Join("execroot", "__main__", relativeRoot) + if pathComponents := strings.Split(path, "/"); len(pathComponents) >= 3 && + pathComponents[0] == "bazel-out" && pathComponents[2] == "bin" { + // If the path starts with something like: bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/ + // make it relative to that folder. bazel-out/volatile-status.txt is an example + // of something that starts with bazel-out but is not relative to the bin folder + relativeRootPath = filepath.Join("execroot", "__main__", pathComponents[0], pathComponents[1], pathComponents[2], relativeRoot) + } + + var relPath string + if relPath, err = filepath.Rel(relativeRootPath, validatedPath); err != nil || strings.HasPrefix(relPath, "../") { + // We failed to make this path relative to execroot/__main__, fall back to a non-relative path + // One case where this happens is when path is ../bazel_tools/something + relativeRootPath = "" + relPath = validatedPath + } - outputPath := OutputPath{basePath{"", ""}, + outputPath := OutputPath{ + basePath{"", ""}, ctx.Config().soongOutDir, - ctx.Config().BazelContext.OutputBase()} + ctx.Config().BazelContext.OutputBase(), + } return BazelOutPath{ - OutputPath: outputPath.withRel(validatedExecRootPath), + // .withRel() appends its argument onto the current path, and only the most + // recently appended part is returned by outputPath.rel(). + // So outputPath.rel() will return relPath. + OutputPath: outputPath.withRel(relativeRootPath).withRel(relPath), } } +// PathForBazelOut returns a BazelOutPath representing the path under an output directory dedicated to +// bazel-owned outputs. +func PathForBazelOut(ctx PathContext, path string) BazelOutPath { + return PathForBazelOutRelative(ctx, "", path) +} + // PathsForBazelOut returns a list of paths representing the paths under an output directory // dedicated to Bazel-owned outputs. func PathsForBazelOut(ctx PathContext, paths []string) Paths { diff --git a/android/bazel_paths_test.go b/android/bazel_paths_test.go new file mode 100644 index 000000000..b047511c7 --- /dev/null +++ b/android/bazel_paths_test.go @@ -0,0 +1,108 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "path/filepath" + "testing" +) + +type TestBazelPathContext struct{} + +func (*TestBazelPathContext) Config() Config { + cfg := NullConfig("out", "out/soong") + cfg.BazelContext = MockBazelContext{ + OutputBaseDir: "out/bazel", + } + return cfg +} + +func (*TestBazelPathContext) AddNinjaFileDeps(deps ...string) { + panic("Unimplemented") +} + +func TestPathForBazelOut(t *testing.T) { + ctx := &TestBazelPathContext{} + out := PathForBazelOut(ctx, "foo/bar/baz/boq.txt") + expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt") + if out.String() != expectedPath { + t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) + } + + expectedRelPath := "foo/bar/baz/boq.txt" + if out.Rel() != expectedRelPath { + t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) + } +} + +func TestPathForBazelOutRelative(t *testing.T) { + ctx := &TestBazelPathContext{} + out := PathForBazelOutRelative(ctx, "foo/bar", "foo/bar/baz/boq.txt") + + expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/foo/bar/baz/boq.txt") + if out.String() != expectedPath { + t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) + } + + expectedRelPath := "baz/boq.txt" + if out.Rel() != expectedRelPath { + t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) + } +} + +func TestPathForBazelOutRelativeUnderBinFolder(t *testing.T) { + ctx := &TestBazelPathContext{} + out := PathForBazelOutRelative(ctx, "foo/bar", "bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt") + + expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/__main__/bazel-out/linux_x86_64-fastbuild-ST-b4ef1c4402f9/bin/foo/bar/baz/boq.txt") + if out.String() != expectedPath { + t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) + } + + expectedRelPath := "baz/boq.txt" + if out.Rel() != expectedRelPath { + t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) + } +} + +func TestPathForBazelOutOutsideOfExecroot(t *testing.T) { + ctx := &TestBazelPathContext{} + out := PathForBazelOut(ctx, "../bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar") + + expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar") + if out.String() != expectedPath { + t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) + } + + expectedRelPath := "execroot/bazel_tools/linux_x86_64-fastbuild/bin/tools/android/java_base_extras.jar" + if out.Rel() != expectedRelPath { + t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) + } +} + +func TestPathForBazelOutRelativeWithParentDirectoryRoot(t *testing.T) { + ctx := &TestBazelPathContext{} + out := PathForBazelOutRelative(ctx, "../bazel_tools", "../bazel_tools/foo/bar/baz.sh") + + expectedPath := filepath.Join(ctx.Config().BazelContext.OutputBase(), "execroot/bazel_tools/foo/bar/baz.sh") + if out.String() != expectedPath { + t.Errorf("incorrect OutputPath: expected %q, got %q", expectedPath, out.String()) + } + + expectedRelPath := "foo/bar/baz.sh" + if out.Rel() != expectedRelPath { + t.Errorf("incorrect OutputPath.Rel(): expected %q, got %q", expectedRelPath, out.Rel()) + } +} diff --git a/android/bazel_test.go b/android/bazel_test.go index 482df2abd..e14649e3f 100644 --- a/android/bazel_test.go +++ b/android/bazel_test.go @@ -14,11 +14,12 @@ package android import ( - "android/soong/android/allowlists" - "android/soong/bazel" "fmt" "testing" + "android/soong/android/allowlists" + "android/soong/bazel" + "github.com/google/blueprint" "github.com/google/blueprint/proptools" ) @@ -386,3 +387,37 @@ func TestBp2BuildAllowlist(t *testing.T) { }) } } + +func TestBp2buildAllowList(t *testing.T) { + allowlist := getBp2BuildAllowList() + for k, v := range allowlists.Bp2buildDefaultConfig { + if allowlist.defaultConfig[k] != v { + t.Errorf("bp2build default config of %s: expected: %v, got: %v", k, v, allowlist.defaultConfig[k]) + } + } + for k, v := range allowlists.Bp2buildKeepExistingBuildFile { + if allowlist.keepExistingBuildFile[k] != v { + t.Errorf("bp2build keep existing build file of %s: expected: %v, got: %v", k, v, allowlist.keepExistingBuildFile[k]) + } + } + for _, k := range allowlists.Bp2buildModuleTypeAlwaysConvertList { + if !allowlist.moduleTypeAlwaysConvert[k] { + t.Errorf("bp2build module type always convert of %s: expected: true, got: %v", k, allowlist.moduleTypeAlwaysConvert[k]) + } + } + for _, k := range allowlists.Bp2buildModuleDoNotConvertList { + if !allowlist.moduleDoNotConvert[k] { + t.Errorf("bp2build module do not convert of %s: expected: true, got: %v", k, allowlist.moduleDoNotConvert[k]) + } + } + for _, k := range allowlists.Bp2buildCcLibraryStaticOnlyList { + if !allowlist.ccLibraryStaticOnly[k] { + t.Errorf("bp2build cc library static only of %s: expected: true, got: %v", k, allowlist.ccLibraryStaticOnly[k]) + } + } + for _, k := range allowlists.MixedBuildsDisabledList { + if !allowlist.mixedBuildsDisabled[k] { + t.Errorf("bp2build mix build disabled of %s: expected: true, got: %v", k, allowlist.mixedBuildsDisabled[k]) + } + } +} diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go index 6339a7181..acebdbbff 100644 --- a/android/buildinfo_prop.go +++ b/android/buildinfo_prop.go @@ -89,6 +89,7 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { writeProp("ro.build.version.security_patch", config.PlatformSecurityPatch()) writeProp("ro.build.version.base_os", config.PlatformBaseOS()) writeProp("ro.build.version.min_supported_target_sdk", config.PlatformMinSupportedTargetSdkVersion()) + writeProp("ro.build.version.known_codenames", config.PlatformVersionKnownCodenames()) if config.Eng() { writeProp("ro.build.type", "eng") @@ -109,7 +110,6 @@ func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) { writeProp("ro.build.display.id", $BUILD_DISPLAY_ID) writeProp("ro.build.version.incremental", $BUILD_NUMBER) writeProp("ro.build.version.preview_sdk_fingerprint", $PLATFORM_PREVIEW_SDK_FINGERPRINT) - writeProp("ro.build.version.known_codenames", $PLATFORM_VERSION_KNOWN_CODENAMES) writeProp("ro.build.version.release_or_preview_display", $PLATFORM_DISPLAY_VERSION) writeProp("ro.build.date", `$DATE`) writeProp("ro.build.date.utc", `$DATE +%s`) diff --git a/android/config.go b/android/config.go index 250c312b7..47346fcf1 100644 --- a/android/config.go +++ b/android/config.go @@ -170,6 +170,10 @@ type config struct { ninjaFileDepsSet sync.Map OncePer + + mixedBuildsLock sync.Mutex + mixedBuildEnabledModules map[string]struct{} + mixedBuildDisabledModules map[string]struct{} } type deviceConfig struct { @@ -375,7 +379,9 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string // passed to PathForSource or PathForModuleSrc. TestAllowNonExistentPaths: true, - BazelContext: noopBazelContext{}, + BazelContext: noopBazelContext{}, + mixedBuildDisabledModules: make(map[string]struct{}), + mixedBuildEnabledModules: make(map[string]struct{}), } config.deviceConfig = &deviceConfig{ config: config, @@ -410,7 +416,7 @@ func modifyTestConfigToSupportArchMutator(testConfig Config) { config.BuildOSTarget = config.Targets[config.BuildOS][0] config.BuildOSCommonTarget = getCommonTargets(config.Targets[config.BuildOS])[0] config.AndroidCommonTarget = getCommonTargets(config.Targets[Android])[0] - config.AndroidFirstDeviceTarget = firstTarget(config.Targets[Android], "lib64", "lib32")[0] + config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0] config.TestProductVariables.DeviceArch = proptools.StringPtr("arm64") config.TestProductVariables.DeviceArchVariant = proptools.StringPtr("armv8-a") config.TestProductVariables.DeviceSecondaryArch = proptools.StringPtr("arm") @@ -466,8 +472,10 @@ func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir strin runGoTests: runGoTests, multilibConflicts: make(map[ArchType]bool), - moduleListFile: moduleListFile, - fs: pathtools.NewOsFs(absSrcDir), + moduleListFile: moduleListFile, + fs: pathtools.NewOsFs(absSrcDir), + mixedBuildDisabledModules: make(map[string]struct{}), + mixedBuildEnabledModules: make(map[string]struct{}), } config.deviceConfig = &deviceConfig{ @@ -546,11 +554,11 @@ func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir strin // Compilation targets for Android. if len(config.Targets[Android]) > 0 { config.AndroidCommonTarget = getCommonTargets(config.Targets[Android])[0] - config.AndroidFirstDeviceTarget = firstTarget(config.Targets[Android], "lib64", "lib32")[0] + config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0] } config.BazelContext, err = NewBazelContext(config) - config.bp2buildPackageConfig = bp2buildAllowlist + config.bp2buildPackageConfig = getBp2BuildAllowList() return Config{config}, err } @@ -690,6 +698,10 @@ func (c *config) IsEnvFalse(key string) bool { return value == "0" || value == "n" || value == "no" || value == "off" || value == "false" } +func (c *config) TargetsJava17() bool { + return c.IsEnvTrue("EXPERIMENTAL_TARGET_JAVA_VERSION_17") +} + // EnvDeps returns the environment variables this build depends on. The first // call to this function blocks future reads from the environment. func (c *config) EnvDeps() map[string]string { @@ -781,6 +793,10 @@ func (c *config) PlatformVersionLastStable() string { return String(c.productVariables.Platform_version_last_stable) } +func (c *config) PlatformVersionKnownCodenames() string { + return String(c.productVariables.Platform_version_known_codenames) +} + func (c *config) MinSupportedSdkVersion() ApiLevel { return uncheckedFinalApiLevel(19) } @@ -1486,6 +1502,10 @@ func (c *config) MissingUsesLibraries() []string { return c.productVariables.MissingUsesLibraries } +func (c *config) TargetMultitreeUpdateMeta() bool { + return c.productVariables.MultitreeUpdateMeta +} + func (c *deviceConfig) DeviceArch() string { return String(c.config.productVariables.DeviceArch) } @@ -1677,6 +1697,10 @@ func (c *deviceConfig) BuildBrokenInputDir(name string) bool { return InList(name, c.config.productVariables.BuildBrokenInputDirModules) } +func (c *deviceConfig) BuildBrokenDepfile() bool { + return Bool(c.config.productVariables.BuildBrokenDepfile) +} + func (c *deviceConfig) RequiresInsecureExecmemForSwiftshader() bool { return c.config.productVariables.RequiresInsecureExecmemForSwiftshader } @@ -2034,3 +2058,14 @@ func (c *config) RBEWrapper() string { func (c *config) UseHostMusl() bool { return Bool(c.productVariables.HostMusl) } + +func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) { + moduleName := ctx.Module().Name() + c.mixedBuildsLock.Lock() + defer c.mixedBuildsLock.Unlock() + if useBazel { + c.mixedBuildEnabledModules[moduleName] = struct{}{} + } else { + c.mixedBuildDisabledModules[moduleName] = struct{}{} + } +} diff --git a/android/deapexer.go b/android/deapexer.go index 265f5312b..6a93f6087 100644 --- a/android/deapexer.go +++ b/android/deapexer.go @@ -15,6 +15,8 @@ package android import ( + "strings" + "github.com/google/blueprint" ) @@ -148,12 +150,19 @@ type RequiresFilesFromPrebuiltApexTag interface { func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { var di *DeapexerInfo ctx.VisitDirectDepsWithTag(DeapexerTag, func(m Module) { - p := ctx.OtherModuleProvider(m, DeapexerProvider).(DeapexerInfo) + c := ctx.OtherModuleProvider(m, DeapexerProvider).(DeapexerInfo) + p := &c if di != nil { + // If two DeapexerInfo providers have been found then check if they are + // equivalent. If they are then use the selected one, otherwise fail. + if selected := equivalentDeapexerInfoProviders(di, p); selected != nil { + di = selected + return + } ctx.ModuleErrorf("Multiple installable prebuilt APEXes provide ambiguous deapexers: %s and %s", di.ApexModuleName(), p.ApexModuleName()) } - di = &p + di = p }) if di != nil { return di @@ -162,3 +171,33 @@ func FindDeapexerProviderForModule(ctx ModuleContext) *DeapexerInfo { ctx.ModuleErrorf("No prebuilt APEX provides a deapexer module for APEX variant %s", ai.ApexVariationName) return nil } + +// removeCompressedApexSuffix removes the _compressed suffix from the name if present. +func removeCompressedApexSuffix(name string) string { + return strings.TrimSuffix(name, "_compressed") +} + +// equivalentDeapexerInfoProviders checks to make sure that the two DeapexerInfo structures are +// equivalent. +// +// At the moment <x> and <x>_compressed APEXes are treated as being equivalent. +// +// If they are not equivalent then this returns nil, otherwise, this returns the DeapexerInfo that +// should be used by the build, which is always the uncompressed one. That ensures that the behavior +// of the build is not dependent on which prebuilt APEX is visited first. +func equivalentDeapexerInfoProviders(p1 *DeapexerInfo, p2 *DeapexerInfo) *DeapexerInfo { + n1 := removeCompressedApexSuffix(p1.ApexModuleName()) + n2 := removeCompressedApexSuffix(p2.ApexModuleName()) + + // If the names don't match then they are not equivalent. + if n1 != n2 { + return nil + } + + // Select the uncompressed APEX. + if n1 == removeCompressedApexSuffix(n1) { + return p1 + } else { + return p2 + } +} diff --git a/android/defaults.go b/android/defaults.go index 8b121f6d9..54f44bcec 100644 --- a/android/defaults.go +++ b/android/defaults.go @@ -15,6 +15,8 @@ package android import ( + "bytes" + "fmt" "reflect" "github.com/google/blueprint" @@ -67,9 +69,11 @@ type Defaultable interface { // Set the property structures into which defaults will be added. setProperties(props []interface{}, variableProperties interface{}) - // Apply defaults from the supplied Defaults to the property structures supplied to + // Apply defaults from the supplied DefaultsModule to the property structures supplied to // setProperties(...). - applyDefaults(TopDownMutatorContext, []Defaults) + applyDefaults(TopDownMutatorContext, []DefaultsModule) + + applySingleDefaultsWithTracker(EarlyModuleContext, DefaultsModule, defaultsTrackerFunc) // Set the hook to be called after any defaults have been applied. // @@ -115,9 +119,23 @@ type DefaultsVisibilityProperties struct { Defaults_visibility []string } +// AdditionalDefaultsProperties contains properties of defaults modules which +// can have other defaults applied. +type AdditionalDefaultsProperties struct { + + // The list of properties set by the default whose values must not be changed by any module that + // applies these defaults. It is an error if a property is not supported by the defaults module or + // has not been set to a non-zero value. If this contains "*" then that must be the only entry in + // which case all properties that are set on this defaults will be protected (except the + // protected_properties and visibility. + Protected_properties []string +} + type DefaultsModuleBase struct { DefaultableModuleBase + defaultsProperties AdditionalDefaultsProperties + // Included to support setting bazel_module.label for multiple Soong modules to the same Bazel // target. This is primarily useful for modules that were architecture specific and instead are // handled in Bazel as a select(). @@ -151,6 +169,18 @@ type Defaults interface { // DefaultsModuleBase will type-assert to the Defaults interface. isDefaults() bool + // additionalDefaultableProperties returns additional properties provided by the defaults which + // can themselves have defaults applied. + additionalDefaultableProperties() []interface{} + + // protectedProperties returns the names of the properties whose values cannot be changed by a + // module that applies these defaults. + protectedProperties() []string + + // setProtectedProperties sets the names of the properties whose values cannot be changed by a + // module that applies these defaults. + setProtectedProperties(protectedProperties []string) + // Get the structures containing the properties for which defaults can be provided. properties() []interface{} @@ -167,6 +197,18 @@ type DefaultsModule interface { Bazelable } +func (d *DefaultsModuleBase) additionalDefaultableProperties() []interface{} { + return []interface{}{&d.defaultsProperties} +} + +func (d *DefaultsModuleBase) protectedProperties() []string { + return d.defaultsProperties.Protected_properties +} + +func (d *DefaultsModuleBase) setProtectedProperties(protectedProperties []string) { + d.defaultsProperties.Protected_properties = protectedProperties +} + func (d *DefaultsModuleBase) properties() []interface{} { return d.defaultableProperties } @@ -190,6 +232,10 @@ func InitDefaultsModule(module DefaultsModule) { &ApexProperties{}, &distProperties{}) + // Additional properties of defaults modules that can themselves have + // defaults applied. + module.AddProperties(module.additionalDefaultableProperties()...) + // Bazel module must be initialized _before_ Defaults to be included in cc_defaults module. InitBazelModule(module) initAndroidModuleBase(module) @@ -218,6 +264,57 @@ func InitDefaultsModule(module DefaultsModule) { // The applicable licenses property for defaults is 'licenses'. setPrimaryLicensesProperty(module, "licenses", &commonProperties.Licenses) + AddLoadHook(module, func(ctx LoadHookContext) { + + protectedProperties := module.protectedProperties() + if len(protectedProperties) == 0 { + return + } + + propertiesAvailable := map[string]struct{}{} + propertiesSet := map[string]struct{}{} + + // A defaults tracker which will keep track of which properties have been set on this module. + collector := func(defaults DefaultsModule, property string, dstValue interface{}, srcValue interface{}) bool { + value := reflect.ValueOf(dstValue) + propertiesAvailable[property] = struct{}{} + if !value.IsZero() { + propertiesSet[property] = struct{}{} + } + // Skip all the properties so that there are no changes to the defaults. + return false + } + + // Try and apply this module's defaults to itself, so that the properties can be collected but + // skip all the properties so it doesn't actually do anything. + module.applySingleDefaultsWithTracker(ctx, module, collector) + + if InList("*", protectedProperties) { + if len(protectedProperties) != 1 { + ctx.PropertyErrorf("protected_properties", `if specified then "*" must be the only property listed`) + return + } + + // Do not automatically protect the protected_properties property. + delete(propertiesSet, "protected_properties") + + // Or the visibility property. + delete(propertiesSet, "visibility") + + // Replace the "*" with the names of all the properties that have been set. + protectedProperties = SortedStringKeys(propertiesSet) + module.setProtectedProperties(protectedProperties) + } else { + for _, property := range protectedProperties { + if _, ok := propertiesAvailable[property]; !ok { + ctx.PropertyErrorf(property, "property is not supported by this module type %q", + ctx.ModuleType()) + } else if _, ok := propertiesSet[property]; !ok { + ctx.PropertyErrorf(property, "is not set; protected properties must be explicitly set") + } + } + } + }) } var _ Defaults = (*DefaultsModuleBase)(nil) @@ -269,35 +366,204 @@ func applyNamespacedVariableDefaults(defaultDep Defaults, ctx TopDownMutatorCont b.setNamespacedVariableProps(dst) } +// defaultValueInfo contains information about each default value that applies to a protected +// property. +type defaultValueInfo struct { + // The DefaultsModule providing the value, which may be defined on that module or applied as a + // default from other modules. + module Module + + // The default value, as returned by getComparableValue + defaultValue reflect.Value +} + +// protectedPropertyInfo contains information about each property that has to be protected when +// applying defaults. +type protectedPropertyInfo struct { + // True if the property was set on the module to which defaults are applied, this is an error. + propertySet bool + + // The original value of the property on the module, as returned by getComparableValue. + originalValue reflect.Value + + // A list of defaults for the property that are being applied. + defaultValues []defaultValueInfo +} + +// getComparableValue takes a reflect.Value that may be a pointer to another value and returns a +// reflect.Value to the underlying data or the original if was not a pointer or was nil. The +// returned values can then be compared for equality. +func getComparableValue(value reflect.Value) reflect.Value { + if value.IsZero() { + return value + } + for value.Kind() == reflect.Ptr { + value = value.Elem() + } + return value +} + func (defaultable *DefaultableModuleBase) applyDefaults(ctx TopDownMutatorContext, - defaultsList []Defaults) { + defaultsList []DefaultsModule) { + + // Collate information on all the properties protected by each of the default modules applied + // to this module. + allProtectedProperties := map[string]*protectedPropertyInfo{} + for _, defaults := range defaultsList { + for _, property := range defaults.protectedProperties() { + info := allProtectedProperties[property] + if info == nil { + info = &protectedPropertyInfo{} + allProtectedProperties[property] = info + } + } + } + + // If there are any protected properties then collate information about attempts to change them. + var protectedPropertyInfoCollector defaultsTrackerFunc + if len(allProtectedProperties) > 0 { + protectedPropertyInfoCollector = func(defaults DefaultsModule, property string, + dstValue interface{}, srcValue interface{}) bool { + + // If the property is not protected then return immediately. + info := allProtectedProperties[property] + if info == nil { + return true + } + + currentValue := reflect.ValueOf(dstValue) + if info.defaultValues == nil { + info.propertySet = !currentValue.IsZero() + info.originalValue = getComparableValue(currentValue) + } + + defaultValue := reflect.ValueOf(srcValue) + if !defaultValue.IsZero() { + info.defaultValues = append(info.defaultValues, + defaultValueInfo{defaults, getComparableValue(defaultValue)}) + } + + return true + } + } for _, defaults := range defaultsList { if ctx.Config().runningAsBp2Build { applyNamespacedVariableDefaults(defaults, ctx) } - for _, prop := range defaultable.defaultableProperties { - if prop == defaultable.defaultableVariableProperties { - defaultable.applyDefaultVariableProperties(ctx, defaults, prop) + + defaultable.applySingleDefaultsWithTracker(ctx, defaults, protectedPropertyInfoCollector) + } + + // Check the status of any protected properties. + for property, info := range allProtectedProperties { + if len(info.defaultValues) == 0 { + // No defaults were applied to the protected properties. Possibly because this module type + // does not support any of them. + continue + } + + // Check to make sure that there are no conflicts between the defaults. + conflictingDefaults := false + previousDefaultValue := reflect.ValueOf(false) + for _, defaultInfo := range info.defaultValues { + defaultValue := defaultInfo.defaultValue + if previousDefaultValue.IsZero() { + previousDefaultValue = defaultValue + } else if !reflect.DeepEqual(previousDefaultValue.Interface(), defaultValue.Interface()) { + conflictingDefaults = true + break + } + } + + if conflictingDefaults { + var buf bytes.Buffer + for _, defaultInfo := range info.defaultValues { + buf.WriteString(fmt.Sprintf("\n defaults module %q provides value %#v", + ctx.OtherModuleName(defaultInfo.module), defaultInfo.defaultValue)) + } + result := buf.String() + ctx.ModuleErrorf("has conflicting default values for protected property %q:%s", property, result) + continue + } + + // Now check to see whether there the current module tried to override/append to the defaults. + if info.propertySet { + originalValue := info.originalValue + // Just compare against the first defaults. + defaultValue := info.defaultValues[0].defaultValue + defaults := info.defaultValues[0].module + + if originalValue.Kind() == reflect.Slice { + ctx.ModuleErrorf("attempts to append %q to protected property %q's value of %q defined in module %q", + originalValue, + property, + defaultValue, + ctx.OtherModuleName(defaults)) } else { - defaultable.applyDefaultProperties(ctx, defaults, prop) + same := reflect.DeepEqual(originalValue.Interface(), defaultValue.Interface()) + message := "" + if same { + message = fmt.Sprintf(" with a matching value (%#v) so this property can simply be removed.", originalValue) + } else { + message = fmt.Sprintf(" with a different value (override %#v with %#v) so removing the property may necessitate other changes.", defaultValue, originalValue) + } + ctx.ModuleErrorf("attempts to override protected property %q defined in module %q%s", + property, + ctx.OtherModuleName(defaults), message) } } } } +func (defaultable *DefaultableModuleBase) applySingleDefaultsWithTracker(ctx EarlyModuleContext, defaults DefaultsModule, tracker defaultsTrackerFunc) { + for _, prop := range defaultable.defaultableProperties { + var err error + if prop == defaultable.defaultableVariableProperties { + err = defaultable.applyDefaultVariableProperties(defaults, prop, tracker) + } else { + err = defaultable.applyDefaultProperties(defaults, prop, tracker) + } + if err != nil { + if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { + ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) + } else { + panic(err) + } + } + } +} + +// defaultsTrackerFunc is the type of a function that can be used to track how defaults are applied. +type defaultsTrackerFunc func(defaults DefaultsModule, property string, + dstValue interface{}, srcValue interface{}) bool + +// filterForTracker wraps a defaultsTrackerFunc in a proptools.ExtendPropertyFilterFunc +func filterForTracker(defaults DefaultsModule, tracker defaultsTrackerFunc) proptools.ExtendPropertyFilterFunc { + if tracker == nil { + return nil + } + return func(property string, + dstField, srcField reflect.StructField, + dstValue, srcValue interface{}) (bool, error) { + + apply := tracker(defaults, property, dstValue, srcValue) + return apply, nil + } +} + // Product variable properties need special handling, the type of the filtered product variable // property struct may not be identical between the defaults module and the defaultable module. // Use PrependMatchingProperties to apply whichever properties match. -func (defaultable *DefaultableModuleBase) applyDefaultVariableProperties(ctx TopDownMutatorContext, - defaults Defaults, defaultableProp interface{}) { +func (defaultable *DefaultableModuleBase) applyDefaultVariableProperties(defaults DefaultsModule, + defaultableProp interface{}, tracker defaultsTrackerFunc) error { if defaultableProp == nil { - return + return nil } defaultsProp := defaults.productVariableProperties() if defaultsProp == nil { - return + return nil } dst := []interface{}{ @@ -307,31 +573,26 @@ func (defaultable *DefaultableModuleBase) applyDefaultVariableProperties(ctx Top proptools.CloneEmptyProperties(reflect.ValueOf(defaultsProp)).Interface(), } - err := proptools.PrependMatchingProperties(dst, defaultsProp, nil) - if err != nil { - if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { - ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) - } else { - panic(err) - } - } + filter := filterForTracker(defaults, tracker) + + return proptools.PrependMatchingProperties(dst, defaultsProp, filter) } -func (defaultable *DefaultableModuleBase) applyDefaultProperties(ctx TopDownMutatorContext, - defaults Defaults, defaultableProp interface{}) { +func (defaultable *DefaultableModuleBase) applyDefaultProperties(defaults DefaultsModule, + defaultableProp interface{}, checker defaultsTrackerFunc) error { + + filter := filterForTracker(defaults, checker) for _, def := range defaults.properties() { if proptools.TypeEqual(defaultableProp, def) { - err := proptools.PrependProperties(defaultableProp, def, nil) + err := proptools.PrependProperties(defaultableProp, def, filter) if err != nil { - if propertyErr, ok := err.(*proptools.ExtendPropertyError); ok { - ctx.PropertyErrorf(propertyErr.Property, "%s", propertyErr.Err.Error()) - } else { - panic(err) - } + return err } } } + + return nil } func RegisterDefaultsPreArchMutators(ctx RegisterMutatorsContext) { @@ -348,12 +609,12 @@ func defaultsDepsMutator(ctx BottomUpMutatorContext) { func defaultsMutator(ctx TopDownMutatorContext) { if defaultable, ok := ctx.Module().(Defaultable); ok { if len(defaultable.defaults().Defaults) > 0 { - var defaultsList []Defaults + var defaultsList []DefaultsModule seen := make(map[Defaults]bool) ctx.WalkDeps(func(module, parent Module) bool { if ctx.OtherModuleDependencyTag(module) == DefaultsDepTag { - if defaults, ok := module.(Defaults); ok { + if defaults, ok := module.(DefaultsModule); ok { if !seen[defaults] { seen[defaults] = true defaultsList = append(defaultsList, defaults) diff --git a/android/defaults_test.go b/android/defaults_test.go index a7542abb3..d80f40cc6 100644 --- a/android/defaults_test.go +++ b/android/defaults_test.go @@ -19,7 +19,14 @@ import ( ) type defaultsTestProperties struct { - Foo []string + Foo []string + Bar []string + Nested struct { + Fizz *bool + } + Other struct { + Buzz *string + } } type defaultsTestModule struct { @@ -130,3 +137,167 @@ func TestDefaultsAllowMissingDependencies(t *testing.T) { // TODO: missing transitive defaults is currently not handled _ = missingTransitiveDefaults } + +func TestProtectedProperties_ProtectedPropertyNotSet(t *testing.T) { + bp := ` + defaults { + name: "transitive", + protected_properties: ["foo"], + } + ` + + GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).ExtendWithErrorHandler(FixtureExpectsAtLeastOneErrorMatchingPattern( + "module \"transitive\": foo: is not set; protected properties must be explicitly set")). + RunTest(t) +} + +func TestProtectedProperties_ProtectedPropertyNotLeaf(t *testing.T) { + bp := ` + defaults { + name: "transitive", + protected_properties: ["nested"], + nested: { + fizz: true, + }, + } + ` + + GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).ExtendWithErrorHandler(FixtureExpectsAtLeastOneErrorMatchingPattern( + `\Qmodule "transitive": nested: property is not supported by this module type "defaults"\E`)). + RunTest(t) +} + +// TestProtectedProperties_ApplyDefaults makes sure that the protected_properties property has +// defaults applied. +func TestProtectedProperties_HasDefaultsApplied(t *testing.T) { + + bp := ` + defaults { + name: "transitive", + protected_properties: ["foo"], + foo: ["transitive"], + } + + defaults { + name: "defaults", + defaults: ["transitive"], + protected_properties: ["bar"], + bar: ["defaults"], + } + ` + + result := GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).RunTest(t) + + defaults := result.Module("defaults", "").(DefaultsModule) + AssertDeepEquals(t, "defaults protected properties", []string{"foo", "bar"}, defaults.protectedProperties()) +} + +// TestProtectedProperties_ProtectAllProperties makes sure that protected_properties: ["*"] protects +// all properties. +func TestProtectedProperties_ProtectAllProperties(t *testing.T) { + + bp := ` + defaults { + name: "transitive", + protected_properties: ["other.buzz"], + other: { + buzz: "transitive", + }, + } + + defaults { + name: "defaults", + defaults: ["transitive"], + visibility: ["//visibility:private"], + protected_properties: ["*"], + foo: ["other"], + bar: ["defaults"], + nested: { + fizz: true, + } + } + ` + + result := GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).RunTest(t) + + defaults := result.Module("defaults", "").(DefaultsModule) + AssertDeepEquals(t, "defaults protected properties", []string{"other.buzz", "bar", "foo", "nested.fizz"}, + defaults.protectedProperties()) +} + +func TestProtectedProperties_DetectedOverride(t *testing.T) { + bp := ` + defaults { + name: "defaults", + protected_properties: ["foo", "nested.fizz"], + foo: ["defaults"], + nested: { + fizz: true, + }, + } + + test { + name: "foo", + defaults: ["defaults"], + foo: ["module"], + nested: { + fizz: false, + }, + } + ` + + GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).ExtendWithErrorHandler(FixtureExpectsAllErrorsToMatchAPattern( + []string{ + `\Qmodule "foo": attempts to append ["module"] to protected property "foo"'s value of ["defaults"] defined in module "defaults"\E`, + `\Qmodule "foo": attempts to override protected property "nested.fizz" defined in module "defaults" with a different value (override true with false) so removing the property may necessitate other changes.\E`, + })).RunTest(t) +} + +func TestProtectedProperties_DefaultsConflict(t *testing.T) { + bp := ` + defaults { + name: "defaults1", + protected_properties: ["other.buzz"], + other: { + buzz: "value", + }, + } + + defaults { + name: "defaults2", + protected_properties: ["other.buzz"], + other: { + buzz: "another", + }, + } + + test { + name: "foo", + defaults: ["defaults1", "defaults2"], + } + ` + + GroupFixturePreparers( + prepareForDefaultsTest, + FixtureWithRootAndroidBp(bp), + ).ExtendWithErrorHandler(FixtureExpectsAtLeastOneErrorMatchingPattern( + `\Qmodule "foo": has conflicting default values for protected property "other.buzz": + defaults module "defaults1" provides value "value" + defaults module "defaults2" provides value "another"\E`, + )).RunTest(t) +} diff --git a/android/filegroup.go b/android/filegroup.go index 50356d1c3..9e5769a82 100644 --- a/android/filegroup.go +++ b/android/filegroup.go @@ -18,6 +18,7 @@ import ( "strings" "android/soong/bazel" + "android/soong/bazel/cquery" "github.com/google/blueprint" ) @@ -101,6 +102,7 @@ type fileGroup struct { srcs Paths } +var _ MixedBuildBuildable = (*fileGroup)(nil) var _ SourceFileProducer = (*fileGroup)(nil) // filegroup contains a list of files that are referenced by other modules @@ -114,33 +116,21 @@ func FileGroupFactory() Module { return module } -func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) { - if !fg.MixedBuildsEnabled(ctx) { - return - } +var _ blueprint.JSONActionSupplier = (*fileGroup)(nil) - archVariant := ctx.Arch().String() - osVariant := ctx.Os() - if len(fg.Srcs()) == 1 && fg.Srcs()[0].Base() == fg.Name() { - // This will be a regular file target, not filegroup, in Bazel. - // See FilegroupBp2Build for more information. - archVariant = Common.String() - osVariant = CommonOS +func (fg *fileGroup) JSONActions() []blueprint.JSONAction { + ins := make([]string, 0, len(fg.srcs)) + outs := make([]string, 0, len(fg.srcs)) + for _, p := range fg.srcs { + ins = append(ins, p.String()) + outs = append(outs, p.Rel()) } - - bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{archVariant, osVariant}) - if !ok { - return - } - - bazelOuts := make(Paths, 0, len(filePaths)) - for _, p := range filePaths { - src := PathForBazelOut(ctx, p) - bazelOuts = append(bazelOuts, src) + return []blueprint.JSONAction{ + blueprint.JSONAction{ + Inputs: ins, + Outputs: outs, + }, } - - fg.srcs = bazelOuts } func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) { @@ -148,8 +138,6 @@ func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) { if fg.properties.Path != nil { fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path)) } - - fg.maybeGenerateBazelBuildActions(ctx) } func (fg *fileGroup) Srcs() Paths { @@ -161,3 +149,37 @@ func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) { ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " ")) } } + +func (fg *fileGroup) QueueBazelCall(ctx BaseModuleContext) { + bazelCtx := ctx.Config().BazelContext + + bazelCtx.QueueBazelRequest( + fg.GetBazelLabel(ctx, fg), + cquery.GetOutputFiles, + configKey{Common.String(), CommonOS}) +} + +func (fg *fileGroup) IsMixedBuildSupported(ctx BaseModuleContext) bool { + return true +} + +func (fg *fileGroup) ProcessBazelQueryResponse(ctx ModuleContext) { + fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs) + if fg.properties.Path != nil { + fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path)) + } + + bazelCtx := ctx.Config().BazelContext + filePaths, err := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{Common.String(), CommonOS}) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + bazelOuts := make(Paths, 0, len(filePaths)) + for _, p := range filePaths { + bazelOuts = append(bazelOuts, PathForBazelOutRelative(ctx, ctx.ModuleDir(), p)) + } + + fg.srcs = bazelOuts +} diff --git a/android/fixture.go b/android/fixture.go index 728f0318f..0690a5a6c 100644 --- a/android/fixture.go +++ b/android/fixture.go @@ -586,6 +586,18 @@ func FixtureExpectsAllErrorsToMatchAPattern(patterns []string) FixtureErrorHandl }) } +// FixtureExpectsOneErrorPattern returns an error handler that will cause the test to fail +// if there is more than one error or the error does not match the pattern. +// +// If the test fails this handler will call `result.FailNow()` which will exit the goroutine within +// which the test is being run which means that the RunTest() method will not return. +func FixtureExpectsOneErrorPattern(pattern string) FixtureErrorHandler { + return FixtureCustomErrorHandler(func(t *testing.T, result *TestResult) { + t.Helper() + CheckErrorsAgainstExpectations(t, result.Errs, []string{pattern}) + }) +} + // FixtureCustomErrorHandler creates a custom error handler func FixtureCustomErrorHandler(function func(t *testing.T, result *TestResult)) FixtureErrorHandler { return simpleErrorHandler{ diff --git a/android/gen_notice.go b/android/gen_notice.go new file mode 100644 index 000000000..e2b839f69 --- /dev/null +++ b/android/gen_notice.go @@ -0,0 +1,212 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "fmt" + "strings" + + "github.com/google/blueprint/proptools" +) + +func init() { + RegisterGenNoticeBuildComponents(InitRegistrationContext) +} + +// Register the gen_notice module type. +func RegisterGenNoticeBuildComponents(ctx RegistrationContext) { + ctx.RegisterSingletonType("gen_notice_build_rules", GenNoticeBuildRulesFactory) + ctx.RegisterModuleType("gen_notice", GenNoticeFactory) +} + +type genNoticeBuildRules struct{} + +func (s *genNoticeBuildRules) GenerateBuildActions(ctx SingletonContext) { + ctx.VisitAllModules(func(m Module) { + gm, ok := m.(*genNoticeModule) + if !ok { + return + } + if len(gm.missing) > 0 { + missingReferencesRule(ctx, gm) + return + } + out := BuildNoticeTextOutputFromLicenseMetadata + if proptools.Bool(gm.properties.Xml) { + out = BuildNoticeXmlOutputFromLicenseMetadata + } else if proptools.Bool(gm.properties.Html) { + out = BuildNoticeHtmlOutputFromLicenseMetadata + } + defaultName := "" + if len(gm.properties.For) > 0 { + defaultName = gm.properties.For[0] + } + + modules := make([]Module, 0) + for _, name := range gm.properties.For { + mods := ctx.ModuleVariantsFromName(gm, name) + for _, mod := range mods { + if mod == nil { + continue + } + modules = append(modules, mod) + } + } + if ctx.Failed() { + return + } + out(ctx, gm.output, ctx.ModuleName(gm), + proptools.StringDefault(gm.properties.ArtifactName, defaultName), + []string{ + ctx.Config().OutDir() + "/", + ctx.Config().SoongOutDir() + "/", + }, modules...) + }) +} + +func GenNoticeBuildRulesFactory() Singleton { + return &genNoticeBuildRules{} +} + +type genNoticeProperties struct { + // For specifies the modules for which to generate a notice file. + For []string + // ArtifactName specifies the internal name to use for the notice file. + // It appears in the "used by:" list for targets whose entire name is stripped by --strip_prefix. + ArtifactName *string + // Stem specifies the base name of the output file. + Stem *string `android:"arch_variant"` + // Html indicates an html-format file is needed. The default is text. Can be Html or Xml but not both. + Html *bool + // Xml indicates an xml-format file is needed. The default is text. Can be Html or Xml but not both. + Xml *bool + // Gzipped indicates the output file must be compressed with gzip. Will append .gz to suffix if not there. + Gzipped *bool + // Suffix specifies the file extension to use. Defaults to .html for html, .xml for xml, or no extension for text. + Suffix *string + // Visibility specifies where this license can be used + Visibility []string +} + +type genNoticeModule struct { + ModuleBase + DefaultableModuleBase + + properties genNoticeProperties + + output OutputPath + missing []string +} + +func (m *genNoticeModule) DepsMutator(ctx BottomUpMutatorContext) { + if proptools.Bool(m.properties.Html) && proptools.Bool(m.properties.Xml) { + ctx.ModuleErrorf("can be html or xml but not both") + } + if !ctx.Config().AllowMissingDependencies() { + var missing []string + // Verify the modules for which to generate notices exist. + for _, otherMod := range m.properties.For { + if !ctx.OtherModuleExists(otherMod) { + missing = append(missing, otherMod) + } + } + if len(missing) == 1 { + ctx.PropertyErrorf("for", "no %q module exists", missing[0]) + } else if len(missing) > 1 { + ctx.PropertyErrorf("for", "modules \"%s\" do not exist", strings.Join(missing, "\", \"")) + } + } +} + +func (m *genNoticeModule) getStem() string { + stem := m.base().BaseModuleName() + if m.properties.Stem != nil { + stem = proptools.String(m.properties.Stem) + } + return stem +} + +func (m *genNoticeModule) getSuffix() string { + suffix := "" + if m.properties.Suffix == nil { + if proptools.Bool(m.properties.Html) { + suffix = ".html" + } else if proptools.Bool(m.properties.Xml) { + suffix = ".xml" + } + } else { + suffix = proptools.String(m.properties.Suffix) + } + if proptools.Bool(m.properties.Gzipped) && !strings.HasSuffix(suffix, ".gz") { + suffix += ".gz" + } + return suffix +} + +func (m *genNoticeModule) GenerateAndroidBuildActions(ctx ModuleContext) { + if ctx.Config().AllowMissingDependencies() { + // Verify the modules for which to generate notices exist. + for _, otherMod := range m.properties.For { + if !ctx.OtherModuleExists(otherMod) { + m.missing = append(m.missing, otherMod) + } + } + m.missing = append(m.missing, ctx.GetMissingDependencies()...) + m.missing = FirstUniqueStrings(m.missing) + } + out := m.getStem() + m.getSuffix() + m.output = PathForModuleOut(ctx, out).OutputPath +} + +func GenNoticeFactory() Module { + module := &genNoticeModule{} + + base := module.base() + module.AddProperties(&base.nameProperties, &module.properties) + + // The visibility property needs to be checked and parsed by the visibility module. + setPrimaryVisibilityProperty(module, "visibility", &module.properties.Visibility) + + initAndroidModuleBase(module) + InitDefaultableModule(module) + + return module +} + +var _ OutputFileProducer = (*genNoticeModule)(nil) + +// Implements OutputFileProducer +func (m *genNoticeModule) OutputFiles(tag string) (Paths, error) { + if tag == "" { + return Paths{m.output}, nil + } + return nil, fmt.Errorf("unrecognized tag %q", tag) +} + +// missingReferencesRule emits an ErrorRule for missing module references. +func missingReferencesRule(ctx BuilderContext, m *genNoticeModule) { + if len(m.missing) < 1 { + panic(fmt.Errorf("missing references rule requested with no missing references")) + } + + ctx.Build(pctx, BuildParams{ + Rule: ErrorRule, + Output: m.output, + Description: "notice for " + proptools.StringDefault(m.properties.ArtifactName, "container"), + Args: map[string]string{ + "error": m.Name() + " references missing module(s): " + strings.Join(m.missing, ", "), + }, + }) +} diff --git a/android/gen_notice_test.go b/android/gen_notice_test.go new file mode 100644 index 000000000..4ad2ecfa9 --- /dev/null +++ b/android/gen_notice_test.go @@ -0,0 +1,164 @@ +package android + +import ( + "testing" + + "github.com/google/blueprint" +) + +var genNoticeTests = []struct { + name string + fs MockFS + expectedErrors []string +}{ + { + name: "gen_notice must not accept licenses property", + fs: map[string][]byte{ + "top/Android.bp": []byte(` + gen_notice { + name: "top_license", + licenses: ["other_license"], + }`), + }, + expectedErrors: []string{ + `unrecognized property "licenses"`, + }, + }, + { + name: "bad gen_notice", + fs: map[string][]byte{ + "top/Android.bp": []byte(` + gen_notice { + name: "top_notice", + for: ["top_rule"], + }`), + "other/Android.bp": []byte(` + mock_genrule { + name: "other_rule", + dep: ["top_notice"], + }`), + }, + expectedErrors: []string{ + `module "top_notice": for: no "top_rule" module exists`, + }, + }, + { + name: "doubly bad gen_notice", + fs: map[string][]byte{ + "top/Android.bp": []byte(` + gen_notice { + name: "top_notice", + for: ["top_rule", "other_rule"], + }`), + }, + expectedErrors: []string{ + `module "top_notice": for: modules "top_rule", "other_rule" do not exist`, + }, + }, + { + name: "good gen_notice", + fs: map[string][]byte{ + "top/Android.bp": []byte(` + gen_notice { + name: "top_notice", + for: ["top_rule"], + } + + mock_genrule { + name: "top_rule", + dep: ["top_notice"], + }`), + "other/Android.bp": []byte(` + mock_genrule { + name: "other_rule", + dep: ["top_notice"], + }`), + }, + }, + { + name: "multiple license kinds", + fs: map[string][]byte{ + "top/Android.bp": []byte(` + gen_notice { + name: "top_notice", + for: ["top_rule"], + } + + gen_notice { + name: "top_html_notice", + html: true, + for: ["top_rule"], + } + + gen_notice { + name: "top_xml_notice", + xml: true, + for: ["top_notice"], + } + + mock_genrule { + name: "top_rule", + dep: [ + "top_notice", + "top_html_notice", + "top_xml_notice", + ], + }`), + "other/Android.bp": []byte(` + mock_genrule { + name: "other_rule", + dep: ["top_xml_notice"], + }`), + }, + }, +} + +func TestGenNotice(t *testing.T) { + for _, test := range genNoticeTests { + t.Run(test.name, func(t *testing.T) { + GroupFixturePreparers( + PrepareForTestWithGenNotice, + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.RegisterModuleType("mock_genrule", newMockGenruleModule) + }), + test.fs.AddToFixture(), + ). + ExtendWithErrorHandler(FixtureExpectsAllErrorsToMatchAPattern(test.expectedErrors)). + RunTest(t) + }) + } +} + +type mockGenruleProperties struct { + Dep []string +} + +type mockGenruleModule struct { + ModuleBase + DefaultableModuleBase + + properties mockGenruleProperties +} + +func newMockGenruleModule() Module { + m := &mockGenruleModule{} + m.AddProperties(&m.properties) + InitAndroidArchModule(m, HostAndDeviceSupported, MultilibCommon) + InitDefaultableModule(m) + return m +} + +type genruleDepTag struct { + blueprint.BaseDependencyTag +} + +func (j *mockGenruleModule) DepsMutator(ctx BottomUpMutatorContext) { + m, ok := ctx.Module().(Module) + if !ok { + return + } + ctx.AddDependency(m, genruleDepTag{}, j.properties.Dep...) +} + +func (p *mockGenruleModule) GenerateAndroidBuildActions(ModuleContext) { +} diff --git a/android/license_metadata.go b/android/license_metadata.go index 6a5b0dafb..f2ab0a44c 100644 --- a/android/license_metadata.go +++ b/android/license_metadata.go @@ -105,7 +105,7 @@ func buildLicenseMetadata(ctx ModuleContext, licenseMetadataFile WritablePath) { if p := base.commonProperties.Effective_package_name; p != nil { args = append(args, - "-p "+proptools.NinjaAndShellEscape(*p)) + `-p `+proptools.NinjaAndShellEscapeIncludingSpaces(*p)) } args = append(args, diff --git a/android/licenses.go b/android/licenses.go index bd14b26ca..c47b3e63d 100644 --- a/android/licenses.go +++ b/android/licenses.go @@ -303,6 +303,7 @@ func exemptFromRequiredApplicableLicensesProperty(module Module) bool { switch reflect.TypeOf(module).String() { case "*android.licenseModule": // is a license, doesn't need one case "*android.licenseKindModule": // is a license, doesn't need one + case "*android.genNoticeModule": // contains license texts as data case "*android.NamespaceModule": // just partitions things, doesn't add anything case "*android.soongConfigModuleTypeModule": // creates aliases for modules with licenses case "*android.soongConfigModuleTypeImport": // creates aliases for modules with licenses @@ -330,6 +331,8 @@ func init() { func licensesMakeVarsProvider(ctx MakeVarsContext) { ctx.Strict("BUILD_LICENSE_METADATA", ctx.Config().HostToolPath(ctx, "build_license_metadata").String()) + ctx.Strict("COPY_LICENSE_METADATA", + ctx.Config().HostToolPath(ctx, "copy_license_metadata").String()) ctx.Strict("HTMLNOTICE", ctx.Config().HostToolPath(ctx, "htmlnotice").String()) ctx.Strict("XMLNOTICE", ctx.Config().HostToolPath(ctx, "xmlnotice").String()) ctx.Strict("TEXTNOTICE", ctx.Config().HostToolPath(ctx, "textnotice").String()) diff --git a/android/makevars.go b/android/makevars.go index ece7091b6..a74185a5c 100644 --- a/android/makevars.go +++ b/android/makevars.go @@ -472,7 +472,8 @@ func (s *makeVarsSingleton) writeInstalls(installs, symlinks []katiInstall) []by fmt.Fprintf(buf, "\tchmod +x $@\n") } if extraFiles := install.extraFiles; extraFiles != nil { - fmt.Fprintf(buf, "\tunzip -qDD -d '%s' '%s'\n", extraFiles.dir.String(), extraFiles.zip.String()) + fmt.Fprintf(buf, "\t( unzip -qDD -d '%s' '%s' 2>&1 | grep -v \"zipfile is empty\"; exit $${PIPESTATUS[0]} ) || \\\n", extraFiles.dir.String(), extraFiles.zip.String()) + fmt.Fprintf(buf, "\t ( code=$$?; if [ $$code -ne 0 -a $$code -ne 1 ]; then exit $$code; fi )\n") } fmt.Fprintln(buf) } diff --git a/android/metrics.go b/android/metrics.go index 9038bde9f..1580f82b1 100644 --- a/android/metrics.go +++ b/android/metrics.go @@ -17,6 +17,7 @@ package android import ( "io/ioutil" "runtime" + "sort" "github.com/google/blueprint/metrics" "google.golang.org/protobuf/proto" @@ -78,6 +79,23 @@ func collectMetrics(config Config, eventHandler metrics.EventHandler) *soong_met } metrics.Events = append(metrics.Events, &perfInfo) } + mixedBuildsInfo := soong_metrics_proto.MixedBuildsInfo{} + mixedBuildEnabledModules := make([]string, 0, len(config.mixedBuildEnabledModules)) + for module, _ := range config.mixedBuildEnabledModules { + mixedBuildEnabledModules = append(mixedBuildEnabledModules, module) + } + + mixedBuildDisabledModules := make([]string, 0, len(config.mixedBuildDisabledModules)) + for module, _ := range config.mixedBuildDisabledModules { + mixedBuildDisabledModules = append(mixedBuildDisabledModules, module) + } + // Sorted for deterministic output. + sort.Strings(mixedBuildEnabledModules) + sort.Strings(mixedBuildDisabledModules) + + mixedBuildsInfo.MixedBuildEnabledModules = mixedBuildEnabledModules + mixedBuildsInfo.MixedBuildDisabledModules = mixedBuildDisabledModules + metrics.MixedBuildsInfo = &mixedBuildsInfo return metrics } diff --git a/android/module.go b/android/module.go index ab68e24f1..292508199 100644 --- a/android/module.go +++ b/android/module.go @@ -1177,37 +1177,103 @@ func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *topDownMutator data := &attrs.Data - required := depsToLabelList(props.Required) archVariantProps := mod.GetArchVariantProperties(ctx, &commonProperties{}) var enabledProperty bazel.BoolAttribute - if props.Enabled != nil { - enabledProperty.Value = props.Enabled + + onlyAndroid := false + neitherHostNorDevice := false + + osSupport := map[string]bool{} + + // if the target is enabled and supports arch variance, determine the defaults based on the module + // type's host or device property and host_supported/device_supported properties + if mod.commonProperties.ArchSpecific { + moduleSupportsDevice := mod.DeviceSupported() + moduleSupportsHost := mod.HostSupported() + if moduleSupportsHost && !moduleSupportsDevice { + // for host only, we specify as unsupported on android rather than listing all host osSupport + // TODO(b/220874839): consider replacing this with a constraint that covers all host osSupport + // instead + enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false)) + } else if moduleSupportsDevice && !moduleSupportsHost { + enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(true)) + // specify as a positive to ensure any target-specific enabled can be resolved + // also save that a target is only android, as if there is only the positive restriction on + // android, it'll be dropped, so we may need to add it back later + onlyAndroid = true + } else if !moduleSupportsHost && !moduleSupportsDevice { + neitherHostNorDevice = true + } + + for _, os := range OsTypeList() { + if os.Class == Host { + osSupport[os.Name] = moduleSupportsHost + } else if os.Class == Device { + osSupport[os.Name] = moduleSupportsDevice + } + } } + if neitherHostNorDevice { + // we can't build this, disable + enabledProperty.Value = proptools.BoolPtr(false) + } else if props.Enabled != nil { + enabledProperty.SetValue(props.Enabled) + if !*props.Enabled { + for os, enabled := range osSupport { + if val := enabledProperty.SelectValue(bazel.OsConfigurationAxis, os); enabled && val != nil && *val { + // if this should be disabled by default, clear out any enabling we've done + enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, os, nil) + } + } + } + } + + required := depsToLabelList(props.Required) for axis, configToProps := range archVariantProps { for config, _props := range configToProps { if archProps, ok := _props.(*commonProperties); ok { - required.SetSelectValue(axis, config, depsToLabelList(archProps.Required).Value) - if archProps.Enabled != nil { - enabledProperty.SetSelectValue(axis, config, archProps.Enabled) + // TODO(b/234748998) Remove this requiredFiltered workaround when aapt2 converts successfully + requiredFiltered := archProps.Required + if name == "apexer" { + requiredFiltered = make([]string, 0, len(archProps.Required)) + for _, req := range archProps.Required { + if req != "aapt2" && req != "apexer" { + requiredFiltered = append(requiredFiltered, req) + } + } + } + required.SetSelectValue(axis, config, depsToLabelList(requiredFiltered).Value) + if !neitherHostNorDevice { + if archProps.Enabled != nil { + if axis != bazel.OsConfigurationAxis || osSupport[config] { + enabledProperty.SetSelectValue(axis, config, archProps.Enabled) + } + } } } } } - if enabledPropertyOverrides.Value != nil { - enabledProperty.Value = enabledPropertyOverrides.Value - } - for _, axis := range enabledPropertyOverrides.SortedConfigurationAxes() { - configToBools := enabledPropertyOverrides.ConfigurableValues[axis] - for cfg, val := range configToBools { - enabledProperty.SetSelectValue(axis, cfg, &val) + if !neitherHostNorDevice { + if enabledPropertyOverrides.Value != nil { + enabledProperty.Value = enabledPropertyOverrides.Value + } + for _, axis := range enabledPropertyOverrides.SortedConfigurationAxes() { + configToBools := enabledPropertyOverrides.ConfigurableValues[axis] + for cfg, val := range configToBools { + if axis != bazel.OsConfigurationAxis || osSupport[cfg] { + enabledProperty.SetSelectValue(axis, cfg, &val) + } + } } } productConfigEnabledLabels := []bazel.Label{} - if !proptools.BoolDefault(enabledProperty.Value, true) { + // TODO(b/234497586): Soong config variables and product variables have different overriding behavior, we + // should handle it correctly + if !proptools.BoolDefault(enabledProperty.Value, true) && !neitherHostNorDevice { // If the module is not enabled by default, then we can check if a // product variable enables it productConfigEnabledLabels = productVariableConfigEnableLabels(ctx) @@ -1224,11 +1290,6 @@ func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *topDownMutator productConfigEnabledLabels, nil, }) - moduleSupportsDevice := mod.commonProperties.HostOrDeviceSupported&deviceSupported == deviceSupported - if mod.commonProperties.HostOrDeviceSupported != NeitherHostNorDeviceSupported && !moduleSupportsDevice { - enabledProperty.SetSelectValue(bazel.OsConfigurationAxis, Android.Name, proptools.BoolPtr(false)) - } - platformEnabledAttribute, err := enabledProperty.ToLabelListAttribute( bazel.LabelList{[]bazel.Label{bazel.Label{Label: "@platforms//:incompatible"}}, nil}, bazel.LabelList{[]bazel.Label{}, nil}) @@ -1236,6 +1297,13 @@ func (attrs *CommonAttributes) fillCommonBp2BuildModuleAttrs(ctx *topDownMutator ctx.ModuleErrorf("Error processing platform enabled attribute: %s", err) } + // if android is the only arch/os enabled, then add a restriction to only be compatible with android + if platformEnabledAttribute.IsNil() && onlyAndroid { + l := bazel.LabelAttribute{} + l.SetValue(bazel.Label{Label: bazel.OsConfigurationAxis.SelectKey(Android.Name)}) + platformEnabledAttribute.Add(&l) + } + data.Append(required) constraints := constraintAttributes{} @@ -2275,7 +2343,11 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) return } - m.module.GenerateAndroidBuildActions(ctx) + if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled { + mixedBuildMod.ProcessBazelQueryResponse(ctx) + } else { + m.module.GenerateAndroidBuildActions(ctx) + } if ctx.Failed() { return } @@ -2331,6 +2403,18 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.variables = ctx.variables } +func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) { + if !ctx.Config().BazelContext.BazelEnabled() { + return nil, false + } + if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok { + if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) { + return mixedBuildMod, true + } + } + return nil, false +} + // Check the supplied dist structure to make sure that it is valid. // // property - the base property, e.g. dist or dists[1], which is combined with the @@ -2446,7 +2530,7 @@ type baseModuleContext struct { bazelConversionMode bool } -func (b *baseModuleContext) BazelConversionMode() bool { +func (b *baseModuleContext) isBazelConversionMode() bool { return b.bazelConversionMode } func (b *baseModuleContext) OtherModuleName(m blueprint.Module) string { @@ -2835,7 +2919,7 @@ func (b *baseModuleContext) GetDirectDep(name string) (blueprint.Module, bluepri } func (b *baseModuleContext) ModuleFromName(name string) (blueprint.Module, bool) { - if !b.BazelConversionMode() { + if !b.isBazelConversionMode() { panic("cannot call ModuleFromName if not in bazel conversion mode") } if moduleName, _ := SrcIsModuleWithTag(name); moduleName != "" { @@ -3208,8 +3292,9 @@ func (m *moduleContext) installFile(installPath InstallPath, name string, srcPat extraCmds := "" if extraZip != nil { - extraCmds += fmt.Sprintf(" && unzip -qDD -d '%s' '%s'", + extraCmds += fmt.Sprintf(" && ( unzip -qDD -d '%s' '%s' 2>&1 | grep -v \"zipfile is empty\"; exit $${PIPESTATUS[0]} )", extraZip.dir.String(), extraZip.zip.String()) + extraCmds += " || ( code=$$?; if [ $$code -ne 0 -a $$code -ne 1 ]; then exit $$code; fi )" implicitDeps = append(implicitDeps, extraZip.zip) } diff --git a/android/mutator.go b/android/mutator.go index 02a614353..9e4aa59ad 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -93,6 +93,7 @@ type RegisterMutatorsContext interface { TopDown(name string, m TopDownMutator) MutatorHandle BottomUp(name string, m BottomUpMutator) MutatorHandle BottomUpBlueprint(name string, m blueprint.BottomUpMutator) MutatorHandle + Transition(name string, m TransitionMutator) } type RegisterMutatorFunc func(RegisterMutatorsContext) @@ -232,9 +233,6 @@ type BaseMutatorContext interface { // Rename all variants of a module. The new name is not visible to calls to ModuleName, // AddDependency or OtherModuleName until after this mutator pass is complete. Rename(name string) - - // BazelConversionMode returns whether this mutator is being run as part of Bazel Conversion. - BazelConversionMode() bool } type TopDownMutator func(TopDownMutatorContext) @@ -424,6 +422,182 @@ func (x *registerMutatorsContext) BottomUpBlueprint(name string, m blueprint.Bot return mutator } +type IncomingTransitionContext interface { + // Module returns the target of the dependency edge for which the transition + // is being computed + Module() Module + + // Config returns the configuration for the build. + Config() Config +} + +type OutgoingTransitionContext interface { + // Module returns the target of the dependency edge for which the transition + // is being computed + Module() Module + + // DepTag() Returns the dependency tag through which this dependency is + // reached + DepTag() blueprint.DependencyTag +} + +// Transition mutators implement a top-down mechanism where a module tells its +// direct dependencies what variation they should be built in but the dependency +// has the final say. +// +// When implementing a transition mutator, one needs to implement four methods: +// - Split() that tells what variations a module has by itself +// - OutgoingTransition() where a module tells what it wants from its +// dependency +// - IncomingTransition() where a module has the final say about its own +// variation +// - Mutate() that changes the state of a module depending on its variation +// +// That the effective variation of module B when depended on by module A is the +// composition the outgoing transition of module A and the incoming transition +// of module B. +// +// the outgoing transition should not take the properties of the dependency into +// account, only those of the module that depends on it. For this reason, the +// dependency is not even passed into it as an argument. Likewise, the incoming +// transition should not take the properties of the depending module into +// account and is thus not informed about it. This makes for a nice +// decomposition of the decision logic. +// +// A given transition mutator only affects its own variation; other variations +// stay unchanged along the dependency edges. +// +// Soong makes sure that all modules are created in the desired variations and +// that dependency edges are set up correctly. This ensures that "missing +// variation" errors do not happen and allows for more flexible changes in the +// value of the variation among dependency edges (as oppposed to bottom-up +// mutators where if module A in variation X depends on module B and module B +// has that variation X, A must depend on variation X of B) +// +// The limited power of the context objects passed to individual mutators +// methods also makes it more difficult to shoot oneself in the foot. Complete +// safety is not guaranteed because no one prevents individual transition +// mutators from mutating modules in illegal ways and for e.g. Split() or +// Mutate() to run their own visitations of the transitive dependency of the +// module and both of these are bad ideas, but it's better than no guardrails at +// all. +// +// This model is pretty close to Bazel's configuration transitions. The mapping +// between concepts in Soong and Bazel is as follows: +// - Module == configured target +// - Variant == configuration +// - Variation name == configuration flag +// - Variation == configuration flag value +// - Outgoing transition == attribute transition +// - Incoming transition == rule transition +// +// The Split() method does not have a Bazel equivalent and Bazel split +// transitions do not have a Soong equivalent. +// +// Mutate() does not make sense in Bazel due to the different models of the +// two systems: when creating new variations, Soong clones the old module and +// thus some way is needed to change it state whereas Bazel creates each +// configuration of a given configured target anew. +type TransitionMutator interface { + // Split returns the set of variations that should be created for a module no + // matter who depends on it. Used when Make depends on a particular variation + // or when the module knows its variations just based on information given to + // it in the Blueprint file. This method should not mutate the module it is + // called on. + Split(ctx BaseModuleContext) []string + + // Called on a module to determine which variation it wants from its direct + // dependencies. The dependency itself can override this decision. This method + // should not mutate the module itself. + OutgoingTransition(ctx OutgoingTransitionContext, sourceVariation string) string + + // Called on a module to determine which variation it should be in based on + // the variation modules that depend on it want. This gives the module a final + // say about its own variations. This method should not mutate the module + // itself. + IncomingTransition(ctx IncomingTransitionContext, incomingVariation string) string + + // Called after a module was split into multiple variations on each variation. + // It should not split the module any further but adding new dependencies is + // fine. Unlike all the other methods on TransitionMutator, this method is + // allowed to mutate the module. + Mutate(ctx BottomUpMutatorContext, variation string) +} + +type androidTransitionMutator struct { + finalPhase bool + bazelConversionMode bool + mutator TransitionMutator +} + +func (a *androidTransitionMutator) Split(ctx blueprint.BaseModuleContext) []string { + if m, ok := ctx.Module().(Module); ok { + moduleContext := m.base().baseModuleContextFactory(ctx) + moduleContext.bazelConversionMode = a.bazelConversionMode + return a.mutator.Split(&moduleContext) + } else { + return []string{""} + } +} + +type outgoingTransitionContextImpl struct { + bp blueprint.OutgoingTransitionContext +} + +func (c *outgoingTransitionContextImpl) Module() Module { + return c.bp.Module().(Module) +} + +func (c *outgoingTransitionContextImpl) DepTag() blueprint.DependencyTag { + return c.bp.DepTag() +} + +func (a *androidTransitionMutator) OutgoingTransition(ctx blueprint.OutgoingTransitionContext, sourceVariation string) string { + if _, ok := ctx.Module().(Module); ok { + return a.mutator.OutgoingTransition(&outgoingTransitionContextImpl{bp: ctx}, sourceVariation) + } else { + return "" + } +} + +type incomingTransitionContextImpl struct { + bp blueprint.IncomingTransitionContext +} + +func (c *incomingTransitionContextImpl) Module() Module { + return c.bp.Module().(Module) +} + +func (c *incomingTransitionContextImpl) Config() Config { + return c.bp.Config().(Config) +} + +func (a *androidTransitionMutator) IncomingTransition(ctx blueprint.IncomingTransitionContext, incomingVariation string) string { + if _, ok := ctx.Module().(Module); ok { + return a.mutator.IncomingTransition(&incomingTransitionContextImpl{bp: ctx}, incomingVariation) + } else { + return "" + } +} + +func (a *androidTransitionMutator) Mutate(ctx blueprint.BottomUpMutatorContext, variation string) { + if am, ok := ctx.Module().(Module); ok { + a.mutator.Mutate(bottomUpMutatorContextFactory(ctx, am, a.finalPhase, a.bazelConversionMode), variation) + } +} + +func (x *registerMutatorsContext) Transition(name string, m TransitionMutator) { + atm := &androidTransitionMutator{ + finalPhase: x.finalPhase, + bazelConversionMode: x.bazelConversionMode, + mutator: m, + } + mutator := &mutator{ + name: name, + transitionMutator: atm} + x.mutators = append(x.mutators, mutator) +} + func (x *registerMutatorsContext) mutatorName(name string) string { if x.bazelConversionMode { return name + "_bp2build" @@ -459,6 +633,8 @@ func (mutator *mutator) register(ctx *Context) { handle = blueprintCtx.RegisterBottomUpMutator(mutator.name, mutator.bottomUpMutator) } else if mutator.topDownMutator != nil { handle = blueprintCtx.RegisterTopDownMutator(mutator.name, mutator.topDownMutator) + } else if mutator.transitionMutator != nil { + blueprintCtx.RegisterTransitionMutator(mutator.name, mutator.transitionMutator) } if mutator.parallel { handle.Parallel() @@ -626,28 +802,11 @@ func (b *bottomUpMutatorContext) SetDefaultDependencyVariation(variation *string func (b *bottomUpMutatorContext) AddVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { - if b.bazelConversionMode { - _, noSelfDeps := RemoveFromList(b.ModuleName(), names) - if len(noSelfDeps) == 0 { - return []blueprint.Module(nil) - } - // In Bazel conversion mode, mutators should not have created any variants. So, when adding a - // dependency, the variations would not exist and the dependency could not be added, by - // specifying no variations, we will allow adding the dependency to succeed. - return b.bp.AddFarVariationDependencies(nil, tag, noSelfDeps...) - } - return b.bp.AddVariationDependencies(variations, tag, names...) } func (b *bottomUpMutatorContext) AddFarVariationDependencies(variations []blueprint.Variation, tag blueprint.DependencyTag, names ...string) []blueprint.Module { - if b.bazelConversionMode { - // In Bazel conversion mode, mutators should not have created any variants. So, when adding a - // dependency, the variations would not exist and the dependency could not be added, by - // specifying no variations, we will allow adding the dependency to succeed. - return b.bp.AddFarVariationDependencies(nil, tag, names...) - } return b.bp.AddFarVariationDependencies(variations, tag, names...) } diff --git a/android/namespace_test.go b/android/namespace_test.go index ea399da06..87d13206b 100644 --- a/android/namespace_test.go +++ b/android/namespace_test.go @@ -15,7 +15,6 @@ package android import ( - "errors" "path/filepath" "reflect" "testing" @@ -24,577 +23,555 @@ import ( ) func TestDependingOnModuleInSameNamespace(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + } + test_module { + name: "a", + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - a := getModule(ctx, "a") - b := getModule(ctx, "b") - if !dependsOn(ctx, b, a) { + a := getModule(result, "a") + b := getModule(result, "b") + if !dependsOn(result, b, a) { t.Errorf("module b does not depend on module a in the same namespace") } } func TestDependingOnModuleInRootNamespace(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ ".": ` - test_module { - name: "b", - deps: ["a"], - } - test_module { - name: "a", - } + test_module { + name: "b", + deps: ["a"], + } + test_module { + name: "a", + } `, - }, - ) + }), + ).RunTest(t) - a := getModule(ctx, "a") - b := getModule(ctx, "b") - if !dependsOn(ctx, b, a) { + a := getModule(result, "a") + b := getModule(result, "b") + if !dependsOn(result, b, a) { t.Errorf("module b in root namespace does not depend on module a in the root namespace") } } func TestImplicitlyImportRootNamespace(t *testing.T) { - _ = setupTest(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ ".": ` - test_module { - name: "a", - } + test_module { + name: "a", + } `, "dir1": ` - soong_namespace { - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - // setupTest will report any errors + // RunTest will report any errors } func TestDependingOnBlueprintModuleInRootNamespace(t *testing.T) { - _ = setupTest(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ ".": ` - blueprint_test_module { - name: "a", - } + blueprint_test_module { + name: "a", + } `, "dir1": ` - soong_namespace { - } - blueprint_test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + } + blueprint_test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - // setupTest will report any errors + // RunTest will report any errors } func TestDependingOnModuleInImportedNamespace(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir2": ` - soong_namespace { - imports: ["dir1"], - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + imports: ["dir1"], + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - a := getModule(ctx, "a") - b := getModule(ctx, "b") - if !dependsOn(ctx, b, a) { + a := getModule(result, "a") + b := getModule(result, "b") + if !dependsOn(result, b, a) { t.Errorf("module b does not depend on module a in the same namespace") } } func TestDependingOnModuleInNonImportedNamespace(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir2": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir3": ` - soong_namespace { - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) - - expectedErrors := []error{ - errors.New( - `dir3/Android.bp:4:4: "b" depends on undefined module "a" + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir3/Android.bp:4:5: "b" depends on undefined module "a" Module "b" is defined in namespace "dir3" which can read these 2 namespaces: ["dir3" "."] -Module "a" can be found in these namespaces: ["dir1" "dir2"]`), - } - - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } +Module "a" can be found in these namespaces: ["dir1" "dir2"]\E`)). + RunTest(t) } func TestDependingOnModuleByFullyQualifiedReference(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir2": ` - soong_namespace { - } - test_module { - name: "b", - deps: ["//dir1:a"], - } + soong_namespace { + } + test_module { + name: "b", + deps: ["//dir1:a"], + } `, - }, - ) - a := getModule(ctx, "a") - b := getModule(ctx, "b") - if !dependsOn(ctx, b, a) { + }), + ).RunTest(t) + + a := getModule(result, "a") + b := getModule(result, "b") + if !dependsOn(result, b, a) { t.Errorf("module b does not depend on module a") } } func TestSameNameInTwoNamespaces(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - id: "1", - } - test_module { - name: "b", - deps: ["a"], - id: "2", - } + soong_namespace { + } + test_module { + name: "a", + id: "1", + } + test_module { + name: "b", + deps: ["a"], + id: "2", + } `, "dir2": ` - soong_namespace { - } - test_module { - name: "a", - id:"3", - } - test_module { - name: "b", - deps: ["a"], - id:"4", - } + soong_namespace { + } + test_module { + name: "a", + id:"3", + } + test_module { + name: "b", + deps: ["a"], + id:"4", + } `, - }, - ) + }), + ).RunTest(t) - one := findModuleById(ctx, "1") - two := findModuleById(ctx, "2") - three := findModuleById(ctx, "3") - four := findModuleById(ctx, "4") - if !dependsOn(ctx, two, one) { + one := findModuleById(result, "1") + two := findModuleById(result, "2") + three := findModuleById(result, "3") + four := findModuleById(result, "4") + if !dependsOn(result, two, one) { t.Fatalf("Module 2 does not depend on module 1 in its namespace") } - if dependsOn(ctx, two, three) { + if dependsOn(result, two, three) { t.Fatalf("Module 2 depends on module 3 in another namespace") } - if !dependsOn(ctx, four, three) { + if !dependsOn(result, four, three) { t.Fatalf("Module 4 does not depend on module 3 in its namespace") } - if dependsOn(ctx, four, one) { + if dependsOn(result, four, one) { t.Fatalf("Module 4 depends on module 1 in another namespace") } } func TestSearchOrder(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - id: "1", - } + soong_namespace { + } + test_module { + name: "a", + id: "1", + } `, "dir2": ` - soong_namespace { - } - test_module { - name: "a", - id:"2", - } - test_module { - name: "b", - id:"3", - } + soong_namespace { + } + test_module { + name: "a", + id:"2", + } + test_module { + name: "b", + id:"3", + } `, "dir3": ` - soong_namespace { - } - test_module { - name: "a", - id:"4", - } - test_module { - name: "b", - id:"5", - } - test_module { - name: "c", - id:"6", - } + soong_namespace { + } + test_module { + name: "a", + id:"4", + } + test_module { + name: "b", + id:"5", + } + test_module { + name: "c", + id:"6", + } `, ".": ` - test_module { - name: "a", - id: "7", - } - test_module { - name: "b", - id: "8", - } - test_module { - name: "c", - id: "9", - } - test_module { - name: "d", - id: "10", - } + test_module { + name: "a", + id: "7", + } + test_module { + name: "b", + id: "8", + } + test_module { + name: "c", + id: "9", + } + test_module { + name: "d", + id: "10", + } `, "dir4": ` - soong_namespace { - imports: ["dir1", "dir2", "dir3"] - } - test_module { - name: "test_me", - id:"0", - deps: ["a", "b", "c", "d"], - } + soong_namespace { + imports: ["dir1", "dir2", "dir3"] + } + test_module { + name: "test_me", + id:"0", + deps: ["a", "b", "c", "d"], + } `, - }, - ) + }), + ).RunTest(t) - testMe := findModuleById(ctx, "0") - if !dependsOn(ctx, testMe, findModuleById(ctx, "1")) { + testMe := findModuleById(result, "0") + if !dependsOn(result, testMe, findModuleById(result, "1")) { t.Errorf("test_me doesn't depend on id 1") } - if !dependsOn(ctx, testMe, findModuleById(ctx, "3")) { + if !dependsOn(result, testMe, findModuleById(result, "3")) { t.Errorf("test_me doesn't depend on id 3") } - if !dependsOn(ctx, testMe, findModuleById(ctx, "6")) { + if !dependsOn(result, testMe, findModuleById(result, "6")) { t.Errorf("test_me doesn't depend on id 6") } - if !dependsOn(ctx, testMe, findModuleById(ctx, "10")) { + if !dependsOn(result, testMe, findModuleById(result, "10")) { t.Errorf("test_me doesn't depend on id 10") } - if numDeps(ctx, testMe) != 4 { - t.Errorf("num dependencies of test_me = %v, not 4\n", numDeps(ctx, testMe)) + if numDeps(result, testMe) != 4 { + t.Errorf("num dependencies of test_me = %v, not 4\n", numDeps(result, testMe)) } } func TestTwoNamespacesCanImportEachOther(t *testing.T) { - _ = setupTest(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - imports: ["dir2"] - } - test_module { - name: "a", - } - test_module { - name: "c", - deps: ["b"], - } + soong_namespace { + imports: ["dir2"] + } + test_module { + name: "a", + } + test_module { + name: "c", + deps: ["b"], + } `, "dir2": ` - soong_namespace { - imports: ["dir1"], - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + imports: ["dir1"], + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - // setupTest will report any errors + // RunTest will report any errors } func TestImportingNonexistentNamespace(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - imports: ["a_nonexistent_namespace"] - } - test_module { - name: "a", - deps: ["a_nonexistent_module"] - } + soong_namespace { + imports: ["a_nonexistent_namespace"] + } + test_module { + name: "a", + deps: ["a_nonexistent_module"] + } `, - }, - ) - - // should complain about the missing namespace and not complain about the unresolvable dependency - expectedErrors := []error{ - errors.New(`dir1/Android.bp:2:4: module "soong_namespace": namespace a_nonexistent_namespace does not exist`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } + }), + ). + // should complain about the missing namespace and not complain about the unresolvable dependency + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir1/Android.bp:2:5: module "soong_namespace": namespace a_nonexistent_namespace does not exist\E`)). + RunTest(t) } func TestNamespacesDontInheritParentNamespaces(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir1/subdir1": ` - soong_namespace { - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) - - expectedErrors := []error{ - errors.New(`dir1/subdir1/Android.bp:4:4: "b" depends on undefined module "a" + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir1/subdir1/Android.bp:4:5: "b" depends on undefined module "a" Module "b" is defined in namespace "dir1/subdir1" which can read these 2 namespaces: ["dir1/subdir1" "."] -Module "a" can be found in these namespaces: ["dir1"]`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } +Module "a" can be found in these namespaces: ["dir1"]\E`)). + RunTest(t) } func TestModulesDoReceiveParentNamespace(t *testing.T) { - _ = setupTest(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir1/subdir": ` - test_module { - name: "b", - deps: ["a"], - } + test_module { + name: "b", + deps: ["a"], + } `, - }, - ) + }), + ).RunTest(t) - // setupTest will report any errors + // RunTest will report any errors } func TestNamespaceImportsNotTransitive(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - } + soong_namespace { + } + test_module { + name: "a", + } `, "dir2": ` - soong_namespace { - imports: ["dir1"], - } - test_module { - name: "b", - deps: ["a"], - } + soong_namespace { + imports: ["dir1"], + } + test_module { + name: "b", + deps: ["a"], + } `, "dir3": ` - soong_namespace { - imports: ["dir2"], - } - test_module { - name: "c", - deps: ["a"], - } + soong_namespace { + imports: ["dir2"], + } + test_module { + name: "c", + deps: ["a"], + } `, - }, - ) - - expectedErrors := []error{ - errors.New(`dir3/Android.bp:5:4: "c" depends on undefined module "a" + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir3/Android.bp:5:5: "c" depends on undefined module "a" Module "c" is defined in namespace "dir3" which can read these 3 namespaces: ["dir3" "dir2" "."] -Module "a" can be found in these namespaces: ["dir1"]`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } +Module "a" can be found in these namespaces: ["dir1"]\E`)). + RunTest(t) } func TestTwoNamepacesInSameDir(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - soong_namespace { - } + soong_namespace { + } + soong_namespace { + } `, - }, - ) - - expectedErrors := []error{ - errors.New(`dir1/Android.bp:4:4: namespace dir1 already exists`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir1/Android.bp:4:5: namespace dir1 already exists\E`)). + RunTest(t) } func TestNamespaceNotAtTopOfFile(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - test_module { - name: "a" - } - soong_namespace { - } + test_module { + name: "a" + } + soong_namespace { + } `, - }, - ) - - expectedErrors := []error{ - errors.New(`dir1/Android.bp:5:4: a namespace must be the first module in the file`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir1/Android.bp:5:5: a namespace must be the first module in the file\E`)). + RunTest(t) } func TestTwoModulesWithSameNameInSameNamespace(t *testing.T) { - _, errs := setupTestExpectErrs(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a" - } - test_module { - name: "a" - } + soong_namespace { + } + test_module { + name: "a" + } + test_module { + name: "a" + } `, - }, - ) - - expectedErrors := []error{ - errors.New(`dir1/Android.bp:7:4: module "a" already defined - dir1/Android.bp:4:4 <-- previous definition here`), - } - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } + }), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern(`\Qdir1/Android.bp:7:5: module "a" already defined + dir1/Android.bp:4:5 <-- previous definition here\E`)). + RunTest(t) } func TestDeclaringNamespaceInNonAndroidBpFile(t *testing.T) { - _, errs := setupTestFromFiles(t, - map[string][]byte{ - "Android.bp": []byte(` + GroupFixturePreparers( + prepareForTestWithNamespace, + FixtureWithRootAndroidBp(` build = ["include.bp"] - `), - "include.bp": []byte(` + `), + FixtureAddTextFile("include.bp", ` soong_namespace { } - `), - }, - ) - - expectedErrors := []error{ - errors.New(`include.bp:2:5: A namespace may only be declared in a file named Android.bp`), - } - - if len(errs) != 1 || errs[0].Error() != expectedErrors[0].Error() { - t.Errorf("Incorrect errors. Expected:\n%v\n, got:\n%v\n", expectedErrors, errs) - } + `), + ). + ExtendWithErrorHandler(FixtureExpectsOneErrorPattern( + `\Qinclude.bp:2:5: A namespace may only be declared in a file named Android.bp\E`, + )). + RunTest(t) } // so that the generated .ninja file will have consistent names func TestConsistentNamespaceNames(t *testing.T) { - ctx := setupTest(t, - map[string]string{ + result := GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": "soong_namespace{}", "dir2": "soong_namespace{}", "dir3": "soong_namespace{}", - }) + }), + ).RunTest(t) - ns1, _ := ctx.NameResolver.namespaceAt("dir1") - ns2, _ := ctx.NameResolver.namespaceAt("dir2") - ns3, _ := ctx.NameResolver.namespaceAt("dir3") + ns1, _ := result.NameResolver.namespaceAt("dir1") + ns2, _ := result.NameResolver.namespaceAt("dir2") + ns3, _ := result.NameResolver.namespaceAt("dir3") actualIds := []string{ns1.id, ns2.id, ns3.id} expectedIds := []string{"1", "2", "3"} if !reflect.DeepEqual(actualIds, expectedIds) { @@ -604,103 +581,88 @@ func TestConsistentNamespaceNames(t *testing.T) { // so that the generated .ninja file will have consistent names func TestRename(t *testing.T) { - _ = setupTest(t, - map[string]string{ + GroupFixturePreparers( + prepareForTestWithNamespace, + dirBpToPreparer(map[string]string{ "dir1": ` - soong_namespace { - } - test_module { - name: "a", - deps: ["c"], - } - test_module { - name: "b", - rename: "c", - } - `}) - // setupTest will report any errors -} - -// some utils to support the tests + soong_namespace { + } + test_module { + name: "a", + deps: ["c"], + } + test_module { + name: "b", + rename: "c", + } + `, + }), + ).RunTest(t) -func mockFiles(bps map[string]string) (files map[string][]byte) { - files = make(map[string][]byte, len(bps)) - files["Android.bp"] = []byte("") - for dir, text := range bps { - files[filepath.Join(dir, "Android.bp")] = []byte(text) - } - return files + // RunTest will report any errors } -func setupTestFromFiles(t *testing.T, bps MockFS) (ctx *TestContext, errs []error) { - result := GroupFixturePreparers( - FixtureModifyContext(func(ctx *TestContext) { - ctx.RegisterModuleType("test_module", newTestModule) - ctx.Context.RegisterModuleType("blueprint_test_module", newBlueprintTestModule) - ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { - ctx.BottomUp("rename", renameMutator) - }) - }), - PrepareForTestWithNamespace, - bps.AddToFixture(), - ). - // Ignore errors for now so tests can check them later. - ExtendWithErrorHandler(FixtureIgnoreErrors). - RunTest(t) +// some utils to support the tests - return result.TestContext, result.Errs -} +var prepareForTestWithNamespace = GroupFixturePreparers( + FixtureRegisterWithContext(registerNamespaceBuildComponents), + FixtureRegisterWithContext(func(ctx RegistrationContext) { + ctx.PreArchMutators(RegisterNamespaceMutator) + }), + FixtureModifyContext(func(ctx *TestContext) { + ctx.RegisterModuleType("test_module", newTestModule) + ctx.Context.RegisterModuleType("blueprint_test_module", newBlueprintTestModule) + ctx.PreDepsMutators(func(ctx RegisterMutatorsContext) { + ctx.BottomUp("rename", renameMutator) + }) + }), +) -func setupTestExpectErrs(t *testing.T, bps map[string]string) (ctx *TestContext, errs []error) { - files := make(map[string][]byte, len(bps)) +// dirBpToPreparer takes a map from directory to the contents of the Android.bp file and produces a +// FixturePreparer. +func dirBpToPreparer(bps map[string]string) FixturePreparer { + files := make(MockFS, len(bps)) files["Android.bp"] = []byte("") for dir, text := range bps { files[filepath.Join(dir, "Android.bp")] = []byte(text) } - return setupTestFromFiles(t, files) -} - -func setupTest(t *testing.T, bps map[string]string) (ctx *TestContext) { - t.Helper() - ctx, errs := setupTestExpectErrs(t, bps) - FailIfErrored(t, errs) - return ctx + return files.AddToFixture() } -func dependsOn(ctx *TestContext, module TestingModule, possibleDependency TestingModule) bool { +func dependsOn(result *TestResult, module TestingModule, possibleDependency TestingModule) bool { depends := false visit := func(dependency blueprint.Module) { if dependency == possibleDependency.module { depends = true } } - ctx.VisitDirectDeps(module.module, visit) + result.VisitDirectDeps(module.module, visit) return depends } -func numDeps(ctx *TestContext, module TestingModule) int { +func numDeps(result *TestResult, module TestingModule) int { count := 0 visit := func(dependency blueprint.Module) { count++ } - ctx.VisitDirectDeps(module.module, visit) + result.VisitDirectDeps(module.module, visit) return count } -func getModule(ctx *TestContext, moduleName string) TestingModule { - return ctx.ModuleForTests(moduleName, "") +func getModule(result *TestResult, moduleName string) TestingModule { + return result.ModuleForTests(moduleName, "") } -func findModuleById(ctx *TestContext, id string) (module TestingModule) { +func findModuleById(result *TestResult, id string) (module TestingModule) { visit := func(candidate blueprint.Module) { testModule, ok := candidate.(*testModule) if ok { if testModule.properties.Id == id { - module = newTestingModule(ctx.config, testModule) + module = newTestingModule(result.config, testModule) } } } - ctx.VisitAllModules(visit) + result.VisitAllModules(visit) return module } @@ -747,7 +709,7 @@ type blueprintTestModule struct { } } -func (b *blueprintTestModule) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string { +func (b *blueprintTestModule) DynamicDependencies(_ blueprint.DynamicDependerModuleContext) []string { return b.properties.Deps } diff --git a/android/notices.go b/android/notices.go index 2a4c17cd8..b9c1682e3 100644 --- a/android/notices.go +++ b/android/notices.go @@ -15,31 +15,97 @@ package android import ( + "fmt" + "path/filepath" "strings" ) -// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's -// generated license metadata file. -func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { - depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", ".")) - rule := NewRuleBuilder(pctx, ctx) - rule.Command(). - BuiltTool("textnotice"). - FlagWithOutput("-o ", outputFile). - FlagWithDepFile("-d ", depsFile). - Input(ctx.Module().base().licenseMetadataFile) - rule.Build("text_notice", "container notice file") +func modulesOutputDirs(ctx BuilderContext, modules ...Module) []string { + dirs := make([]string, 0, len(modules)) + for _, module := range modules { + paths, err := outputFilesForModule(ctx, module, "") + if err != nil { + continue + } + for _, path := range paths { + if path != nil { + dirs = append(dirs, filepath.Dir(path.String())) + } + } + } + return SortedUniqueStrings(dirs) } -// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based on the module's -// generated license metadata file. -func BuildNoticeHtmlOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { +func modulesLicenseMetadata(ctx BuilderContext, modules ...Module) Paths { + result := make(Paths, 0, len(modules)) + for _, module := range modules { + if mf := module.base().licenseMetadataFile; mf != nil { + result = append(result, mf) + } + } + return result +} + +// buildNoticeOutputFromLicenseMetadata writes out a notice file. +func buildNoticeOutputFromLicenseMetadata( + ctx BuilderContext, tool, ruleName string, outputFile WritablePath, + libraryName string, stripPrefix []string, modules ...Module) { depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", ".")) rule := NewRuleBuilder(pctx, ctx) - rule.Command(). - BuiltTool("htmlnotice"). + if len(modules) == 0 { + if mctx, ok := ctx.(ModuleContext); ok { + modules = []Module{mctx.Module()} + } else { + panic(fmt.Errorf("%s %q needs a module to generate the notice for", ruleName, libraryName)) + } + } + if libraryName == "" { + libraryName = modules[0].Name() + } + cmd := rule.Command(). + BuiltTool(tool). FlagWithOutput("-o ", outputFile). - FlagWithDepFile("-d ", depsFile). - Input(ctx.Module().base().licenseMetadataFile) - rule.Build("html_notice", "container notice file") + FlagWithDepFile("-d ", depsFile) + if len(stripPrefix) > 0 { + cmd = cmd.FlagForEachArg("--strip_prefix ", stripPrefix) + } + outputs := modulesOutputDirs(ctx, modules...) + if len(outputs) > 0 { + cmd = cmd.FlagForEachArg("--strip_prefix ", outputs) + } + if libraryName != "" { + cmd = cmd.FlagWithArg("--product ", libraryName) + } + cmd = cmd.Inputs(modulesLicenseMetadata(ctx, modules...)) + rule.Build(ruleName, "container notice file") +} + +// BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based +// on the license metadata files for the input `modules` defaulting to the +// current context module if none given. +func BuildNoticeTextOutputFromLicenseMetadata( + ctx BuilderContext, outputFile WritablePath, ruleName, libraryName string, + stripPrefix []string, modules ...Module) { + buildNoticeOutputFromLicenseMetadata(ctx, "textnotice", "text_notice_"+ruleName, + outputFile, libraryName, stripPrefix, modules...) +} + +// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based +// on the license metadata files for the input `modules` defaulting to the +// current context module if none given. +func BuildNoticeHtmlOutputFromLicenseMetadata( + ctx BuilderContext, outputFile WritablePath, ruleName, libraryName string, + stripPrefix []string, modules ...Module) { + buildNoticeOutputFromLicenseMetadata(ctx, "htmlnotice", "html_notice_"+ruleName, + outputFile, libraryName, stripPrefix, modules...) +} + +// BuildNoticeXmlOutputFromLicenseMetadata writes out a notice text file based +// on the license metadata files for the input `modules` defaulting to the +// current context module if none given. +func BuildNoticeXmlOutputFromLicenseMetadata( + ctx BuilderContext, outputFile WritablePath, ruleName, libraryName string, + stripPrefix []string, modules ...Module) { + buildNoticeOutputFromLicenseMetadata(ctx, "xmlnotice", "xml_notice_"+ruleName, + outputFile, libraryName, stripPrefix, modules...) } diff --git a/android/paths.go b/android/paths.go index e7829b961..d5cec9af7 100644 --- a/android/paths.go +++ b/android/paths.go @@ -1057,7 +1057,8 @@ func safePathForSource(ctx PathContext, pathComponents ...string) (SourcePath, e } // absolute path already checked by validateSafePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case api surface gen files for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } @@ -1073,7 +1074,8 @@ func pathForSource(ctx PathContext, pathComponents ...string) (SourcePath, error } // absolute path already checked by validatePath - if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) { + // special-case for now + if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") { return ret, fmt.Errorf("source path %q is in output", ret.String()) } @@ -1737,10 +1739,16 @@ func pathForInstall(ctx PathContext, os OsType, arch ArchType, partition string, partionPaths = []string{"target", "product", ctx.Config().DeviceName(), partition} } else { osName := os.String() - if os == Linux || os == LinuxMusl { + if os == Linux { // instead of linux_glibc osName = "linux" } + if os == LinuxMusl && ctx.Config().UseHostMusl() { + // When using musl instead of glibc, use "linux" instead of "linux_musl". When cross + // compiling we will still use "linux_musl". + osName = "linux" + } + // SOONG_HOST_OUT is set to out/host/$(HOST_OS)-$(HOST_PREBUILT_ARCH) // and HOST_PREBUILT_ARCH is forcibly set to x86 even on x86_64 hosts. We don't seem // to have a plan to fix it (see the comment in build/make/core/envsetup.mk). diff --git a/android/prebuilt.go b/android/prebuilt.go index 584348767..4e4fa42ee 100644 --- a/android/prebuilt.go +++ b/android/prebuilt.go @@ -110,6 +110,18 @@ func RemoveOptionalPrebuiltPrefix(name string) string { return strings.TrimPrefix(name, "prebuilt_") } +// RemoveOptionalPrebuiltPrefixFromBazelLabel removes the "prebuilt_" prefix from the *target name* of a Bazel label. +// This differs from RemoveOptionalPrebuiltPrefix in that it does not remove it from the start of the string, but +// instead removes it from the target name itself. +func RemoveOptionalPrebuiltPrefixFromBazelLabel(label string) string { + splitLabel := strings.Split(label, ":") + bazelModuleNameNoPrebuilt := RemoveOptionalPrebuiltPrefix(splitLabel[1]) + return strings.Join([]string{ + splitLabel[0], + bazelModuleNameNoPrebuilt, + }, ":") +} + func (p *Prebuilt) Name(name string) string { return PrebuiltNameFromSource(name) } diff --git a/android/register.go b/android/register.go index c50583322..4ff8fff97 100644 --- a/android/register.go +++ b/android/register.go @@ -96,10 +96,11 @@ var singletons sortableComponents var preSingletons sortableComponents type mutator struct { - name string - bottomUpMutator blueprint.BottomUpMutator - topDownMutator blueprint.TopDownMutator - parallel bool + name string + bottomUpMutator blueprint.BottomUpMutator + topDownMutator blueprint.TopDownMutator + transitionMutator blueprint.TransitionMutator + parallel bool } var _ sortableComponent = &mutator{} diff --git a/android/sdk.go b/android/sdk.go index 1d63d7a94..2dc0bd7f3 100644 --- a/android/sdk.go +++ b/android/sdk.go @@ -23,24 +23,8 @@ import ( "github.com/google/blueprint/proptools" ) -// RequiredSdks provides access to the set of SDKs required by an APEX and its contents. -// -// Extracted from SdkAware to make it easier to define custom subsets of the -// SdkAware interface and improve code navigation within the IDE. -// -// In addition to its use in SdkAware this interface must also be implemented by -// APEX to specify the SDKs required by that module and its contents. e.g. APEX -// is expected to implement RequiredSdks() by reading its own properties like -// `uses_sdks`. -type RequiredSdks interface { - // RequiredSdks returns the set of SDKs required by an APEX and its contents. - RequiredSdks() SdkRefs -} - // sdkAwareWithoutModule is provided simply to improve code navigation with the IDE. type sdkAwareWithoutModule interface { - RequiredSdks - // SdkMemberComponentName will return the name to use for a component of this module based on the // base name of this module. // @@ -81,7 +65,6 @@ type sdkAwareWithoutModule interface { ContainingSdk() SdkRef MemberName() string - BuildWithSdks(sdks SdkRefs) } // SdkAware is the interface that must be supported by any module to become a member of SDK or to be @@ -150,9 +133,6 @@ type sdkProperties struct { // The SDK that this module is a member of. nil if it is not a member of any SDK ContainingSdk *SdkRef `blueprint:"mutated"` - // The list of SDK names and versions that are used to build this module - RequiredSdks SdkRefs `blueprint:"mutated"` - // Name of the module that this sdk member is representing Sdk_member_name *string } @@ -208,16 +188,6 @@ func (s *SdkBase) MemberName() string { return proptools.String(s.properties.Sdk_member_name) } -// BuildWithSdks is used to mark that this module has to be built with the given SDK(s). -func (s *SdkBase) BuildWithSdks(sdks SdkRefs) { - s.properties.RequiredSdks = sdks -} - -// RequiredSdks returns the SDK(s) that this module has to be built with -func (s *SdkBase) RequiredSdks() SdkRefs { - return s.properties.RequiredSdks -} - // InitSdkAwareModule initializes the SdkBase struct. This must be called by all modules including // SdkBase. func InitSdkAwareModule(m SdkAware) { @@ -700,6 +670,9 @@ type SdkMemberType interface { // host OS variant explicitly and disable all other host OS'es. IsHostOsDependent() bool + // SupportedLinkages returns the names of the linkage variants supported by this module. + SupportedLinkages() []string + // AddDependencies adds dependencies from the SDK module to all the module variants the member // type contributes to the SDK. `names` is the list of module names given in the member type // property (as returned by SdkPropertyName()) in the SDK module. The exact set of variants @@ -763,6 +736,9 @@ type SdkMemberType interface { // SupportedTraits returns the set of traits supported by this member type. SupportedTraits() SdkMemberTraitSet + + // Overrides returns whether type overrides other SdkMemberType + Overrides(SdkMemberType) bool } var _ sdkRegisterable = (SdkMemberType)(nil) @@ -786,6 +762,13 @@ type SdkDependencyContext interface { type SdkMemberTypeBase struct { PropertyName string + // Property names that this SdkMemberTypeBase can override, this is useful when a module type is a + // superset of another module type. + OverridesPropertyNames map[string]bool + + // The names of linkage variants supported by this module. + SupportedLinkageNames []string + // When set to true BpPropertyNotRequired indicates that the member type does not require the // property to be specifiable in an Android.bp file. BpPropertyNotRequired bool @@ -826,6 +809,14 @@ func (b *SdkMemberTypeBase) SupportedTraits() SdkMemberTraitSet { return NewSdkMemberTraitSet(b.Traits) } +func (b *SdkMemberTypeBase) Overrides(other SdkMemberType) bool { + return b.OverridesPropertyNames[other.SdkPropertyName()] +} + +func (b *SdkMemberTypeBase) SupportedLinkages() []string { + return b.SupportedLinkageNames +} + // registeredModuleExportsMemberTypes is the set of registered SdkMemberTypes for module_exports // modules. var registeredModuleExportsMemberTypes = &sdkRegistry{} @@ -991,3 +982,10 @@ type ExportedComponentsInfo struct { } var ExportedComponentsInfoProvider = blueprint.NewProvider(ExportedComponentsInfo{}) + +// AdditionalSdkInfo contains additional properties to add to the generated SDK info file. +type AdditionalSdkInfo struct { + Properties map[string]interface{} +} + +var AdditionalSdkInfoProvider = blueprint.NewProvider(AdditionalSdkInfo{}) diff --git a/android/singleton.go b/android/singleton.go index 7ff96c9d5..7c6cf4fd6 100644 --- a/android/singleton.go +++ b/android/singleton.go @@ -29,6 +29,10 @@ type SingletonContext interface { ModuleType(module blueprint.Module) string BlueprintFile(module blueprint.Module) string + // ModuleVariantsFromName returns the list of module variants named `name` in the same namespace as `referer` enforcing visibility rules. + // Allows generating build actions for `referer` based on the metadata for `name` deferred until the singleton context. + ModuleVariantsFromName(referer Module, name string) []Module + // ModuleProvider returns the value, if any, for the provider for a module. If the value for the // provider was not set it returns the zero value of the type of the provider, which means the // return value can always be type-asserted to the type of the provider. The return value should @@ -251,3 +255,30 @@ func (s *singletonContextAdaptor) PrimaryModule(module Module) Module { func (s *singletonContextAdaptor) FinalModule(module Module) Module { return s.SingletonContext.FinalModule(module).(Module) } + +func (s *singletonContextAdaptor) ModuleVariantsFromName(referer Module, name string) []Module { + // get qualified module name for visibility enforcement + qualified := createQualifiedModuleName(s.ModuleName(referer), s.ModuleDir(referer)) + + modules := s.SingletonContext.ModuleVariantsFromName(referer, name) + result := make([]Module, 0, len(modules)) + for _, m := range modules { + if module, ok := m.(Module); ok { + // enforce visibility + depName := s.ModuleName(module) + depDir := s.ModuleDir(module) + depQualified := qualifiedModuleName{depDir, depName} + // Targets are always visible to other targets in their own package. + if depQualified.pkg != qualified.pkg { + rule := effectiveVisibilityRules(s.Config(), depQualified) + if !rule.matches(qualified) { + s.ModuleErrorf(referer, "module %q references %q which is not visible to this module\nYou may need to add %q to its visibility", + referer.Name(), depQualified, "//"+s.ModuleDir(referer)) + continue + } + } + result = append(result, module) + } + } + return result +} diff --git a/android/testing.go b/android/testing.go index ac02db9af..85bdca475 100644 --- a/android/testing.go +++ b/android/testing.go @@ -83,6 +83,8 @@ var PrepareForTestWithLicenses = GroupFixturePreparers( FixtureRegisterWithContext(registerLicenseMutators), ) +var PrepareForTestWithGenNotice = FixtureRegisterWithContext(RegisterGenNoticeBuildComponents) + func registerLicenseMutators(ctx RegistrationContext) { ctx.PreArchMutators(RegisterLicensesPackageMapper) ctx.PreArchMutators(RegisterLicensesPropertyGatherer) diff --git a/android/util.go b/android/util.go index 47c45833b..a0f716047 100644 --- a/android/util.go +++ b/android/util.go @@ -32,6 +32,12 @@ func CopyOf(s []string) []string { // JoinWithPrefix prepends the prefix to each string in the list and // returns them joined together with " " as separator. func JoinWithPrefix(strs []string, prefix string) string { + return JoinWithPrefixAndSeparator(strs, prefix, " ") +} + +// JoinWithPrefixAndSeparator prepends the prefix to each string in the list and +// returns them joined together with the given separator. +func JoinWithPrefixAndSeparator(strs []string, prefix string, sep string) string { if len(strs) == 0 { return "" } @@ -40,7 +46,7 @@ func JoinWithPrefix(strs []string, prefix string) string { buf.WriteString(prefix) buf.WriteString(strs[0]) for i := 1; i < len(strs); i++ { - buf.WriteString(" ") + buf.WriteString(sep) buf.WriteString(prefix) buf.WriteString(strs[i]) } diff --git a/android/variable.go b/android/variable.go index 373891ae4..50fc304d2 100644 --- a/android/variable.go +++ b/android/variable.go @@ -200,6 +200,7 @@ type productVariables struct { Platform_min_supported_target_sdk_version *string `json:",omitempty"` Platform_base_os *string `json:",omitempty"` Platform_version_last_stable *string `json:",omitempty"` + Platform_version_known_codenames *string `json:",omitempty"` DeviceName *string `json:",omitempty"` DeviceProduct *string `json:",omitempty"` @@ -352,6 +353,8 @@ type productVariables struct { RecoverySnapshotDirsIncluded []string `json:",omitempty"` HostFakeSnapshotEnabled bool `json:",omitempty"` + MultitreeUpdateMeta bool `json:",omitempty"` + BoardVendorSepolicyDirs []string `json:",omitempty"` BoardOdmSepolicyDirs []string `json:",omitempty"` BoardReqdMaskPolicy []string `json:",omitempty"` @@ -426,6 +429,7 @@ type productVariables struct { ShippingApiLevel *string `json:",omitempty"` + BuildBrokenDepfile *bool `json:",omitempty"` BuildBrokenEnforceSyspropOwner bool `json:",omitempty"` BuildBrokenTrebleSyspropNeverallow bool `json:",omitempty"` BuildBrokenVendorPropertyNamespace bool `json:",omitempty"` @@ -461,12 +465,13 @@ func (v *productVariables) SetDefaultConfig() { *v = productVariables{ BuildNumberFile: stringPtr("build_number.txt"), - Platform_version_name: stringPtr("S"), - Platform_sdk_version: intPtr(30), - Platform_sdk_codename: stringPtr("S"), - Platform_sdk_final: boolPtr(false), - Platform_version_active_codenames: []string{"S"}, - Platform_vndk_version: stringPtr("S"), + Platform_version_name: stringPtr("S"), + Platform_base_sdk_extension_version: intPtr(30), + Platform_sdk_version: intPtr(30), + Platform_sdk_codename: stringPtr("S"), + Platform_sdk_final: boolPtr(false), + Platform_version_active_codenames: []string{"S"}, + Platform_vndk_version: stringPtr("S"), HostArch: stringPtr("x86_64"), HostSecondaryArch: stringPtr("x86"), diff --git a/android/visibility.go b/android/visibility.go index 5d1be6b47..b20959944 100644 --- a/android/visibility.go +++ b/android/visibility.go @@ -234,7 +234,7 @@ func RegisterVisibilityRuleEnforcer(ctx RegisterMutatorsContext) { // Checks the per-module visibility rule lists before defaults expansion. func visibilityRuleChecker(ctx BottomUpMutatorContext) { - qualified := createQualifiedModuleName(ctx) + qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir()) if m, ok := ctx.Module().(Module); ok { visibilityProperties := m.visibilityProperties() for _, p := range visibilityProperties { @@ -435,7 +435,7 @@ func visibilityRuleEnforcer(ctx TopDownMutatorContext) { return } - qualified := createQualifiedModuleName(ctx) + qualified := createQualifiedModuleName(ctx.ModuleName(), ctx.ModuleDir()) // Visit all the dependencies making sure that this module has access to them all. ctx.VisitDirectDeps(func(dep Module) { @@ -486,9 +486,7 @@ func effectiveVisibilityRules(config Config, qualified qualifiedModuleName) comp return rule } -func createQualifiedModuleName(ctx BaseModuleContext) qualifiedModuleName { - moduleName := ctx.ModuleName() - dir := ctx.ModuleDir() +func createQualifiedModuleName(moduleName, dir string) qualifiedModuleName { qualified := qualifiedModuleName{dir, moduleName} return qualified } diff --git a/android/visibility_test.go b/android/visibility_test.go index 714c92a71..a66f0b698 100644 --- a/android/visibility_test.go +++ b/android/visibility_test.go @@ -135,20 +135,35 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//visibility:public"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], }`), "top/nested/Android.bp": []byte(` mock_library { name: "libnested", deps: ["libexample"], + } + + gen_notice { + name: "nested-notice", + for: ["libexample"], }`), "other/Android.bp": []byte(` mock_library { name: "libother", deps: ["libexample"], + } + + gen_notice { + name: "other-notice", + for: ["libexample"], }`), }, }, @@ -162,7 +177,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//visibility:private"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -186,6 +201,44 @@ var visibilityTests = []struct { }, }, { + // Verify that //visibility:private allows the module to be referenced from the current + // directory only. + name: "//visibility:private (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + } + + mock_library { + name: "libsamepackage", + deps: ["libexample"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "nested-notice" references "//top:libexample" which is not visible to this` + + ` module\nYou may need to add "//top/nested" to its visibility`, + `module "other-notice" references "//top:libexample" which is not visible to this module\n` + + `You may need to add "//other" to its visibility`, + }, + }, + { // Verify that :__pkg__ allows the module to be referenced from the current directory only. name: ":__pkg__", fs: MockFS{ @@ -194,7 +247,7 @@ var visibilityTests = []struct { name: "libexample", visibility: [":__pkg__"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -218,6 +271,36 @@ var visibilityTests = []struct { }, }, { + // Verify that :__pkg__ allows the module to be referenced from the current directory only. + name: ":__pkg__ (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: [":__pkg__"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "nested-notice" references "//top:libexample" which is not visible to this module`, + `module "other-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { // Verify that //top/nested allows the module to be referenced from the current directory and // the top/nested directory only, not a subdirectory of top/nested and not peak directory. name: "//top/nested", @@ -227,7 +310,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//top/nested"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -256,6 +339,42 @@ var visibilityTests = []struct { }, }, { + // Verify that //top/nested allows the module to be referenced from the current directory and + // the top/nested directory only, not a subdirectory of top/nested and not peak directory. + name: "//top/nested (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "top/nested/again/Android.bp": []byte(` + gen_notice { + name: "nestedagain-notice", + for: ["libexample"], + }`), + "peak/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "other-notice" references "//top:libexample" which is not visible to this module`, + `module "nestedagain-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { // Verify that :__subpackages__ allows the module to be referenced from the current directory // and sub directories but nowhere else. name: ":__subpackages__", @@ -265,7 +384,7 @@ var visibilityTests = []struct { name: "libexample", visibility: [":__subpackages__"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -287,6 +406,36 @@ var visibilityTests = []struct { }, }, { + // Verify that :__subpackages__ allows the module to be referenced from the current directory + // and sub directories but nowhere else. + name: ":__subpackages__ (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: [":__subpackages__"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "peak/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "other-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { // Verify that //top/nested:__subpackages__ allows the module to be referenced from the current // directory and sub directories but nowhere else. name: "//top/nested:__subpackages__", @@ -296,7 +445,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//top/nested:__subpackages__", "//other"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -318,6 +467,36 @@ var visibilityTests = []struct { }, }, { + // Verify that //top/nested:__subpackages__ allows the module to be referenced from the current + // directory and sub directories but nowhere else. + name: "//top/nested:__subpackages__ (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested:__subpackages__", "//other"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "top/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "other-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { // Verify that ["//top/nested", "//peak:__subpackages"] allows the module to be referenced from // the current directory, top/nested and peak and all its subpackages. name: `["//top/nested", "//peak:__subpackages__"]`, @@ -327,7 +506,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//top/nested", "//peak:__subpackages__"], } - + mock_library { name: "libsamepackage", deps: ["libexample"], @@ -345,6 +524,33 @@ var visibilityTests = []struct { }, }, { + // Verify that ["//top/nested", "//peak:__subpackages"] allows the module to be referenced from + // the current directory, top/nested and peak and all its subpackages. + name: `["//top/nested", "//peak:__subpackages__ (notices)"]`, + fs: MockFS{ + "top/Android.bp": []byte(` + mock_library { + name: "libexample", + visibility: ["//top/nested", "//peak:__subpackages__"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "peak/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + }, + { // Verify that //vendor... cannot be used outside vendor apart from //vendor:__subpackages__ name: `//vendor`, fs: MockFS{ @@ -353,7 +559,7 @@ var visibilityTests = []struct { name: "libexample", visibility: ["//vendor:__subpackages__"], } - + mock_library { name: "libsamepackage", visibility: ["//vendor/apps/AcmeSettings"], @@ -418,6 +624,45 @@ var visibilityTests = []struct { }, }, { + // Check that visibility is the union of the defaults modules. + name: "defaults union, basic (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//other"], + } + mock_library { + name: "libexample", + visibility: ["//top/nested"], + defaults: ["libexample_defaults"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "defaults union, multiple defaults", fs: MockFS{ "top/Android.bp": []byte(` @@ -459,6 +704,47 @@ var visibilityTests = []struct { }, }, { + name: "defaults union, multiple defaults (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//other"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//top/nested"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "//visibility:public mixed with other in defaults", fs: MockFS{ "top/Android.bp": []byte(` @@ -500,6 +786,29 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:public overriding defaults (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + defaults: ["libexample_defaults"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + effectiveVisibility: map[qualifiedModuleName][]string{ + qualifiedModuleName{pkg: "top", name: "libexample"}: {"//visibility:public"}, + }, + }, + { name: "//visibility:public mixed with other from different defaults 1", fs: MockFS{ "top/Android.bp": []byte(` @@ -523,6 +832,34 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:public mixed with other from different defaults 1", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//namespace"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + }, + { name: "//visibility:public mixed with other from different defaults 2", fs: MockFS{ "top/Android.bp": []byte(` @@ -546,6 +883,29 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:public mixed with other from different defaults 2 (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//visibility:public"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + }, + { name: "//visibility:private in defaults", fs: MockFS{ "top/Android.bp": []byte(` @@ -580,6 +940,39 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:private in defaults (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + defaults: ["libexample_defaults"], + } + + gen_notice { + name: "libexample-notice", + for: ["libexample"], + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "nested-notice" references "//top:libexample" which is not visible to this module`, + `module "other-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "//visibility:private mixed with other in defaults", fs: MockFS{ "top/Android.bp": []byte(` @@ -706,6 +1099,27 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:override discards //visibility:private (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //visibility:private + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + }, + }, + { name: "//visibility:override discards //visibility:public", fs: MockFS{ "top/Android.bp": []byte(` @@ -735,6 +1149,35 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:override discards //visibility:public (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //visibility:public + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + "namespace/Android.bp": []byte(` + gen_notice { + name: "namespace-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "namespace-notice" references "//top:libexample" which is not visible to this module\nYou may need to add "//namespace" to its visibility`, + }, + }, + { name: "//visibility:override discards defaults supplied rules", fs: MockFS{ "top/Android.bp": []byte(` @@ -764,6 +1207,35 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:override discards defaults supplied rules (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//namespace"], + } + mock_library { + name: "libexample", + // Make this visibility to //other but not //namespace + visibility: ["//visibility:override", "//other"], + defaults: ["libexample_defaults"], + }`), + "other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libexample"], + }`), + "namespace/Android.bp": []byte(` + gen_notice { + name: "namespace-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "namespace-notice" references "//top:libexample" which is not visible to this module\nYou may need to add "//namespace" to its visibility`, + }, + }, + { name: "//visibility:override can override //visibility:public with //visibility:private", fs: MockFS{ "top/Android.bp": []byte(` @@ -787,6 +1259,29 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:override can override //visibility:public with //visibility:private (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:public"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:override", "//visibility:private"], + defaults: ["libexample_defaults"], + }`), + "namespace/Android.bp": []byte(` + gen_notice { + name: "namespace-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "namespace-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "//visibility:override can override //visibility:private with //visibility:public", fs: MockFS{ "top/Android.bp": []byte(` @@ -807,6 +1302,26 @@ var visibilityTests = []struct { }, }, { + name: "//visibility:override can override //visibility:private with //visibility:public (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:override", "//visibility:public"], + defaults: ["libexample_defaults"], + }`), + "namespace/Android.bp": []byte(` + gen_notice { + name: "namespace-notice", + for: ["libexample"], + }`), + }, + }, + { name: "//visibility:private mixed with itself", fs: MockFS{ "top/Android.bp": []byte(` @@ -834,6 +1349,33 @@ var visibilityTests = []struct { ` visible to this module`, }, }, + { + name: "//visibility:private mixed with itself (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_defaults { + name: "libexample_defaults_1", + visibility: ["//visibility:private"], + } + mock_defaults { + name: "libexample_defaults_2", + visibility: ["//visibility:private"], + } + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + defaults: ["libexample_defaults_1", "libexample_defaults_2"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, // Defaults module's defaults_visibility tests { @@ -903,6 +1445,28 @@ var visibilityTests = []struct { }, }, { + // This test relies on the default visibility being legacy_public. + name: "package default_visibility property used when no visibility specified (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + mock_library { + name: "libexample", + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "package default_visibility public does not override visibility private", fs: MockFS{ "top/Android.bp": []byte(` @@ -926,6 +1490,28 @@ var visibilityTests = []struct { }, }, { + name: "package default_visibility public does not override visibility private (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: ["//visibility:public"], + } + + mock_library { + name: "libexample", + visibility: ["//visibility:private"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "package default_visibility private does not override visibility public", fs: MockFS{ "top/Android.bp": []byte(` @@ -945,6 +1531,25 @@ var visibilityTests = []struct { }, }, { + name: "package default_visibility private does not override visibility public (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + } + + mock_library { + name: "libexample", + visibility: ["//visibility:public"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + }, + { name: "package default_visibility :__subpackages__", fs: MockFS{ "top/Android.bp": []byte(` @@ -972,6 +1577,32 @@ var visibilityTests = []struct { }, }, { + name: "package default_visibility :__subpackages__ (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: [":__subpackages__"], + } + + mock_library { + name: "libexample", + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "package default_visibility inherited to subpackages", fs: MockFS{ "top/Android.bp": []byte(` @@ -981,7 +1612,7 @@ var visibilityTests = []struct { mock_library { name: "libexample", - visibility: [":__subpackages__"], + visibility: [":__subpackages__"], }`), "top/nested/Android.bp": []byte(` mock_library { @@ -1000,6 +1631,38 @@ var visibilityTests = []struct { }, }, { + name: "package default_visibility inherited to subpackages (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: ["//outsider"], + } + + mock_library { + name: "libexample", + visibility: [":__subpackages__"], + }`), + "top/nested/Android.bp": []byte(` + mock_library { + name: "libnested", + deps: ["libexample"], + } + + gen_notice { + name: "nested-notice", + for: ["libexample"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libexample", "libnested"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top:libexample" which is not visible to this module`, + }, + }, + { name: "package default_visibility inherited to subpackages", fs: MockFS{ "top/Android.bp": []byte(` @@ -1030,6 +1693,41 @@ var visibilityTests = []struct { }, }, { + name: "package default_visibility inherited to subpackages (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + package { + default_visibility: ["//visibility:private"], + }`), + "top/nested/Android.bp": []byte(` + package { + default_visibility: ["//outsider"], + } + + mock_library { + name: "libnested", + }`), + "top/other/Android.bp": []byte(` + mock_library { + name: "libother", + } + + gen_notice { + name: "other-notice", + for: ["libother"], + }`), + "outsider/Android.bp": []byte(` + gen_notice { + name: "outsider-notice", + for: ["libother", "libnested"], + }`), + }, + expectedErrors: []string{ + `module "outsider-notice" references "//top/other:libother" which is not visible to this` + + ` module\nYou may need to add "//outsider" to its visibility`, + }, + }, + { name: "verify that prebuilt dependencies are ignored for visibility reasons (not preferred)", fs: MockFS{ "prebuilts/Android.bp": []byte(` @@ -1052,6 +1750,28 @@ var visibilityTests = []struct { }, }, { + name: "verify that prebuilt dependencies are ignored for visibility reasons (not preferred) (notices)", + fs: MockFS{ + "prebuilts/Android.bp": []byte(` + prebuilt { + name: "module", + visibility: ["//top/other"], + }`), + "top/sources/source_file": nil, + "top/sources/Android.bp": []byte(` + source { + name: "module", + visibility: ["//top/other"], + }`), + "top/other/source_file": nil, + "top/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["module"], + }`), + }, + }, + { name: "verify that prebuilt dependencies are ignored for visibility reasons (preferred)", fs: MockFS{ "prebuilts/Android.bp": []byte(` @@ -1075,6 +1795,29 @@ var visibilityTests = []struct { }, }, { + name: "verify that prebuilt dependencies are ignored for visibility reasons (preferred) (notices)", + fs: MockFS{ + "prebuilts/Android.bp": []byte(` + prebuilt { + name: "module", + visibility: ["//top/other"], + prefer: true, + }`), + "top/sources/source_file": nil, + "top/sources/Android.bp": []byte(` + source { + name: "module", + visibility: ["//top/other"], + }`), + "top/other/source_file": nil, + "top/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["module"], + }`), + }, + }, + { name: "ensure visibility properties are checked for correctness", fs: MockFS{ "top/Android.bp": []byte(` @@ -1137,6 +1880,30 @@ var visibilityTests = []struct { }`), }, }, + { + name: "automatic visibility inheritance enabled (notices)", + fs: MockFS{ + "top/Android.bp": []byte(` + mock_parent { + name: "parent", + visibility: ["//top/nested"], + child: { + name: "libchild", + visibility: ["//top/other"], + }, + }`), + "top/nested/Android.bp": []byte(` + gen_notice { + name: "nested-notice", + for: ["libchild"], + }`), + "top/other/Android.bp": []byte(` + gen_notice { + name: "other-notice", + for: ["libchild"], + }`), + }, + }, } func TestVisibility(t *testing.T) { @@ -1147,6 +1914,7 @@ func TestVisibility(t *testing.T) { // registration order. PrepareForTestWithArchMutator, PrepareForTestWithDefaults, + PrepareForTestWithGenNotice, PrepareForTestWithOverrides, PrepareForTestWithPackageModule, PrepareForTestWithPrebuilts, diff --git a/android_sdk/sdk_repo_host.go b/android_sdk/sdk_repo_host.go index f050a2e87..280dae89c 100644 --- a/android_sdk/sdk_repo_host.go +++ b/android_sdk/sdk_repo_host.go @@ -107,6 +107,7 @@ func (s *sdkRepoHost) DepsMutator(ctx android.BottomUpMutatorContext) { func (s *sdkRepoHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { dir := android.PathForModuleOut(ctx, "zip") + outputZipFile := dir.Join(ctx, "output.zip") builder := android.NewRuleBuilder(pctx, ctx). Sbox(dir, android.PathForModuleOut(ctx, "out.sbox.textproto")). SandboxInputs() @@ -123,7 +124,12 @@ func (s *sdkRepoHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { s.CopySpecsToDir(ctx, builder, packageSpecs, dir) noticeFile := android.PathForModuleOut(ctx, "NOTICES.txt") - android.BuildNoticeTextOutputFromLicenseMetadata(ctx, noticeFile) + android.BuildNoticeTextOutputFromLicenseMetadata( + ctx, noticeFile, "", "", + []string{ + android.PathForModuleInstall(ctx, "sdk-repo").String() + "/", + outputZipFile.String(), + }) builder.Command().Text("cp"). Input(noticeFile). Text(filepath.Join(dir.String(), "NOTICE.txt")) @@ -209,7 +215,6 @@ func (s *sdkRepoHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { } // Zip up our temporary directory as the sdk-repo - outputZipFile := dir.Join(ctx, "output.zip") builder.Command(). BuiltTool("soong_zip"). FlagWithOutput("-o ", outputZipFile). diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go index 803032649..8afbe7edc 100644 --- a/androidmk/parser/make_strings.go +++ b/androidmk/parser/make_strings.go @@ -234,10 +234,10 @@ func (ms *MakeString) splitNFunc(n int, splitFunc func(s string, n int) []string if n != 0 { split := splitFunc(s, n) if n != -1 { - if len(split) > n { + if len(split) > n || len(split) == 0 { panic("oops!") } else { - n -= len(split) + n -= len(split) - 1 } } curMs.appendString(split[0]) @@ -279,7 +279,7 @@ func (ms *MakeString) TrimRightOne() { func (ms *MakeString) EndsWith(ch rune) bool { s := ms.Strings[len(ms.Strings)-1] - return s[len(s)-1] == uint8(ch) + return len(s) > 0 && s[len(s)-1] == uint8(ch) } func (ms *MakeString) ReplaceLiteral(input string, output string) { diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go index fbb289bb7..7e842a51b 100644 --- a/androidmk/parser/make_strings_test.go +++ b/androidmk/parser/make_strings_test.go @@ -75,6 +75,16 @@ var splitNTestCases = []struct { genMakeString(""), }, }, + { + // "x$(var1)y bar" + in: genMakeString("x", "var1", "y bar"), + sep: " ", + n: 2, + expected: []*MakeString{ + genMakeString("x", "var1", "y"), + genMakeString("bar"), + }, + }, } func TestMakeStringSplitN(t *testing.T) { @@ -217,6 +227,36 @@ func TestMakeStringWords(t *testing.T) { } } +var endsWithTestCases = []struct { + in *MakeString + endsWith rune + expected bool +}{ + { + in: genMakeString("foo", "X", "bar ="), + endsWith: '=', + expected: true, + }, + { + in: genMakeString("foo", "X", "bar ="), + endsWith: ':', + expected: false, + }, + { + in: genMakeString("foo", "X", ""), + endsWith: '=', + expected: false, + }, +} + +func TestMakeStringEndsWith(t *testing.T) { + for _, test := range endsWithTestCases { + if test.in.EndsWith(test.endsWith) != test.expected { + t.Errorf("with:\n%q\nexpected:\n%t\ngot:\n%t", test.in.Dump(), test.expected, !test.expected) + } + } +} + func dumpArray(a []*MakeString) string { ret := make([]string, len(a)) diff --git a/apex/Android.bp b/apex/Android.bp index 41224ecd5..fcdf8e6be 100644 --- a/apex/Android.bp +++ b/apex/Android.bp @@ -14,6 +14,7 @@ bootstrap_go_package { "soong-cc", "soong-filesystem", "soong-java", + "soong-multitree", "soong-provenance", "soong-python", "soong-rust", @@ -24,6 +25,7 @@ bootstrap_go_package { "apex.go", "apex_singleton.go", "builder.go", + "constants.go", "deapexer.go", "key.go", "prebuilt.go", diff --git a/apex/androidmk.go b/apex/androidmk.go index e094a1276..938c8edd3 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -412,6 +412,7 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { fmt.Fprintln(w, ".PHONY:", goal) fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", goal, a.installedFilesFile.String(), distFile) + fmt.Fprintf(w, "$(call declare-0p-target,%s)\n", a.installedFilesFile.String()) } for _, dist := range data.Entries.GetDistForGoals(a) { fmt.Fprintf(w, dist) diff --git a/apex/apex.go b/apex/apex.go index 76af1b82e..5678b06bf 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -19,6 +19,7 @@ package apex import ( "fmt" "path/filepath" + "regexp" "sort" "strings" @@ -33,6 +34,7 @@ import ( prebuilt_etc "android/soong/etc" "android/soong/filesystem" "android/soong/java" + "android/soong/multitree" "android/soong/python" "android/soong/rust" "android/soong/sh" @@ -48,7 +50,7 @@ func registerApexBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("apex_vndk", vndkApexBundleFactory) ctx.RegisterModuleType("apex_defaults", defaultsFactory) ctx.RegisterModuleType("prebuilt_apex", PrebuiltFactory) - ctx.RegisterModuleType("override_apex", overrideApexFactory) + ctx.RegisterModuleType("override_apex", OverrideApexFactory) ctx.RegisterModuleType("apex_set", apexSetFactory) ctx.PreArchMutators(registerPreArchMutators) @@ -158,12 +160,6 @@ type apexBundleProperties struct { // or else conflicting build rules may be created. Multi_install_skip_symbol_files *bool - // List of SDKs that are used to build this APEX. A reference to an SDK should be either - // `name#version` or `name` which is an alias for `name#current`. If left empty, - // `platform#current` is implied. This value affects all modules included in this APEX. In - // other words, they are also built with the SDKs specified here. - Uses_sdks []string - // The type of APEX to build. Controls what the APEX payload is. Either 'image', 'zip' or // 'both'. When set to image, contents are stored in a filesystem image inside a zip // container. When set to zip, contents are stored in a zip container directly. This type is @@ -358,6 +354,7 @@ type apexBundle struct { android.OverridableModuleBase android.SdkBase android.BazelModuleBase + multitree.ExportableModuleBase // Properties properties apexBundleProperties @@ -611,30 +608,34 @@ type dependencyTag struct { sourceOnly bool } -func (d dependencyTag) ReplaceSourceWithPrebuilt() bool { +func (d *dependencyTag) String() string { + return fmt.Sprintf("apex.dependencyTag{%q}", d.name) +} + +func (d *dependencyTag) ReplaceSourceWithPrebuilt() bool { return !d.sourceOnly } var _ android.ReplaceSourceWithPrebuilt = &dependencyTag{} var ( - androidAppTag = dependencyTag{name: "androidApp", payload: true} - bpfTag = dependencyTag{name: "bpf", payload: true} - certificateTag = dependencyTag{name: "certificate"} - executableTag = dependencyTag{name: "executable", payload: true} - fsTag = dependencyTag{name: "filesystem", payload: true} - bcpfTag = dependencyTag{name: "bootclasspathFragment", payload: true, sourceOnly: true} - sscpfTag = dependencyTag{name: "systemserverclasspathFragment", payload: true, sourceOnly: true} - compatConfigTag = dependencyTag{name: "compatConfig", payload: true, sourceOnly: true} - javaLibTag = dependencyTag{name: "javaLib", payload: true} - jniLibTag = dependencyTag{name: "jniLib", payload: true} - keyTag = dependencyTag{name: "key"} - prebuiltTag = dependencyTag{name: "prebuilt", payload: true} - rroTag = dependencyTag{name: "rro", payload: true} - sharedLibTag = dependencyTag{name: "sharedLib", payload: true} - testForTag = dependencyTag{name: "test for"} - testTag = dependencyTag{name: "test", payload: true} - shBinaryTag = dependencyTag{name: "shBinary", payload: true} + androidAppTag = &dependencyTag{name: "androidApp", payload: true} + bpfTag = &dependencyTag{name: "bpf", payload: true} + certificateTag = &dependencyTag{name: "certificate"} + executableTag = &dependencyTag{name: "executable", payload: true} + fsTag = &dependencyTag{name: "filesystem", payload: true} + bcpfTag = &dependencyTag{name: "bootclasspathFragment", payload: true, sourceOnly: true} + sscpfTag = &dependencyTag{name: "systemserverclasspathFragment", payload: true, sourceOnly: true} + compatConfigTag = &dependencyTag{name: "compatConfig", payload: true, sourceOnly: true} + javaLibTag = &dependencyTag{name: "javaLib", payload: true} + jniLibTag = &dependencyTag{name: "jniLib", payload: true} + keyTag = &dependencyTag{name: "key"} + prebuiltTag = &dependencyTag{name: "prebuilt", payload: true} + rroTag = &dependencyTag{name: "rro", payload: true} + sharedLibTag = &dependencyTag{name: "sharedLib", payload: true} + testForTag = &dependencyTag{name: "test for"} + testTag = &dependencyTag{name: "test", payload: true} + shBinaryTag = &dependencyTag{name: "shBinary", payload: true} ) // TODO(jiyong): shorten this function signature @@ -789,19 +790,6 @@ func (a *apexBundle) DepsMutator(ctx android.BottomUpMutatorContext) { commonVariation := ctx.Config().AndroidCommonTarget.Variations() ctx.AddFarVariationDependencies(commonVariation, fsTag, a.properties.Filesystems...) ctx.AddFarVariationDependencies(commonVariation, compatConfigTag, a.properties.Compat_configs...) - - // Marks that this APEX (in fact all the modules in it) has to be built with the given SDKs. - // This field currently isn't used. - // TODO(jiyong): consider dropping this feature - // TODO(jiyong): ensure that all apexes are with non-empty uses_sdks - if len(a.properties.Uses_sdks) > 0 { - sdkRefs := []android.SdkRef{} - for _, str := range a.properties.Uses_sdks { - parsed := android.ParseSdkRef(ctx, str, "uses_sdks") - sdkRefs = append(sdkRefs, parsed) - } - a.BuildWithSdks(sdkRefs) - } } // DepsMutator for the overridden properties. @@ -966,7 +954,6 @@ func (a *apexBundle) ApexInfoMutator(mctx android.TopDownMutatorContext) { apexInfo := android.ApexInfo{ ApexVariationName: apexVariationName, MinSdkVersion: minSdkVersion, - RequiredSdks: a.RequiredSdks(), Updatable: a.Updatable(), UsePlatformApis: a.UsePlatformApis(), InApexVariants: []string{apexVariationName}, @@ -994,6 +981,7 @@ type ApexInfoMutator interface { // apexInfoMutator delegates the work of identifying which modules need an ApexInfo and apex // specific variant to modules that support the ApexInfoMutator. +// It also propagates updatable=true to apps of updatable apexes func apexInfoMutator(mctx android.TopDownMutatorContext) { if !mctx.Module().Enabled() { return @@ -1001,8 +989,8 @@ func apexInfoMutator(mctx android.TopDownMutatorContext) { if a, ok := mctx.Module().(ApexInfoMutator); ok { a.ApexInfoMutator(mctx) - return } + enforceAppUpdatability(mctx) } // apexStrictUpdatibilityLintMutator propagates strict_updatability_linting to transitive deps of a mainline module @@ -1033,6 +1021,22 @@ func apexStrictUpdatibilityLintMutator(mctx android.TopDownMutatorContext) { } } +// enforceAppUpdatability propagates updatable=true to apps of updatable apexes +func enforceAppUpdatability(mctx android.TopDownMutatorContext) { + if !mctx.Module().Enabled() { + return + } + if apex, ok := mctx.Module().(*apexBundle); ok && apex.Updatable() { + // checking direct deps is sufficient since apex->apk is a direct edge, even when inherited via apex_defaults + mctx.VisitDirectDeps(func(module android.Module) { + // ignore android_test_app + if app, ok := module.(*java.AndroidApp); ok { + app.SetUpdatable(true) + } + }) + } +} + // TODO: b/215736885 Whittle the denylist // Transitive deps of certain mainline modules baseline NewApi errors // Skip these mainline modules for now @@ -1334,7 +1338,7 @@ func apexFlattenedMutator(mctx android.BottomUpMutatorContext) { var _ android.DepIsInSameApex = (*apexBundle)(nil) // Implements android.DepInInSameApex -func (a *apexBundle) DepIsInSameApex(ctx android.BaseModuleContext, dep android.Module) bool { +func (a *apexBundle) DepIsInSameApex(_ android.BaseModuleContext, _ android.Module) bool { // direct deps of an APEX bundle are all part of the APEX bundle // TODO(jiyong): shouldn't we look into the payload field of the dependencyTag? return true @@ -1359,6 +1363,21 @@ func (a *apexBundle) OutputFiles(tag string) (android.Paths, error) { } } +var _ multitree.Exportable = (*apexBundle)(nil) + +func (a *apexBundle) Exportable() bool { + if a.properties.ApexType == flattenedApex { + return false + } + return true +} + +func (a *apexBundle) TaggedOutputs() map[string]android.Paths { + ret := make(map[string]android.Paths) + ret["apex"] = android.Paths{a.outputFile} + return ret +} + var _ cc.Coverage = (*apexBundle)(nil) // Implements cc.Coverage @@ -1449,19 +1468,19 @@ func (a *apexBundle) EnableSanitizer(sanitizerName string) { } } -func (a *apexBundle) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool { +func (a *apexBundle) IsSanitizerEnabled(config android.Config, sanitizerName string) bool { if android.InList(sanitizerName, a.properties.SanitizerNames) { return true } // Then follow the global setting - globalSanitizerNames := []string{} + var globalSanitizerNames []string if a.Host() { - globalSanitizerNames = ctx.Config().SanitizeHost() + globalSanitizerNames = config.SanitizeHost() } else { - arches := ctx.Config().SanitizeDeviceArch() + arches := config.SanitizeDeviceArch() if len(arches) == 0 || android.InList(a.Arch().ArchType.Name, arches) { - globalSanitizerNames = ctx.Config().SanitizeDevice() + globalSanitizerNames = config.SanitizeDevice() } } return android.InList(sanitizerName, globalSanitizerNames) @@ -1656,7 +1675,20 @@ type androidApp interface { var _ androidApp = (*java.AndroidApp)(nil) var _ androidApp = (*java.AndroidAppImport)(nil) -const APEX_VERSION_PLACEHOLDER = "__APEX_VERSION_PLACEHOLDER__" +func sanitizedBuildIdForPath(ctx android.BaseModuleContext) string { + buildId := ctx.Config().BuildId() + + // The build ID is used as a suffix for a filename, so ensure that + // the set of characters being used are sanitized. + // - any word character: [a-zA-Z0-9_] + // - dots: . + // - dashes: - + validRegex := regexp.MustCompile(`^[\w\.\-\_]+$`) + if !validRegex.MatchString(buildId) { + ctx.ModuleErrorf("Unable to use build id %s as filename suffix, valid characters are [a-z A-Z 0-9 _ . -].", buildId) + } + return buildId +} func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexFile { appDir := "app" @@ -1667,7 +1699,7 @@ func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexF // TODO(b/224589412, b/226559955): Ensure that the subdirname is suffixed // so that PackageManager correctly invalidates the existing installed apk // in favour of the new APK-in-APEX. See bugs for more information. - dirInApex := filepath.Join(appDir, aapp.InstallApkName()+"@"+APEX_VERSION_PLACEHOLDER) + dirInApex := filepath.Join(appDir, aapp.InstallApkName()+"@"+sanitizedBuildIdForPath(ctx)) fileToCopy := aapp.OutputFile() af := newApexFile(ctx, fileToCopy, aapp.BaseModuleName(), dirInApex, app, aapp) @@ -1724,7 +1756,7 @@ func (a *apexBundle) WalkPayloadDeps(ctx android.ModuleContext, do android.Paylo if _, ok := depTag.(android.ExcludeFromApexContentsTag); ok { return false } - if dt, ok := depTag.(dependencyTag); ok && !dt.payload { + if dt, ok := depTag.(*dependencyTag); ok && !dt.payload { return false } @@ -1758,6 +1790,382 @@ func (f fsType) string() string { } } +type visitorContext struct { + // all the files that will be included in this APEX + filesInfo []apexFile + + // native lib dependencies + provideNativeLibs []string + requireNativeLibs []string + + handleSpecialLibs bool +} + +func (vctx *visitorContext) normalizeFileInfo() { + encountered := make(map[string]apexFile) + for _, f := range vctx.filesInfo { + dest := filepath.Join(f.installDir, f.builtFile.Base()) + if e, ok := encountered[dest]; !ok { + encountered[dest] = f + } else { + // If a module is directly included and also transitively depended on + // consider it as directly included. + e.transitiveDep = e.transitiveDep && f.transitiveDep + encountered[dest] = e + } + } + vctx.filesInfo = vctx.filesInfo[:0] + for _, v := range encountered { + vctx.filesInfo = append(vctx.filesInfo, v) + } + sort.Slice(vctx.filesInfo, func(i, j int) bool { + // Sort by destination path so as to ensure consistent ordering even if the source of the files + // changes. + return vctx.filesInfo[i].path() < vctx.filesInfo[j].path() + }) +} + +func (a *apexBundle) depVisitor(vctx *visitorContext, ctx android.ModuleContext, child, parent blueprint.Module) bool { + depTag := ctx.OtherModuleDependencyTag(child) + if _, ok := depTag.(android.ExcludeFromApexContentsTag); ok { + return false + } + if mod, ok := child.(android.Module); ok && !mod.Enabled() { + return false + } + depName := ctx.OtherModuleName(child) + if _, isDirectDep := parent.(*apexBundle); isDirectDep { + switch depTag { + case sharedLibTag, jniLibTag: + isJniLib := depTag == jniLibTag + switch ch := child.(type) { + case *cc.Module: + fi := apexFileForNativeLibrary(ctx, ch, vctx.handleSpecialLibs) + fi.isJniLib = isJniLib + vctx.filesInfo = append(vctx.filesInfo, fi) + // Collect the list of stub-providing libs except: + // - VNDK libs are only for vendors + // - bootstrap bionic libs are treated as provided by system + if ch.HasStubsVariants() && !a.vndkApex && !cc.InstallToBootstrap(ch.BaseModuleName(), ctx.Config()) { + vctx.provideNativeLibs = append(vctx.provideNativeLibs, fi.stem()) + } + return true // track transitive dependencies + case *rust.Module: + fi := apexFileForRustLibrary(ctx, ch) + fi.isJniLib = isJniLib + vctx.filesInfo = append(vctx.filesInfo, fi) + return true // track transitive dependencies + default: + propertyName := "native_shared_libs" + if isJniLib { + propertyName = "jni_libs" + } + ctx.PropertyErrorf(propertyName, "%q is not a cc_library or cc_library_shared module", depName) + } + case executableTag: + switch ch := child.(type) { + case *cc.Module: + vctx.filesInfo = append(vctx.filesInfo, apexFileForExecutable(ctx, ch)) + return true // track transitive dependencies + case *python.Module: + if ch.HostToolPath().Valid() { + vctx.filesInfo = append(vctx.filesInfo, apexFileForPyBinary(ctx, ch)) + } + case bootstrap.GoBinaryTool: + if a.Host() { + vctx.filesInfo = append(vctx.filesInfo, apexFileForGoBinary(ctx, depName, ch)) + } + case *rust.Module: + vctx.filesInfo = append(vctx.filesInfo, apexFileForRustExecutable(ctx, ch)) + return true // track transitive dependencies + default: + ctx.PropertyErrorf("binaries", + "%q is neither cc_binary, rust_binary, (embedded) py_binary, (host) blueprint_go_binary, nor (host) bootstrap_go_binary", depName) + } + case shBinaryTag: + if csh, ok := child.(*sh.ShBinary); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForShBinary(ctx, csh)) + } else { + ctx.PropertyErrorf("sh_binaries", "%q is not a sh_binary module", depName) + } + case bcpfTag: + bcpfModule, ok := child.(*java.BootclasspathFragmentModule) + if !ok { + ctx.PropertyErrorf("bootclasspath_fragments", "%q is not a bootclasspath_fragment module", depName) + return false + } + + vctx.filesInfo = append(vctx.filesInfo, apexBootclasspathFragmentFiles(ctx, child)...) + for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { + a.requiredDeps = append(a.requiredDeps, makeModuleName) + } + return true + case sscpfTag: + if _, ok := child.(*java.SystemServerClasspathModule); !ok { + ctx.PropertyErrorf("systemserverclasspath_fragments", + "%q is not a systemserverclasspath_fragment module", depName) + return false + } + if af := apexClasspathFragmentProtoFile(ctx, child); af != nil { + vctx.filesInfo = append(vctx.filesInfo, *af) + } + return true + case javaLibTag: + switch child.(type) { + case *java.Library, *java.SdkLibrary, *java.DexImport, *java.SdkLibraryImport, *java.Import: + af := apexFileForJavaModule(ctx, child.(javaModule)) + if !af.ok() { + ctx.PropertyErrorf("java_libs", "%q is not configured to be compiled into dex", depName) + return false + } + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + default: + ctx.PropertyErrorf("java_libs", "%q of type %q is not supported", depName, ctx.OtherModuleType(child)) + } + case androidAppTag: + switch ap := child.(type) { + case *java.AndroidApp: + vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + return true // track transitive dependencies + case *java.AndroidAppImport: + vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + case *java.AndroidTestHelperApp: + vctx.filesInfo = append(vctx.filesInfo, apexFileForAndroidApp(ctx, ap)) + case *java.AndroidAppSet: + appDir := "app" + if ap.Privileged() { + appDir = "priv-app" + } + // TODO(b/224589412, b/226559955): Ensure that the dirname is + // suffixed so that PackageManager correctly invalidates the + // existing installed apk in favour of the new APK-in-APEX. + // See bugs for more information. + appDirName := filepath.Join(appDir, ap.BaseModuleName()+"@"+sanitizedBuildIdForPath(ctx)) + af := newApexFile(ctx, ap.OutputFile(), ap.BaseModuleName(), appDirName, appSet, ap) + af.certificate = java.PresignedCertificate + vctx.filesInfo = append(vctx.filesInfo, af) + default: + ctx.PropertyErrorf("apps", "%q is not an android_app module", depName) + } + case rroTag: + if rro, ok := child.(java.RuntimeResourceOverlayModule); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForRuntimeResourceOverlay(ctx, rro)) + } else { + ctx.PropertyErrorf("rros", "%q is not an runtime_resource_overlay module", depName) + } + case bpfTag: + if bpfProgram, ok := child.(bpf.BpfModule); ok { + filesToCopy, _ := bpfProgram.OutputFiles("") + apex_sub_dir := bpfProgram.SubDir() + for _, bpfFile := range filesToCopy { + vctx.filesInfo = append(vctx.filesInfo, apexFileForBpfProgram(ctx, bpfFile, apex_sub_dir, bpfProgram)) + } + } else { + ctx.PropertyErrorf("bpfs", "%q is not a bpf module", depName) + } + case fsTag: + if fs, ok := child.(filesystem.Filesystem); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForFilesystem(ctx, fs.OutputPath(), fs)) + } else { + ctx.PropertyErrorf("filesystems", "%q is not a filesystem module", depName) + } + case prebuiltTag: + if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) + } else { + ctx.PropertyErrorf("prebuilts", "%q is not a prebuilt_etc module", depName) + } + case compatConfigTag: + if compatConfig, ok := child.(java.PlatformCompatConfigIntf); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForCompatConfig(ctx, compatConfig, depName)) + } else { + ctx.PropertyErrorf("compat_configs", "%q is not a platform_compat_config module", depName) + } + case testTag: + if ccTest, ok := child.(*cc.Module); ok { + if ccTest.IsTestPerSrcAllTestsVariation() { + // Multiple-output test module (where `test_per_src: true`). + // + // `ccTest` is the "" ("all tests") variation of a `test_per_src` module. + // We do not add this variation to `filesInfo`, as it has no output; + // however, we do add the other variations of this module as indirect + // dependencies (see below). + } else { + // Single-output test module (where `test_per_src: false`). + af := apexFileForExecutable(ctx, ccTest) + af.class = nativeTest + vctx.filesInfo = append(vctx.filesInfo, af) + } + return true // track transitive dependencies + } else { + ctx.PropertyErrorf("tests", "%q is not a cc module", depName) + } + case keyTag: + if key, ok := child.(*apexKey); ok { + a.privateKeyFile = key.privateKeyFile + a.publicKeyFile = key.publicKeyFile + } else { + ctx.PropertyErrorf("key", "%q is not an apex_key module", depName) + } + case certificateTag: + if dep, ok := child.(*java.AndroidAppCertificate); ok { + a.containerCertificateFile = dep.Certificate.Pem + a.containerPrivateKeyFile = dep.Certificate.Key + } else { + ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", depName) + } + case android.PrebuiltDepTag: + // If the prebuilt is force disabled, remember to delete the prebuilt file + // that might have been installed in the previous builds + if prebuilt, ok := child.(prebuilt); ok && prebuilt.isForceDisabled() { + a.prebuiltFileToDelete = prebuilt.InstallFilename() + } + } + return false + } + + if a.vndkApex { + return false + } + + // indirect dependencies + am, ok := child.(android.ApexModule) + if !ok { + return false + } + // We cannot use a switch statement on `depTag` here as the checked + // tags used below are private (e.g. `cc.sharedDepTag`). + if cc.IsSharedDepTag(depTag) || cc.IsRuntimeDepTag(depTag) { + if ch, ok := child.(*cc.Module); ok { + if ch.UseVndk() && proptools.Bool(a.properties.Use_vndk_as_stable) && ch.IsVndk() { + vctx.requireNativeLibs = append(vctx.requireNativeLibs, ":vndk") + return false + } + af := apexFileForNativeLibrary(ctx, ch, vctx.handleSpecialLibs) + af.transitiveDep = true + + // Always track transitive dependencies for host. + if a.Host() { + vctx.filesInfo = append(vctx.filesInfo, af) + return true + } + + abInfo := ctx.Provider(ApexBundleInfoProvider).(ApexBundleInfo) + if !abInfo.Contents.DirectlyInApex(depName) && (ch.IsStubs() || ch.HasStubsVariants()) { + // If the dependency is a stubs lib, don't include it in this APEX, + // but make sure that the lib is installed on the device. + // In case no APEX is having the lib, the lib is installed to the system + // partition. + // + // Always include if we are a host-apex however since those won't have any + // system libraries. + if !am.DirectlyInAnyApex() { + // we need a module name for Make + name := ch.ImplementationModuleNameForMake(ctx) + ch.Properties.SubName + if !android.InList(name, a.requiredDeps) { + a.requiredDeps = append(a.requiredDeps, name) + } + } + vctx.requireNativeLibs = append(vctx.requireNativeLibs, af.stem()) + // Don't track further + return false + } + + // If the dep is not considered to be in the same + // apex, don't add it to filesInfo so that it is not + // included in this APEX. + // TODO(jiyong): move this to at the top of the + // else-if clause for the indirect dependencies. + // Currently, that's impossible because we would + // like to record requiredNativeLibs even when + // DepIsInSameAPex is false. We also shouldn't do + // this for host. + // + // TODO(jiyong): explain why the same module is passed in twice. + // Switching the first am to parent breaks lots of tests. + if !android.IsDepInSameApex(ctx, am, am) { + return false + } + + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + } else if rm, ok := child.(*rust.Module); ok { + af := apexFileForRustLibrary(ctx, rm) + af.transitiveDep = true + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + } + } else if cc.IsTestPerSrcDepTag(depTag) { + if ch, ok := child.(*cc.Module); ok { + af := apexFileForExecutable(ctx, ch) + // Handle modules created as `test_per_src` variations of a single test module: + // use the name of the generated test binary (`fileToCopy`) instead of the name + // of the original test module (`depName`, shared by all `test_per_src` + // variations of that module). + af.androidMkModuleName = filepath.Base(af.builtFile.String()) + // these are not considered transitive dep + af.transitiveDep = false + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + } + } else if cc.IsHeaderDepTag(depTag) { + // nothing + } else if java.IsJniDepTag(depTag) { + // Because APK-in-APEX embeds jni_libs transitively, we don't need to track transitive deps + } else if java.IsXmlPermissionsFileDepTag(depTag) { + if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { + vctx.filesInfo = append(vctx.filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) + } + } else if rust.IsDylibDepTag(depTag) { + if rustm, ok := child.(*rust.Module); ok && rustm.IsInstallableToApex() { + af := apexFileForRustLibrary(ctx, rustm) + af.transitiveDep = true + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + } + } else if rust.IsRlibDepTag(depTag) { + // Rlib is statically linked, but it might have shared lib + // dependencies. Track them. + return true + } else if java.IsBootclasspathFragmentContentDepTag(depTag) { + // Add the contents of the bootclasspath fragment to the apex. + switch child.(type) { + case *java.Library, *java.SdkLibrary: + javaModule := child.(javaModule) + af := apexFileForBootclasspathFragmentContentModule(ctx, parent, javaModule) + if !af.ok() { + ctx.PropertyErrorf("bootclasspath_fragments", + "bootclasspath_fragment content %q is not configured to be compiled into dex", depName) + return false + } + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + default: + ctx.PropertyErrorf("bootclasspath_fragments", + "bootclasspath_fragment content %q of type %q is not supported", depName, ctx.OtherModuleType(child)) + } + } else if java.IsSystemServerClasspathFragmentContentDepTag(depTag) { + // Add the contents of the systemserverclasspath fragment to the apex. + switch child.(type) { + case *java.Library, *java.SdkLibrary: + af := apexFileForJavaModule(ctx, child.(javaModule)) + vctx.filesInfo = append(vctx.filesInfo, af) + return true // track transitive dependencies + default: + ctx.PropertyErrorf("systemserverclasspath_fragments", + "systemserverclasspath_fragment content %q of type %q is not supported", depName, ctx.OtherModuleType(child)) + } + } else if _, ok := depTag.(android.CopyDirectlyInAnyApexTag); ok { + // nothing + } else if depTag == android.DarwinUniversalVariantTag { + // nothing + } else if am.CanHaveApexVariants() && am.IsInstallableToApex() { + ctx.ModuleErrorf("unexpected tag %s for indirect dependency %q", android.PrettyPrintTag(depTag), depName) + } + return false +} + // Creates build rules for an APEX. It consists of the following major steps: // // 1) do some validity checks such as apex_available, min_sdk_version, etc. @@ -1780,386 +2188,23 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { //////////////////////////////////////////////////////////////////////////////////////////// // 2) traverse the dependency tree to collect apexFile structs from them. - // all the files that will be included in this APEX - var filesInfo []apexFile - - // native lib dependencies - var provideNativeLibs []string - var requireNativeLibs []string - - handleSpecialLibs := !android.Bool(a.properties.Ignore_system_library_special_case) - // Collect the module directory for IDE info in java/jdeps.go. a.modulePaths = append(a.modulePaths, ctx.ModuleDir()) // TODO(jiyong): do this using WalkPayloadDeps // TODO(jiyong): make this clean!!! - ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool { - depTag := ctx.OtherModuleDependencyTag(child) - if _, ok := depTag.(android.ExcludeFromApexContentsTag); ok { - return false - } - if mod, ok := child.(android.Module); ok && !mod.Enabled() { - return false - } - depName := ctx.OtherModuleName(child) - if _, isDirectDep := parent.(*apexBundle); isDirectDep { - switch depTag { - case sharedLibTag, jniLibTag: - isJniLib := depTag == jniLibTag - if c, ok := child.(*cc.Module); ok { - fi := apexFileForNativeLibrary(ctx, c, handleSpecialLibs) - fi.isJniLib = isJniLib - filesInfo = append(filesInfo, fi) - // Collect the list of stub-providing libs except: - // - VNDK libs are only for vendors - // - bootstrap bionic libs are treated as provided by system - if c.HasStubsVariants() && !a.vndkApex && !cc.InstallToBootstrap(c.BaseModuleName(), ctx.Config()) { - provideNativeLibs = append(provideNativeLibs, fi.stem()) - } - return true // track transitive dependencies - } else if r, ok := child.(*rust.Module); ok { - fi := apexFileForRustLibrary(ctx, r) - fi.isJniLib = isJniLib - filesInfo = append(filesInfo, fi) - return true // track transitive dependencies - } else { - propertyName := "native_shared_libs" - if isJniLib { - propertyName = "jni_libs" - } - ctx.PropertyErrorf(propertyName, "%q is not a cc_library or cc_library_shared module", depName) - } - case executableTag: - if cc, ok := child.(*cc.Module); ok { - filesInfo = append(filesInfo, apexFileForExecutable(ctx, cc)) - return true // track transitive dependencies - } else if py, ok := child.(*python.Module); ok && py.HostToolPath().Valid() { - filesInfo = append(filesInfo, apexFileForPyBinary(ctx, py)) - } else if gb, ok := child.(bootstrap.GoBinaryTool); ok && a.Host() { - filesInfo = append(filesInfo, apexFileForGoBinary(ctx, depName, gb)) - } else if rust, ok := child.(*rust.Module); ok { - filesInfo = append(filesInfo, apexFileForRustExecutable(ctx, rust)) - return true // track transitive dependencies - } else { - ctx.PropertyErrorf("binaries", "%q is neither cc_binary, rust_binary, (embedded) py_binary, (host) blueprint_go_binary, nor (host) bootstrap_go_binary", depName) - } - case shBinaryTag: - if sh, ok := child.(*sh.ShBinary); ok { - filesInfo = append(filesInfo, apexFileForShBinary(ctx, sh)) - } else { - ctx.PropertyErrorf("sh_binaries", "%q is not a sh_binary module", depName) - } - case bcpfTag: - { - bcpfModule, ok := child.(*java.BootclasspathFragmentModule) - if !ok { - ctx.PropertyErrorf("bootclasspath_fragments", "%q is not a bootclasspath_fragment module", depName) - return false - } - - filesToAdd := apexBootclasspathFragmentFiles(ctx, child) - filesInfo = append(filesInfo, filesToAdd...) - for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() { - a.requiredDeps = append(a.requiredDeps, makeModuleName) - } - return true - } - case sscpfTag: - { - if _, ok := child.(*java.SystemServerClasspathModule); !ok { - ctx.PropertyErrorf("systemserverclasspath_fragments", "%q is not a systemserverclasspath_fragment module", depName) - return false - } - if af := apexClasspathFragmentProtoFile(ctx, child); af != nil { - filesInfo = append(filesInfo, *af) - } - return true - } - case javaLibTag: - switch child.(type) { - case *java.Library, *java.SdkLibrary, *java.DexImport, *java.SdkLibraryImport, *java.Import: - af := apexFileForJavaModule(ctx, child.(javaModule)) - if !af.ok() { - ctx.PropertyErrorf("java_libs", "%q is not configured to be compiled into dex", depName) - return false - } - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - default: - ctx.PropertyErrorf("java_libs", "%q of type %q is not supported", depName, ctx.OtherModuleType(child)) - } - case androidAppTag: - if ap, ok := child.(*java.AndroidApp); ok { - filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) - return true // track transitive dependencies - } else if ap, ok := child.(*java.AndroidAppImport); ok { - filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) - } else if ap, ok := child.(*java.AndroidTestHelperApp); ok { - filesInfo = append(filesInfo, apexFileForAndroidApp(ctx, ap)) - } else if ap, ok := child.(*java.AndroidAppSet); ok { - appDir := "app" - if ap.Privileged() { - appDir = "priv-app" - } - // TODO(b/224589412, b/226559955): Ensure that the dirname is - // suffixed so that PackageManager correctly invalidates the - // existing installed apk in favour of the new APK-in-APEX. - // See bugs for more information. - appDirName := filepath.Join(appDir, ap.BaseModuleName()+"@"+APEX_VERSION_PLACEHOLDER) - af := newApexFile(ctx, ap.OutputFile(), ap.BaseModuleName(), appDirName, appSet, ap) - af.certificate = java.PresignedCertificate - filesInfo = append(filesInfo, af) - } else { - ctx.PropertyErrorf("apps", "%q is not an android_app module", depName) - } - case rroTag: - if rro, ok := child.(java.RuntimeResourceOverlayModule); ok { - filesInfo = append(filesInfo, apexFileForRuntimeResourceOverlay(ctx, rro)) - } else { - ctx.PropertyErrorf("rros", "%q is not an runtime_resource_overlay module", depName) - } - case bpfTag: - if bpfProgram, ok := child.(bpf.BpfModule); ok { - filesToCopy, _ := bpfProgram.OutputFiles("") - apex_sub_dir := bpfProgram.SubDir() - for _, bpfFile := range filesToCopy { - filesInfo = append(filesInfo, apexFileForBpfProgram(ctx, bpfFile, apex_sub_dir, bpfProgram)) - } - } else { - ctx.PropertyErrorf("bpfs", "%q is not a bpf module", depName) - } - case fsTag: - if fs, ok := child.(filesystem.Filesystem); ok { - filesInfo = append(filesInfo, apexFileForFilesystem(ctx, fs.OutputPath(), fs)) - } else { - ctx.PropertyErrorf("filesystems", "%q is not a filesystem module", depName) - } - case prebuiltTag: - if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { - filesInfo = append(filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) - } else { - ctx.PropertyErrorf("prebuilts", "%q is not a prebuilt_etc module", depName) - } - case compatConfigTag: - if compatConfig, ok := child.(java.PlatformCompatConfigIntf); ok { - filesInfo = append(filesInfo, apexFileForCompatConfig(ctx, compatConfig, depName)) - } else { - ctx.PropertyErrorf("compat_configs", "%q is not a platform_compat_config module", depName) - } - case testTag: - if ccTest, ok := child.(*cc.Module); ok { - if ccTest.IsTestPerSrcAllTestsVariation() { - // Multiple-output test module (where `test_per_src: true`). - // - // `ccTest` is the "" ("all tests") variation of a `test_per_src` module. - // We do not add this variation to `filesInfo`, as it has no output; - // however, we do add the other variations of this module as indirect - // dependencies (see below). - } else { - // Single-output test module (where `test_per_src: false`). - af := apexFileForExecutable(ctx, ccTest) - af.class = nativeTest - filesInfo = append(filesInfo, af) - } - return true // track transitive dependencies - } else { - ctx.PropertyErrorf("tests", "%q is not a cc module", depName) - } - case keyTag: - if key, ok := child.(*apexKey); ok { - a.privateKeyFile = key.privateKeyFile - a.publicKeyFile = key.publicKeyFile - } else { - ctx.PropertyErrorf("key", "%q is not an apex_key module", depName) - } - return false - case certificateTag: - if dep, ok := child.(*java.AndroidAppCertificate); ok { - a.containerCertificateFile = dep.Certificate.Pem - a.containerPrivateKeyFile = dep.Certificate.Key - } else { - ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", depName) - } - case android.PrebuiltDepTag: - // If the prebuilt is force disabled, remember to delete the prebuilt file - // that might have been installed in the previous builds - if prebuilt, ok := child.(prebuilt); ok && prebuilt.isForceDisabled() { - a.prebuiltFileToDelete = prebuilt.InstallFilename() - } - } - } else if !a.vndkApex { - // indirect dependencies - if am, ok := child.(android.ApexModule); ok { - // We cannot use a switch statement on `depTag` here as the checked - // tags used below are private (e.g. `cc.sharedDepTag`). - if cc.IsSharedDepTag(depTag) || cc.IsRuntimeDepTag(depTag) { - if cc, ok := child.(*cc.Module); ok { - if cc.UseVndk() && proptools.Bool(a.properties.Use_vndk_as_stable) && cc.IsVndk() { - requireNativeLibs = append(requireNativeLibs, ":vndk") - return false - } - af := apexFileForNativeLibrary(ctx, cc, handleSpecialLibs) - af.transitiveDep = true - - // Always track transitive dependencies for host. - if a.Host() { - filesInfo = append(filesInfo, af) - return true - } - - abInfo := ctx.Provider(ApexBundleInfoProvider).(ApexBundleInfo) - if !abInfo.Contents.DirectlyInApex(depName) && (cc.IsStubs() || cc.HasStubsVariants()) { - // If the dependency is a stubs lib, don't include it in this APEX, - // but make sure that the lib is installed on the device. - // In case no APEX is having the lib, the lib is installed to the system - // partition. - // - // Always include if we are a host-apex however since those won't have any - // system libraries. - if !am.DirectlyInAnyApex() { - // we need a module name for Make - name := cc.ImplementationModuleNameForMake(ctx) + cc.Properties.SubName - if !android.InList(name, a.requiredDeps) { - a.requiredDeps = append(a.requiredDeps, name) - } - } - requireNativeLibs = append(requireNativeLibs, af.stem()) - // Don't track further - return false - } - - // If the dep is not considered to be in the same - // apex, don't add it to filesInfo so that it is not - // included in this APEX. - // TODO(jiyong): move this to at the top of the - // else-if clause for the indirect dependencies. - // Currently, that's impossible because we would - // like to record requiredNativeLibs even when - // DepIsInSameAPex is false. We also shouldn't do - // this for host. - // - // TODO(jiyong): explain why the same module is passed in twice. - // Switching the first am to parent breaks lots of tests. - if !android.IsDepInSameApex(ctx, am, am) { - return false - } - - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - } else if rm, ok := child.(*rust.Module); ok { - af := apexFileForRustLibrary(ctx, rm) - af.transitiveDep = true - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - } - } else if cc.IsTestPerSrcDepTag(depTag) { - if cc, ok := child.(*cc.Module); ok { - af := apexFileForExecutable(ctx, cc) - // Handle modules created as `test_per_src` variations of a single test module: - // use the name of the generated test binary (`fileToCopy`) instead of the name - // of the original test module (`depName`, shared by all `test_per_src` - // variations of that module). - af.androidMkModuleName = filepath.Base(af.builtFile.String()) - // these are not considered transitive dep - af.transitiveDep = false - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - } - } else if cc.IsHeaderDepTag(depTag) { - // nothing - } else if java.IsJniDepTag(depTag) { - // Because APK-in-APEX embeds jni_libs transitively, we don't need to track transitive deps - return false - } else if java.IsXmlPermissionsFileDepTag(depTag) { - if prebuilt, ok := child.(prebuilt_etc.PrebuiltEtcModule); ok { - filesInfo = append(filesInfo, apexFileForPrebuiltEtc(ctx, prebuilt, depName)) - } - } else if rust.IsDylibDepTag(depTag) { - if rustm, ok := child.(*rust.Module); ok && rustm.IsInstallableToApex() { - af := apexFileForRustLibrary(ctx, rustm) - af.transitiveDep = true - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - } - } else if rust.IsRlibDepTag(depTag) { - // Rlib is statically linked, but it might have shared lib - // dependencies. Track them. - return true - } else if java.IsBootclasspathFragmentContentDepTag(depTag) { - // Add the contents of the bootclasspath fragment to the apex. - switch child.(type) { - case *java.Library, *java.SdkLibrary: - javaModule := child.(javaModule) - af := apexFileForBootclasspathFragmentContentModule(ctx, parent, javaModule) - if !af.ok() { - ctx.PropertyErrorf("bootclasspath_fragments", "bootclasspath_fragment content %q is not configured to be compiled into dex", depName) - return false - } - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - default: - ctx.PropertyErrorf("bootclasspath_fragments", "bootclasspath_fragment content %q of type %q is not supported", depName, ctx.OtherModuleType(child)) - } - } else if java.IsSystemServerClasspathFragmentContentDepTag(depTag) { - // Add the contents of the systemserverclasspath fragment to the apex. - switch child.(type) { - case *java.Library, *java.SdkLibrary: - af := apexFileForJavaModule(ctx, child.(javaModule)) - filesInfo = append(filesInfo, af) - return true // track transitive dependencies - default: - ctx.PropertyErrorf("systemserverclasspath_fragments", "systemserverclasspath_fragment content %q of type %q is not supported", depName, ctx.OtherModuleType(child)) - } - } else if _, ok := depTag.(android.CopyDirectlyInAnyApexTag); ok { - // nothing - } else if depTag == android.DarwinUniversalVariantTag { - // nothing - } else if am.CanHaveApexVariants() && am.IsInstallableToApex() { - ctx.ModuleErrorf("unexpected tag %s for indirect dependency %q", android.PrettyPrintTag(depTag), depName) - } - } - } - return false - }) + vctx := visitorContext{handleSpecialLibs: !android.Bool(a.properties.Ignore_system_library_special_case)} + ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool { return a.depVisitor(&vctx, ctx, child, parent) }) + vctx.normalizeFileInfo() if a.privateKeyFile == nil { ctx.PropertyErrorf("key", "private_key for %q could not be found", String(a.overridableProperties.Key)) return } - // Remove duplicates in filesInfo - removeDup := func(filesInfo []apexFile) []apexFile { - encountered := make(map[string]apexFile) - for _, f := range filesInfo { - dest := filepath.Join(f.installDir, f.builtFile.Base()) - if e, ok := encountered[dest]; !ok { - encountered[dest] = f - } else { - // If a module is directly included and also transitively depended on - // consider it as directly included. - e.transitiveDep = e.transitiveDep && f.transitiveDep - encountered[dest] = e - } - } - var result []apexFile - for _, v := range encountered { - result = append(result, v) - } - return result - } - filesInfo = removeDup(filesInfo) - - // Sort to have consistent build rules - sort.Slice(filesInfo, func(i, j int) bool { - // Sort by destination path so as to ensure consistent ordering even if the source of the files - // changes. - return filesInfo[i].path() < filesInfo[j].path() - }) - //////////////////////////////////////////////////////////////////////////////////////////// // 3) some fields in apexBundle struct are configured a.installDir = android.PathForModuleInstall(ctx, "apex") - a.filesInfo = filesInfo + a.filesInfo = vctx.filesInfo // Set suffix and primaryApexType depending on the ApexType buildFlattenedAsDefault := ctx.Config().FlattenApex() @@ -2235,7 +2280,7 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { //////////////////////////////////////////////////////////////////////////////////////////// // 4) generate the build rules to create the APEX. This is done in builder.go. - a.buildManifest(ctx, provideNativeLibs, requireNativeLibs) + a.buildManifest(ctx, vctx.provideNativeLibs, vctx.requireNativeLibs) if a.properties.ApexType == flattenedApex { a.buildFlattenedApex(ctx) } else { @@ -2372,6 +2417,7 @@ func newApexBundle() *apexBundle { android.InitSdkAwareModule(module) android.InitOverridableModule(module, &module.overridableProperties.Overrides) android.InitBazelModule(module) + multitree.InitExportableModule(module) return module } @@ -2422,24 +2468,109 @@ func DefaultsFactory(props ...interface{}) android.Module { type OverrideApex struct { android.ModuleBase android.OverrideModuleBase + android.BazelModuleBase } -func (o *OverrideApex) GenerateAndroidBuildActions(ctx android.ModuleContext) { +func (o *OverrideApex) GenerateAndroidBuildActions(_ android.ModuleContext) { // All the overrides happen in the base module. } // override_apex is used to create an apex module based on another apex module by overriding some of // its properties. -func overrideApexFactory() android.Module { +func OverrideApexFactory() android.Module { m := &OverrideApex{} m.AddProperties(&overridableProperties{}) android.InitAndroidMultiTargetsArchModule(m, android.DeviceSupported, android.MultilibCommon) android.InitOverrideModule(m) + android.InitBazelModule(m) return m } +func (o *OverrideApex) ConvertWithBp2build(ctx android.TopDownMutatorContext) { + if ctx.ModuleType() != "override_apex" { + return + } + + baseApexModuleName := o.OverrideModuleBase.GetOverriddenModuleName() + baseModule, baseApexExists := ctx.ModuleFromName(baseApexModuleName) + if !baseApexExists { + panic(fmt.Errorf("Base apex module doesn't exist: %s", baseApexModuleName)) + } + + a, baseModuleIsApex := baseModule.(*apexBundle) + if !baseModuleIsApex { + panic(fmt.Errorf("Base module is not apex module: %s", baseApexModuleName)) + } + attrs, props := convertWithBp2build(a, ctx) + + for _, p := range o.GetProperties() { + overridableProperties, ok := p.(*overridableProperties) + if !ok { + continue + } + + // Manifest is either empty or a file in the directory of base APEX and is not overridable. + // After it is converted in convertWithBp2build(baseApex, ctx), + // the attrs.Manifest.Value.Label is the file path relative to the directory + // of base apex. So the following code converts it to a label that looks like + // <package of base apex>:<path of manifest file> if base apex and override + // apex are not in the same package. + baseApexPackage := ctx.OtherModuleDir(a) + overrideApexPackage := ctx.ModuleDir() + if baseApexPackage != overrideApexPackage { + attrs.Manifest.Value.Label = "//" + baseApexPackage + ":" + attrs.Manifest.Value.Label + } + + // Key + if overridableProperties.Key != nil { + attrs.Key = bazel.LabelAttribute{} + attrs.Key.SetValue(android.BazelLabelForModuleDepSingle(ctx, *overridableProperties.Key)) + } + + // Certificate + if overridableProperties.Certificate != nil { + attrs.Certificate = bazel.LabelAttribute{} + attrs.Certificate.SetValue(android.BazelLabelForModuleDepSingle(ctx, *overridableProperties.Certificate)) + } + + // Prebuilts + if overridableProperties.Prebuilts != nil { + prebuiltsLabelList := android.BazelLabelForModuleDeps(ctx, overridableProperties.Prebuilts) + attrs.Prebuilts = bazel.MakeLabelListAttribute(prebuiltsLabelList) + } + + // Compressible + if overridableProperties.Compressible != nil { + attrs.Compressible = bazel.BoolAttribute{Value: overridableProperties.Compressible} + } + + // Package name + // + // e.g. com.android.adbd's package name is com.android.adbd, but + // com.google.android.adbd overrides the package name to com.google.android.adbd + // + // TODO: this can be overridden from the product configuration, see + // getOverrideManifestPackageName and + // PRODUCT_MANIFEST_PACKAGE_NAME_OVERRIDES. + // + // Instead of generating the BUILD files differently based on the product config + // at the point of conversion, this should be handled by the BUILD file loading + // from the soong_injection's product_vars, so product config is decoupled from bp2build. + if overridableProperties.Package_name != "" { + attrs.Package_name = &overridableProperties.Package_name + } + + // Logging parent + if overridableProperties.Logging_parent != "" { + attrs.Logging_parent = &overridableProperties.Logging_parent + } + } + + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: o.Name()}, &attrs) +} + /////////////////////////////////////////////////////////////////////////////////////////////////// // Vality check routines // @@ -2678,7 +2809,7 @@ func (a *apexBundle) checkStaticExecutables(ctx android.ModuleContext) { // A small list of exceptions where static executables are allowed in APEXes. func isStaticExecutableAllowed(apex string, exec string) bool { m := map[string][]string{ - "com.android.runtime": []string{ + "com.android.runtime": { "linker", "linkerconfig", }, @@ -3143,33 +3274,6 @@ func makeApexAvailableBaseline() map[string][]string { // // Module separator // - m["com.android.permission"] = []string{ - "car-ui-lib", - "iconloader", - "kotlin-annotations", - "kotlin-stdlib", - "kotlin-stdlib-jdk7", - "kotlin-stdlib-jdk8", - "kotlinx-coroutines-android", - "kotlinx-coroutines-android-nodeps", - "kotlinx-coroutines-core", - "kotlinx-coroutines-core-nodeps", - "permissioncontroller-statsd", - "GooglePermissionController", - "PermissionController", - "SettingsLibActionBarShadow", - "SettingsLibAppPreference", - "SettingsLibBarChartPreference", - "SettingsLibLayoutPreference", - "SettingsLibProgressBar", - "SettingsLibSearchWidget", - "SettingsLibSettingsTheme", - "SettingsLibRestrictedLockUtils", - "SettingsLibHelpUtils", - } - // - // Module separator - // m["com.android.runtime"] = []string{ "bionic_libc_platform_headers", "libarm-optimized-routines-math", @@ -3342,11 +3446,11 @@ func createBcpPermittedPackagesRules(bcpPermittedPackages map[string][]string) [ // Adding code to the bootclasspath in new packages will cause issues on module update. func qBcpPackages() map[string][]string { return map[string][]string{ - "conscrypt": []string{ + "conscrypt": { "android.net.ssl", "com.android.org.conscrypt", }, - "updatable-media": []string{ + "updatable-media": { "android.media", }, } @@ -3356,32 +3460,32 @@ func qBcpPackages() map[string][]string { // Adding code to the bootclasspath in new packages will cause issues on module update. func rBcpPackages() map[string][]string { return map[string][]string{ - "framework-mediaprovider": []string{ + "framework-mediaprovider": { "android.provider", }, - "framework-permission": []string{ + "framework-permission": { "android.permission", "android.app.role", "com.android.permission", "com.android.role", }, - "framework-sdkextensions": []string{ + "framework-sdkextensions": { "android.os.ext", }, - "framework-statsd": []string{ + "framework-statsd": { "android.app", "android.os", "android.util", "com.android.internal.statsd", "com.android.server.stats", }, - "framework-wifi": []string{ + "framework-wifi": { "com.android.server.wifi", "com.android.wifi.x", "android.hardware.wifi", "android.net.wifi", }, - "framework-tethering": []string{ + "framework-tethering": { "android.net", }, } @@ -3403,6 +3507,8 @@ type bazelApexBundleAttributes struct { Native_shared_libs_32 bazel.LabelListAttribute Native_shared_libs_64 bazel.LabelListAttribute Compressible bazel.BoolAttribute + Package_name *string + Logging_parent *string } type convertedNativeSharedLibs struct { @@ -3417,10 +3523,13 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { return } + attrs, props := convertWithBp2build(a, ctx) + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: a.Name()}, &attrs) +} + +func convertWithBp2build(a *apexBundle, ctx android.TopDownMutatorContext) (bazelApexBundleAttributes, bazel.BazelTargetModuleProperties) { var manifestLabelAttribute bazel.LabelAttribute - if a.properties.Manifest != nil { - manifestLabelAttribute.SetValue(android.BazelLabelForModuleSrcSingle(ctx, *a.properties.Manifest)) - } + manifestLabelAttribute.SetValue(android.BazelLabelForModuleSrcSingle(ctx, proptools.StringDefault(a.properties.Manifest, "apex_manifest.json"))) var androidManifestLabelAttribute bazel.LabelAttribute if a.properties.AndroidManifest != nil { @@ -3428,8 +3537,15 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } var fileContextsLabelAttribute bazel.LabelAttribute - if a.properties.File_contexts != nil { + if a.properties.File_contexts == nil { + // See buildFileContexts(), if file_contexts is not specified the default one is used, which is //system/sepolicy/apex:<module name>-file_contexts + fileContextsLabelAttribute.SetValue(android.BazelLabelForModuleDepSingle(ctx, a.Name()+"-file_contexts")) + } else if strings.HasPrefix(*a.properties.File_contexts, ":") { + // File_contexts is a module fileContextsLabelAttribute.SetValue(android.BazelLabelForModuleDepSingle(ctx, *a.properties.File_contexts)) + } else { + // File_contexts is a file + fileContextsLabelAttribute.SetValue(android.BazelLabelForModuleSrcSingle(ctx, *a.properties.File_contexts)) } // TODO(b/219503907) this would need to be set to a.MinSdkVersionValue(ctx) but @@ -3487,7 +3603,17 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { compressibleAttribute.Value = a.overridableProperties.Compressible } - attrs := &bazelApexBundleAttributes{ + var packageName *string + if a.overridableProperties.Package_name != "" { + packageName = &a.overridableProperties.Package_name + } + + var loggingParent *string + if a.overridableProperties.Logging_parent != "" { + loggingParent = &a.overridableProperties.Logging_parent + } + + attrs := bazelApexBundleAttributes{ Manifest: manifestLabelAttribute, Android_manifest: androidManifestLabelAttribute, File_contexts: fileContextsLabelAttribute, @@ -3501,14 +3627,16 @@ func (a *apexBundle) ConvertWithBp2build(ctx android.TopDownMutatorContext) { Binaries: binariesLabelListAttribute, Prebuilts: prebuiltsLabelListAttribute, Compressible: compressibleAttribute, + Package_name: packageName, + Logging_parent: loggingParent, } props := bazel.BazelTargetModuleProperties{ Rule_class: "apex", - Bzl_load_location: "//build/bazel/rules:apex.bzl", + Bzl_load_location: "//build/bazel/rules/apex:apex.bzl", } - ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: a.Name()}, attrs) + return attrs, props } // The following conversions are based on this table where the rows are the compile_multilib diff --git a/apex/apex_test.go b/apex/apex_test.go index 4a52115db..dbe918010 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -223,6 +223,7 @@ var prepareForApexTest = android.GroupFixturePreparers( // not because of these tests specifically (it's not used by the tests) variables.Platform_version_active_codenames = []string{"Q", "Tiramisu"} variables.Platform_vndk_version = proptools.StringPtr("29") + variables.BuildId = proptools.StringPtr("TEST.BUILD_ID") }), ) @@ -682,7 +683,7 @@ func TestDefaults(t *testing.T) { "etc/myetc", "javalib/myjar.jar", "lib64/mylib.so", - "app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk", + "app/AppFoo@TEST.BUILD_ID/AppFoo.apk", "overlay/blue/rro.apk", "etc/bpf/bpf.o", "etc/bpf/bpf2.o", @@ -2393,6 +2394,7 @@ func TestApexMinSdkVersion_ErrorIfDepIsNewer_Java(t *testing.T) { key: "myapex.key", apps: ["AppFoo"], min_sdk_version: "29", + updatable: false, } apex_key { @@ -5682,8 +5684,8 @@ func TestApexWithApps(t *testing.T) { apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] - ensureContains(t, copyCmds, "image.apex/app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk") - ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@__APEX_VERSION_PLACEHOLDER__/AppFooPriv.apk") + ensureContains(t, copyCmds, "image.apex/app/AppFoo@TEST.BUILD_ID/AppFoo.apk") + ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@TEST.BUILD_ID/AppFooPriv.apk") appZipRule := ctx.ModuleForTests("AppFoo", "android_common_apex10000").Description("zip jni libs") // JNI libraries are uncompressed @@ -5700,6 +5702,36 @@ func TestApexWithApps(t *testing.T) { } } +func TestApexWithAppImportBuildId(t *testing.T) { + invalidBuildIds := []string{"../", "a b", "a/b", "a/b/../c", "/a"} + for _, id := range invalidBuildIds { + message := fmt.Sprintf("Unable to use build id %s as filename suffix", id) + fixture := android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.BuildId = proptools.StringPtr(id) + }) + testApexError(t, message, `apex { + name: "myapex", + key: "myapex.key", + apps: ["AppFooPrebuilt"], + updatable: false, + } + + apex_key { + name: "myapex.key", + public_key: "testkey.avbpubkey", + private_key: "testkey.pem", + } + + android_app_import { + name: "AppFooPrebuilt", + apk: "PrebuiltAppFoo.apk", + presigned: true, + apex_available: ["myapex"], + } + `, fixture) + } +} + func TestApexWithAppImports(t *testing.T) { ctx := testApex(t, ` apex { @@ -5745,8 +5777,8 @@ func TestApexWithAppImports(t *testing.T) { apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] - ensureContains(t, copyCmds, "image.apex/app/AppFooPrebuilt@__APEX_VERSION_PLACEHOLDER__/AppFooPrebuilt.apk") - ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPrivPrebuilt@__APEX_VERSION_PLACEHOLDER__/AwesomePrebuiltAppFooPriv.apk") + ensureContains(t, copyCmds, "image.apex/app/AppFooPrebuilt@TEST.BUILD_ID/AppFooPrebuilt.apk") + ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPrivPrebuilt@TEST.BUILD_ID/AwesomePrebuiltAppFooPriv.apk") } func TestApexWithAppImportsPrefer(t *testing.T) { @@ -5787,7 +5819,7 @@ func TestApexWithAppImportsPrefer(t *testing.T) { })) ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{ - "app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFooPrebuilt.apk", + "app/AppFoo@TEST.BUILD_ID/AppFooPrebuilt.apk", }) } @@ -5820,7 +5852,7 @@ func TestApexWithTestHelperApp(t *testing.T) { apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] - ensureContains(t, copyCmds, "image.apex/app/TesterHelpAppFoo@__APEX_VERSION_PLACEHOLDER__/TesterHelpAppFoo.apk") + ensureContains(t, copyCmds, "image.apex/app/TesterHelpAppFoo@TEST.BUILD_ID/TesterHelpAppFoo.apk") } func TestApexPropertiesShouldBeDefaultable(t *testing.T) { @@ -5899,7 +5931,7 @@ func TestApexAvailable_DirectDep(t *testing.T) { func TestApexAvailable_IndirectDep(t *testing.T) { // libbbaz is an indirect dep testApexError(t, `requires "libbaz" that doesn't list the APEX under 'apex_available'.\n\nDependency path: -.*via tag apex\.dependencyTag.*name:sharedLib.* +.*via tag apex\.dependencyTag\{"sharedLib"\} .*-> libfoo.*link:shared.* .*via tag cc\.libraryDependencyTag.*Kind:sharedLibraryDependency.* .*-> libbar.*link:shared.* @@ -6195,6 +6227,9 @@ func TestOverrideApex(t *testing.T) { name: "mybootclasspath_fragment", contents: ["bcplib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -6209,6 +6244,9 @@ func TestOverrideApex(t *testing.T) { name: "override_bootclasspath_fragment", contents: ["override_bcplib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -6263,8 +6301,8 @@ func TestOverrideApex(t *testing.T) { apexRule := module.Rule("apexRule") copyCmds := apexRule.Args["copy_commands"] - ensureNotContains(t, copyCmds, "image.apex/app/app@__APEX_VERSION_PLACEHOLDER__/app.apk") - ensureContains(t, copyCmds, "image.apex/app/override_app@__APEX_VERSION_PLACEHOLDER__/override_app.apk") + ensureNotContains(t, copyCmds, "image.apex/app/app@TEST.BUILD_ID/app.apk") + ensureContains(t, copyCmds, "image.apex/app/override_app@TEST.BUILD_ID/override_app.apk") ensureNotContains(t, copyCmds, "image.apex/etc/bpf/bpf.o") ensureContains(t, copyCmds, "image.apex/etc/bpf/override_bpf.o") @@ -7168,7 +7206,7 @@ func TestAppBundle(t *testing.T) { content := bundleConfigRule.Args["content"] ensureContains(t, content, `"compression":{"uncompressed_glob":["apex_payload.img","apex_manifest.*"]}`) - ensureContains(t, content, `"apex_config":{"apex_embedded_apk_config":[{"package_name":"com.android.foo","path":"app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk"}]}`) + ensureContains(t, content, `"apex_config":{"apex_embedded_apk_config":[{"package_name":"com.android.foo","path":"app/AppFoo@TEST.BUILD_ID/AppFoo.apk"}]}`) } func TestAppSetBundle(t *testing.T) { @@ -7199,9 +7237,9 @@ func TestAppSetBundle(t *testing.T) { if len(copyCmds) != 3 { t.Fatalf("Expected 3 commands, got %d in:\n%s", len(copyCmds), s) } - ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__$") - ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__$") - ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__ .*/AppSet.zip$") + ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@TEST.BUILD_ID$") + ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@TEST.BUILD_ID$") + ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@TEST.BUILD_ID .*/AppSet.zip$") } func TestAppSetBundlePrebuilt(t *testing.T) { @@ -7262,6 +7300,9 @@ func testNoUpdatableJarsInBootImage(t *testing.T, errmsg string, preparer androi apex_available: [ "some-non-updatable-apex", ], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -7320,6 +7361,9 @@ func testNoUpdatableJarsInBootImage(t *testing.T, errmsg string, preparer androi apex_available: [ "com.android.art.debug", ], + hidden_api: { + split_packages: ["*"], + }, } apex_key { @@ -7396,7 +7440,7 @@ func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, preparer android.F return result.TestContext } -func TestDuplicateDeapexeresFromPrebuiltApexes(t *testing.T) { +func TestDuplicateDeapexersFromPrebuiltApexes(t *testing.T) { preparers := android.GroupFixturePreparers( java.PrepareForTestWithJavaDefaultModules, PrepareForTestWithApexBuildComponents, @@ -7465,6 +7509,107 @@ func TestDuplicateDeapexeresFromPrebuiltApexes(t *testing.T) { }) } +func TestDuplicateButEquivalentDeapexersFromPrebuiltApexes(t *testing.T) { + preparers := android.GroupFixturePreparers( + java.PrepareForTestWithJavaDefaultModules, + PrepareForTestWithApexBuildComponents, + ) + + bpBase := ` + apex_set { + name: "com.android.myapex", + installable: true, + exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + set: "myapex.apks", + } + + apex_set { + name: "com.android.myapex_compressed", + apex_name: "com.android.myapex", + installable: true, + exported_bootclasspath_fragments: ["my-bootclasspath-fragment"], + set: "myapex_compressed.apks", + } + + prebuilt_bootclasspath_fragment { + name: "my-bootclasspath-fragment", + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + hidden_api: { + annotation_flags: "annotation-flags.csv", + metadata: "metadata.csv", + index: "index.csv", + signature_patterns: "signature_patterns.csv", + }, + %s + } + ` + + t.Run("java_import", func(t *testing.T) { + result := preparers.RunTestWithBp(t, + fmt.Sprintf(bpBase, `contents: ["libfoo"]`)+` + java_import { + name: "libfoo", + jars: ["libfoo.jar"], + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + } + `) + + module := result.Module("libfoo", "android_common_com.android.myapex") + usesLibraryDep := module.(java.UsesLibraryDependency) + android.AssertPathRelativeToTopEquals(t, "dex jar path", + "out/soong/.intermediates/com.android.myapex.deapexer/android_common/deapexer/javalib/libfoo.jar", + usesLibraryDep.DexJarBuildPath().Path()) + }) + + t.Run("java_sdk_library_import", func(t *testing.T) { + result := preparers.RunTestWithBp(t, + fmt.Sprintf(bpBase, `contents: ["libfoo"]`)+` + java_sdk_library_import { + name: "libfoo", + public: { + jars: ["libbar.jar"], + }, + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + compile_dex: true, + } + `) + + module := result.Module("libfoo", "android_common_com.android.myapex") + usesLibraryDep := module.(java.UsesLibraryDependency) + android.AssertPathRelativeToTopEquals(t, "dex jar path", + "out/soong/.intermediates/com.android.myapex.deapexer/android_common/deapexer/javalib/libfoo.jar", + usesLibraryDep.DexJarBuildPath().Path()) + }) + + t.Run("prebuilt_bootclasspath_fragment", func(t *testing.T) { + _ = preparers.RunTestWithBp(t, fmt.Sprintf(bpBase, ` + image_name: "art", + contents: ["libfoo"], + `)+` + java_sdk_library_import { + name: "libfoo", + public: { + jars: ["libbar.jar"], + }, + apex_available: [ + "com.android.myapex", + "com.android.myapex_compressed", + ], + compile_dex: true, + } + `) + }) +} + func TestUpdatable_should_set_min_sdk_version(t *testing.T) { testApexError(t, `"myapex" .*: updatable: updatable APEXes should set min_sdk_version`, ` apex { @@ -8663,6 +8808,9 @@ func TestApexJavaCoverage(t *testing.T) { name: "mybootclasspathfragment", contents: ["mybootclasspathlib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -8983,6 +9131,9 @@ func TestSdkLibraryCanHaveHigherMinSdkVersion(t *testing.T) { name: "mybootclasspathfragment", contents: ["mybootclasspathlib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_sdk_library { @@ -9083,6 +9234,9 @@ func TestSdkLibraryCanHaveHigherMinSdkVersion(t *testing.T) { name: "mybootclasspathfragment", contents: ["mybootclasspathlib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_sdk_library { @@ -9302,6 +9456,9 @@ func TestApexStrictUpdtabilityLintBcpFragmentDeps(t *testing.T) { name: "mybootclasspathfragment", contents: ["myjavalib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { name: "myjavalib", @@ -9324,6 +9481,69 @@ func TestApexStrictUpdtabilityLintBcpFragmentDeps(t *testing.T) { } } +// updatable apexes should propagate updatable=true to its apps +func TestUpdatableApexEnforcesAppUpdatability(t *testing.T) { + bp := ` + apex { + name: "myapex", + key: "myapex.key", + updatable: %v, + apps: [ + "myapp", + ], + min_sdk_version: "30", + } + apex_key { + name: "myapex.key", + } + android_app { + name: "myapp", + updatable: %v, + apex_available: [ + "myapex", + ], + sdk_version: "current", + min_sdk_version: "30", + } + ` + testCases := []struct { + name string + apex_is_updatable_bp bool + app_is_updatable_bp bool + app_is_updatable_expected bool + }{ + { + name: "Non-updatable apex respects updatable property of non-updatable app", + apex_is_updatable_bp: false, + app_is_updatable_bp: false, + app_is_updatable_expected: false, + }, + { + name: "Non-updatable apex respects updatable property of updatable app", + apex_is_updatable_bp: false, + app_is_updatable_bp: true, + app_is_updatable_expected: true, + }, + { + name: "Updatable apex respects updatable property of updatable app", + apex_is_updatable_bp: true, + app_is_updatable_bp: true, + app_is_updatable_expected: true, + }, + { + name: "Updatable apex sets updatable=true on non-updatable app", + apex_is_updatable_bp: true, + app_is_updatable_bp: false, + app_is_updatable_expected: true, + }, + } + for _, testCase := range testCases { + result := testApex(t, fmt.Sprintf(bp, testCase.apex_is_updatable_bp, testCase.app_is_updatable_bp)) + myapp := result.ModuleForTests("myapp", "android_common").Module().(*java.AndroidApp) + android.AssertBoolEquals(t, testCase.name, testCase.app_is_updatable_expected, myapp.Updatable()) + } +} + func TestMain(m *testing.M) { os.Exit(m.Run()) } diff --git a/apex/bootclasspath_fragment_test.go b/apex/bootclasspath_fragment_test.go index ce6b7f730..b298dac5d 100644 --- a/apex/bootclasspath_fragment_test.go +++ b/apex/bootclasspath_fragment_test.go @@ -110,6 +110,9 @@ func TestBootclasspathFragments(t *testing.T) { apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } `, ) @@ -209,6 +212,9 @@ func TestBootclasspathFragments_FragmentDependency(t *testing.T) { apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } bootclasspath_fragment { @@ -220,6 +226,9 @@ func TestBootclasspathFragments_FragmentDependency(t *testing.T) { module: "art-bootclasspath-fragment", }, ], + hidden_api: { + split_packages: ["*"], + }, } `, ) @@ -361,6 +370,9 @@ func TestBootclasspathFragmentInArtApex(t *testing.T) { apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } `, contentsInsert(contents)) @@ -853,6 +865,9 @@ func TestBootclasspathFragmentContentsNoName(t *testing.T) { apex_available: [ "myapex", ], + hidden_api: { + split_packages: ["*"], + }, } `) @@ -959,6 +974,9 @@ func TestBootclasspathFragment_HiddenAPIList(t *testing.T) { apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } apex { @@ -1010,6 +1028,9 @@ func TestBootclasspathFragment_HiddenAPIList(t *testing.T) { module: "art-bootclasspath-fragment", }, ], + hidden_api: { + split_packages: ["*"], + }, } `) @@ -1123,6 +1144,9 @@ func TestBootclasspathFragment_AndroidNonUpdatable(t *testing.T) { apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } apex { @@ -1175,6 +1199,9 @@ func TestBootclasspathFragment_AndroidNonUpdatable(t *testing.T) { module: "art-bootclasspath-fragment", }, ], + hidden_api: { + split_packages: ["*"], + }, } `) @@ -1282,6 +1309,9 @@ func TestBootclasspathFragment_AndroidNonUpdatable_AlwaysUsePrebuiltSdks(t *test apex_available: [ "com.android.art", ], + hidden_api: { + split_packages: ["*"], + }, } apex { @@ -1334,6 +1364,9 @@ func TestBootclasspathFragment_AndroidNonUpdatable_AlwaysUsePrebuiltSdks(t *test module: "art-bootclasspath-fragment", }, ], + hidden_api: { + split_packages: ["*"], + }, } `) diff --git a/apex/builder.go b/apex/builder.go index d4765d022..7e2b924c7 100644 --- a/apex/builder.go +++ b/apex/builder.go @@ -82,6 +82,11 @@ var ( Description: "prepare ${out}", }, "provideNativeLibs", "requireNativeLibs", "opt") + stripCommentsApexManifestRule = pctx.StaticRule("stripCommentsApexManifestRule", blueprint.RuleParams{ + Command: `sed '/^\s*\/\//d' $in > $out`, + Description: "strip lines starting with // ${in}=>${out}", + }) + stripApexManifestRule = pctx.StaticRule("stripApexManifestRule", blueprint.RuleParams{ Command: `rm -f $out && ${conv_apex_manifest} strip $in -o $out`, CommandDeps: []string{"${conv_apex_manifest}"}, @@ -107,16 +112,14 @@ var ( `--canned_fs_config ${canned_fs_config} ` + `--include_build_info ` + `--payload_type image ` + - `--key ${key} ` + - `--apex_version_placeholder ${apex_version_placeholder} ` + - `${opt_flags} ${image_dir} ${out} `, + `--key ${key} ${opt_flags} ${image_dir} ${out} `, CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}", "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", "${sload_f2fs}", "${make_erofs}", "${soong_zip}", "${zipalign}", "${aapt2}", "prebuilts/sdk/current/public/android.jar"}, Rspfile: "${out}.copy_commands", RspfileContent: "${copy_commands}", Description: "APEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type", "apex_version_placeholder") + }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type") zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{ Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` + @@ -124,13 +127,12 @@ var ( `APEXER_TOOL_PATH=${tool_path} ` + `${apexer} --force --manifest ${manifest} ` + `--payload_type zip ` + - `--apex_version_placeholder ${apex_version_placeholder} ` + `${image_dir} ${out} `, CommandDeps: []string{"${apexer}", "${merge_zips}", "${soong_zip}", "${zipalign}", "${aapt2}"}, Rspfile: "${out}.copy_commands", RspfileContent: "${copy_commands}", Description: "ZipAPEX ${image_dir} => ${out}", - }, "tool_path", "image_dir", "copy_commands", "manifest", "apex_version_placeholder") + }, "tool_path", "image_dir", "copy_commands", "manifest") apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule", blueprint.RuleParams{ @@ -208,10 +210,17 @@ func (a *apexBundle) buildManifest(ctx android.ModuleContext, provideNativeLibs, optCommands = append(optCommands, "-a jniLibs "+strings.Join(jniLibs, " ")) } + manifestJsonCommentsStripped := android.PathForModuleOut(ctx, "apex_manifest_comments_stripped.json") + ctx.Build(pctx, android.BuildParams{ + Rule: stripCommentsApexManifestRule, + Input: src, + Output: manifestJsonCommentsStripped, + }) + manifestJsonFullOut := android.PathForModuleOut(ctx, "apex_manifest_full.json") ctx.Build(pctx, android.BuildParams{ Rule: apexManifestRule, - Input: src, + Input: manifestJsonCommentsStripped, Output: manifestJsonFullOut, Args: map[string]string{ "provideNativeLibs": strings.Join(provideNativeLibs, " "), @@ -621,7 +630,12 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { // Create a NOTICE file, and embed it as an asset file in the APEX. a.htmlGzNotice = android.PathForModuleOut(ctx, "NOTICE.html.gz") - android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, a.htmlGzNotice) + android.BuildNoticeHtmlOutputFromLicenseMetadata( + ctx, a.htmlGzNotice, "", "", + []string{ + android.PathForModuleInstall(ctx).String() + "/", + android.PathForModuleInPartitionInstall(ctx, "apex").String() + "/", + }) noticeAssetPath := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") builder := android.NewRuleBuilder(pctx, ctx) builder.Command().Text("cp"). @@ -653,6 +667,8 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { optFlags = append(optFlags, "--manifest_json "+a.manifestJsonOut.String()) } + optFlags = append(optFlags, "--apex_version "+defaultManifestVersion) + optFlags = append(optFlags, "--payload_fs_type "+a.payloadFsType.string()) ctx.Build(pctx, android.BuildParams{ @@ -661,15 +677,14 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { Output: unsignedOutputFile, Description: "apex (" + apexType.name() + ")", Args: map[string]string{ - "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, - "image_dir": imageDir.String(), - "copy_commands": strings.Join(copyCommands, " && "), - "manifest": a.manifestPbOut.String(), - "file_contexts": fileContexts.String(), - "canned_fs_config": cannedFsConfig.String(), - "key": a.privateKeyFile.String(), - "opt_flags": strings.Join(optFlags, " "), - "apex_version_placeholder": APEX_VERSION_PLACEHOLDER, + "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, + "image_dir": imageDir.String(), + "copy_commands": strings.Join(copyCommands, " && "), + "manifest": a.manifestPbOut.String(), + "file_contexts": fileContexts.String(), + "canned_fs_config": cannedFsConfig.String(), + "key": a.privateKeyFile.String(), + "opt_flags": strings.Join(optFlags, " "), }, }) @@ -761,11 +776,10 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) { Output: unsignedOutputFile, Description: "apex (" + apexType.name() + ")", Args: map[string]string{ - "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, - "image_dir": imageDir.String(), - "copy_commands": strings.Join(copyCommands, " && "), - "manifest": a.manifestPbOut.String(), - "apex_version_placeholder": APEX_VERSION_PLACEHOLDER, + "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir, + "image_dir": imageDir.String(), + "copy_commands": strings.Join(copyCommands, " && "), + "manifest": a.manifestPbOut.String(), }, }) } diff --git a/apex/classpath_element_test.go b/apex/classpath_element_test.go index 60f18bd57..9142eed99 100644 --- a/apex/classpath_element_test.go +++ b/apex/classpath_element_test.go @@ -88,6 +88,9 @@ func TestCreateClasspathElements(t *testing.T) { "baz", "quuz", ], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -134,6 +137,9 @@ func TestCreateClasspathElements(t *testing.T) { contents: [ "bar", ], + hidden_api: { + split_packages: ["*"], + }, } java_library { diff --git a/apex/constants.go b/apex/constants.go new file mode 100644 index 000000000..c68edb724 --- /dev/null +++ b/apex/constants.go @@ -0,0 +1,36 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apex + +// This file contains branch specific constants. They are stored in a separate +// file to minimise the potential of merge conflicts between branches when +// the code from the package is changed. + +// The default manifest version for all the modules on this branch. +// This version code will be used only if there is no version field in the +// module's apex_manifest.json. Release branches have their version injected +// into apex_manifest.json by the tooling and will not use the version set +// here. Developers can also set the version field locally in the +// apex_manifest.json to build a module with a specific version. +// +// The value follows the schema from go/mainline-version-codes, and is chosen +// based on the branch such that the builds from testing and development +// branches will have a version higher than the prebuilts. +// Versions per branch: +// * x-dev - xx0090000 (where xx is the branch SDK level) +// * AOSP - xx9990000 +// * x-mainline-prod - xx9990000 +// * master - 990090000 +const defaultManifestVersion = "339990000" diff --git a/apex/key.go b/apex/key.go index 829410ed2..9c5bb05e7 100644 --- a/apex/key.go +++ b/apex/key.go @@ -230,7 +230,7 @@ func apexKeyBp2BuildInternal(ctx android.TopDownMutatorContext, module *apexKey) props := bazel.BazelTargetModuleProperties{ Rule_class: "apex_key", - Bzl_load_location: "//build/bazel/rules:apex_key.bzl", + Bzl_load_location: "//build/bazel/rules/apex:apex_key.bzl", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs) diff --git a/apex/platform_bootclasspath_test.go b/apex/platform_bootclasspath_test.go index 06c39ee4b..4b48da8e2 100644 --- a/apex/platform_bootclasspath_test.go +++ b/apex/platform_bootclasspath_test.go @@ -125,6 +125,7 @@ func TestPlatformBootclasspath_Fragments(t *testing.T) { unsupported_packages: [ "bar-unsupported-packages.txt", ], + split_packages: ["*"], }, } @@ -274,6 +275,9 @@ func TestPlatformBootclasspathDependencies(t *testing.T) { "baz", "quuz", ], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -317,6 +321,9 @@ func TestPlatformBootclasspathDependencies(t *testing.T) { name: "my-bootclasspath-fragment", contents: ["bar"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } apex_key { @@ -482,6 +489,9 @@ func TestPlatformBootclasspath_AlwaysUsePrebuiltSdks(t *testing.T) { contents: [ "foo", "bar", ], + hidden_api: { + split_packages: ["*"], + }, } prebuilt_bootclasspath_fragment { @@ -599,6 +609,9 @@ func TestPlatformBootclasspath_IncludesRemainingApexJars(t *testing.T) { generate_classpaths_proto: false, contents: ["foo"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -656,6 +669,9 @@ func TestBootJarNotInApex(t *testing.T) { contents: [ "foo", ], + hidden_api: { + split_packages: ["*"], + }, } platform_bootclasspath { @@ -696,6 +712,9 @@ func TestBootFragmentNotInApex(t *testing.T) { bootclasspath_fragment { name: "not-in-apex-fragment", contents: ["foo"], + hidden_api: { + split_packages: ["*"], + }, } platform_bootclasspath { @@ -746,6 +765,9 @@ func TestNonBootJarInFragment(t *testing.T) { name: "apex-fragment", contents: ["foo", "bar"], apex_available:[ "myapex" ], + hidden_api: { + split_packages: ["*"], + }, } platform_bootclasspath { diff --git a/bazel/aquery.go b/bazel/aquery.go index fd8cf677d..1d1f49cdf 100644 --- a/bazel/aquery.go +++ b/bazel/aquery.go @@ -15,26 +15,34 @@ package bazel import ( + "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "path/filepath" + "reflect" "regexp" + "sort" "strings" "github.com/google/blueprint/proptools" ) +type artifactId int +type depsetId int +type pathFragmentId int + // artifact contains relevant portions of Bazel's aquery proto, Artifact. // Represents a single artifact, whether it's a source file or a derived output file. type artifact struct { - Id int - PathFragmentId int + Id artifactId + PathFragmentId pathFragmentId } type pathFragment struct { - Id int + Id pathFragmentId Label string - ParentId int + ParentId pathFragmentId } // KeyValuePair represents Bazel's aquery proto, KeyValuePair. @@ -43,13 +51,26 @@ type KeyValuePair struct { Value string } +// AqueryDepset is a depset definition from Bazel's aquery response. This is +// akin to the `depSetOfFiles` in the response proto, except: +// * direct artifacts are enumerated by full path instead of by ID +// * it has a hash of the depset contents, instead of an int ID (for determinism) +// A depset is a data structure for efficient transitive handling of artifact +// paths. A single depset consists of one or more artifact paths and one or +// more "child" depsets. +type AqueryDepset struct { + ContentHash string + DirectArtifacts []string + TransitiveDepSetHashes []string +} + // depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles. // Represents a data structure containing one or more files. Depsets in Bazel are an efficient // data structure for storing large numbers of file paths. type depSetOfFiles struct { - Id int - DirectArtifactIds []int - TransitiveDepSetIds []int + Id depsetId + DirectArtifactIds []artifactId + TransitiveDepSetIds []depsetId } // action contains relevant portions of Bazel's aquery proto, Action. @@ -57,9 +78,9 @@ type depSetOfFiles struct { type action struct { Arguments []string EnvironmentVariables []KeyValuePair - InputDepSetIds []int + InputDepSetIds []depsetId Mnemonic string - OutputIds []int + OutputIds []artifactId TemplateContent string Substitutions []KeyValuePair } @@ -79,33 +100,40 @@ type BuildStatement struct { Command string Depfile *string OutputPaths []string - InputPaths []string SymlinkPaths []string Env []KeyValuePair Mnemonic string + + // Inputs of this build statement, either as unexpanded depsets or expanded + // input paths. There should be no overlap between these fields; an input + // path should either be included as part of an unexpanded depset or a raw + // input path string, but not both. + InputDepsetHashes []string + InputPaths []string } // A helper type for aquery processing which facilitates retrieval of path IDs from their // less readable Bazel structures (depset and path fragment). type aqueryArtifactHandler struct { - // Maps middleman artifact Id to input artifact depset ID. - // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, - // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then, - // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for - // that action instead. - middlemanIdToDepsetIds map[int][]int - // Maps depset Id to depset struct. - depsetIdToDepset map[int]depSetOfFiles + // Switches to true if any depset contains only `bazelToolsDependencySentinel` + bazelToolsDependencySentinelNeeded bool + // Maps depset id to AqueryDepset, a representation of depset which is + // post-processed for middleman artifact handling, unhandled artifact + // dropping, content hashing, etc. + depsetIdToAqueryDepset map[depsetId]AqueryDepset + // Maps content hash to AqueryDepset. + depsetHashToAqueryDepset map[string]AqueryDepset + // depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening // may be an expensive operation. - depsetIdToArtifactIdsCache map[int][]int - // Maps artifact Id to fully expanded path. - artifactIdToPath map[int]string + depsetHashToArtifactPathsCache map[string][]string + // Maps artifact ids to fully expanded paths. + artifactIdToPath map[artifactId]string } // The tokens should be substituted with the value specified here, instead of the // one returned in 'substitutions' of TemplateExpand action. -var TemplateActionOverriddenTokens = map[string]string{ +var templateActionOverriddenTokens = map[string]string{ // Uses "python3" for %python_binary% instead of the value returned by aquery // which is "py3wrapper.sh". See removePy3wrapperScript. "%python_binary%": "python3", @@ -115,15 +143,25 @@ var TemplateActionOverriddenTokens = map[string]string{ var manifestFilePattern = regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$") // The file name of py3wrapper.sh, which is used by py_binary targets. -var py3wrapperFileName = "/py3wrapper.sh" +const py3wrapperFileName = "/py3wrapper.sh" -func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) { - pathFragments := map[int]pathFragment{} - for _, pathFragment := range aqueryResult.PathFragments { - pathFragments[pathFragment.Id] = pathFragment +// A file to be put into depsets that are otherwise empty +const bazelToolsDependencySentinel = "BAZEL_TOOLS_DEPENDENCY_SENTINEL" + +func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V { + m := map[K]V{} + for _, v := range values { + m[keyFn(v)] = v } + return m +} + +func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) { + pathFragments := indexBy(aqueryResult.PathFragments, func(pf pathFragment) pathFragmentId { + return pf.Id + }) - artifactIdToPath := map[int]string{} + artifactIdToPath := map[artifactId]string{} for _, artifact := range aqueryResult.Artifacts { artifactPath, err := expandPathFragment(artifact.PathFragmentId, pathFragments) if err != nil { @@ -132,13 +170,12 @@ func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler artifactIdToPath[artifact.Id] = artifactPath } - depsetIdToDepset := map[int]depSetOfFiles{} - for _, depset := range aqueryResult.DepSetOfFiles { - depsetIdToDepset[depset.Id] = depset - } - - // Do a pass through all actions to identify which artifacts are middleman artifacts. - middlemanIdToDepsetIds := map[int][]int{} + // Map middleman artifact ContentHash to input artifact depset ID. + // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example, + // if we find a middleman action which has inputs [foo, bar], and output [baz_middleman], then, + // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for + // that action instead. + middlemanIdToDepsetIds := map[artifactId][]depsetId{} for _, actionEntry := range aqueryResult.Actions { if actionEntry.Mnemonic == "Middleman" { for _, outputId := range actionEntry.OutputIds { @@ -146,196 +183,408 @@ func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler } } } - return &aqueryArtifactHandler{ - middlemanIdToDepsetIds: middlemanIdToDepsetIds, - depsetIdToDepset: depsetIdToDepset, - depsetIdToArtifactIdsCache: map[int][]int{}, - artifactIdToPath: artifactIdToPath, - }, nil -} -func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error) { - inputPaths := []string{} + depsetIdToDepset := indexBy(aqueryResult.DepSetOfFiles, func(d depSetOfFiles) depsetId { + return d.Id + }) - for _, inputDepSetId := range depsetIds { - inputArtifacts, err := a.artifactIdsFromDepsetId(inputDepSetId) + aqueryHandler := aqueryArtifactHandler{ + depsetIdToAqueryDepset: map[depsetId]AqueryDepset{}, + depsetHashToAqueryDepset: map[string]AqueryDepset{}, + depsetHashToArtifactPathsCache: map[string][]string{}, + artifactIdToPath: artifactIdToPath, + } + + // Validate and adjust aqueryResult.DepSetOfFiles values. + for _, depset := range aqueryResult.DepSetOfFiles { + _, err := aqueryHandler.populateDepsetMaps(depset, middlemanIdToDepsetIds, depsetIdToDepset) if err != nil { return nil, err } - for _, inputId := range inputArtifacts { - if middlemanInputDepsetIds, isMiddlemanArtifact := a.middlemanIdToDepsetIds[inputId]; isMiddlemanArtifact { - // Add all inputs from middleman actions which created middleman artifacts which are - // in the inputs for this action. - swappedInputPaths, err := a.getInputPaths(middlemanInputDepsetIds) - if err != nil { - return nil, err - } - inputPaths = append(inputPaths, swappedInputPaths...) - } else { - inputPath, exists := a.artifactIdToPath[inputId] - if !exists { - return nil, fmt.Errorf("undefined input artifactId %d", inputId) - } - inputPaths = append(inputPaths, inputPath) - } - } } - // TODO(b/197135294): Clean up this custom runfiles handling logic when - // SourceSymlinkManifest and SymlinkTree actions are supported. - filteredInputPaths := filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths) + return &aqueryHandler, nil +} + +// Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given +// depset. +func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlemanIdToDepsetIds map[artifactId][]depsetId, depsetIdToDepset map[depsetId]depSetOfFiles) (AqueryDepset, error) { + if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depset.Id]; containsDepset { + return aqueryDepset, nil + } + transitiveDepsetIds := depset.TransitiveDepSetIds + var directArtifactPaths []string + for _, artifactId := range depset.DirectArtifactIds { + path, pathExists := a.artifactIdToPath[artifactId] + if !pathExists { + return AqueryDepset{}, fmt.Errorf("undefined input artifactId %d", artifactId) + } + // Filter out any inputs which are universally dropped, and swap middleman + // artifacts with their corresponding depsets. + if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[artifactId]; isMiddleman { + // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts. + transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...) + } else if strings.HasSuffix(path, py3wrapperFileName) || + manifestFilePattern.MatchString(path) || + strings.HasPrefix(path, "../bazel_tools") { + // Drop these artifacts. + // See go/python-binary-host-mixed-build for more details. + // 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of + // Bazel py_binary targets, so there is no Ninja build statements generated for creating it. + // 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, + // but it doesn't contain sufficient information so no Ninja build statements are generated + // for creating it. + // So in mixed build mode, when these two are used as input of some Ninja build statement, + // since there is no build statement to create them, they should be removed from input paths. + // TODO(b/197135294): Clean up this custom runfiles handling logic when + // SourceSymlinkManifest and SymlinkTree actions are supported. + // 3) ../bazel_tools: they have MODIFY timestamp 10years in the future and would cause the + // containing depset to always be considered newer than their outputs. + } else { + directArtifactPaths = append(directArtifactPaths, path) + } + } - return filteredInputPaths, nil + var childDepsetHashes []string + for _, childDepsetId := range transitiveDepsetIds { + childDepset, exists := depsetIdToDepset[childDepsetId] + if !exists { + return AqueryDepset{}, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id) + } + childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset) + if err != nil { + return AqueryDepset{}, err + } + childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash) + } + if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 { + // We could omit this depset altogether but that requires cleanup on + // transitive dependents. + // As a simpler alternative, we use this sentinel file as a dependency. + directArtifactPaths = append(directArtifactPaths, bazelToolsDependencySentinel) + a.bazelToolsDependencySentinelNeeded = true + } + aqueryDepset := AqueryDepset{ + ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes), + DirectArtifacts: directArtifactPaths, + TransitiveDepSetHashes: childDepsetHashes, + } + a.depsetIdToAqueryDepset[depset.Id] = aqueryDepset + a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset + return aqueryDepset, nil } -// See go/python-binary-host-mixed-build for more details. -// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of -// Bazel py_binary targets, so there is no Ninja build statements generated for creating it. -// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets, -// but it doesn't contain sufficient information so no Ninja build statements are generated -// for creating it. -// So in mixed build mode, when these two are used as input of some Ninja build statement, -// since there is no build statement to create them, they should be removed from input paths. -func filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths []string) []string { - filteredInputPaths := []string{} - for _, path := range inputPaths { - if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) { - continue +// getInputPaths flattens the depsets of the given IDs and returns all transitive +// input paths contained in these depsets. +// This is a potentially expensive operation, and should not be invoked except +// for actions which need specialized input handling. +func (a *aqueryArtifactHandler) getInputPaths(depsetIds []depsetId) ([]string, error) { + var inputPaths []string + + for _, inputDepSetId := range depsetIds { + depset := a.depsetIdToAqueryDepset[inputDepSetId] + inputArtifacts, err := a.artifactPathsFromDepsetHash(depset.ContentHash) + if err != nil { + return nil, err + } + for _, inputPath := range inputArtifacts { + inputPaths = append(inputPaths, inputPath) } - filteredInputPaths = append(filteredInputPaths, path) } - return filteredInputPaths + + return inputPaths, nil } -func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) { - if result, exists := a.depsetIdToArtifactIdsCache[depsetId]; exists { +func (a *aqueryArtifactHandler) artifactPathsFromDepsetHash(depsetHash string) ([]string, error) { + if result, exists := a.depsetHashToArtifactPathsCache[depsetHash]; exists { return result, nil } - if depset, exists := a.depsetIdToDepset[depsetId]; exists { - result := depset.DirectArtifactIds - for _, childId := range depset.TransitiveDepSetIds { - childArtifactIds, err := a.artifactIdsFromDepsetId(childId) + if depset, exists := a.depsetHashToAqueryDepset[depsetHash]; exists { + result := depset.DirectArtifacts + for _, childHash := range depset.TransitiveDepSetHashes { + childArtifactIds, err := a.artifactPathsFromDepsetHash(childHash) if err != nil { return nil, err } result = append(result, childArtifactIds...) } - a.depsetIdToArtifactIdsCache[depsetId] = result + a.depsetHashToArtifactPathsCache[depsetHash] = result return result, nil } else { - return nil, fmt.Errorf("undefined input depsetId %d", depsetId) + return nil, fmt.Errorf("undefined input depset hash %s", depsetHash) } } -// AqueryBuildStatements returns an array of BuildStatements which should be registered (and output -// to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel -// aquery invocation). -func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) { - buildStatements := []BuildStatement{} - +// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset +// which should be registered (and output to a ninja file) to correspond with Bazel's +// action graph, as described by the given action graph json proto. +// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets +// are one-to-one with Bazel's depSetOfFiles objects. +func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) { var aqueryResult actionGraphContainer err := json.Unmarshal(aqueryJsonProto, &aqueryResult) if err != nil { - return nil, err + return nil, nil, err } aqueryHandler, err := newAqueryHandler(aqueryResult) if err != nil { - return nil, err + return nil, nil, err + } + + var buildStatements []BuildStatement + if aqueryHandler.bazelToolsDependencySentinelNeeded { + buildStatements = append(buildStatements, BuildStatement{ + Command: fmt.Sprintf("touch '%s'", bazelToolsDependencySentinel), + OutputPaths: []string{bazelToolsDependencySentinel}, + Mnemonic: bazelToolsDependencySentinel, + }) } for _, actionEntry := range aqueryResult.Actions { if shouldSkipAction(actionEntry) { continue } - outputPaths := []string{} - var depfile *string - for _, outputId := range actionEntry.OutputIds { - outputPath, exists := aqueryHandler.artifactIdToPath[outputId] - if !exists { - return nil, fmt.Errorf("undefined outputId %d", outputId) - } - ext := filepath.Ext(outputPath) - if ext == ".d" { - if depfile != nil { - return nil, fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) - } else { - depfile = &outputPath - } - } else { - outputPaths = append(outputPaths, outputPath) - } - } - inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds) - if err != nil { - return nil, err - } - - buildStatement := BuildStatement{ - Command: strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " "), - Depfile: depfile, - OutputPaths: outputPaths, - InputPaths: inputPaths, - Env: actionEntry.EnvironmentVariables, - Mnemonic: actionEntry.Mnemonic, - } + var buildStatement BuildStatement if isSymlinkAction(actionEntry) { - if len(inputPaths) != 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) - } - out := outputPaths[0] - outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) - out = proptools.ShellEscapeIncludingSpaces(out) - in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) - // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). - buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) - buildStatement.SymlinkPaths = outputPaths[:] + buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry) } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 { - if len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) - } - expandedTemplateContent := expandTemplateContent(actionEntry) - // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, - // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might - // change \n to space and mess up the format of Python programs. - // sed is used to convert \\n back to \n before saving to output file. - // See go/python-binary-host-mixed-build for more details. - command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, - escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) - buildStatement.Command = command + buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry) } else if isPythonZipperAction(actionEntry) { - if len(inputPaths) < 1 || len(outputPaths) != 1 { - return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) - } - buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement) - buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0]) - // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. - // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, - // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. - // - // The following logic relies on that Bazel aquery output returns actions in the order that - // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions - // in that order, the following logic might not find the build statement generated for Python binary - // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. - // See go/python-binary-host-mixed-build for more details. - pythonZipFilePath := outputPaths[0] - pyBinaryFound := false - for i, _ := range buildStatements { - if len(buildStatements[i].OutputPaths) == 1 && buildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { - buildStatements[i].InputPaths = append(buildStatements[i].InputPaths, pythonZipFilePath) - pyBinaryFound = true - } - } - if !pyBinaryFound { - return nil, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) - } + buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements) } else if len(actionEntry.Arguments) < 1 { - return nil, fmt.Errorf("received action with no command: [%v]", buildStatement) + return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic) + } else { + buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry) + } + + if err != nil { + return nil, nil, err } buildStatements = append(buildStatements, buildStatement) } - return buildStatements, nil + depsetsByHash := map[string]AqueryDepset{} + var depsets []AqueryDepset + for _, aqueryDepset := range aqueryHandler.depsetIdToAqueryDepset { + if prevEntry, hasKey := depsetsByHash[aqueryDepset.ContentHash]; hasKey { + // Two depsets collide on hash. Ensure that their contents are identical. + if !reflect.DeepEqual(aqueryDepset, prevEntry) { + return nil, nil, fmt.Errorf("two different depsets have the same hash: %v, %v", prevEntry, aqueryDepset) + } + } else { + depsetsByHash[aqueryDepset.ContentHash] = aqueryDepset + depsets = append(depsets, aqueryDepset) + } + } + + // Build Statements and depsets must be sorted by their content hash to + // preserve determinism between builds (this will result in consistent ninja file + // output). Note they are not sorted by their original IDs nor their Bazel ordering, + // as Bazel gives nondeterministic ordering / identifiers in aquery responses. + sort.Slice(buildStatements, func(i, j int) bool { + // For build statements, compare output lists. In Bazel, each output file + // may only have one action which generates it, so this will provide + // a deterministic ordering. + outputs_i := buildStatements[i].OutputPaths + outputs_j := buildStatements[j].OutputPaths + if len(outputs_i) != len(outputs_j) { + return len(outputs_i) < len(outputs_j) + } + if len(outputs_i) == 0 { + // No outputs for these actions, so compare commands. + return buildStatements[i].Command < buildStatements[j].Command + } + // There may be multiple outputs, but the output ordering is deterministic. + return outputs_i[0] < outputs_j[0] + }) + sort.Slice(depsets, func(i, j int) bool { + return depsets[i].ContentHash < depsets[j].ContentHash + }) + return buildStatements, depsets, nil +} + +// depsetContentHash computes and returns a SHA256 checksum of the contents of +// the given depset. This content hash may serve as the depset's identifier. +// Using a content hash for an identifier is superior for determinism. (For example, +// using an integer identifier which depends on the order in which the depsets are +// created would result in nondeterministic depset IDs.) +func depsetContentHash(directPaths []string, transitiveDepsetHashes []string) string { + h := sha256.New() + // Use newline as delimiter, as paths cannot contain newline. + h.Write([]byte(strings.Join(directPaths, "\n"))) + h.Write([]byte(strings.Join(transitiveDepsetHashes, ""))) + fullHash := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + return fullHash +} + +func (a *aqueryArtifactHandler) depsetContentHashes(inputDepsetIds []depsetId) ([]string, error) { + var hashes []string + for _, depsetId := range inputDepsetIds { + if aqueryDepset, exists := a.depsetIdToAqueryDepset[depsetId]; !exists { + return nil, fmt.Errorf("undefined input depsetId %d", depsetId) + } else { + hashes = append(hashes, aqueryDepset.ContentHash) + } + } + return hashes, nil +} + +func (a *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) (BuildStatement, error) { + command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") + inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + outputPaths, depfile, err := a.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputDepsetHashes: inputDepsetHashes, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (a *aqueryArtifactHandler) pythonZipperActionBuildStatement(actionEntry action, prevBuildStatements []BuildStatement) (BuildStatement, error) { + inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + outputPaths, depfile, err := a.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + if len(inputPaths) < 1 || len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths) + } + command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ") + inputPaths, command = removePy3wrapperScript(inputPaths, command) + command = addCommandForPyBinaryRunfilesDir(command, outputPaths[0]) + // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements. + // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input, + // which is not sufficient without the python zip file from which runfiles directory is created for py_binary. + // + // The following logic relies on that Bazel aquery output returns actions in the order that + // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions + // in that order, the following logic might not find the build statement generated for Python binary + // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output. + // See go/python-binary-host-mixed-build for more details. + pythonZipFilePath := outputPaths[0] + pyBinaryFound := false + for i := range prevBuildStatements { + if len(prevBuildStatements[i].OutputPaths) == 1 && prevBuildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath { + prevBuildStatements[i].InputPaths = append(prevBuildStatements[i].InputPaths, pythonZipFilePath) + pyBinaryFound = true + } + } + if !pyBinaryFound { + return BuildStatement{}, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths) + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputPaths: inputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (a *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, depfile, err := a.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + if len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths) + } + expandedTemplateContent := expandTemplateContent(actionEntry) + // The expandedTemplateContent is escaped for being used in double quotes and shell unescape, + // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might + // change \n to space and mess up the format of Python programs. + // sed is used to convert \\n back to \n before saving to output file. + // See go/python-binary-host-mixed-build for more details. + command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`, + escapeCommandlineArgument(expandedTemplateContent), outputPaths[0]) + inputDepsetHashes, err := a.depsetContentHashes(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputDepsetHashes: inputDepsetHashes, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + } + return buildStatement, nil +} + +func (a *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) { + outputPaths, depfile, err := a.getOutputPaths(actionEntry) + if err != nil { + return BuildStatement{}, err + } + + inputPaths, err := a.getInputPaths(actionEntry.InputDepSetIds) + if err != nil { + return BuildStatement{}, err + } + if len(inputPaths) != 1 || len(outputPaths) != 1 { + return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths) + } + out := outputPaths[0] + outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out)) + out = proptools.ShellEscapeIncludingSpaces(out) + in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0])) + // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`). + command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in) + symlinkPaths := outputPaths[:] + + buildStatement := BuildStatement{ + Command: command, + Depfile: depfile, + OutputPaths: outputPaths, + InputPaths: inputPaths, + Env: actionEntry.EnvironmentVariables, + Mnemonic: actionEntry.Mnemonic, + SymlinkPaths: symlinkPaths, + } + return buildStatement, nil +} + +func (a *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths []string, depfile *string, err error) { + for _, outputId := range actionEntry.OutputIds { + outputPath, exists := a.artifactIdToPath[outputId] + if !exists { + err = fmt.Errorf("undefined outputId %d", outputId) + return + } + ext := filepath.Ext(outputPath) + if ext == ".d" { + if depfile != nil { + err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath) + return + } else { + depfile = &outputPath + } + } else { + outputPaths = append(outputPaths, outputPath) + } + } + return } // expandTemplateContent substitutes the tokens in a template. @@ -343,7 +592,7 @@ func expandTemplateContent(actionEntry action) string { replacerString := []string{} for _, pair := range actionEntry.Substitutions { value := pair.Value - if val, ok := TemplateActionOverriddenTokens[pair.Key]; ok { + if val, ok := templateActionOverriddenTokens[pair.Key]; ok { value = val } replacerString = append(replacerString, pair.Key, value) @@ -372,10 +621,10 @@ func escapeCommandlineArgument(str string) string { // removed from input paths and command of creating python zip file. // See go/python-binary-host-mixed-build for more details. // TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode. -func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) { +func removePy3wrapperScript(inputPaths []string, command string) (newInputPaths []string, newCommand string) { // Remove from inputs - filteredInputPaths := []string{} - for _, path := range bs.InputPaths { + var filteredInputPaths []string + for _, path := range inputPaths { if !strings.HasSuffix(path, py3wrapperFileName) { filteredInputPaths = append(filteredInputPaths, path) } @@ -384,7 +633,7 @@ func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newComma // Remove from command line var re = regexp.MustCompile(`\S*` + py3wrapperFileName) - newCommand = re.ReplaceAllString(bs.Command, "") + newCommand = re.ReplaceAllString(command, "") return } @@ -395,18 +644,18 @@ func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newComma // so MANIFEST file could not be created, which also blocks the creation of runfiles directory. // See go/python-binary-host-mixed-build for more details. // TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action. -func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string { +func addCommandForPyBinaryRunfilesDir(oldCommand string, zipFilePath string) string { // Unzip the zip file, zipFilePath looks like <python_binary>.zip runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles" - command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName) + command := fmt.Sprintf("%s x %s -d %s", "../bazel_tools/tools/zip/zipper/zipper", zipFilePath, runfilesDirName) // Create a symbolic link in <python_binary>.runfiles/, which is the expected structure // when running the python binary stub script. command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName) - return bs.Command + " && " + command + return oldCommand + " && " + command } func isSymlinkAction(a action) bool { - return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" + return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink" || a.Mnemonic == "ExecutableSymlink" } func isTemplateExpandAction(a action) bool { @@ -437,11 +686,14 @@ func shouldSkipAction(a action) bool { if a.Mnemonic == "FileWrite" { return true } + if a.Mnemonic == "BaselineCoverage" { + return true + } return false } -func expandPathFragment(id int, pathFragmentsMap map[int]pathFragment) (string, error) { - labels := []string{} +func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]pathFragment) (string, error) { + var labels []string currId := id // Only positive IDs are valid for path fragments. An ID of zero indicates a terminal node. for currId > 0 { diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go index 68e50c21a..c759d56ee 100644 --- a/bazel/aquery_test.go +++ b/bazel/aquery_test.go @@ -17,6 +17,7 @@ package bazel import ( "fmt" "reflect" + "sort" "testing" ) @@ -24,28 +25,14 @@ func TestAqueryMultiArchGenrule(t *testing.T) { // This input string is retrieved from a real build of bionic-related genrules. const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 6 - }, { - "id": 3, - "pathFragmentId": 8 - }, { - "id": 4, - "pathFragmentId": 12 - }, { - "id": 5, - "pathFragmentId": 19 - }, { - "id": 6, - "pathFragmentId": 20 - }, { - "id": 7, - "pathFragmentId": 21 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 6 }, + { "id": 3, "pathFragmentId": 8 }, + { "id": 4, "pathFragmentId": 12 }, + { "id": 5, "pathFragmentId": 19 }, + { "id": 6, "pathFragmentId": 20 }, + { "id": 7, "pathFragmentId": 21 }], "actions": [{ "targetId": 1, "actionKey": "ab53f6ecbdc2ee8cb8812613b63205464f1f5083f6dca87081a0a398c0f1ecf7", @@ -99,132 +86,48 @@ func TestAqueryMultiArchGenrule(t *testing.T) { "outputIds": [7], "primaryOutputId": 7 }], - "targets": [{ - "id": 1, - "label": "@sourceroot//bionic/libc:syscalls-arm", - "ruleClassId": 1 - }, { - "id": 2, - "label": "@sourceroot//bionic/libc:syscalls-x86", - "ruleClassId": 1 - }, { - "id": 3, - "label": "@sourceroot//bionic/libc:syscalls-x86_64", - "ruleClassId": 1 - }, { - "id": 4, - "label": "@sourceroot//bionic/libc:syscalls-arm64", - "ruleClassId": 1 - }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2, 3] - }, { - "id": 2, - "directArtifactIds": [1, 2, 3] - }, { - "id": 3, - "directArtifactIds": [1, 2, 3] - }, { - "id": 4, - "directArtifactIds": [1, 2, 3] - }], + "targets": [ + { "id": 1, "label": "@sourceroot//bionic/libc:syscalls-arm", "ruleClassId": 1 }, + { "id": 2, "label": "@sourceroot//bionic/libc:syscalls-x86", "ruleClassId": 1 }, + { "id": 3, "label": "@sourceroot//bionic/libc:syscalls-x86_64", "ruleClassId": 1 }, + { "id": 4, "label": "@sourceroot//bionic/libc:syscalls-arm64", "ruleClassId": 1 }], + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2, 3] }, + { "id": 2, "directArtifactIds": [1, 2, 3] }, + { "id": 3, "directArtifactIds": [1, 2, 3] }, + { "id": 4, "directArtifactIds": [1, 2, 3] }], "configuration": [{ "id": 1, "mnemonic": "k8-fastbuild", "platformName": "k8", "checksum": "485c362832c178e367d972177f68e69e0981e51e67ef1c160944473db53fe046" }], - "ruleClasses": [{ - "id": 1, - "name": "genrule" - }], - "pathFragments": [{ - "id": 5, - "label": ".." - }, { - "id": 4, - "label": "sourceroot", - "parentId": 5 - }, { - "id": 3, - "label": "bionic", - "parentId": 4 - }, { - "id": 2, - "label": "libc", - "parentId": 3 - }, { - "id": 1, - "label": "SYSCALLS.TXT", - "parentId": 2 - }, { - "id": 7, - "label": "tools", - "parentId": 2 - }, { - "id": 6, - "label": "gensyscalls.py", - "parentId": 7 - }, { - "id": 11, - "label": "bazel_tools", - "parentId": 5 - }, { - "id": 10, - "label": "tools", - "parentId": 11 - }, { - "id": 9, - "label": "genrule", - "parentId": 10 - }, { - "id": 8, - "label": "genrule-setup.sh", - "parentId": 9 - }, { - "id": 18, - "label": "bazel-out" - }, { - "id": 17, - "label": "sourceroot", - "parentId": 18 - }, { - "id": 16, - "label": "k8-fastbuild", - "parentId": 17 - }, { - "id": 15, - "label": "bin", - "parentId": 16 - }, { - "id": 14, - "label": "bionic", - "parentId": 15 - }, { - "id": 13, - "label": "libc", - "parentId": 14 - }, { - "id": 12, - "label": "syscalls-arm.S", - "parentId": 13 - }, { - "id": 19, - "label": "syscalls-x86.S", - "parentId": 13 - }, { - "id": 20, - "label": "syscalls-x86_64.S", - "parentId": 13 - }, { - "id": 21, - "label": "syscalls-arm64.S", - "parentId": 13 - }] + "ruleClasses": [{ "id": 1, "name": "genrule"}], + "pathFragments": [ + { "id": 5, "label": ".." }, + { "id": 4, "label": "sourceroot", "parentId": 5 }, + { "id": 3, "label": "bionic", "parentId": 4 }, + { "id": 2, "label": "libc", "parentId": 3 }, + { "id": 1, "label": "SYSCALLS.TXT", "parentId": 2 }, + { "id": 7, "label": "tools", "parentId": 2 }, + { "id": 6, "label": "gensyscalls.py", "parentId": 7 }, + { "id": 11, "label": "bazel_tools", "parentId": 5 }, + { "id": 10, "label": "tools", "parentId": 11 }, + { "id": 9, "label": "genrule", "parentId": 10 }, + { "id": 8, "label": "genrule-setup.sh", "parentId": 9 }, + { "id": 18, "label": "bazel-out" }, + { "id": 17, "label": "sourceroot", "parentId": 18 }, + { "id": 16, "label": "k8-fastbuild", "parentId": 17 }, + { "id": 15, "label": "bin", "parentId": 16 }, + { "id": 14, "label": "bionic", "parentId": 15 }, + { "id": 13, "label": "libc", "parentId": 14 }, + { "id": 12, "label": "syscalls-arm.S", "parentId": 13 }, + { "id": 19, "label": "syscalls-x86.S", "parentId": 13 }, + { "id": 20, "label": "syscalls-x86_64.S", "parentId": 13 }, + { "id": 21, "label": "syscalls-arm64.S", "parentId": 13 }] }` - actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) - expectedBuildStatements := []BuildStatement{} + actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) + var expectedBuildStatements []BuildStatement for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} { expectedBuildStatements = append(expectedBuildStatements, BuildStatement{ @@ -234,30 +137,33 @@ func TestAqueryMultiArchGenrule(t *testing.T) { OutputPaths: []string{ fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch), }, - InputPaths: []string{ - "../sourceroot/bionic/libc/SYSCALLS.TXT", - "../sourceroot/bionic/libc/tools/gensyscalls.py", - "../bazel_tools/tools/genrule/genrule-setup.sh", - }, Env: []KeyValuePair{ - KeyValuePair{Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, + {Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"}, }, Mnemonic: "Genrule", }) } assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) + + expectedFlattenedInputs := []string{ + "../sourceroot/bionic/libc/SYSCALLS.TXT", + "../sourceroot/bionic/libc/tools/gensyscalls.py", + } + // In this example, each depset should have the same expected inputs. + for _, actualDepset := range actualDepsets { + actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) + } + } } func TestInvalidOutputId(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -267,33 +173,23 @@ func TestInvalidOutputId(t *testing.T) { "outputIds": [3], "primaryOutputId": 3 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined outputId 3") } -func TestInvalidInputDepsetId(t *testing.T) { +func TestInvalidInputDepsetIdFromAction(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -303,33 +199,23 @@ func TestInvalidInputDepsetId(t *testing.T) { "outputIds": [1], "primaryOutputId": 1 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined input depsetId 2") } -func TestInvalidInputArtifactId(t *testing.T) { +func TestInvalidInputDepsetIdFromDepset(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -339,33 +225,49 @@ func TestInvalidInputArtifactId(t *testing.T) { "outputIds": [1], "primaryOutputId": 1 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 3] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2], "transitiveDepSetIds": [42] }], + "pathFragments": [ + { "id": 1, "label": "one"}, + { "id": 2, "label": "two" }] +}` + + _, _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)") +} + +func TestInvalidInputArtifactId(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], + "actions": [{ + "targetId": 1, + "actionKey": "x", + "mnemonic": "x", + "arguments": ["touch", "foo"], + "inputDepSetIds": [1], + "outputIds": [1], + "primaryOutputId": 1 }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 3] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined input artifactId 3") } func TestInvalidPathFragmentId(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -375,37 +277,24 @@ func TestInvalidPathFragmentId(t *testing.T) { "outputIds": [1], "primaryOutputId": 1 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two", - "parentId": 3 - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two", "parentId": 3 }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, "undefined path fragment id 3") } func TestDepfiles(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }, { - "id": 3, - "pathFragmentId": 3 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -415,23 +304,15 @@ func TestDepfiles(t *testing.T) { "outputIds": [2, 3], "primaryOutputId": 2 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2, 3] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }, { - "id": 3, - "label": "two.d" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2, 3] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }, + { "id": 3, "label": "two.d" }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } @@ -451,19 +332,11 @@ func TestDepfiles(t *testing.T) { func TestMultipleDepfiles(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }, { - "id": 3, - "pathFragmentId": 3 - }, { - "id": 4, - "pathFragmentId": 4 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }, + { "id": 4, "pathFragmentId": 4 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -477,22 +350,14 @@ func TestMultipleDepfiles(t *testing.T) { "id": 1, "directArtifactIds": [1, 2, 3, 4] }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "two" - }, { - "id": 3, - "label": "two.d" - }, { - "id": 4, - "label": "other.d" - }] + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "two" }, + { "id": 3, "label": "two.d" }, + { "id": 4, "label": "other.d" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`) } @@ -501,70 +366,28 @@ func TestTransitiveInputDepsets(t *testing.T) { // a single action with many inputs given via a deep depset. const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 7 - }, { - "id": 3, - "pathFragmentId": 8 - }, { - "id": 4, - "pathFragmentId": 9 - }, { - "id": 5, - "pathFragmentId": 10 - }, { - "id": 6, - "pathFragmentId": 11 - }, { - "id": 7, - "pathFragmentId": 12 - }, { - "id": 8, - "pathFragmentId": 13 - }, { - "id": 9, - "pathFragmentId": 14 - }, { - "id": 10, - "pathFragmentId": 15 - }, { - "id": 11, - "pathFragmentId": 16 - }, { - "id": 12, - "pathFragmentId": 17 - }, { - "id": 13, - "pathFragmentId": 18 - }, { - "id": 14, - "pathFragmentId": 19 - }, { - "id": 15, - "pathFragmentId": 20 - }, { - "id": 16, - "pathFragmentId": 21 - }, { - "id": 17, - "pathFragmentId": 22 - }, { - "id": 18, - "pathFragmentId": 23 - }, { - "id": 19, - "pathFragmentId": 24 - }, { - "id": 20, - "pathFragmentId": 25 - }, { - "id": 21, - "pathFragmentId": 26 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 7 }, + { "id": 3, "pathFragmentId": 8 }, + { "id": 4, "pathFragmentId": 9 }, + { "id": 5, "pathFragmentId": 10 }, + { "id": 6, "pathFragmentId": 11 }, + { "id": 7, "pathFragmentId": 12 }, + { "id": 8, "pathFragmentId": 13 }, + { "id": 9, "pathFragmentId": 14 }, + { "id": 10, "pathFragmentId": 15 }, + { "id": 11, "pathFragmentId": 16 }, + { "id": 12, "pathFragmentId": 17 }, + { "id": 13, "pathFragmentId": 18 }, + { "id": 14, "pathFragmentId": 19 }, + { "id": 15, "pathFragmentId": 20 }, + { "id": 16, "pathFragmentId": 21 }, + { "id": 17, "pathFragmentId": 22 }, + { "id": 18, "pathFragmentId": 23 }, + { "id": 19, "pathFragmentId": 24 }, + { "id": 20, "pathFragmentId": 25 }, + { "id": 21, "pathFragmentId": 26 }], "actions": [{ "targetId": 1, "actionKey": "3b826d17fadbbbcd8313e456b90ec47c078c438088891dd45b4adbcd8889dc50", @@ -575,197 +398,174 @@ func TestTransitiveInputDepsets(t *testing.T) { "outputIds": [21], "primaryOutputId": 21 }], - "depSetOfFiles": [{ - "id": 3, - "directArtifactIds": [1, 2, 3, 4, 5] - }, { - "id": 4, - "directArtifactIds": [6, 7, 8, 9, 10] - }, { - "id": 2, - "transitiveDepSetIds": [3, 4], - "directArtifactIds": [11, 12, 13, 14, 15] - }, { - "id": 5, - "directArtifactIds": [16, 17, 18, 19] - }, { - "id": 1, - "transitiveDepSetIds": [2, 5], - "directArtifactIds": [20] - }], - "pathFragments": [{ - "id": 6, - "label": "bazel-out" - }, { - "id": 5, - "label": "sourceroot", - "parentId": 6 - }, { - "id": 4, - "label": "k8-fastbuild", - "parentId": 5 - }, { - "id": 3, - "label": "bin", - "parentId": 4 - }, { - "id": 2, - "label": "testpkg", - "parentId": 3 - }, { - "id": 1, - "label": "test_1", - "parentId": 2 - }, { - "id": 7, - "label": "test_2", - "parentId": 2 - }, { - "id": 8, - "label": "test_3", - "parentId": 2 - }, { - "id": 9, - "label": "test_4", - "parentId": 2 - }, { - "id": 10, - "label": "test_5", - "parentId": 2 - }, { - "id": 11, - "label": "test_6", - "parentId": 2 - }, { - "id": 12, - "label": "test_7", - "parentId": 2 - }, { - "id": 13, - "label": "test_8", - "parentId": 2 - }, { - "id": 14, - "label": "test_9", - "parentId": 2 - }, { - "id": 15, - "label": "test_10", - "parentId": 2 - }, { - "id": 16, - "label": "test_11", - "parentId": 2 - }, { - "id": 17, - "label": "test_12", - "parentId": 2 - }, { - "id": 18, - "label": "test_13", - "parentId": 2 - }, { - "id": 19, - "label": "test_14", - "parentId": 2 - }, { - "id": 20, - "label": "test_15", - "parentId": 2 - }, { - "id": 21, - "label": "test_16", - "parentId": 2 - }, { - "id": 22, - "label": "test_17", - "parentId": 2 - }, { - "id": 23, - "label": "test_18", - "parentId": 2 - }, { - "id": 24, - "label": "test_19", - "parentId": 2 - }, { - "id": 25, - "label": "test_root", - "parentId": 2 - }, { - "id": 26, - "label": "test_out", - "parentId": 2 - }] + "depSetOfFiles": [ + { "id": 3, "directArtifactIds": [1, 2, 3, 4, 5] }, + { "id": 4, "directArtifactIds": [6, 7, 8, 9, 10] }, + { "id": 2, "transitiveDepSetIds": [3, 4], "directArtifactIds": [11, 12, 13, 14, 15] }, + { "id": 5, "directArtifactIds": [16, 17, 18, 19] }, + { "id": 1, "transitiveDepSetIds": [2, 5], "directArtifactIds": [20] }], + "pathFragments": [ + { "id": 6, "label": "bazel-out" }, + { "id": 5, "label": "sourceroot", "parentId": 6 }, + { "id": 4, "label": "k8-fastbuild", "parentId": 5 }, + { "id": 3, "label": "bin", "parentId": 4 }, + { "id": 2, "label": "testpkg", "parentId": 3 }, + { "id": 1, "label": "test_1", "parentId": 2 }, + { "id": 7, "label": "test_2", "parentId": 2 }, + { "id": 8, "label": "test_3", "parentId": 2 }, + { "id": 9, "label": "test_4", "parentId": 2 }, + { "id": 10, "label": "test_5", "parentId": 2 }, + { "id": 11, "label": "test_6", "parentId": 2 }, + { "id": 12, "label": "test_7", "parentId": 2 }, + { "id": 13, "label": "test_8", "parentId": 2 }, + { "id": 14, "label": "test_9", "parentId": 2 }, + { "id": 15, "label": "test_10", "parentId": 2 }, + { "id": 16, "label": "test_11", "parentId": 2 }, + { "id": 17, "label": "test_12", "parentId": 2 }, + { "id": 18, "label": "test_13", "parentId": 2 }, + { "id": 19, "label": "test_14", "parentId": 2 }, + { "id": 20, "label": "test_15", "parentId": 2 }, + { "id": 21, "label": "test_16", "parentId": 2 }, + { "id": 22, "label": "test_17", "parentId": 2 }, + { "id": 23, "label": "test_18", "parentId": 2 }, + { "id": 24, "label": "test_19", "parentId": 2 }, + { "id": 25, "label": "test_root", "parentId": 2 }, + { "id": 26,"label": "test_out", "parentId": 2 }] }` - actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString)) - // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs - // are given via a deep depset, but the depset is flattened when returned as a - // BuildStatement slice. - inputPaths := []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root"} - for i := 1; i < 20; i++ { - inputPaths = append(inputPaths, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) - } + actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) + expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'", OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"}, - InputPaths: inputPaths, Mnemonic: "Action", }, } assertBuildStatements(t, expectedBuildStatements, actualbuildStatements) + + // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs + // are given via a deep depset, but the depset is flattened when returned as a + // BuildStatement slice. + var expectedFlattenedInputs []string + for i := 1; i < 20; i++ { + expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i)) + } + expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root") + + actualDepsetHashes := actualbuildStatements[0].InputDepsetHashes + actualFlattenedInputs := flattenDepsets(actualDepsetHashes, actualDepsets) + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) + } } -func TestMiddlemenAction(t *testing.T) { - const inputString = ` -{ +func TestBazelOutRemovalFromInputDepsets(t *testing.T) { + const inputString = `{ "artifacts": [{ "id": 1, - "pathFragmentId": 1 + "pathFragmentId": 10 }, { "id": 2, - "pathFragmentId": 2 + "pathFragmentId": 20 }, { "id": 3, - "pathFragmentId": 3 + "pathFragmentId": 30 }, { "id": 4, - "pathFragmentId": 4 - }, { - "id": 5, - "pathFragmentId": 5 - }, { - "id": 6, - "pathFragmentId": 6 + "pathFragmentId": 40 + }], + "depSetOfFiles": [{ + "id": 1111, + "directArtifactIds": [3 , 4] + }], + "actions": [{ + "targetId": 100, + "actionKey": "x", + "inputDepSetIds": [1111], + "mnemonic": "x", + "arguments": ["bogus", "command"], + "outputIds": [2], + "primaryOutputId": 1 }], "pathFragments": [{ - "id": 1, - "label": "middleinput_one" - }, { - "id": 2, - "label": "middleinput_two" + "id": 10, + "label": "input" }, { - "id": 3, - "label": "middleman_artifact" + "id": 20, + "label": "output" }, { - "id": 4, - "label": "maininput_one" + "id": 30, + "label": "dep1", + "parentId": 50 }, { - "id": 5, - "label": "maininput_two" + "id": 40, + "label": "dep2", + "parentId": 60 }, { - "id": 6, - "label": "output" - }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1, 2] + "id": 50, + "label": "bazel_tools", + "parentId": 60 }, { - "id": 2, - "directArtifactIds": [3, 4, 5] - }], + "id": 60, + "label": ".." + }] +}` + actualBuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString)) + if len(actualDepsets) != 1 { + t.Errorf("expected 1 depset but found %#v", actualDepsets) + return + } + dep2Found := false + for _, dep := range flattenDepsets([]string{actualDepsets[0].ContentHash}, actualDepsets) { + if dep == "../bazel_tools/dep1" { + t.Errorf("dependency %s expected to be removed but still exists", dep) + } else if dep == "../dep2" { + dep2Found = true + } + } + if !dep2Found { + t.Errorf("dependency ../dep2 expected but not found") + } + + expectedBuildStatement := BuildStatement{ + Command: "bogus command", + OutputPaths: []string{"output"}, + Mnemonic: "x", + } + buildStatementFound := false + for _, actualBuildStatement := range actualBuildStatements { + if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" { + buildStatementFound = true + break + } + } + if !buildStatementFound { + t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements) + return + } +} + +func TestMiddlemenAction(t *testing.T) { + const inputString = ` +{ + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }, + { "id": 4, "pathFragmentId": 4 }, + { "id": 5, "pathFragmentId": 5 }, + { "id": 6, "pathFragmentId": 6 }], + "pathFragments": [ + { "id": 1, "label": "middleinput_one" }, + { "id": 2, "label": "middleinput_two" }, + { "id": 3, "label": "middleman_artifact" }, + { "id": 4, "label": "maininput_one" }, + { "id": 5, "label": "maininput_two" }, + { "id": 6, "label": "output" }], + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1, 2] }, + { "id": 2, "directArtifactIds": [3, 4, 5] }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -785,36 +585,81 @@ func TestMiddlemenAction(t *testing.T) { }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actualBuildStatements, actualDepsets, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } - if expected := 1; len(actual) != expected { - t.Fatalf("Expected %d build statements, got %d", expected, len(actual)) + if expected := 1; len(actualBuildStatements) != expected { + t.Fatalf("Expected %d build statements, got %d", expected, len(actualBuildStatements)) } - bs := actual[0] - expectedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} - if !reflect.DeepEqual(bs.InputPaths, expectedInputs) { - t.Errorf("Expected main action inputs %q, but got %q", expectedInputs, bs.InputPaths) + expectedDepsetFiles := [][]string{ + {"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}, + {"middleinput_one", "middleinput_two"}, + } + assertFlattenedDepsets(t, actualDepsets, expectedDepsetFiles) + + bs := actualBuildStatements[0] + if len(bs.InputPaths) > 0 { + t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths) } expectedOutputs := []string{"output"} if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) { t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths) } + + expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"} + actualFlattenedInputs := flattenDepsets(bs.InputDepsetHashes, actualDepsets) + + if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) { + t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs) + } +} + +// Returns the contents of given depsets in concatenated post order. +func flattenDepsets(depsetHashesToFlatten []string, allDepsets []AqueryDepset) []string { + depsetsByHash := map[string]AqueryDepset{} + for _, depset := range allDepsets { + depsetsByHash[depset.ContentHash] = depset + } + var result []string + for _, depsetId := range depsetHashesToFlatten { + result = append(result, flattenDepset(depsetId, depsetsByHash)...) + } + return result +} + +// Returns the contents of a given depset in post order. +func flattenDepset(depsetHashToFlatten string, allDepsets map[string]AqueryDepset) []string { + depset := allDepsets[depsetHashToFlatten] + var result []string + for _, depsetId := range depset.TransitiveDepSetHashes { + result = append(result, flattenDepset(depsetId, allDepsets)...) + } + result = append(result, depset.DirectArtifacts...) + return result +} + +func assertFlattenedDepsets(t *testing.T, actualDepsets []AqueryDepset, expectedDepsetFiles [][]string) { + t.Helper() + if len(actualDepsets) != len(expectedDepsetFiles) { + t.Errorf("Expected %d depsets, but got %d depsets", len(expectedDepsetFiles), len(actualDepsets)) + } + for i, actualDepset := range actualDepsets { + actualFlattenedInputs := flattenDepsets([]string{actualDepset.ContentHash}, actualDepsets) + if !reflect.DeepEqual(actualFlattenedInputs, expectedDepsetFiles[i]) { + t.Errorf("Expected depset files: %v, but got %v", expectedDepsetFiles[i], actualFlattenedInputs) + } + } } func TestSimpleSymlink(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 3 - }, { - "id": 2, - "pathFragmentId": 5 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 3 }, + { "id": 2, "pathFragmentId": 5 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -823,40 +668,24 @@ func TestSimpleSymlink(t *testing.T) { "outputIds": [2], "primaryOutputId": 2 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "file_subdir", - "parentId": 1 - }, { - "id": 3, - "label": "file", - "parentId": 2 - }, { - "id": 4, - "label": "symlink_subdir", - "parentId": 1 - }, { - "id": 5, - "label": "symlink", - "parentId": 4 - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "file_subdir", "parentId": 1 }, + { "id": 3, "label": "file", "parentId": 2 }, + { "id": 4, "label": "symlink_subdir", "parentId": 1 }, + { "id": 5, "label": "symlink", "parentId": 4 }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "mkdir -p one/symlink_subdir && " + "rm -f one/symlink_subdir/symlink && " + "ln -sf $PWD/one/file_subdir/file one/symlink_subdir/symlink", @@ -872,13 +701,9 @@ func TestSimpleSymlink(t *testing.T) { func TestSymlinkQuotesPaths(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 3 - }, { - "id": 2, - "pathFragmentId": 5 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 3 }, + { "id": 2, "pathFragmentId": 5 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -887,40 +712,24 @@ func TestSymlinkQuotesPaths(t *testing.T) { "outputIds": [2], "primaryOutputId": 2 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1] - }], - "pathFragments": [{ - "id": 1, - "label": "one" - }, { - "id": 2, - "label": "file subdir", - "parentId": 1 - }, { - "id": 3, - "label": "file", - "parentId": 2 - }, { - "id": 4, - "label": "symlink subdir", - "parentId": 1 - }, { - "id": 5, - "label": "symlink", - "parentId": 4 - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1] }], + "pathFragments": [ + { "id": 1, "label": "one" }, + { "id": 2, "label": "file subdir", "parentId": 1 }, + { "id": 3, "label": "file", "parentId": 2 }, + { "id": 4, "label": "symlink subdir", "parentId": 1 }, + { "id": 5, "label": "symlink", "parentId": 4 }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "mkdir -p 'one/symlink subdir' && " + "rm -f 'one/symlink subdir/symlink' && " + "ln -sf $PWD/'one/file subdir/file' 'one/symlink subdir/symlink'", @@ -936,16 +745,10 @@ func TestSymlinkQuotesPaths(t *testing.T) { func TestSymlinkMultipleInputs(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }, { - "id": 3, - "pathFragmentId": 3 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -954,39 +757,24 @@ func TestSymlinkMultipleInputs(t *testing.T) { "outputIds": [3], "primaryOutputId": 3 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1,2] - }], - "pathFragments": [{ - "id": 1, - "label": "file" - }, { - "id": 2, - "label": "other_file" - }, { - "id": 3, - "label": "symlink" - }] + "depSetOfFiles": [{ "id": 1, "directArtifactIds": [1,2] }], + "pathFragments": [ + { "id": 1, "label": "file" }, + { "id": 2, "label": "other_file" }, + { "id": 3, "label": "symlink" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]`) } func TestSymlinkMultipleOutputs(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }, { - "id": 2, - "pathFragmentId": 2 - }, { - "id": 3, - "pathFragmentId": 3 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -995,23 +783,15 @@ func TestSymlinkMultipleOutputs(t *testing.T) { "outputIds": [2,3], "primaryOutputId": 2 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [1] - }], - "pathFragments": [{ - "id": 1, - "label": "file" - }, { - "id": 2, - "label": "symlink" - }, { - "id": 3, - "label": "other_symlink" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [1] }], + "pathFragments": [ + { "id": 1, "label": "file" }, + { "id": 2, "label": "symlink" }, + { "id": 3, "label": "other_symlink" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`) } @@ -1031,28 +811,22 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) { "primaryOutputId": 1, "executionPlatform": "//build/bazel/platforms:linux_x86_64", "templateContent": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [{ - "key": "%token1%", - "value": "abcd" - },{ - "key": "%python_binary%", - "value": "python3" - }] + "substitutions": [ + { "key": "%token1%", "value": "abcd" }, + { "key": "%python_binary%", "value": "python3" }] }], - "pathFragments": [{ - "id": 1, - "label": "template_file" - }] + "pathFragments": [ + { "id": 1, "label": "template_file" }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > template_file && " + "chmod a+x template_file'", OutputPaths: []string{"template_file"}, @@ -1065,10 +839,8 @@ func TestTemplateExpandActionSubstitutions(t *testing.T) { func TestTemplateExpandActionNoOutput(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -1077,46 +849,28 @@ func TestTemplateExpandActionNoOutput(t *testing.T) { "primaryOutputId": 1, "executionPlatform": "//build/bazel/platforms:linux_x86_64", "templateContent": "Test template substitutions: %token1%, %python_binary%", - "substitutions": [{ - "key": "%token1%", - "value": "abcd" - },{ - "key": "%python_binary%", - "value": "python3" - }] + "substitutions": [ + { "key": "%token1%", "value": "abcd" }, + { "key": "%python_binary%", "value": "python3" }] }], - "pathFragments": [{ - "id": 1, - "label": "template_file" - }] + "pathFragments": [ + { "id": 1, "label": "template_file" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1 output to template expand action, got: output []`) } func TestPythonZipperActionSuccess(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - },{ - "id": 2, - "pathFragmentId": 2 - },{ - "id": 3, - "pathFragmentId": 3 - },{ - "id": 4, - "pathFragmentId": 4 - },{ - "id": 5, - "pathFragmentId": 10 - },{ - "id": 10, - "pathFragmentId": 20 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }, + { "id": 4, "pathFragmentId": 4 }, + { "id": 5, "pathFragmentId": 10 }, + { "id": 10, "pathFragmentId": 20 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -1143,93 +897,46 @@ func TestPythonZipperActionSuccess(t *testing.T) { "inputDepSetIds": [1], "primaryOutputId": 2 }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [4, 3, 5] - }], - "pathFragments": [{ - "id": 1, - "label": "python_binary" - },{ - "id": 2, - "label": "python_binary.zip" - },{ - "id": 3, - "label": "python_binary.py" - },{ - "id": 9, - "label": ".." - }, { - "id": 8, - "label": "bazel_tools", - "parentId": 9 - }, { - "id": 7, - "label": "tools", - "parentId": 8 - }, { - "id": 6, - "label": "zip", - "parentId": 7 - }, { - "id": 5, - "label": "zipper", - "parentId": 6 - }, { - "id": 4, - "label": "zipper", - "parentId": 5 - },{ - "id": 16, - "label": "bazel-out" - },{ - "id": 15, - "label": "bazel_tools", - "parentId": 16 - }, { - "id": 14, - "label": "k8-fastbuild", - "parentId": 15 - }, { - "id": 13, - "label": "bin", - "parentId": 14 - }, { - "id": 12, - "label": "tools", - "parentId": 13 - }, { - "id": 11, - "label": "python", - "parentId": 12 - }, { - "id": 10, - "label": "py3wrapper.sh", - "parentId": 11 - },{ - "id": 20, - "label": "python_binary" - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [4, 3, 5] }], + "pathFragments": [ + { "id": 1, "label": "python_binary" }, + { "id": 2, "label": "python_binary.zip" }, + { "id": 3, "label": "python_binary.py" }, + { "id": 9, "label": ".." }, + { "id": 8, "label": "bazel_tools", "parentId": 9 }, + { "id": 7, "label": "tools", "parentId": 8 }, + { "id": 6, "label": "zip", "parentId": 7 }, + { "id": 5, "label": "zipper", "parentId": 6 }, + { "id": 4, "label": "zipper", "parentId": 5 }, + { "id": 16, "label": "bazel-out" }, + { "id": 15, "label": "bazel_tools", "parentId": 16 }, + { "id": 14, "label": "k8-fastbuild", "parentId": 15 }, + { "id": 13, "label": "bin", "parentId": 14 }, + { "id": 12, "label": "tools", "parentId": 13 }, + { "id": 11, "label": "python", "parentId": 12 }, + { "id": 10, "label": "py3wrapper.sh", "parentId": 11 }, + { "id": 20, "label": "python_binary" }] }` - actual, err := AqueryBuildStatements([]byte(inputString)) + actual, _, err := AqueryBuildStatements([]byte(inputString)) if err != nil { t.Errorf("Unexpected error %q", err) } expectedBuildStatements := []BuildStatement{ - BuildStatement{ + { Command: "/bin/bash -c 'echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" > python_binary && " + "chmod a+x python_binary'", InputPaths: []string{"python_binary.zip"}, OutputPaths: []string{"python_binary"}, Mnemonic: "TemplateExpand", }, - BuildStatement{ + { Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " + "__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " + "../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles", - InputPaths: []string{"../bazel_tools/tools/zip/zipper/zipper", "python_binary.py"}, + InputPaths: []string{"python_binary.py"}, OutputPaths: []string{"python_binary.zip"}, Mnemonic: "PythonZipper", }, @@ -1240,13 +947,9 @@ func TestPythonZipperActionSuccess(t *testing.T) { func TestPythonZipperActionNoInput(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - },{ - "id": 2, - "pathFragmentId": 2 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -1256,37 +959,23 @@ func TestPythonZipperActionNoInput(t *testing.T) { "outputIds": [2], "primaryOutputId": 2 }], - "pathFragments": [{ - "id": 1, - "label": "python_binary" - },{ - "id": 2, - "label": "python_binary.zip" - }] + "pathFragments": [ + { "id": 1, "label": "python_binary" }, + { "id": 2, "label": "python_binary.zip" }] }` - _, err := AqueryBuildStatements([]byte(inputString)) + _, _, err := AqueryBuildStatements([]byte(inputString)) assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`) } func TestPythonZipperActionNoOutput(t *testing.T) { const inputString = ` { - "artifacts": [{ - "id": 1, - "pathFragmentId": 1 - },{ - "id": 2, - "pathFragmentId": 2 - },{ - "id": 3, - "pathFragmentId": 3 - },{ - "id": 4, - "pathFragmentId": 4 - },{ - "id": 5, - "pathFragmentId": 10 - }], + "artifacts": [ + { "id": 1, "pathFragmentId": 1 }, + { "id": 2, "pathFragmentId": 2 }, + { "id": 3, "pathFragmentId": 3 }, + { "id": 4, "pathFragmentId": 4 }, + { "id": 5, "pathFragmentId": 10 }], "actions": [{ "targetId": 1, "actionKey": "x", @@ -1295,73 +984,28 @@ func TestPythonZipperActionNoOutput(t *testing.T) { "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"], "inputDepSetIds": [1] }], - "depSetOfFiles": [{ - "id": 1, - "directArtifactIds": [4, 3, 5] - }], - "pathFragments": [{ - "id": 1, - "label": "python_binary" - },{ - "id": 2, - "label": "python_binary.zip" - },{ - "id": 3, - "label": "python_binary.py" - },{ - "id": 9, - "label": ".." - }, { - "id": 8, - "label": "bazel_tools", - "parentId": 9 - }, { - "id": 7, - "label": "tools", - "parentId": 8 - }, { - "id": 6, - "label": "zip", - "parentId": 7 - }, { - "id": 5, - "label": "zipper", - "parentId": 6 - }, { - "id": 4, - "label": "zipper", - "parentId": 5 - },{ - "id": 16, - "label": "bazel-out" - },{ - "id": 15, - "label": "bazel_tools", - "parentId": 16 - }, { - "id": 14, - "label": "k8-fastbuild", - "parentId": 15 - }, { - "id": 13, - "label": "bin", - "parentId": 14 - }, { - "id": 12, - "label": "tools", - "parentId": 13 - }, { - "id": 11, - "label": "python", - "parentId": 12 - }, { - "id": 10, - "label": "py3wrapper.sh", - "parentId": 11 - }] + "depSetOfFiles": [ + { "id": 1, "directArtifactIds": [4, 3, 5]}], + "pathFragments": [ + { "id": 1, "label": "python_binary" }, + { "id": 2, "label": "python_binary.zip" }, + { "id": 3, "label": "python_binary.py" }, + { "id": 9, "label": ".." }, + { "id": 8, "label": "bazel_tools", "parentId": 9 }, + { "id": 7, "label": "tools", "parentId": 8 }, + { "id": 6, "label": "zip", "parentId": 7 }, + { "id": 5, "label": "zipper", "parentId": 6 }, + { "id": 4, "label": "zipper", "parentId": 5 }, + { "id": 16, "label": "bazel-out" }, + { "id": 15, "label": "bazel_tools", "parentId": 16 }, + { "id": 14, "label": "k8-fastbuild", "parentId": 15 }, + { "id": 13, "label": "bin", "parentId": 14 }, + { "id": 12, "label": "tools", "parentId": 13 }, + { "id": 11, "label": "python", "parentId": 12 }, + { "id": 10, "label": "py3wrapper.sh", "parentId": 11 }] }` - _, err := AqueryBuildStatements([]byte(inputString)) - assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`) + _, _, err := AqueryBuildStatements([]byte(inputString)) + assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["python_binary.py"], output []`) } func assertError(t *testing.T, err error, expected string) { @@ -1382,50 +1026,54 @@ func assertBuildStatements(t *testing.T, expected []BuildStatement, actual []Bui len(expected), len(actual), expected, actual) return } -ACTUAL_LOOP: - for _, actualStatement := range actual { - for _, expectedStatement := range expected { - if buildStatementEquals(actualStatement, expectedStatement) { - continue ACTUAL_LOOP - } + type compareFn = func(i int, j int) bool + byCommand := func(slice []BuildStatement) compareFn { + return func(i int, j int) bool { + return slice[i].Command < slice[j].Command + } + } + sort.SliceStable(expected, byCommand(expected)) + sort.SliceStable(actual, byCommand(actual)) + for i, actualStatement := range actual { + expectedStatement := expected[i] + if differingField := buildStatementEquals(actualStatement, expectedStatement); differingField != "" { + t.Errorf("%s differs\nunexpected build statement %#v.\nexpected: %#v", + differingField, actualStatement, expectedStatement) + return } - t.Errorf("unexpected build statement %#v.\n expected: %#v", - actualStatement, expected) - return } } -func buildStatementEquals(first BuildStatement, second BuildStatement) bool { +func buildStatementEquals(first BuildStatement, second BuildStatement) string { if first.Mnemonic != second.Mnemonic { - return false + return "Mnemonic" } if first.Command != second.Command { - return false + return "Command" } // Ordering is significant for environment variables. if !reflect.DeepEqual(first.Env, second.Env) { - return false + return "Env" } // Ordering is irrelevant for input and output paths, so compare sets. - if !reflect.DeepEqual(stringSet(first.InputPaths), stringSet(second.InputPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.InputPaths), sortedStrings(second.InputPaths)) { + return "InputPaths" } - if !reflect.DeepEqual(stringSet(first.OutputPaths), stringSet(second.OutputPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.OutputPaths), sortedStrings(second.OutputPaths)) { + return "OutputPaths" } - if !reflect.DeepEqual(stringSet(first.SymlinkPaths), stringSet(second.SymlinkPaths)) { - return false + if !reflect.DeepEqual(sortedStrings(first.SymlinkPaths), sortedStrings(second.SymlinkPaths)) { + return "SymlinkPaths" } if first.Depfile != second.Depfile { - return false + return "Depfile" } - return true + return "" } -func stringSet(stringSlice []string) map[string]struct{} { - stringMap := make(map[string]struct{}) - for _, s := range stringSlice { - stringMap[s] = struct{}{} - } - return stringMap +func sortedStrings(stringSlice []string) []string { + sorted := make([]string, len(stringSlice)) + copy(sorted, stringSlice) + sort.Strings(sorted) + return sorted } diff --git a/bazel/configurability.go b/bazel/configurability.go index 7355ac7d3..0ab49eb12 100644 --- a/bazel/configurability.go +++ b/bazel/configurability.go @@ -43,6 +43,8 @@ const ( osArchDarwinX86_64 = "darwin_x86_64" osArchLinuxX86 = "linux_glibc_x86" osArchLinuxX86_64 = "linux_glibc_x86_64" + osArchLinuxMuslArm = "linux_musl_arm" + osArchLinuxMuslArm64 = "linux_musl_arm64" osArchLinuxMuslX86 = "linux_musl_x86" osArchLinuxMuslX86_64 = "linux_musl_x86_64" osArchLinuxBionicArm64 = "linux_bionic_arm64" @@ -101,6 +103,8 @@ var ( osArchDarwinX86_64: "//build/bazel/platforms/os_arch:darwin_x86_64", osArchLinuxX86: "//build/bazel/platforms/os_arch:linux_glibc_x86", osArchLinuxX86_64: "//build/bazel/platforms/os_arch:linux_glibc_x86_64", + osArchLinuxMuslArm: "//build/bazel/platforms/os_arch:linux_musl_arm", + osArchLinuxMuslArm64: "//build/bazel/platforms/os_arch:linux_musl_arm64", osArchLinuxMuslX86: "//build/bazel/platforms/os_arch:linux_musl_x86", osArchLinuxMuslX86_64: "//build/bazel/platforms/os_arch:linux_musl_x86_64", osArchLinuxBionicArm64: "//build/bazel/platforms/os_arch:linux_bionic_arm64", diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go index 5d00b0b29..f5435f25f 100644 --- a/bazel/cquery/request_type.go +++ b/bazel/cquery/request_type.go @@ -132,7 +132,7 @@ else: sharedLibraries = [] rootSharedLibraries = [] -shared_info_tag = "@rules_cc//examples:experimental_cc_shared_library.bzl%CcSharedLibraryInfo" +shared_info_tag = "@_builtins//:common/cc/experimental_cc_shared_library.bzl%CcSharedLibraryInfo" if shared_info_tag in providers(target): shared_info = providers(target)[shared_info_tag] for lib in shared_info.linker_input.libraries: diff --git a/bazel/properties.go b/bazel/properties.go index f9560319e..e29b9e137 100644 --- a/bazel/properties.go +++ b/bazel/properties.go @@ -409,6 +409,11 @@ func (ba BoolAttribute) HasConfigurableValues() bool { return false } +// SetValue sets value for the no config axis +func (ba *BoolAttribute) SetValue(value *bool) { + ba.SetSelectValue(NoConfigAxis, "", value) +} + // SetSelectValue sets value for the given axis/config. func (ba *BoolAttribute) SetSelectValue(axis ConfigurationAxis, config string, value *bool) { axis.validateConfig(config) @@ -652,6 +657,11 @@ func MakeLabelListAttribute(value LabelList) LabelListAttribute { } } +// MakeSingleLabelListAttribute initializes a LabelListAttribute as a non-arch specific list with 1 element, the given Label. +func MakeSingleLabelListAttribute(value Label) LabelListAttribute { + return MakeLabelListAttribute(MakeLabelList([]Label{value})) +} + func (lla *LabelListAttribute) SetValue(list LabelList) { lla.SetSelectValue(NoConfigAxis, "", list) } diff --git a/bp2build/Android.bp b/bp2build/Android.bp index e0ce19477..34548ed95 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -51,6 +51,7 @@ bootstrap_go_package { "conversion_test.go", "filegroup_conversion_test.go", "genrule_conversion_test.go", + "gensrcs_conversion_test.go", "java_binary_host_conversion_test.go", "java_import_conversion_test.go", "java_library_conversion_test.go", diff --git a/bp2build/android_app_certificate_conversion_test.go b/bp2build/android_app_certificate_conversion_test.go index 035a3529e..173b4e485 100644 --- a/bp2build/android_app_certificate_conversion_test.go +++ b/bp2build/android_app_certificate_conversion_test.go @@ -42,7 +42,7 @@ android_app_certificate { } `, expectedBazelTargets: []string{ - makeBazelTarget("android_app_certificate", "com.android.apogee.cert", attrNameToString{ + makeBazelTargetNoRestrictions("android_app_certificate", "com.android.apogee.cert", attrNameToString{ "certificate": `"chamber_of_secrets_dir"`, }), }}) diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go index 3824586c6..a216c9d11 100644 --- a/bp2build/android_app_conversion_test.go +++ b/bp2build/android_app_conversion_test.go @@ -74,7 +74,8 @@ android_app { package_name: "com.google", resource_dirs: ["resa", "resb"], manifest: "manifest/AndroidManifest.xml", - static_libs: ["static_lib_dep"] + static_libs: ["static_lib_dep"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -87,6 +88,7 @@ android_app { ]`, "custom_package": `"com.google"`, "deps": `[":static_lib_dep"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }}) } diff --git a/bp2build/apex_conversion_test.go b/bp2build/apex_conversion_test.go index 90571893c..7bc379fac 100644 --- a/bp2build/apex_conversion_test.go +++ b/bp2build/apex_conversion_test.go @@ -18,6 +18,7 @@ import ( "android/soong/android" "android/soong/apex" "android/soong/cc" + "android/soong/etc" "android/soong/java" "android/soong/sh" @@ -39,11 +40,31 @@ func registerApexModuleTypes(ctx android.RegistrationContext) { ctx.RegisterModuleType("apex_key", apex.ApexKeyFactory) ctx.RegisterModuleType("android_app_certificate", java.AndroidAppCertificateFactory) ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("prebuilt_etc", etc.PrebuiltEtcFactory) +} + +func runOverrideApexTestCase(t *testing.T, tc bp2buildTestCase) { + t.Helper() + runBp2BuildTestCase(t, registerOverrideApexModuleTypes, tc) +} + +func registerOverrideApexModuleTypes(ctx android.RegistrationContext) { + // CC module types needed as they can be APEX dependencies + cc.RegisterCCBuildComponents(ctx) + + ctx.RegisterModuleType("sh_binary", sh.ShBinaryFactory) + ctx.RegisterModuleType("cc_binary", cc.BinaryFactory) + ctx.RegisterModuleType("cc_library", cc.LibraryFactory) + ctx.RegisterModuleType("apex_key", apex.ApexKeyFactory) + ctx.RegisterModuleType("android_app_certificate", java.AndroidAppCertificateFactory) + ctx.RegisterModuleType("filegroup", android.FileGroupFactory) + ctx.RegisterModuleType("apex", apex.BundleFactory) + ctx.RegisterModuleType("prebuilt_etc", etc.PrebuiltEtcFactory) } func TestApexBundleSimple(t *testing.T) { runApexTestCase(t, bp2buildTestCase{ - description: "apex - example with all props", + description: "apex - example with all props, file_context is a module in same Android.bp", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, filesystem: map[string]string{}, @@ -71,22 +92,20 @@ cc_library { bazel_module: { bp2build_available: false }, } -// TODO(b/194878861): Add bp2build support for prebuilt_etc -cc_library { - name: "pretend_prebuilt_1", +prebuilt_etc { + name: "prebuilt_1", bazel_module: { bp2build_available: false }, } -// TODO(b/194878861): Add bp2build support for prebuilt_etc -cc_library { - name: "pretend_prebuilt_2", +prebuilt_etc { + name: "prebuilt_2", bazel_module: { bp2build_available: false }, } filegroup { name: "com.android.apogee-file_contexts", srcs: [ - "com.android.apogee-file_contexts", + "com.android.apogee-file_contexts", ], bazel_module: { bp2build_available: false }, } @@ -98,7 +117,7 @@ apex { name: "com.android.apogee", manifest: "apogee_manifest.json", androidManifest: "ApogeeAndroidManifest.xml", - file_contexts: "com.android.apogee-file_contexts", + file_contexts: ":com.android.apogee-file_contexts", min_sdk_version: "29", key: "com.android.apogee.key", certificate: "com.android.apogee.certificate", @@ -114,9 +133,11 @@ apex { "sh_binary_2", ], prebuilts: [ - "pretend_prebuilt_1", - "pretend_prebuilt_2", + "prebuilt_1", + "prebuilt_2", ], + package_name: "com.android.apogee.test.package", + logging_parent: "logging.parent", } `, expectedBazelTargets: []string{ @@ -148,11 +169,92 @@ apex { "//conditions:default": [], })`, "prebuilts": `[ - ":pretend_prebuilt_1", - ":pretend_prebuilt_2", + ":prebuilt_1", + ":prebuilt_2", ]`, - "updatable": "False", - "compressible": "False", + "updatable": "False", + "compressible": "False", + "package_name": `"com.android.apogee.test.package"`, + "logging_parent": `"logging.parent"`, + }), + }}) +} + +func TestApexBundleSimple_fileContextsInAnotherAndroidBp(t *testing.T) { + runApexTestCase(t, bp2buildTestCase{ + description: "apex - file contexts is a module in another Android.bp", + moduleTypeUnderTest: "apex", + moduleTypeUnderTestFactory: apex.BundleFactory, + filesystem: map[string]string{ + "a/b/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ + "com.android.apogee-file_contexts", + ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + file_contexts: ":com.android.apogee-file_contexts", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.android.apogee", attrNameToString{ + "file_contexts": `"//a/b:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_fileContextsIsFile(t *testing.T) { + runApexTestCase(t, bp2buildTestCase{ + description: "apex - file contexts is a file", + moduleTypeUnderTest: "apex", + moduleTypeUnderTestFactory: apex.BundleFactory, + filesystem: map[string]string{}, + blueprint: ` +apex { + name: "com.android.apogee", + file_contexts: "file_contexts_file", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.android.apogee", attrNameToString{ + "file_contexts": `"file_contexts_file"`, + "manifest": `"apex_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_fileContextsIsNotSpecified(t *testing.T) { + runApexTestCase(t, bp2buildTestCase{ + description: "apex - file contexts is not specified", + moduleTypeUnderTest: "apex", + moduleTypeUnderTestFactory: apex.BundleFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ + "com.android.apogee-file_contexts", + ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: ` +apex { + name: "com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, }), }}) } @@ -162,8 +264,16 @@ func TestApexBundleCompileMultilibBoth(t *testing.T) { description: "apex - example with compile_multilib=both", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, - blueprint: createMultilibBlueprint("both"), + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: createMultilibBlueprint("both"), expectedBazelTargets: []string{ makeBazelTarget("apex", "com.android.apogee", attrNameToString{ "native_shared_libs_32": `[ @@ -187,6 +297,8 @@ func TestApexBundleCompileMultilibBoth(t *testing.T) { ], "//conditions:default": [], })`, + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, }), }}) } @@ -196,8 +308,16 @@ func TestApexBundleCompileMultilibFirst(t *testing.T) { description: "apex - example with compile_multilib=first", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, - blueprint: createMultilibBlueprint("first"), + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: createMultilibBlueprint("first"), expectedBazelTargets: []string{ makeBazelTarget("apex", "com.android.apogee", attrNameToString{ "native_shared_libs_32": `select({ @@ -226,6 +346,8 @@ func TestApexBundleCompileMultilibFirst(t *testing.T) { ], "//conditions:default": [], })`, + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, }), }}) } @@ -235,8 +357,16 @@ func TestApexBundleCompileMultilib32(t *testing.T) { description: "apex - example with compile_multilib=32", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, - blueprint: createMultilibBlueprint("32"), + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: createMultilibBlueprint("32"), expectedBazelTargets: []string{ makeBazelTarget("apex", "com.android.apogee", attrNameToString{ "native_shared_libs_32": `[ @@ -247,6 +377,8 @@ func TestApexBundleCompileMultilib32(t *testing.T) { "//build/bazel/platforms/arch:x86": [":native_shared_lib_2"], "//conditions:default": [], })`, + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, }), }}) } @@ -256,8 +388,16 @@ func TestApexBundleCompileMultilib64(t *testing.T) { description: "apex - example with compile_multilib=64", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, - blueprint: createMultilibBlueprint("64"), + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: createMultilibBlueprint("64"), expectedBazelTargets: []string{ makeBazelTarget("apex", "com.android.apogee", attrNameToString{ "native_shared_libs_64": `select({ @@ -273,6 +413,8 @@ func TestApexBundleCompileMultilib64(t *testing.T) { ], "//conditions:default": [], })`, + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, }), }}) } @@ -282,7 +424,15 @@ func TestApexBundleDefaultPropertyValues(t *testing.T) { description: "apex - default property values", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, blueprint: ` apex { name: "com.android.apogee", @@ -290,7 +440,8 @@ apex { } `, expectedBazelTargets: []string{makeBazelTarget("apex", "com.android.apogee", attrNameToString{ - "manifest": `"apogee_manifest.json"`, + "manifest": `"apogee_manifest.json"`, + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, }), }}) } @@ -300,7 +451,15 @@ func TestApexBundleHasBazelModuleProps(t *testing.T) { description: "apex - has bazel module props", moduleTypeUnderTest: "apex", moduleTypeUnderTestFactory: apex.BundleFactory, - filesystem: map[string]string{}, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +} +`, + }, blueprint: ` apex { name: "apogee", @@ -309,7 +468,8 @@ apex { } `, expectedBazelTargets: []string{makeBazelTarget("apex", "apogee", attrNameToString{ - "manifest": `"manifest.json"`, + "manifest": `"manifest.json"`, + "file_contexts": `"//system/sepolicy/apex:apogee-file_contexts"`, }), }}) } @@ -363,3 +523,496 @@ apex { }, }` } + +func TestBp2BuildOverrideApex(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{}, + blueprint: ` +apex_key { + name: "com.android.apogee.key", + public_key: "com.android.apogee.avbpubkey", + private_key: "com.android.apogee.pem", + bazel_module: { bp2build_available: false }, +} + +android_app_certificate { + name: "com.android.apogee.certificate", + certificate: "com.android.apogee", + bazel_module: { bp2build_available: false }, +} + +cc_library { + name: "native_shared_lib_1", + bazel_module: { bp2build_available: false }, +} + +cc_library { + name: "native_shared_lib_2", + bazel_module: { bp2build_available: false }, +} + +prebuilt_etc { + name: "prebuilt_1", + bazel_module: { bp2build_available: false }, +} + +prebuilt_etc { + name: "prebuilt_2", + bazel_module: { bp2build_available: false }, +} + +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ + "com.android.apogee-file_contexts", + ], + bazel_module: { bp2build_available: false }, +} + +cc_binary { name: "cc_binary_1", bazel_module: { bp2build_available: false } } +sh_binary { name: "sh_binary_2", bazel_module: { bp2build_available: false } } + +apex { + name: "com.android.apogee", + manifest: "apogee_manifest.json", + androidManifest: "ApogeeAndroidManifest.xml", + file_contexts: ":com.android.apogee-file_contexts", + min_sdk_version: "29", + key: "com.android.apogee.key", + certificate: "com.android.apogee.certificate", + updatable: false, + installable: false, + compressible: false, + native_shared_libs: [ + "native_shared_lib_1", + "native_shared_lib_2", + ], + binaries: [ + "cc_binary_1", + "sh_binary_2", + ], + prebuilts: [ + "prebuilt_1", + "prebuilt_2", + ], + bazel_module: { bp2build_available: false }, +} + +apex_key { + name: "com.google.android.apogee.key", + public_key: "com.google.android.apogee.avbpubkey", + private_key: "com.google.android.apogee.pem", + bazel_module: { bp2build_available: false }, +} + +android_app_certificate { + name: "com.google.android.apogee.certificate", + certificate: "com.google.android.apogee", + bazel_module: { bp2build_available: false }, +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", + key: "com.google.android.apogee.key", + certificate: "com.google.android.apogee.certificate", + prebuilts: [], + compressible: true, +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "android_manifest": `"ApogeeAndroidManifest.xml"`, + "binaries": `[ + ":cc_binary_1", + ":sh_binary_2", + ]`, + "certificate": `":com.google.android.apogee.certificate"`, + "file_contexts": `":com.android.apogee-file_contexts"`, + "installable": "False", + "key": `":com.google.android.apogee.key"`, + "manifest": `"apogee_manifest.json"`, + "min_sdk_version": `"29"`, + "native_shared_libs_32": `[ + ":native_shared_lib_1", + ":native_shared_lib_2", + ]`, + "native_shared_libs_64": `select({ + "//build/bazel/platforms/arch:arm64": [ + ":native_shared_lib_1", + ":native_shared_lib_2", + ], + "//build/bazel/platforms/arch:x86_64": [ + ":native_shared_lib_1", + ":native_shared_lib_2", + ], + "//conditions:default": [], + })`, + "prebuilts": `[]`, + "updatable": "False", + "compressible": "True", + }), + }}) +} + +func TestApexBundleSimple_manifestIsEmpty_baseApexOverrideApexInDifferentAndroidBp(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - manifest of base apex is empty, base apex and override_apex is in different Android.bp", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + "a/b/Android.bp": ` +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: ` +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"//a/b:apex_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_manifestIsSet_baseApexOverrideApexInDifferentAndroidBp(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - manifest of base apex is set, base apex and override_apex is in different Android.bp", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + "a/b/Android.bp": ` +apex { + name: "com.android.apogee", + manifest: "apogee_manifest.json", + bazel_module: { bp2build_available: false }, +} +`, + }, + blueprint: ` +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"//a/b:apogee_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_manifestIsEmpty_baseApexOverrideApexInSameAndroidBp(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - manifest of base apex is empty, base apex and override_apex is in same Android.bp", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_manifestIsSet_baseApexOverrideApexInSameAndroidBp(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - manifest of base apex is set, base apex and override_apex is in same Android.bp", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + manifest: "apogee_manifest.json", + bazel_module: { bp2build_available: false }, +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apogee_manifest.json"`, + }), + }}) +} + +func TestApexBundleSimple_packageNameOverride(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - override package name", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", + package_name: "com.google.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "package_name": `"com.google.android.apogee"`, + }), + }}) +} + +func TestApexBundleSimple_NoPrebuiltsOverride(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - no override", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +prebuilt_etc { + name: "prebuilt_file", + bazel_module: { bp2build_available: false }, +} + +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, + prebuilts: ["prebuilt_file"] +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "prebuilts": `[":prebuilt_file"]`, + }), + }}) +} + +func TestApexBundleSimple_PrebuiltsOverride(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - ooverride", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +prebuilt_etc { + name: "prebuilt_file", + bazel_module: { bp2build_available: false }, +} + +prebuilt_etc { + name: "prebuilt_file2", + bazel_module: { bp2build_available: false }, +} + +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, + prebuilts: ["prebuilt_file"] +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", + prebuilts: ["prebuilt_file2"] +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "prebuilts": `[":prebuilt_file2"]`, + }), + }}) +} + +func TestApexBundleSimple_PrebuiltsOverrideEmptyList(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - override with empty list", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +prebuilt_etc { + name: "prebuilt_file", + bazel_module: { bp2build_available: false }, +} + +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, + prebuilts: ["prebuilt_file"] +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", + prebuilts: [], +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "prebuilts": `[]`, + }), + }}) +} + +func TestApexBundleSimple_NoLoggingParentOverride(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - logging_parent - no override", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, + logging_parent: "foo.bar.baz", +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "logging_parent": `"foo.bar.baz"`, + }), + }}) +} + +func TestApexBundleSimple_LoggingParentOverride(t *testing.T) { + runOverrideApexTestCase(t, bp2buildTestCase{ + description: "override_apex - logging_parent - override", + moduleTypeUnderTest: "override_apex", + moduleTypeUnderTestFactory: apex.OverrideApexFactory, + filesystem: map[string]string{ + "system/sepolicy/apex/Android.bp": ` +filegroup { + name: "com.android.apogee-file_contexts", + srcs: [ "apogee-file_contexts", ], + bazel_module: { bp2build_available: false }, +}`, + }, + blueprint: ` +apex { + name: "com.android.apogee", + bazel_module: { bp2build_available: false }, + logging_parent: "foo.bar.baz", +} + +override_apex { + name: "com.google.android.apogee", + base: ":com.android.apogee", + logging_parent: "foo.bar.baz.override", +} +`, + expectedBazelTargets: []string{ + makeBazelTarget("apex", "com.google.android.apogee", attrNameToString{ + "file_contexts": `"//system/sepolicy/apex:com.android.apogee-file_contexts"`, + "manifest": `"apex_manifest.json"`, + "logging_parent": `"foo.bar.baz.override"`, + }), + }}) +} diff --git a/bp2build/apex_key_conversion_test.go b/bp2build/apex_key_conversion_test.go index 1d949901c..dfa96a290 100644 --- a/bp2build/apex_key_conversion_test.go +++ b/bp2build/apex_key_conversion_test.go @@ -42,7 +42,7 @@ apex_key { private_key: "com.android.apogee.pem", } `, - expectedBazelTargets: []string{makeBazelTarget("apex_key", "com.android.apogee.key", attrNameToString{ + expectedBazelTargets: []string{makeBazelTargetNoRestrictions("apex_key", "com.android.apogee.key", attrNameToString{ "private_key": `"com.android.apogee.pem"`, "public_key": `"com.android.apogee.avbpubkey"`, }), diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 0f3ca79b5..19209f634 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -201,7 +201,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) { config := android.TestConfig(buildDir, nil, testCase.bp, nil) ctx := android.NewTestContext(config) - ctx.RegisterModuleType("custom", customModuleFactory) + ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice) ctx.Register() _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) @@ -501,6 +501,215 @@ custom { } } +func TestBp2buildHostAndDevice(t *testing.T) { + testCases := []bp2buildTestCase{ + { + description: "host and device, device only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.DeviceSupported), + }, + }, + { + description: "host and device, both", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: true, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{}), + }, + }, + { + description: "host and device, host explicitly disabled", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.DeviceSupported), + }, + }, + { + description: "host and device, neither", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: false, + device_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{ + "target_compatible_with": `["@platforms//:incompatible"]`, + }), + }, + }, + { + description: "host and device, neither, cannot override with product_var", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: false, + device_supported: false, + product_variables: { unbundled_build: { enabled: true } }, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{ + "target_compatible_with": `["@platforms//:incompatible"]`, + }), + }, + }, + { + description: "host and device, both, disabled overrided with product_var", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: true, + device_supported: true, + enabled: false, + product_variables: { unbundled_build: { enabled: true } }, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{ + "target_compatible_with": `["//build/bazel/product_variables:unbundled_build"]`, + }), + }, + }, + { + description: "host and device, neither, cannot override with arch enabled", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: false, + device_supported: false, + arch: { x86: { enabled: true } }, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{ + "target_compatible_with": `["@platforms//:incompatible"]`, + }), + }, + }, + { + description: "host and device, host only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, + blueprint: `custom { + name: "foo", + host_supported: true, + device_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.HostSupported), + }, + }, + { + description: "host only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostSupported, + blueprint: `custom { + name: "foo", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.HostSupported), + }, + }, + { + description: "device only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryDeviceSupported, + blueprint: `custom { + name: "foo", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.DeviceSupported), + }, + }, + { + description: "host and device default, default", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDeviceDefault, + blueprint: `custom { + name: "foo", + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{}), + }, + }, + { + description: "host and device default, device only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDeviceDefault, + blueprint: `custom { + name: "foo", + host_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.DeviceSupported), + }, + }, + { + description: "host and device default, host only", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDeviceDefault, + blueprint: `custom { + name: "foo", + device_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetHostOrDevice("custom", "foo", attrNameToString{}, android.HostSupported), + }, + }, + { + description: "host and device default, neither", + moduleTypeUnderTest: "custom", + moduleTypeUnderTestFactory: customModuleFactoryHostAndDeviceDefault, + blueprint: `custom { + name: "foo", + host_supported: false, + device_supported: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTargetNoRestrictions("custom", "foo", attrNameToString{ + "target_compatible_with": `["@platforms//:incompatible"]`, + }), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + runBp2BuildTestCaseSimple(t, tc) + }) + } +} + func TestLoadStatements(t *testing.T) { testCases := []struct { bazelTargets BazelTargets @@ -610,6 +819,7 @@ func TestGenerateBazelTargetModules_OneToMany_LoadedFromStarlark(t *testing.T) { { bp: `custom { name: "bar", + host_supported: true, one_to_many_prop: true, bazel_module: { bp2build_available: true }, }`, @@ -634,7 +844,7 @@ load("//build/bazel/rules:rules.bzl", "my_library")`, for _, testCase := range testCases { config := android.TestConfig(buildDir, nil, testCase.bp, nil) ctx := android.NewTestContext(config) - ctx.RegisterModuleType("custom", customModuleFactory) + ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice) ctx.RegisterForBazelConversion() _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) @@ -680,7 +890,7 @@ func TestModuleTypeBp2Build(t *testing.T) { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{}), + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{}), }, }, { @@ -693,7 +903,7 @@ func TestModuleTypeBp2Build(t *testing.T) { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{}), + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{}), }, }, { @@ -706,7 +916,7 @@ func TestModuleTypeBp2Build(t *testing.T) { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "a", "b", @@ -725,7 +935,7 @@ func TestModuleTypeBp2Build(t *testing.T) { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `["b"]`, }), }, @@ -740,7 +950,7 @@ func TestModuleTypeBp2Build(t *testing.T) { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "other/a.txt", "other/b.txt", @@ -772,7 +982,7 @@ func TestModuleTypeBp2Build(t *testing.T) { "other/file": "", }, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "a.txt", "b.txt", @@ -801,7 +1011,7 @@ func TestModuleTypeBp2Build(t *testing.T) { }`, }, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "//other:foo", "c", @@ -884,7 +1094,7 @@ func TestAllowlistingBp2buildTargetsExplicitly(t *testing.T) { { description: "generates more than 1 target if needed", moduleTypeUnderTest: "custom", - moduleTypeUnderTestFactory: customModuleFactory, + moduleTypeUnderTestFactory: customModuleFactoryHostAndDevice, bp: `custom { name: "foo", one_to_many_prop: true, @@ -1092,7 +1302,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { "other/BUILD.bazel": `// definition for fg_bar`, }, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{}), + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{}), `// definition for fg_bar`, }, }, @@ -1118,7 +1328,7 @@ func TestCombineBuildFilesBp2buildTargets(t *testing.T) { }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_bar", map[string]string{}), + makeBazelTargetNoRestrictions("filegroup", "fg_bar", map[string]string{}), `// BUILD file`, }, }, @@ -1203,7 +1413,7 @@ func TestGlobExcludeSrcs(t *testing.T) { "dir/f.txt": "", }, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "a.txt", "b.txt", @@ -1234,7 +1444,7 @@ func TestGlobExcludeSrcs(t *testing.T) { "dir/subdir/f.txt": "", }, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "srcs": `[ "a.txt", "//dir/subdir:e.txt", @@ -1265,7 +1475,7 @@ filegroup { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "data": `[":reqd"]`, }), }, @@ -1296,6 +1506,7 @@ python_library { "//conditions:default": [], })`, "srcs_version": `"PY3"`, + "imports": `["."]`, }), }, }, @@ -1321,6 +1532,7 @@ python_library { ":reqd", ]`, "srcs_version": `"PY3"`, + "imports": `["."]`, }), }, }, @@ -1335,7 +1547,7 @@ filegroup { bazel_module: { bp2build_available: true }, }`, expectedBazelTargets: []string{ - makeBazelTarget("filegroup", "fg_foo", map[string]string{ + makeBazelTargetNoRestrictions("filegroup", "fg_foo", map[string]string{ "data": `[":reqd"]`, }), }, diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go index 037564bb1..4794269e4 100644 --- a/bp2build/cc_binary_conversion_test.go +++ b/bp2build/cc_binary_conversion_test.go @@ -34,10 +34,11 @@ type testBazelTarget struct { attrs attrNameToString } -func generateBazelTargetsForTest(targets []testBazelTarget) []string { +func generateBazelTargetsForTest(targets []testBazelTarget, hod android.HostOrDeviceSupported) []string { ret := make([]string, 0, len(targets)) for _, t := range targets { - ret = append(ret, makeBazelTarget(t.typ, t.name, t.attrs)) + attrs := t.attrs.clone() + ret = append(ret, makeBazelTargetHostOrDevice(t.typ, t.name, attrs, hod)) } return ret } @@ -65,42 +66,33 @@ func runCcBinaryTests(t *testing.T, tc ccBinaryBp2buildTestCase) { runCcHostBinaryTestCase(t, tc) } -func runCcBinaryTestCase(t *testing.T, tc ccBinaryBp2buildTestCase) { +func runCcBinaryTestCase(t *testing.T, testCase ccBinaryBp2buildTestCase) { t.Helper() moduleTypeUnderTest := "cc_binary" - testCase := bp2buildTestCase{ - expectedBazelTargets: generateBazelTargetsForTest(tc.targets), - moduleTypeUnderTest: moduleTypeUnderTest, - moduleTypeUnderTestFactory: cc.BinaryFactory, - description: fmt.Sprintf("%s %s", moduleTypeUnderTest, tc.description), - blueprint: binaryReplacer.Replace(tc.blueprint), - } - t.Run(testCase.description, func(t *testing.T) { + + description := fmt.Sprintf("%s %s", moduleTypeUnderTest, testCase.description) + t.Run(description, func(t *testing.T) { t.Helper() - runBp2BuildTestCase(t, registerCcBinaryModuleTypes, testCase) + runBp2BuildTestCase(t, registerCcBinaryModuleTypes, bp2buildTestCase{ + expectedBazelTargets: generateBazelTargetsForTest(testCase.targets, android.DeviceSupported), + moduleTypeUnderTest: moduleTypeUnderTest, + moduleTypeUnderTestFactory: cc.BinaryFactory, + description: description, + blueprint: binaryReplacer.Replace(testCase.blueprint), + }) }) } -func runCcHostBinaryTestCase(t *testing.T, tc ccBinaryBp2buildTestCase) { +func runCcHostBinaryTestCase(t *testing.T, testCase ccBinaryBp2buildTestCase) { t.Helper() - testCase := tc - for i, tar := range testCase.targets { - switch tar.typ { - case "cc_binary", "proto_library", "cc_lite_proto_library": - tar.attrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - testCase.targets[i] = tar - } moduleTypeUnderTest := "cc_binary_host" - t.Run(testCase.description, func(t *testing.T) { + description := fmt.Sprintf("%s %s", moduleTypeUnderTest, testCase.description) + t.Run(description, func(t *testing.T) { runBp2BuildTestCase(t, registerCcBinaryModuleTypes, bp2buildTestCase{ - expectedBazelTargets: generateBazelTargetsForTest(testCase.targets), + expectedBazelTargets: generateBazelTargetsForTest(testCase.targets, android.HostSupported), moduleTypeUnderTest: moduleTypeUnderTest, moduleTypeUnderTestFactory: cc.BinaryHostFactory, - description: fmt.Sprintf("%s %s", moduleTypeUnderTest, tc.description), + description: description, blueprint: hostBinaryReplacer.Replace(testCase.blueprint), }) }) @@ -505,3 +497,49 @@ func TestCcBinaryStaticProto(t *testing.T) { }, }) } + +func TestCcBinaryConvertLex(t *testing.T) { + runCcBinaryTests(t, ccBinaryBp2buildTestCase{ + description: `.l and .ll sources converted to .c and .cc`, + blueprint: ` +{rule_name} { + name: "foo", + srcs: ["foo.c", "bar.cc", "foo1.l", "foo2.l", "bar1.ll", "bar2.ll"], + lex: { flags: ["--foo_opt", "--bar_opt"] }, + include_build_directory: false, +} +`, + targets: []testBazelTarget{ + {"genlex", "foo_genlex_l", attrNameToString{ + "srcs": `[ + "foo1.l", + "foo2.l", + ]`, + "lexopts": `[ + "--foo_opt", + "--bar_opt", + ]`, + }}, + {"genlex", "foo_genlex_ll", attrNameToString{ + "srcs": `[ + "bar1.ll", + "bar2.ll", + ]`, + "lexopts": `[ + "--foo_opt", + "--bar_opt", + ]`, + }}, + {"cc_binary", "foo", attrNameToString{ + "srcs": `[ + "bar.cc", + ":foo_genlex_ll", + ]`, + "srcs_c": `[ + "foo.c", + ":foo_genlex_l", + ]`, + }}, + }, + }) +} diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go index 2775a104e..2cc2207df 100644 --- a/bp2build/cc_library_conversion_test.go +++ b/bp2build/cc_library_conversion_test.go @@ -959,11 +959,12 @@ func TestCcLibraryFeatures(t *testing.T) { "features": `[ "disable_pack_relocations", "-no_undefined_symbols", + "-coverage", ]`, "srcs": `["a.cpp"]`, })...) expected_targets = append(expected_targets, makeCcLibraryTargets("b", attrNameToString{ - "features": `select({ + "features": `["-coverage"] + select({ "//build/bazel/platforms/arch:x86_64": [ "disable_pack_relocations", "-no_undefined_symbols", @@ -994,6 +995,7 @@ cc_library { pack_relocations: false, allow_undefined_symbols: true, include_build_directory: false, + native_coverage: false, } cc_library { @@ -1006,6 +1008,7 @@ cc_library { }, }, include_build_directory: false, + native_coverage: false, } cc_library { @@ -1879,76 +1882,78 @@ func TestCcLibraryCppStdWithGnuExtensions_ConvertsToFeatureAttr(t *testing.T) { // not set, only emit if gnu_extensions is disabled. the default (gnu+17 // is set in the toolchain.) {cpp_std: "", gnu_extensions: "", bazel_cpp_std: ""}, - {cpp_std: "", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c99"}, + {cpp_std: "", gnu_extensions: "false", bazel_cpp_std: "cpp_std_default_no_gnu", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "", gnu_extensions: "true", bazel_cpp_std: ""}, // experimental defaults to gnu++2a - {cpp_std: "experimental", gnu_extensions: "", bazel_cpp_std: "gnu++2a"}, - {cpp_std: "experimental", gnu_extensions: "false", bazel_cpp_std: "c++2a", bazel_c_std: "c99"}, - {cpp_std: "experimental", gnu_extensions: "true", bazel_cpp_std: "gnu++2a"}, + {cpp_std: "experimental", gnu_extensions: "", bazel_cpp_std: "cpp_std_experimental"}, + {cpp_std: "experimental", gnu_extensions: "false", bazel_cpp_std: "cpp_std_experimental_no_gnu", bazel_c_std: "c_std_default_no_gnu"}, + {cpp_std: "experimental", gnu_extensions: "true", bazel_cpp_std: "cpp_std_experimental"}, // Explicitly setting a c++ std does not use replace gnu++ std even if // gnu_extensions is true. // "c++11", {cpp_std: "c++11", gnu_extensions: "", bazel_cpp_std: "c++11"}, - {cpp_std: "c++11", gnu_extensions: "false", bazel_cpp_std: "c++11", bazel_c_std: "c99"}, + {cpp_std: "c++11", gnu_extensions: "false", bazel_cpp_std: "c++11", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "c++11", gnu_extensions: "true", bazel_cpp_std: "c++11"}, // "c++17", {cpp_std: "c++17", gnu_extensions: "", bazel_cpp_std: "c++17"}, - {cpp_std: "c++17", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c99"}, + {cpp_std: "c++17", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "c++17", gnu_extensions: "true", bazel_cpp_std: "c++17"}, // "c++2a", {cpp_std: "c++2a", gnu_extensions: "", bazel_cpp_std: "c++2a"}, - {cpp_std: "c++2a", gnu_extensions: "false", bazel_cpp_std: "c++2a", bazel_c_std: "c99"}, + {cpp_std: "c++2a", gnu_extensions: "false", bazel_cpp_std: "c++2a", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "c++2a", gnu_extensions: "true", bazel_cpp_std: "c++2a"}, // "c++98", {cpp_std: "c++98", gnu_extensions: "", bazel_cpp_std: "c++98"}, - {cpp_std: "c++98", gnu_extensions: "false", bazel_cpp_std: "c++98", bazel_c_std: "c99"}, + {cpp_std: "c++98", gnu_extensions: "false", bazel_cpp_std: "c++98", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "c++98", gnu_extensions: "true", bazel_cpp_std: "c++98"}, // gnu++ is replaced with c++ if gnu_extensions is explicitly false. // "gnu++11", {cpp_std: "gnu++11", gnu_extensions: "", bazel_cpp_std: "gnu++11"}, - {cpp_std: "gnu++11", gnu_extensions: "false", bazel_cpp_std: "c++11", bazel_c_std: "c99"}, + {cpp_std: "gnu++11", gnu_extensions: "false", bazel_cpp_std: "c++11", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "gnu++11", gnu_extensions: "true", bazel_cpp_std: "gnu++11"}, // "gnu++17", {cpp_std: "gnu++17", gnu_extensions: "", bazel_cpp_std: "gnu++17"}, - {cpp_std: "gnu++17", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c99"}, + {cpp_std: "gnu++17", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c_std_default_no_gnu"}, {cpp_std: "gnu++17", gnu_extensions: "true", bazel_cpp_std: "gnu++17"}, // some c_std test cases - {c_std: "experimental", gnu_extensions: "", bazel_c_std: "gnu11"}, - {c_std: "experimental", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c11"}, - {c_std: "experimental", gnu_extensions: "true", bazel_c_std: "gnu11"}, + {c_std: "experimental", gnu_extensions: "", bazel_c_std: "c_std_experimental"}, + {c_std: "experimental", gnu_extensions: "false", bazel_cpp_std: "cpp_std_default_no_gnu", bazel_c_std: "c_std_experimental_no_gnu"}, + {c_std: "experimental", gnu_extensions: "true", bazel_c_std: "c_std_experimental"}, {c_std: "gnu11", cpp_std: "gnu++17", gnu_extensions: "", bazel_cpp_std: "gnu++17", bazel_c_std: "gnu11"}, {c_std: "gnu11", cpp_std: "gnu++17", gnu_extensions: "false", bazel_cpp_std: "c++17", bazel_c_std: "c11"}, {c_std: "gnu11", cpp_std: "gnu++17", gnu_extensions: "true", bazel_cpp_std: "gnu++17", bazel_c_std: "gnu11"}, } for i, tc := range testCases { - name_prefix := fmt.Sprintf("a_%v", i) - cppStdProp := "" - if tc.cpp_std != "" { - cppStdProp = fmt.Sprintf(" cpp_std: \"%s\",", tc.cpp_std) - } - cStdProp := "" - if tc.c_std != "" { - cStdProp = fmt.Sprintf(" c_std: \"%s\",", tc.c_std) - } - gnuExtensionsProp := "" - if tc.gnu_extensions != "" { - gnuExtensionsProp = fmt.Sprintf(" gnu_extensions: %s,", tc.gnu_extensions) - } - attrs := attrNameToString{} - if tc.bazel_cpp_std != "" { - attrs["cpp_std"] = fmt.Sprintf(`"%s"`, tc.bazel_cpp_std) - } - if tc.bazel_c_std != "" { - attrs["c_std"] = fmt.Sprintf(`"%s"`, tc.bazel_c_std) - } - - runCcLibraryTestCase(t, bp2buildTestCase{ - description: fmt.Sprintf( - "cc_library with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), - moduleTypeUnderTest: "cc_library", - moduleTypeUnderTestFactory: cc.LibraryFactory, - blueprint: soongCcLibraryPreamble + fmt.Sprintf(` + name := fmt.Sprintf("cpp std: %q, c std: %q, gnu_extensions: %q", tc.cpp_std, tc.c_std, tc.gnu_extensions) + t.Run(name, func(t *testing.T) { + name_prefix := fmt.Sprintf("a_%v", i) + cppStdProp := "" + if tc.cpp_std != "" { + cppStdProp = fmt.Sprintf(" cpp_std: \"%s\",", tc.cpp_std) + } + cStdProp := "" + if tc.c_std != "" { + cStdProp = fmt.Sprintf(" c_std: \"%s\",", tc.c_std) + } + gnuExtensionsProp := "" + if tc.gnu_extensions != "" { + gnuExtensionsProp = fmt.Sprintf(" gnu_extensions: %s,", tc.gnu_extensions) + } + attrs := attrNameToString{} + if tc.bazel_cpp_std != "" { + attrs["cpp_std"] = fmt.Sprintf(`"%s"`, tc.bazel_cpp_std) + } + if tc.bazel_c_std != "" { + attrs["c_std"] = fmt.Sprintf(`"%s"`, tc.bazel_c_std) + } + + runCcLibraryTestCase(t, bp2buildTestCase{ + description: fmt.Sprintf( + "cc_library with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), + moduleTypeUnderTest: "cc_library", + moduleTypeUnderTestFactory: cc.LibraryFactory, + blueprint: soongCcLibraryPreamble + fmt.Sprintf(` cc_library { name: "%s_full", %s // cpp_std: *string @@ -1957,15 +1962,15 @@ cc_library { include_build_directory: false, } `, name_prefix, cppStdProp, cStdProp, gnuExtensionsProp), - expectedBazelTargets: makeCcLibraryTargets(name_prefix+"_full", attrs), - }) - - runCcLibraryStaticTestCase(t, bp2buildTestCase{ - description: fmt.Sprintf( - "cc_library_static with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), - moduleTypeUnderTest: "cc_library_static", - moduleTypeUnderTestFactory: cc.LibraryStaticFactory, - blueprint: soongCcLibraryPreamble + fmt.Sprintf(` + expectedBazelTargets: makeCcLibraryTargets(name_prefix+"_full", attrs), + }) + + runCcLibraryStaticTestCase(t, bp2buildTestCase{ + description: fmt.Sprintf( + "cc_library_static with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + blueprint: soongCcLibraryPreamble + fmt.Sprintf(` cc_library_static { name: "%s_static", %s // cpp_std: *string @@ -1974,17 +1979,17 @@ cc_library_static { include_build_directory: false, } `, name_prefix, cppStdProp, cStdProp, gnuExtensionsProp), - expectedBazelTargets: []string{ - makeBazelTarget("cc_library_static", name_prefix+"_static", attrs), - }, - }) - - runCcLibrarySharedTestCase(t, bp2buildTestCase{ - description: fmt.Sprintf( - "cc_library_shared with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), - moduleTypeUnderTest: "cc_library_shared", - moduleTypeUnderTestFactory: cc.LibrarySharedFactory, - blueprint: soongCcLibraryPreamble + fmt.Sprintf(` + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_static", name_prefix+"_static", attrs), + }, + }) + + runCcLibrarySharedTestCase(t, bp2buildTestCase{ + description: fmt.Sprintf( + "cc_library_shared with cpp_std: %s and gnu_extensions: %s", tc.cpp_std, tc.gnu_extensions), + moduleTypeUnderTest: "cc_library_shared", + moduleTypeUnderTestFactory: cc.LibrarySharedFactory, + blueprint: soongCcLibraryPreamble + fmt.Sprintf(` cc_library_shared { name: "%s_shared", %s // cpp_std: *string @@ -1993,9 +1998,10 @@ cc_library_shared { include_build_directory: false, } `, name_prefix, cppStdProp, cStdProp, gnuExtensionsProp), - expectedBazelTargets: []string{ - makeBazelTarget("cc_library_shared", name_prefix+"_shared", attrs), - }, + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_shared", name_prefix+"_shared", attrs), + }, + }) }) } } @@ -2278,6 +2284,7 @@ func TestCcLibraryDisabledArchAndTarget(t *testing.T) { blueprint: soongCcProtoPreamble + `cc_library { name: "foo", srcs: ["foo.cpp"], + host_supported: true, target: { darwin: { enabled: false, @@ -2313,6 +2320,7 @@ func TestCcLibraryDisabledArchAndTargetWithDefault(t *testing.T) { name: "foo", srcs: ["foo.cpp"], enabled: false, + host_supported: true, target: { darwin: { enabled: true, @@ -2377,6 +2385,7 @@ func TestCcLibraryStaticDisabledForSomeArch(t *testing.T) { moduleTypeUnderTestFactory: cc.LibraryFactory, blueprint: soongCcProtoPreamble + `cc_library { name: "foo", + host_supported: true, srcs: ["foo.cpp"], shared: { enabled: false @@ -2457,3 +2466,51 @@ func TestCcLibraryEscapeLdflags(t *testing.T) { }), }) } + +func TestCcLibraryConvertLex(t *testing.T) { + runCcLibraryTestCase(t, bp2buildTestCase{ + moduleTypeUnderTest: "cc_library", + moduleTypeUnderTestFactory: cc.LibraryFactory, + filesystem: map[string]string{ + "foo.c": "", + "bar.cc": "", + "foo1.l": "", + "bar1.ll": "", + "foo2.l": "", + "bar2.ll": "", + }, + blueprint: `cc_library { + name: "foo_lib", + srcs: ["foo.c", "bar.cc", "foo1.l", "foo2.l", "bar1.ll", "bar2.ll"], + lex: { flags: ["--foo_flags"] }, + include_build_directory: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: append([]string{ + makeBazelTarget("genlex", "foo_lib_genlex_l", attrNameToString{ + "srcs": `[ + "foo1.l", + "foo2.l", + ]`, + "lexopts": `["--foo_flags"]`, + }), + makeBazelTarget("genlex", "foo_lib_genlex_ll", attrNameToString{ + "srcs": `[ + "bar1.ll", + "bar2.ll", + ]`, + "lexopts": `["--foo_flags"]`, + }), + }, + makeCcLibraryTargets("foo_lib", attrNameToString{ + "srcs": `[ + "bar.cc", + ":foo_lib_genlex_ll", + ]`, + "srcs_c": `[ + "foo.c", + ":foo_lib_genlex_l", + ]`, + })...), + }) +} diff --git a/bp2build/cc_library_headers_conversion_test.go b/bp2build/cc_library_headers_conversion_test.go index e5bb12010..641984b52 100644 --- a/bp2build/cc_library_headers_conversion_test.go +++ b/bp2build/cc_library_headers_conversion_test.go @@ -84,18 +84,6 @@ func TestCcLibraryHeadersSimple(t *testing.T) { }, blueprint: soongCcLibraryHeadersPreamble + ` cc_library_headers { - name: "lib-1", - export_include_dirs: ["lib-1"], - bazel_module: { bp2build_available: false }, -} - -cc_library_headers { - name: "lib-2", - export_include_dirs: ["lib-2"], - bazel_module: { bp2build_available: false }, -} - -cc_library_headers { name: "foo_headers", export_include_dirs: ["dir-1", "dir-2"], header_libs: ["lib-1", "lib-2"], @@ -128,12 +116,8 @@ cc_library_headers { "//build/bazel/platforms/arch:x86_64": ["arch_x86_64_exported_include_dir"], "//conditions:default": [], })`, - "implementation_deps": `[ - ":lib-1", - ":lib-2", - ]`, - "sdk_version": `"current"`, - "min_sdk_version": `"29"`, + "sdk_version": `"current"`, + "min_sdk_version": `"29"`, }), }, }) @@ -173,18 +157,34 @@ cc_library_headers { cc_library_headers { name: "foo_headers", header_libs: ["base-lib"], + export_header_lib_headers: ["base-lib"], target: { - android: { header_libs: ["android-lib"] }, - darwin: { header_libs: ["darwin-lib"] }, - linux_bionic: { header_libs: ["linux_bionic-lib"] }, - linux_glibc: { header_libs: ["linux-lib"] }, - windows: { header_libs: ["windows-lib"] }, + android: { + header_libs: ["android-lib"], + export_header_lib_headers: ["android-lib"], + }, + darwin: { + header_libs: ["darwin-lib"], + export_header_lib_headers: ["darwin-lib"], + }, + linux_bionic: { + header_libs: ["linux_bionic-lib"], + export_header_lib_headers: ["linux_bionic-lib"], + }, + linux_glibc: { + header_libs: ["linux-lib"], + export_header_lib_headers: ["linux-lib"], + }, + windows: { + header_libs: ["windows-lib"], + export_header_lib_headers: ["windows-lib"], + }, }, include_build_directory: false, }`, expectedBazelTargets: []string{ makeBazelTarget("cc_library_headers", "foo_headers", attrNameToString{ - "implementation_deps": `[":base-lib"] + select({ + "deps": `[":base-lib"] + select({ "//build/bazel/platforms/os:android": [":android-lib"], "//build/bazel/platforms/os:darwin": [":darwin-lib"], "//build/bazel/platforms/os:linux": [":linux-lib"], @@ -228,10 +228,6 @@ cc_library_headers { "//build/bazel/platforms/os:android": [":exported-lib"], "//conditions:default": [], })`, - "implementation_deps": `select({ - "//build/bazel/platforms/os:android": [":android-lib"], - "//conditions:default": [], - })`, }), }, }) @@ -328,3 +324,69 @@ cc_library_headers { }, }) } + +func TestCcLibraryHeadersExportedStaticLibHeadersReexported(t *testing.T) { + runCcLibraryHeadersTestCase(t, bp2buildTestCase{ + description: "cc_library_headers exported_static_lib_headers is reexported", + moduleTypeUnderTest: "cc_library_headers", + moduleTypeUnderTestFactory: cc.LibraryHeaderFactory, + filesystem: map[string]string{}, + blueprint: soongCcLibraryHeadersPreamble + ` +cc_library_headers { + name: "foo_headers", + export_static_lib_headers: ["foo_export"], + static_libs: ["foo_export", "foo_no_reexport"], + bazel_module: { bp2build_available: true }, +} +` + simpleModuleDoNotConvertBp2build("cc_library_headers", "foo_export"), + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_headers", "foo_headers", attrNameToString{ + "deps": `[":foo_export"]`, + }), + }, + }) +} + +func TestCcLibraryHeadersExportedSharedLibHeadersReexported(t *testing.T) { + runCcLibraryHeadersTestCase(t, bp2buildTestCase{ + description: "cc_library_headers exported_shared_lib_headers is reexported", + moduleTypeUnderTest: "cc_library_headers", + moduleTypeUnderTestFactory: cc.LibraryHeaderFactory, + filesystem: map[string]string{}, + blueprint: soongCcLibraryHeadersPreamble + ` +cc_library_headers { + name: "foo_headers", + export_shared_lib_headers: ["foo_export"], + shared_libs: ["foo_export", "foo_no_reexport"], + bazel_module: { bp2build_available: true }, +} +` + simpleModuleDoNotConvertBp2build("cc_library_headers", "foo_export"), + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_headers", "foo_headers", attrNameToString{ + "deps": `[":foo_export"]`, + }), + }, + }) +} + +func TestCcLibraryHeadersExportedHeaderLibHeadersReexported(t *testing.T) { + runCcLibraryHeadersTestCase(t, bp2buildTestCase{ + description: "cc_library_headers exported_header_lib_headers is reexported", + moduleTypeUnderTest: "cc_library_headers", + moduleTypeUnderTestFactory: cc.LibraryHeaderFactory, + filesystem: map[string]string{}, + blueprint: soongCcLibraryHeadersPreamble + ` +cc_library_headers { + name: "foo_headers", + export_header_lib_headers: ["foo_export"], + header_libs: ["foo_export", "foo_no_reexport"], + bazel_module: { bp2build_available: true }, +} +` + simpleModuleDoNotConvertBp2build("cc_library_headers", "foo_export"), + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_headers", "foo_headers", attrNameToString{ + "deps": `[":foo_export"]`, + }), + }, + }) +} diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go index 7c2c10077..be096168c 100644 --- a/bp2build/cc_library_shared_conversion_test.go +++ b/bp2build/cc_library_shared_conversion_test.go @@ -520,3 +520,52 @@ cc_library_shared { })}, }) } + +func TestCcLibrarySharedConvertLex(t *testing.T) { + runCcLibrarySharedTestCase(t, bp2buildTestCase{ + description: "cc_library_shared with lex files", + moduleTypeUnderTest: "cc_library_shared", + moduleTypeUnderTestFactory: cc.LibrarySharedFactory, + filesystem: map[string]string{ + "foo.c": "", + "bar.cc": "", + "foo1.l": "", + "bar1.ll": "", + "foo2.l": "", + "bar2.ll": "", + }, + blueprint: `cc_library_shared { + name: "foo_lib", + srcs: ["foo.c", "bar.cc", "foo1.l", "foo2.l", "bar1.ll", "bar2.ll"], + lex: { flags: ["--foo_flags"] }, + include_build_directory: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTarget("genlex", "foo_lib_genlex_l", attrNameToString{ + "srcs": `[ + "foo1.l", + "foo2.l", + ]`, + "lexopts": `["--foo_flags"]`, + }), + makeBazelTarget("genlex", "foo_lib_genlex_ll", attrNameToString{ + "srcs": `[ + "bar1.ll", + "bar2.ll", + ]`, + "lexopts": `["--foo_flags"]`, + }), + makeBazelTarget("cc_library_shared", "foo_lib", attrNameToString{ + "srcs": `[ + "bar.cc", + ":foo_lib_genlex_ll", + ]`, + "srcs_c": `[ + "foo.c", + ":foo_lib_genlex_l", + ]`, + }), + }, + }) +} diff --git a/bp2build/cc_library_static_conversion_test.go b/bp2build/cc_library_static_conversion_test.go index be10e866d..36c46a427 100644 --- a/bp2build/cc_library_static_conversion_test.go +++ b/bp2build/cc_library_static_conversion_test.go @@ -18,6 +18,7 @@ import ( "android/soong/android" "android/soong/cc" "android/soong/genrule" + "fmt" "testing" ) @@ -204,8 +205,8 @@ cc_library_static { ":whole_static_lib_1", ":whole_static_lib_2", ]`, - "sdk_version": `"current"`, - "min_sdk_version": `"29"`, + "sdk_version": `"current"`, + "min_sdk_version": `"29"`, }), }, }) @@ -1489,3 +1490,72 @@ func TestCcLibraryStaticStdInFlags(t *testing.T) { }, }) } + +func TestCcLibraryStaticStl(t *testing.T) { + testCases := []struct { + desc string + prop string + attr attrNameToString + }{ + { + desc: "c++_shared deduped to libc++", + prop: `stl: "c++_shared",`, + attr: attrNameToString{ + "stl": `"libc++"`, + }, + }, + { + desc: "libc++ to libc++", + prop: `stl: "libc++",`, + attr: attrNameToString{ + "stl": `"libc++"`, + }, + }, + { + desc: "c++_static to libc++_static", + prop: `stl: "c++_static",`, + attr: attrNameToString{ + "stl": `"libc++_static"`, + }, + }, + { + desc: "libc++_static to libc++_static", + prop: `stl: "libc++_static",`, + attr: attrNameToString{ + "stl": `"libc++_static"`, + }, + }, + { + desc: "system to system", + prop: `stl: "system",`, + attr: attrNameToString{ + "stl": `"system"`, + }, + }, + { + desc: "none to none", + prop: `stl: "none",`, + attr: attrNameToString{ + "stl": `"none"`, + }, + }, + { + desc: "empty to empty", + attr: attrNameToString{}, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(*testing.T) { + runCcLibraryStaticTestCase(t, bp2buildTestCase{ + blueprint: fmt.Sprintf(`cc_library_static { + name: "foo", + include_build_directory: false, + %s +}`, tc.prop), + expectedBazelTargets: []string{ + makeBazelTarget("cc_library_static", "foo", tc.attr), + }, + }) + }) + } +} diff --git a/bp2build/cc_object_conversion_test.go b/bp2build/cc_object_conversion_test.go index ea5808665..52688d9e9 100644 --- a/bp2build/cc_object_conversion_test.go +++ b/bp2build/cc_object_conversion_test.go @@ -414,6 +414,7 @@ func TestCcObjectSelectOnLinuxAndBionicArchs(t *testing.T) { "bionic_arm64.cpp", ], "//build/bazel/platforms/os_arch:linux_glibc_x86": ["linux_x86.cpp"], + "//build/bazel/platforms/os_arch:linux_musl_arm64": ["linux_arm64.cpp"], "//build/bazel/platforms/os_arch:linux_musl_x86": ["linux_x86.cpp"], "//conditions:default": [], })`, diff --git a/bp2build/cc_prebuilt_library_static_test.go b/bp2build/cc_prebuilt_library_static_test.go index 3feb1f155..59839c884 100644 --- a/bp2build/cc_prebuilt_library_static_test.go +++ b/bp2build/cc_prebuilt_library_static_test.go @@ -96,3 +96,52 @@ cc_prebuilt_library_static { expectedErr: fmt.Errorf("Expected at most one source file"), }) } + +func TestCcLibraryStaticConvertLex(t *testing.T) { + runCcLibrarySharedTestCase(t, bp2buildTestCase{ + description: "cc_library_static with lex files", + moduleTypeUnderTest: "cc_library_static", + moduleTypeUnderTestFactory: cc.LibraryStaticFactory, + filesystem: map[string]string{ + "foo.c": "", + "bar.cc": "", + "foo1.l": "", + "bar1.ll": "", + "foo2.l": "", + "bar2.ll": "", + }, + blueprint: `cc_library_static { + name: "foo_lib", + srcs: ["foo.c", "bar.cc", "foo1.l", "foo2.l", "bar1.ll", "bar2.ll"], + lex: { flags: ["--foo_flags"] }, + include_build_directory: false, + bazel_module: { bp2build_available: true }, +}`, + expectedBazelTargets: []string{ + makeBazelTarget("genlex", "foo_lib_genlex_l", attrNameToString{ + "srcs": `[ + "foo1.l", + "foo2.l", + ]`, + "lexopts": `["--foo_flags"]`, + }), + makeBazelTarget("genlex", "foo_lib_genlex_ll", attrNameToString{ + "srcs": `[ + "bar1.ll", + "bar2.ll", + ]`, + "lexopts": `["--foo_flags"]`, + }), + makeBazelTarget("cc_library_static", "foo_lib", attrNameToString{ + "srcs": `[ + "bar.cc", + ":foo_lib_genlex_ll", + ]`, + "srcs_c": `[ + "foo.c", + ":foo_lib_genlex_l", + ]`, + }), + }, + }) +} diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 1790dd7b8..4b013d913 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -152,6 +152,7 @@ var ( "target": true, // interface prop type is not supported yet. "visibility": true, // Bazel has native visibility semantics. Handle later. "features": true, // There is already a built-in attribute 'features' which cannot be overridden. + "for": true, // reserved keyword, b/233579439 } ) diff --git a/bp2build/genrule_conversion_test.go b/bp2build/genrule_conversion_test.go index 9244b9922..45048924e 100644 --- a/bp2build/genrule_conversion_test.go +++ b/bp2build/genrule_conversion_test.go @@ -56,6 +56,7 @@ func TestGenruleCliVariableReplacement(t *testing.T) { moduleType string factory android.ModuleFactory genDir string + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -66,16 +67,19 @@ func TestGenruleCliVariableReplacement(t *testing.T) { moduleType: "cc_genrule", factory: cc.GenRuleFactory, genDir: "$(RULEDIR)", + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, genDir: "$(RULEDIR)", + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, genDir: "$(RULEDIR)", + hod: android.HostSupported, }, } @@ -104,15 +108,8 @@ func TestGenruleCliVariableReplacement(t *testing.T) { "tools": `[":foo.tool"]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -131,6 +128,7 @@ func TestGenruleLocationsLabel(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -139,14 +137,17 @@ func TestGenruleLocationsLabel(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -183,18 +184,9 @@ func TestGenruleLocationsLabel(t *testing.T) { "srcs": `["foo_tool.in"]`, } - if tc.moduleType == "java_genrule_host" { - compatibilityAttrs := `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - fooAttrs["target_compatible_with"] = compatibilityAttrs - fooToolsAttrs["target_compatible_with"] = compatibilityAttrs - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", fooAttrs), - makeBazelTarget("genrule", "foo.tools", fooToolsAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", fooAttrs, tc.hod), + makeBazelTargetHostOrDevice("genrule", "foo.tools", fooToolsAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -213,6 +205,7 @@ func TestGenruleLocationsAbsoluteLabel(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -221,14 +214,17 @@ func TestGenruleLocationsAbsoluteLabel(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -249,15 +245,8 @@ func TestGenruleLocationsAbsoluteLabel(t *testing.T) { "tools": `["//other:foo.tool"]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -277,6 +266,7 @@ func TestGenruleSrcsLocationsAbsoluteLabel(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -285,14 +275,17 @@ func TestGenruleSrcsLocationsAbsoluteLabel(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -313,15 +306,8 @@ func TestGenruleSrcsLocationsAbsoluteLabel(t *testing.T) { "tools": `["//other:foo.tool"]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -341,6 +327,7 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -349,14 +336,17 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -380,15 +370,8 @@ func TestGenruleLocationLabelShouldSubstituteFirstToolLabel(t *testing.T) { ]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -408,6 +391,7 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -416,14 +400,17 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -447,15 +434,8 @@ func TestGenruleLocationsLabelShouldSubstituteFirstToolLabel(t *testing.T) { ]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -475,6 +455,7 @@ func TestGenruleWithoutToolsOrToolFiles(t *testing.T) { testCases := []struct { moduleType string factory android.ModuleFactory + hod android.HostOrDeviceSupported }{ { moduleType: "genrule", @@ -483,14 +464,17 @@ func TestGenruleWithoutToolsOrToolFiles(t *testing.T) { { moduleType: "cc_genrule", factory: cc.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule", factory: java.GenRuleFactory, + hod: android.DeviceSupported, }, { moduleType: "java_genrule_host", factory: java.GenRuleFactoryHost, + hod: android.HostSupported, }, } @@ -509,15 +493,8 @@ func TestGenruleWithoutToolsOrToolFiles(t *testing.T) { "srcs": `["foo.in"]`, } - if tc.moduleType == "java_genrule_host" { - moduleAttrs["target_compatible_with"] = `select({ - "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], - "//conditions:default": [], - })` - } - expectedBazelTargets := []string{ - makeBazelTarget("genrule", "foo", moduleAttrs), + makeBazelTargetHostOrDevice("genrule", "foo", moduleAttrs, tc.hod), } t.Run(tc.moduleType, func(t *testing.T) { @@ -549,7 +526,7 @@ genrule { } `, expectedBazelTargets: []string{ - makeBazelTarget("genrule", "gen", attrNameToString{ + makeBazelTargetNoRestrictions("genrule", "gen", attrNameToString{ "cmd": `"do-something $(SRCS) $(OUTS)"`, "outs": `["out"]`, "srcs": `["in1"]`, @@ -574,7 +551,7 @@ genrule { } `, expectedBazelTargets: []string{ - makeBazelTarget("genrule", "gen", attrNameToString{ + makeBazelTargetNoRestrictions("genrule", "gen", attrNameToString{ "cmd": `"do-something $(SRCS) $(OUTS)"`, "outs": `[ "out-from-defaults", @@ -607,7 +584,7 @@ genrule { } `, expectedBazelTargets: []string{ - makeBazelTarget("genrule", "gen", attrNameToString{ + makeBazelTargetNoRestrictions("genrule", "gen", attrNameToString{ "cmd": `"cp $(SRCS) $(OUTS)"`, "outs": `["out"]`, "srcs": `["in1"]`, @@ -644,7 +621,7 @@ genrule { } `, expectedBazelTargets: []string{ - makeBazelTarget("genrule", "gen", attrNameToString{ + makeBazelTargetNoRestrictions("genrule", "gen", attrNameToString{ "cmd": `"cmd1 $(SRCS) $(OUTS)"`, "outs": `[ "out-from-3", diff --git a/bp2build/gensrcs_conversion_test.go b/bp2build/gensrcs_conversion_test.go new file mode 100644 index 000000000..76826631a --- /dev/null +++ b/bp2build/gensrcs_conversion_test.go @@ -0,0 +1,80 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "android/soong/genrule" + "testing" +) + +func TestGensrcs(t *testing.T) { + testcases := []struct { + name string + bp string + expectedBazelAttrs attrNameToString + }{ + { + name: "gensrcs with common usage of properties", + bp: ` + gensrcs { + name: "foo", + srcs: ["test/input.txt", ":external_files"], + tool_files: ["program.py"], + cmd: "$(location program.py) $(in) $(out)", + output_extension: "out", + bazel_module: { bp2build_available: true }, + }`, + expectedBazelAttrs: attrNameToString{ + "srcs": `[ + "test/input.txt", + ":external_files__BP2BUILD__MISSING__DEP", + ]`, + "tools": `["program.py"]`, + "output_extension": `"out"`, + "cmd": `"$(location program.py) $(SRC) $(OUT)"`, + }, + }, + { + name: "gensrcs with out_extension unset", + bp: ` + gensrcs { + name: "foo", + srcs: ["input.txt"], + cmd: "cat $(in) > $(out)", + bazel_module: { bp2build_available: true }, + }`, + expectedBazelAttrs: attrNameToString{ + "srcs": `["input.txt"]`, + "cmd": `"cat $(SRC) > $(OUT)"`, + }, + }, + } + + for _, test := range testcases { + expectedBazelTargets := []string{ + makeBazelTargetNoRestrictions("gensrcs", "foo", test.expectedBazelAttrs), + } + t.Run(test.name, func(t *testing.T) { + runBp2BuildTestCase(t, func(ctx android.RegistrationContext) {}, + bp2buildTestCase{ + moduleTypeUnderTest: "gensrcs", + moduleTypeUnderTestFactory: genrule.GenSrcsFactory, + blueprint: test.bp, + expectedBazelTargets: expectedBazelTargets, + }) + }) + } +} diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go index 4fc07e014..d7a76a89f 100644 --- a/bp2build/java_binary_host_conversion_test.go +++ b/bp2build/java_binary_host_conversion_test.go @@ -52,6 +52,7 @@ func TestJavaBinaryHost(t *testing.T) { jni_libs: ["jni-lib-1"], javacflags: ["-Xdoclint:all/protected"], bazel_module: { bp2build_available: true }, + java_version: "8", }`, expectedBazelTargets: []string{ makeBazelTarget("java_binary", "java-binary-host-1", attrNameToString{ @@ -59,7 +60,10 @@ func TestJavaBinaryHost(t *testing.T) { "main_class": `"com.android.test.MainClass"`, "deps": `["//other:jni-lib-1"]`, "jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`, - "javacopts": `["-Xdoclint:all/protected"]`, + "javacopts": `[ + "-Xdoclint:all/protected", + "-source 1.8 -target 1.8", + ]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go index ccc52ef12..e4d9cbcac 100644 --- a/bp2build/java_library_conversion_test.go +++ b/bp2build/java_library_conversion_test.go @@ -158,6 +158,22 @@ java_plugin { }) } +func TestJavaLibraryJavaVersion(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + blueprint: `java_library { + name: "java-lib-1", + srcs: ["a.java"], + java_version: "11", +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "srcs": `["a.java"]`, + "javacopts": `["-source 11 -target 11"]`, + }), + }, + }) +} + func TestJavaLibraryErrorproneJavacflagsEnabledManually(t *testing.T) { runJavaLibraryTestCase(t, bp2buildTestCase{ blueprint: `java_library { @@ -251,3 +267,108 @@ func TestJavaLibraryLogTags(t *testing.T) { }), }}) } + +func TestJavaLibraryResources(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + filesystem: map[string]string{ + "res/a.res": "", + "res/b.res": "", + "res/dir1/b.res": "", + }, + blueprint: `java_library { + name: "java-lib-1", + java_resources: ["res/a.res", "res/b.res"], +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "resources": `[ + "res/a.res", + "res/b.res", + ]`, + }), + }, + }) +} + +func TestJavaLibraryResourceDirs(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + filesystem: map[string]string{ + "res/a.res": "", + "res/b.res": "", + "res/dir1/b.res": "", + }, + blueprint: `java_library { + name: "java-lib-1", + java_resource_dirs: ["res"], +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "resource_strip_prefix": `"res"`, + "resources": `[ + "res/a.res", + "res/b.res", + "res/dir1/b.res", + ]`, + }), + }, + }) +} + +func TestJavaLibraryResourcesExcludeDir(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + filesystem: map[string]string{ + "res/a.res": "", + "res/exclude/b.res": "", + }, + blueprint: `java_library { + name: "java-lib-1", + java_resource_dirs: ["res"], + exclude_java_resource_dirs: ["res/exclude"], +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "resource_strip_prefix": `"res"`, + "resources": `["res/a.res"]`, + }), + }, + }) +} + +func TestJavaLibraryResourcesExcludeFile(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + filesystem: map[string]string{ + "res/a.res": "", + "res/dir1/b.res": "", + "res/dir1/exclude.res": "", + }, + blueprint: `java_library { + name: "java-lib-1", + java_resource_dirs: ["res"], + exclude_java_resources: ["res/dir1/exclude.res"], +}`, + expectedBazelTargets: []string{ + makeBazelTarget("java_library", "java-lib-1", attrNameToString{ + "resource_strip_prefix": `"res"`, + "resources": `[ + "res/a.res", + "res/dir1/b.res", + ]`, + }), + }, + }) +} + +func TestJavaLibraryResourcesFailsWithMultipleDirs(t *testing.T) { + runJavaLibraryTestCase(t, bp2buildTestCase{ + filesystem: map[string]string{ + "res/a.res": "", + "res1/a.res": "", + }, + blueprint: `java_library { + name: "java-lib-1", + java_resource_dirs: ["res", "res1"], +}`, + expectedErr: fmt.Errorf("bp2build does not support more than one directory in java_resource_dirs (b/226423379)"), + expectedBazelTargets: []string{}, + }) +} diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go index 73abdd2ea..83cc5519a 100644 --- a/bp2build/java_library_host_conversion_test.go +++ b/bp2build/java_library_host_conversion_test.go @@ -43,6 +43,7 @@ java_library_host { name: "java-lib-host-2", srcs: ["c.java"], bazel_module: { bp2build_available: true }, + java_version: "9", }`, expectedBazelTargets: []string{ makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{ @@ -54,7 +55,8 @@ java_library_host { })`, }), makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{ - "srcs": `["c.java"]`, + "javacopts": `["-source 1.9 -target 1.9"]`, + "srcs": `["c.java"]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/bp2build/java_plugin_conversion_test.go b/bp2build/java_plugin_conversion_test.go index c2a2182bd..dc763e763 100644 --- a/bp2build/java_plugin_conversion_test.go +++ b/bp2build/java_plugin_conversion_test.go @@ -39,6 +39,7 @@ func TestJavaPlugin(t *testing.T) { libs: ["java-lib-1"], static_libs: ["java-lib-2"], bazel_module: { bp2build_available: true }, + java_version: "7", } java_library { @@ -66,6 +67,7 @@ java_library { "a.java", "b.java", ]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/bp2build/java_proto_conversion_test.go b/bp2build/java_proto_conversion_test.go index 67f80446e..c6feeb8dc 100644 --- a/bp2build/java_proto_conversion_test.go +++ b/bp2build/java_proto_conversion_test.go @@ -102,6 +102,7 @@ func TestJavaProtoDefault(t *testing.T) { blueprint: `java_library_static { name: "java-protos", srcs: ["a.proto"], + java_version: "7", } `, expectedBazelTargets: []string{ @@ -115,7 +116,8 @@ func TestJavaProtoDefault(t *testing.T) { "deps": `[":java-protos_proto"]`, }), makeBazelTarget("java_library", "java-protos", attrNameToString{ - "exports": `[":java-protos_java_proto_lite"]`, + "exports": `[":java-protos_java_proto_lite"]`, + "javacopts": `["-source 1.7 -target 1.7"]`, }), }, }) diff --git a/bp2build/python_binary_conversion_test.go b/bp2build/python_binary_conversion_test.go index dfa11d1be..22bd028e7 100644 --- a/bp2build/python_binary_conversion_test.go +++ b/bp2build/python_binary_conversion_test.go @@ -43,9 +43,10 @@ func TestPythonBinaryHostSimple(t *testing.T) { }`, expectedBazelTargets: []string{ makeBazelTarget("py_binary", "foo", attrNameToString{ - "data": `["files/data.txt"]`, - "deps": `[":bar"]`, - "main": `"a.py"`, + "data": `["files/data.txt"]`, + "deps": `[":bar"]`, + "main": `"a.py"`, + "imports": `["."]`, "srcs": `[ "a.py", "b/c.py", @@ -83,6 +84,7 @@ func TestPythonBinaryHostPy2(t *testing.T) { expectedBazelTargets: []string{ makeBazelTarget("py_binary", "foo", attrNameToString{ "python_version": `"PY2"`, + "imports": `["."]`, "srcs": `["a.py"]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], @@ -116,7 +118,8 @@ func TestPythonBinaryHostPy3(t *testing.T) { expectedBazelTargets: []string{ // python_version is PY3 by default. makeBazelTarget("py_binary", "foo", attrNameToString{ - "srcs": `["a.py"]`, + "imports": `["."]`, + "srcs": `["a.py"]`, "target_compatible_with": `select({ "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], "//conditions:default": [], @@ -148,6 +151,7 @@ func TestPythonBinaryHostArchVariance(t *testing.T) { }`, expectedBazelTargets: []string{ makeBazelTarget("py_binary", "foo-arm", attrNameToString{ + "imports": `["."]`, "srcs": `select({ "//build/bazel/platforms/arch:arm": ["arm.py"], "//build/bazel/platforms/arch:x86": ["x86.py"], diff --git a/bp2build/python_library_conversion_test.go b/bp2build/python_library_conversion_test.go index 356d52e8c..f51f1068b 100644 --- a/bp2build/python_library_conversion_test.go +++ b/bp2build/python_library_conversion_test.go @@ -2,6 +2,7 @@ package bp2build import ( "fmt" + "strings" "testing" "android/soong/android" @@ -16,6 +17,8 @@ type pythonLibBp2BuildTestCase struct { filesystem map[string]string blueprint string expectedBazelTargets []testBazelTarget + dir string + expectedError error } func convertPythonLibTestCaseToBp2build_Host(tc pythonLibBp2BuildTestCase) bp2buildTestCase { @@ -34,11 +37,19 @@ func convertPythonLibTestCaseToBp2build(tc pythonLibBp2BuildTestCase) bp2buildTe for _, t := range tc.expectedBazelTargets { bp2BuildTargets = append(bp2BuildTargets, makeBazelTarget(t.typ, t.name, t.attrs)) } + // Copy the filesystem so that we can change stuff in it later without it + // affecting the original pythonLibBp2BuildTestCase + filesystemCopy := make(map[string]string) + for k, v := range tc.filesystem { + filesystemCopy[k] = v + } return bp2buildTestCase{ description: tc.description, - filesystem: tc.filesystem, + filesystem: filesystemCopy, blueprint: tc.blueprint, expectedBazelTargets: bp2BuildTargets, + dir: tc.dir, + expectedErr: tc.expectedError, } } @@ -47,6 +58,11 @@ func runPythonLibraryTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { testCase := convertPythonLibTestCaseToBp2build(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library") + for name, contents := range testCase.filesystem { + if strings.HasSuffix(name, "Android.bp") { + testCase.filesystem[name] = fmt.Sprintf(contents, "python_library") + } + } testCase.moduleTypeUnderTest = "python_library" testCase.moduleTypeUnderTestFactory = python.PythonLibraryFactory @@ -58,6 +74,11 @@ func runPythonLibraryHostTestCase(t *testing.T, tc pythonLibBp2BuildTestCase) { testCase := convertPythonLibTestCaseToBp2build_Host(tc) testCase.description = fmt.Sprintf(testCase.description, "python_library_host") testCase.blueprint = fmt.Sprintf(testCase.blueprint, "python_library_host") + for name, contents := range testCase.filesystem { + if strings.HasSuffix(name, "Android.bp") { + testCase.filesystem[name] = fmt.Sprintf(contents, "python_library_host") + } + } testCase.moduleTypeUnderTest = "python_library_host" testCase.moduleTypeUnderTestFactory = python.PythonLibraryHostFactory runBp2BuildTestCase(t, func(ctx android.RegistrationContext) { @@ -109,6 +130,7 @@ func TestSimplePythonLib(t *testing.T) { "b/d.py", ]`, "srcs_version": `"PY3"`, + "imports": `["."]`, }, }, }, @@ -136,6 +158,7 @@ func TestSimplePythonLib(t *testing.T) { attrs: attrNameToString{ "srcs": `["a.py"]`, "srcs_version": `"PY2"`, + "imports": `["."]`, }, }, }, @@ -163,6 +186,7 @@ func TestSimplePythonLib(t *testing.T) { attrs: attrNameToString{ "srcs": `["a.py"]`, "srcs_version": `"PY3"`, + "imports": `["."]`, }, }, }, @@ -189,11 +213,54 @@ func TestSimplePythonLib(t *testing.T) { typ: "py_library", name: "foo", attrs: attrNameToString{ - "srcs": `["a.py"]`, + "srcs": `["a.py"]`, + "imports": `["."]`, + }, + }, + }, + }, + { + description: "%s: pkg_path in a subdirectory of the same name converts correctly", + dir: "mylib/subpackage", + filesystem: map[string]string{ + "mylib/subpackage/a.py": "", + "mylib/subpackage/Android.bp": `%s { + name: "foo", + srcs: ["a.py"], + pkg_path: "mylib/subpackage", + + bazel_module: { bp2build_available: true }, + }`, + }, + blueprint: `%s {name: "bar"}`, + expectedBazelTargets: []testBazelTarget{ + { + // srcs_version is PY2ANDPY3 by default. + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["a.py"]`, + "imports": `["../.."]`, + "srcs_version": `"PY3"`, }, }, }, }, + { + description: "%s: pkg_path in a subdirectory of a different name fails", + dir: "mylib/subpackage", + filesystem: map[string]string{ + "mylib/subpackage/a.py": "", + "mylib/subpackage/Android.bp": `%s { + name: "foo", + srcs: ["a.py"], + pkg_path: "mylib/subpackage2", + bazel_module: { bp2build_available: true }, + }`, + }, + blueprint: `%s {name: "bar"}`, + expectedError: fmt.Errorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in."), + }, } for _, tc := range testCases { @@ -232,6 +299,50 @@ func TestPythonArchVariance(t *testing.T) { "//conditions:default": [], })`, "srcs_version": `"PY3"`, + "imports": `["."]`, + }, + }, + }, + }) +} + +func TestPythonLibraryWithProtobufs(t *testing.T) { + runPythonLibraryTestCases(t, pythonLibBp2BuildTestCase{ + description: "test %s protobuf", + filesystem: map[string]string{ + "dir/mylib.py": "", + "dir/myproto.proto": "", + }, + blueprint: `%s { + name: "foo", + srcs: [ + "dir/mylib.py", + "dir/myproto.proto", + ], + }`, + expectedBazelTargets: []testBazelTarget{ + { + typ: "proto_library", + name: "foo_proto", + attrs: attrNameToString{ + "srcs": `["dir/myproto.proto"]`, + }, + }, + { + typ: "py_proto_library", + name: "foo_py_proto", + attrs: attrNameToString{ + "deps": `[":foo_proto"]`, + }, + }, + { + typ: "py_library", + name: "foo", + attrs: attrNameToString{ + "srcs": `["dir/mylib.py"]`, + "srcs_version": `"PY3"`, + "imports": `["."]`, + "deps": `[":foo_py_proto"]`, }, }, }, diff --git a/bp2build/soong_config_module_type_conversion_test.go b/bp2build/soong_config_module_type_conversion_test.go index b1e1fb2ee..8460caeb3 100644 --- a/bp2build/soong_config_module_type_conversion_test.go +++ b/bp2build/soong_config_module_type_conversion_test.go @@ -49,6 +49,7 @@ soong_config_module_type { custom_cc_library_static { name: "foo", bazel_module: { bp2build_available: true }, + host_supported: true, soong_config_variables: { feature1: { conditions_default: { @@ -94,6 +95,7 @@ soong_config_module_type_import { custom_cc_library_static { name: "foo", bazel_module: { bp2build_available: true }, + host_supported: true, soong_config_variables: { feature1: { conditions_default: { @@ -141,6 +143,7 @@ soong_config_module_type { custom_cc_library_static { name: "foo", bazel_module: { bp2build_available: true }, + host_supported: true, soong_config_variables: { board: { soc_a: { @@ -200,6 +203,7 @@ soong_config_module_type { custom_cc_library_static { name: "foo", bazel_module: { bp2build_available: true }, + host_supported: true, soong_config_variables: { feature1: { conditions_default: { @@ -268,6 +272,7 @@ soong_config_module_type { custom_cc_library_static { name: "foo", bazel_module: { bp2build_available: true }, + host_supported: true, soong_config_variables: { board: { soc_a: { @@ -356,6 +361,7 @@ cc_library_static { name: "lib", defaults: ["foo_defaults_2"], bazel_module: { bp2build_available: true }, + host_supported: true, } ` @@ -429,12 +435,14 @@ cc_library_static { name: "lib", defaults: ["foo_defaults", "bar_defaults"], bazel_module: { bp2build_available: true }, + host_supported: true, } cc_library_static { name: "lib2", defaults: ["bar_defaults", "foo_defaults"], bazel_module: { bp2build_available: true }, + host_supported: true, } ` @@ -550,6 +558,7 @@ cc_library_static { name: "lib", defaults: ["foo_defaults", "qux_defaults"], bazel_module: { bp2build_available: true }, + host_supported: true, } ` @@ -615,6 +624,7 @@ library_linking_strategy_cc_defaults { library_linking_strategy_cc_defaults { name: "library_linking_strategy_merged_defaults", defaults: ["library_linking_strategy_lib_a_defaults"], + host_supported: true, soong_config_variables: { library_linking_strategy: { prefer_static: { @@ -714,6 +724,7 @@ library_linking_strategy_cc_defaults { cc_binary { name: "library_linking_strategy_sample_binary", + host_supported: true, srcs: ["library_linking_strategy.cc"], defaults: ["library_linking_strategy_sample_defaults"], }` @@ -800,6 +811,7 @@ alphabet_cc_defaults { cc_binary { name: "alphabet_binary", + host_supported: true, srcs: ["main.cc"], defaults: ["alphabet_sample_cc_defaults"], }` @@ -861,6 +873,7 @@ alphabet_cc_defaults { cc_binary { name: "alphabet_binary", srcs: ["main.cc"], + host_supported: true, defaults: ["alphabet_sample_cc_defaults"], enabled: false, arch: { @@ -958,6 +971,7 @@ soong_config_module_type { alphabet_cc_defaults { name: "alphabet_sample_cc_defaults", + host_supported: true, soong_config_variables: { special_build: { enabled: true, diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go index 15a63356e..818d7aece 100644 --- a/bp2build/symlink_forest.go +++ b/bp2build/symlink_forest.go @@ -90,6 +90,26 @@ func symlinkIntoForest(topdir, dst, src string) { } } +func isDir(path string, fi os.FileInfo) bool { + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + return fi.IsDir() + } + + fi2, statErr := os.Stat(path) + if statErr == nil { + return fi2.IsDir() + } + + // Check if this is a dangling symlink. If so, treat it like a file, not a dir. + _, lstatErr := os.Lstat(path) + if lstatErr != nil { + fmt.Fprintf(os.Stderr, "Cannot stat or lstat '%s': %s\n%s\n", path, statErr, lstatErr) + os.Exit(1) + } + + return false +} + // Recursively plants a symlink forest at forestDir. The symlink tree will // contain every file in buildFilesDir and srcDir excluding the files in // exclude. Collects every directory encountered during the traversal of srcDir @@ -145,8 +165,18 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir continue } + sDir := false + bDir := false + if sExists { + sDir = isDir(shared.JoinPath(topdir, srcChild), srcChildEntry) + } + + if bExists { + bDir = isDir(shared.JoinPath(topdir, buildFilesChild), buildFilesChildEntry) + } + if !sExists { - if buildFilesChildEntry.IsDir() && excludeChild != nil { + if bDir && excludeChild != nil { // Not in the source tree, but we have to exclude something from under // this subtree, so descend plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) @@ -155,7 +185,7 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir symlinkIntoForest(topdir, forestChild, buildFilesChild) } } else if !bExists { - if srcChildEntry.IsDir() && excludeChild != nil { + if sDir && excludeChild != nil { // Not in the build file tree, but we have to exclude something from // under this subtree, so descend plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) @@ -163,10 +193,10 @@ func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir // Not in the build file tree, symlink source tree, carry on symlinkIntoForest(topdir, forestChild, srcChild) } - } else if srcChildEntry.IsDir() && buildFilesChildEntry.IsDir() { + } else if sDir && bDir { // Both are directories. Descend. plantSymlinkForestRecursive(topdir, forestChild, buildFilesChild, srcChild, excludeChild, acc, okay) - } else if !srcChildEntry.IsDir() && !buildFilesChildEntry.IsDir() { + } else if !sDir && !bDir { // Neither is a directory. Prioritize BUILD files generated by bp2build // over any BUILD file imported into external/. fmt.Fprintf(os.Stderr, "Both '%s' and '%s' exist, symlinking the former to '%s'\n", diff --git a/bp2build/testing.go b/bp2build/testing.go index 029ba4938..580bac4a5 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -119,8 +119,8 @@ func runBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.Regi return } - errs := append(parseErrs, resolveDepsErrs...) - if tc.expectedErr != nil && checkError(t, errs, tc.expectedErr) { + parseAndResolveErrs := append(parseErrs, resolveDepsErrs...) + if tc.expectedErr != nil && checkError(t, parseAndResolveErrs, tc.expectedErr) { return } @@ -135,7 +135,7 @@ func runBp2BuildTestCase(t *testing.T, registerModuleTypes func(ctx android.Regi if checkError(t, errs, tc.expectedErr) { return } else { - t.Errorf("Expected error: %q, got: %q", tc.expectedErr, errs) + t.Errorf("Expected error: %q, got: %q and %q", tc.expectedErr, errs, parseAndResolveErrs) } } else { android.FailIfErrored(t, errs) @@ -213,12 +213,36 @@ func customModuleFactoryBase() android.Module { return module } -func customModuleFactory() android.Module { +func customModuleFactoryHostAndDevice() android.Module { m := customModuleFactoryBase() android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibBoth) return m } +func customModuleFactoryDeviceSupported() android.Module { + m := customModuleFactoryBase() + android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibBoth) + return m +} + +func customModuleFactoryHostSupported() android.Module { + m := customModuleFactoryBase() + android.InitAndroidArchModule(m, android.HostSupported, android.MultilibBoth) + return m +} + +func customModuleFactoryHostAndDeviceDefault() android.Module { + m := customModuleFactoryBase() + android.InitAndroidArchModule(m, android.HostAndDeviceDefault, android.MultilibBoth) + return m +} + +func customModuleFactoryNeitherHostNorDeviceSupported() android.Module { + m := customModuleFactoryBase() + android.InitAndroidArchModule(m, android.NeitherHostNorDeviceSupported, android.MultilibBoth) + return m +} + type testProps struct { Test_prop struct { Test_string_prop string @@ -355,7 +379,7 @@ func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) (BazelTa } func registerCustomModuleForBp2buildConversion(ctx *android.TestContext) { - ctx.RegisterModuleType("custom", customModuleFactory) + ctx.RegisterModuleType("custom", customModuleFactoryHostAndDevice) ctx.RegisterForBazelConversion() } @@ -369,7 +393,29 @@ func simpleModuleDoNotConvertBp2build(typ, name string) string { type attrNameToString map[string]string -func makeBazelTarget(typ, name string, attrs attrNameToString) string { +func (a attrNameToString) clone() attrNameToString { + newAttrs := make(attrNameToString, len(a)) + for k, v := range a { + newAttrs[k] = v + } + return newAttrs +} + +// makeBazelTargetNoRestrictions returns bazel target build file definition that can be host or +// device specific, or independent of host/device. +func makeBazelTargetHostOrDevice(typ, name string, attrs attrNameToString, hod android.HostOrDeviceSupported) string { + if _, ok := attrs["target_compatible_with"]; !ok { + switch hod { + case android.HostSupported: + attrs["target_compatible_with"] = `select({ + "//build/bazel/platforms/os:android": ["@platforms//:incompatible"], + "//conditions:default": [], + })` + case android.DeviceSupported: + attrs["target_compatible_with"] = `["//build/bazel/platforms/os:android"]` + } + } + attrStrings := make([]string, 0, len(attrs)+1) attrStrings = append(attrStrings, fmt.Sprintf(` name = "%s",`, name)) for _, k := range android.SortedStringKeys(attrs) { @@ -379,3 +425,16 @@ func makeBazelTarget(typ, name string, attrs attrNameToString) string { %s )`, typ, strings.Join(attrStrings, "\n")) } + +// makeBazelTargetNoRestrictions returns bazel target build file definition that does not add a +// target_compatible_with. This is useful for module types like filegroup and genrule that arch not +// arch variant +func makeBazelTargetNoRestrictions(typ, name string, attrs attrNameToString) string { + return makeBazelTargetHostOrDevice(typ, name, attrs, android.HostAndDeviceDefault) +} + +// makeBazelTargetNoRestrictions returns bazel target build file definition that is device specific +// as this is the most common default in Soong. +func makeBazelTarget(typ, name string, attrs attrNameToString) string { + return makeBazelTargetHostOrDevice(typ, name, attrs, android.DeviceSupported) +} diff --git a/build_kzip.bash b/build_kzip.bash index aff2d6d03..6219021cf 100755 --- a/build_kzip.bash +++ b/build_kzip.bash @@ -36,7 +36,7 @@ export KYTHE_JAVA_SOURCE_BATCH_SIZE KYTHE_KZIP_ENCODING declare -r out="${OUT_DIR:-out}" # Build extraction files for C++ and Java. Build `merge_zips` which we use later. -build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java +build/soong/soong_ui.bash --build-mode --all-modules --dir=$PWD -k merge_zips xref_cxx xref_java xref_rust # Build extraction file for Go the files in build/{blueprint,soong} directories. declare -r abspath_out=$(realpath "${out}") @@ -44,7 +44,7 @@ declare -r go_extractor=$(realpath prebuilts/build-tools/linux-x86/bin/go_extrac declare -r go_root=$(realpath prebuilts/go/linux-x86) declare -r source_root=$PWD -# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified +# TODO(asmundak): Until b/182183061 is fixed, default corpus has to be specified # in the rules file. Generate this file on the fly with corpus value set from the # environment variable. for dir in blueprint soong; do diff --git a/cc/Android.bp b/cc/Android.bp index b105e7c18..ce944674b 100644 --- a/cc/Android.bp +++ b/cc/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "soong-etc", "soong-fuzz", "soong-genrule", + "soong-multitree", "soong-snapshot", "soong-tradefed", ], @@ -65,6 +66,7 @@ bootstrap_go_package { "library.go", "library_headers.go", "library_sdk_member.go", + "library_stub.go", "native_bridge_sdk_trait.go", "object.go", "test.go", @@ -89,11 +91,13 @@ bootstrap_go_package { ], testSrcs: [ "afdo_test.go", + "binary_test.go", "cc_test.go", "compiler_test.go", "gen_test.go", "genrule_test.go", "library_headers_test.go", + "library_stub_test.go", "library_test.go", "object_test.go", "prebuilt_test.go", diff --git a/cc/binary.go b/cc/binary.go index c5017c1dd..b2f248282 100644 --- a/cc/binary.go +++ b/cc/binary.go @@ -17,6 +17,7 @@ package cc import ( "path/filepath" + "android/soong/bazel/cquery" "github.com/google/blueprint" "github.com/google/blueprint/proptools" @@ -182,7 +183,7 @@ func (binary *binaryDecorator) linkerDeps(ctx DepsContext, deps Deps) Deps { } } - if !binary.static() && inList("libc", deps.StaticLibs) && !ctx.BazelConversionMode() { + if !binary.static() && inList("libc", deps.StaticLibs) { ctx.ModuleErrorf("statically linking libc to dynamic executable, please remove libc\n" + "from static libs or set static_executable: true") } @@ -562,25 +563,32 @@ func (binary *binaryDecorator) verifyHostBionicLinker(ctx ModuleContext, in, lin } type ccBinaryBazelHandler struct { - android.BazelHandler - module *Module } -func (handler *ccBinaryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +var _ BazelHandler = (*ccBinaryBazelHandler)(nil) + +func (handler *ccBinaryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - if len(filePaths) != 1 { - ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths) - return false - } - outputFilePath := android.PathForBazelOut(ctx, filePaths[0]) - handler.module.outputFile = android.OptionalPathForPath(outputFilePath) - // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped. - handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath + bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} + +func (handler *ccBinaryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + if len(filePaths) != 1 { + ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths) + return } - return ok + outputFilePath := android.PathForBazelOut(ctx, filePaths[0]) + handler.module.outputFile = android.OptionalPathForPath(outputFilePath) + // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped. + handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath } func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) { diff --git a/cc/binary_test.go b/cc/binary_test.go index 8ec387182..cba5974fb 100644 --- a/cc/binary_test.go +++ b/cc/binary_test.go @@ -49,3 +49,23 @@ cc_binary { expectedUnStrippedFile := "outputbase/execroot/__main__/foo" android.AssertStringEquals(t, "Unstripped output file", expectedUnStrippedFile, unStrippedFilePath.String()) } + +func TestBinaryLinkerScripts(t *testing.T) { + result := PrepareForIntegrationTestWithCc.RunTestWithBp(t, ` + cc_binary { + name: "foo", + srcs: ["foo.cc"], + linker_scripts: ["foo.ld", "bar.ld"], + }`) + + binFoo := result.ModuleForTests("foo", "android_arm64_armv8-a").Rule("ld") + + android.AssertStringListContains(t, "missing dependency on linker_scripts", + binFoo.Implicits.Strings(), "foo.ld") + android.AssertStringListContains(t, "missing dependency on linker_scripts", + binFoo.Implicits.Strings(), "bar.ld") + android.AssertStringDoesContain(t, "missing flag for linker_scripts", + binFoo.Args["ldFlags"], "-Wl,--script,foo.ld") + android.AssertStringDoesContain(t, "missing flag for linker_scripts", + binFoo.Args["ldFlags"], "-Wl,--script,bar.ld") +} diff --git a/cc/bp2build.go b/cc/bp2build.go index cc378b3b4..fa30d096b 100644 --- a/cc/bp2build.go +++ b/cc/bp2build.go @@ -29,6 +29,8 @@ import ( const ( cSrcPartition = "c" asSrcPartition = "as" + lSrcPartition = "l" + llSrcPartition = "ll" cppSrcPartition = "cpp" protoSrcPartition = "proto" ) @@ -53,6 +55,8 @@ type staticOrSharedAttributes struct { Enabled bazel.BoolAttribute + Native_coverage bazel.BoolAttribute + sdkAttributes } @@ -76,6 +80,12 @@ func groupSrcsByExtension(ctx android.BazelConversionPathContext, srcs bazel.Lab protoSrcPartition: android.ProtoSrcLabelPartition, cSrcPartition: bazel.LabelPartition{Extensions: []string{".c"}, LabelMapper: addSuffixForFilegroup("_c_srcs")}, asSrcPartition: bazel.LabelPartition{Extensions: []string{".s", ".S"}, LabelMapper: addSuffixForFilegroup("_as_srcs")}, + // TODO(http://b/231968910): If there is ever a filegroup target that + // contains .l or .ll files we will need to find a way to add a + // LabelMapper for these that identifies these filegroups and + // converts them appropriately + lSrcPartition: bazel.LabelPartition{Extensions: []string{".l"}}, + llSrcPartition: bazel.LabelPartition{Extensions: []string{".ll"}}, // C++ is the "catch-all" group, and comprises generated sources because we don't // know the language of these sources until the genrule is executed. cppSrcPartition: bazel.LabelPartition{Extensions: []string{".cpp", ".cc", ".cxx", ".mm"}, LabelMapper: addSuffixForFilegroup("_cpp_srcs"), Keep_remainder: true}, @@ -167,21 +177,17 @@ func bp2buildParseStaticOrSharedProps(ctx android.BazelConversionPathContext, mo attrs.System_dynamic_deps.ForceSpecifyEmptyList = true if isStatic { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StaticProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*StaticProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Static) - } + bp2BuildPropParseHelper(ctx, module, &StaticProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*StaticProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Static) } - } + }) } else { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &SharedProperties{}) { - for config, props := range configToProps { - if staticOrSharedProps, ok := props.(*SharedProperties); ok { - setAttrs(axis, config, staticOrSharedProps.Shared) - } + bp2BuildPropParseHelper(ctx, module, &SharedProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if staticOrSharedProps, ok := props.(*SharedProperties); ok { + setAttrs(axis, config, staticOrSharedProps.Shared) } - } + }) } partitionedSrcs := groupSrcsByExtension(ctx, attrs.Srcs) @@ -289,6 +295,11 @@ type compilerAttributes struct { cppFlags bazel.StringListAttribute srcs bazel.LabelListAttribute + // Lex sources and options + lSrcs bazel.LabelListAttribute + llSrcs bazel.LabelListAttribute + lexopts bazel.StringListAttribute + hdrs bazel.LabelListAttribute rtti bazel.BoolAttribute @@ -359,21 +370,19 @@ func (ca *compilerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversi } func (ca *compilerAttributes) convertStlProps(ctx android.ArchVariantContext, module *Module) { - stlPropsByArch := module.GetArchVariantProperties(ctx, &StlProperties{}) - for _, configToProps := range stlPropsByArch { - for _, props := range configToProps { - if stlProps, ok := props.(*StlProperties); ok { - if stlProps.Stl == nil { - continue - } - if ca.stl == nil { - ca.stl = stlProps.Stl - } else if ca.stl != stlProps.Stl { - ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) - } + bp2BuildPropParseHelper(ctx, module, &StlProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stlProps, ok := props.(*StlProperties); ok { + if stlProps.Stl == nil { + return + } + if ca.stl == nil { + stl := deduplicateStlInput(*stlProps.Stl) + ca.stl = &stl + } else if ca.stl != stlProps.Stl { + ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl) } } - } + }) } func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -414,6 +423,8 @@ func (ca *compilerAttributes) finalize(ctx android.BazelConversionPathContext, i ca.srcs = partitionedSrcs[cppSrcPartition] ca.cSrcs = partitionedSrcs[cSrcPartition] ca.asSrcs = partitionedSrcs[asSrcPartition] + ca.lSrcs = partitionedSrcs[lSrcPartition] + ca.llSrcs = partitionedSrcs[llSrcPartition] ca.absoluteIncludes.DeduplicateAxesFromBase() ca.localIncludes.DeduplicateAxesFromBase() @@ -436,32 +447,33 @@ func parseSrcs(ctx android.BazelConversionPathContext, props *BaseCompilerProper return bazel.AppendBazelLabelLists(allSrcsLabelList, generatedSrcsLabelList), anySrcs } -func bp2buildResolveCppStdValue(c_std *string, cpp_std *string, gnu_extensions *bool) (*string, *string) { - var cStdVal, cppStdVal string +func bp2buildStdVal(std *string, prefix string, useGnu bool) *string { + defaultVal := prefix + "_std_default" // If c{,pp}std properties are not specified, don't generate them in the BUILD file. // Defaults are handled by the toolchain definition. // However, if gnu_extensions is false, then the default gnu-to-c version must be specified. - if cpp_std != nil { - cppStdVal = parseCppStd(cpp_std) - } else if gnu_extensions != nil && !*gnu_extensions { - cppStdVal = "c++17" - } - if c_std != nil { - cStdVal = parseCStd(c_std) - } else if gnu_extensions != nil && !*gnu_extensions { - cStdVal = "c99" + stdVal := proptools.StringDefault(std, defaultVal) + if stdVal == "experimental" || stdVal == defaultVal { + if stdVal == "experimental" { + stdVal = prefix + "_std_experimental" + } + if !useGnu { + stdVal += "_no_gnu" + } + } else if !useGnu { + stdVal = gnuToCReplacer.Replace(stdVal) } - cStdVal, cppStdVal = maybeReplaceGnuToC(gnu_extensions, cStdVal, cppStdVal) - var c_std_prop, cpp_std_prop *string - if cStdVal != "" { - c_std_prop = &cStdVal - } - if cppStdVal != "" { - cpp_std_prop = &cppStdVal + if stdVal == defaultVal { + return nil } + return &stdVal +} + +func bp2buildResolveCppStdValue(c_std *string, cpp_std *string, gnu_extensions *bool) (*string, *string) { + useGnu := useGnuExtensions(gnu_extensions) - return c_std_prop, cpp_std_prop + return bp2buildStdVal(c_std, "c", useGnu), bp2buildStdVal(cpp_std, "cpp", useGnu) } // packageFromLabel extracts package from a fully-qualified or relative Label and whether the label @@ -522,7 +534,9 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) var allHdrs []string if baseCompilerProps, ok := archVariantCompilerProps[axis][config].(*BaseCompilerProperties); ok { allHdrs = baseCompilerProps.Generated_headers - + if baseCompilerProps.Lex != nil { + compilerAttrs.lexopts.SetSelectValue(axis, config, baseCompilerProps.Lex.Flags) + } (&compilerAttrs).bp2buildForAxisAndConfig(ctx, axis, config, baseCompilerProps) } @@ -557,10 +571,15 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) } } } - compilerAttrs.convertStlProps(ctx, module) (&linkerAttrs).convertStripProps(ctx, module) + if module.coverage != nil && module.coverage.Properties.Native_coverage != nil && + !Bool(module.coverage.Properties.Native_coverage) { + // Native_coverage is arch neutral + (&linkerAttrs).features.Append(bazel.MakeStringListAttribute([]string{"-coverage"})) + } + productVariableProps := android.ProductVariableProperties(ctx) (&compilerAttrs).convertProductVariables(ctx, productVariableProps) @@ -577,6 +596,10 @@ func bp2BuildParseBaseProps(ctx android.Bp2buildMutatorContext, module *Module) (&linkerAttrs).wholeArchiveDeps.Add(protoDep.wholeStaticLib) (&linkerAttrs).implementationWholeArchiveDeps.Add(protoDep.implementationWholeStaticLib) + convertedLSrcs := bp2BuildLex(ctx, module.Name(), compilerAttrs) + (&compilerAttrs).srcs.Add(&convertedLSrcs.srcName) + (&compilerAttrs).cSrcs.Add(&convertedLSrcs.cSrcName) + return baseAttributes{ compilerAttrs, linkerAttrs, @@ -691,6 +714,13 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion la.additionalLinkerInputs.SetSelectValue(axis, config, bazel.LabelList{Includes: []bazel.Label{label}}) linkerFlags = append(linkerFlags, fmt.Sprintf("-Wl,--version-script,$(location %s)", label.Label)) } + + if props.Dynamic_list != nil { + label := android.BazelLabelForModuleSrcSingle(ctx, *props.Dynamic_list) + la.additionalLinkerInputs.SetSelectValue(axis, config, bazel.LabelList{Includes: []bazel.Label{label}}) + linkerFlags = append(linkerFlags, fmt.Sprintf("-Wl,--dynamic-list,$(location %s)", label.Label)) + } + la.linkopts.SetSelectValue(axis, config, linkerFlags) la.useLibcrt.SetSelectValue(axis, config, props.libCrt()) @@ -713,17 +743,15 @@ func (la *linkerAttributes) bp2buildForAxisAndConfig(ctx android.BazelConversion } func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) { - for axis, configToProps := range module.GetArchVariantProperties(ctx, &StripProperties{}) { - for config, props := range configToProps { - if stripProperties, ok := props.(*StripProperties); ok { - la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) - la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) - la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) - la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) - la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) - } + bp2BuildPropParseHelper(ctx, module, &StripProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if stripProperties, ok := props.(*StripProperties); ok { + la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols) + la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list) + la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame) + la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All) + la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None) } - } + }) } func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) { @@ -837,40 +865,23 @@ type BazelIncludes struct { SystemIncludes bazel.StringListAttribute } -func bp2BuildParseExportedIncludes(ctx android.BazelConversionPathContext, module *Module, existingIncludes BazelIncludes) BazelIncludes { - libraryDecorator := module.linker.(*libraryDecorator) - return bp2BuildParseExportedIncludesHelper(ctx, module, libraryDecorator, &existingIncludes) -} - -// Bp2buildParseExportedIncludesForPrebuiltLibrary returns a BazelIncludes with Bazel-ified values -// to export includes from the underlying module's properties. -func Bp2BuildParseExportedIncludesForPrebuiltLibrary(ctx android.BazelConversionPathContext, module *Module) BazelIncludes { - prebuiltLibraryLinker := module.linker.(*prebuiltLibraryLinker) - libraryDecorator := prebuiltLibraryLinker.libraryDecorator - return bp2BuildParseExportedIncludesHelper(ctx, module, libraryDecorator, nil) -} - -// bp2BuildParseExportedIncludes creates a string list attribute contains the -// exported included directories of a module. -func bp2BuildParseExportedIncludesHelper(ctx android.BazelConversionPathContext, module *Module, libraryDecorator *libraryDecorator, includes *BazelIncludes) BazelIncludes { +func bp2BuildParseExportedIncludes(ctx android.BazelConversionPathContext, module *Module, includes *BazelIncludes) BazelIncludes { var exported BazelIncludes if includes != nil { exported = *includes } else { exported = BazelIncludes{} } - for axis, configToProps := range module.GetArchVariantProperties(ctx, &FlagExporterProperties{}) { - for config, props := range configToProps { - if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { - if len(flagExporterProperties.Export_include_dirs) > 0 { - exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) - } - if len(flagExporterProperties.Export_system_include_dirs) > 0 { - exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) - } + bp2BuildPropParseHelper(ctx, module, &FlagExporterProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + if flagExporterProperties, ok := props.(*FlagExporterProperties); ok { + if len(flagExporterProperties.Export_include_dirs) > 0 { + exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...))) + } + if len(flagExporterProperties.Export_system_include_dirs) > 0 { + exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...))) } } - } + }) exported.AbsoluteIncludes.DeduplicateAxesFromBase() exported.Includes.DeduplicateAxesFromBase() exported.SystemIncludes.DeduplicateAxesFromBase() @@ -938,22 +949,19 @@ type binaryLinkerAttrs struct { func bp2buildBinaryLinkerProps(ctx android.BazelConversionPathContext, m *Module) binaryLinkerAttrs { attrs := binaryLinkerAttrs{} - archVariantProps := m.GetArchVariantProperties(ctx, &BinaryLinkerProperties{}) - for axis, configToProps := range archVariantProps { - for _, p := range configToProps { - props := p.(*BinaryLinkerProperties) - staticExecutable := props.Static_executable - if axis == bazel.NoConfigAxis { - if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { - attrs.Linkshared = &linkBinaryShared - } - } else if staticExecutable != nil { - // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a - // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling - ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") + bp2BuildPropParseHelper(ctx, m, &BinaryLinkerProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) { + linkerProps := props.(*BinaryLinkerProperties) + staticExecutable := linkerProps.Static_executable + if axis == bazel.NoConfigAxis { + if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared { + attrs.Linkshared = &linkBinaryShared } + } else if staticExecutable != nil { + // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a + // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling + ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values") } - } + }) return attrs } diff --git a/cc/builder.go b/cc/builder.go index 525b1a14f..70bbd6abb 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -202,36 +202,22 @@ var ( }, "clangBin", "format") - // Rule for invoking clang-tidy (a clang-based linter). - clangTidyDep, clangTidyDepRE = pctx.RemoteStaticRules("clangTidyDep", - blueprint.RuleParams{ - Depfile: "$out", - Deps: blueprint.DepsGCC, - Command: "${config.CcWrapper}$ccCmd $cFlags -E -o /dev/null $in " + - "-MQ $tidyFile -MD -MF $out", - CommandDeps: []string{"$ccCmd"}, - }, - &remoteexec.REParams{ - Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, - ExecStrategy: "${config.REClangTidyExecStrategy}", - Inputs: []string{"$in"}, - Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, - }, []string{"ccCmd", "cFlags", "tidyFile"}, []string{}) - + // Rules for invoking clang-tidy (a clang-based linter). clangTidy, clangTidyRE = pctx.RemoteStaticRules("clangTidy", blueprint.RuleParams{ Depfile: "${out}.d", Deps: blueprint.DepsGCC, - Command: "cp ${out}.dep ${out}.d && " + - "$tidyVars$reTemplate${config.ClangBin}/clang-tidy $tidyFlags $in -- $cFlags && " + - "touch $out", - CommandDeps: []string{"${config.ClangBin}/clang-tidy"}, + Command: "CLANG_CMD=$clangCmd TIDY_FILE=$out " + + "$tidyVars$reTemplate${config.ClangBin}/clang-tidy.sh $in $tidyFlags -- $cFlags", + CommandDeps: []string{"${config.ClangBin}/clang-tidy.sh", "$ccCmd", "$tidyCmd"}, }, &remoteexec.REParams{ Labels: map[string]string{"type": "lint", "tool": "clang-tidy", "lang": "cpp"}, ExecStrategy: "${config.REClangTidyExecStrategy}", - Inputs: []string{"$in", "${out}.dep"}, - EnvironmentVariables: []string{"TIDY_TIMEOUT"}, + Inputs: []string{"$in"}, + OutputFiles: []string{"${out}", "${out}.d"}, + ToolchainInputs: []string{"$ccCmd", "$tidyCmd"}, + EnvironmentVariables: []string{"CLANG_CMD", "TIDY_FILE", "TIDY_TIMEOUT"}, // Although clang-tidy has an option to "fix" source files, that feature is hardly useable // under parallel compilation and RBE. So we assume no OutputFiles here. // The clang-tidy fix option is best run locally in single thread. @@ -239,7 +225,7 @@ var ( // (1) New timestamps trigger clang and clang-tidy compilations again. // (2) Changing source files caused concurrent clang or clang-tidy jobs to crash. Platform: map[string]string{remoteexec.PoolKey: "${config.REClangTidyPool}"}, - }, []string{"cFlags", "tidyFlags", "tidyVars"}, []string{}) + }, []string{"cFlags", "ccCmd", "clangCmd", "tidyCmd", "tidyFlags", "tidyVars"}, []string{}) _ = pctx.SourcePathVariable("yasmCmd", "prebuilts/misc/${config.HostPrebuiltTag}/yasm/yasm") @@ -636,6 +622,7 @@ func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs continue } + // ccCmd is "clang" or "clang++" ccDesc := ccCmd ccCmd = "${config.ClangBin}/" + ccCmd @@ -681,43 +668,30 @@ func transformSourceToObj(ctx ModuleContext, subdir string, srcFiles, noTidySrcs // Even with tidy, some src file could be skipped by noTidySrcsMap. if tidy && !noTidySrcsMap[srcFile.String()] { tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy") - tidyDepFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy.dep") tidyFiles = append(tidyFiles, tidyFile) + tidyCmd := "${config.ClangBin}/clang-tidy" - ruleDep := clangTidyDep rule := clangTidy if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CLANG_TIDY") { - ruleDep = clangTidyDepRE rule = clangTidyRE } sharedCFlags := shareFlags("cFlags", moduleFlags) srcRelPath := srcFile.Rel() - // Add the .tidy.d rule - ctx.Build(pctx, android.BuildParams{ - Rule: ruleDep, - Description: "clang-tidy-dep " + srcRelPath, - Output: tidyDepFile, - Input: srcFile, - Implicits: cFlagsDeps, - OrderOnly: pathDeps, - Args: map[string]string{ - "ccCmd": ccCmd, - "cFlags": sharedCFlags, - "tidyFile": tidyFile.String(), - }, - }) - // Add the .tidy rule with order only dependency on the .tidy.d file + // Add the .tidy rule ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: "clang-tidy " + srcRelPath, Output: tidyFile, Input: srcFile, Implicits: cFlagsDeps, - OrderOnly: append(android.Paths{}, tidyDepFile), + OrderOnly: pathDeps, Args: map[string]string{ "cFlags": sharedCFlags, + "ccCmd": ccCmd, + "clangCmd": ccDesc, + "tidyCmd": tidyCmd, "tidyFlags": shareFlags("tidyFlags", config.TidyFlagsForSrcFile(srcFile, flags.tidyFlags)), "tidyVars": tidyVars, // short and not shared }, @@ -944,13 +918,13 @@ func unzipRefDump(ctx android.ModuleContext, zippedRefDump android.Path, baseNam return outputFile } -// sourceAbiDiff registers a build statement to compare linked sAbi dump files (.ldump). +// sourceAbiDiff registers a build statement to compare linked sAbi dump files (.lsdump). func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceDump android.Path, - baseName, exportedHeaderFlags string, checkAllApis, isLlndk, isNdk, isVndkExt bool) android.OptionalPath { + baseName, exportedHeaderFlags string, diffFlags []string, + checkAllApis, isLlndk, isNdk, isVndkExt bool) android.OptionalPath { outputFile := android.PathForModuleOut(ctx, baseName+".abidiff") libName := strings.TrimSuffix(baseName, filepath.Ext(baseName)) - createReferenceDumpFlags := "" var extraFlags []string if checkAllApis { @@ -961,22 +935,14 @@ func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceD "-allow-unreferenced-elf-symbol-changes") } - if exportedHeaderFlags == "" { - extraFlags = append(extraFlags, "-advice-only") - } - if isLlndk || isNdk { - createReferenceDumpFlags = "--llndk" - if isLlndk { - // TODO(b/130324828): "-consider-opaque-types-different" should apply to - // both LLNDK and NDK shared libs. However, a known issue in header-abi-diff - // breaks libaaudio. Remove the if-guard after the issue is fixed. - extraFlags = append(extraFlags, "-consider-opaque-types-different") - } + extraFlags = append(extraFlags, "-consider-opaque-types-different") } if isVndkExt { extraFlags = append(extraFlags, "-allow-extensions") } + // TODO(b/232891473): Simplify the above logic with diffFlags. + extraFlags = append(extraFlags, diffFlags...) ctx.Build(pctx, android.BuildParams{ Rule: sAbiDiff, @@ -989,7 +955,7 @@ func sourceAbiDiff(ctx android.ModuleContext, inputDump android.Path, referenceD "libName": libName, "arch": ctx.Arch().ArchType.Name, "extraFlags": strings.Join(extraFlags, " "), - "createReferenceDumpFlags": createReferenceDumpFlags, + "createReferenceDumpFlags": "", }, }) return android.OptionalPathForPath(outputFile) @@ -48,7 +48,6 @@ func RegisterCCBuildComponents(ctx android.RegistrationContext) { ctx.BottomUp("vndk", VndkMutator).Parallel() ctx.BottomUp("link", LinkageMutator).Parallel() ctx.BottomUp("test_per_src", TestPerSrcMutator).Parallel() - ctx.BottomUp("version_selector", versionSelectorMutator).Parallel() ctx.BottomUp("version", versionMutator).Parallel() ctx.BottomUp("begin", BeginMutator).Parallel() ctx.BottomUp("sysprop_cc", SyspropMutator).Parallel() @@ -96,6 +95,10 @@ type Deps struct { HeaderLibs []string RuntimeLibs []string + // UnexportedStaticLibs are static libraries that are also passed to -Wl,--exclude-libs= to + // prevent automatically exporting symbols. + UnexportedStaticLibs []string + // Used for data dependencies adjacent to tests DataLibs []string DataBins []string @@ -157,6 +160,7 @@ type PathDeps struct { GeneratedDeps android.Paths Flags []string + LdFlags []string IncludeDirs android.Paths SystemIncludeDirs android.Paths ReexportedDirs android.Paths @@ -679,6 +683,9 @@ type libraryDependencyTag struct { // Whether or not this dependency has to be followed for the apex variants excludeInApex bool + + // If true, don't automatically export symbols from the static library into a shared library. + unexportedSymbols bool } // header returns true if the libraryDependencyTag is tagging a header lib dependency. @@ -773,6 +780,19 @@ func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool { return ok && ccDepTag == testPerSrcDepTag } +// bazelHandler is the interface for a helper object related to deferring to Bazel for +// processing a cc module (during Bazel mixed builds). Individual module types should define +// their own bazel handler if they support being handled by Bazel. +type BazelHandler interface { + // QueueBazelCall invokes request-queueing functions on the BazelContext + //so that these requests are handled when Bazel's cquery is invoked. + QueueBazelCall(ctx android.BaseModuleContext, label string) + + // ProcessBazelQueryResponse uses information retrieved from Bazel to set properties + // on the current module with given label. + ProcessBazelQueryResponse(ctx android.ModuleContext, label string) +} + // Module contains the properties and members used by all C/C++ module types, and implements // the blueprint.Module interface. It delegates to compiler, linker, and installer interfaces // to construct the output file. Behavior can be customized with a Customizer, or "decorator", @@ -812,7 +832,7 @@ type Module struct { compiler compiler linker linker installer installer - bazelHandler android.BazelHandler + bazelHandler BazelHandler features []feature stl *stl @@ -970,6 +990,7 @@ func (c *Module) Shared() bool { return library.shared() } } + panic(fmt.Errorf("Shared() called on non-library module: %q", c.BaseModuleName())) } @@ -1774,31 +1795,58 @@ func GetSubnameProperty(actx android.ModuleContext, c LinkableInterface) string return subName } -// Returns true if Bazel was successfully used for the analysis of this module. -func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool { +var _ android.MixedBuildBuildable = (*Module)(nil) + +func (c *Module) getBazelModuleLabel(ctx android.BaseModuleContext) string { var bazelModuleLabel string if c.typ() == fullLibrary && c.static() { // cc_library is a special case in bp2build; two targets are generated -- one for each // of the shared and static variants. The shared variant keeps the module name, but the // static variant uses a different suffixed name. - bazelModuleLabel = bazelLabelForStaticModule(actx, c) + bazelModuleLabel = bazelLabelForStaticModule(ctx, c) } else { - bazelModuleLabel = c.GetBazelLabel(actx, c) + bazelModuleLabel = c.GetBazelLabel(ctx, c) + } + labelNoPrebuilt := bazelModuleLabel + if c.IsPrebuilt() { + labelNoPrebuilt = android.RemoveOptionalPrebuiltPrefixFromBazelLabel(bazelModuleLabel) + } + return labelNoPrebuilt +} + +func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) { + c.bazelHandler.QueueBazelCall(ctx, c.getBazelModuleLabel(ctx)) +} + +func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { + return c.bazelHandler != nil +} + +func (c *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { + bazelModuleLabel := c.getBazelModuleLabel(ctx) + + c.bazelHandler.ProcessBazelQueryResponse(ctx, bazelModuleLabel) + + c.Properties.SubName = GetSubnameProperty(ctx, c) + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + if !apexInfo.IsForPlatform() { + c.hideApexVariantFromMake = true } - bazelActionsUsed := false - // Mixed builds mode is disabled for modules outside of device OS. - // TODO(b/200841190): Support non-device OS in mixed builds. - if c.MixedBuildsEnabled(actx) && c.bazelHandler != nil { - bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel) + c.makeLinkType = GetMakeLinkType(ctx, c) + + mctx := &moduleContext{ + ModuleContext: ctx, + moduleContextImpl: moduleContextImpl{ + mod: c, + }, } - return bazelActionsUsed + mctx.ctx = mctx + + c.maybeInstall(mctx, apexInfo) } func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { - // TODO(cparsons): Any logic in this method occurring prior to querying Bazel should be - // requested from Bazel instead. - // Handle the case of a test module split by `test_per_src` mutator. // // The `test_per_src` mutator adds an extra variation named "", depending on all the other @@ -1825,11 +1873,6 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { } ctx.ctx = ctx - if c.maybeGenerateBazelActions(actx) { - c.maybeInstall(ctx, apexInfo) - return - } - deps := c.depsToPaths(ctx) if ctx.Failed() { return @@ -1887,6 +1930,8 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { flags.Local.CommonFlags = append(flags.Local.CommonFlags, "-isystem "+dir.String()) } + flags.Local.LdFlags = append(flags.Local.LdFlags, deps.LdFlags...) + c.flags = flags // We need access to all the flags seen by a source file. if c.sabi != nil { @@ -2032,12 +2077,6 @@ func (c *Module) deps(ctx DepsContext) Deps { deps.HeaderLibs = android.LastUniqueStrings(deps.HeaderLibs) deps.RuntimeLibs = android.LastUniqueStrings(deps.RuntimeLibs) - // In Bazel conversion mode, we dependency and build validations will occur in Bazel, so there is - // no need to do so in Soong. - if ctx.BazelConversionMode() { - return deps - } - for _, lib := range deps.ReexportSharedLibHeaders { if !inList(lib, deps.SharedLibs) { ctx.PropertyErrorf("export_shared_lib_headers", "Shared library not in shared_libs: '%s'", lib) @@ -2117,7 +2156,7 @@ func AddSharedLibDependenciesWithVersions(ctx android.BottomUpMutatorContext, mo variations = append([]blueprint.Variation(nil), variations...) - if version != "" && CanBeOrLinkAgainstVersionVariants(mod) { + if version != "" && canBeOrLinkAgainstVersionVariants(mod) { // Version is explicitly specified. i.e. libFoo#30 variations = append(variations, blueprint.Variation{Mutator: "version", Variation: version}) if tag, ok := depTag.(libraryDependencyTag); ok { @@ -2340,6 +2379,13 @@ func (c *Module) DepsMutator(actx android.BottomUpMutatorContext) { }, depTag, RewriteSnapshotLib(lib, GetSnapshot(c, &snapshotInfo, actx).StaticLibs)) } + for _, lib := range deps.UnexportedStaticLibs { + depTag := libraryDependencyTag{Kind: staticLibraryDependency, Order: lateLibraryDependency, unexportedSymbols: true} + actx.AddVariationDependencies([]blueprint.Variation{ + {Mutator: "link", Variation: "static"}, + }, depTag, RewriteSnapshotLib(lib, GetSnapshot(c, &snapshotInfo, actx).StaticLibs)) + } + for _, lib := range deps.LateSharedLibs { if inList(lib, sharedLibNames) { // This is to handle the case that some of the late shared libs (libc, libdl, libm, ...) @@ -2838,6 +2884,10 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { panic(fmt.Errorf("unexpected library dependency order %d", libDepTag.Order)) } } + if libDepTag.unexportedSymbols { + depPaths.LdFlags = append(depPaths.LdFlags, + "-Wl,--exclude-libs="+staticLibraryInfo.StaticLibrary.Base()) + } } if libDepTag.static() && !libDepTag.wholeStatic { diff --git a/cc/cc_test.go b/cc/cc_test.go index 09cc352a2..b6d196cc2 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -3722,6 +3722,25 @@ func TestMinSdkVersionInClangTriple(t *testing.T) { android.AssertStringDoesContain(t, "min sdk version", cFlags, "-target aarch64-linux-android29") } +func TestNonDigitMinSdkVersionInClangTriple(t *testing.T) { + bp := ` + cc_library_shared { + name: "libfoo", + srcs: ["foo.c"], + min_sdk_version: "S", + } + ` + result := android.GroupFixturePreparers( + prepareForCcTest, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + variables.Platform_version_active_codenames = []string{"UpsideDownCake", "Tiramisu"} + }), + ).RunTestWithBp(t, bp) + ctx := result.TestContext + cFlags := ctx.ModuleForTests("libfoo", "android_arm64_armv8-a_shared").Rule("cc").Args["cFlags"] + android.AssertStringDoesContain(t, "min sdk version", cFlags, "-target aarch64-linux-android31") +} + func TestIncludeDirsExporting(t *testing.T) { // Trim spaces from the beginning, end and immediately after any newline characters. Leaves @@ -4041,8 +4060,8 @@ func TestIncludeDirectoryOrdering(t *testing.T) { conly := []string{"-fPIC", "${config.CommonGlobalConlyflags}"} cppOnly := []string{"-fPIC", "${config.CommonGlobalCppflags}", "${config.DeviceGlobalCppflags}", "${config.ArmCppflags}"} - cflags := []string{"-Wall", "-Werror", "-std=candcpp"} - cstd := []string{"-std=gnu99", "-std=conly"} + cflags := []string{"-Werror", "-std=candcpp"} + cstd := []string{"-std=gnu11", "-std=conly"} cppstd := []string{"-std=gnu++17", "-std=cpp", "-fno-rtti"} lastIncludes := []string{ @@ -4076,7 +4095,7 @@ func TestIncludeDirectoryOrdering(t *testing.T) { { name: "assemble", src: "foo.s", - expected: combineSlices(baseExpectedFlags, []string{"-D__ASSEMBLY__"}, expectedIncludes, lastIncludes), + expected: combineSlices(baseExpectedFlags, []string{"${config.CommonGlobalAsflags}"}, expectedIncludes, lastIncludes), }, } diff --git a/cc/check.go b/cc/check.go index a357a9751..3d290a94b 100644 --- a/cc/check.go +++ b/cc/check.go @@ -87,6 +87,8 @@ func CheckBadLinkerFlags(ctx BaseModuleContext, prop string, flags []string) { ctx.PropertyErrorf(prop, "Bad flag: `%s` is not allowed", flag) } else if strings.HasPrefix(flag, "-Wl,--version-script") { ctx.PropertyErrorf(prop, "Bad flag: `%s`, use version_script instead", flag) + } else if flag == "-T" || strings.HasPrefix(flag, "--script") { + ctx.PropertyErrorf(prop, "Bad flag: `%s`, use linker_scripts instead", flag) } else if flag == "--coverage" { ctx.PropertyErrorf(prop, "Bad flag: `%s`, use native_coverage instead", flag) } else if strings.Contains(flag, " ") { diff --git a/cc/compiler.go b/cc/compiler.go index eb5458fc6..3c904b8f9 100644 --- a/cc/compiler.go +++ b/cc/compiler.go @@ -295,8 +295,12 @@ func addToModuleList(ctx ModuleContext, key android.OnceKey, module string) { getNamedMapForConfig(ctx.Config(), key).Store(module, true) } +func useGnuExtensions(gnuExtensions *bool) bool { + return proptools.BoolDefault(gnuExtensions, true) +} + func maybeReplaceGnuToC(gnuExtensions *bool, cStd string, cppStd string) (string, string) { - if gnuExtensions != nil && *gnuExtensions == false { + if !useGnuExtensions(gnuExtensions) { cStd = gnuToCReplacer.Replace(cStd) cppStd = gnuToCReplacer.Replace(cppStd) } @@ -453,7 +457,8 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps if version == "" || version == "current" { target += strconv.Itoa(android.FutureApiLevelInt) } else { - target += version + apiLevel := nativeApiLevelOrPanic(ctx, version) + target += apiLevel.String() } } @@ -489,7 +494,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps } } - flags.Global.AsFlags = append(flags.Global.AsFlags, "-D__ASSEMBLY__") + flags.Global.AsFlags = append(flags.Global.AsFlags, "${config.CommonGlobalAsflags}") flags.Global.CppFlags = append(flags.Global.CppFlags, tc.Cppflags()) @@ -589,10 +594,9 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags, deps addToModuleList(ctx, modulesUsingWnoErrorKey, module) } else if !inList("-Werror", flags.Local.CFlags) && !inList("-Werror", flags.Local.CppFlags) { if warningsAreAllowed(ctx.ModuleDir()) { - addToModuleList(ctx, modulesAddedWallKey, module) - flags.Local.CFlags = append([]string{"-Wall"}, flags.Local.CFlags...) + addToModuleList(ctx, modulesWarningsAllowedKey, module) } else { - flags.Local.CFlags = append([]string{"-Wall", "-Werror"}, flags.Local.CFlags...) + flags.Local.CFlags = append([]string{"-Werror"}, flags.Local.CFlags...) } } } diff --git a/cc/config/Android.bp b/cc/config/Android.bp index 1a21c1361..64a121e30 100644 --- a/cc/config/Android.bp +++ b/cc/config/Android.bp @@ -24,6 +24,7 @@ bootstrap_go_package { "x86_device.go", "x86_64_device.go", + "arm_linux_host.go", "darwin_host.go", "x86_linux_host.go", "x86_linux_bionic_host.go", diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go index dfe143f95..66087e6f0 100644 --- a/cc/config/arm64_device.go +++ b/cc/config/arm64_device.go @@ -33,9 +33,7 @@ var ( }, "armv8-a-branchprot": []string{ "-march=armv8-a", - // Disable BTI until drm vendors stop using OS libraries as sources - // of gadgets (https://issuetracker.google.com/216395195). - "-mbranch-protection=pac-ret", + "-mbranch-protection=standard", }, "armv8-2a": []string{ "-march=armv8.2-a", diff --git a/cc/config/arm64_linux_host.go b/cc/config/arm64_linux_host.go index 5c7f926f8..2d316e671 100644 --- a/cc/config/arm64_linux_host.go +++ b/cc/config/arm64_linux_host.go @@ -64,25 +64,25 @@ func init() { // toolchain config for ARM64 Linux CrossHost. Almost everything is the same as the ARM64 Android // target. The overridden methods below show the differences. -type toolchainLinuxArm64 struct { +type toolchainLinuxBionicArm64 struct { toolchainArm64 } -func (toolchainLinuxArm64) ClangTriple() string { +func (toolchainLinuxBionicArm64) ClangTriple() string { // Note the absence of "-android" suffix. The compiler won't define __ANDROID__ return "aarch64-linux" } -func (toolchainLinuxArm64) Cflags() string { +func (toolchainLinuxBionicArm64) Cflags() string { // The inherited flags + extra flags return "${config.Arm64Cflags} ${config.LinuxBionicArm64Cflags}" } -func (toolchainLinuxArm64) CrtBeginSharedBinary() []string { +func (toolchainLinuxBionicArm64) CrtBeginSharedBinary() []string { return linuxArm64CrtBeginSharedBinary } -func linuxArm64ToolchainFactory(arch android.Arch) Toolchain { +func linuxBionicArm64ToolchainFactory(arch android.Arch) Toolchain { archVariant := "armv8-a" // for host, default to armv8-a toolchainCflags := []string{arm64ArchVariantCflagsVar[archVariant]} @@ -90,7 +90,7 @@ func linuxArm64ToolchainFactory(arch android.Arch) Toolchain { // the host CPU needs the fix extraLdflags := "-Wl,--fix-cortex-a53-843419" - ret := toolchainLinuxArm64{} + ret := toolchainLinuxBionicArm64{} // add the extra ld and lld flags ret.toolchainArm64.ldflags = strings.Join([]string{ @@ -108,5 +108,5 @@ func linuxArm64ToolchainFactory(arch android.Arch) Toolchain { } func init() { - registerToolchainFactory(android.LinuxBionic, android.Arm64, linuxArm64ToolchainFactory) + registerToolchainFactory(android.LinuxBionic, android.Arm64, linuxBionicArm64ToolchainFactory) } diff --git a/cc/config/arm_linux_host.go b/cc/config/arm_linux_host.go new file mode 100644 index 000000000..525fb5d5d --- /dev/null +++ b/cc/config/arm_linux_host.go @@ -0,0 +1,174 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "android/soong/android" + +var ( + linuxArmCflags = []string{ + "-march=armv7a", + } + + linuxArm64Cflags = []string{} + + linuxArmLdflags = []string{ + "-march=armv7a", + } + + linuxArm64Ldflags = []string{} +) + +func init() { + exportedVars.ExportStringListStaticVariable("LinuxArmCflags", linuxArmCflags) + exportedVars.ExportStringListStaticVariable("LinuxArm64Cflags", linuxArm64Cflags) + exportedVars.ExportStringListStaticVariable("LinuxArmLdflags", linuxArmLdflags) + exportedVars.ExportStringListStaticVariable("LinuxArmLldflags", linuxArmLdflags) + exportedVars.ExportStringListStaticVariable("LinuxArm64Ldflags", linuxArm64Ldflags) + exportedVars.ExportStringListStaticVariable("LinuxArm64Lldflags", linuxArm64Ldflags) + + exportedVars.ExportStringListStaticVariable("LinuxArmYasmFlags", []string{"-f elf32 -m arm"}) + exportedVars.ExportStringListStaticVariable("LinuxArm64YasmFlags", []string{"-f elf64 -m aarch64"}) + +} + +// Musl arm+arm64 +type toolchainLinuxArm struct { + toolchain32Bit + toolchainLinux +} + +type toolchainLinuxArm64 struct { + toolchain64Bit + toolchainLinux +} + +func (t *toolchainLinuxArm) Name() string { + return "arm" +} + +func (t *toolchainLinuxArm64) Name() string { + return "arm64" +} + +func (t *toolchainLinuxArm) Cflags() string { + return "${config.LinuxCflags} ${config.LinuxArmCflags}" +} + +func (t *toolchainLinuxArm) Cppflags() string { + return "" +} + +func (t *toolchainLinuxArm64) Cflags() string { + return "${config.LinuxCflags} ${config.LinuxArm64Cflags}" +} + +func (t *toolchainLinuxArm64) Cppflags() string { + return "" +} + +func (t *toolchainLinuxArm) Ldflags() string { + return "${config.LinuxLdflags} ${config.LinuxArmLdflags}" +} + +func (t *toolchainLinuxArm) Lldflags() string { + return "${config.LinuxLldflags} ${config.LinuxArmLldflags}" +} + +func (t *toolchainLinuxArm64) Ldflags() string { + return "${config.LinuxLdflags} ${config.LinuxArm64Ldflags}" +} + +func (t *toolchainLinuxArm64) Lldflags() string { + return "${config.LinuxLldflags} ${config.LinuxArm64Lldflags}" +} + +func (t *toolchainLinuxArm) YasmFlags() string { + return "${config.LinuxArmYasmFlags}" +} + +func (t *toolchainLinuxArm64) YasmFlags() string { + return "${config.LinuxArm64YasmFlags}" +} + +func (toolchainLinuxArm) LibclangRuntimeLibraryArch() string { + return "arm" +} + +func (toolchainLinuxArm64) LibclangRuntimeLibraryArch() string { + return "arm64" +} + +func (t *toolchainLinuxArm) InstructionSetFlags(isa string) (string, error) { + // TODO: Is no thumb OK? + return t.toolchainBase.InstructionSetFlags("") +} + +type toolchainLinuxMuslArm struct { + toolchainLinuxArm + toolchainMusl +} + +type toolchainLinuxMuslArm64 struct { + toolchainLinuxArm64 + toolchainMusl +} + +func (t *toolchainLinuxMuslArm) ClangTriple() string { + return "arm-linux-musleabihf" +} + +func (t *toolchainLinuxMuslArm) Cflags() string { + return t.toolchainLinuxArm.Cflags() + " " + t.toolchainMusl.Cflags() +} + +func (t *toolchainLinuxMuslArm) Ldflags() string { + return t.toolchainLinuxArm.Ldflags() + " " + t.toolchainMusl.Ldflags() +} + +func (t *toolchainLinuxMuslArm) Lldflags() string { + return t.toolchainLinuxArm.Lldflags() + " " + t.toolchainMusl.Lldflags() +} + +func (t *toolchainLinuxMuslArm64) ClangTriple() string { + return "aarch64-linux-musl" +} + +func (t *toolchainLinuxMuslArm64) Cflags() string { + return t.toolchainLinuxArm64.Cflags() + " " + t.toolchainMusl.Cflags() +} + +func (t *toolchainLinuxMuslArm64) Ldflags() string { + return t.toolchainLinuxArm64.Ldflags() + " " + t.toolchainMusl.Ldflags() +} + +func (t *toolchainLinuxMuslArm64) Lldflags() string { + return t.toolchainLinuxArm64.Lldflags() + " " + t.toolchainMusl.Lldflags() +} + +var toolchainLinuxMuslArmSingleton Toolchain = &toolchainLinuxMuslArm{} +var toolchainLinuxMuslArm64Singleton Toolchain = &toolchainLinuxMuslArm64{} + +func linuxMuslArmToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslArmSingleton +} + +func linuxMuslArm64ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslArm64Singleton +} + +func init() { + registerToolchainFactory(android.LinuxMusl, android.Arm, linuxMuslArmToolchainFactory) + registerToolchainFactory(android.LinuxMusl, android.Arm64, linuxMuslArm64ToolchainFactory) +} diff --git a/cc/config/global.go b/cc/config/global.go index 3caf32792..c5fde5560 100644 --- a/cc/config/global.go +++ b/cc/config/global.go @@ -117,6 +117,13 @@ var ( commonGlobalConlyflags = []string{} + commonGlobalAsflags = []string{ + "-D__ASSEMBLY__", + // TODO(b/235105792): override global -fdebug-default-version=5, it is causing $TMPDIR to + // end up in the dwarf data for crtend_so.S. + "-fdebug-default-version=4", + } + deviceGlobalCflags = []string{ "-ffunction-sections", "-fdata-sections", @@ -191,7 +198,6 @@ var ( "-Werror=int-in-bool-context", "-Werror=int-to-pointer-cast", "-Werror=pointer-to-int-cast", - "-Werror=string-compare", "-Werror=xor-used-as-pow", // http://b/161386391 for -Wno-void-pointer-to-enum-cast "-Wno-void-pointer-to-enum-cast", @@ -225,7 +231,6 @@ var ( "-Wno-misleading-indentation", // http://b/153746954 "-Wno-zero-as-null-pointer-constant", // http://b/68236239 "-Wno-deprecated-anon-enum-enum-conversion", // http://b/153746485 - "-Wno-string-compare", // http://b/153764102 "-Wno-pessimizing-move", // http://b/154270751 // New warnings to be fixed after clang-r399163 "-Wno-non-c-typedef-for-linkage", // http://b/161304145 @@ -279,15 +284,15 @@ var ( "-w", } - CStdVersion = "gnu99" + CStdVersion = "gnu11" CppStdVersion = "gnu++17" - ExperimentalCStdVersion = "gnu11" + ExperimentalCStdVersion = "gnu17" ExperimentalCppStdVersion = "gnu++2a" // prebuilts/clang default settings. ClangDefaultBase = "prebuilts/clang/host" - ClangDefaultVersion = "clang-r450784d" - ClangDefaultShortVersion = "14.0.6" + ClangDefaultVersion = "clang-r450784e" + ClangDefaultShortVersion = "14.0.7" // Directories with warnings from Android.bp files. WarningAllowedProjects = []string{ @@ -315,6 +320,7 @@ func init() { } exportedVars.ExportStringListStaticVariable("CommonGlobalConlyflags", commonGlobalConlyflags) + exportedVars.ExportStringListStaticVariable("CommonGlobalAsflags", commonGlobalAsflags) exportedVars.ExportStringListStaticVariable("DeviceGlobalCppflags", deviceGlobalCppflags) exportedVars.ExportStringListStaticVariable("DeviceGlobalLdflags", deviceGlobalLdflags) exportedVars.ExportStringListStaticVariable("DeviceGlobalLldflags", deviceGlobalLldflags) @@ -372,6 +378,11 @@ func init() { exportedVars.ExportStringListStaticVariable("CommonGlobalCppflags", commonGlobalCppflags) exportedVars.ExportStringListStaticVariable("ExternalCflags", extraExternalCflags) + exportedVars.ExportString("CStdVersion", CStdVersion) + exportedVars.ExportString("CppStdVersion", CppStdVersion) + exportedVars.ExportString("ExperimentalCStdVersion", ExperimentalCStdVersion) + exportedVars.ExportString("ExperimentalCppStdVersion", ExperimentalCppStdVersion) + // Everything in these lists is a crime against abstraction and dependency tracking. // Do not add anything to this list. commonGlobalIncludes := []string{ diff --git a/cc/config/tidy.go b/cc/config/tidy.go index ba1043b0d..674edaddd 100644 --- a/cc/config/tidy.go +++ b/cc/config/tidy.go @@ -19,6 +19,37 @@ import ( "strings" ) +var ( + // Some clang-tidy checks have bugs or don't work for Android. + // They are disabled here, overriding any locally selected checks. + globalNoCheckList = []string{ + // https://b.corp.google.com/issues/153464409 + // many local projects enable cert-* checks, which + // trigger bugprone-reserved-identifier. + "-bugprone-reserved-identifier*,-cert-dcl51-cpp,-cert-dcl37-c", + // http://b/153757728 + "-readability-qualified-auto", + // http://b/193716442 + "-bugprone-implicit-widening-of-multiplication-result", + // Too many existing functions trigger this rule, and fixing it requires large code + // refactoring. The cost of maintaining this tidy rule outweighs the benefit it brings. + "-bugprone-easily-swappable-parameters", + // http://b/216364337 - TODO: Follow-up after compiler update to + // disable or fix individual instances. + "-cert-err33-c", + } + + // Some clang-tidy checks are included in some tidy_checks_as_errors lists, + // but not all warnings are fixed/suppressed yet. These checks are not + // disabled in the TidyGlobalNoChecks list, so we can see them and fix/suppress them. + globalNoErrorCheckList = []string{ + // http://b/155034563 + "-bugprone-signed-char-misuse", + // http://b/155034972 + "-bugprone-branch-clone", + } +) + func init() { // Many clang-tidy checks like altera-*, llvm-*, modernize-* // are not designed for Android source code or creating too @@ -35,17 +66,26 @@ func init() { "bugprone-*", "cert-*", "clang-diagnostic-unused-command-line-argument", - "google-*", + // Select only google-* checks that do not have thousands of warnings. + // Add more such checks when we clean up source code. + // "google-build-using-namespace", + // "google-default-arguments", + // "google-explicit-constructor", + // "google-global-names-in-headers", + // "google-runtime-int", + "google-build-explicit-make-pair", + "google-build-namespaces", + "google-runtime-operator", + "google-upgrade-*", "misc-*", "performance-*", "portability-*", "-bugprone-easily-swappable-parameters", "-bugprone-narrowing-conversions", - "-google-readability*", - "-google-runtime-references", "-misc-no-recursion", "-misc-non-private-member-variables-in-classes", "-misc-unused-parameters", + "-performance-no-int-to-ptr", // the following groups are excluded by -* // -altera-* // -cppcoreguidelines-* @@ -78,16 +118,21 @@ func init() { return strings.Join([]string{ "-*", "clang-diagnostic-unused-command-line-argument", - "google*", - "-google-build-using-namespace", - "-google-default-arguments", - "-google-explicit-constructor", - "-google-readability*", - "-google-runtime-int", - "-google-runtime-references", + "google-build-explicit-make-pair", + "google-build-namespaces", + "google-runtime-operator", + "google-upgrade-*", }, ",") }) + pctx.VariableFunc("TidyGlobalNoChecks", func(ctx android.PackageVarContext) string { + return strings.Join(globalNoCheckList, ",") + }) + + pctx.VariableFunc("TidyGlobalNoErrorChecks", func(ctx android.PackageVarContext) string { + return strings.Join(globalNoErrorCheckList, ",") + }) + // To reduce duplicate warnings from the same header files, // header-filter will contain only the module directory and // those specified by DEFAULT_TIDY_HEADER_DIRS. @@ -122,6 +167,7 @@ var DefaultLocalTidyChecks = []PathBasedTidyCheck{ {"hardware/qcom", tidyExternalVendor}, {"vendor/", tidyExternalVendor}, {"vendor/google", tidyDefault}, + {"vendor/google_arc/libs/org.chromium.arc.mojom", tidyExternalVendor}, {"vendor/google_devices", tidyExternalVendor}, } @@ -145,6 +191,22 @@ func TidyChecksForDir(dir string) string { return tidyDefault } +// Returns a globally disabled tidy checks, overriding locally selected checks. +func TidyGlobalNoChecks() string { + if len(globalNoCheckList) > 0 { + return ",${config.TidyGlobalNoChecks}" + } + return "" +} + +// Returns a globally allowed/no-error tidy checks, appended to -warnings-as-errors. +func TidyGlobalNoErrorChecks() string { + if len(globalNoErrorCheckList) > 0 { + return ",${config.TidyGlobalNoErrorChecks}" + } + return "" +} + func TidyFlagsForSrcFile(srcFile android.Path, flags string) string { // Disable clang-analyzer-* checks globally for generated source files // because some of them are too huge. Local .bp files can add wanted diff --git a/cc/config/toolchain.go b/cc/config/toolchain.go index 7175fdc1a..253bb06d2 100644 --- a/cc/config/toolchain.go +++ b/cc/config/toolchain.go @@ -36,19 +36,10 @@ type toolchainContext interface { Arch() android.Arch } -type conversionContext interface { - BazelConversionMode() bool -} - func FindToolchainWithContext(ctx toolchainContext) Toolchain { t, err := findToolchain(ctx.Os(), ctx.Arch()) if err != nil { - if c, ok := ctx.(conversionContext); ok && c.BazelConversionMode() { - // TODO(b/179123288): determine conversion for toolchain - return &toolchainX86_64{} - } else { - panic(err) - } + panic(err) } return t } diff --git a/cc/config/vndk.go b/cc/config/vndk.go index 492cd9844..dd612ce63 100644 --- a/cc/config/vndk.go +++ b/cc/config/vndk.go @@ -17,100 +17,9 @@ package config // List of VNDK libraries that have different core variant and vendor variant. // For these libraries, the vendor variants must be installed even if the device // has VndkUseCoreVariant set. -// TODO(b/150578172): clean up unstable and non-versioned aidl module +// Note that AIDL-generated modules must use vendor variants by default. var VndkMustUseVendorVariantList = []string{ - "android.hardware.authsecret-V1-ndk", - "android.hardware.authsecret-V1-ndk_platform", - "android.hardware.authsecret-ndk_platform", - "android.hardware.authsecret-unstable-ndk_platform", - "android.hardware.automotive.occupant_awareness-V1-ndk", - "android.hardware.automotive.occupant_awareness-V1-ndk_platform", - "android.hardware.automotive.occupant_awareness-ndk_platform", - "android.hardware.gnss-V1-ndk", - "android.hardware.gnss-V1-ndk_platform", - "android.hardware.gnss-ndk_platform", - "android.hardware.gnss-unstable-ndk_platform", - "android.hardware.health-V1-ndk", - "android.hardware.health-ndk", - "android.hardware.health.storage-V1-ndk", - "android.hardware.health.storage-V1-ndk_platform", - "android.hardware.health.storage-ndk_platform", - "android.hardware.health.storage-unstable-ndk_platform", - "android.hardware.identity-V2-ndk_platform", - "android.hardware.identity-V3-ndk", - "android.hardware.identity-V3-ndk_platform", - "android.hardware.identity-ndk_platform", - "android.hardware.light-V1-ndk", - "android.hardware.light-V1-ndk_platform", - "android.hardware.light-ndk_platform", - "android.hardware.memtrack-V1-ndk", - "android.hardware.memtrack-V1-ndk_platform", - "android.hardware.memtrack-ndk_platform", - "android.hardware.memtrack-unstable-ndk_platform", "android.hardware.nfc@1.2", - "android.hardware.oemlock-V1-ndk", - "android.hardware.oemlock-V1-ndk_platform", - "android.hardware.oemlock-ndk_platform", - "android.hardware.oemlock-unstable-ndk_platform", - "android.hardware.power-V1-ndk_platform", - "android.hardware.power-V2-ndk", - "android.hardware.power-V2-ndk_platform", - "android.hardware.power-ndk_platform", - "android.hardware.power.stats-V1-ndk", - "android.hardware.power.stats-V1-ndk_platform", - "android.hardware.power.stats-ndk_platform", - "android.hardware.power.stats-unstable-ndk_platform", - "android.hardware.rebootescrow-V1-ndk", - "android.hardware.rebootescrow-V1-ndk_platform", - "android.hardware.rebootescrow-ndk_platform", - "android.hardware.radio-V1-ndk", - "android.hardware.radio-V1-ndk_platform", - "android.hardware.radio.config-V1-ndk", - "android.hardware.radio.config-V1-ndk_platform", - "android.hardware.radio.data-V1-ndk", - "android.hardware.radio.data-V1-ndk_platform", - "android.hardware.radio.messaging-V1-ndk", - "android.hardware.radio.messaging-V1-ndk_platform", - "android.hardware.radio.modem-V1-ndk", - "android.hardware.radio.modem-V1-ndk_platform", - "android.hardware.radio.network-V1-ndk", - "android.hardware.radio.network-V1-ndk_platform", - "android.hardware.radio.sim-V1-ndk", - "android.hardware.radio.sim-V1-ndk_platform", - "android.hardware.radio.voice-V1-ndk", - "android.hardware.radio.voice-V1-ndk_platform", - "android.hardware.security.keymint-V1-ndk", - "android.hardware.security.keymint-V1-ndk_platform", - "android.hardware.security.keymint-ndk_platform", - "android.hardware.security.keymint-unstable-ndk_platform", - "android.hardware.security.secureclock-V1-ndk", - "android.hardware.security.secureclock-V1-ndk_platform", - "android.hardware.security.secureclock-ndk_platform", - "android.hardware.security.secureclock-unstable-ndk_platform", - "android.hardware.security.sharedsecret-V1-ndk", - "android.hardware.security.sharedsecret-V1-ndk_platform", - "android.hardware.security.sharedsecret-ndk_platform", - "android.hardware.security.sharedsecret-unstable-ndk_platform", - "android.hardware.vibrator-V1-ndk_platform", - "android.hardware.vibrator-V2-ndk", - "android.hardware.vibrator-V2-ndk_platform", - "android.hardware.vibrator-ndk_platform", - "android.hardware.weaver-V1-ndk", - "android.hardware.weaver-V1-ndk_platform", - "android.hardware.weaver-ndk_platform", - "android.hardware.weaver-unstable-ndk_platform", - "android.system.suspend-V1-ndk", - "android.system.keystore2-V1-ndk", - "android.se.omapi-V1-ndk_platform", - "android.se.omapi-ndk_platform", - "android.se.omapi-unstable-ndk_platform", - "android.hardware.wifi.hostapd-V1-ndk", - "android.hardware.wifi.hostapd-V1-ndk_platform", - "android.hardware.wifi.supplicant-V1-ndk", - "android.system.keystore2-V1-ndk_platform", - "android.system.keystore2-ndk_platform", - "android.system.keystore2-unstable-ndk_platform", - "android.system.suspend-V1-ndk_platform", "libbinder", "libcrypto", "libexpat", @@ -18,6 +18,7 @@ import ( "path/filepath" "strings" + "android/soong/bazel" "github.com/google/blueprint" "android/soong/android" @@ -169,6 +170,41 @@ func genLex(ctx android.ModuleContext, lexFile android.Path, outFile android.Mod }) } +type LexAttrs struct { + Srcs bazel.LabelListAttribute + Lexopts bazel.StringListAttribute +} + +type LexNames struct { + cSrcName bazel.LabelAttribute + srcName bazel.LabelAttribute +} + +func bp2BuildLex(ctx android.Bp2buildMutatorContext, moduleName string, ca compilerAttributes) LexNames { + names := LexNames{} + if !ca.lSrcs.IsEmpty() { + names.cSrcName = createLexTargetModule(ctx, moduleName+"_genlex_l", ca.lSrcs, ca.lexopts) + } + if !ca.llSrcs.IsEmpty() { + names.srcName = createLexTargetModule(ctx, moduleName+"_genlex_ll", ca.llSrcs, ca.lexopts) + } + return names +} + +func createLexTargetModule(ctx android.Bp2buildMutatorContext, name string, srcs bazel.LabelListAttribute, opts bazel.StringListAttribute) bazel.LabelAttribute { + ctx.CreateBazelTargetModule( + bazel.BazelTargetModuleProperties{ + Rule_class: "genlex", + Bzl_load_location: "//build/bazel/rules/cc:flex.bzl", + }, + android.CommonAttributes{Name: name}, + &LexAttrs{ + Srcs: srcs, + Lexopts: opts, + }) + return bazel.LabelAttribute{Value: &bazel.Label{Label: ":" + name}} +} + func genSysprop(ctx android.ModuleContext, syspropFile android.Path) (android.Path, android.Paths) { headerFile := android.PathForModuleGen(ctx, "sysprop", "include", syspropFile.Rel()+".h") publicHeaderFile := android.PathForModuleGen(ctx, "sysprop/public", "include", syspropFile.Rel()+".h") diff --git a/cc/installer.go b/cc/installer.go index 2522610d9..e2c0e7b8d 100644 --- a/cc/installer.go +++ b/cc/installer.go @@ -31,7 +31,7 @@ type InstallerProperties struct { Install_in_root *bool `android:"arch_variant"` // Install output directly in {partition}/xbin - Install_in_xbin *bool `android:"arch_vvariant"` + Install_in_xbin *bool `android:"arch_variant"` } type installLocation int diff --git a/cc/libbuildversion/tests/Android.bp b/cc/libbuildversion/tests/Android.bp index 0e97fedff..c616a3351 100644 --- a/cc/libbuildversion/tests/Android.bp +++ b/cc/libbuildversion/tests/Android.bp @@ -35,6 +35,16 @@ cc_defaults { dir: "host/", }, }, + linux_musl_x86: { + dist: { + dir: "host32/", + }, + }, + linux_musl_x86_64: { + dist: { + dir: "host/", + }, + }, linux_glibc_x86: { dist: { dir: "host32/", diff --git a/cc/library.go b/cc/library.go index 0abcb6f39..c445a42ca 100644 --- a/cc/library.go +++ b/cc/library.go @@ -110,6 +110,9 @@ type LibraryProperties struct { // Run checks on all APIs (in addition to the ones referred by // one of exported ELF symbols.) Check_all_apis *bool + + // Extra flags passed to header-abi-diff + Diff_flags []string } // Inject boringssl hash into the shared library. This is only intended for use by external/boringssl. @@ -289,7 +292,7 @@ func libraryBp2Build(ctx android.TopDownMutatorContext, m *Module) { baseAttributes := bp2BuildParseBaseProps(ctx, m) compilerAttrs := baseAttributes.compilerAttributes linkerAttrs := baseAttributes.linkerAttributes - exportedIncludes := bp2BuildParseExportedIncludes(ctx, m, compilerAttrs.includes) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, m, &compilerAttrs.includes) srcs := compilerAttrs.srcs @@ -642,18 +645,18 @@ type libraryDecorator struct { } type ccLibraryBazelHandler struct { - android.BazelHandler - module *Module } +var _ BazelHandler = (*ccLibraryBazelHandler)(nil) + // generateStaticBazelBuildActions constructs the StaticLibraryInfo Soong // provider from a Bazel shared library's CcInfo provider. -func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { +func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) { rootStaticArchives := ccInfo.RootStaticArchives if len(rootStaticArchives) != 1 { ctx.ModuleErrorf("expected exactly one root archive file for '%s', but got %s", label, rootStaticArchives) - return false + return } outputFilePath := android.PathForBazelOut(ctx, rootStaticArchives[0]) handler.module.outputFile = android.OptionalPathForPath(outputFilePath) @@ -679,17 +682,17 @@ func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx androi Build(), }) - return true + return } // generateSharedBazelBuildActions constructs the SharedLibraryInfo Soong // provider from a Bazel shared library's CcInfo provider. -func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { +func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) { rootDynamicLibraries := ccInfo.RootDynamicLibraries if len(rootDynamicLibraries) != 1 { ctx.ModuleErrorf("expected exactly one root dynamic library file for '%s', but got %s", label, rootDynamicLibraries) - return false + return } outputFilePath := android.PathForBazelOut(ctx, rootDynamicLibraries[0]) handler.module.outputFile = android.OptionalPathForPath(outputFilePath) @@ -709,30 +712,27 @@ func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx androi // TODO(b/190524881): Include transitive static libraries in this provider to support // static libraries with deps. The provider key for this is TransitiveStaticLibrariesForOrdering. }) - return true } -func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (handler *ccLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) - return false - } - if !ok { - return ok + return } if handler.module.static() { - if ok := handler.generateStaticBazelBuildActions(ctx, label, ccInfo); !ok { - return false - } + handler.generateStaticBazelBuildActions(ctx, label, ccInfo) } else if handler.module.Shared() { - if ok := handler.generateSharedBazelBuildActions(ctx, label, ccInfo); !ok { - return false - } + handler.generateSharedBazelBuildActions(ctx, label, ccInfo) } else { - return false + ctx.ModuleErrorf("Unhandled bazel case for %s (neither shared nor static!)", ctx.ModuleName()) } handler.module.linker.(*libraryDecorator).setFlagExporterInfoFromCcInfo(ctx, ccInfo) @@ -746,7 +746,6 @@ func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.Modu // implementation. i.(*libraryDecorator).collectedSnapshotHeaders = android.Paths{} } - return ok } func (library *libraryDecorator) setFlagExporterInfoFromCcInfo(ctx android.ModuleContext, ccInfo cquery.CcInfo) { @@ -926,7 +925,6 @@ func (library *libraryDecorator) linkerFlags(ctx ModuleContext, flags Flags) Fla if ctx.Darwin() { f = append(f, "-dynamiclib", - "-single_module", "-install_name @rpath/"+libName+flags.Toolchain.ShlibSuffix(), ) if ctx.Arch().ArchType == android.X86 { @@ -1638,6 +1636,7 @@ func (library *libraryDecorator) linkSAbiDumpFiles(ctx ModuleContext, objs Objec if refAbiDumpFile != nil { library.sAbiDiff = sourceAbiDiff(ctx, library.sAbiOutputFile.Path(), refAbiDumpFile, fileName, exportedHeaderFlags, + library.Properties.Header_abi_checker.Diff_flags, Bool(library.Properties.Header_abi_checker.Check_all_apis), ctx.IsLlndk(), ctx.isNdk(ctx.Config()), ctx.IsVndkExt()) } @@ -2344,7 +2343,7 @@ func createPerApiVersionVariations(mctx android.BottomUpMutatorContext, minSdkVe } } -func CanBeOrLinkAgainstVersionVariants(module interface { +func canBeOrLinkAgainstVersionVariants(module interface { Host() bool InRamdisk() bool InVendorRamdisk() bool @@ -2352,15 +2351,14 @@ func CanBeOrLinkAgainstVersionVariants(module interface { return !module.Host() && !module.InRamdisk() && !module.InVendorRamdisk() } -func CanBeVersionVariant(module interface { +func canBeVersionVariant(module interface { Host() bool InRamdisk() bool InVendorRamdisk() bool - InRecovery() bool CcLibraryInterface() bool Shared() bool }) bool { - return CanBeOrLinkAgainstVersionVariants(module) && + return canBeOrLinkAgainstVersionVariants(module) && module.CcLibraryInterface() && module.Shared() } @@ -2371,37 +2369,41 @@ func moduleLibraryInterface(module blueprint.Module) libraryInterface { return nil } -// versionSelector normalizes the versions in the Stubs.Versions property into MutatedProperties.AllStubsVersions. -func versionSelectorMutator(mctx android.BottomUpMutatorContext) { - if library := moduleLibraryInterface(mctx.Module()); library != nil && CanBeVersionVariant(mctx.Module().(*Module)) { - if library.buildShared() { - versions := library.stubsVersions(mctx) - if len(versions) > 0 { - normalizeVersions(mctx, versions) - if mctx.Failed() { - return - } - // Set the versions on the pre-mutated module so they can be read by any llndk modules that - // depend on the implementation library and haven't been mutated yet. - library.setAllStubsVersions(versions) - } - } +// setStubsVersions normalizes the versions in the Stubs.Versions property into MutatedProperties.AllStubsVersions. +func setStubsVersions(mctx android.BottomUpMutatorContext, library libraryInterface, module *Module) { + if !library.buildShared() || !canBeVersionVariant(module) { + return + } + versions := library.stubsVersions(mctx) + if len(versions) <= 0 { + return } + normalizeVersions(mctx, versions) + if mctx.Failed() { + return + } + // Set the versions on the pre-mutated module so they can be read by any llndk modules that + // depend on the implementation library and haven't been mutated yet. + library.setAllStubsVersions(versions) } // versionMutator splits a module into the mandatory non-stubs variant // (which is unnamed) and zero or more stubs variants. func versionMutator(mctx android.BottomUpMutatorContext) { - if library := moduleLibraryInterface(mctx.Module()); library != nil && CanBeVersionVariant(mctx.Module().(*Module)) { + if mctx.Os() != android.Android { + return + } + + m, ok := mctx.Module().(*Module) + if library := moduleLibraryInterface(mctx.Module()); library != nil && canBeVersionVariant(m) { + setStubsVersions(mctx, library, m) + createVersionVariations(mctx, library.allStubsVersions()) return } - if m, ok := mctx.Module().(*Module); ok { + if ok { if m.SplitPerApiLevel() && m.IsSdkVariant() { - if mctx.Os() != android.Android { - return - } createPerApiVersionVariations(mctx, m.MinSdkVersion()) } } @@ -2433,7 +2435,6 @@ func maybeInjectBoringSSLHash(ctx android.ModuleContext, outputFile android.Modu rule := android.NewRuleBuilder(pctx, ctx) rule.Command(). BuiltTool("bssl_inject_hash"). - Flag("-sha256"). FlagWithInput("-in-object ", outputFile). FlagWithOutput("-o ", hashedOutputfile) rule.Build("injectCryptoHash", "inject crypto hash") @@ -2447,7 +2448,7 @@ func sharedOrStaticLibraryBp2Build(ctx android.TopDownMutatorContext, module *Mo compilerAttrs := baseAttributes.compilerAttributes linkerAttrs := baseAttributes.linkerAttributes - exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, compilerAttrs.includes) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, &compilerAttrs.includes) // Append shared/static{} stanza properties. These won't be specified on // cc_library_* itself, but may be specified in cc_defaults that this module diff --git a/cc/library_headers.go b/cc/library_headers.go index 41ebcc766..970d8d1a6 100644 --- a/cc/library_headers.go +++ b/cc/library_headers.go @@ -17,6 +17,7 @@ package cc import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) func init() { @@ -47,28 +48,30 @@ func RegisterLibraryHeadersBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("cc_prebuilt_library_headers", prebuiltLibraryHeaderFactory) } -type libraryHeaderBazelHander struct { - android.BazelHandler - +type libraryHeaderBazelHandler struct { module *Module library *libraryDecorator } -func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +var _ BazelHandler = (*libraryHeaderBazelHandler)(nil) + +func (handler *libraryHeaderBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (h *libraryHeaderBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) - return false - } - if !ok { - return false + ctx.ModuleErrorf(err.Error()) + return } outputPaths := ccInfo.OutputFiles if len(outputPaths) != 1 { ctx.ModuleErrorf("expected exactly one output file for %q, but got %q", label, outputPaths) - return false + return } outputPath := android.PathForBazelOut(ctx, outputPaths[0]) @@ -83,8 +86,6 @@ func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleC // validation will fail. For now, set this to an empty list. // TODO(cparsons): More closely mirror the collectHeadersForSnapshot implementation. h.library.collectedSnapshotHeaders = android.Paths{} - - return true } // cc_library_headers contains a set of c/c++ headers which are imported by @@ -96,7 +97,7 @@ func LibraryHeaderFactory() android.Module { library.HeaderOnly() module.sdkMemberTypes = []android.SdkMemberType{headersLibrarySdkMemberType} module.bazelable = true - module.bazelHandler = &libraryHeaderBazelHander{module: module, library: library} + module.bazelHandler = &libraryHeaderBazelHandler{module: module, library: library} return module.Init() } @@ -122,14 +123,14 @@ type bazelCcLibraryHeadersAttributes struct { func libraryHeadersBp2Build(ctx android.TopDownMutatorContext, module *Module) { baseAttributes := bp2BuildParseBaseProps(ctx, module) - exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, baseAttributes.includes) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, &baseAttributes.includes) linkerAttrs := baseAttributes.linkerAttributes + (&linkerAttrs.deps).Append(linkerAttrs.dynamicDeps) attrs := &bazelCcLibraryHeadersAttributes{ Export_includes: exportedIncludes.Includes, Export_absolute_includes: exportedIncludes.AbsoluteIncludes, Export_system_includes: exportedIncludes.SystemIncludes, - Implementation_deps: linkerAttrs.implementationDeps, Deps: linkerAttrs.deps, System_dynamic_deps: linkerAttrs.systemDynamicDeps, Hdrs: baseAttributes.hdrs, diff --git a/cc/library_sdk_member.go b/cc/library_sdk_member.go index 8988de2e5..1bcbdc55d 100644 --- a/cc/library_sdk_member.go +++ b/cc/library_sdk_member.go @@ -27,32 +27,33 @@ import ( var sharedLibrarySdkMemberType = &librarySdkMemberType{ SdkMemberTypeBase: android.SdkMemberTypeBase{ - PropertyName: "native_shared_libs", - SupportsSdk: true, - HostOsDependent: true, + PropertyName: "native_shared_libs", + SupportsSdk: true, + HostOsDependent: true, + SupportedLinkageNames: []string{"shared"}, }, prebuiltModuleType: "cc_prebuilt_library_shared", - linkTypes: []string{"shared"}, } var staticLibrarySdkMemberType = &librarySdkMemberType{ SdkMemberTypeBase: android.SdkMemberTypeBase{ - PropertyName: "native_static_libs", - SupportsSdk: true, - HostOsDependent: true, + PropertyName: "native_static_libs", + SupportsSdk: true, + HostOsDependent: true, + SupportedLinkageNames: []string{"static"}, }, prebuiltModuleType: "cc_prebuilt_library_static", - linkTypes: []string{"static"}, } var staticAndSharedLibrarySdkMemberType = &librarySdkMemberType{ SdkMemberTypeBase: android.SdkMemberTypeBase{ - PropertyName: "native_libs", - SupportsSdk: true, - HostOsDependent: true, + PropertyName: "native_libs", + OverridesPropertyNames: map[string]bool{"native_shared_libs": true, "native_static_libs": true}, + SupportsSdk: true, + HostOsDependent: true, + SupportedLinkageNames: []string{"static", "shared"}, }, prebuiltModuleType: "cc_prebuilt_library", - linkTypes: []string{"static", "shared"}, } func init() { @@ -69,9 +70,6 @@ type librarySdkMemberType struct { noOutputFiles bool // True if there are no srcs files. - // The set of link types supported. A set of "static", "shared", or nil to - // skip link type variations. - linkTypes []string } func (mt *librarySdkMemberType) AddDependencies(ctx android.SdkDependencyContext, dependencyTag blueprint.DependencyTag, names []string) { @@ -165,12 +163,12 @@ func (mt *librarySdkMemberType) AddDependencies(ctx android.SdkDependencyContext // Add any additional dependencies needed. variations = append(variations, dependency.imageVariations...) - if mt.linkTypes == nil { + if mt.SupportedLinkageNames == nil { // No link types are supported so add a dependency directly. ctx.AddFarVariationDependencies(variations, dependencyTag, name) } else { // Otherwise, add a dependency on each supported link type in turn. - for _, linkType := range mt.linkTypes { + for _, linkType := range mt.SupportedLinkageNames { libVariations := append(variations, blueprint.Variation{Mutator: "link", Variation: linkType}) // If this is for the device and a shared link type then add a dependency onto the diff --git a/cc/library_stub.go b/cc/library_stub.go new file mode 100644 index 000000000..4d0148df4 --- /dev/null +++ b/cc/library_stub.go @@ -0,0 +1,163 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + "android/soong/android" + "android/soong/multitree" +) + +func init() { + RegisterLibraryStubBuildComponents(android.InitRegistrationContext) +} + +func RegisterLibraryStubBuildComponents(ctx android.RegistrationContext) { + // cc_api_stub_library shares a lot of ndk_library, and this will be refactored later + ctx.RegisterModuleType("cc_api_stub_library", CcApiStubLibraryFactory) + ctx.RegisterModuleType("cc_api_contribution", CcApiContributionFactory) +} + +func CcApiStubLibraryFactory() android.Module { + module, decorator := NewLibrary(android.DeviceSupported) + apiStubDecorator := &apiStubDecorator{ + libraryDecorator: decorator, + } + apiStubDecorator.BuildOnlyShared() + + module.compiler = apiStubDecorator + module.linker = apiStubDecorator + module.installer = nil + module.library = apiStubDecorator + module.Properties.HideFromMake = true // TODO: remove + + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth) + module.AddProperties(&module.Properties, + &apiStubDecorator.properties, + &apiStubDecorator.MutatedProperties, + &apiStubDecorator.apiStubLibraryProperties) + return module +} + +type apiStubLiraryProperties struct { + Imported_includes []string `android:"path"` +} + +type apiStubDecorator struct { + *libraryDecorator + properties libraryProperties + apiStubLibraryProperties apiStubLiraryProperties +} + +func (compiler *apiStubDecorator) stubsVersions(ctx android.BaseMutatorContext) []string { + firstVersion := String(compiler.properties.First_version) + return ndkLibraryVersions(ctx, android.ApiLevelOrPanic(ctx, firstVersion)) +} + +func (decorator *apiStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects { + if decorator.stubsVersion() == "" { + decorator.setStubsVersion("current") + } // TODO: fix + symbolFile := String(decorator.properties.Symbol_file) + nativeAbiResult := parseNativeAbiDefinition(ctx, symbolFile, + android.ApiLevelOrPanic(ctx, decorator.stubsVersion()), + "") + return compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc) +} + +func (decorator *apiStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objects Objects) android.Path { + decorator.reexportDirs(android.PathsForModuleSrc(ctx, decorator.apiStubLibraryProperties.Imported_includes)...) + return decorator.libraryDecorator.link(ctx, flags, deps, objects) +} + +func init() { + pctx.HostBinToolVariable("gen_api_surface_build_files", "gen_api_surface_build_files") +} + +type CcApiContribution struct { + android.ModuleBase + properties ccApiContributionProperties +} + +type ccApiContributionProperties struct { + Symbol_file *string `android:"path"` + First_version *string + Export_include_dir *string +} + +func CcApiContributionFactory() android.Module { + module := &CcApiContribution{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + return module +} + +// Do some simple validations +// Majority of the build rules will be created in the ctx of the api surface this module contributes to +func (contrib *CcApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) { + if contrib.properties.Symbol_file == nil { + ctx.PropertyErrorf("symbol_file", "%v does not have symbol file", ctx.ModuleName()) + } + if contrib.properties.First_version == nil { + ctx.PropertyErrorf("first_version", "%v does not have first_version for stub variants", ctx.ModuleName()) + } +} + +// Path is out/soong/.export/ but will be different in final multi-tree layout +func outPathApiSurface(ctx android.ModuleContext, myModuleName string, pathComponent string) android.OutputPath { + return android.PathForOutput(ctx, ".export", ctx.ModuleName(), myModuleName, pathComponent) +} + +func (contrib *CcApiContribution) CopyFilesWithTag(apiSurfaceContext android.ModuleContext) map[string]android.Paths { + // copy map.txt for now + // hardlinks cannot be created since nsjail creates a different mountpoint for out/ + myDir := apiSurfaceContext.OtherModuleDir(contrib) + genMapTxt := outPathApiSurface(apiSurfaceContext, contrib.Name(), String(contrib.properties.Symbol_file)) + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Description: "import map.txt file", + Input: android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Symbol_file)), + Output: genMapTxt, + }) + + outputs := make(map[string]android.Paths) + outputs["map"] = []android.Path{genMapTxt} + + if contrib.properties.Export_include_dir != nil { + includeDir := android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Export_include_dir)) + outputs["export_include_dir"] = []android.Path{includeDir} + } + return outputs +} + +var _ multitree.ApiContribution = (*CcApiContribution)(nil) + +/* +func (contrib *CcApiContribution) GenerateBuildFiles(apiSurfaceContext android.ModuleContext) android.Paths { + genAndroidBp := outPathApiSurface(apiSurfaceContext, contrib.Name(), "Android.bp") + + // generate Android.bp + apiSurfaceContext.Build(pctx, android.BuildParams{ + Rule: genApiSurfaceBuildFiles, + Description: "generate API surface build files", + Outputs: []android.WritablePath{genAndroidBp}, + Args: map[string]string{ + "name": contrib.Name() + "." + apiSurfaceContext.ModuleName(), //e.g. liblog.ndk + "symbol_file": String(contrib.properties.Symbol_file), + "first_version": String(contrib.properties.First_version), + }, + }) + return []android.Path{genAndroidBp} +} +*/ diff --git a/cc/library_stub_test.go b/cc/library_stub_test.go new file mode 100644 index 000000000..15b56d227 --- /dev/null +++ b/cc/library_stub_test.go @@ -0,0 +1,108 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import ( + _ "fmt" + _ "sort" + + "testing" + + "android/soong/android" + "android/soong/multitree" +) + +func TestCcApiStubLibraryOutputFiles(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + outputs := result.ModuleForTests("foo", "android_arm64_armv8-a_shared").AllOutputs() + expected_file_suffixes := []string{".c", "stub.map", ".o", ".so"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in output", true, android.SuffixInList(outputs, expected_file_suffix)) + } +} + +func TestCcApiStubLibraryVariants(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := prepareForCcTest.RunTestWithBp(t, bp) + variants := result.ModuleVariantsForTests("foo") + expected_variants := []string{"29", "30", "S", "Tiramisu"} //TODO: make this test deterministic by using fixtures + for _, expected_variant := range expected_variants { + android.AssertBoolEquals(t, expected_variant+" variant not found in foo", true, android.SubstringInList(variants, expected_variant)) + } +} + +func TestCcLibraryUsesCcApiStubLibrary(t *testing.T) { + bp := ` + cc_api_stub_library { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + cc_library { + name: "foo_user", + shared_libs: [ + "foo#29", + ], + } + + ` + prepareForCcTest.RunTestWithBp(t, bp) +} + +func TestApiSurfaceOutputs(t *testing.T) { + bp := ` + api_surface { + name: "mysdk", + contributions: [ + "foo", + ], + } + + cc_api_contribution { + name: "foo", + symbol_file: "foo.map.txt", + first_version: "29", + } + ` + result := android.GroupFixturePreparers( + prepareForCcTest, + multitree.PrepareForTestWithApiSurface, + ).RunTestWithBp(t, bp) + mysdk := result.ModuleForTests("mysdk", "") + + actual_surface_inputs := mysdk.Rule("phony").BuildParams.Inputs.Strings() + expected_file_suffixes := []string{"mysdk/foo/foo.map.txt"} + for _, expected_file_suffix := range expected_file_suffixes { + android.AssertBoolEquals(t, expected_file_suffix+" file not found in input", true, android.SuffixInList(actual_surface_inputs, expected_file_suffix)) + } + + // check args/inputs to rule + /*api_surface_gen_rule_args := result.ModuleForTests("mysdk", "").Rule("genApiSurfaceBuildFiles").Args + android.AssertStringEquals(t, "name", "foo.mysdk", api_surface_gen_rule_args["name"]) + android.AssertStringEquals(t, "symbol_file", "foo.map.txt", api_surface_gen_rule_args["symbol_file"])*/ +} diff --git a/cc/linkable.go b/cc/linkable.go index 6bec30c54..2316d865c 100644 --- a/cc/linkable.go +++ b/cc/linkable.go @@ -22,16 +22,9 @@ type PlatformSanitizeable interface { // than left undefined. IsSanitizerExplicitlyDisabled(t SanitizerType) bool - // SanitizeDep returns the value of the SanitizeDep flag, which is set if a module is a dependency of a - // sanitized module. - SanitizeDep() bool - // SetSanitizer enables or disables the specified sanitizer type if it's supported, otherwise this should panic. SetSanitizer(t SanitizerType, b bool) - // SetSanitizerDep returns true if the module is statically linked. - SetSanitizeDep(b bool) - // StaticallyLinked returns true if the module is statically linked. StaticallyLinked() bool @@ -113,6 +106,9 @@ type LinkableInterface interface { UnstrippedOutputFile() android.Path CoverageFiles() android.Paths + // CoverageOutputFile returns the output archive of gcno coverage information files. + CoverageOutputFile() android.OptionalPath + NonCcVariants() bool SelectedStl() string @@ -140,6 +136,12 @@ type LinkableInterface interface { UseSdk() bool + // IsNdk returns true if the library is in the configs known NDK list. + IsNdk(config android.Config) bool + + // IsStubs returns true if the this is a stubs library. + IsStubs() bool + // IsLlndk returns true for both LLNDK (public) and LLNDK-private libs. IsLlndk() bool diff --git a/cc/linker.go b/cc/linker.go index bea65d441..76a60ca7f 100644 --- a/cc/linker.go +++ b/cc/linker.go @@ -227,6 +227,9 @@ type BaseLinkerProperties struct { // local file name to pass to the linker as --dynamic-list Dynamic_list *string `android:"path,arch_variant"` + // local files to pass to the linker as --script + Linker_scripts []string `android:"path,arch_variant"` + // list of static libs that should not be used to build this module Exclude_static_libs []string `android:"arch_variant"` @@ -386,9 +389,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { } deps.SystemSharedLibs = linker.Properties.System_shared_libs - // In Bazel conversion mode, variations have not been specified, so SystemSharedLibs may - // inaccuarately appear unset, which can cause issues with circular dependencies. - if deps.SystemSharedLibs == nil && !ctx.BazelConversionMode() { + if deps.SystemSharedLibs == nil { // Provide a default system_shared_libs if it is unspecified. Note: If an // empty list [] is specified, it implies that the module declines the // default system_shared_libs. @@ -398,7 +399,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { if ctx.toolchain().Bionic() { // libclang_rt.builtins has to be last on the command line if !Bool(linker.Properties.No_libcrt) && !ctx.header() { - deps.LateStaticLibs = append(deps.LateStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) + deps.UnexportedStaticLibs = append(deps.UnexportedStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) } if inList("libdl", deps.SharedLibs) { @@ -421,7 +422,7 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { } } else if ctx.toolchain().Musl() { if !Bool(linker.Properties.No_libcrt) && !ctx.header() { - deps.LateStaticLibs = append(deps.LateStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) + deps.UnexportedStaticLibs = append(deps.UnexportedStaticLibs, config.BuiltinsRuntimeLibrary(ctx.toolchain())) } } @@ -435,11 +436,6 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps { } func (linker *baseLinker) useClangLld(ctx ModuleContext) bool { - // Clang lld is not ready for for Darwin host executables yet. - // See https://lld.llvm.org/AtomLLD.html for status of lld for Mach-O. - if ctx.Darwin() { - return false - } if linker.Properties.Use_clang_lld != nil { return Bool(linker.Properties.Use_clang_lld) } @@ -529,10 +525,6 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { } } - if ctx.toolchain().LibclangRuntimeLibraryArch() != "" { - flags.Global.LdFlags = append(flags.Global.LdFlags, "-Wl,--exclude-libs="+config.BuiltinsRuntimeLibrary(ctx.toolchain())+".a") - } - CheckBadLinkerFlags(ctx, "ldflags", linker.Properties.Ldflags) flags.Local.LdFlags = append(flags.Local.LdFlags, proptools.NinjaAndShellEscapeList(linker.Properties.Ldflags)...) @@ -602,6 +594,17 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags { flags.LdFlagsDeps = append(flags.LdFlagsDeps, dynamicList.Path()) } } + + linkerScriptPaths := android.PathsForModuleSrc(ctx, linker.Properties.Linker_scripts) + if len(linkerScriptPaths) > 0 && (ctx.Darwin() || ctx.Windows()) { + ctx.PropertyErrorf("linker_scripts", "Only supported for ELF files") + } else { + for _, linkerScriptPath := range linkerScriptPaths { + flags.Local.LdFlags = append(flags.Local.LdFlags, + "-Wl,--script,"+linkerScriptPath.String()) + flags.LdFlagsDeps = append(flags.LdFlagsDeps, linkerScriptPath) + } + } } return flags diff --git a/cc/makevars.go b/cc/makevars.go index 6752f8cde..815443626 100644 --- a/cc/makevars.go +++ b/cc/makevars.go @@ -25,7 +25,7 @@ import ( ) var ( - modulesAddedWallKey = android.NewOnceKey("ModulesAddedWall") + modulesWarningsAllowedKey = android.NewOnceKey("ModulesWarningsAllowed") modulesUsingWnoErrorKey = android.NewOnceKey("ModulesUsingWnoError") modulesMissingProfileFileKey = android.NewOnceKey("ModulesMissingProfileFile") ) @@ -119,7 +119,7 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("LSDUMP_PATHS", strings.Join(lsdumpPaths, " ")) ctx.Strict("ANDROID_WARNING_ALLOWED_PROJECTS", makeStringOfWarningAllowedProjects()) - ctx.Strict("SOONG_MODULES_ADDED_WALL", makeStringOfKeys(ctx, modulesAddedWallKey)) + ctx.Strict("SOONG_MODULES_WARNINGS_ALLOWED", makeStringOfKeys(ctx, modulesWarningsAllowedKey)) ctx.Strict("SOONG_MODULES_USING_WNO_ERROR", makeStringOfKeys(ctx, modulesUsingWnoErrorKey)) ctx.Strict("SOONG_MODULES_MISSING_PGO_PROFILE_FILE", makeStringOfKeys(ctx, modulesMissingProfileFileKey)) diff --git a/cc/ndk_library.go b/cc/ndk_library.go index 5ef41eae5..0879257a5 100644 --- a/cc/ndk_library.go +++ b/cc/ndk_library.go @@ -93,7 +93,7 @@ var ( type libraryProperties struct { // Relative path to the symbol map. // An example file can be seen here: TODO(danalbert): Make an example. - Symbol_file *string + Symbol_file *string `android:"path"` // The first API level a library was available. A library will be generated // for every API level beginning with this one. @@ -284,6 +284,10 @@ func parseNativeAbiDefinition(ctx ModuleContext, symbolFile string, } func compileStubLibrary(ctx ModuleContext, flags Flags, src android.Path) Objects { + // libc/libm stubs libraries end up mismatching with clang's internal definition of these + // functions (which have noreturn attributes and other things). Because we just want to create a + // stub with symbol definitions, and types aren't important in C, ignore the mismatch. + flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, "-fno-builtin") return compileObjs(ctx, flagsToBuilderFlags(flags), "", android.Paths{src}, nil, nil, nil, nil) } diff --git a/cc/object.go b/cc/object.go index bd5bd4517..65a11e005 100644 --- a/cc/object.go +++ b/cc/object.go @@ -19,6 +19,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) // @@ -37,7 +38,6 @@ var ccObjectSdkMemberType = &librarySdkMemberType{ SupportsSdk: true, }, prebuiltModuleType: "cc_prebuilt_object", - linkTypes: nil, } type objectLinker struct { @@ -46,23 +46,30 @@ type objectLinker struct { } type objectBazelHandler struct { - android.BazelHandler - module *Module } -func (handler *objectBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +var _ BazelHandler = (*objectBazelHandler)(nil) + +func (handler *objectBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { bazelCtx := ctx.Config().BazelContext - objPaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - if len(objPaths) != 1 { - ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths) - return false - } + bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} - handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0])) +func (handler *objectBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + bazelCtx := ctx.Config().BazelContext + objPaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + if len(objPaths) != 1 { + ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths) + return } - return ok + + handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0])) } type ObjectLinkerProperties struct { @@ -41,7 +41,6 @@ var pgoProfileProjectsConfigKey = android.NewOnceKey("PgoProfileProjects") const profileInstrumentFlag = "-fprofile-generate=/data/local/tmp" const profileUseInstrumentFormat = "-fprofile-use=%s" -const profileUseSamplingFormat = "-fprofile-sample-accurate -fprofile-sample-use=%s" func getPgoProfileProjects(config android.DeviceConfig) []string { return config.OnceStringSlice(pgoProfileProjectsConfigKey, func() []string { @@ -56,12 +55,11 @@ func recordMissingProfileFile(ctx BaseModuleContext, missing string) { type PgoProperties struct { Pgo struct { Instrumentation *bool - Sampling *bool `android:"arch_variant"` Profile_file *string `android:"arch_variant"` Benchmarks []string Enable_profile_use *bool `android:"arch_variant"` // Additional compiler flags to use when building this module - // for profiling (either instrumentation or sampling). + // for profiling. Cflags []string `android:"arch_variant"` } `android:"arch_variant"` @@ -79,10 +77,6 @@ func (props *PgoProperties) isInstrumentation() bool { return props.Pgo.Instrumentation != nil && *props.Pgo.Instrumentation == true } -func (props *PgoProperties) isSampling() bool { - return props.Pgo.Sampling != nil && *props.Pgo.Sampling == true -} - func (pgo *pgo) props() []interface{} { return []interface{}{&pgo.Properties} } @@ -135,18 +129,8 @@ func (props *PgoProperties) getPgoProfileFile(ctx BaseModuleContext) android.Opt return android.OptionalPathForPath(nil) } -func (props *PgoProperties) profileUseFlag(ctx ModuleContext, file string) string { - if props.isInstrumentation() { - return fmt.Sprintf(profileUseInstrumentFormat, file) - } - if props.isSampling() { - return fmt.Sprintf(profileUseSamplingFormat, file) - } - return "" -} - func (props *PgoProperties) profileUseFlags(ctx ModuleContext, file string) []string { - flags := []string{props.profileUseFlag(ctx, file)} + flags := []string{fmt.Sprintf(profileUseInstrumentFormat, file)} flags = append(flags, profileUseOtherFlags...) return flags } @@ -169,19 +153,14 @@ func (props *PgoProperties) addProfileUseFlags(ctx ModuleContext, flags Flags) F // if profileFile gets updated flags.CFlagsDeps = append(flags.CFlagsDeps, profileFilePath) flags.LdFlagsDeps = append(flags.LdFlagsDeps, profileFilePath) - - if props.isSampling() { - flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,-mllvm,-no-warn-sample-unused=true") - } } return flags } func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool { isInstrumentation := props.isInstrumentation() - isSampling := props.isSampling() - profileKindPresent := isInstrumentation || isSampling + profileKindPresent := isInstrumentation filePresent := props.Pgo.Profile_file != nil benchmarksPresent := len(props.Pgo.Benchmarks) > 0 @@ -194,7 +173,7 @@ func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool { if !profileKindPresent || !filePresent { var missing []string if !profileKindPresent { - missing = append(missing, "profile kind (either \"instrumentation\" or \"sampling\" property)") + missing = append(missing, "profile kind") } if !filePresent { missing = append(missing, "profile_file property") @@ -208,14 +187,6 @@ func (props *PgoProperties) isPGO(ctx BaseModuleContext) bool { ctx.ModuleErrorf("Instrumentation PGO specification is missing benchmark property") } - if isSampling { - ctx.ModuleErrorf("Sampling PGO is deprecated, use AFDO instead") - } - - if isSampling && isInstrumentation { - ctx.PropertyErrorf("pgo", "Exactly one of \"instrumentation\" and \"sampling\" properties must be set") - } - return true } diff --git a/cc/prebuilt.go b/cc/prebuilt.go index f54c6f8d6..8c404d30f 100644 --- a/cc/prebuilt.go +++ b/cc/prebuilt.go @@ -20,6 +20,7 @@ import ( "android/soong/android" "android/soong/bazel" + "android/soong/bazel/cquery" ) func init() { @@ -54,6 +55,13 @@ type prebuiltLinkerProperties struct { // This is needed only if this library is linked by other modules in build time. // Only makes sense for the Windows target. Windows_import_lib *string `android:"path,arch_variant"` + + // MixedBuildsDisabled is true if and only if building this prebuilt is explicitly disabled in mixed builds for either + // its static or shared version on the current build variant. This is to prevent Bazel targets for build variants with + // which either the static or shared version is incompatible from participating in mixed buiods. Please note that this + // is an override and does not fully determine whether Bazel or Soong will be used. For the full determination, see + // cc.ProcessBazelQueryResponse, cc.QueueBazelCall, and cc.MixedBuildsDisabled. + MixedBuildsDisabled bool `blueprint:"mutated"` } type prebuiltLinker struct { @@ -243,6 +251,7 @@ func (p *prebuiltLibraryLinker) nativeCoverage() bool { func (p *prebuiltLibraryLinker) disablePrebuilt() { p.properties.Srcs = nil + p.properties.MixedBuildsDisabled = true } // Implements versionedInterface @@ -254,6 +263,7 @@ func NewPrebuiltLibrary(hod android.HostOrDeviceSupported, srcsProperty string) module, library := NewLibrary(hod) module.compiler = nil module.bazelable = true + module.bazelHandler = &prebuiltLibraryBazelHandler{module: module, library: library} prebuilt := &prebuiltLibraryLinker{ libraryDecorator: library, @@ -309,8 +319,6 @@ func PrebuiltSharedTestLibraryFactory() android.Module { func NewPrebuiltSharedLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { module, library := NewPrebuiltLibrary(hod, "srcs") library.BuildOnlyShared() - module.bazelable = true - module.bazelHandler = &prebuiltSharedLibraryBazelHandler{module: module, library: library} // Prebuilt shared libraries can be included in APEXes android.InitApexModule(module) @@ -328,8 +336,7 @@ func PrebuiltStaticLibraryFactory() android.Module { func NewPrebuiltStaticLibrary(hod android.HostOrDeviceSupported) (*Module, *libraryDecorator) { module, library := NewPrebuiltLibrary(hod, "srcs") library.BuildOnlyStatic() - module.bazelable = true - module.bazelHandler = &prebuiltStaticLibraryBazelHandler{module: module, library: library} + return module, library } @@ -354,7 +361,7 @@ func prebuiltLibraryBp2Build(ctx android.TopDownMutatorContext, module *Module) func prebuiltLibraryStaticBp2Build(ctx android.TopDownMutatorContext, module *Module, fullBuild bool) { prebuiltAttrs := Bp2BuildParsePrebuiltLibraryProps(ctx, module, true) - exportedIncludes := Bp2BuildParseExportedIncludesForPrebuiltLibrary(ctx, module) + exportedIncludes := bp2BuildParseExportedIncludes(ctx, module, nil) attrs := &bazelPrebuiltLibraryStaticAttributes{ Static_library: prebuiltAttrs.Src, @@ -405,22 +412,48 @@ type prebuiltObjectLinker struct { properties prebuiltObjectProperties } -type prebuiltStaticLibraryBazelHandler struct { - android.BazelHandler - +type prebuiltLibraryBazelHandler struct { module *Module library *libraryDecorator } -func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +var _ BazelHandler = (*prebuiltLibraryBazelHandler)(nil) + +func (h *prebuiltLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) { + if h.module.linker.(*prebuiltLibraryLinker).properties.MixedBuildsDisabled { + return + } + bazelCtx := ctx.Config().BazelContext + bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx)) +} + +func (h *prebuiltLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) { + if h.module.linker.(*prebuiltLibraryLinker).properties.MixedBuildsDisabled { + return + } bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) + ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err) + ctx.ModuleErrorf(err.Error()) + return } - if !ok { - return false + + if h.module.static() { + if ok := h.processStaticBazelQueryResponse(ctx, label, ccInfo); !ok { + return + } + } else if h.module.Shared() { + if ok := h.processSharedBazelQueryResponse(ctx, label, ccInfo); !ok { + return + } + } else { + return } + + h.module.maybeUnhideFromMake() +} + +func (h *prebuiltLibraryBazelHandler) processStaticBazelQueryResponse(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { staticLibs := ccInfo.CcStaticLibraryFiles if len(staticLibs) > 1 { ctx.ModuleErrorf("expected 1 static library from bazel target %q, got %s", label, staticLibs) @@ -455,24 +488,9 @@ func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx androi return true } -type prebuiltSharedLibraryBazelHandler struct { - android.BazelHandler - - module *Module - library *libraryDecorator -} - -func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { - bazelCtx := ctx.Config().BazelContext - ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx)) - if err != nil { - ctx.ModuleErrorf("Error getting Bazel CcInfo for %s: %s", label, err) - } - if !ok { - return false - } +func (h *prebuiltLibraryBazelHandler) processSharedBazelQueryResponse(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool { sharedLibs := ccInfo.CcSharedLibraryFiles - if len(sharedLibs) != 1 { + if len(sharedLibs) > 1 { ctx.ModuleErrorf("expected 1 shared library from bazel target %s, got %q", label, sharedLibs) return false } @@ -482,11 +500,6 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi // TODO(eakammer):Add stub-related flags if this library is a stub library. // h.library.exportVersioningMacroIfNeeded(ctx) - // Dependencies on this library will expect collectedSnapshotHeaders to be set, otherwise - // validation will fail. For now, set this to an empty list. - // TODO(cparsons): More closely mirror the collectHeadersForSnapshot implementation. - h.library.collectedSnapshotHeaders = android.Paths{} - if len(sharedLibs) == 0 { h.module.outputFile = android.OptionalPath{} return true @@ -514,7 +527,6 @@ func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx androi h.library.setFlagExporterInfoFromCcInfo(ctx, ccInfo) h.module.maybeUnhideFromMake() - return true } diff --git a/cc/prebuilt_test.go b/cc/prebuilt_test.go index 901f45825..e959157bf 100644 --- a/cc/prebuilt_test.go +++ b/cc/prebuilt_test.go @@ -381,6 +381,149 @@ func TestPrebuiltLibrarySanitized(t *testing.T) { assertString(t, static2.OutputFile().Path().Base(), "libf.hwasan.a") } +func TestPrebuiltLibraryWithBazel(t *testing.T) { + const bp = ` +cc_prebuilt_library { + name: "foo", + shared: { + srcs: ["foo.so"], + }, + static: { + srcs: ["foo.a"], + }, + bazel_module: { label: "//foo/bar:bar" }, +}` + outBaseDir := "outputbase" + result := android.GroupFixturePreparers( + prepareForPrebuiltTest, + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: outBaseDir, + LabelToCcInfo: map[string]cquery.CcInfo{ + "//foo/bar:bar": cquery.CcInfo{ + CcSharedLibraryFiles: []string{"foo.so"}, + }, + "//foo/bar:bar_bp2build_cc_library_static": cquery.CcInfo{ + CcStaticLibraryFiles: []string{"foo.a"}, + }, + }, + } + }), + ).RunTestWithBp(t, bp) + sharedFoo := result.ModuleForTests("foo", "android_arm_armv7-a-neon_shared").Module() + pathPrefix := outBaseDir + "/execroot/__main__/" + + sharedInfo := result.ModuleProvider(sharedFoo, SharedLibraryInfoProvider).(SharedLibraryInfo) + android.AssertPathRelativeToTopEquals(t, + "prebuilt library shared target path did not exist or did not match expected. If the base path is what does not match, it is likely that Soong built this module instead of Bazel.", + pathPrefix+"foo.so", sharedInfo.SharedLibrary) + + outputFiles, err := sharedFoo.(android.OutputFileProducer).OutputFiles("") + if err != nil { + t.Errorf("Unexpected error getting cc_object outputfiles %s", err) + } + expectedOutputFiles := []string{pathPrefix + "foo.so"} + android.AssertDeepEquals(t, + "prebuilt library shared target output files did not match expected.", + expectedOutputFiles, outputFiles.Strings()) + + staticFoo := result.ModuleForTests("foo", "android_arm_armv7-a-neon_static").Module() + staticInfo := result.ModuleProvider(staticFoo, StaticLibraryInfoProvider).(StaticLibraryInfo) + android.AssertPathRelativeToTopEquals(t, + "prebuilt library static target path did not exist or did not match expected. If the base path is what does not match, it is likely that Soong built this module instead of Bazel.", + pathPrefix+"foo.a", staticInfo.StaticLibrary) + + staticOutputFiles, err := staticFoo.(android.OutputFileProducer).OutputFiles("") + if err != nil { + t.Errorf("Unexpected error getting cc_object staticOutputFiles %s", err) + } + expectedStaticOutputFiles := []string{pathPrefix + "foo.a"} + android.AssertDeepEquals(t, + "prebuilt library static target output files did not match expected.", + expectedStaticOutputFiles, staticOutputFiles.Strings()) +} + +func TestPrebuiltLibraryWithBazelStaticDisabled(t *testing.T) { + const bp = ` +cc_prebuilt_library { + name: "foo", + shared: { + srcs: ["foo.so"], + }, + static: { + enabled: false + }, + bazel_module: { label: "//foo/bar:bar" }, +}` + outBaseDir := "outputbase" + result := android.GroupFixturePreparers( + prepareForPrebuiltTest, + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: outBaseDir, + LabelToCcInfo: map[string]cquery.CcInfo{ + "//foo/bar:bar": cquery.CcInfo{ + CcSharedLibraryFiles: []string{"foo.so"}, + }, + }, + } + }), + ).RunTestWithBp(t, bp) + sharedFoo := result.ModuleForTests("foo", "android_arm_armv7-a-neon_shared").Module() + pathPrefix := outBaseDir + "/execroot/__main__/" + + sharedInfo := result.ModuleProvider(sharedFoo, SharedLibraryInfoProvider).(SharedLibraryInfo) + android.AssertPathRelativeToTopEquals(t, + "prebuilt library shared target path did not exist or did not match expected. If the base path is what does not match, it is likely that Soong built this module instead of Bazel.", + pathPrefix+"foo.so", sharedInfo.SharedLibrary) + + outputFiles, err := sharedFoo.(android.OutputFileProducer).OutputFiles("") + if err != nil { + t.Errorf("Unexpected error getting cc_object outputfiles %s", err) + } + expectedOutputFiles := []string{pathPrefix + "foo.so"} + android.AssertDeepEquals(t, + "prebuilt library shared target output files did not match expected.", + expectedOutputFiles, outputFiles.Strings()) +} + +func TestPrebuiltLibraryStaticWithBazel(t *testing.T) { + const bp = ` +cc_prebuilt_library_static { + name: "foo", + srcs: ["foo.so"], + bazel_module: { label: "//foo/bar:bar" }, +}` + outBaseDir := "outputbase" + result := android.GroupFixturePreparers( + prepareForPrebuiltTest, + android.FixtureModifyConfig(func(config android.Config) { + config.BazelContext = android.MockBazelContext{ + OutputBaseDir: outBaseDir, + LabelToCcInfo: map[string]cquery.CcInfo{ + "//foo/bar:bar": cquery.CcInfo{ + CcStaticLibraryFiles: []string{"foo.so"}, + }, + }, + } + }), + ).RunTestWithBp(t, bp) + staticFoo := result.ModuleForTests("foo", "android_arm_armv7-a-neon_static").Module() + pathPrefix := outBaseDir + "/execroot/__main__/" + + info := result.ModuleProvider(staticFoo, StaticLibraryInfoProvider).(StaticLibraryInfo) + android.AssertPathRelativeToTopEquals(t, + "prebuilt library static path did not match expected. If the base path is what does not match, it is likely that Soong built this module instead of Bazel.", + pathPrefix+"foo.so", info.StaticLibrary) + + outputFiles, err := staticFoo.(android.OutputFileProducer).OutputFiles("") + if err != nil { + t.Errorf("Unexpected error getting cc_object outputfiles %s", err) + } + expectedOutputFiles := []string{pathPrefix + "foo.so"} + android.AssertDeepEquals(t, "prebuilt library static output files did not match expected.", expectedOutputFiles, outputFiles.Strings()) +} + func TestPrebuiltLibrarySharedWithBazelWithoutToc(t *testing.T) { const bp = ` cc_prebuilt_library_shared { diff --git a/cc/sanitize.go b/cc/sanitize.go index 53169de48..86472a2cc 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -153,9 +153,10 @@ func (t SanitizerType) name() string { func (t SanitizerType) registerMutators(ctx android.RegisterMutatorsContext) { switch t { - case Asan, Hwasan, Fuzzer, scs, tsan, cfi: - ctx.TopDown(t.variationName()+"_deps", sanitizerDepsMutator(t)) - ctx.BottomUp(t.variationName(), sanitizerMutator(t)) + case cfi, Hwasan, Asan, tsan, Fuzzer, scs: + sanitizer := &sanitizerSplitMutator{t} + ctx.TopDown(t.variationName()+"_markapexes", sanitizer.markSanitizableApexesMutator) + ctx.Transition(t.variationName(), sanitizer) case Memtag_heap, intOverflow: // do nothing default: @@ -276,7 +277,6 @@ type SanitizeUserProps struct { type SanitizeProperties struct { Sanitize SanitizeUserProps `android:"arch_variant"` SanitizerEnabled bool `blueprint:"mutated"` - SanitizeDep bool `blueprint:"mutated"` MinimalRuntimeDep bool `blueprint:"mutated"` BuiltinsDep bool `blueprint:"mutated"` UbsanRuntimeDep bool `blueprint:"mutated"` @@ -430,7 +430,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } // Enable Memtag for all components in the include paths (for Aarch64 only) - if ctx.Arch().ArchType == android.Arm64 { + if ctx.Arch().ArchType == android.Arm64 && ctx.toolchain().Bionic() { if ctx.Config().MemtagHeapSyncEnabledForPath(ctx.ModuleDir()) { if s.Memtag_heap == nil { s.Memtag_heap = proptools.BoolPtr(true) @@ -460,17 +460,17 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } // HWASan requires AArch64 hardware feature (top-byte-ignore). - if ctx.Arch().ArchType != android.Arm64 { + if ctx.Arch().ArchType != android.Arm64 || !ctx.toolchain().Bionic() { s.Hwaddress = nil } // SCS is only implemented on AArch64. - if ctx.Arch().ArchType != android.Arm64 { + if ctx.Arch().ArchType != android.Arm64 || !ctx.toolchain().Bionic() { s.Scs = nil } // Memtag_heap is only implemented on AArch64. - if ctx.Arch().ArchType != android.Arm64 { + if ctx.Arch().ArchType != android.Arm64 || !ctx.toolchain().Bionic() { s.Memtag_heap = nil } @@ -588,13 +588,6 @@ func toDisableUnsignedShiftBaseChange(flags []string) bool { } func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { - minimalRuntimeLib := config.UndefinedBehaviorSanitizerMinimalRuntimeLibrary(ctx.toolchain()) + ".a" - - if sanitize.Properties.MinimalRuntimeDep { - flags.Local.LdFlags = append(flags.Local.LdFlags, - "-Wl,--exclude-libs,"+minimalRuntimeLib) - } - if !sanitize.Properties.SanitizerEnabled && !sanitize.Properties.UbsanRuntimeDep { return flags } @@ -717,14 +710,14 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { // Host sanitizers only link symbols in the final executable, so // there will always be undefined symbols in intermediate libraries. _, flags.Global.LdFlags = removeFromList("-Wl,--no-undefined", flags.Global.LdFlags) - - // non-Bionic toolchain prebuilts are missing UBSan's vptr and function san - flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") } - if enableMinimalRuntime(sanitize) { - flags.Local.CFlags = append(flags.Local.CFlags, strings.Join(minimalRuntimeFlags, " ")) - flags.Local.LdFlags = append(flags.Local.LdFlags, "-Wl,--exclude-libs,"+minimalRuntimeLib) + if !ctx.toolchain().Bionic() { + // non-Bionic toolchain prebuilts are missing UBSan's vptr and function san. + // Musl toolchain prebuilts have vptr and function sanitizers, but enabling them + // implicitly enables RTTI which causes RTTI mismatch issues with dependencies. + + flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=vptr,function") } if Bool(sanitize.Properties.Sanitize.Fuzzer) { @@ -735,6 +728,11 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags) Flags { } else { flags.Local.CFlags = append(flags.Local.CFlags, "-fsanitize-trap=all", "-ftrap-function=abort") } + + if enableMinimalRuntime(sanitize) { + flags.Local.CFlags = append(flags.Local.CFlags, strings.Join(minimalRuntimeFlags, " ")) + } + // http://b/119329758, Android core does not boot up with this sanitizer yet. if toDisableImplicitIntegerChange(flags.Local.CFlags) { flags.Local.CFlags = append(flags.Local.CFlags, "-fno-sanitize=implicit-integer-sign-change") @@ -906,7 +904,7 @@ func (m *Module) SanitizableDepTagChecker() SantizableDependencyTagChecker { // Determines if the current module is a static library going to be captured // as vendor snapshot. Such modules must create both cfi and non-cfi variants, // except for ones which explicitly disable cfi. -func needsCfiForVendorSnapshot(mctx android.TopDownMutatorContext) bool { +func needsCfiForVendorSnapshot(mctx android.BaseModuleContext) bool { if snapshot.IsVendorProprietaryModule(mctx) { return false } @@ -934,64 +932,232 @@ func needsCfiForVendorSnapshot(mctx android.TopDownMutatorContext) bool { !c.IsSanitizerExplicitlyDisabled(cfi) } -// Propagate sanitizer requirements down from binaries -func sanitizerDepsMutator(t SanitizerType) func(android.TopDownMutatorContext) { - return func(mctx android.TopDownMutatorContext) { - if c, ok := mctx.Module().(PlatformSanitizeable); ok { - enabled := c.IsSanitizerEnabled(t) - if t == cfi && needsCfiForVendorSnapshot(mctx) { - // We shouldn't change the result of isSanitizerEnabled(cfi) to correctly - // determine defaultVariation in sanitizerMutator below. - // Instead, just mark SanitizeDep to forcefully create cfi variant. +type sanitizerSplitMutator struct { + sanitizer SanitizerType +} + +// If an APEX is sanitized or not depends on whether it contains at least one +// sanitized module. Transition mutators cannot propagate information up the +// dependency graph this way, so we need an auxiliary mutator to do so. +func (s *sanitizerSplitMutator) markSanitizableApexesMutator(ctx android.TopDownMutatorContext) { + if sanitizeable, ok := ctx.Module().(Sanitizeable); ok { + enabled := sanitizeable.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + ctx.VisitDirectDeps(func(dep android.Module) { + if c, ok := dep.(*Module); ok && c.sanitize.isSanitizerEnabled(s.sanitizer) { enabled = true - c.SetSanitizeDep(true) } - if enabled { - isSanitizableDependencyTag := c.SanitizableDepTagChecker() - mctx.WalkDeps(func(child, parent android.Module) bool { - if !isSanitizableDependencyTag(mctx.OtherModuleDependencyTag(child)) { - return false - } - if d, ok := child.(PlatformSanitizeable); ok && d.SanitizePropDefined() && - !d.SanitizeNever() && - !d.IsSanitizerExplicitlyDisabled(t) { - if t == cfi || t == Hwasan || t == scs || t == Asan { - if d.StaticallyLinked() && d.SanitizerSupported(t) { - // Rust does not support some of these sanitizers, so we need to check if it's - // supported before setting this true. - d.SetSanitizeDep(true) - } - } else { - d.SetSanitizeDep(true) - } - } - return true - }) + }) + + if enabled { + sanitizeable.EnableSanitizer(s.sanitizer.name()) + } + } +} + +func (s *sanitizerSplitMutator) Split(ctx android.BaseModuleContext) []string { + if c, ok := ctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { + if s.sanitizer == cfi && needsCfiForVendorSnapshot(ctx) { + return []string{"", s.sanitizer.variationName()} + } + + // If the given sanitizer is not requested in the .bp file for a module, it + // won't automatically build the sanitized variation. + if !c.IsSanitizerEnabled(s.sanitizer) { + return []string{""} + } + + if c.Binary() { + // If a sanitizer is enabled for a binary, we do not build the version + // without the sanitizer + return []string{s.sanitizer.variationName()} + } else if c.StaticallyLinked() || c.Header() { + // For static libraries, we build both versions. Some Make modules + // apparently depend on this behavior. + return []string{"", s.sanitizer.variationName()} + } else { + // We only build the requested variation of dynamic libraries + return []string{s.sanitizer.variationName()} + } + } + + if _, ok := ctx.Module().(JniSanitizeable); ok { + // TODO: this should call into JniSanitizable.IsSanitizerEnabledForJni but + // that is short-circuited for now + return []string{""} + } + + // If an APEX has a sanitized dependency, we build the APEX in the sanitized + // variation. This is useful because such APEXes require extra dependencies. + if sanitizeable, ok := ctx.Module().(Sanitizeable); ok { + enabled := sanitizeable.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + if enabled { + return []string{s.sanitizer.variationName()} + } else { + return []string{""} + } + } + + if c, ok := ctx.Module().(*Module); ok { + //TODO: When Rust modules have vendor support, enable this path for PlatformSanitizeable + + // Check if it's a snapshot module supporting sanitizer + if ss, ok := c.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + return []string{"", s.sanitizer.variationName()} + } else { + return []string{""} + } + } + + return []string{""} +} + +func (s *sanitizerSplitMutator) OutgoingTransition(ctx android.OutgoingTransitionContext, sourceVariation string) string { + if c, ok := ctx.Module().(PlatformSanitizeable); ok { + if !c.SanitizableDepTagChecker()(ctx.DepTag()) { + // If the dependency is through a non-sanitizable tag, use the + // non-sanitized variation + return "" + } + + return sourceVariation + } else if _, ok := ctx.Module().(JniSanitizeable); ok { + // TODO: this should call into JniSanitizable.IsSanitizerEnabledForJni but + // that is short-circuited for now + return "" + } else { + // Otherwise, do not rock the boat. + return sourceVariation + } +} + +func (s *sanitizerSplitMutator) IncomingTransition(ctx android.IncomingTransitionContext, incomingVariation string) string { + if d, ok := ctx.Module().(PlatformSanitizeable); ok { + if dm, ok := ctx.Module().(*Module); ok { + if ss, ok := dm.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + return incomingVariation } - } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { - // If it's a Java module with native dependencies through jni, - // set the sanitizer for them - if jniSanitizeable, ok := mctx.Module().(JniSanitizeable); ok { - if jniSanitizeable.IsSanitizerEnabledForJni(mctx, t.name()) { - mctx.VisitDirectDeps(func(child android.Module) { - if c, ok := child.(PlatformSanitizeable); ok && - mctx.OtherModuleDependencyTag(child) == JniFuzzLibTag && - c.SanitizePropDefined() && - !c.SanitizeNever() && - !c.IsSanitizerExplicitlyDisabled(t) { - c.SetSanitizeDep(true) - } - }) + } + + if !d.SanitizePropDefined() || + d.SanitizeNever() || + d.IsSanitizerExplicitlyDisabled(s.sanitizer) || + !d.SanitizerSupported(s.sanitizer) { + // If a module opts out of a sanitizer, use its non-sanitized variation + return "" + } + + // Binaries are always built in the variation they requested. + if d.Binary() { + if d.IsSanitizerEnabled(s.sanitizer) { + return s.sanitizer.variationName() + } else { + return "" + } + } + + // If a shared library requests to be sanitized, it will be built for that + // sanitizer. Otherwise, some sanitizers propagate through shared library + // dependency edges, some do not. + if !d.StaticallyLinked() && !d.Header() { + if d.IsSanitizerEnabled(s.sanitizer) { + return s.sanitizer.variationName() + } + + if s.sanitizer == cfi || s.sanitizer == Hwasan || s.sanitizer == scs || s.sanitizer == Asan { + return "" + } + } + + // Static and header libraries inherit whether they are sanitized from the + // module they are linked into + return incomingVariation + } else if d, ok := ctx.Module().(Sanitizeable); ok { + // If an APEX contains a sanitized module, it will be built in the variation + // corresponding to that sanitizer. + enabled := d.IsSanitizerEnabled(ctx.Config(), s.sanitizer.name()) + if enabled { + return s.sanitizer.variationName() + } + + return incomingVariation + } + + return "" +} + +func (s *sanitizerSplitMutator) Mutate(mctx android.BottomUpMutatorContext, variationName string) { + sanitizerVariation := variationName == s.sanitizer.variationName() + + if c, ok := mctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { + sanitizerEnabled := c.IsSanitizerEnabled(s.sanitizer) + + oneMakeVariation := false + if c.StaticallyLinked() || c.Header() { + if s.sanitizer != cfi && s.sanitizer != scs && s.sanitizer != Hwasan { + // These sanitizers export only one variation to Make. For the rest, + // Make targets can depend on both the sanitized and non-sanitized + // versions. + oneMakeVariation = true + } + } else if !c.Binary() { + // Shared library. These are the sanitizers that do propagate through shared + // library dependencies and therefore can cause multiple variations of a + // shared library to be built. + if s.sanitizer != cfi && s.sanitizer != Hwasan && s.sanitizer != scs && s.sanitizer != Asan { + oneMakeVariation = true + } + } + + if oneMakeVariation { + if sanitizerEnabled != sanitizerVariation { + c.SetPreventInstall() + c.SetHideFromMake() + } + } + + if sanitizerVariation { + c.SetSanitizer(s.sanitizer, true) + + // CFI is incompatible with ASAN so disable it in ASAN variations + if s.sanitizer.incompatibleWithCfi() { + cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) + if mctx.Device() && cfiSupported { + c.SetSanitizer(cfi, false) } } - // If an APEX module includes a lib which is enabled for a sanitizer T, then - // the APEX module is also enabled for the same sanitizer type. - mctx.VisitDirectDeps(func(child android.Module) { - if c, ok := child.(*Module); ok && c.sanitize.isSanitizerEnabled(t) { - sanitizeable.EnableSanitizer(t.name()) + // locate the asan libraries under /data/asan + if !c.Binary() && !c.StaticallyLinked() && !c.Header() && mctx.Device() && s.sanitizer == Asan && sanitizerEnabled { + c.SetInSanitizerDir() + } + + if c.StaticallyLinked() && c.ExportedToMake() { + if s.sanitizer == Hwasan { + hwasanStaticLibs(mctx.Config()).add(c, c.Module().Name()) + } else if s.sanitizer == cfi { + cfiStaticLibs(mctx.Config()).add(c, c.Module().Name()) + } + } + } else if c.IsSanitizerEnabled(s.sanitizer) { + // Disable the sanitizer for the non-sanitized variation + c.SetSanitizer(s.sanitizer, false) + } + } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok { + // If an APEX has sanitized dependencies, it gets a few more dependencies + if sanitizerVariation { + sanitizeable.AddSanitizerDependencies(mctx, s.sanitizer.name()) + } + } else if c, ok := mctx.Module().(*Module); ok { + if ss, ok := c.linker.(snapshotSanitizer); ok && ss.isSanitizerEnabled(s.sanitizer) { + c.linker.(snapshotSanitizer).setSanitizerVariation(s.sanitizer, sanitizerVariation) + + // Export the static lib name to make + if c.static() && c.ExportedToMake() { + if s.sanitizer == cfi { + // use BaseModuleName which is the name for Make. + cfiStaticLibs(mctx.Config()).add(c, c.BaseModuleName()) } - }) + } } } } @@ -1219,7 +1385,7 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { } // static executable gets static runtime libs - depTag := libraryDependencyTag{Kind: staticLibraryDependency} + depTag := libraryDependencyTag{Kind: staticLibraryDependency, unexportedSymbols: true} variations := append(mctx.Target().Variations(), blueprint.Variation{Mutator: "link", Variation: "static"}) if c.Device() { @@ -1291,7 +1457,7 @@ func sanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { type Sanitizeable interface { android.Module - IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool + IsSanitizerEnabled(config android.Config, sanitizerName string) bool EnableSanitizer(sanitizerName string) AddSanitizerDependencies(ctx android.BottomUpMutatorContext, sanitizerName string) } @@ -1317,10 +1483,6 @@ func (c *Module) IsSanitizerEnabled(t SanitizerType) bool { return c.sanitize.isSanitizerEnabled(t) } -func (c *Module) SanitizeDep() bool { - return c.sanitize.Properties.SanitizeDep -} - func (c *Module) StaticallyLinked() bool { return c.static() } @@ -1337,124 +1499,8 @@ func (c *Module) SetSanitizer(t SanitizerType, b bool) { } } -func (c *Module) SetSanitizeDep(b bool) { - if c.sanitize != nil { - c.sanitize.Properties.SanitizeDep = b - } -} - var _ PlatformSanitizeable = (*Module)(nil) -// Create sanitized variants for modules that need them -func sanitizerMutator(t SanitizerType) func(android.BottomUpMutatorContext) { - return func(mctx android.BottomUpMutatorContext) { - if c, ok := mctx.Module().(PlatformSanitizeable); ok && c.SanitizePropDefined() { - - // Make sure we're not setting CFI to any value if it's not supported. - cfiSupported := mctx.Module().(PlatformSanitizeable).SanitizerSupported(cfi) - - if c.Binary() && c.IsSanitizerEnabled(t) { - modules := mctx.CreateVariations(t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, true) - } else if c.IsSanitizerEnabled(t) || c.SanitizeDep() { - isSanitizerEnabled := c.IsSanitizerEnabled(t) - if c.StaticallyLinked() || c.Header() || t == Fuzzer { - // Static and header libs are split into non-sanitized and sanitized variants. - // Shared libs are not split. However, for asan and fuzzer, we split even for shared - // libs because a library sanitized for asan/fuzzer can't be linked from a library - // that isn't sanitized for asan/fuzzer. - // - // Note for defaultVariation: since we don't split for shared libs but for static/header - // libs, it is possible for the sanitized variant of a static/header lib to depend - // on non-sanitized variant of a shared lib. Such unfulfilled variation causes an - // error when the module is split. defaultVariation is the name of the variation that - // will be used when such a dangling dependency occurs during the split of the current - // module. By setting it to the name of the sanitized variation, the dangling dependency - // is redirected to the sanitized variant of the dependent module. - defaultVariation := t.variationName() - // Not all PlatformSanitizeable modules support the CFI sanitizer - mctx.SetDefaultDependencyVariation(&defaultVariation) - - modules := mctx.CreateVariations("", t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, false) - modules[1].(PlatformSanitizeable).SetSanitizer(t, true) - modules[0].(PlatformSanitizeable).SetSanitizeDep(false) - modules[1].(PlatformSanitizeable).SetSanitizeDep(false) - - if mctx.Device() && t.incompatibleWithCfi() && cfiSupported { - // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that - // are incompatible with cfi - modules[1].(PlatformSanitizeable).SetSanitizer(cfi, false) - } - - // For cfi/scs/hwasan, we can export both sanitized and un-sanitized variants - // to Make, because the sanitized version has a different suffix in name. - // For other types of sanitizers, suppress the variation that is disabled. - if t != cfi && t != scs && t != Hwasan { - if isSanitizerEnabled { - modules[0].(PlatformSanitizeable).SetPreventInstall() - modules[0].(PlatformSanitizeable).SetHideFromMake() - } else { - modules[1].(PlatformSanitizeable).SetPreventInstall() - modules[1].(PlatformSanitizeable).SetHideFromMake() - } - } - - // Export the static lib name to make - if c.StaticallyLinked() && c.ExportedToMake() { - if t == cfi { - cfiStaticLibs(mctx.Config()).add(c, c.Module().Name()) - } else if t == Hwasan { - hwasanStaticLibs(mctx.Config()).add(c, c.Module().Name()) - } - } - } else { - // Shared libs are not split. Only the sanitized variant is created. - modules := mctx.CreateVariations(t.variationName()) - modules[0].(PlatformSanitizeable).SetSanitizer(t, true) - modules[0].(PlatformSanitizeable).SetSanitizeDep(false) - - // locate the asan libraries under /data/asan - if mctx.Device() && t == Asan && isSanitizerEnabled { - modules[0].(PlatformSanitizeable).SetInSanitizerDir() - } - - if mctx.Device() && t.incompatibleWithCfi() && cfiSupported { - // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that - // are incompatible with cfi - modules[0].(PlatformSanitizeable).SetSanitizer(cfi, false) - } - } - } - c.SetSanitizeDep(false) - } else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok && sanitizeable.IsSanitizerEnabled(mctx, t.name()) { - // APEX and Java fuzz modules fall here - sanitizeable.AddSanitizerDependencies(mctx, t.name()) - mctx.CreateVariations(t.variationName()) - } else if c, ok := mctx.Module().(*Module); ok { - //TODO: When Rust modules have vendor support, enable this path for PlatformSanitizeable - - // Check if it's a snapshot module supporting sanitizer - if s, ok := c.linker.(snapshotSanitizer); ok && s.isSanitizerEnabled(t) { - // Set default variation as above. - defaultVariation := t.variationName() - mctx.SetDefaultDependencyVariation(&defaultVariation) - modules := mctx.CreateVariations("", t.variationName()) - modules[0].(*Module).linker.(snapshotSanitizer).setSanitizerVariation(t, false) - modules[1].(*Module).linker.(snapshotSanitizer).setSanitizerVariation(t, true) - - // Export the static lib name to make - if c.static() && c.ExportedToMake() { - if t == cfi { - // use BaseModuleName which is the name for Make. - cfiStaticLibs(mctx.Config()).add(c, c.BaseModuleName()) - } - } - } - } - } -} - type sanitizerStaticLibsMap struct { // libsMap contains one list of modules per each image and each arch. // e.g. libs[vendor]["arm"] contains arm modules installed to vendor @@ -1529,12 +1575,10 @@ func enableMinimalRuntime(sanitize *sanitize) bool { if !Bool(sanitize.Properties.Sanitize.Address) && !Bool(sanitize.Properties.Sanitize.Hwaddress) && !Bool(sanitize.Properties.Sanitize.Fuzzer) && - (Bool(sanitize.Properties.Sanitize.Integer_overflow) || len(sanitize.Properties.Sanitize.Misc_undefined) > 0 || Bool(sanitize.Properties.Sanitize.Undefined) || Bool(sanitize.Properties.Sanitize.All_undefined)) && - !(Bool(sanitize.Properties.Sanitize.Diag.Integer_overflow) || Bool(sanitize.Properties.Sanitize.Diag.Cfi) || Bool(sanitize.Properties.Sanitize.Diag.Undefined) || diff --git a/cc/sanitize_test.go b/cc/sanitize_test.go index c1ca03408..5d7e7d80a 100644 --- a/cc/sanitize_test.go +++ b/cc/sanitize_test.go @@ -16,6 +16,7 @@ package cc import ( "fmt" + "runtime" "strings" "testing" @@ -201,6 +202,125 @@ func TestAsan(t *testing.T) { t.Run("device", func(t *testing.T) { check(t, result, "android_arm64_armv8-a") }) } +func TestUbsan(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("requires linux") + } + + bp := ` + cc_binary { + name: "bin_with_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libnoubsan", + ], + sanitize: { + undefined: true, + } + } + + cc_binary { + name: "bin_depends_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libubsan", + "libnoubsan", + ], + } + + cc_binary { + name: "bin_no_ubsan", + host_supported: true, + shared_libs: [ + "libshared", + ], + static_libs: [ + "libstatic", + "libnoubsan", + ], + } + + cc_library_shared { + name: "libshared", + host_supported: true, + shared_libs: ["libtransitive"], + } + + cc_library_shared { + name: "libtransitive", + host_supported: true, + } + + cc_library_static { + name: "libubsan", + host_supported: true, + sanitize: { + undefined: true, + } + } + + cc_library_static { + name: "libstatic", + host_supported: true, + } + + cc_library_static { + name: "libnoubsan", + host_supported: true, + } + ` + + result := android.GroupFixturePreparers( + prepareForCcTest, + ).RunTestWithBp(t, bp) + + check := func(t *testing.T, result *android.TestResult, variant string) { + staticVariant := variant + "_static" + + minimalRuntime := result.ModuleForTests("libclang_rt.ubsan_minimal", staticVariant) + + // The binaries, one with ubsan and one without + binWithUbsan := result.ModuleForTests("bin_with_ubsan", variant) + binDependsUbsan := result.ModuleForTests("bin_depends_ubsan", variant) + binNoUbsan := result.ModuleForTests("bin_no_ubsan", variant) + + android.AssertStringListContains(t, "missing libclang_rt.ubsan_minimal in bin_with_ubsan static libs", + strings.Split(binWithUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListContains(t, "missing libclang_rt.ubsan_minimal in bin_depends_ubsan static libs", + strings.Split(binDependsUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListDoesNotContain(t, "unexpected libclang_rt.ubsan_minimal in bin_no_ubsan static libs", + strings.Split(binNoUbsan.Rule("ld").Args["libFlags"], " "), + minimalRuntime.OutputFiles(t, "")[0].String()) + + android.AssertStringListContains(t, "missing -Wl,--exclude-libs for minimal runtime in bin_with_ubsan", + strings.Split(binWithUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + + android.AssertStringListContains(t, "missing -Wl,--exclude-libs for minimal runtime in bin_depends_ubsan static libs", + strings.Split(binDependsUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + + android.AssertStringListDoesNotContain(t, "unexpected -Wl,--exclude-libs for minimal runtime in bin_no_ubsan static libs", + strings.Split(binNoUbsan.Rule("ld").Args["ldFlags"], " "), + "-Wl,--exclude-libs="+minimalRuntime.OutputFiles(t, "")[0].Base()) + } + + t.Run("host", func(t *testing.T) { check(t, result, result.Config.BuildOSTarget.String()) }) + t.Run("device", func(t *testing.T) { check(t, result, "android_arm64_armv8-a") }) +} + type MemtagNoteType int const ( @@ -25,6 +25,16 @@ func getNdkStlFamily(m LinkableInterface) string { return family } +func deduplicateStlInput(stl string) string { + switch stl { + case "c++_shared": + return "libc++" + case "c++_static": + return "libc++_static" + } + return stl +} + func getNdkStlFamilyAndLinkType(m LinkableInterface) (string, string) { stl := m.SelectedStl() switch stl { @@ -66,18 +76,18 @@ func (stl *stl) begin(ctx BaseModuleContext) { } else if ctx.header() { s = "none" } + if s == "none" { + return "" + } + s = deduplicateStlInput(s) if ctx.useSdk() && ctx.Device() { switch s { case "", "system": return "ndk_system" - case "c++_shared", "c++_static": - return "ndk_lib" + s case "libc++": return "ndk_libc++_shared" case "libc++_static": return "ndk_libc++_static" - case "none": - return "" default: ctx.ModuleErrorf("stl: %q is not a supported STL with sdk_version set", s) return "" @@ -87,8 +97,6 @@ func (stl *stl) begin(ctx BaseModuleContext) { case "libc++", "libc++_static", "": // Only use static libc++ for Windows. return "libc++_static" - case "none": - return "" default: ctx.ModuleErrorf("stl: %q is not a supported STL for windows", s) return "" @@ -97,12 +105,6 @@ func (stl *stl) begin(ctx BaseModuleContext) { switch s { case "libc++", "libc++_static": return s - case "c++_shared": - return "libc++" - case "c++_static": - return "libc++_static" - case "none": - return "" case "", "system": if ctx.static() { return "libc++_static" diff --git a/cc/test.go b/cc/test.go index ead7877e2..57035711e 100644 --- a/cc/test.go +++ b/cc/test.go @@ -30,7 +30,8 @@ type TestLinkerProperties struct { // if set, build against the gtest library. Defaults to true. Gtest *bool - // if set, use the isolated gtest runner. Defaults to false. + // if set, use the isolated gtest runner. Defaults to true if gtest is also true and the arch is Windows, false + // otherwise. Isolated *bool } @@ -256,6 +257,13 @@ func (test *testDecorator) gtest() bool { return BoolDefault(test.LinkerProperties.Gtest, true) } +func (test *testDecorator) isolated(ctx BaseModuleContext) bool { + if !ctx.Windows() { + return BoolDefault(test.LinkerProperties.Isolated, false) + } + return BoolDefault(test.LinkerProperties.Isolated, false) +} + func (test *testDecorator) testBinary() bool { return true } @@ -288,7 +296,7 @@ func (test *testDecorator) linkerDeps(ctx BaseModuleContext, deps Deps) Deps { if test.gtest() { if ctx.useSdk() && ctx.Device() { deps.StaticLibs = append(deps.StaticLibs, "libgtest_main_ndk_c++", "libgtest_ndk_c++") - } else if BoolDefault(test.LinkerProperties.Isolated, false) { + } else if test.isolated(ctx) { deps.StaticLibs = append(deps.StaticLibs, "libgtest_isolated_main") // The isolated library requires liblog, but adding it // as a static library means unit tests cannot override @@ -424,7 +432,7 @@ func (test *testBinary) install(ctx ModuleContext, file android.Path) { var options []tradefed.Option configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.StopServicesSetup", options}) } - if Bool(test.testDecorator.LinkerProperties.Isolated) { + if test.isolated(ctx) { configs = append(configs, tradefed.Option{Name: "not-shardable", Value: "true"}) } if test.Properties.Test_options.Run_test_as != nil { diff --git a/cc/testing.go b/cc/testing.go index 32f7c6080..077fcda3c 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -29,6 +29,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) { RegisterBinaryBuildComponents(ctx) RegisterLibraryBuildComponents(ctx) RegisterLibraryHeadersBuildComponents(ctx) + RegisterLibraryStubBuildComponents(ctx) ctx.RegisterModuleType("cc_benchmark", BenchmarkFactory) ctx.RegisterModuleType("cc_object", ObjectFactory) @@ -72,7 +73,6 @@ func commonDefaultModules() string { nocrt: true, system_shared_libs: [], stl: "none", - srcs: [""], check_elf_files: false, sanitize: { never: true, @@ -83,6 +83,7 @@ func commonDefaultModules() string { name: "libcompiler_rt-extras", defaults: ["toolchain_libs_defaults"], vendor_ramdisk_available: true, + srcs: [""], } cc_prebuilt_library_static { @@ -92,11 +93,13 @@ func commonDefaultModules() string { vendor_available: true, vendor_ramdisk_available: true, native_bridge_supported: true, + srcs: [""], } cc_prebuilt_library_shared { name: "libclang_rt.hwasan", defaults: ["toolchain_libs_defaults"], + srcs: [""], } cc_prebuilt_library_static { @@ -107,6 +110,7 @@ func commonDefaultModules() string { ], vendor_ramdisk_available: true, native_bridge_supported: true, + srcs: [""], } cc_prebuilt_library_static { @@ -115,17 +119,34 @@ func commonDefaultModules() string { "linux_bionic_supported", "toolchain_libs_defaults", ], + srcs: [""], } // Needed for sanitizer cc_prebuilt_library_shared { name: "libclang_rt.ubsan_standalone", defaults: ["toolchain_libs_defaults"], + srcs: [""], } cc_prebuilt_library_static { name: "libclang_rt.ubsan_minimal", defaults: ["toolchain_libs_defaults"], + host_supported: true, + target: { + android_arm64: { + srcs: ["libclang_rt.ubsan_minimal.android_arm64.a"], + }, + android_arm: { + srcs: ["libclang_rt.ubsan_minimal.android_arm.a"], + }, + linux_glibc_x86_64: { + srcs: ["libclang_rt.ubsan_minimal.x86_64.a"], + }, + linux_glibc_x86: { + srcs: ["libclang_rt.ubsan_minimal.x86.a"], + }, + }, } cc_library { @@ -545,6 +566,11 @@ var PrepareForTestWithCcDefaultModules = android.GroupFixturePreparers( "defaults/cc/common/crtend_so.c": nil, "defaults/cc/common/crtend.c": nil, "defaults/cc/common/crtbrand.c": nil, + + "defaults/cc/common/libclang_rt.ubsan_minimal.android_arm64.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.android_arm.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.x86_64.a": nil, + "defaults/cc/common/libclang_rt.ubsan_minimal.x86.a": nil, }.AddToFixture(), // Place the default cc test modules that are common to all platforms in a location that will not diff --git a/cc/tidy.go b/cc/tidy.go index 03e967d7c..6b5d5728c 100644 --- a/cc/tidy.go +++ b/cc/tidy.go @@ -15,6 +15,7 @@ package cc import ( + "fmt" "path/filepath" "regexp" "strings" @@ -62,6 +63,11 @@ func (tidy *tidyFeature) props() []interface{} { return []interface{}{&tidy.Properties} } +// Set this const to true when all -warnings-as-errors in tidy_flags +// are replaced with tidy_checks_as_errors. +// Then, that old style usage will be obsolete and an error. +const NoWarningsAsErrorsInTidyFlags = true + func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { CheckBadTidyFlags(ctx, "tidy_flags", tidy.Properties.Tidy_flags) CheckBadTidyChecks(ctx, "tidy_checks", tidy.Properties.Tidy_checks) @@ -96,10 +102,15 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { if !android.SubstringInList(flags.TidyFlags, "-header-filter=") { defaultDirs := ctx.Config().Getenv("DEFAULT_TIDY_HEADER_DIRS") headerFilter := "-header-filter=" + // Default header filter should include only the module directory, + // not the out/soong/.../ModuleDir/... + // Otherwise, there will be too many warnings from generated files in out/... + // If a module wants to see warnings in the generated source files, + // it should specify its own -header-filter flag. if defaultDirs == "" { - headerFilter += ctx.ModuleDir() + "/" + headerFilter += "^" + ctx.ModuleDir() + "/" } else { - headerFilter += "\"(" + ctx.ModuleDir() + "/|" + defaultDirs + ")\"" + headerFilter += "\"(^" + ctx.ModuleDir() + "/|" + defaultDirs + ")\"" } flags.TidyFlags = append(flags.TidyFlags, headerFilter) } @@ -139,61 +150,54 @@ func (tidy *tidyFeature) flags(ctx ModuleContext, flags Flags) Flags { tidyChecks += config.TidyChecksForDir(ctx.ModuleDir()) } if len(tidy.Properties.Tidy_checks) > 0 { - tidyChecks = tidyChecks + "," + strings.Join(esc(ctx, "tidy_checks", - config.ClangRewriteTidyChecks(tidy.Properties.Tidy_checks)), ",") + // If Tidy_checks contains "-*", ignore all checks before "-*". + localChecks := tidy.Properties.Tidy_checks + ignoreGlobalChecks := false + for n, check := range tidy.Properties.Tidy_checks { + if check == "-*" { + ignoreGlobalChecks = true + localChecks = tidy.Properties.Tidy_checks[n:] + } + } + if ignoreGlobalChecks { + tidyChecks = "-checks=" + strings.Join(esc(ctx, "tidy_checks", + config.ClangRewriteTidyChecks(localChecks)), ",") + } else { + tidyChecks = tidyChecks + "," + strings.Join(esc(ctx, "tidy_checks", + config.ClangRewriteTidyChecks(localChecks)), ",") + } } + tidyChecks = tidyChecks + config.TidyGlobalNoChecks() if ctx.Windows() { // https://b.corp.google.com/issues/120614316 // mingw32 has cert-dcl16-c warning in NO_ERROR, // which is used in many Android files. - tidyChecks = tidyChecks + ",-cert-dcl16-c" + tidyChecks += ",-cert-dcl16-c" } - // https://b.corp.google.com/issues/153464409 - // many local projects enable cert-* checks, which - // trigger bugprone-reserved-identifier. - tidyChecks = tidyChecks + ",-bugprone-reserved-identifier*,-cert-dcl51-cpp,-cert-dcl37-c" - // http://b/153757728 - tidyChecks = tidyChecks + ",-readability-qualified-auto" - // http://b/155034563 - tidyChecks = tidyChecks + ",-bugprone-signed-char-misuse" - // http://b/155034972 - tidyChecks = tidyChecks + ",-bugprone-branch-clone" - // http://b/193716442 - tidyChecks = tidyChecks + ",-bugprone-implicit-widening-of-multiplication-result" - // Too many existing functions trigger this rule, and fixing it requires large code - // refactoring. The cost of maintaining this tidy rule outweighs the benefit it brings. - tidyChecks = tidyChecks + ",-bugprone-easily-swappable-parameters" - // http://b/216364337 - TODO: Follow-up after compiler update to - // disable or fix individual instances. - tidyChecks = tidyChecks + ",-cert-err33-c" + flags.TidyFlags = append(flags.TidyFlags, tidyChecks) - if ctx.Config().IsEnvTrue("WITH_TIDY") { - // WITH_TIDY=1 enables clang-tidy globally. There could be many unexpected - // warnings from new checks and many local tidy_checks_as_errors and - // -warnings-as-errors can break a global build. - // So allow all clang-tidy warnings. - inserted := false - for i, s := range flags.TidyFlags { - if strings.Contains(s, "-warnings-as-errors=") { - // clang-tidy accepts only one -warnings-as-errors - // replace the old one - re := regexp.MustCompile(`'?-?-warnings-as-errors=[^ ]* *`) - newFlag := re.ReplaceAllString(s, "") - if newFlag == "" { - flags.TidyFlags[i] = "-warnings-as-errors=-*" - } else { - flags.TidyFlags[i] = newFlag + " -warnings-as-errors=-*" - } - inserted = true - break + // Embedding -warnings-as-errors in tidy_flags is error-prone. + // It should be replaced with the tidy_checks_as_errors list. + for i, s := range flags.TidyFlags { + if strings.Contains(s, "-warnings-as-errors=") { + if NoWarningsAsErrorsInTidyFlags { + ctx.PropertyErrorf("tidy_flags", "should not contain "+s+"; use tidy_checks_as_errors instead.") + } else { + fmt.Printf("%s: warning: module %s's tidy_flags should not contain %s, which is replaced with -warnings-as-errors=-*; use tidy_checks_as_errors for your own as-error warnings instead.\n", + ctx.BlueprintsFile(), ctx.ModuleName(), s) + flags.TidyFlags[i] = "-warnings-as-errors=-*" } + break // there is at most one -warnings-as-errors } - if !inserted { - flags.TidyFlags = append(flags.TidyFlags, "-warnings-as-errors=-*") - } - } else if len(tidy.Properties.Tidy_checks_as_errors) > 0 { - tidyChecksAsErrors := "-warnings-as-errors=" + strings.Join(esc(ctx, "tidy_checks_as_errors", tidy.Properties.Tidy_checks_as_errors), ",") + } + // Default clang-tidy flags does not contain -warning-as-errors. + // If a module has tidy_checks_as_errors, add the list to -warnings-as-errors + // and then append the TidyGlobalNoErrorChecks. + if len(tidy.Properties.Tidy_checks_as_errors) > 0 { + tidyChecksAsErrors := "-warnings-as-errors=" + + strings.Join(esc(ctx, "tidy_checks_as_errors", tidy.Properties.Tidy_checks_as_errors), ",") + + config.TidyGlobalNoErrorChecks() flags.TidyFlags = append(flags.TidyFlags, tidyChecksAsErrors) } return flags diff --git a/cc/tidy_test.go b/cc/tidy_test.go index 339b30281..7036ecb1a 100644 --- a/cc/tidy_test.go +++ b/cc/tidy_test.go @@ -16,11 +16,159 @@ package cc import ( "fmt" + "strings" "testing" "android/soong/android" ) +func TestTidyFlagsWarningsAsErrors(t *testing.T) { + // The "tidy_flags" property should not contain -warnings-as-errors. + type testCase struct { + libName, bp string + errorMsg string // a negative test; must have error message + flags []string // must have substrings in tidyFlags + noFlags []string // must not have substrings in tidyFlags + } + + testCases := []testCase{ + { + "libfoo1", + `cc_library_shared { // no warnings-as-errors, good tidy_flags + name: "libfoo1", + srcs: ["foo.c"], + tidy_flags: ["-header-filter=dir1/"], + }`, + "", + []string{"-header-filter=dir1/"}, + []string{"-warnings-as-errors"}, + }, + { + "libfoo2", + `cc_library_shared { // good use of tidy_checks_as_errors + name: "libfoo2", + srcs: ["foo.c"], + tidy_checks_as_errors: ["xyz-*", "abc"], + }`, + "", + []string{ + "-header-filter=^", // there is a default header filter + "-warnings-as-errors='xyz-*',abc,${config.TidyGlobalNoErrorChecks}", + }, + []string{}, + }, + } + if NoWarningsAsErrorsInTidyFlags { + testCases = append(testCases, testCase{ + "libfoo3", + `cc_library_shared { // bad use of -warnings-as-errors in tidy_flags + name: "libfoo3", + srcs: ["foo.c"], + tidy_flags: [ + "-header-filters=.*", + "-warnings-as-errors=xyz-*", + ], + }`, + `module "libfoo3" .*: tidy_flags: should not contain .*;` + + ` use tidy_checks_as_errors instead`, + []string{}, + []string{}, + }) + } + for _, test := range testCases { + if test.errorMsg != "" { + testCcError(t, test.errorMsg, test.bp) + continue + } + variant := "android_arm64_armv8-a_shared" + ctx := testCc(t, test.bp) + t.Run("caseTidyFlags", func(t *testing.T) { + flags := ctx.ModuleForTests(test.libName, variant).Rule("clangTidy").Args["tidyFlags"] + for _, flag := range test.flags { + if !strings.Contains(flags, flag) { + t.Errorf("tidyFlags %v for %s does not contain %s.", flags, test.libName, flag) + } + } + for _, flag := range test.noFlags { + if strings.Contains(flags, flag) { + t.Errorf("tidyFlags %v for %s should not contain %s.", flags, test.libName, flag) + } + } + }) + } +} + +func TestTidyChecks(t *testing.T) { + // The "tidy_checks" property defines additional checks appended + // to global default. But there are some checks disabled after + // the local tidy_checks. + bp := ` + cc_library_shared { // has global checks + extraGlobalChecks + name: "libfoo_1", + srcs: ["foo.c"], + } + cc_library_shared { // has only local checks + extraGlobalChecks + name: "libfoo_2", + srcs: ["foo.c"], + tidy_checks: ["-*", "xyz-*"], + } + cc_library_shared { // has global checks + local checks + extraGlobalChecks + name: "libfoo_3", + srcs: ["foo.c"], + tidy_checks: ["-abc*", "xyz-*", "mycheck"], + } + cc_library_shared { // has only local checks after "-*" + extraGlobalChecks + name: "libfoo_4", + srcs: ["foo.c"], + tidy_checks: ["-abc*", "xyz-*", "mycheck", "-*", "xyz-*"], + }` + ctx := testCc(t, bp) + + globalChecks := "-checks=${config.TidyDefaultGlobalChecks}," + firstXyzChecks := "-checks='-*','xyz-*'," + localXyzChecks := "'-*','xyz-*'" + localAbcChecks := "'-abc*','xyz-*',mycheck" + extraGlobalChecks := ",${config.TidyGlobalNoChecks}" + testCases := []struct { + libNumber int // 1,2,3,... + checks []string // must have substrings in -checks + noChecks []string // must not have substrings in -checks + }{ + {1, []string{globalChecks, extraGlobalChecks}, []string{localXyzChecks, localAbcChecks}}, + {2, []string{firstXyzChecks, extraGlobalChecks}, []string{globalChecks, localAbcChecks}}, + {3, []string{globalChecks, localAbcChecks, extraGlobalChecks}, []string{localXyzChecks}}, + {4, []string{firstXyzChecks, extraGlobalChecks}, []string{globalChecks, localAbcChecks}}, + } + t.Run("caseTidyChecks", func(t *testing.T) { + variant := "android_arm64_armv8-a_shared" + for _, test := range testCases { + libName := fmt.Sprintf("libfoo_%d", test.libNumber) + flags := ctx.ModuleForTests(libName, variant).Rule("clangTidy").Args["tidyFlags"] + splitFlags := strings.Split(flags, " ") + foundCheckFlag := false + for _, flag := range splitFlags { + if strings.HasPrefix(flag, "-checks=") { + foundCheckFlag = true + for _, check := range test.checks { + if !strings.Contains(flag, check) { + t.Errorf("tidyFlags for %s does not contain %s.", libName, check) + } + } + for _, check := range test.noChecks { + if strings.Contains(flag, check) { + t.Errorf("tidyFlags for %s should not contain %s.", libName, check) + } + } + break + } + } + if !foundCheckFlag { + t.Errorf("tidyFlags for %s does not contain -checks=.", libName) + } + } + }) +} + func TestWithTidy(t *testing.T) { // When WITH_TIDY=1 or (ALLOW_LOCAL_TIDY_TRUE=1 and local tidy:true) // a C++ library should depend on .tidy files. diff --git a/cmd/path_interposer/main.go b/cmd/path_interposer/main.go index a4fe3e489..8b9de5246 100644 --- a/cmd/path_interposer/main.go +++ b/cmd/path_interposer/main.go @@ -12,6 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +// This tool tries to prohibit access to tools on the system on which the build +// is run. +// +// The rationale is that if the build uses a binary that is not shipped in the +// source tree, it is unknowable which version of that binary will be installed +// and therefore the output of the build will be unpredictable. Therefore, we +// should make every effort to use only tools under our control. +// +// This is currently implemented by a "sandbox" that sets $PATH to a specific, +// single directory and creates a symlink for every binary in $PATH in it. That +// symlink will point to path_interposer, which then uses an embedded +// configuration to determine whether to allow access to the binary (in which +// case it calls the original executable) or not (in which case it fails). It +// can also optionally log invocations. +// +// This, of course, does not help if one invokes the tool in question with its +// full path. package main import ( diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 4b3161b33..8a3d6e037 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -129,44 +129,27 @@ func newConfig(availableEnv map[string]string) android.Config { return configuration } -// Bazel-enabled mode. Soong runs in two passes. -// First pass: Analyze the build tree, but only store all bazel commands -// needed to correctly evaluate the tree in the second pass. -// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite -// the incorrect results from the first pass, and file I/O is expensive. -func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { - firstCtx.EventHandler.Begin("mixed_build") - defer firstCtx.EventHandler.End("mixed_build") - - firstCtx.EventHandler.Begin("prepare") - bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration) - firstCtx.EventHandler.End("prepare") - - firstCtx.EventHandler.Begin("bazel") - // Invoke bazel commands and save results for second pass. - if err := configuration.BazelContext.InvokeBazel(); err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) - } - // Second pass: Full analysis, using the bazel command results. Output ninja file. - secondConfig, err := android.ConfigForAdditionalRun(configuration) - if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) +// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a +// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata +// for modules that should be handled by Bazel. +func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) { + ctx.EventHandler.Begin("mixed_build") + defer ctx.EventHandler.End("mixed_build") + + bazelHook := func() error { + ctx.EventHandler.Begin("bazel") + defer ctx.EventHandler.End("bazel") + return configuration.BazelContext.InvokeBazel(configuration) } - firstCtx.EventHandler.End("bazel") + ctx.SetBeforePrepareBuildActionsHook(bazelHook) - secondCtx := newContext(secondConfig) - secondCtx.EventHandler = firstCtx.EventHandler - secondCtx.EventHandler.Begin("analyze") - ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig) + ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration) ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - secondCtx.EventHandler.End("analyze") - globListFiles := writeBuildGlobsNinjaFile(secondCtx, configuration.SoongOutDir(), configuration) + globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration) ninjaDeps = append(ninjaDeps, globListFiles...) - writeDepFile(cmdlineArgs.OutFile, *secondCtx.EventHandler, ninjaDeps) + writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps) } // Run the code-generation phase to convert BazelTargetModules to BUILD files. @@ -183,8 +166,7 @@ func runQueryView(queryviewDir, queryviewMarker string, configuration android.Co touch(shared.JoinPath(topDir, queryviewMarker)) } -func writeMetrics(configuration android.Config, eventHandler metrics.EventHandler) { - metricsDir := configuration.Getenv("LOG_DIR") +func writeMetrics(configuration android.Config, eventHandler metrics.EventHandler, metricsDir string) { if len(metricsDir) < 1 { fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n") os.Exit(1) @@ -238,7 +220,7 @@ func writeDepFile(outputFile string, eventHandler metrics.EventHandler, ninjaDep // doChosenActivity runs Soong for a specific activity, like bp2build, queryview // or the actual Soong build for the build.ninja file. Returns the top level // output file of the specific activity. -func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { +func doChosenActivity(configuration android.Config, extraNinjaDeps []string, logDir string) string { mixedModeBuild := configuration.BazelContext.BazelEnabled() generateBazelWorkspace := bp2buildMarker != "" generateQueryView := bazelQueryViewDir != "" @@ -302,7 +284,7 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str } } - writeMetrics(configuration, *ctx.EventHandler) + writeMetrics(configuration, *ctx.EventHandler, logDir) return cmdlineArgs.OutFile } @@ -358,7 +340,11 @@ func main() { extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve")) } - finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) + // Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will + // change between every CI build, so tracking it would require re-running Soong for every build. + logDir := availableEnv["LOG_DIR"] + + finalOutputFile := doChosenActivity(configuration, extraNinjaDeps, logDir) writeUsedEnvironmentFile(configuration, finalOutputFile) } @@ -460,6 +446,8 @@ func getTemporaryExcludes() []string { // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel excludes = append(excludes, "external/autotest/venv/autotest_lib") + excludes = append(excludes, "external/autotest/autotest_lib") + excludes = append(excludes, "external/autotest/client/autotest_lib/client") // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison // It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore @@ -492,88 +480,88 @@ func getExistingBazelRelatedFiles(topDir string) ([]string, error) { // Bazel BUILD files instead of Ninja files. func runBp2Build(configuration android.Config, extraNinjaDeps []string) { eventHandler := metrics.EventHandler{} - eventHandler.Begin("bp2build") - - // Register an alternate set of singletons and mutators for bazel - // conversion for Bazel conversion. - bp2buildCtx := android.NewContext(configuration) - - // Soong internals like LoadHooks behave differently when running as - // bp2build. This is the bit to differentiate between Soong-as-Soong and - // Soong-as-bp2build. - bp2buildCtx.SetRunningAsBp2build() - - // Propagate "allow misssing dependencies" bit. This is normally set in - // newContext(), but we create bp2buildCtx without calling that method. - bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) - bp2buildCtx.SetNameInterface(newNameResolver(configuration)) - bp2buildCtx.RegisterForBazelConversion() - - // The bp2build process is a purely functional process that only depends on - // Android.bp files. It must not depend on the values of per-build product - // configurations or variables, since those will generate different BUILD - // files based on how the user has configured their tree. - bp2buildCtx.SetModuleListFile(cmdlineArgs.ModuleListFile) - modulePaths, err := bp2buildCtx.ListModulePaths(".") - if err != nil { - panic(err) - } - - extraNinjaDeps = append(extraNinjaDeps, modulePaths...) + var metrics bp2build.CodegenMetrics + eventHandler.Do("bp2build", func() { + + // Register an alternate set of singletons and mutators for bazel + // conversion for Bazel conversion. + bp2buildCtx := android.NewContext(configuration) + + // Soong internals like LoadHooks behave differently when running as + // bp2build. This is the bit to differentiate between Soong-as-Soong and + // Soong-as-bp2build. + bp2buildCtx.SetRunningAsBp2build() + + // Propagate "allow misssing dependencies" bit. This is normally set in + // newContext(), but we create bp2buildCtx without calling that method. + bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) + bp2buildCtx.SetNameInterface(newNameResolver(configuration)) + bp2buildCtx.RegisterForBazelConversion() + + // The bp2build process is a purely functional process that only depends on + // Android.bp files. It must not depend on the values of per-build product + // configurations or variables, since those will generate different BUILD + // files based on how the user has configured their tree. + bp2buildCtx.SetModuleListFile(cmdlineArgs.ModuleListFile) + modulePaths, err := bp2buildCtx.ListModulePaths(".") + if err != nil { + panic(err) + } - // Run the loading and analysis pipeline to prepare the graph of regular - // Modules parsed from Android.bp files, and the BazelTargetModules mapped - // from the regular Modules. - blueprintArgs := cmdlineArgs - ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration) - ninjaDeps = append(ninjaDeps, extraNinjaDeps...) + extraNinjaDeps = append(extraNinjaDeps, modulePaths...) - globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx, configuration.SoongOutDir(), configuration) - ninjaDeps = append(ninjaDeps, globListFiles...) + // Run the loading and analysis pipeline to prepare the graph of regular + // Modules parsed from Android.bp files, and the BazelTargetModules mapped + // from the regular Modules. + blueprintArgs := cmdlineArgs + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration) + ninjaDeps = append(ninjaDeps, extraNinjaDeps...) - // Run the code-generation phase to convert BazelTargetModules to BUILD files - // and print conversion metrics to the user. - codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) - metrics := bp2build.Codegen(codegenContext) + globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx, configuration.SoongOutDir(), configuration) + ninjaDeps = append(ninjaDeps, globListFiles...) - generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build") - workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace") + // Run the code-generation phase to convert BazelTargetModules to BUILD files + // and print conversion metrics to the user. + codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) + metrics = bp2build.Codegen(codegenContext) - excludes := []string{ - "bazel-bin", - "bazel-genfiles", - "bazel-out", - "bazel-testlogs", - "bazel-" + filepath.Base(topDir), - } + generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build") + workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace") - if outDir[0] != '/' { - excludes = append(excludes, outDir) - } + excludes := []string{ + "bazel-bin", + "bazel-genfiles", + "bazel-out", + "bazel-testlogs", + "bazel-" + filepath.Base(topDir), + } - existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) - if err != nil { - fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err) - os.Exit(1) - } + if outDir[0] != '/' { + excludes = append(excludes, outDir) + } - pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(topDir, generatedRoot, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE")) - excludes = append(excludes, pathsToIgnoredBuildFiles...) + existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err) + os.Exit(1) + } - excludes = append(excludes, getTemporaryExcludes()...) + pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(topDir, generatedRoot, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE")) + excludes = append(excludes, pathsToIgnoredBuildFiles...) - symlinkForestDeps := bp2build.PlantSymlinkForest( - topDir, workspaceRoot, generatedRoot, ".", excludes) + excludes = append(excludes, getTemporaryExcludes()...) - ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) - ninjaDeps = append(ninjaDeps, symlinkForestDeps...) + symlinkForestDeps := bp2build.PlantSymlinkForest( + topDir, workspaceRoot, generatedRoot, ".", excludes) - writeDepFile(bp2buildMarker, eventHandler, ninjaDeps) + ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) + ninjaDeps = append(ninjaDeps, symlinkForestDeps...) - // Create an empty bp2build marker file. - touch(shared.JoinPath(topDir, bp2buildMarker)) + writeDepFile(bp2buildMarker, eventHandler, ninjaDeps) - eventHandler.End("bp2build") + // Create an empty bp2build marker file. + touch(shared.JoinPath(topDir, bp2buildMarker)) + }) // Only report metrics when in bp2build mode. The metrics aren't relevant // for queryview, since that's a total repo-wide conversion and there's a diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go index a03a86a49..16f994d55 100644 --- a/cmd/soong_ui/main.go +++ b/cmd/soong_ui/main.go @@ -221,6 +221,7 @@ func main() { } defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...) defer met.Dump(soongMetricsFile) + defer build.CheckProdCreds(buildCtx, config) } // Read the time at the starting point. diff --git a/compliance/copy_license_metadata/Android.bp b/compliance/copy_license_metadata/Android.bp new file mode 100644 index 000000000..83019eb2f --- /dev/null +++ b/compliance/copy_license_metadata/Android.bp @@ -0,0 +1,30 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "copy_license_metadata", + srcs: [ + "copy_license_metadata.go", + ], + deps: [ + "license_metadata_proto", + "golang-protobuf-proto", + "golang-protobuf-encoding-prototext", + "soong-response", + ], +} diff --git a/compliance/copy_license_metadata/copy_license_metadata.go b/compliance/copy_license_metadata/copy_license_metadata.go new file mode 100644 index 000000000..36b94898c --- /dev/null +++ b/compliance/copy_license_metadata/copy_license_metadata.go @@ -0,0 +1,144 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "strings" + + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" + + "android/soong/compliance/license_metadata_proto" + "android/soong/response" +) + +func newMultiString(flags *flag.FlagSet, name, usage string) *multiString { + var f multiString + flags.Var(&f, name, usage) + return &f +} + +type multiString []string + +func (ms *multiString) String() string { return strings.Join(*ms, ", ") } +func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil } + +func main() { + var expandedArgs []string + for _, arg := range os.Args[1:] { + if strings.HasPrefix(arg, "@") { + f, err := os.Open(strings.TrimPrefix(arg, "@")) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + respArgs, err := response.ReadRspFile(f) + f.Close() + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + expandedArgs = append(expandedArgs, respArgs...) + } else { + expandedArgs = append(expandedArgs, arg) + } + } + + flags := flag.NewFlagSet("flags", flag.ExitOnError) + + installed := flags.String("i", "", "installed target") + sources := newMultiString(flags, "s", "source (input) file") + dep := flags.String("d", "", "license metadata file dependency") + outFile := flags.String("o", "", "output file") + + flags.Parse(expandedArgs) + + if len(*dep) == 0 || len(*installed) == 0 || len(*sources) == 0 { + flags.Usage() + if len(*dep) == 0 { + fmt.Fprintf(os.Stderr, "source license metadata (-d flag) required\n") + } + if len(*sources) == 0 { + fmt.Fprintf(os.Stderr, "source copy (-s flag required\n") + } + if len(*installed) == 0 { + fmt.Fprintf(os.Stderr, "installed copy (-i flag) required\n") + } + os.Exit(1) + } + + src_metadata := license_metadata_proto.LicenseMetadata{} + err := readMetadata(*dep, &src_metadata) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) + os.Exit(2) + } + + metadata := src_metadata + metadata.Built = nil + metadata.InstallMap = nil + metadata.Installed = []string{*installed} + metadata.Sources = *sources + metadata.Deps = []*license_metadata_proto.AnnotatedDependency{&license_metadata_proto.AnnotatedDependency{ + File: proto.String(*dep), + Annotations: []string{"static"}, + }} + + err = writeMetadata(*outFile, &metadata) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) + os.Exit(2) + } +} + +func readMetadata(file string, metadata *license_metadata_proto.LicenseMetadata) error { + if file == "" { + return fmt.Errorf("source metadata file (-d) required") + } + buf, err := ioutil.ReadFile(file) + if err != nil { + return fmt.Errorf("error reading textproto %q: %w", file, err) + } + + err = prototext.Unmarshal(buf, metadata) + if err != nil { + return fmt.Errorf("error unmarshalling textproto: %w", err) + } + + return nil +} + +func writeMetadata(file string, metadata *license_metadata_proto.LicenseMetadata) error { + buf, err := prototext.MarshalOptions{Multiline: true}.Marshal(metadata) + if err != nil { + return fmt.Errorf("error marshalling textproto: %w", err) + } + + if file != "" { + err = ioutil.WriteFile(file, buf, 0666) + if err != nil { + return fmt.Errorf("error writing textproto %q: %w", file, err) + } + } else { + _, _ = os.Stdout.Write(buf) + } + + return nil +} diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go index 36513b64b..7bc9ab261 100644 --- a/dexpreopt/class_loader_context.go +++ b/dexpreopt/class_loader_context.go @@ -196,10 +196,6 @@ type ClassLoaderContext struct { // If the library is optional or required. Optional bool - // If the library is implicitly infered by Soong (as opposed to explicitly added via `uses_libs` - // or `optional_uses_libs`. - Implicit bool - // On-host build path to the library dex file (used in dex2oat argument --class-loader-context). Host android.Path @@ -290,9 +286,8 @@ const UnknownInstallLibraryPath = "error" const AnySdkVersion int = android.FutureApiLevelInt // Add class loader context for the given library to the map entry for the given SDK version. -func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathContext, sdkVer int, - lib string, optional, implicit bool, hostPath, installPath android.Path, - nestedClcMap ClassLoaderContextMap) error { +func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathContext, sdkVer int, lib string, + optional bool, hostPath, installPath android.Path, nestedClcMap ClassLoaderContextMap) error { // For prebuilts, library should have the same name as the source module. lib = android.RemoveOptionalPrebuiltPrefix(lib) @@ -341,7 +336,6 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont clcMap[sdkVer] = append(clcMap[sdkVer], &ClassLoaderContext{ Name: lib, Optional: optional, - Implicit: implicit, Host: hostPath, Device: devicePath, Subcontexts: subcontexts, @@ -354,10 +348,9 @@ func (clcMap ClassLoaderContextMap) addContext(ctx android.ModuleInstallPathCont // about paths). For the subset of libraries that are used in dexpreopt, their build/install paths // are validated later before CLC is used (in validateClassLoaderContext). func (clcMap ClassLoaderContextMap) AddContext(ctx android.ModuleInstallPathContext, sdkVer int, - lib string, optional, implicit bool, hostPath, installPath android.Path, - nestedClcMap ClassLoaderContextMap) { + lib string, optional bool, hostPath, installPath android.Path, nestedClcMap ClassLoaderContextMap) { - err := clcMap.addContext(ctx, sdkVer, lib, optional, implicit, hostPath, installPath, nestedClcMap) + err := clcMap.addContext(ctx, sdkVer, lib, optional, hostPath, installPath, nestedClcMap) if err != nil { ctx.ModuleErrorf(err.Error()) } @@ -401,15 +394,13 @@ func (clcMap ClassLoaderContextMap) AddContextMap(otherClcMap ClassLoaderContext // included). This is the list of libraries that should be in the <uses-library> tags in the // manifest. Some of them may be present in the source manifest, others are added by manifest_fixer. // Required and optional libraries are in separate lists. -func (clcMap ClassLoaderContextMap) usesLibs(implicit bool) (required []string, optional []string) { +func (clcMap ClassLoaderContextMap) UsesLibs() (required []string, optional []string) { if clcMap != nil { clcs := clcMap[AnySdkVersion] required = make([]string, 0, len(clcs)) optional = make([]string, 0, len(clcs)) for _, clc := range clcs { - if implicit && !clc.Implicit { - // Skip, this is an explicit library and we need only the implicit ones. - } else if clc.Optional { + if clc.Optional { optional = append(optional, clc.Name) } else { required = append(required, clc.Name) @@ -419,14 +410,6 @@ func (clcMap ClassLoaderContextMap) usesLibs(implicit bool) (required []string, return required, optional } -func (clcMap ClassLoaderContextMap) UsesLibs() ([]string, []string) { - return clcMap.usesLibs(false) -} - -func (clcMap ClassLoaderContextMap) ImplicitUsesLibs() ([]string, []string) { - return clcMap.usesLibs(true) -} - func (clcMap ClassLoaderContextMap) Dump() string { jsonCLC := toJsonClassLoaderContext(clcMap) bytes, err := json.MarshalIndent(jsonCLC, "", " ") @@ -631,7 +614,6 @@ func computeClassLoaderContextRec(clcs []*ClassLoaderContext) (string, string, a type jsonClassLoaderContext struct { Name string Optional bool - Implicit bool Host string Device string Subcontexts []*jsonClassLoaderContext @@ -664,7 +646,6 @@ func fromJsonClassLoaderContextRec(ctx android.PathContext, jClcs []*jsonClassLo clcs = append(clcs, &ClassLoaderContext{ Name: clc.Name, Optional: clc.Optional, - Implicit: clc.Implicit, Host: constructPath(ctx, clc.Host), Device: clc.Device, Subcontexts: fromJsonClassLoaderContextRec(ctx, clc.Subcontexts), @@ -678,6 +659,9 @@ func toJsonClassLoaderContext(clcMap ClassLoaderContextMap) jsonClassLoaderConte jClcMap := make(jsonClassLoaderContextMap) for sdkVer, clcs := range clcMap { sdkVerStr := fmt.Sprintf("%d", sdkVer) + if sdkVer == AnySdkVersion { + sdkVerStr = "any" + } jClcMap[sdkVerStr] = toJsonClassLoaderContextRec(clcs) } return jClcMap @@ -697,7 +681,6 @@ func toJsonClassLoaderContextRec(clcs []*ClassLoaderContext) []*jsonClassLoaderC jClcs[i] = &jsonClassLoaderContext{ Name: clc.Name, Optional: clc.Optional, - Implicit: clc.Implicit, Host: host, Device: clc.Device, Subcontexts: toJsonClassLoaderContextRec(clc.Subcontexts), diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go index 5d3a9d943..8b3c0130c 100644 --- a/dexpreopt/class_loader_context_test.go +++ b/dexpreopt/class_loader_context_test.go @@ -50,34 +50,33 @@ func TestCLC(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, AnySdkVersion, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) // Add some libraries with nested subcontexts. m1 := make(ClassLoaderContextMap) - m1.AddContext(ctx, AnySdkVersion, "a1", optional, implicit, buildPath(ctx, "a1"), installPath(ctx, "a1"), nil) - m1.AddContext(ctx, AnySdkVersion, "b1", optional, implicit, buildPath(ctx, "b1"), installPath(ctx, "b1"), nil) + m1.AddContext(ctx, AnySdkVersion, "a1", optional, buildPath(ctx, "a1"), installPath(ctx, "a1"), nil) + m1.AddContext(ctx, AnySdkVersion, "b1", optional, buildPath(ctx, "b1"), installPath(ctx, "b1"), nil) m2 := make(ClassLoaderContextMap) - m2.AddContext(ctx, AnySdkVersion, "a2", optional, implicit, buildPath(ctx, "a2"), installPath(ctx, "a2"), nil) - m2.AddContext(ctx, AnySdkVersion, "b2", optional, implicit, buildPath(ctx, "b2"), installPath(ctx, "b2"), nil) - m2.AddContext(ctx, AnySdkVersion, "c2", optional, implicit, buildPath(ctx, "c2"), installPath(ctx, "c2"), m1) + m2.AddContext(ctx, AnySdkVersion, "a2", optional, buildPath(ctx, "a2"), installPath(ctx, "a2"), nil) + m2.AddContext(ctx, AnySdkVersion, "b2", optional, buildPath(ctx, "b2"), installPath(ctx, "b2"), nil) + m2.AddContext(ctx, AnySdkVersion, "c2", optional, buildPath(ctx, "c2"), installPath(ctx, "c2"), m1) m3 := make(ClassLoaderContextMap) - m3.AddContext(ctx, AnySdkVersion, "a3", optional, implicit, buildPath(ctx, "a3"), installPath(ctx, "a3"), nil) - m3.AddContext(ctx, AnySdkVersion, "b3", optional, implicit, buildPath(ctx, "b3"), installPath(ctx, "b3"), nil) + m3.AddContext(ctx, AnySdkVersion, "a3", optional, buildPath(ctx, "a3"), installPath(ctx, "a3"), nil) + m3.AddContext(ctx, AnySdkVersion, "b3", optional, buildPath(ctx, "b3"), installPath(ctx, "b3"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), m2) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), m2) // When the same library is both in conditional and unconditional context, it should be removed // from conditional context. - m.AddContext(ctx, 42, "f", optional, implicit, buildPath(ctx, "f"), installPath(ctx, "f"), nil) - m.AddContext(ctx, AnySdkVersion, "f", optional, implicit, buildPath(ctx, "f"), installPath(ctx, "f"), nil) + m.AddContext(ctx, 42, "f", optional, buildPath(ctx, "f"), installPath(ctx, "f"), nil) + m.AddContext(ctx, AnySdkVersion, "f", optional, buildPath(ctx, "f"), installPath(ctx, "f"), nil) // Merge map with implicit root library that is among toplevel contexts => does nothing. m.AddContextMap(m1, "c") @@ -86,12 +85,12 @@ func TestCLC(t *testing.T) { m.AddContextMap(m3, "m_g") // Compatibility libraries with unknown install paths get default paths. - m.AddContext(ctx, 29, AndroidHidlManager, optional, implicit, buildPath(ctx, AndroidHidlManager), nil, nil) - m.AddContext(ctx, 29, AndroidHidlBase, optional, implicit, buildPath(ctx, AndroidHidlBase), nil, nil) + m.AddContext(ctx, 29, AndroidHidlManager, optional, buildPath(ctx, AndroidHidlManager), nil, nil) + m.AddContext(ctx, 29, AndroidHidlBase, optional, buildPath(ctx, AndroidHidlBase), nil, nil) // Add "android.test.mock" to conditional CLC, observe that is gets removed because it is only // needed as a compatibility library if "android.test.runner" is in CLC as well. - m.AddContext(ctx, 30, AndroidTestMock, optional, implicit, buildPath(ctx, AndroidTestMock), nil, nil) + m.AddContext(ctx, 30, AndroidTestMock, optional, buildPath(ctx, AndroidTestMock), nil, nil) valid, validationError := validateClassLoaderContext(m) @@ -165,12 +164,11 @@ func TestCLC(t *testing.T) { func TestCLCJson(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 29, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, 30, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 29, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 30, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), nil) jsonCLC := toJsonClassLoaderContext(m) restored := fromJsonClassLoaderContext(ctx, jsonCLC) android.AssertIntEquals(t, "The size of the maps should be the same.", len(m), len(restored)) @@ -191,13 +189,12 @@ func TestCLCJson(t *testing.T) { func testCLCUnknownPath(t *testing.T, whichPath string) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) if whichPath == "build" { - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, nil, nil, nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, nil, nil, nil) } else { - m.AddContext(ctx, AnySdkVersion, "a", optional, implicit, buildPath(ctx, "a"), nil, nil) + m.AddContext(ctx, AnySdkVersion, "a", optional, buildPath(ctx, "a"), nil, nil) } // The library should be added to <uses-library> tags by the manifest_fixer. @@ -232,11 +229,10 @@ func TestCLCUnknownInstallPath(t *testing.T) { func TestCLCNestedConditional(t *testing.T) { ctx := testContext() optional := false - implicit := true m1 := make(ClassLoaderContextMap) - m1.AddContext(ctx, 42, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m1.AddContext(ctx, 42, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) m := make(ClassLoaderContextMap) - err := m.addContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), m1) + err := m.addContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), m1) checkError(t, err, "nested class loader context shouldn't have conditional part") } @@ -245,12 +241,11 @@ func TestCLCNestedConditional(t *testing.T) { func TestCLCSdkVersionOrder(t *testing.T) { ctx := testContext() optional := false - implicit := true m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 29, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - m.AddContext(ctx, 30, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) - m.AddContext(ctx, AnySdkVersion, "d", optional, implicit, buildPath(ctx, "d"), installPath(ctx, "d"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 29, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 30, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + m.AddContext(ctx, AnySdkVersion, "d", optional, buildPath(ctx, "d"), installPath(ctx, "d"), nil) valid, validationError := validateClassLoaderContext(m) @@ -287,7 +282,6 @@ func TestCLCSdkVersionOrder(t *testing.T) { func TestCLCMExcludeLibs(t *testing.T) { ctx := testContext() const optional = false - const implicit = true excludeLibs := func(t *testing.T, m ClassLoaderContextMap, excluded_libs ...string) ClassLoaderContextMap { // Dump the CLCM before creating a new copy that excludes a specific set of libraries. @@ -305,7 +299,7 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("exclude nothing", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) a := excludeLibs(t, m) @@ -314,7 +308,6 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "a", "Optional": false, - "Implicit": true, "Host": "out/soong/a.jar", "Device": "/system/a.jar", "Subcontexts": [] @@ -325,8 +318,8 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("one item from list", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) a := excludeLibs(t, m, "a") @@ -335,7 +328,6 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "b", "Optional": false, - "Implicit": true, "Host": "out/soong/b.jar", "Device": "/system/b.jar", "Subcontexts": [] @@ -347,8 +339,8 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("all items from a list", func(t *testing.T) { m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil) - m.AddContext(ctx, 28, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, 28, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) a := excludeLibs(t, m, "a", "b") @@ -357,11 +349,11 @@ func TestCLCMExcludeLibs(t *testing.T) { t.Run("items from a subcontext", func(t *testing.T) { s := make(ClassLoaderContextMap) - s.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil) - s.AddContext(ctx, AnySdkVersion, "c", optional, implicit, buildPath(ctx, "c"), installPath(ctx, "c"), nil) + s.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + s.AddContext(ctx, AnySdkVersion, "c", optional, buildPath(ctx, "c"), installPath(ctx, "c"), nil) m := make(ClassLoaderContextMap) - m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), s) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), s) a := excludeLibs(t, m, "b") @@ -370,14 +362,12 @@ func TestCLCMExcludeLibs(t *testing.T) { { "Name": "a", "Optional": false, - "Implicit": true, "Host": "out/soong/a.jar", "Device": "/system/a.jar", "Subcontexts": [ { "Name": "c", "Optional": false, - "Implicit": true, "Host": "out/soong/c.jar", "Device": "/system/c.jar", "Subcontexts": [] @@ -389,6 +379,35 @@ func TestCLCMExcludeLibs(t *testing.T) { }) } +// Test that CLC is correctly serialized to JSON. +func TestCLCtoJSON(t *testing.T) { + ctx := testContext() + optional := false + m := make(ClassLoaderContextMap) + m.AddContext(ctx, 28, "a", optional, buildPath(ctx, "a"), installPath(ctx, "a"), nil) + m.AddContext(ctx, AnySdkVersion, "b", optional, buildPath(ctx, "b"), installPath(ctx, "b"), nil) + android.AssertStringEquals(t, "output CLCM ", `{ + "28": [ + { + "Name": "a", + "Optional": false, + "Host": "out/soong/a.jar", + "Device": "/system/a.jar", + "Subcontexts": [] + } + ], + "any": [ + { + "Name": "b", + "Optional": false, + "Host": "out/soong/b.jar", + "Device": "/system/b.jar", + "Subcontexts": [] + } + ] +}`, m.Dump()) +} + func checkError(t *testing.T, have error, want string) { if have == nil { t.Errorf("\nwant error: '%s'\nhave: none", want) diff --git a/dexpreopt/config.go b/dexpreopt/config.go index 153b0256f..64cd46a76 100644 --- a/dexpreopt/config.go +++ b/dexpreopt/config.go @@ -36,8 +36,6 @@ type GlobalConfig struct { PreoptWithUpdatableBcp bool // If updatable boot jars are included in dexpreopt or not. - UseArtImage bool // use the art image (use other boot class path dex files without image) - HasSystemOther bool // store odex files that match PatternsOnSystemOther on the system_other partition PatternsOnSystemOther []string // patterns (using '%' to denote a prefix match) to put odex on the system_other partition @@ -252,8 +250,9 @@ func ParseGlobalConfig(ctx android.PathContext, data []byte) (*GlobalConfig, err } type globalConfigAndRaw struct { - global *GlobalConfig - data []byte + global *GlobalConfig + data []byte + pathErrors []error } // GetGlobalConfig returns the global dexpreopt.config that's created in the @@ -274,16 +273,26 @@ func GetGlobalConfigRawData(ctx android.PathContext) []byte { var globalConfigOnceKey = android.NewOnceKey("DexpreoptGlobalConfig") var testGlobalConfigOnceKey = android.NewOnceKey("TestDexpreoptGlobalConfig") +type pathContextErrorCollector struct { + android.PathContext + errors []error +} + +func (p *pathContextErrorCollector) Errorf(format string, args ...interface{}) { + p.errors = append(p.errors, fmt.Errorf(format, args...)) +} + func getGlobalConfigRaw(ctx android.PathContext) globalConfigAndRaw { - return ctx.Config().Once(globalConfigOnceKey, func() interface{} { + config := ctx.Config().Once(globalConfigOnceKey, func() interface{} { if data, err := ctx.Config().DexpreoptGlobalConfig(ctx); err != nil { panic(err) } else if data != nil { - globalConfig, err := ParseGlobalConfig(ctx, data) + pathErrorCollectorCtx := &pathContextErrorCollector{PathContext: ctx} + globalConfig, err := ParseGlobalConfig(pathErrorCollectorCtx, data) if err != nil { panic(err) } - return globalConfigAndRaw{globalConfig, data} + return globalConfigAndRaw{globalConfig, data, pathErrorCollectorCtx.errors} } // No global config filename set, see if there is a test config set @@ -293,16 +302,35 @@ func getGlobalConfigRaw(ctx android.PathContext) globalConfigAndRaw { DisablePreopt: true, DisablePreoptBootImages: true, DisableGenerateProfile: true, - }, nil} + }, nil, nil} }) }).(globalConfigAndRaw) + + // Avoid non-deterministic errors by reporting cached path errors on all callers. + for _, err := range config.pathErrors { + if ctx.Config().AllowMissingDependencies() { + // When AllowMissingDependencies it set, report errors through AddMissingDependencies. + // If AddMissingDependencies doesn't exist on the current context (for example when + // called with a SingletonContext), just swallow the errors since there is no way to + // report them. + if missingDepsCtx, ok := ctx.(interface { + AddMissingDependencies(missingDeps []string) + }); ok { + missingDepsCtx.AddMissingDependencies([]string{err.Error()}) + } + } else { + android.ReportPathErrorf(ctx, "%w", err) + } + } + + return config } // SetTestGlobalConfig sets a GlobalConfig that future calls to GetGlobalConfig // will return. It must be called before the first call to GetGlobalConfig for // the config. func SetTestGlobalConfig(config android.Config, globalConfig *GlobalConfig) { - config.Once(testGlobalConfigOnceKey, func() interface{} { return globalConfigAndRaw{globalConfig, nil} }) + config.Once(testGlobalConfigOnceKey, func() interface{} { return globalConfigAndRaw{globalConfig, nil, nil} }) } // This struct is required to convert ModuleConfig from/to JSON. diff --git a/docs/tidy.md b/docs/tidy.md index 890c3a036..2eb8234c7 100644 --- a/docs/tidy.md +++ b/docs/tidy.md @@ -31,7 +31,7 @@ The default global clang-tidy checks and flags are defined in The global default can be overwritten by module properties in Android.bp. -### `tidy` and `tidy_checks` +### `tidy`, `tidy_checks`, and `ALLOW_LOCAL_TIDY_TRUE` For example, in [system/bpf/Android.bp](https://android.googlesource.com/platform/system/bpf/+/refs/heads/master/Android.bp), @@ -52,8 +52,16 @@ cc_defaults { } ``` That means in normal builds, even without `WITH_TIDY=1`, -the modules that use `bpf_defaults` will run clang-tidy +the modules that use `bpf_defaults` _should_ run clang-tidy over C/C++ source files with the given `tidy_checks`. + +However since clang-tidy warnings and its runtime cost might +not be wanted by all people, the default is to ignore the +`tidy:true` property unless the environment variable +`ALLOW_LOCAL_TIDY_TRUE` is set to true or 1. +To run clang-tidy on all modules that should be tested with clang-tidy, +`ALLOW_LOCAL_TIDY_TRUE` or `WITH_TIDY` should be set to true or 1. + Note that `clang-analyzer-security*` is included in `tidy_checks` but not all `clang-analyzer-*` checks. Check `cert-err34-c` is disabled, although `cert-*` is selected. @@ -80,6 +88,9 @@ cc_test_library { } ``` +Note that `tidy:false` always disables clang-tidy, no matter +`ALLOW_LOCAL_TIDY_TRUE` is set or not. + ### `tidy_checks_as_errors` The global tidy checks are enabled as warnings. diff --git a/filesystem/Android.bp b/filesystem/Android.bp index 38684d3b2..dfcd405d3 100644 --- a/filesystem/Android.bp +++ b/filesystem/Android.bp @@ -12,9 +12,11 @@ bootstrap_go_package { "soong-linkerconfig", ], srcs: [ + "avb_add_hash_footer.go", "bootimg.go", "filesystem.go", "logical_partition.go", + "raw_binary.go", "system_image.go", "vbmeta.go", "testing.go", diff --git a/filesystem/avb_add_hash_footer.go b/filesystem/avb_add_hash_footer.go new file mode 100644 index 000000000..af3bdbe1a --- /dev/null +++ b/filesystem/avb_add_hash_footer.go @@ -0,0 +1,149 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystem + +import ( + "fmt" + "strconv" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +func init() { + android.RegisterModuleType("avb_add_hash_footer", avbAddHashFooterFactory) +} + +type avbAddHashFooter struct { + android.ModuleBase + + properties avbAddHashFooterProperties + + output android.OutputPath + installDir android.InstallPath +} + +type avbAddHashFooterProperties struct { + // Source file of this image. Can reference a genrule type module with the ":module" syntax. + Src *string `android:"path,arch_variant"` + + // Set the name of the output. Defaults to <module_name>.img. + Filename *string + + // Name of the image partition. Defaults to the name of this module. + Partition_name *string + + // Size of the partition. Defaults to dynamically calculating the size. + Partition_size *int64 + + // Path to the private key that avbtool will use to sign this image. + Private_key *string `android:"path"` + + // Algorithm that avbtool will use to sign this image. Default is SHA256_RSA4096. + Algorithm *string + + // The salt in hex. Required for reproducible builds. + Salt *string +} + +// The AVB footer adds verification information to the image. +func avbAddHashFooterFactory() android.Module { + module := &avbAddHashFooter{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +func (a *avbAddHashFooter) installFileName() string { + return proptools.StringDefault(a.properties.Filename, a.BaseModuleName()+".img") +} + +func (a *avbAddHashFooter) GenerateAndroidBuildActions(ctx android.ModuleContext) { + builder := android.NewRuleBuilder(pctx, ctx) + + if a.properties.Src == nil { + ctx.PropertyErrorf("src", "missing source file") + return + } + input := android.PathForModuleSrc(ctx, proptools.String(a.properties.Src)) + a.output = android.PathForModuleOut(ctx, a.installFileName()).OutputPath + builder.Command().Text("cp").Input(input).Output(a.output) + + cmd := builder.Command().BuiltTool("avbtool").Text("add_hash_footer") + + partition_name := proptools.StringDefault(a.properties.Partition_name, a.BaseModuleName()) + cmd.FlagWithArg("--partition_name ", partition_name) + + if a.properties.Partition_size == nil { + cmd.Flag("--dynamic_partition_size") + } else { + partition_size := proptools.Int(a.properties.Partition_size) + cmd.FlagWithArg("--partition_size ", strconv.Itoa(partition_size)) + } + + key := android.PathForModuleSrc(ctx, proptools.String(a.properties.Private_key)) + cmd.FlagWithInput("--key ", key) + + algorithm := proptools.StringDefault(a.properties.Algorithm, "SHA256_RSA4096") + cmd.FlagWithArg("--algorithm ", algorithm) + + if a.properties.Salt == nil { + ctx.PropertyErrorf("salt", "missing salt value") + return + } + cmd.FlagWithArg("--salt ", proptools.String(a.properties.Salt)) + + cmd.FlagWithOutput("--image ", a.output) + + builder.Build("avbAddHashFooter", fmt.Sprintf("avbAddHashFooter %s", ctx.ModuleName())) + + a.installDir = android.PathForModuleInstall(ctx, "etc") + ctx.InstallFile(a.installDir, a.installFileName(), a.output) +} + +var _ android.AndroidMkEntriesProvider = (*avbAddHashFooter)(nil) + +// Implements android.AndroidMkEntriesProvider +func (a *avbAddHashFooter) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{android.AndroidMkEntries{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(a.output), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", a.installDir.String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", a.installFileName()) + }, + }, + }} +} + +var _ Filesystem = (*avbAddHashFooter)(nil) + +func (a *avbAddHashFooter) OutputPath() android.Path { + return a.output +} + +func (a *avbAddHashFooter) SignedOutputPath() android.Path { + return a.OutputPath() // always signed +} + +// TODO(b/185115783): remove when not needed as input to a prebuilt_etc rule +var _ android.SourceFileProducer = (*avbAddHashFooter)(nil) + +// Implements android.SourceFileProducer +func (a *avbAddHashFooter) Srcs() android.Paths { + return append(android.Paths{}, a.output) +} diff --git a/filesystem/raw_binary.go b/filesystem/raw_binary.go new file mode 100644 index 000000000..1544ea788 --- /dev/null +++ b/filesystem/raw_binary.go @@ -0,0 +1,121 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystem + +import ( + "fmt" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +var ( + toRawBinary = pctx.AndroidStaticRule("toRawBinary", + blueprint.RuleParams{ + Command: "${objcopy} --output-target=binary ${in} ${out} &&" + + "chmod -x ${out}", + CommandDeps: []string{"$objcopy"}, + }, + "objcopy") +) + +func init() { + pctx.Import("android/soong/cc/config") + + android.RegisterModuleType("raw_binary", rawBinaryFactory) +} + +type rawBinary struct { + android.ModuleBase + + properties rawBinaryProperties + + output android.OutputPath + installDir android.InstallPath +} + +type rawBinaryProperties struct { + // Set the name of the output. Defaults to <module_name>.bin. + Stem *string + + // Name of input executable. Can be a name of a target. + Src *string `android:"path,arch_variant"` +} + +func rawBinaryFactory() android.Module { + module := &rawBinary{} + module.AddProperties(&module.properties) + android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst) + return module +} + +func (r *rawBinary) DepsMutator(ctx android.BottomUpMutatorContext) { + // do nothing +} + +func (r *rawBinary) installFileName() string { + return proptools.StringDefault(r.properties.Stem, r.BaseModuleName()+".bin") +} + +func (r *rawBinary) GenerateAndroidBuildActions(ctx android.ModuleContext) { + inputFile := android.PathForModuleSrc(ctx, proptools.String(r.properties.Src)) + outputFile := android.PathForModuleOut(ctx, r.installFileName()).OutputPath + + ctx.Build(pctx, android.BuildParams{ + Rule: toRawBinary, + Description: "raw binary " + outputFile.Base(), + Output: outputFile, + Input: inputFile, + Args: map[string]string{ + "objcopy": "${config.ClangBin}/llvm-objcopy", + }, + }) + + r.output = outputFile + r.installDir = android.PathForModuleInstall(ctx, "etc") + ctx.InstallFile(r.installDir, r.installFileName(), r.output) +} + +var _ android.AndroidMkEntriesProvider = (*rawBinary)(nil) + +// Implements android.AndroidMkEntriesProvider +func (r *rawBinary) AndroidMkEntries() []android.AndroidMkEntries { + return []android.AndroidMkEntries{{ + Class: "ETC", + OutputFile: android.OptionalPathForPath(r.output), + }} +} + +var _ Filesystem = (*rawBinary)(nil) + +func (r *rawBinary) OutputPath() android.Path { + return r.output +} + +func (r *rawBinary) SignedOutputPath() android.Path { + return nil +} + +var _ android.OutputFileProducer = (*rawBinary)(nil) + +// Implements android.OutputFileProducer +func (r *rawBinary) OutputFiles(tag string) (android.Paths, error) { + if tag == "" { + return []android.Path{r.output}, nil + } + return nil, fmt.Errorf("unsupported module reference tag %q", tag) +} diff --git a/finder/finder.go b/finder/finder.go index b4834b16b..c5196c8de 100644 --- a/finder/finder.go +++ b/finder/finder.go @@ -94,6 +94,10 @@ type CacheParams struct { // RootDirs are the root directories used to initiate the search RootDirs []string + // Whether symlinks are followed. If set, symlinks back to their own parent + // directory don't work. + FollowSymlinks bool + // ExcludeDirs are directory names that if encountered are removed from the search ExcludeDirs []string @@ -1415,9 +1419,14 @@ func (f *Finder) listDirSync(dir *pathMap) { // If stat fails this is probably a broken or dangling symlink, treat it as a file. subfiles = append(subfiles, child.Name()) } else if childStat.IsDir() { - // Skip symlink dirs. - // We don't have to support symlink dirs because - // that would cause duplicates. + // Skip symlink dirs if not requested otherwise. Android has a number + // of symlinks creating infinite source trees which would otherwise get + // us in an infinite loop. + // TODO(b/197349722): Revisit this once symlink loops are banned in the + // source tree. + if f.cacheMetadata.Config.FollowSymlinks { + subdirs = append(subdirs, child.Name()) + } } else { // We do have to support symlink files because the link name might be // different than the target name diff --git a/finder/finder_test.go b/finder/finder_test.go index 788dbdd35..8f73719a6 100644 --- a/finder/finder_test.go +++ b/finder/finder_test.go @@ -90,6 +90,7 @@ func runSimpleTest(t *testing.T, existentPaths []string, expectedMatches []strin CacheParams{ "/cwd", []string{root}, + false, nil, nil, []string{"findme.txt", "skipme.txt"}, @@ -121,6 +122,7 @@ func runTestWithSuffixes(t *testing.T, existentPaths []string, expectedMatches [ CacheParams{ "/cwd", []string{root}, + false, nil, nil, []string{"findme.txt", "skipme.txt"}, diff --git a/genrule/genrule.go b/genrule/genrule.go index c52ddee53..b796877ef 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" + "android/soong/bazel/cquery" "github.com/google/blueprint" "github.com/google/blueprint/bootstrap" "github.com/google/blueprint/proptools" @@ -189,6 +190,8 @@ type Module struct { modulePaths []string } +var _ android.MixedBuildBuildable = (*Module)(nil) + type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask type generateTask struct { @@ -249,27 +252,36 @@ func toolDepsMutator(ctx android.BottomUpMutatorContext) { } } -// Returns true if information was available from Bazel, false if bazel invocation still needs to occur. -func (c *Module) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool { +func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) { + g.generateCommonBuildActions(ctx) + + label := g.GetBazelLabel(ctx, g) bazelCtx := ctx.Config().BazelContext - filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) - if ok { - var bazelOutputFiles android.Paths - exportIncludeDirs := map[string]bool{} - for _, bazelOutputFile := range filePaths { - bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile)) - exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true - } - c.outputFiles = bazelOutputFiles - c.outputDeps = bazelOutputFiles - for includePath, _ := range exportIncludeDirs { - c.exportedIncludeDirs = append(c.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath)) - } + filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx)) + if err != nil { + ctx.ModuleErrorf(err.Error()) + return + } + + var bazelOutputFiles android.Paths + exportIncludeDirs := map[string]bool{} + for _, bazelOutputFile := range filePaths { + bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOutRelative(ctx, ctx.ModuleDir(), bazelOutputFile)) + exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true + } + g.outputFiles = bazelOutputFiles + g.outputDeps = bazelOutputFiles + for includePath, _ := range exportIncludeDirs { + g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath)) } - return ok } -func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { +// generateCommonBuildActions contains build action generation logic +// common to both the mixed build case and the legacy case of genrule processing. +// To fully support genrule in mixed builds, the contents of this function should +// approach zero; there should be no genrule action registration done directly +// by Soong logic in the mixed-build case. +func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) { g.subName = ctx.ModuleSubDir() // Collect the module directory for IDE info in java/jdeps.go. @@ -575,31 +587,49 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { } g.outputFiles = outputFiles.Paths() +} - bazelModuleLabel := g.GetBazelLabel(ctx, g) - bazelActionsUsed := false - if g.MixedBuildsEnabled(ctx) { - bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel) +func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Allowlist genrule to use depfile until we have a solution to remove it. + // TODO(b/235582219): Remove allowlist for genrule + if ctx.ModuleType() == "gensrcs" && + !ctx.DeviceConfig().BuildBrokenDepfile() && + Bool(g.properties.Depfile) { + ctx.PropertyErrorf( + "depfile", + "Deprecated to ensure the module type is convertible to Bazel. "+ + "Try specifying the dependencies explicitly so that there is no need to use depfile. "+ + "If not possible, the escape hatch is to use BUILD_BROKEN_DEPFILE to bypass the error.") } - if !bazelActionsUsed { - // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of - // the genrules on AOSP. That will make things simpler to look at the graph in the common - // case. For larger sets of outputs, inject a phony target in between to limit ninja file - // growth. - if len(g.outputFiles) <= 6 { - g.outputDeps = g.outputFiles - } else { - phonyFile := android.PathForModuleGen(ctx, "genrule-phony") - ctx.Build(pctx, android.BuildParams{ - Rule: blueprint.Phony, - Output: phonyFile, - Inputs: g.outputFiles, - }) - g.outputDeps = android.Paths{phonyFile} - } + + g.generateCommonBuildActions(ctx) + + // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of + // the genrules on AOSP. That will make things simpler to look at the graph in the common + // case. For larger sets of outputs, inject a phony target in between to limit ninja file + // growth. + if len(g.outputFiles) <= 6 { + g.outputDeps = g.outputFiles + } else { + phonyFile := android.PathForModuleGen(ctx, "genrule-phony") + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: phonyFile, + Inputs: g.outputFiles, + }) + g.outputDeps = android.Paths{phonyFile} } } +func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) { + bazelCtx := ctx.Config().BazelContext + bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx)) +} + +func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool { + return true +} + // Collect information for opening IDE project files in java/jdeps.go. func (g *Module) IDEInfo(dpInfo *android.IdeInfo) { dpInfo.Srcs = append(dpInfo.Srcs, g.Srcs().Strings()...) @@ -787,6 +817,7 @@ func NewGenSrcs() *Module { func GenSrcsFactory() android.Module { m := NewGenSrcs() android.InitAndroidModule(m) + android.InitBazelModule(m) return m } @@ -798,6 +829,13 @@ type genSrcsProperties struct { Shard_size *int64 } +type bazelGensrcsAttributes struct { + Srcs bazel.LabelListAttribute + Output_extension *string + Tools bazel.LabelListAttribute + Cmd string +} + const defaultShardSize = 50 func NewGenRule() *Module { @@ -862,8 +900,14 @@ func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { // Replace in and out variables with $< and $@ var cmd string if m.properties.Cmd != nil { - cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1) - cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1) + if ctx.ModuleType() == "gensrcs" { + cmd = strings.ReplaceAll(*m.properties.Cmd, "$(in)", "$(SRC)") + cmd = strings.ReplaceAll(cmd, "$(out)", "$(OUT)") + } else { + cmd = strings.Replace(*m.properties.Cmd, "$(in)", "$(SRCS)", -1) + cmd = strings.Replace(cmd, "$(out)", "$(OUTS)", -1) + } + genDir := "$(GENDIR)" if t := ctx.ModuleType(); t == "cc_genrule" || t == "java_genrule" || t == "java_genrule_host" { genDir = "$(RULEDIR)" @@ -883,30 +927,50 @@ func (m *Module) ConvertWithBp2build(ctx android.TopDownMutatorContext) { } } - // The Out prop is not in an immediately accessible field - // in the Module struct, so use GetProperties and cast it - // to the known struct prop. - var outs []string - for _, propIntf := range m.GetProperties() { - if props, ok := propIntf.(*genRuleProperties); ok { - outs = props.Out - break + if ctx.ModuleType() == "gensrcs" { + // The Output_extension prop is not in an immediately accessible field + // in the Module struct, so use GetProperties and cast it + // to the known struct prop. + var outputExtension *string + for _, propIntf := range m.GetProperties() { + if props, ok := propIntf.(*genSrcsProperties); ok { + outputExtension = props.Output_extension + break + } } + props := bazel.BazelTargetModuleProperties{ + Rule_class: "gensrcs", + Bzl_load_location: "//build/bazel/rules:gensrcs.bzl", + } + attrs := &bazelGensrcsAttributes{ + Srcs: srcs, + Output_extension: outputExtension, + Cmd: cmd, + Tools: tools, + } + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) + } else { + // The Out prop is not in an immediately accessible field + // in the Module struct, so use GetProperties and cast it + // to the known struct prop. + var outs []string + for _, propIntf := range m.GetProperties() { + if props, ok := propIntf.(*genRuleProperties); ok { + outs = props.Out + break + } + } + attrs := &bazelGenruleAttributes{ + Srcs: srcs, + Outs: outs, + Cmd: cmd, + Tools: tools, + } + props := bazel.BazelTargetModuleProperties{ + Rule_class: "genrule", + } + ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) } - - attrs := &bazelGenruleAttributes{ - Srcs: srcs, - Outs: outs, - Cmd: cmd, - Tools: tools, - } - - props := bazel.BazelTargetModuleProperties{ - Rule_class: "genrule", - } - - // Create the BazelTargetModule. - ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: m.Name()}, attrs) } var Bool = proptools.Bool diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 1b5cef2fe..b9be1f70f 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -15,6 +15,7 @@ package genrule import ( + "fmt" "os" "regexp" "testing" @@ -626,6 +627,73 @@ func TestGenSrcs(t *testing.T) { } } +func TestGensrcsBuildBrokenDepfile(t *testing.T) { + tests := []struct { + name string + prop string + BuildBrokenDepfile *bool + err string + }{ + { + name: `error when BuildBrokenDepfile is set to false`, + prop: ` + depfile: true, + cmd: "cat $(in) > $(out) && cat $(depfile)", + `, + BuildBrokenDepfile: proptools.BoolPtr(false), + err: "depfile: Deprecated to ensure the module type is convertible to Bazel", + }, + { + name: `error when BuildBrokenDepfile is not set`, + prop: ` + depfile: true, + cmd: "cat $(in) > $(out) && cat $(depfile)", + `, + err: "depfile: Deprecated to ensure the module type is convertible to Bazel.", + }, + { + name: `no error when BuildBrokenDepfile is explicitly set to true`, + prop: ` + depfile: true, + cmd: "cat $(in) > $(out) && cat $(depfile)", + `, + BuildBrokenDepfile: proptools.BoolPtr(true), + }, + { + name: `no error if depfile is not set`, + prop: ` + cmd: "cat $(in) > $(out)", + `, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + bp := fmt.Sprintf(` + gensrcs { + name: "foo", + srcs: ["data.txt"], + %s + }`, test.prop) + + var expectedErrors []string + if test.err != "" { + expectedErrors = append(expectedErrors, test.err) + } + android.GroupFixturePreparers( + prepareForGenRuleTest, + android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) { + if test.BuildBrokenDepfile != nil { + variables.BuildBrokenDepfile = test.BuildBrokenDepfile + } + }), + ). + ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(expectedErrors)). + RunTestWithBp(t, bp) + }) + + } +} + func TestGenruleDefaults(t *testing.T) { bp := ` genrule_defaults { @@ -16,4 +16,4 @@ replace github.com/google/go-cmp v0.5.5 => ../../external/go-cmp // Indirect dep from go-cmp exclude golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 -go 1.15 +go 1.18 diff --git a/java/Android.bp b/java/Android.bp index df0d1eb3d..e25accf2a 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -87,6 +87,7 @@ bootstrap_go_package { "dexpreopt_bootjars_test.go", "droiddoc_test.go", "droidstubs_test.go", + "genrule_test.go", "hiddenapi_singleton_test.go", "jacoco_test.go", "java_test.go", @@ -97,6 +98,7 @@ bootstrap_go_package { "platform_compat_config_test.go", "plugin_test.go", "prebuilt_apis_test.go", + "proto_test.go", "rro_test.go", "sdk_test.go", "sdk_library_test.go", diff --git a/java/android_manifest.go b/java/android_manifest.go index 3fa3520ad..a297b2c10 100644 --- a/java/android_manifest.go +++ b/java/android_manifest.go @@ -96,9 +96,9 @@ func ManifestFixer(ctx android.ModuleContext, manifest android.Path, } if params.ClassLoaderContexts != nil { - // manifest_fixer should add only the implicit SDK libraries inferred by Soong, not those added - // explicitly via `uses_libs`/`optional_uses_libs`. - requiredUsesLibs, optionalUsesLibs := params.ClassLoaderContexts.ImplicitUsesLibs() + // Libraries propagated via `uses_libs`/`optional_uses_libs` are also added (they may be + // propagated from dependencies). + requiredUsesLibs, optionalUsesLibs := params.ClassLoaderContexts.UsesLibs() for _, usesLib := range requiredUsesLibs { args = append(args, "--uses-library", usesLib) diff --git a/java/androidmk.go b/java/androidmk.go index f6ea6a930..82ef4137e 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -324,7 +324,7 @@ func (binary *Binary) AndroidMkEntries() []android.AndroidMkEntries { } func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { - if app.hideApexVariantFromMake || app.appProperties.HideFromMake { + if app.hideApexVariantFromMake || app.IsHideFromMake() { return []android.AndroidMkEntries{android.AndroidMkEntries{ Disabled: true, }} @@ -424,8 +424,8 @@ func (app *AndroidApp) AndroidMkEntries() []android.AndroidMkEntries { func (a *AndroidApp) getOverriddenPackages() []string { var overridden []string - if len(a.appProperties.Overrides) > 0 { - overridden = append(overridden, a.appProperties.Overrides...) + if len(a.overridableAppProperties.Overrides) > 0 { + overridden = append(overridden, a.overridableAppProperties.Overrides...) } // When APK name is overridden via PRODUCT_PACKAGE_NAME_OVERRIDES // ensure that the original name is overridden. @@ -542,6 +542,9 @@ func (dstubs *Droidstubs) AndroidMkEntries() []android.AndroidMkEntries { if !outputFile.Valid() { outputFile = android.OptionalPathForPath(dstubs.apiFile) } + if !outputFile.Valid() { + outputFile = android.OptionalPathForPath(dstubs.apiVersionsXml) + } return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "JAVA_LIBRARIES", OutputFile: outputFile, @@ -620,6 +623,7 @@ func (dstubs *Droidstubs) AndroidMkEntries() []android.AndroidMkEntries { if dstubs.apiLintReport != nil { fmt.Fprintf(w, "$(call dist-for-goals,%s,%s:%s)\n", dstubs.Name()+"-api-lint", dstubs.apiLintReport.String(), "apilint/"+dstubs.Name()+"-lint-report.txt") + fmt.Fprintf(w, "$(call declare-0p-target,%s)\n", dstubs.apiLintReport.String()) } } if dstubs.checkNullabilityWarningsTimestamp != nil { diff --git a/java/androidmk_test.go b/java/androidmk_test.go index 246c0eb07..197da4f38 100644 --- a/java/androidmk_test.go +++ b/java/androidmk_test.go @@ -206,3 +206,49 @@ func TestAndroidTestHelperApp_LocalDisableTestConfig(t *testing.T) { t.Errorf("Unexpected flag value - expected: %q, actual: %q", expected, actual) } } + +func TestGetOverriddenPackages(t *testing.T) { + ctx, _ := testJava( + t, ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + overrides: ["qux"] + } + + override_android_app { + name: "foo_override", + base: "foo", + overrides: ["bar"] + } + `) + + expectedVariants := []struct { + name string + moduleName string + variantName string + overrides []string + }{ + { + name: "foo", + moduleName: "foo", + variantName: "android_common", + overrides: []string{"qux"}, + }, + { + name: "foo", + moduleName: "foo_override", + variantName: "android_common_foo_override", + overrides: []string{"bar", "foo"}, + }, + } + + for _, expected := range expectedVariants { + mod := ctx.ModuleForTests(expected.name, expected.variantName).Module() + entries := android.AndroidMkEntriesForTest(t, ctx, mod)[0] + actual := entries.EntryMap["LOCAL_OVERRIDES_PACKAGES"] + + android.AssertDeepEquals(t, "overrides property", expected.overrides, actual) + } +} diff --git a/java/app.go b/java/app.go index 2b52eab15..23a9816b4 100755 --- a/java/app.go +++ b/java/app.go @@ -63,13 +63,6 @@ type appProperties struct { // list of resource labels to generate individual resource packages Package_splits []string - // Names of modules to be overridden. Listed modules can only be other binaries - // (in Make or Soong). - // This does not completely prevent installation of the overridden binaries, but if both - // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed - // from PRODUCT_PACKAGES. - Overrides []string - // list of native libraries that will be provided in or alongside the resulting jar Jni_libs []string `android:"arch_variant"` @@ -106,7 +99,6 @@ type appProperties struct { // cc.Coverage related properties PreventInstall bool `blueprint:"mutated"` - HideFromMake bool `blueprint:"mutated"` IsCoverageVariant bool `blueprint:"mutated"` // Whether this app is considered mainline updatable or not. When set to true, this will enforce @@ -133,6 +125,13 @@ type overridableAppProperties struct { // Whether to rename the package in resources to the override name rather than the base name. Defaults to true. Rename_resources_package *bool + + // Names of modules to be overridden. Listed modules can only be other binaries + // (in Make or Soong). + // This does not completely prevent installation of the overridden binaries, but if both + // binaries would be installed by default (in PRODUCT_PACKAGES) the other binary will be removed + // from PRODUCT_PACKAGES. + Overrides []string } type AndroidApp struct { @@ -299,10 +298,6 @@ func (a *AndroidApp) checkAppSdkVersions(ctx android.ModuleContext) { // If an updatable APK sets min_sdk_version, min_sdk_vesion of JNI libs should match with it. // This check is enforced for "updatable" APKs (including APK-in-APEX). -// b/155209650: until min_sdk_version is properly supported, use sdk_version instead. -// because, sdk_version is overridden by min_sdk_version (if set as smaller) -// and sdkLinkType is checked with dependencies so we can be sure that the whole dependency tree -// will meet the requirements. func (a *AndroidApp) checkJniLibsSdkVersion(ctx android.ModuleContext, minSdkVersion android.ApiLevel) { // It's enough to check direct JNI deps' sdk_version because all transitive deps from JNI deps are checked in cc.checkLinkType() ctx.VisitDirectDeps(func(m android.Module) { @@ -313,10 +308,10 @@ func (a *AndroidApp) checkJniLibsSdkVersion(ctx android.ModuleContext, minSdkVer // The domain of cc.sdk_version is "current" and <number> // We can rely on android.SdkSpec to convert it to <number> so that "current" is // handled properly regardless of sdk finalization. - jniSdkVersion, err := android.SdkSpecFrom(ctx, dep.SdkVersion()).EffectiveVersion(ctx) + jniSdkVersion, err := android.SdkSpecFrom(ctx, dep.MinSdkVersion()).EffectiveVersion(ctx) if err != nil || minSdkVersion.LessThan(jniSdkVersion) { - ctx.OtherModuleErrorf(dep, "sdk_version(%v) is higher than min_sdk_version(%v) of the containing android_app(%v)", - dep.SdkVersion(), minSdkVersion, ctx.ModuleName()) + ctx.OtherModuleErrorf(dep, "min_sdk_version(%v) is higher than min_sdk_version(%v) of the containing android_app(%v)", + dep.MinSdkVersion(), minSdkVersion, ctx.ModuleName()) return } @@ -586,20 +581,18 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { } a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir) + a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) + + var noticeAssetPath android.WritablePath if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { - noticeFile := android.PathForModuleOut(ctx, "NOTICE.html.gz") - android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, noticeFile) - noticeAssetPath := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") - builder := android.NewRuleBuilder(pctx, ctx) - builder.Command().Text("cp"). - Input(noticeFile). - Output(noticeAssetPath) - builder.Build("notice_dir", "Building notice dir") + // The rule to create the notice file can't be generated yet, as the final output path + // for the apk isn't known yet. Add the path where the notice file will be generated to the + // aapt rules now before calling aaptBuildActions, the rule to create the notice file will + // be generated later. + noticeAssetPath = android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") a.aapt.noticeFile = android.OptionalPathForPath(noticeAssetPath) } - a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) - // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) @@ -671,6 +664,23 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.extraOutputFiles = append(a.extraOutputFiles, v4SignatureFile) } + if a.aapt.noticeFile.Valid() { + // Generating the notice file rule has to be here after a.outputFile is known. + noticeFile := android.PathForModuleOut(ctx, "NOTICE.html.gz") + android.BuildNoticeHtmlOutputFromLicenseMetadata( + ctx, noticeFile, "", "", + []string{ + a.installDir.String() + "/", + android.PathForModuleInstall(ctx).String() + "/", + a.outputFile.String(), + }) + builder := android.NewRuleBuilder(pctx, ctx) + builder.Command().Text("cp"). + Input(noticeFile). + Output(noticeAssetPath) + builder.Build("notice_dir", "Building notice dir") + } + for _, split := range a.aapt.splits { // Sign the split APKs packageFile := android.PathForModuleOut(ctx, a.installApkName+"_"+split.suffix+".apk") @@ -730,7 +740,7 @@ func collectAppDeps(ctx android.ModuleContext, app appDepsInterface, tag := ctx.OtherModuleDependencyTag(module) if IsJniDepTag(tag) || cc.IsSharedDepTag(tag) { - if dep, ok := module.(*cc.Module); ok { + if dep, ok := module.(cc.LinkableInterface); ok { if dep.IsNdk(ctx.Config()) || dep.IsStubs() { return false } @@ -842,6 +852,10 @@ func (a *AndroidApp) Updatable() bool { return Bool(a.appProperties.Updatable) } +func (a *AndroidApp) SetUpdatable(val bool) { + a.appProperties.Updatable = &val +} + func (a *AndroidApp) getCertString(ctx android.BaseModuleContext) string { certificate, overridden := ctx.DeviceConfig().OverrideCertificateFor(ctx.ModuleName()) if overridden { @@ -880,10 +894,6 @@ func (a *AndroidApp) SetPreventInstall() { a.appProperties.PreventInstall = true } -func (a *AndroidApp) HideFromMake() { - a.appProperties.HideFromMake = true -} - func (a *AndroidApp) MarkAsCoverageVariant(coverage bool) { a.appProperties.IsCoverageVariant = coverage } @@ -900,6 +910,7 @@ func AndroidAppFactory() android.Module { module.Module.dexProperties.Optimize.Shrink = proptools.BoolPtr(true) module.Module.properties.Instrument = true + module.Module.properties.Supports_static_instrumentation = true module.Module.properties.Installable = proptools.BoolPtr(true) module.addHostAndDeviceProperties() @@ -912,7 +923,7 @@ func AndroidAppFactory() android.Module { android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) - android.InitOverridableModule(module, &module.appProperties.Overrides) + android.InitOverridableModule(module, &module.overridableAppProperties.Overrides) android.InitApexModule(module) android.InitBazelModule(module) @@ -1016,9 +1027,10 @@ func (a *AndroidTest) OverridablePropertiesDepsMutator(ctx android.BottomUpMutat func AndroidTestFactory() android.Module { module := &AndroidTest{} - module.Module.dexProperties.Optimize.EnabledByDefault = true + module.Module.dexProperties.Optimize.EnabledByDefault = false module.Module.properties.Instrument = true + module.Module.properties.Supports_static_instrumentation = true module.Module.properties.Installable = proptools.BoolPtr(true) module.appProperties.Use_embedded_native_libs = proptools.BoolPtr(true) module.appProperties.AlwaysPackageNativeLibs = true @@ -1035,7 +1047,7 @@ func AndroidTestFactory() android.Module { android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon) android.InitDefaultableModule(module) - android.InitOverridableModule(module, &module.appProperties.Overrides) + android.InitOverridableModule(module, &module.overridableAppProperties.Overrides) return module } @@ -1069,6 +1081,7 @@ func (a *AndroidTestHelperApp) InstallInTestcases() bool { func AndroidTestHelperAppFactory() android.Module { module := &AndroidTestHelperApp{} + // TODO(b/192032291): Disable by default after auditing downstream usage. module.Module.dexProperties.Optimize.EnabledByDefault = true module.Module.properties.Installable = proptools.BoolPtr(true) @@ -1225,28 +1238,17 @@ func (u *usesLibrary) addLib(lib string, optional bool) { func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, hasFrameworkLibs bool) { if !ctx.Config().UnbundledBuild() || ctx.Config().UnbundledBuildImage() { - reqTag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, false, false) - ctx.AddVariationDependencies(nil, reqTag, u.usesLibraryProperties.Uses_libs...) - - optTag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, true, false) - ctx.AddVariationDependencies(nil, optTag, u.presentOptionalUsesLibs(ctx)...) - + ctx.AddVariationDependencies(nil, usesLibReqTag, u.usesLibraryProperties.Uses_libs...) + ctx.AddVariationDependencies(nil, usesLibOptTag, u.presentOptionalUsesLibs(ctx)...) // Only add these extra dependencies if the module depends on framework libs. This avoids // creating a cyclic dependency: // e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res. if hasFrameworkLibs { - // Add implicit <uses-library> dependencies on compatibility libraries. Some of them are - // optional, and some required --- this depends on the most common usage of the library - // and may be wrong for some apps (they need explicit `uses_libs`/`optional_uses_libs`). - - compat28OptTag := makeUsesLibraryDependencyTag(28, true, true) - ctx.AddVariationDependencies(nil, compat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) - - compat29ReqTag := makeUsesLibraryDependencyTag(29, false, true) - ctx.AddVariationDependencies(nil, compat29ReqTag, dexpreopt.CompatUsesLibs29...) - - compat30OptTag := makeUsesLibraryDependencyTag(30, true, true) - ctx.AddVariationDependencies(nil, compat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) + // Dexpreopt needs paths to the dex jars of these libraries in order to construct + // class loader context for dex2oat. Add them as a dependency with a special tag. + ctx.AddVariationDependencies(nil, usesLibCompat29ReqTag, dexpreopt.CompatUsesLibs29...) + ctx.AddVariationDependencies(nil, usesLibCompat28OptTag, dexpreopt.OptionalCompatUsesLibs28...) + ctx.AddVariationDependencies(nil, usesLibCompat30OptTag, dexpreopt.OptionalCompatUsesLibs30...) } } } @@ -1305,7 +1307,7 @@ func (u *usesLibrary) classLoaderContextForUsesLibDeps(ctx android.ModuleContext replaceInList(u.usesLibraryProperties.Uses_libs, dep, libName) replaceInList(u.usesLibraryProperties.Optional_uses_libs, dep, libName) } - clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, tag.implicit, + clcMap.AddContext(ctx, tag.sdkVersion, libName, tag.optional, lib.DexJarBuildPath().PathOrNil(), lib.DexJarInstallPath(), lib.ClassLoaderContexts()) } else if ctx.Config().AllowMissingDependencies() { diff --git a/java/app_test.go b/java/app_test.go index 6a4508cd6..c4ac4dfdb 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -427,7 +427,8 @@ func TestUpdatableApps_JniLibShouldBeBuiltAgainstMinSdkVersion(t *testing.T) { name: "libjni", stl: "none", system_shared_libs: [], - sdk_version: "29", + sdk_version: "current", + min_sdk_version: "29", } ` fs := map[string][]byte{ @@ -481,12 +482,13 @@ func TestUpdatableApps_ErrorIfJniLibDoesntSupportMinSdkVersion(t *testing.T) { name: "libjni", stl: "none", sdk_version: "current", + min_sdk_version: "current", } ` - testJavaError(t, `"libjni" .*: sdk_version\(current\) is higher than min_sdk_version\(29\)`, bp) + testJavaError(t, `"libjni" .*: min_sdk_version\(current\) is higher than min_sdk_version\(29\)`, bp) } -func TestUpdatableApps_ErrorIfDepSdkVersionIsHigher(t *testing.T) { +func TestUpdatableApps_ErrorIfDepMinSdkVersionIsHigher(t *testing.T) { bp := cc.GatherRequiredDepsForTest(android.Android) + ` android_app { name: "foo", @@ -503,6 +505,7 @@ func TestUpdatableApps_ErrorIfDepSdkVersionIsHigher(t *testing.T) { shared_libs: ["libbar"], system_shared_libs: [], sdk_version: "27", + min_sdk_version: "27", } cc_library { @@ -510,6 +513,7 @@ func TestUpdatableApps_ErrorIfDepSdkVersionIsHigher(t *testing.T) { stl: "none", system_shared_libs: [], sdk_version: "current", + min_sdk_version: "current", } ` testJavaError(t, `"libjni" .*: links "libbar" built against newer API version "current"`, bp) @@ -1962,7 +1966,7 @@ func TestOverrideAndroidApp(t *testing.T) { // Check if the overrides field values are correctly aggregated. mod := variant.Module().(*AndroidApp) - android.AssertDeepEquals(t, "overrides property", expected.overrides, mod.appProperties.Overrides) + android.AssertDeepEquals(t, "overrides property", expected.overrides, mod.overridableAppProperties.Overrides) // Test Overridable property: Logging_parent logging_parent := mod.aapt.LoggingParent @@ -1980,6 +1984,99 @@ func TestOverrideAndroidApp(t *testing.T) { } } +func TestOverrideAndroidAppOverrides(t *testing.T) { + ctx, _ := testJava( + t, ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + overrides: ["qux"] + } + + android_app { + name: "bar", + srcs: ["b.java"], + sdk_version: "current", + overrides: ["foo"] + } + + override_android_app { + name: "foo_override", + base: "foo", + overrides: ["bar"] + } + `) + + expectedVariants := []struct { + name string + moduleName string + variantName string + overrides []string + }{ + { + name: "foo", + moduleName: "foo", + variantName: "android_common", + overrides: []string{"qux"}, + }, + { + name: "bar", + moduleName: "bar", + variantName: "android_common", + overrides: []string{"foo"}, + }, + { + name: "foo", + moduleName: "foo_override", + variantName: "android_common_foo_override", + overrides: []string{"bar", "foo"}, + }, + } + for _, expected := range expectedVariants { + variant := ctx.ModuleForTests(expected.name, expected.variantName) + + // Check if the overrides field values are correctly aggregated. + mod := variant.Module().(*AndroidApp) + android.AssertDeepEquals(t, "overrides property", expected.overrides, mod.overridableAppProperties.Overrides) + } +} + +func TestOverrideAndroidAppWithPrebuilt(t *testing.T) { + result := PrepareForTestWithJavaDefaultModules.RunTestWithBp( + t, ` + android_app { + name: "foo", + srcs: ["a.java"], + sdk_version: "current", + } + + override_android_app { + name: "bar", + base: "foo", + } + + android_app_import { + name: "bar", + prefer: true, + apk: "bar.apk", + presigned: true, + } + `) + + // An app that has an override that also has a prebuilt should not be hidden. + foo := result.ModuleForTests("foo", "android_common") + if foo.Module().IsHideFromMake() { + t.Errorf("expected foo to have HideFromMake false") + } + + // An override that also has a prebuilt should be hidden. + barOverride := result.ModuleForTests("foo", "android_common_bar") + if !barOverride.Module().IsHideFromMake() { + t.Errorf("expected bar override variant of foo to have HideFromMake true") + } +} + func TestOverrideAndroidAppStem(t *testing.T) { ctx, _ := testJava(t, ` android_app { @@ -2160,9 +2257,9 @@ func TestOverrideAndroidTest(t *testing.T) { // Check if the overrides field values are correctly aggregated. mod := variant.Module().(*AndroidTest) - if !reflect.DeepEqual(expected.overrides, mod.appProperties.Overrides) { + if !reflect.DeepEqual(expected.overrides, mod.overridableAppProperties.Overrides) { t.Errorf("Incorrect overrides property value, expected: %q, got: %q", - expected.overrides, mod.appProperties.Overrides) + expected.overrides, mod.overridableAppProperties.Overrides) } // Check if javac classpath has the correct jar file path. This checks instrumentation_for overrides. @@ -2505,12 +2602,20 @@ func TestUsesLibraries(t *testing.T) { prebuilt := result.ModuleForTests("prebuilt", "android_common") // Test that implicit dependencies on java_sdk_library instances are passed to the manifest. - // This should not include explicit `uses_libs`/`optional_uses_libs` entries. + // These also include explicit `uses_libs`/`optional_uses_libs` entries, as they may be + // propagated from dependencies. actualManifestFixerArgs := app.Output("manifest_fixer/AndroidManifest.xml").Args["args"] expectManifestFixerArgs := `--extract-native-libs=true ` + `--uses-library qux ` + `--uses-library quuz ` + - `--uses-library runtime-library` + `--uses-library foo ` + + `--uses-library com.non.sdk.lib ` + + `--uses-library runtime-library ` + + `--uses-library runtime-required-x ` + + `--uses-library runtime-required-y ` + + `--optional-uses-library bar ` + + `--optional-uses-library runtime-optional-x ` + + `--optional-uses-library runtime-optional-y` android.AssertStringDoesContain(t, "manifest_fixer args", actualManifestFixerArgs, expectManifestFixerArgs) // Test that all libraries are verified (library order matters). diff --git a/java/base.go b/java/base.go index 4932c4831..c399c4063 100644 --- a/java/base.go +++ b/java/base.go @@ -170,6 +170,9 @@ type CommonProperties struct { } Instrument bool `blueprint:"mutated"` + // If true, then the module supports statically including the jacocoagent + // into the library. + Supports_static_instrumentation bool `blueprint:"mutated"` // List of files to include in the META-INF/services folder of the resulting jar. Services []string `android:"path,arch_variant"` @@ -602,7 +605,8 @@ func (j *Module) shouldInstrument(ctx android.BaseModuleContext) bool { } func (j *Module) shouldInstrumentStatic(ctx android.BaseModuleContext) bool { - return j.shouldInstrument(ctx) && + return j.properties.Supports_static_instrumentation && + j.shouldInstrument(ctx) && (ctx.Config().IsEnvTrue("EMMA_INSTRUMENT_STATIC") || ctx.Config().UnbundledBuild()) } @@ -720,8 +724,10 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { if component, ok := dep.(SdkLibraryComponentDependency); ok { if lib := component.OptionalSdkLibraryImplementation(); lib != nil { // Add library as optional if it's one of the optional compatibility libs. - optional := android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) - tag := makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, optional, true) + tag := usesLibReqTag + if android.InList(*lib, dexpreopt.OptionalCompatUsesLibs) { + tag = usesLibOptTag + } ctx.AddVariationDependencies(nil, tag, *lib) } } @@ -742,9 +748,7 @@ func (j *Module) deps(ctx android.BottomUpMutatorContext) { // Kotlin files ctx.AddVariationDependencies(nil, kotlinStdlibTag, "kotlin-stdlib", "kotlin-stdlib-jdk7", "kotlin-stdlib-jdk8") - if len(j.properties.Plugins) > 0 { - ctx.AddVariationDependencies(nil, kotlinAnnotationsTag, "kotlin-annotations") - } + ctx.AddVariationDependencies(nil, kotlinAnnotationsTag, "kotlin-annotations") } // Framework libraries need special handling in static coverage builds: they should not have @@ -1016,6 +1020,7 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { ctx.PropertyErrorf("common_srcs", "common_srcs must be .kt files") } + nonGeneratedSrcJars := srcFiles.FilterByExt(".srcjar") srcFiles = j.genSources(ctx, srcFiles, flags) // Collect javac flags only after computing the full set of srcFiles to @@ -1100,8 +1105,6 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { flags.classpath = append(flags.classpath, deps.kotlinStdlib...) flags.classpath = append(flags.classpath, deps.kotlinAnnotations...) - flags.dexClasspath = append(flags.dexClasspath, deps.kotlinAnnotations...) - flags.kotlincClasspath = append(flags.kotlincClasspath, flags.bootClasspath...) flags.kotlincClasspath = append(flags.kotlincClasspath, flags.classpath...) @@ -1133,9 +1136,12 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { // Jar kotlin classes into the final jar after javac if BoolDefault(j.properties.Static_kotlin_stdlib, true) { kotlinJars = append(kotlinJars, deps.kotlinStdlib...) + kotlinJars = append(kotlinJars, deps.kotlinAnnotations...) kotlinHeaderJars = append(kotlinHeaderJars, deps.kotlinStdlib...) + kotlinHeaderJars = append(kotlinHeaderJars, deps.kotlinAnnotations...) } else { flags.dexClasspath = append(flags.dexClasspath, deps.kotlinStdlib...) + flags.dexClasspath = append(flags.dexClasspath, deps.kotlinAnnotations...) } } @@ -1412,17 +1418,18 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { j.implementationAndResourcesJar = implementationAndResourcesJar // Enable dex compilation for the APEX variants, unless it is disabled explicitly + compileDex := j.dexProperties.Compile_dex apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) if j.DirectlyInAnyApex() && !apexInfo.IsForPlatform() { - if j.dexProperties.Compile_dex == nil { - j.dexProperties.Compile_dex = proptools.BoolPtr(true) + if compileDex == nil { + compileDex = proptools.BoolPtr(true) } if j.deviceProperties.Hostdex == nil { j.deviceProperties.Hostdex = proptools.BoolPtr(true) } } - if ctx.Device() && (Bool(j.properties.Installable) || Bool(j.dexProperties.Compile_dex)) { + if ctx.Device() && (Bool(j.properties.Installable) || Bool(compileDex)) { if j.hasCode(ctx) { if j.shouldInstrumentStatic(ctx) { j.dexer.extraProguardFlagFiles = append(j.dexer.extraProguardFlagFiles, @@ -1506,8 +1513,8 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { } j.linter.name = ctx.ModuleName() - j.linter.srcs = srcFiles - j.linter.srcJars = srcJars + j.linter.srcs = append(srcFiles, nonGeneratedSrcJars...) + j.linter.srcJars, _ = android.FilterPathList(srcJars, nonGeneratedSrcJars) j.linter.classpath = append(append(android.Paths(nil), flags.bootClasspath...), flags.classpath...) j.linter.classes = j.implementationJarFile j.linter.minSdkVersion = lintSDKVersion(j.MinSdkVersion(ctx)) @@ -1926,6 +1933,9 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { case bootClasspathTag: deps.bootClasspath = append(deps.bootClasspath, dep.HeaderJars...) case libTag, instrumentationForTag: + if _, ok := module.(*Plugin); ok { + ctx.ModuleErrorf("a java_plugin (%s) cannot be used as a libs dependency", otherName) + } deps.classpath = append(deps.classpath, dep.HeaderJars...) deps.dexClasspath = append(deps.dexClasspath, dep.HeaderJars...) deps.aidlIncludeDirs = append(deps.aidlIncludeDirs, dep.AidlIncludeDirs...) @@ -1934,6 +1944,9 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { case java9LibTag: deps.java9Classpath = append(deps.java9Classpath, dep.HeaderJars...) case staticLibTag: + if _, ok := module.(*Plugin); ok { + ctx.ModuleErrorf("a java_plugin (%s) cannot be used as a static_libs dependency", otherName) + } deps.classpath = append(deps.classpath, dep.HeaderJars...) deps.staticJars = append(deps.staticJars, dep.ImplementationJars...) deps.staticHeaderJars = append(deps.staticHeaderJars, dep.HeaderJars...) diff --git a/java/bootclasspath.go b/java/bootclasspath.go index 52ce77d46..f4cef7fa6 100644 --- a/java/bootclasspath.go +++ b/java/bootclasspath.go @@ -84,6 +84,9 @@ func addDependencyOntoApexModulePair(ctx android.BottomUpMutatorContext, apex st } } + target := ctx.Module().Target() + variations = append(variations, target.Variations()...) + addedDep := false if ctx.OtherModuleDependencyVariantExists(variations, name) { ctx.AddFarVariationDependencies(variations, tag, name) diff --git a/java/bootclasspath_fragment.go b/java/bootclasspath_fragment.go index c3a5d5f70..0591012fd 100644 --- a/java/bootclasspath_fragment.go +++ b/java/bootclasspath_fragment.go @@ -42,6 +42,7 @@ func init() { func registerBootclasspathFragmentBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("bootclasspath_fragment", bootclasspathFragmentFactory) + ctx.RegisterModuleType("bootclasspath_fragment_test", testBootclasspathFragmentFactory) ctx.RegisterModuleType("prebuilt_bootclasspath_fragment", prebuiltBootclasspathFragmentFactory) } @@ -227,6 +228,9 @@ type BootclasspathFragmentModule struct { android.SdkBase ClasspathFragmentBase + // True if this fragment is for testing purposes. + testFragment bool + properties bootclasspathFragmentProperties sourceOnlyProperties SourceOnlyBootclasspathProperties @@ -273,7 +277,7 @@ func bootclasspathFragmentFactory() android.Module { android.InitApexModule(m) android.InitSdkAwareModule(m) initClasspathFragment(m, BOOTCLASSPATH) - android.InitAndroidArchModule(m, android.HostAndDeviceSupported, android.MultilibCommon) + android.InitAndroidArchModule(m, android.DeviceSupported, android.MultilibCommon) android.AddLoadHook(m, func(ctx android.LoadHookContext) { // If code coverage has been enabled for the framework then append the properties with @@ -298,6 +302,12 @@ func bootclasspathFragmentFactory() android.Module { return m } +func testBootclasspathFragmentFactory() android.Module { + m := bootclasspathFragmentFactory().(*BootclasspathFragmentModule) + m.testFragment = true + return m +} + // bootclasspathFragmentInitContentsFromImage will initialize the contents property from the image_name if // necessary. func bootclasspathFragmentInitContentsFromImage(ctx android.EarlyModuleContext, m *BootclasspathFragmentModule) { @@ -815,6 +825,26 @@ func (b *BootclasspathFragmentModule) createHiddenAPIFlagInput(ctx android.Modul return input } +// isTestFragment returns true if the current module is a test bootclasspath_fragment. +func (b *BootclasspathFragmentModule) isTestFragment() bool { + if b.testFragment { + return true + } + + // TODO(b/194063708): Once test fragments all use bootclasspath_fragment_test + // Some temporary exceptions until all test fragments use the + // bootclasspath_fragment_test module type. + name := b.BaseModuleName() + if strings.HasPrefix(name, "test_") { + return true + } + if name == "apex.apexd_test_bootclasspath-fragment" { + return true + } + + return false +} + // produceHiddenAPIOutput produces the hidden API all-flags.csv file (and supporting files) // for the fragment as well as encoding the flags in the boot dex jars. func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput { @@ -828,11 +858,18 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC packagePrefixes := b.sourceOnlyProperties.Hidden_api.Package_prefixes singlePackages := b.sourceOnlyProperties.Hidden_api.Single_packages if splitPackages != nil || packagePrefixes != nil || singlePackages != nil { - if splitPackages == nil { - splitPackages = []string{"*"} - } output.SignaturePatternsPath = buildRuleSignaturePatternsFile( ctx, output.AllFlagsPath, splitPackages, packagePrefixes, singlePackages) + } else if !b.isTestFragment() { + ctx.ModuleErrorf(`Must specify at least one of the split_packages, package_prefixes and single_packages properties + If this is a new bootclasspath_fragment or you are unsure what to do add the + the following to the bootclasspath_fragment: + hidden_api: {split_packages: ["*"]}, + and then run the following: + m analyze_bcpf && analyze_bcpf --bcpf %q + it will analyze the bootclasspath_fragment and provide hints as to what you + should specify here. If you are happy with its suggestions then you can add + the --fix option and it will fix them for you.`, b.BaseModuleName()) } return output diff --git a/java/bootclasspath_fragment_test.go b/java/bootclasspath_fragment_test.go index d3de675d8..83beb6d23 100644 --- a/java/bootclasspath_fragment_test.go +++ b/java/bootclasspath_fragment_test.go @@ -121,6 +121,9 @@ func TestBootclasspathFragment_Coverage(t *testing.T) { ], }, }, + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -201,6 +204,9 @@ func TestBootclasspathFragment_StubLibs(t *testing.T) { core_platform_api: { stub_libs: ["mycoreplatform.stubs"], }, + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -278,3 +284,64 @@ func TestBootclasspathFragment_StubLibs(t *testing.T) { android.AssertPathsRelativeToTopEquals(t, "widest dex stubs jar", expectedWidestPaths, info.TransitiveStubDexJarsByScope.StubDexJarsForWidestAPIScope()) } + +func TestBootclasspathFragment_Test(t *testing.T) { + result := android.GroupFixturePreparers( + prepareForTestWithBootclasspathFragment, + PrepareForTestWithJavaSdkLibraryFiles, + FixtureWithLastReleaseApis("mysdklibrary"), + ).RunTestWithBp(t, ` + bootclasspath_fragment { + name: "myfragment", + contents: ["mysdklibrary"], + hidden_api: { + split_packages: [], + }, + } + + bootclasspath_fragment { + name: "test_fragment", + contents: ["mysdklibrary"], + hidden_api: { + split_packages: [], + }, + } + + bootclasspath_fragment { + name: "apex.apexd_test_bootclasspath-fragment", + contents: ["mysdklibrary"], + hidden_api: { + split_packages: [], + }, + } + + bootclasspath_fragment_test { + name: "a_test_fragment", + contents: ["mysdklibrary"], + hidden_api: { + split_packages: [], + }, + } + + + java_sdk_library { + name: "mysdklibrary", + srcs: ["a.java"], + shared_library: false, + public: {enabled: true}, + system: {enabled: true}, + } + `) + + fragment := result.Module("myfragment", "android_common").(*BootclasspathFragmentModule) + android.AssertBoolEquals(t, "not a test fragment", false, fragment.isTestFragment()) + + fragment = result.Module("test_fragment", "android_common").(*BootclasspathFragmentModule) + android.AssertBoolEquals(t, "is a test fragment by prefix", true, fragment.isTestFragment()) + + fragment = result.Module("a_test_fragment", "android_common").(*BootclasspathFragmentModule) + android.AssertBoolEquals(t, "is a test fragment by type", true, fragment.isTestFragment()) + + fragment = result.Module("apex.apexd_test_bootclasspath-fragment", "android_common").(*BootclasspathFragmentModule) + android.AssertBoolEquals(t, "is a test fragment by name", true, fragment.isTestFragment()) +} diff --git a/java/config/config.go b/java/config/config.go index 46c91a2f6..1d4b242f9 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -68,6 +68,11 @@ var ( "-J-XX:+TieredCompilation", "-J-XX:TieredStopAtLevel=1", } + dexerJavaVmFlagsList = []string{ + `-JXX:OnError="cat hs_err_pid%p.log"`, + "-JXX:CICompilerCount=6", + "-JXX:+UseDynamicNumberOfGCThreads", + } ) func init() { @@ -83,19 +88,16 @@ func init() { // D8 invocations are shorter lived, so we restrict their JIT tiering relative to R8. // Note that the `-JXX` prefix syntax is specific to the R8/D8 invocation wrappers. - exportedVars.ExportStringListStaticVariable("D8Flags", []string{ - `-JXX:OnError="cat hs_err_pid%p.log"`, - "-JXX:CICompilerCount=6", - "-JXX:+UseDynamicNumberOfGCThreads", + exportedVars.ExportStringListStaticVariable("D8Flags", append([]string{ + "-JXmx2048M", "-JXX:+TieredCompilation", "-JXX:TieredStopAtLevel=1", - }) - - exportedVars.ExportStringListStaticVariable("R8Flags", []string{ - `-JXX:OnError="cat hs_err_pid%p.log"`, - "-JXX:CICompilerCount=6", - "-JXX:+UseDynamicNumberOfGCThreads", - }) + }, dexerJavaVmFlagsList...)) + exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{ + "-JXmx2048M", + // Disable this optimization as it can impact weak reference semantics. See b/233432839. + "-JDcom.android.tools.r8.disableEnqueuerDeferredTracing=true", + }, dexerJavaVmFlagsList...)) exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{ `-Xmaxerrs 9999999`, @@ -157,7 +159,7 @@ func init() { pctx.HostBinToolVariable("ZipSyncCmd", "zipsync") pctx.HostBinToolVariable("ApiCheckCmd", "apicheck") pctx.HostBinToolVariable("D8Cmd", "d8") - pctx.HostBinToolVariable("R8Cmd", "r8-compat-proguard") + pctx.HostBinToolVariable("R8Cmd", "r8") pctx.HostBinToolVariable("HiddenAPICmd", "hiddenapi") pctx.HostBinToolVariable("ExtractApksCmd", "extract_apks") pctx.VariableFunc("TurbineJar", func(ctx android.PackageVarContext) string { @@ -175,7 +177,7 @@ func init() { pctx.HostJavaToolVariable("MetalavaJar", "metalava.jar") pctx.HostJavaToolVariable("DokkaJar", "dokka.jar") pctx.HostJavaToolVariable("JetifierJar", "jetifier.jar") - pctx.HostJavaToolVariable("R8Jar", "r8-compat-proguard.jar") + pctx.HostJavaToolVariable("R8Jar", "r8.jar") pctx.HostJavaToolVariable("D8Jar", "d8.jar") pctx.HostBinToolVariable("SoongJavacWrapper", "soong_javac_wrapper") diff --git a/java/config/makevars.go b/java/config/makevars.go index bc6848fc3..273aca0b4 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -43,9 +43,10 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("JAVADOC", "${JavadocCmd}") ctx.Strict("COMMON_JDK_FLAGS", "${CommonJdkFlags}") - ctx.Strict("DX", "${D8Cmd}") - ctx.Strict("DX_COMMAND", "${D8Cmd} -JXms16M -JXmx2048M") - ctx.Strict("R8_COMPAT_PROGUARD", "${R8Cmd}") + ctx.Strict("D8", "${D8Cmd}") + ctx.Strict("R8", "${R8Cmd}") + ctx.Strict("D8_COMMAND", "${D8Cmd} ${D8Flags}") + ctx.Strict("R8_COMMAND", "${R8Cmd} ${R8Flags}") ctx.Strict("TURBINE", "${TurbineJar}") @@ -78,9 +79,6 @@ func makeVarsProvider(ctx android.MakeVarsContext) { ctx.Strict("CLASS2NONSDKLIST", "${Class2NonSdkList}") ctx.Strict("HIDDENAPI", "${HiddenAPI}") - ctx.Strict("D8_FLAGS", "${D8Flags}") - ctx.Strict("R8_FLAGS", "${R8Flags}") - ctx.Strict("AIDL", "${AidlCmd}") ctx.Strict("AAPT2", "${Aapt2Cmd}") ctx.Strict("ZIPALIGN", "${ZipAlign}") diff --git a/java/core-libraries/Android.bp b/java/core-libraries/Android.bp index cf3974601..513c6061b 100644 --- a/java/core-libraries/Android.bp +++ b/java/core-libraries/Android.bp @@ -138,11 +138,29 @@ java_library { }, } +// Same as core-module-lib-stubs-for-system-modules, but android annotations are +// stripped. This is used by the Java toolchain, while the annotated stub is to +// be used by Kotlin one. +java_library { + name: "core-module-lib-stubs-for-system-modules-no-annotations", + visibility: ["//visibility:private"], + static_libs: [ + "core-module-lib-stubs-for-system-modules", + ], + sdk_version: "none", + system_modules: "none", + dist: { + dest: "system-modules/module-lib/core-for-system-modules-no-annotations.jar", + targets: dist_targets, + }, + jarjar_rules: "jarjar-strip-annotations-rules.txt", +} + // Used when compiling higher-level code with sdk_version "module_current" java_system_modules { name: "core-module-lib-stubs-system-modules", libs: [ - "core-module-lib-stubs-for-system-modules", + "core-module-lib-stubs-for-system-modules-no-annotations", ], visibility: ["//visibility:public"], } @@ -174,6 +192,24 @@ java_library { patch_module: "java.base", } +// Same as legacy.core.platform.api.stubs, but android annotations are +// stripped. This is used by the Java toolchain, while the annotated stub is to +// be used by Kotlin one. +java_library { + name: "legacy.core.platform.api.no.annotations.stubs", + visibility: core_platform_visibility, + hostdex: true, + compile_dex: true, + + sdk_version: "none", + system_modules: "none", + static_libs: [ + "legacy.core.platform.api.stubs", + ], + patch_module: "java.base", + jarjar_rules: "jarjar-strip-annotations-rules.txt", +} + java_library { name: "stable.core.platform.api.stubs", visibility: core_platform_visibility, @@ -191,12 +227,30 @@ java_library { patch_module: "java.base", } +// Same as stable.core.platform.api.stubs, but android annotations are +// stripped. This is used by the Java toolchain, while the annotated stub is to +// be used by Kotlin one. +java_library { + name: "stable.core.platform.api.no.annotations.stubs", + visibility: core_platform_visibility, + hostdex: true, + compile_dex: true, + + sdk_version: "none", + system_modules: "none", + static_libs: [ + "stable.core.platform.api.stubs", + ], + patch_module: "java.base", + jarjar_rules: "jarjar-strip-annotations-rules.txt", +} + // Used when compiling higher-level code against *.core.platform.api.stubs. java_system_modules { name: "legacy-core-platform-api-stubs-system-modules", visibility: core_platform_visibility, libs: [ - "legacy.core.platform.api.stubs", + "legacy.core.platform.api.no.annotations.stubs", // This one is not on device but it's needed when javac compiles code // containing lambdas. "core-lambda-stubs-for-system-modules", @@ -212,7 +266,7 @@ java_system_modules { name: "stable-core-platform-api-stubs-system-modules", visibility: core_platform_visibility, libs: [ - "stable.core.platform.api.stubs", + "stable.core.platform.api.no.annotations.stubs", // This one is not on device but it's needed when javac compiles code // containing lambdas. "core-lambda-stubs-for-system-modules", diff --git a/java/core-libraries/jarjar-strip-annotations-rules.txt b/java/core-libraries/jarjar-strip-annotations-rules.txt new file mode 100644 index 000000000..a1c261b9a --- /dev/null +++ b/java/core-libraries/jarjar-strip-annotations-rules.txt @@ -0,0 +1,4 @@ +strip-annotation android.annotation.NotNull +strip-annotation android.annotation.Nullable +strip-annotation androidx.annotation.RecentlyNonNull +strip-annotation androidx.annotation.RecentlyNullable diff --git a/java/dex.go b/java/dex.go index 13d6e4a01..c943938e2 100644 --- a/java/dex.go +++ b/java/dex.go @@ -36,8 +36,8 @@ type DexProperties struct { Main_dex_rules []string `android:"path"` Optimize struct { - // If false, disable all optimization. Defaults to true for android_app and android_test - // modules, false for java_library and java_test modules. + // If false, disable all optimization. Defaults to true for android_app and + // android_test_helper_app modules, false for android_test, java_library, and java_test modules. Enabled *bool // True if the module containing this has it set by default. EnabledByDefault bool `blueprint:"mutated"` diff --git a/java/dexpreopt.go b/java/dexpreopt.go index 7c5f055e6..0adaf9917 100644 --- a/java/dexpreopt.go +++ b/java/dexpreopt.go @@ -259,10 +259,6 @@ func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.Wr isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx)) bootImage := defaultBootImageConfig(ctx) - if global.UseArtImage { - bootImage = artBootImageConfig(ctx) - } - dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) targets := ctx.MultiTargets() diff --git a/java/dexpreopt.go_v1 b/java/dexpreopt.go_v1 new file mode 100644 index 000000000..0adaf9917 --- /dev/null +++ b/java/dexpreopt.go_v1 @@ -0,0 +1,404 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" +) + +type DexpreopterInterface interface { + IsInstallable() bool // Structs that embed dexpreopter must implement this. + dexpreoptDisabled(ctx android.BaseModuleContext) bool + DexpreoptBuiltInstalledForApex() []dexpreopterInstall + AndroidMkEntriesForApex() []android.AndroidMkEntries +} + +type dexpreopterInstall struct { + // A unique name to distinguish an output from others for the same java library module. Usually in + // the form of `<arch>-<encoded-path>.odex/vdex/art`. + name string + + // The name of the input java module. + moduleName string + + // The path to the dexpreopt output on host. + outputPathOnHost android.Path + + // The directory on the device for the output to install to. + installDirOnDevice android.InstallPath + + // The basename (the last segment of the path) for the output to install as. + installFileOnDevice string +} + +// The full module name of the output in the makefile. +func (install *dexpreopterInstall) FullModuleName() string { + return install.moduleName + install.SubModuleName() +} + +// The sub-module name of the output in the makefile (the name excluding the java module name). +func (install *dexpreopterInstall) SubModuleName() string { + return "-dexpreopt-" + install.name +} + +// Returns Make entries for installing the file. +// +// This function uses a value receiver rather than a pointer receiver to ensure that the object is +// safe to use in `android.AndroidMkExtraEntriesFunc`. +func (install dexpreopterInstall) ToMakeEntries() android.AndroidMkEntries { + return android.AndroidMkEntries{ + Class: "ETC", + SubName: install.SubModuleName(), + OutputFile: android.OptionalPathForPath(install.outputPathOnHost), + ExtraEntries: []android.AndroidMkExtraEntriesFunc{ + func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) { + entries.SetString("LOCAL_MODULE_PATH", install.installDirOnDevice.String()) + entries.SetString("LOCAL_INSTALLED_MODULE_STEM", install.installFileOnDevice) + entries.SetString("LOCAL_NOT_AVAILABLE_FOR_PLATFORM", "false") + }, + }, + } +} + +type dexpreopter struct { + dexpreoptProperties DexpreoptProperties + + installPath android.InstallPath + uncompressedDex bool + isSDKLibrary bool + isApp bool + isTest bool + isPresignedPrebuilt bool + preventInstall bool + + manifestFile android.Path + statusFile android.WritablePath + enforceUsesLibs bool + classLoaderContexts dexpreopt.ClassLoaderContextMap + + // See the `dexpreopt` function for details. + builtInstalled string + builtInstalledForApex []dexpreopterInstall + + // The config is used for two purposes: + // - Passing dexpreopt information about libraries from Soong to Make. This is needed when + // a <uses-library> is defined in Android.bp, but used in Android.mk (see dex_preopt_config_merger.py). + // Note that dexpreopt.config might be needed even if dexpreopt is disabled for the library itself. + // - Dexpreopt post-processing (using dexpreopt artifacts from a prebuilt system image to incrementally + // dexpreopt another partition). + configPath android.WritablePath +} + +type DexpreoptProperties struct { + Dex_preopt struct { + // If false, prevent dexpreopting. Defaults to true. + Enabled *bool + + // If true, generate an app image (.art file) for this module. + App_image *bool + + // If true, use a checked-in profile to guide optimization. Defaults to false unless + // a matching profile is set or a profile is found in PRODUCT_DEX_PREOPT_PROFILE_DIR + // that matches the name of this module, in which case it is defaulted to true. + Profile_guided *bool + + // If set, provides the path to profile relative to the Android.bp file. If not set, + // defaults to searching for a file that matches the name of this module in the default + // profile location set by PRODUCT_DEX_PREOPT_PROFILE_DIR, or empty if not found. + Profile *string `android:"path"` + } +} + +func init() { + dexpreopt.DexpreoptRunningInSoong = true +} + +func isApexVariant(ctx android.BaseModuleContext) bool { + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + return !apexInfo.IsForPlatform() +} + +func forPrebuiltApex(ctx android.BaseModuleContext) bool { + apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo) + return apexInfo.ForPrebuiltApex +} + +func moduleName(ctx android.BaseModuleContext) string { + // Remove the "prebuilt_" prefix if the module is from a prebuilt because the prefix is not + // expected by dexpreopter. + return android.RemoveOptionalPrebuiltPrefix(ctx.ModuleName()) +} + +func (d *dexpreopter) dexpreoptDisabled(ctx android.BaseModuleContext) bool { + if !ctx.Device() { + return true + } + + if d.isTest { + return true + } + + if !BoolDefault(d.dexpreoptProperties.Dex_preopt.Enabled, true) { + return true + } + + // If the module is from a prebuilt APEX, it shouldn't be installable, but it can still be + // dexpreopted. + if !ctx.Module().(DexpreopterInterface).IsInstallable() && !forPrebuiltApex(ctx) { + return true + } + + if !android.IsModulePreferred(ctx.Module()) { + return true + } + + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisablePreopt { + return true + } + + if inList(moduleName(ctx), global.DisablePreoptModules) { + return true + } + + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + if isApexVariant(ctx) { + // Don't preopt APEX variant module unless the module is an APEX system server jar and we are + // building the entire system image. + if !isApexSystemServerJar || ctx.Config().UnbundledBuild() { + return true + } + } else { + // Don't preopt the platform variant of an APEX system server jar to avoid conflicts. + if isApexSystemServerJar { + return true + } + } + + // TODO: contains no java code + + return false +} + +func dexpreoptToolDepsMutator(ctx android.BottomUpMutatorContext) { + if d, ok := ctx.Module().(DexpreopterInterface); !ok || d.dexpreoptDisabled(ctx) { + return + } + dexpreopt.RegisterToolDeps(ctx) +} + +func (d *dexpreopter) odexOnSystemOther(ctx android.ModuleContext, installPath android.InstallPath) bool { + return dexpreopt.OdexOnSystemOtherByName(moduleName(ctx), android.InstallPathToOnDevicePath(ctx, installPath), dexpreopt.GetGlobalConfig(ctx)) +} + +// Returns the install path of the dex jar of a module. +// +// Do not rely on `ApexInfo.ApexVariationName` because it can be something like "apex1000", rather +// than the `name` in the path `/apex/<name>` as suggested in its comment. +// +// This function is on a best-effort basis. It cannot handle the case where an APEX jar is not a +// system server jar, which is fine because we currently only preopt system server jars for APEXes. +func (d *dexpreopter) getInstallPath( + ctx android.ModuleContext, defaultInstallPath android.InstallPath) android.InstallPath { + global := dexpreopt.GetGlobalConfig(ctx) + if global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) { + dexLocation := dexpreopt.GetSystemServerDexLocation(ctx, global, moduleName(ctx)) + return android.PathForModuleInPartitionInstall(ctx, "", strings.TrimPrefix(dexLocation, "/")) + } + if !d.dexpreoptDisabled(ctx) && isApexVariant(ctx) && + filepath.Base(defaultInstallPath.PartitionDir()) != "apex" { + ctx.ModuleErrorf("unable to get the install path of the dex jar for dexpreopt") + } + return defaultInstallPath +} + +func (d *dexpreopter) dexpreopt(ctx android.ModuleContext, dexJarFile android.WritablePath) { + global := dexpreopt.GetGlobalConfig(ctx) + + // TODO(b/148690468): The check on d.installPath is to bail out in cases where + // the dexpreopter struct hasn't been fully initialized before we're called, + // e.g. in aar.go. This keeps the behaviour that dexpreopting is effectively + // disabled, even if installable is true. + if d.installPath.Base() == "." { + return + } + + dexLocation := android.InstallPathToOnDevicePath(ctx, d.installPath) + + providesUsesLib := moduleName(ctx) + if ulib, ok := ctx.Module().(ProvidesUsesLib); ok { + name := ulib.ProvidesUsesLib() + if name != nil { + providesUsesLib = *name + } + } + + // If it is test, make config files regardless of its dexpreopt setting. + // The config files are required for apps defined in make which depend on the lib. + if d.isTest && d.dexpreoptDisabled(ctx) { + return + } + + isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + + bootImage := defaultBootImageConfig(ctx) + dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) + + targets := ctx.MultiTargets() + if len(targets) == 0 { + // assume this is a java library, dexpreopt for all arches for now + for _, target := range ctx.Config().Targets[android.Android] { + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } + } + if isSystemServerJar && !d.isSDKLibrary { + // If the module is not an SDK library and it's a system server jar, only preopt the primary arch. + targets = targets[:1] + } + } + + var archs []android.ArchType + var images android.Paths + var imagesDeps []android.OutputPaths + for _, target := range targets { + archs = append(archs, target.Arch.ArchType) + variant := bootImage.getVariant(target) + images = append(images, variant.imagePathOnHost) + imagesDeps = append(imagesDeps, variant.imagesDeps) + } + // The image locations for all Android variants are identical. + hostImageLocations, deviceImageLocations := bootImage.getAnyAndroidVariant().imageLocations() + + var profileClassListing android.OptionalPath + var profileBootListing android.OptionalPath + profileIsTextListing := false + if BoolDefault(d.dexpreoptProperties.Dex_preopt.Profile_guided, true) { + // If dex_preopt.profile_guided is not set, default it based on the existence of the + // dexprepot.profile option or the profile class listing. + if String(d.dexpreoptProperties.Dex_preopt.Profile) != "" { + profileClassListing = android.OptionalPathForPath( + android.PathForModuleSrc(ctx, String(d.dexpreoptProperties.Dex_preopt.Profile))) + profileBootListing = android.ExistentPathForSource(ctx, + ctx.ModuleDir(), String(d.dexpreoptProperties.Dex_preopt.Profile)+"-boot") + profileIsTextListing = true + } else if global.ProfileDir != "" { + profileClassListing = android.ExistentPathForSource(ctx, + global.ProfileDir, moduleName(ctx)+".prof") + } + } + + // Full dexpreopt config, used to create dexpreopt build rules. + dexpreoptConfig := &dexpreopt.ModuleConfig{ + Name: moduleName(ctx), + DexLocation: dexLocation, + BuildPath: android.PathForModuleOut(ctx, "dexpreopt", moduleName(ctx)+".jar").OutputPath, + DexPath: dexJarFile, + ManifestPath: android.OptionalPathForPath(d.manifestFile), + UncompressedDex: d.uncompressedDex, + HasApkLibraries: false, + PreoptFlags: nil, + + ProfileClassListing: profileClassListing, + ProfileIsTextListing: profileIsTextListing, + ProfileBootListing: profileBootListing, + + EnforceUsesLibrariesStatusFile: dexpreopt.UsesLibrariesStatusFile(ctx), + EnforceUsesLibraries: d.enforceUsesLibs, + ProvidesUsesLibrary: providesUsesLib, + ClassLoaderContexts: d.classLoaderContexts, + + Archs: archs, + DexPreoptImagesDeps: imagesDeps, + DexPreoptImageLocationsOnHost: hostImageLocations, + DexPreoptImageLocationsOnDevice: deviceImageLocations, + + PreoptBootClassPathDexFiles: dexFiles.Paths(), + PreoptBootClassPathDexLocations: dexLocations, + + PreoptExtractedApk: false, + + NoCreateAppImage: !BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, true), + ForceCreateAppImage: BoolDefault(d.dexpreoptProperties.Dex_preopt.App_image, false), + + PresignedPrebuilt: d.isPresignedPrebuilt, + } + + d.configPath = android.PathForModuleOut(ctx, "dexpreopt", "dexpreopt.config") + dexpreopt.WriteModuleConfig(ctx, dexpreoptConfig, d.configPath) + + if d.dexpreoptDisabled(ctx) { + return + } + + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + + dexpreoptRule, err := dexpreopt.GenerateDexpreoptRule(ctx, globalSoong, global, dexpreoptConfig) + if err != nil { + ctx.ModuleErrorf("error generating dexpreopt rule: %s", err.Error()) + return + } + + dexpreoptRule.Build("dexpreopt", "dexpreopt") + + isApexSystemServerJar := global.AllApexSystemServerJars(ctx).ContainsJar(moduleName(ctx)) + + for _, install := range dexpreoptRule.Installs() { + // Remove the "/" prefix because the path should be relative to $ANDROID_PRODUCT_OUT. + installDir := strings.TrimPrefix(filepath.Dir(install.To), "/") + installBase := filepath.Base(install.To) + arch := filepath.Base(installDir) + installPath := android.PathForModuleInPartitionInstall(ctx, "", installDir) + + if isApexSystemServerJar { + // APEX variants of java libraries are hidden from Make, so their dexpreopt + // outputs need special handling. Currently, for APEX variants of java + // libraries, only those in the system server classpath are handled here. + // Preopting of boot classpath jars in the ART APEX are handled in + // java/dexpreopt_bootjars.go, and other APEX jars are not preopted. + // The installs will be handled by Make as sub-modules of the java library. + d.builtInstalledForApex = append(d.builtInstalledForApex, dexpreopterInstall{ + name: arch + "-" + installBase, + moduleName: moduleName(ctx), + outputPathOnHost: install.From, + installDirOnDevice: installPath, + installFileOnDevice: installBase, + }) + } else if !d.preventInstall { + ctx.InstallFile(installPath, installBase, install.From) + } + } + + if !isApexSystemServerJar { + d.builtInstalled = dexpreoptRule.Installs().String() + } +} + +func (d *dexpreopter) DexpreoptBuiltInstalledForApex() []dexpreopterInstall { + return d.builtInstalledForApex +} + +func (d *dexpreopter) AndroidMkEntriesForApex() []android.AndroidMkEntries { + var entries []android.AndroidMkEntries + for _, install := range d.builtInstalledForApex { + entries = append(entries, install.ToMakeEntries()) + } + return entries +} diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index 3d91aec91..7c4da3ef6 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -213,12 +213,6 @@ import ( // writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, // paths and so on. // -// 2.5. JIT-Zygote configuration -// ----------------------------- -// -// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling -// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage). -// var artApexNames = []string{ "com.android.art", @@ -938,11 +932,8 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " ")) var imageNames []string - // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a - // different way as a part of the ART APEX. However, there is a special JIT-Zygote build - // configuration which uses the primary ART image instead of the Framework boot image - // extension, and it relies on the ART image being exposed to Make. To fix this, it is - // necessary to rework the logic in makefiles. + // The primary ART boot image is exposed to Make for testing (gtests) and benchmarking + // (golem) purposes. for _, current := range append(d.otherImages, image) { imageNames = append(imageNames, current.name) for _, variant := range current.variants { diff --git a/java/dexpreopt_bootjars.go_v1 b/java/dexpreopt_bootjars.go_v1 new file mode 100644 index 000000000..07a357bb5 --- /dev/null +++ b/java/dexpreopt_bootjars.go_v1 @@ -0,0 +1,952 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" + + "github.com/google/blueprint/proptools" +) + +// ================================================================================================= +// WIP - see http://b/177892522 for details +// +// The build support for boot images is currently being migrated away from singleton to modules so +// the documentation may not be strictly accurate. Rather than update the documentation at every +// step which will create a lot of churn the changes that have been made will be listed here and the +// documentation will be updated once it is closer to the final result. +// +// Changes: +// 1) dex_bootjars is now a singleton module and not a plain singleton. +// 2) Boot images are now represented by the boot_image module type. +// 3) The art boot image is called "art-boot-image", the framework boot image is called +// "framework-boot-image". +// 4) They are defined in art/build/boot/Android.bp and frameworks/base/boot/Android.bp +// respectively. +// 5) Each boot_image retrieves the appropriate boot image configuration from the map returned by +// genBootImageConfigs() using the image_name specified in the boot_image module. +// ================================================================================================= + +// This comment describes: +// 1. ART boot images in general (their types, structure, file layout, etc.) +// 2. build system support for boot images +// +// 1. ART boot images +// ------------------ +// +// A boot image in ART is a set of files that contain AOT-compiled native code and a heap snapshot +// of AOT-initialized classes for the bootclasspath Java libraries. A boot image is compiled from a +// set of DEX jars by the dex2oat compiler. A boot image is used for two purposes: 1) it is +// installed on device and loaded at runtime, and 2) other Java libraries and apps are compiled +// against it (compilation may take place either on host, known as "dexpreopt", or on device, known +// as "dexopt"). +// +// A boot image is not a single file, but a collection of interrelated files. Each boot image has a +// number of components that correspond to the Java libraries that constitute it. For each component +// there are multiple files: +// - *.oat or *.odex file with native code (architecture-specific, one per instruction set) +// - *.art file with pre-initialized Java classes (architecture-specific, one per instruction set) +// - *.vdex file with verification metadata for the DEX bytecode (architecture independent) +// +// *.vdex files for the boot images do not contain the DEX bytecode itself, because the +// bootclasspath DEX files are stored on disk in uncompressed and aligned form. Consequently a boot +// image is not self-contained and cannot be used without its DEX files. To simplify the management +// of boot image files, ART uses a certain naming scheme and associates the following metadata with +// each boot image: +// - A stem, which is a symbolic name that is prepended to boot image file names. +// - A location (on-device path to the boot image files). +// - A list of boot image locations (on-device paths to dependency boot images). +// - A set of DEX locations (on-device paths to the DEX files, one location for one DEX file used +// to compile the boot image). +// +// There are two kinds of boot images: +// - primary boot images +// - boot image extensions +// +// 1.1. Primary boot images +// ------------------------ +// +// A primary boot image is compiled for a core subset of bootclasspath Java libraries. It does not +// depend on any other images, and other boot images may depend on it. +// +// For example, assuming that the stem is "boot", the location is /apex/com.android.art/javalib/, +// the set of core bootclasspath libraries is A B C, and the boot image is compiled for ARM targets +// (32 and 64 bits), it will have three components with the following files: +// - /apex/com.android.art/javalib/{arm,arm64}/boot.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-B.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-C.{art,oat,vdex} +// +// The files of the first component are special: they do not have the component name appended after +// the stem. This naming convention dates back to the times when the boot image was not split into +// components, and there were just boot.oat and boot.art. The decision to split was motivated by +// licensing reasons for one of the bootclasspath libraries. +// +// As of November 2020 the only primary boot image in Android is the image in the ART APEX +// com.android.art. The primary ART boot image contains the Core libraries that are part of the ART +// module. When the ART module gets updated, the primary boot image will be updated with it, and all +// dependent images will get invalidated (the checksum of the primary image stored in dependent +// images will not match), unless they are updated in sync with the ART module. +// +// 1.2. Boot image extensions +// -------------------------- +// +// A boot image extension is compiled for a subset of bootclasspath Java libraries (in particular, +// this subset does not include the Core bootclasspath libraries that go into the primary boot +// image). A boot image extension depends on the primary boot image and optionally some other boot +// image extensions. Other images may depend on it. In other words, boot image extensions can form +// acyclic dependency graphs. +// +// The motivation for boot image extensions comes from the Mainline project. Consider a situation +// when the list of bootclasspath libraries is A B C, and both A and B are parts of the Android +// platform, but C is part of an updatable APEX com.android.C. When the APEX is updated, the Java +// code for C might have changed compared to the code that was used to compile the boot image. +// Consequently, the whole boot image is obsolete and invalidated (even though the code for A and B +// that does not depend on C is up to date). To avoid this, the original monolithic boot image is +// split in two parts: the primary boot image that contains A B, and the boot image extension that +// contains C and depends on the primary boot image (extends it). +// +// For example, assuming that the stem is "boot", the location is /system/framework, the set of +// bootclasspath libraries is D E (where D is part of the platform and is located in +// /system/framework, and E is part of a non-updatable APEX com.android.E and is located in +// /apex/com.android.E/javalib), and the boot image is compiled for ARM targets (32 and 64 bits), +// it will have two components with the following files: +// - /system/framework/{arm,arm64}/boot-D.{art,oat,vdex} +// - /system/framework/{arm,arm64}/boot-E.{art,oat,vdex} +// +// As of November 2020 the only boot image extension in Android is the Framework boot image +// extension. It extends the primary ART boot image and contains Framework libraries and other +// bootclasspath libraries from the platform and non-updatable APEXes that are not included in the +// ART image. The Framework boot image extension is updated together with the platform. In the +// future other boot image extensions may be added for some updatable modules. +// +// +// 2. Build system support for boot images +// --------------------------------------- +// +// The primary ART boot image needs to be compiled with one dex2oat invocation that depends on DEX +// jars for the core libraries. Framework boot image extension needs to be compiled with one dex2oat +// invocation that depends on the primary ART boot image and all bootclasspath DEX jars except the +// core libraries as they are already part of the primary ART boot image. +// +// 2.1. Libraries that go in the boot images +// ----------------------------------------- +// +// The contents of each boot image are determined by the PRODUCT variables. The primary ART APEX +// boot image contains libraries listed in the ART_APEX_JARS variable in the AOSP makefiles. The +// Framework boot image extension contains libraries specified in the PRODUCT_BOOT_JARS and +// PRODUCT_BOOT_JARS_EXTRA variables. The AOSP makefiles specify some common Framework libraries, +// but more product-specific libraries can be added in the product makefiles. +// +// Each component of the PRODUCT_BOOT_JARS and PRODUCT_BOOT_JARS_EXTRA variables is a +// colon-separated pair <apex>:<library>, where <apex> is the variant name of a non-updatable APEX, +// "platform" if the library is a part of the platform in the system partition, or "system_ext" if +// it's in the system_ext partition. +// +// In these variables APEXes are identified by their "variant names", i.e. the names they get +// mounted as in /apex on device. In Soong modules that is the name set in the "apex_name" +// properties, which default to the "name" values. For example, many APEXes have both +// com.android.xxx and com.google.android.xxx modules in Soong, but take the same place +// /apex/com.android.xxx at runtime. In these cases the variant name is always com.android.xxx, +// regardless which APEX goes into the product. See also android.ApexInfo.ApexVariationName and +// apex.apexBundleProperties.Apex_name. +// +// A related variable PRODUCT_APEX_BOOT_JARS contains bootclasspath libraries that are in APEXes. +// They are not included in the boot image. The only exception here are ART jars and core-icu4j.jar +// that have been historically part of the boot image and are now in apexes; they are in boot images +// and core-icu4j.jar is generally treated as being part of PRODUCT_BOOT_JARS. +// +// One exception to the above rules are "coverage" builds (a special build flavor which requires +// setting environment variable EMMA_INSTRUMENT_FRAMEWORK=true). In coverage builds the Java code in +// boot image libraries is instrumented, which means that the instrumentation library (jacocoagent) +// needs to be added to the list of bootclasspath DEX jars. +// +// In general, there is a requirement that the source code for a boot image library must be +// available at build time (e.g. it cannot be a stub that has a separate implementation library). +// +// 2.2. Static configs +// ------------------- +// +// Because boot images are used to dexpreopt other Java modules, the paths to boot image files must +// be known by the time dexpreopt build rules for the dependent modules are generated. Boot image +// configs are constructed very early during the build, before build rule generation. The configs +// provide predefined paths to boot image files (these paths depend only on static build +// configuration, such as PRODUCT variables, and use hard-coded directory names). +// +// 2.3. Singleton +// -------------- +// +// Build rules for the boot images are generated with a Soong singleton. Because a singleton has no +// dependencies on other modules, it has to find the modules for the DEX jars using VisitAllModules. +// Soong loops through all modules and compares each module against a list of bootclasspath library +// names. Then it generates build rules that copy DEX jars from their intermediate module-specific +// locations to the hard-coded locations predefined in the boot image configs. +// +// It would be possible to use a module with proper dependencies instead, but that would require +// changes in the way Soong generates variables for Make: a singleton can use one MakeVars() method +// that writes variables to out/soong/make_vars-*.mk, which is included early by the main makefile, +// but module(s) would have to use out/soong/Android-*.mk which has a group of LOCAL_* variables +// for each module, and is included later. +// +// 2.4. Install rules +// ------------------ +// +// The primary boot image and the Framework extension are installed in different ways. The primary +// boot image is part of the ART APEX: it is copied into the APEX intermediate files, packaged +// together with other APEX contents, extracted and mounted on device. The Framework boot image +// extension is installed by the rules defined in makefiles (make/core/dex_preopt_libart.mk). Soong +// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, +// paths and so on. +// + +var artApexNames = []string{ + "com.android.art", + "com.android.art.debug", + "com.android.art.testing", + "com.google.android.art", + "com.google.android.art.debug", + "com.google.android.art.testing", +} + +func init() { + RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext) +} + +// Target-independent description of a boot image. +type bootImageConfig struct { + // If this image is an extension, the image that it extends. + extends *bootImageConfig + + // Image name (used in directory names and ninja rule names). + name string + + // Basename of the image: the resulting filenames are <stem>[-<jar>].{art,oat,vdex}. + stem string + + // Output directory for the image files. + dir android.OutputPath + + // Output directory for the image files with debug symbols. + symbolsDir android.OutputPath + + // Subdirectory where the image files are installed. + installDirOnHost string + + // Subdirectory where the image files on device are installed. + installDirOnDevice string + + // Install path of the boot image profile if it needs to be installed in the APEX, or empty if not + // needed. + profileInstallPathInApex string + + // A list of (location, jar) pairs for the Java modules in this image. + modules android.ConfiguredJarList + + // File paths to jars. + dexPaths android.WritablePaths // for this image + dexPathsDeps android.WritablePaths // for the dependency images and in this image + + // Map from module name (without prebuilt_ prefix) to the predefined build path. + dexPathsByModule map[string]android.WritablePath + + // File path to a zip archive with all image files (or nil, if not needed). + zip android.WritablePath + + // Rules which should be used in make to install the outputs. + profileInstalls android.RuleBuilderInstalls + + // Path to the license metadata file for the module that built the profile. + profileLicenseMetadataFile android.OptionalPath + + // Path to the image profile file on host (or empty, if profile is not generated). + profilePathOnHost android.Path + + // Target-dependent fields. + variants []*bootImageVariant + + // Path of the preloaded classes file. + preloadedClassesFile string +} + +// Target-dependent description of a boot image. +type bootImageVariant struct { + *bootImageConfig + + // Target for which the image is generated. + target android.Target + + // The "locations" of jars. + dexLocations []string // for this image + dexLocationsDeps []string // for the dependency images and in this image + + // Paths to image files. + imagePathOnHost android.OutputPath // first image file path on host + imagePathOnDevice string // first image file path on device + + // All the files that constitute this image variant, i.e. .art, .oat and .vdex files. + imagesDeps android.OutputPaths + + // The path to the primary image variant's imagePathOnHost field, where primary image variant + // means the image variant that this extends. + // + // This is only set for a variant of an image that extends another image. + primaryImages android.OutputPath + + // The paths to the primary image variant's imagesDeps field, where primary image variant + // means the image variant that this extends. + // + // This is only set for a variant of an image that extends another image. + primaryImagesDeps android.Paths + + // Rules which should be used in make to install the outputs on host. + installs android.RuleBuilderInstalls + vdexInstalls android.RuleBuilderInstalls + unstrippedInstalls android.RuleBuilderInstalls + + // Rules which should be used in make to install the outputs on device. + deviceInstalls android.RuleBuilderInstalls + + // Path to the license metadata file for the module that built the image. + licenseMetadataFile android.OptionalPath +} + +// Get target-specific boot image variant for the given boot image config and target. +func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant { + for _, variant := range image.variants { + if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType { + return variant + } + } + return nil +} + +// Return any (the first) variant which is for the device (as opposed to for the host). +func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant { + for _, variant := range image.variants { + if variant.target.Os == android.Android { + return variant + } + } + return nil +} + +// Return the name of a boot image module given a boot image config and a component (module) index. +// A module name is a combination of the Java library name, and the boot image stem (that is stored +// in the config). +func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string { + // The first module of the primary boot image is special: its module name has only the stem, but + // not the library name. All other module names are of the form <stem>-<library name> + m := image.modules.Jar(idx) + name := image.stem + if idx != 0 || image.extends != nil { + name += "-" + android.ModuleStem(m) + } + return name +} + +// Return the name of the first boot image module, or stem if the list of modules is empty. +func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) string { + if image.modules.Len() > 0 { + return image.moduleName(ctx, 0) + } else { + return image.stem + } +} + +// Return filenames for the given boot image component, given the output directory and a list of +// extensions. +func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths { + ret := make(android.OutputPaths, 0, image.modules.Len()*len(exts)) + for i := 0; i < image.modules.Len(); i++ { + name := image.moduleName(ctx, i) + for _, ext := range exts { + ret = append(ret, dir.Join(ctx, name+ext)) + } + } + return ret +} + +// apexVariants returns a list of all *bootImageVariant that could be included in an apex. +func (image *bootImageConfig) apexVariants() []*bootImageVariant { + variants := []*bootImageVariant{} + for _, variant := range image.variants { + // We also generate boot images for host (for testing), but we don't need those in the apex. + // TODO(b/177892522) - consider changing this to check Os.OsClass = android.Device + if variant.target.Os == android.Android { + variants = append(variants, variant) + } + } + return variants +} + +// Returns true if the boot image should be installed in the APEX. +func (image *bootImageConfig) shouldInstallInApex() bool { + return strings.HasPrefix(image.installDirOnDevice, "apex/") +} + +// Return boot image locations (as a list of symbolic paths). +// +// The image "location" is a symbolic path that, with multiarchitecture support, doesn't really +// exist on the device. Typically it is /apex/com.android.art/javalib/boot.art and should be the +// same for all supported architectures on the device. The concrete architecture specific files +// actually end up in architecture-specific sub-directory such as arm, arm64, x86, or x86_64. +// +// For example a physical file /apex/com.android.art/javalib/x86/boot.art has "image location" +// /apex/com.android.art/javalib/boot.art (which is not an actual file). +// +// For a primary boot image the list of locations has a single element. +// +// For a boot image extension the list of locations contains a location for all dependency images +// (including the primary image) and the location of the extension itself. For example, for the +// Framework boot image extension that depends on the primary ART boot image the list contains two +// elements. +// +// The location is passed as an argument to the ART tools like dex2oat instead of the real path. +// ART tools will then reconstruct the architecture-specific real path. +// +func (image *bootImageVariant) imageLocations() (imageLocationsOnHost []string, imageLocationsOnDevice []string) { + if image.extends != nil { + imageLocationsOnHost, imageLocationsOnDevice = image.extends.getVariant(image.target).imageLocations() + } + return append(imageLocationsOnHost, dexpreopt.PathToLocation(image.imagePathOnHost, image.target.Arch.ArchType)), + append(imageLocationsOnDevice, dexpreopt.PathStringToLocation(image.imagePathOnDevice, image.target.Arch.ArchType)) +} + +func dexpreoptBootJarsFactory() android.SingletonModule { + m := &dexpreoptBootJars{} + android.InitAndroidModule(m) + return m +} + +func RegisterDexpreoptBootJarsComponents(ctx android.RegistrationContext) { + ctx.RegisterSingletonModuleType("dex_bootjars", dexpreoptBootJarsFactory) +} + +func SkipDexpreoptBootJars(ctx android.PathContext) bool { + return dexpreopt.GetGlobalConfig(ctx).DisablePreoptBootImages +} + +// Singleton module for generating boot image build rules. +type dexpreoptBootJars struct { + android.SingletonModuleBase + + // Default boot image config (currently always the Framework boot image extension). It should be + // noted that JIT-Zygote builds use ART APEX image instead of the Framework boot image extension, + // but the switch is handled not here, but in the makefiles (triggered with + // DEXPREOPT_USE_ART_IMAGE=true). + defaultBootImage *bootImageConfig + + // Build path to a config file that Soong writes for Make (to be used in makefiles that install + // the default boot image). + dexpreoptConfigForMake android.WritablePath +} + +// Provide paths to boot images for use by modules that depend upon them. +// +// The build rules are created in GenerateSingletonBuildActions(). +func (d *dexpreoptBootJars) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // Placeholder for now. +} + +// Generate build rules for boot images. +func (d *dexpreoptBootJars) GenerateSingletonBuildActions(ctx android.SingletonContext) { + if SkipDexpreoptBootJars(ctx) { + return + } + if dexpreopt.GetCachedGlobalSoongConfig(ctx) == nil { + // No module has enabled dexpreopting, so we assume there will be no boot image to make. + return + } + + d.dexpreoptConfigForMake = android.PathForOutput(ctx, ctx.Config().DeviceName(), "dexpreopt.config") + writeGlobalConfigForMake(ctx, d.dexpreoptConfigForMake) + + global := dexpreopt.GetGlobalConfig(ctx) + if !shouldBuildBootImages(ctx.Config(), global) { + return + } + + defaultImageConfig := defaultBootImageConfig(ctx) + d.defaultBootImage = defaultImageConfig +} + +// shouldBuildBootImages determines whether boot images should be built. +func shouldBuildBootImages(config android.Config, global *dexpreopt.GlobalConfig) bool { + // Skip recompiling the boot image for the second sanitization phase. We'll get separate paths + // and invalidate first-stage artifacts which are crucial to SANITIZE_LITE builds. + // Note: this is technically incorrect. Compiled code contains stack checks which may depend + // on ASAN settings. + if len(config.SanitizeDevice()) == 1 && config.SanitizeDevice()[0] == "address" && global.SanitizeLite { + return false + } + return true +} + +// copyBootJarsToPredefinedLocations generates commands that will copy boot jars to predefined +// paths in the global config. +func copyBootJarsToPredefinedLocations(ctx android.ModuleContext, srcBootDexJarsByModule bootDexJarByModule, dstBootJarsByModule map[string]android.WritablePath) { + // Create the super set of module names. + names := []string{} + names = append(names, android.SortedStringKeys(srcBootDexJarsByModule)...) + names = append(names, android.SortedStringKeys(dstBootJarsByModule)...) + names = android.SortedUniqueStrings(names) + for _, name := range names { + src := srcBootDexJarsByModule[name] + dst := dstBootJarsByModule[name] + + if src == nil { + // A dex boot jar should be provided by the source java module. It needs to be installable or + // have compile_dex=true - cf. assignments to java.Module.dexJarFile. + // + // However, the source java module may be either replaced or overridden (using prefer:true) by + // a prebuilt java module with the same name. In that case the dex boot jar needs to be + // provided by the corresponding prebuilt APEX module. That APEX is the one that refers + // through a exported_(boot|systemserver)classpath_fragments property to a + // prebuilt_(boot|systemserver)classpath_fragment module, which in turn lists the prebuilt + // java module in the contents property. If that chain is broken then this dependency will + // fail. + if !ctx.Config().AllowMissingDependencies() { + ctx.ModuleErrorf("module %s does not provide a dex boot jar (see comment next to this message in Soong for details)", name) + } else { + ctx.AddMissingDependencies([]string{name}) + } + } else if dst == nil { + ctx.ModuleErrorf("module %s is not part of the boot configuration", name) + } else { + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: src, + Output: dst, + }) + } + } +} + +// buildBootImageVariantsForAndroidOs generates rules to build the boot image variants for the +// android.Android OsType and returns a map from the architectures to the paths of the generated +// boot image files. +// +// The paths are returned because they are needed elsewhere in Soong, e.g. for populating an APEX. +func buildBootImageVariantsForAndroidOs(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath) bootImageFilesByArch { + return buildBootImageForOsType(ctx, image, profile, android.Android) +} + +// buildBootImageVariantsForBuildOs generates rules to build the boot image variants for the +// config.BuildOS OsType, i.e. the type of OS on which the build is being running. +// +// The files need to be generated into their predefined location because they are used from there +// both within Soong and outside, e.g. for ART based host side testing and also for use by some +// cloud based tools. However, they are not needed by callers of this function and so the paths do +// not need to be returned from this func, unlike the buildBootImageVariantsForAndroidOs func. +func buildBootImageVariantsForBuildOs(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath) { + buildBootImageForOsType(ctx, image, profile, ctx.Config().BuildOS) +} + +// buildBootImageForOsType takes a bootImageConfig, a profile file and an android.OsType +// boot image files are required for and it creates rules to build the boot image +// files for all the required architectures for them. +// +// It returns a map from android.ArchType to the predefined paths of the boot image files. +func buildBootImageForOsType(ctx android.ModuleContext, image *bootImageConfig, profile android.WritablePath, requiredOsType android.OsType) bootImageFilesByArch { + filesByArch := bootImageFilesByArch{} + for _, variant := range image.variants { + if variant.target.Os == requiredOsType { + buildBootImageVariant(ctx, variant, profile) + filesByArch[variant.target.Arch.ArchType] = variant.imagesDeps.Paths() + } + } + + return filesByArch +} + +// buildBootImageZipInPredefinedLocation generates a zip file containing all the boot image files. +// +// The supplied filesByArch is nil when the boot image files have not been generated. Otherwise, it +// is a map from android.ArchType to the predefined locations. +func buildBootImageZipInPredefinedLocation(ctx android.ModuleContext, image *bootImageConfig, filesByArch bootImageFilesByArch) { + if filesByArch == nil { + return + } + + // Compute the list of files from all the architectures. + zipFiles := android.Paths{} + for _, archType := range android.ArchTypeList() { + zipFiles = append(zipFiles, filesByArch[archType]...) + } + + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command(). + BuiltTool("soong_zip"). + FlagWithOutput("-o ", image.zip). + FlagWithArg("-C ", image.dir.Join(ctx, android.Android.String()).String()). + FlagWithInputList("-f ", zipFiles, " -f ") + + rule.Build("zip_"+image.name, "zip "+image.name+" image") +} + +// Generate boot image build rules for a specific target. +func buildBootImageVariant(ctx android.ModuleContext, image *bootImageVariant, profile android.Path) { + + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + arch := image.target.Arch.ArchType + os := image.target.Os.String() // We need to distinguish host-x86 and device-x86. + symbolsDir := image.symbolsDir.Join(ctx, os, image.installDirOnHost, arch.String()) + symbolsFile := symbolsDir.Join(ctx, image.stem+".oat") + outputDir := image.dir.Join(ctx, os, image.installDirOnHost, arch.String()) + outputPath := outputDir.Join(ctx, image.stem+".oat") + oatLocation := dexpreopt.PathToLocation(outputPath, arch) + imagePath := outputPath.ReplaceExtension(ctx, "art") + + rule := android.NewRuleBuilder(pctx, ctx) + + rule.Command().Text("mkdir").Flag("-p").Flag(symbolsDir.String()) + rule.Command().Text("rm").Flag("-f"). + Flag(symbolsDir.Join(ctx, "*.art").String()). + Flag(symbolsDir.Join(ctx, "*.oat").String()). + Flag(symbolsDir.Join(ctx, "*.invocation").String()) + rule.Command().Text("rm").Flag("-f"). + Flag(outputDir.Join(ctx, "*.art").String()). + Flag(outputDir.Join(ctx, "*.oat").String()). + Flag(outputDir.Join(ctx, "*.invocation").String()) + + cmd := rule.Command() + + extraFlags := ctx.Config().Getenv("ART_BOOT_IMAGE_EXTRA_ARGS") + if extraFlags == "" { + // Use ANDROID_LOG_TAGS to suppress most logging by default... + cmd.Text(`ANDROID_LOG_TAGS="*:e"`) + } else { + // ...unless the boot image is generated specifically for testing, then allow all logging. + cmd.Text(`ANDROID_LOG_TAGS="*:v"`) + } + + invocationPath := outputPath.ReplaceExtension(ctx, "invocation") + + cmd.Tool(globalSoong.Dex2oat). + Flag("--avoid-storing-invocation"). + FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath). + Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatImageXms). + Flag("--runtime-arg").FlagWithArg("-Xmx", global.Dex2oatImageXmx) + + if profile != nil { + cmd.FlagWithInput("--profile-file=", profile) + } + + dirtyImageFile := "frameworks/base/config/dirty-image-objects" + dirtyImagePath := android.ExistentPathForSource(ctx, dirtyImageFile) + if dirtyImagePath.Valid() { + cmd.FlagWithInput("--dirty-image-objects=", dirtyImagePath.Path()) + } + + if image.extends != nil { + // It is a boot image extension, so it needs the boot image it depends on (in this case the + // primary ART APEX image). + artImage := image.primaryImages + cmd. + Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + // Add the path to the first file in the boot image with the arch specific directory removed, + // dex2oat will reconstruct the path to the actual file when it needs it. As the actual path + // to the file cannot be passed to the command make sure to add the actual path as an Implicit + // dependency to ensure that it is built before the command runs. + FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage). + // Similarly, the dex2oat tool will automatically find the paths to other files in the base + // boot image so make sure to add them as implicit dependencies to ensure that they are built + // before this command is run. + Implicits(image.primaryImagesDeps) + } else { + // It is a primary image, so it needs a base address. + cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()) + } + + // We always expect a preloaded classes file to be available. However, if we cannot find it, it's + // OK to not pass the flag to dex2oat. + preloadedClassesPath := android.ExistentPathForSource(ctx, image.preloadedClassesFile) + if preloadedClassesPath.Valid() { + cmd.FlagWithInput("--preloaded-classes=", preloadedClassesPath.Path()) + } + + cmd. + FlagForEachInput("--dex-file=", image.dexPaths.Paths()). + FlagForEachArg("--dex-location=", image.dexLocations). + Flag("--generate-debug-info"). + Flag("--generate-build-id"). + Flag("--image-format=lz4hc"). + FlagWithArg("--oat-symbols=", symbolsFile.String()). + Flag("--strip"). + FlagWithArg("--oat-file=", outputPath.String()). + FlagWithArg("--oat-location=", oatLocation). + FlagWithArg("--image=", imagePath.String()). + FlagWithArg("--instruction-set=", arch.String()). + FlagWithArg("--android-root=", global.EmptyDirectory). + FlagWithArg("--no-inline-from=", "core-oj.jar"). + Flag("--force-determinism"). + Flag("--abort-on-hard-verifier-error") + + // Use the default variant/features for host builds. + // The map below contains only device CPU info (which might be x86 on some devices). + if image.target.Os == android.Android { + cmd.FlagWithArg("--instruction-set-variant=", global.CpuVariant[arch]) + cmd.FlagWithArg("--instruction-set-features=", global.InstructionSetFeatures[arch]) + } + + if global.BootFlags != "" { + cmd.Flag(global.BootFlags) + } + + if extraFlags != "" { + cmd.Flag(extraFlags) + } + + cmd.Textf(`|| ( echo %s ; false )`, proptools.ShellEscape(failureMessage)) + + installDir := filepath.Join("/", image.installDirOnHost, arch.String()) + + var vdexInstalls android.RuleBuilderInstalls + var unstrippedInstalls android.RuleBuilderInstalls + var deviceInstalls android.RuleBuilderInstalls + + for _, artOrOat := range image.moduleFiles(ctx, outputDir, ".art", ".oat") { + cmd.ImplicitOutput(artOrOat) + + // Install the .oat and .art files + rule.Install(artOrOat, filepath.Join(installDir, artOrOat.Base())) + } + + for _, vdex := range image.moduleFiles(ctx, outputDir, ".vdex") { + cmd.ImplicitOutput(vdex) + + // Note that the vdex files are identical between architectures. + // Make rules will create symlinks to share them between architectures. + vdexInstalls = append(vdexInstalls, + android.RuleBuilderInstall{vdex, filepath.Join(installDir, vdex.Base())}) + } + + for _, unstrippedOat := range image.moduleFiles(ctx, symbolsDir, ".oat") { + cmd.ImplicitOutput(unstrippedOat) + + // Install the unstripped oat files. The Make rules will put these in $(TARGET_OUT_UNSTRIPPED) + unstrippedInstalls = append(unstrippedInstalls, + android.RuleBuilderInstall{unstrippedOat, filepath.Join(installDir, unstrippedOat.Base())}) + } + + if image.installDirOnHost != image.installDirOnDevice && !image.shouldInstallInApex() && !ctx.Config().UnbundledBuild() { + installDirOnDevice := filepath.Join("/", image.installDirOnDevice, arch.String()) + for _, file := range image.moduleFiles(ctx, outputDir, ".art", ".oat", ".vdex") { + deviceInstalls = append(deviceInstalls, + android.RuleBuilderInstall{file, filepath.Join(installDirOnDevice, file.Base())}) + } + } + + rule.Build(image.name+"JarsDexpreopt_"+image.target.String(), "dexpreopt "+image.name+" jars "+arch.String()) + + // save output and installed files for makevars + image.installs = rule.Installs() + image.vdexInstalls = vdexInstalls + image.unstrippedInstalls = unstrippedInstalls + image.deviceInstalls = deviceInstalls + image.licenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) +} + +const failureMessage = `ERROR: Dex2oat failed to compile a boot image. +It is likely that the boot classpath is inconsistent. +Rebuild with ART_BOOT_IMAGE_EXTRA_ARGS="--runtime-arg -verbose:verifier" to see verification errors.` + +func bootImageProfileRule(ctx android.ModuleContext, image *bootImageConfig) android.WritablePath { + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisableGenerateProfile { + return nil + } + + defaultProfile := "frameworks/base/config/boot-image-profile.txt" + + rule := android.NewRuleBuilder(pctx, ctx) + + var bootImageProfile android.Path + if len(global.BootImageProfiles) > 1 { + combinedBootImageProfile := image.dir.Join(ctx, "boot-image-profile.txt") + rule.Command().Text("cat").Inputs(global.BootImageProfiles).Text(">").Output(combinedBootImageProfile) + bootImageProfile = combinedBootImageProfile + } else if len(global.BootImageProfiles) == 1 { + bootImageProfile = global.BootImageProfiles[0] + } else if path := android.ExistentPathForSource(ctx, defaultProfile); path.Valid() { + bootImageProfile = path.Path() + } else { + // No profile (not even a default one, which is the case on some branches + // like master-art-host that don't have frameworks/base). + // Return nil and continue without profile. + return nil + } + + profile := image.dir.Join(ctx, "boot.prof") + + rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman). + Flag("--output-profile-type=boot"). + FlagWithInput("--create-profile-from=", bootImageProfile). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.getAnyAndroidVariant().dexLocationsDeps). + FlagWithOutput("--reference-profile-file=", profile) + + if image == defaultBootImageConfig(ctx) { + rule.Install(profile, "/system/etc/boot-image.prof") + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) + } + + rule.Build("bootJarsProfile", "profile boot jars") + + image.profilePathOnHost = profile + + return profile +} + +// bootFrameworkProfileRule generates the rule to create the boot framework profile and +// returns a path to the generated file. +func bootFrameworkProfileRule(ctx android.ModuleContext, image *bootImageConfig) android.WritablePath { + globalSoong := dexpreopt.GetGlobalSoongConfig(ctx) + global := dexpreopt.GetGlobalConfig(ctx) + + if global.DisableGenerateProfile || ctx.Config().UnbundledBuild() { + return nil + } + + defaultProfile := "frameworks/base/config/boot-profile.txt" + bootFrameworkProfile := android.PathForSource(ctx, defaultProfile) + + profile := image.dir.Join(ctx, "boot.bprof") + + rule := android.NewRuleBuilder(pctx, ctx) + rule.Command(). + Text(`ANDROID_LOG_TAGS="*:e"`). + Tool(globalSoong.Profman). + Flag("--output-profile-type=bprof"). + FlagWithInput("--create-profile-from=", bootFrameworkProfile). + FlagForEachInput("--apk=", image.dexPathsDeps.Paths()). + FlagForEachArg("--dex-location=", image.getAnyAndroidVariant().dexLocationsDeps). + FlagWithOutput("--reference-profile-file=", profile) + + rule.Install(profile, "/system/etc/boot-image.bprof") + rule.Build("bootFrameworkProfile", "profile boot framework jars") + image.profileInstalls = append(image.profileInstalls, rule.Installs()...) + image.profileLicenseMetadataFile = android.OptionalPathForPath(ctx.LicenseMetadataFile()) + + return profile +} + +func dumpOatRules(ctx android.ModuleContext, image *bootImageConfig) { + var allPhonies android.Paths + for _, image := range image.variants { + arch := image.target.Arch.ArchType + suffix := arch.String() + // Host and target might both use x86 arch. We need to ensure the names are unique. + if image.target.Os.Class == android.Host { + suffix = "host-" + suffix + } + // Create a rule to call oatdump. + output := android.PathForOutput(ctx, "boot."+suffix+".oatdump.txt") + rule := android.NewRuleBuilder(pctx, ctx) + imageLocationsOnHost, _ := image.imageLocations() + rule.Command(). + BuiltTool("oatdump"). + FlagWithInputList("--runtime-arg -Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). + FlagWithList("--runtime-arg -Xbootclasspath-locations:", image.dexLocationsDeps, ":"). + FlagWithArg("--image=", strings.Join(imageLocationsOnHost, ":")).Implicits(image.imagesDeps.Paths()). + FlagWithOutput("--output=", output). + FlagWithArg("--instruction-set=", arch.String()) + rule.Build("dump-oat-boot-"+suffix, "dump oat boot "+arch.String()) + + // Create a phony rule that depends on the output file and prints the path. + phony := android.PathForPhony(ctx, "dump-oat-boot-"+suffix) + rule = android.NewRuleBuilder(pctx, ctx) + rule.Command(). + Implicit(output). + ImplicitOutput(phony). + Text("echo").FlagWithArg("Output in ", output.String()) + rule.Build("phony-dump-oat-boot-"+suffix, "dump oat boot "+arch.String()) + + allPhonies = append(allPhonies, phony) + } + + phony := android.PathForPhony(ctx, "dump-oat-boot") + ctx.Build(pctx, android.BuildParams{ + Rule: android.Phony, + Output: phony, + Inputs: allPhonies, + Description: "dump-oat-boot", + }) +} + +func writeGlobalConfigForMake(ctx android.SingletonContext, path android.WritablePath) { + data := dexpreopt.GetGlobalConfigRawData(ctx) + + android.WriteFileRule(ctx, path, string(data)) +} + +// Define Make variables for boot image names, paths, etc. These variables are used in makefiles +// (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the +// correct output directories. +func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { + if d.dexpreoptConfigForMake != nil { + ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String()) + ctx.Strict("DEX_PREOPT_SOONG_CONFIG_FOR_MAKE", android.PathForOutput(ctx, "dexpreopt_soong.config").String()) + } + + image := d.defaultBootImage + if image == nil { + return + } + + ctx.Strict("DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED", image.profileInstalls.String()) + if image.profileLicenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_PROFILE_LICENSE_METADATA", image.profileLicenseMetadataFile.String()) + } + + global := dexpreopt.GetGlobalConfig(ctx) + dexPaths, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_FILES", strings.Join(dexPaths.Strings(), " ")) + ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " ")) + + for _, variant := range image.variants { + suffix := "" + if variant.target.Os.Class == android.Host { + suffix = "_host" + } + sfx := suffix + "_" + variant.target.Arch.ArchType.String() + ctx.Strict("DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_"+sfx, variant.vdexInstalls.String()) + ctx.Strict("DEXPREOPT_IMAGE_"+sfx, variant.imagePathOnHost.String()) + ctx.Strict("DEXPREOPT_IMAGE_DEPS_"+sfx, strings.Join(variant.imagesDeps.Strings(), " ")) + ctx.Strict("DEXPREOPT_IMAGE_BUILT_INSTALLED_"+sfx, variant.installs.String()) + ctx.Strict("DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_"+sfx, variant.unstrippedInstalls.String()) + if variant.licenseMetadataFile.Valid() { + ctx.Strict("DEXPREOPT_IMAGE_LICENSE_METADATA_"+sfx, variant.licenseMetadataFile.String()) + } + } + imageLocationsOnHost, imageLocationsOnDevice := image.getAnyAndroidVariant().imageLocations() + ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_ON_HOST", strings.Join(imageLocationsOnHost, ":")) + ctx.Strict("DEXPREOPT_IMAGE_LOCATIONS_ON_DEVICE", strings.Join(imageLocationsOnDevice, ":")) + ctx.Strict("DEXPREOPT_IMAGE_ZIP", image.zip.String()) + + // There used to be multiple images for JIT-Zygote mode, not there's only one. + ctx.Strict("DEXPREOPT_IMAGE_NAMES", image.name) +} diff --git a/java/dexpreopt_config.go_v1 b/java/dexpreopt_config.go_v1 new file mode 100644 index 000000000..d71e2bbfd --- /dev/null +++ b/java/dexpreopt_config.go_v1 @@ -0,0 +1,215 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "path/filepath" + "strings" + + "android/soong/android" + "android/soong/dexpreopt" +) + +// dexpreoptTargets returns the list of targets that are relevant to dexpreopting, which excludes architectures +// supported through native bridge. +func dexpreoptTargets(ctx android.PathContext) []android.Target { + var targets []android.Target + for _, target := range ctx.Config().Targets[android.Android] { + if target.NativeBridge == android.NativeBridgeDisabled { + targets = append(targets, target) + } + } + // We may also need the images on host in order to run host-based tests. + for _, target := range ctx.Config().Targets[ctx.Config().BuildOS] { + targets = append(targets, target) + } + + return targets +} + +var ( + bootImageConfigKey = android.NewOnceKey("bootImageConfig") + bootImageConfigRawKey = android.NewOnceKey("bootImageConfigRaw") + artBootImageName = "art" + frameworkBootImageName = "boot" +) + +func genBootImageConfigRaw(ctx android.PathContext) map[string]*bootImageConfig { + return ctx.Config().Once(bootImageConfigRawKey, func() interface{} { + global := dexpreopt.GetGlobalConfig(ctx) + + artModules := global.ArtApexJars + frameworkModules := global.BootJars.RemoveList(artModules) + + // ART config for the primary boot image in the ART apex. + // It includes the Core Libraries. + artCfg := bootImageConfig{ + name: artBootImageName, + stem: "boot", + installDirOnHost: "apex/art_boot_images/javalib", + installDirOnDevice: "system/framework", + profileInstallPathInApex: "etc/boot-image.prof", + modules: artModules, + preloadedClassesFile: "art/build/boot/preloaded-classes", + } + + // Framework config for the boot image extension. + // It includes framework libraries and depends on the ART config. + frameworkSubdir := "system/framework" + frameworkCfg := bootImageConfig{ + extends: &artCfg, + name: frameworkBootImageName, + stem: "boot", + installDirOnHost: frameworkSubdir, + installDirOnDevice: frameworkSubdir, + modules: frameworkModules, + preloadedClassesFile: "frameworks/base/config/preloaded-classes", + } + + return map[string]*bootImageConfig{ + artBootImageName: &artCfg, + frameworkBootImageName: &frameworkCfg, + } + }).(map[string]*bootImageConfig) +} + +// Construct the global boot image configs. +func genBootImageConfigs(ctx android.PathContext) map[string]*bootImageConfig { + return ctx.Config().Once(bootImageConfigKey, func() interface{} { + targets := dexpreoptTargets(ctx) + deviceDir := android.PathForOutput(ctx, ctx.Config().DeviceName()) + + configs := genBootImageConfigRaw(ctx) + artCfg := configs[artBootImageName] + frameworkCfg := configs[frameworkBootImageName] + + // common to all configs + for _, c := range configs { + c.dir = deviceDir.Join(ctx, "dex_"+c.name+"jars") + c.symbolsDir = deviceDir.Join(ctx, "dex_"+c.name+"jars_unstripped") + + // expands to <stem>.art for primary image and <stem>-<1st module>.art for extension + imageName := c.firstModuleNameOrStem(ctx) + ".art" + + // The path to bootclasspath dex files needs to be known at module + // GenerateAndroidBuildAction time, before the bootclasspath modules have been compiled. + // Set up known paths for them, the singleton rules will copy them there. + // TODO(b/143682396): use module dependencies instead + inputDir := deviceDir.Join(ctx, "dex_"+c.name+"jars_input") + c.dexPaths = c.modules.BuildPaths(ctx, inputDir) + c.dexPathsByModule = c.modules.BuildPathsByModule(ctx, inputDir) + c.dexPathsDeps = c.dexPaths + + // Create target-specific variants. + for _, target := range targets { + arch := target.Arch.ArchType + imageDir := c.dir.Join(ctx, target.Os.String(), c.installDirOnHost, arch.String()) + variant := &bootImageVariant{ + bootImageConfig: c, + target: target, + imagePathOnHost: imageDir.Join(ctx, imageName), + imagePathOnDevice: filepath.Join("/", c.installDirOnDevice, arch.String(), imageName), + imagesDeps: c.moduleFiles(ctx, imageDir, ".art", ".oat", ".vdex"), + dexLocations: c.modules.DevicePaths(ctx.Config(), target.Os), + } + variant.dexLocationsDeps = variant.dexLocations + c.variants = append(c.variants, variant) + } + + c.zip = c.dir.Join(ctx, c.name+".zip") + } + + // specific to the framework config + frameworkCfg.dexPathsDeps = append(artCfg.dexPathsDeps, frameworkCfg.dexPathsDeps...) + for i := range targets { + frameworkCfg.variants[i].primaryImages = artCfg.variants[i].imagePathOnHost + frameworkCfg.variants[i].primaryImagesDeps = artCfg.variants[i].imagesDeps.Paths() + frameworkCfg.variants[i].dexLocationsDeps = append(artCfg.variants[i].dexLocations, frameworkCfg.variants[i].dexLocationsDeps...) + } + + return configs + }).(map[string]*bootImageConfig) +} + +func defaultBootImageConfig(ctx android.PathContext) *bootImageConfig { + return genBootImageConfigs(ctx)[frameworkBootImageName] +} + +// Apex boot config allows to access build/install paths of apex boot jars without going +// through the usual trouble of registering dependencies on those modules and extracting build paths +// from those dependencies. +type apexBootConfig struct { + // A list of apex boot jars. + modules android.ConfiguredJarList + + // A list of predefined build paths to apex boot jars. They are configured very early, + // before the modules for these jars are processed and the actual paths are generated, and + // later on a singleton adds commands to copy actual jars to the predefined paths. + dexPaths android.WritablePaths + + // Map from module name (without prebuilt_ prefix) to the predefined build path. + dexPathsByModule map[string]android.WritablePath + + // A list of dex locations (a.k.a. on-device paths) to the boot jars. + dexLocations []string +} + +var updatableBootConfigKey = android.NewOnceKey("apexBootConfig") + +// Returns apex boot config. +func GetApexBootConfig(ctx android.PathContext) apexBootConfig { + return ctx.Config().Once(updatableBootConfigKey, func() interface{} { + apexBootJars := dexpreopt.GetGlobalConfig(ctx).ApexBootJars + + dir := android.PathForOutput(ctx, ctx.Config().DeviceName(), "apex_bootjars") + dexPaths := apexBootJars.BuildPaths(ctx, dir) + dexPathsByModuleName := apexBootJars.BuildPathsByModule(ctx, dir) + + dexLocations := apexBootJars.DevicePaths(ctx.Config(), android.Android) + + return apexBootConfig{apexBootJars, dexPaths, dexPathsByModuleName, dexLocations} + }).(apexBootConfig) +} + +// Returns a list of paths and a list of locations for the boot jars used in dexpreopt (to be +// passed in -Xbootclasspath and -Xbootclasspath-locations arguments for dex2oat). +func bcpForDexpreopt(ctx android.PathContext, withUpdatable bool) (android.WritablePaths, []string) { + // Non-updatable boot jars (they are used both in the boot image and in dexpreopt). + bootImage := defaultBootImageConfig(ctx) + dexPaths := bootImage.dexPathsDeps + // The dex locations for all Android variants are identical. + dexLocations := bootImage.getAnyAndroidVariant().dexLocationsDeps + + if withUpdatable { + // Apex boot jars (they are used only in dexpreopt, but not in the boot image). + apexBootConfig := GetApexBootConfig(ctx) + dexPaths = append(dexPaths, apexBootConfig.dexPaths...) + dexLocations = append(dexLocations, apexBootConfig.dexLocations...) + } + + return dexPaths, dexLocations +} + +var defaultBootclasspathKey = android.NewOnceKey("defaultBootclasspath") + +var copyOf = android.CopyOf + +func init() { + android.RegisterMakeVarsProvider(pctx, dexpreoptConfigMakevars) +} + +func dexpreoptConfigMakevars(ctx android.MakeVarsContext) { + ctx.Strict("DEXPREOPT_BOOT_JARS_MODULES", strings.Join(defaultBootImageConfig(ctx).modules.CopyOfApexJarPairs(), ":")) +} diff --git a/java/droidstubs.go b/java/droidstubs.go index 3b1f7c041..932fb19ce 100644 --- a/java/droidstubs.go +++ b/java/droidstubs.go @@ -135,6 +135,9 @@ type DroidstubsProperties struct { // if set to true, Metalava will allow framework SDK to contain API levels annotations. Api_levels_annotations_enabled *bool + // Apply the api levels database created by this module rather than generating one in this droidstubs. + Api_levels_module *string + // the dirs which Metalava extracts API levels annotations from. Api_levels_annotations_dirs []string @@ -234,6 +237,7 @@ func (d *Droidstubs) StubsSrcJar() android.Path { var metalavaMergeAnnotationsDirTag = dependencyTag{name: "metalava-merge-annotations-dir"} var metalavaMergeInclusionAnnotationsDirTag = dependencyTag{name: "metalava-merge-inclusion-annotations-dir"} var metalavaAPILevelsAnnotationsDirTag = dependencyTag{name: "metalava-api-levels-annotations-dir"} +var metalavaAPILevelsModuleTag = dependencyTag{name: "metalava-api-levels-module-tag"} func (d *Droidstubs) DepsMutator(ctx android.BottomUpMutatorContext) { d.Javadoc.addDeps(ctx) @@ -255,6 +259,10 @@ func (d *Droidstubs) DepsMutator(ctx android.BottomUpMutatorContext) { ctx.AddDependency(ctx.Module(), metalavaAPILevelsAnnotationsDirTag, apiLevelsAnnotationsDir) } } + + if d.properties.Api_levels_module != nil { + ctx.AddDependency(ctx.Module(), metalavaAPILevelsModuleTag, proptools.String(d.properties.Api_levels_module)) + } } func (d *Droidstubs) stubsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand, stubsDir android.OptionalPath) { @@ -365,21 +373,35 @@ func (d *Droidstubs) inclusionAnnotationsFlags(ctx android.ModuleContext, cmd *a } func (d *Droidstubs) apiLevelsAnnotationsFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { - if !Bool(d.properties.Api_levels_annotations_enabled) { - return + var apiVersions android.Path + if proptools.Bool(d.properties.Api_levels_annotations_enabled) { + d.apiLevelsGenerationFlags(ctx, cmd) + apiVersions = d.apiVersionsXml + } else { + ctx.VisitDirectDepsWithTag(metalavaAPILevelsModuleTag, func(m android.Module) { + if s, ok := m.(*Droidstubs); ok { + apiVersions = s.apiVersionsXml + } else { + ctx.PropertyErrorf("api_levels_module", + "module %q is not a droidstubs module", ctx.OtherModuleName(m)) + } + }) } + if apiVersions != nil { + cmd.FlagWithArg("--current-version ", ctx.Config().PlatformSdkVersion().String()) + cmd.FlagWithArg("--current-codename ", ctx.Config().PlatformSdkCodename()) + cmd.FlagWithInput("--apply-api-levels ", apiVersions) + } +} - d.apiVersionsXml = android.PathForModuleOut(ctx, "metalava", "api-versions.xml") - +func (d *Droidstubs) apiLevelsGenerationFlags(ctx android.ModuleContext, cmd *android.RuleBuilderCommand) { if len(d.properties.Api_levels_annotations_dirs) == 0 { ctx.PropertyErrorf("api_levels_annotations_dirs", "has to be non-empty if api levels annotations was enabled!") } + d.apiVersionsXml = android.PathForModuleOut(ctx, "metalava", "api-versions.xml") cmd.FlagWithOutput("--generate-api-levels ", d.apiVersionsXml) - cmd.FlagWithInput("--apply-api-levels ", d.apiVersionsXml) - cmd.FlagWithArg("--current-version ", ctx.Config().PlatformSdkVersion().String()) - cmd.FlagWithArg("--current-codename ", ctx.Config().PlatformSdkCodename()) filename := proptools.StringDefault(d.properties.Api_levels_jar_filename, "android.jar") @@ -675,87 +697,16 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { zipSyncCleanupCmd(rule, srcJarDir) - rule.Build("metalava", "metalava merged") - if apiCheckEnabled(ctx, d.properties.Check_api.Current, "current") { + d.generateCheckCurrentCheckedInApiIsUpToDateBuildRules(ctx) - if len(d.Javadoc.properties.Out) > 0 { - ctx.PropertyErrorf("out", "out property may not be combined with check_api") - } - - apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) - removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Removed_api_file)) - baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Current.Baseline_file) - - if baselineFile.Valid() { - ctx.PropertyErrorf("baseline_file", "current API check can't have a baseline file. (module %s)", ctx.ModuleName()) - } - - d.checkCurrentApiTimestamp = android.PathForModuleOut(ctx, "metalava", "check_current_api.timestamp") - - rule := android.NewRuleBuilder(pctx, ctx) - - // Diff command line. - // -F matches the closest "opening" line, such as "package android {" - // and " public class Intent {". - diff := `diff -u -F '{ *$'` - - rule.Command().Text("( true") - rule.Command(). - Text(diff). - Input(apiFile).Input(d.apiFile) - - rule.Command(). - Text(diff). - Input(removedApiFile).Input(d.removedApiFile) - - msg := fmt.Sprintf(`\n******************************\n`+ - `You have tried to change the API from what has been previously approved.\n\n`+ - `To make these errors go away, you have two choices:\n`+ - ` 1. You can add '@hide' javadoc comments (and remove @SystemApi/@TestApi/etc)\n`+ - ` to the new methods, etc. shown in the above diff.\n\n`+ - ` 2. You can update current.txt and/or removed.txt by executing the following command:\n`+ - ` m %s-update-current-api\n\n`+ - ` To submit the revised current.txt to the main Android repository,\n`+ - ` you will need approval.\n`+ - `******************************\n`, ctx.ModuleName()) - - rule.Command(). - Text("touch").Output(d.checkCurrentApiTimestamp). - Text(") || ("). - Text("echo").Flag("-e").Flag(`"` + msg + `"`). - Text("; exit 38"). - Text(")") - - rule.Build("metalavaCurrentApiCheck", "check current API") - - d.updateCurrentApiTimestamp = android.PathForModuleOut(ctx, "metalava", "update_current_api.timestamp") - - // update API rule - rule = android.NewRuleBuilder(pctx, ctx) - - rule.Command().Text("( true") - - rule.Command(). - Text("cp").Flag("-f"). - Input(d.apiFile).Flag(apiFile.String()) - - rule.Command(). - Text("cp").Flag("-f"). - Input(d.removedApiFile).Flag(removedApiFile.String()) - - msg = "failed to update public API" - - rule.Command(). - Text("touch").Output(d.updateCurrentApiTimestamp). - Text(") || ("). - Text("echo").Flag("-e").Flag(`"` + msg + `"`). - Text("; exit 38"). - Text(")") - - rule.Build("metalavaCurrentApiUpdate", "update current API") + // Make sure that whenever the API stubs are generated that the current checked in API files are + // checked to make sure that they are up-to-date. + cmd.Validation(d.checkCurrentApiTimestamp) } + rule.Build("metalava", "metalava merged") + if String(d.properties.Check_nullability_warnings) != "" { if d.nullabilityWarningsFile == nil { ctx.PropertyErrorf("check_nullability_warnings", @@ -792,6 +743,84 @@ func (d *Droidstubs) GenerateAndroidBuildActions(ctx android.ModuleContext) { } } +func (d *Droidstubs) generateCheckCurrentCheckedInApiIsUpToDateBuildRules(ctx android.ModuleContext) { + if len(d.Javadoc.properties.Out) > 0 { + ctx.PropertyErrorf("out", "out property may not be combined with check_api") + } + + apiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Api_file)) + removedApiFile := android.PathForModuleSrc(ctx, String(d.properties.Check_api.Current.Removed_api_file)) + baselineFile := android.OptionalPathForModuleSrc(ctx, d.properties.Check_api.Current.Baseline_file) + + if baselineFile.Valid() { + ctx.PropertyErrorf("baseline_file", "current API check can't have a baseline file. (module %s)", ctx.ModuleName()) + } + + d.checkCurrentApiTimestamp = android.PathForModuleOut(ctx, "metalava", "check_current_api.timestamp") + + rule := android.NewRuleBuilder(pctx, ctx) + + // Diff command line. + // -F matches the closest "opening" line, such as "package android {" + // and " public class Intent {". + diff := `diff -u -F '{ *$'` + + rule.Command().Text("( true") + rule.Command(). + Text(diff). + Input(apiFile).Input(d.apiFile) + + rule.Command(). + Text(diff). + Input(removedApiFile).Input(d.removedApiFile) + + msg := fmt.Sprintf(`\n******************************\n`+ + `You have tried to change the API from what has been previously approved.\n\n`+ + `To make these errors go away, you have two choices:\n`+ + ` 1. You can add '@hide' javadoc comments (and remove @SystemApi/@TestApi/etc)\n`+ + ` to the new methods, etc. shown in the above diff.\n\n`+ + ` 2. You can update current.txt and/or removed.txt by executing the following command:\n`+ + ` m %s-update-current-api\n\n`+ + ` To submit the revised current.txt to the main Android repository,\n`+ + ` you will need approval.\n`+ + `******************************\n`, ctx.ModuleName()) + + rule.Command(). + Text("touch").Output(d.checkCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build("metalavaCurrentApiCheck", "check current API") + + d.updateCurrentApiTimestamp = android.PathForModuleOut(ctx, "metalava", "update_current_api.timestamp") + + // update API rule + rule = android.NewRuleBuilder(pctx, ctx) + + rule.Command().Text("( true") + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.apiFile).Flag(apiFile.String()) + + rule.Command(). + Text("cp").Flag("-f"). + Input(d.removedApiFile).Flag(removedApiFile.String()) + + msg = "failed to update public API" + + rule.Command(). + Text("touch").Output(d.updateCurrentApiTimestamp). + Text(") || ("). + Text("echo").Flag("-e").Flag(`"` + msg + `"`). + Text("; exit 38"). + Text(")") + + rule.Build("metalavaCurrentApiUpdate", "update current API") +} + func StubsDefaultsFactory() android.Module { module := &DocDefaults{} diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go index 10d99f3a5..9fdfddeb1 100644 --- a/java/droidstubs_test.go +++ b/java/droidstubs_test.go @@ -46,6 +46,12 @@ func TestDroidstubs(t *testing.T) { api_levels_annotations_enabled: true, api_levels_jar_filename: "android.other.jar", } + + droidstubs { + name: "stubs-applying-api-versions", + srcs: ["bar-doc/a.java"], + api_levels_module: "bar-stubs-other", + } `, map[string][]byte{ "bar-doc/a.java": nil, @@ -53,26 +59,37 @@ func TestDroidstubs(t *testing.T) { testcases := []struct { moduleName string expectedJarFilename string + generate_xml bool high_mem bool }{ { moduleName: "bar-stubs", + generate_xml: true, expectedJarFilename: "android.jar", high_mem: false, }, { moduleName: "bar-stubs-other", + generate_xml: true, expectedJarFilename: "android.other.jar", high_mem: true, }, + { + moduleName: "stubs-applying-api-versions", + generate_xml: false, + }, } for _, c := range testcases { m := ctx.ModuleForTests(c.moduleName, "android_common") manifest := m.Output("metalava.sbox.textproto") sboxProto := android.RuleBuilderSboxProtoForTests(t, manifest) - expected := "--android-jar-pattern ./%/public/" + c.expectedJarFilename - if actual := String(sboxProto.Commands[0].Command); !strings.Contains(actual, expected) { - t.Errorf("For %q, expected metalava argument %q, but was not found %q", c.moduleName, expected, actual) + cmdline := String(sboxProto.Commands[0].Command) + android.AssertStringContainsEquals(t, "api-versions generation flag", cmdline, "--generate-api-levels", c.generate_xml) + if c.expectedJarFilename != "" { + expected := "--android-jar-pattern ./%/public/" + c.expectedJarFilename + if !strings.Contains(cmdline, expected) { + t.Errorf("For %q, expected metalava argument %q, but was not found %q", c.moduleName, expected, cmdline) + } } metalava := m.Rule("metalava") diff --git a/java/fuzz.go b/java/fuzz.go index 584c80b0c..d0f369f2f 100644 --- a/java/fuzz.go +++ b/java/fuzz.go @@ -50,37 +50,19 @@ type JavaFuzzLibrary struct { jniFilePaths android.Paths } -// IsSanitizerEnabled implemented to make JavaFuzzLibrary implement -// cc.Sanitizeable -func (j *JavaFuzzLibrary) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool { - for _, s := range j.jniProperties.Sanitizers { - if sanitizerName == s { - return true - } - } - return false -} - // IsSanitizerEnabledForJni implemented to make JavaFuzzLibrary implement // cc.JniSanitizeable. It returns a bool for whether a cc dependency should be // sanitized for the given sanitizer or not. func (j *JavaFuzzLibrary) IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool { - return j.IsSanitizerEnabled(ctx, sanitizerName) -} - -// EnableSanitizer implemented to make JavaFuzzLibrary implement -// cc.Sanitizeable -func (j *JavaFuzzLibrary) EnableSanitizer(sanitizerName string) { -} - -// AddSanitizerDependencies implemented to make JavaFuzzLibrary implement -// cc.Sanitizeable -func (j *JavaFuzzLibrary) AddSanitizerDependencies(mctx android.BottomUpMutatorContext, sanitizerName string) { + // TODO: once b/231370928 is resolved, please uncomment the loop + // for _, s := range j.jniProperties.Sanitizers { + // if sanitizerName == s { + // return true + // } + // } + return false } -// To verify that JavaFuzzLibrary implements cc.Sanitizeable -var _ cc.Sanitizeable = (*JavaFuzzLibrary)(nil) - func (j *JavaFuzzLibrary) DepsMutator(mctx android.BottomUpMutatorContext) { if len(j.jniProperties.Jni_libs) > 0 { if j.fuzzPackagedModule.FuzzProperties.Fuzz_config == nil { @@ -189,6 +171,10 @@ func (s *javaFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { return } + if javaFuzzModule.Target().HostCross { + return + } + fuzzModuleValidator := fuzz.FuzzModule{ javaFuzzModule.ModuleBase, javaFuzzModule.DefaultableModuleBase, diff --git a/java/hiddenapi.go b/java/hiddenapi.go index 3af5f1c7b..cf9c7ad7a 100644 --- a/java/hiddenapi.go +++ b/java/hiddenapi.go @@ -65,6 +65,8 @@ func (h *hiddenAPI) uncompressDex() *bool { type hiddenAPIModule interface { android.Module hiddenAPIIntf + + MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec } type hiddenAPIIntf interface { @@ -148,7 +150,7 @@ func (h *hiddenAPI) hiddenAPIEncodeDex(ctx android.ModuleContext, dexJar android // Create a copy of the dex jar which has been encoded with hiddenapi flags. flagsCSV := hiddenAPISingletonPaths(ctx).flags outputDir := android.PathForModuleOut(ctx, "hiddenapi").OutputPath - encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, outputDir) + encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, android.SdkSpecNone, outputDir) // Use the encoded dex jar from here onwards. return encodedDex @@ -246,7 +248,7 @@ var hiddenAPIEncodeDexRule = pctx.AndroidStaticRule("hiddenAPIEncodeDex", bluepr // The encode dex rule requires unzipping, encoding and rezipping the classes.dex files along with // all the resources from the input jar. It also ensures that if it was uncompressed in the input // it stays uncompressed in the output. -func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, outputDir android.OutputPath) android.OutputPath { +func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, minSdkVersion android.SdkSpec, outputDir android.OutputPath) android.OutputPath { // The output file has the same name as the input file and is in the output directory. output := outputDir.Join(ctx, dexInput.Base()) @@ -274,6 +276,15 @@ func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Pa hiddenapiFlags = "--no-force-assign-all" } + // If the library is targeted for Q and/or R then make sure that they do not + // have any S+ flags encoded as that will break the runtime. + minApiLevel := minSdkVersion.ApiLevel + if !minApiLevel.IsNone() { + if minApiLevel.LessThanOrEqualTo(android.ApiLevelOrPanic(ctx, "R")) { + hiddenapiFlags = hiddenapiFlags + " --max-hiddenapi-level=max-target-r" + } + } + ctx.Build(pctx, android.BuildParams{ Rule: hiddenAPIEncodeDexRule, Description: "hiddenapi encode dex", diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go index 534a8145f..c90b2ff97 100644 --- a/java/hiddenapi_modular.go +++ b/java/hiddenapi_modular.go @@ -1104,7 +1104,7 @@ func hiddenAPIRulesForBootclasspathFragment(ctx android.ModuleContext, contents for _, name := range android.SortedStringKeys(bootDexInfoByModule) { bootDexInfo := bootDexInfoByModule[name] unencodedDex := bootDexInfo.path - encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, outputDir) + encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, bootDexInfo.minSdkVersion, outputDir) encodedBootDexJarsByModule[name] = encodedDex } @@ -1188,6 +1188,9 @@ type bootDexInfo struct { // Indicates whether the dex jar needs uncompressing before encoding. uncompressDex bool + + // The minimum sdk version that the dex jar will be used on. + minSdkVersion android.SdkSpec } // bootDexInfoByModule is a map from module name (as returned by module.Name()) to the boot dex @@ -1213,6 +1216,7 @@ func extractBootDexInfoFromModules(ctx android.ModuleContext, contents []android bootDexJarsByModule[module.Name()] = bootDexInfo{ path: bootDexJar, uncompressDex: *hiddenAPIModule.uncompressDex(), + minSdkVersion: hiddenAPIModule.MinSdkVersion(ctx), } } diff --git a/java/java.go b/java/java.go index b34d6de8a..2897fd7f5 100644 --- a/java/java.go +++ b/java/java.go @@ -300,19 +300,11 @@ var _ android.LicenseAnnotationsDependencyTag = dependencyTag{} type usesLibraryDependencyTag struct { dependencyTag - - // SDK version in which the library appared as a standalone library. - sdkVersion int - - // If the dependency is optional or required. - optional bool - - // Whether this is an implicit dependency inferred by Soong, or an explicit one added via - // `uses_libs`/`optional_uses_libs` properties. - implicit bool + sdkVersion int // SDK version in which the library appared as a standalone library. + optional bool // If the dependency is optional or required. } -func makeUsesLibraryDependencyTag(sdkVersion int, optional bool, implicit bool) usesLibraryDependencyTag { +func makeUsesLibraryDependencyTag(sdkVersion int, optional bool) usesLibraryDependencyTag { return usesLibraryDependencyTag{ dependencyTag: dependencyTag{ name: fmt.Sprintf("uses-library-%d", sdkVersion), @@ -320,7 +312,6 @@ func makeUsesLibraryDependencyTag(sdkVersion int, optional bool, implicit bool) }, sdkVersion: sdkVersion, optional: optional, - implicit: implicit, } } @@ -351,6 +342,11 @@ var ( syspropPublicStubDepTag = dependencyTag{name: "sysprop public stub"} jniInstallTag = installDependencyTag{name: "jni install"} binaryInstallTag = installDependencyTag{name: "binary install"} + usesLibReqTag = makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, false) + usesLibOptTag = makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion, true) + usesLibCompat28OptTag = makeUsesLibraryDependencyTag(28, true) + usesLibCompat29ReqTag = makeUsesLibraryDependencyTag(29, false) + usesLibCompat30OptTag = makeUsesLibraryDependencyTag(30, true) ) func IsLibDepTag(depTag blueprint.DependencyTag) bool { @@ -472,6 +468,12 @@ func getJavaVersion(ctx android.ModuleContext, javaVersion string, sdkContext an return normalizeJavaVersion(ctx, javaVersion) } else if ctx.Device() { return defaultJavaLanguageVersion(ctx, sdkContext.SdkVersion(ctx)) + } else if ctx.Config().TargetsJava17() { + // Temporary experimental flag to be able to try and build with + // java version 17 options. The flag, if used, just sets Java + // 17 as the default version, leaving any components that + // target an older version intact. + return JAVA_VERSION_17 } else { return JAVA_VERSION_11 } @@ -486,6 +488,7 @@ const ( JAVA_VERSION_8 = 8 JAVA_VERSION_9 = 9 JAVA_VERSION_11 = 11 + JAVA_VERSION_17 = 17 ) func (v javaVersion) String() string { @@ -500,6 +503,8 @@ func (v javaVersion) String() string { return "1.9" case JAVA_VERSION_11: return "11" + case JAVA_VERSION_17: + return "17" default: return "unsupported" } @@ -522,8 +527,10 @@ func normalizeJavaVersion(ctx android.BaseModuleContext, javaVersion string) jav return JAVA_VERSION_9 case "11": return JAVA_VERSION_11 - case "10": - ctx.PropertyErrorf("java_version", "Java language levels 10 is not supported") + case "17": + return JAVA_VERSION_17 + case "10", "12", "13", "14", "15", "16": + ctx.PropertyErrorf("java_version", "Java language level %s is not supported", javaVersion) return JAVA_VERSION_UNSUPPORTED default: ctx.PropertyErrorf("java_version", "Unrecognized Java language level") @@ -596,12 +603,14 @@ func (j *Library) GenerateAndroidBuildActions(ctx android.ModuleContext) { } j.checkSdkVersions(ctx) - j.dexpreopter.installPath = j.dexpreopter.getInstallPath( - ctx, android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar")) - j.dexpreopter.isSDKLibrary = j.deviceProperties.IsSDKLibrary - setUncompressDex(ctx, &j.dexpreopter, &j.dexer) - j.dexpreopter.uncompressedDex = *j.dexProperties.Uncompress_dex - j.classLoaderContexts = j.usesLibrary.classLoaderContextForUsesLibDeps(ctx) + if ctx.Device() { + j.dexpreopter.installPath = j.dexpreopter.getInstallPath( + ctx, android.PathForModuleInstall(ctx, "framework", j.Stem()+".jar")) + j.dexpreopter.isSDKLibrary = j.deviceProperties.IsSDKLibrary + setUncompressDex(ctx, &j.dexpreopter, &j.dexer) + j.dexpreopter.uncompressedDex = *j.dexProperties.Uncompress_dex + j.classLoaderContexts = j.usesLibrary.classLoaderContextForUsesLibDeps(ctx) + } j.compile(ctx, nil) // Collect the module directory for IDE info in java/jdeps.go. @@ -855,7 +864,25 @@ type hostTestProperties struct { Data_native_bins []string `android:"arch_variant"` // list of device binary modules that should be installed alongside the test - Data_device_bins []string `android:"arch_variant"` + // This property only adds the first variant of the dependency + Data_device_bins_first []string `android:"arch_variant"` + + // list of device binary modules that should be installed alongside the test + // This property adds 64bit AND 32bit variants of the dependency + Data_device_bins_both []string `android:"arch_variant"` + + // list of device binary modules that should be installed alongside the test + // This property only adds 64bit variants of the dependency + Data_device_bins_64 []string `android:"arch_variant"` + + // list of device binary modules that should be installed alongside the test + // This property adds 32bit variants of the dependency if available, or else + // defaults to the 64bit variant + Data_device_bins_prefer32 []string `android:"arch_variant"` + + // list of device binary modules that should be installed alongside the test + // This property only adds 32bit variants of the dependency + Data_device_bins_32 []string `android:"arch_variant"` } type testHelperLibraryProperties struct { @@ -922,6 +949,83 @@ func (j *JavaTestImport) InstallInTestcases() bool { return true } +func (j *TestHost) addDataDeviceBinsDeps(ctx android.BottomUpMutatorContext) { + if len(j.testHostProperties.Data_device_bins_first) > 0 { + deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() + ctx.AddFarVariationDependencies(deviceVariations, dataDeviceBinsTag, j.testHostProperties.Data_device_bins_first...) + } + + var maybeAndroid32Target *android.Target + var maybeAndroid64Target *android.Target + android32TargetList := android.FirstTarget(ctx.Config().Targets[android.Android], "lib32") + android64TargetList := android.FirstTarget(ctx.Config().Targets[android.Android], "lib64") + if len(android32TargetList) > 0 { + maybeAndroid32Target = &android32TargetList[0] + } + if len(android64TargetList) > 0 { + maybeAndroid64Target = &android64TargetList[0] + } + + if len(j.testHostProperties.Data_device_bins_both) > 0 { + if maybeAndroid32Target == nil && maybeAndroid64Target == nil { + ctx.PropertyErrorf("data_device_bins_both", "no device targets available. Targets: %q", ctx.Config().Targets) + return + } + if maybeAndroid32Target != nil { + ctx.AddFarVariationDependencies( + maybeAndroid32Target.Variations(), + dataDeviceBinsTag, + j.testHostProperties.Data_device_bins_both..., + ) + } + if maybeAndroid64Target != nil { + ctx.AddFarVariationDependencies( + maybeAndroid64Target.Variations(), + dataDeviceBinsTag, + j.testHostProperties.Data_device_bins_both..., + ) + } + } + + if len(j.testHostProperties.Data_device_bins_prefer32) > 0 { + if maybeAndroid32Target != nil { + ctx.AddFarVariationDependencies( + maybeAndroid32Target.Variations(), + dataDeviceBinsTag, + j.testHostProperties.Data_device_bins_prefer32..., + ) + } else { + if maybeAndroid64Target == nil { + ctx.PropertyErrorf("data_device_bins_prefer32", "no device targets available. Targets: %q", ctx.Config().Targets) + return + } + ctx.AddFarVariationDependencies( + maybeAndroid64Target.Variations(), + dataDeviceBinsTag, + j.testHostProperties.Data_device_bins_prefer32..., + ) + } + } + + if len(j.testHostProperties.Data_device_bins_32) > 0 { + if maybeAndroid32Target == nil { + ctx.PropertyErrorf("data_device_bins_32", "cannot find 32bit device target. Targets: %q", ctx.Config().Targets) + return + } + deviceVariations := maybeAndroid32Target.Variations() + ctx.AddFarVariationDependencies(deviceVariations, dataDeviceBinsTag, j.testHostProperties.Data_device_bins_32...) + } + + if len(j.testHostProperties.Data_device_bins_64) > 0 { + if maybeAndroid64Target == nil { + ctx.PropertyErrorf("data_device_bins_64", "cannot find 64bit device target. Targets: %q", ctx.Config().Targets) + return + } + deviceVariations := maybeAndroid64Target.Variations() + ctx.AddFarVariationDependencies(deviceVariations, dataDeviceBinsTag, j.testHostProperties.Data_device_bins_64...) + } +} + func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { if len(j.testHostProperties.Data_native_bins) > 0 { for _, target := range ctx.MultiTargets() { @@ -929,11 +1033,6 @@ func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { } } - if len(j.testHostProperties.Data_device_bins) > 0 { - deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() - ctx.AddFarVariationDependencies(deviceVariations, dataDeviceBinsTag, j.testHostProperties.Data_device_bins...) - } - if len(j.testProperties.Jni_libs) > 0 { for _, target := range ctx.MultiTargets() { sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"}) @@ -941,6 +1040,8 @@ func (j *TestHost) DepsMutator(ctx android.BottomUpMutatorContext) { } } + j.addDataDeviceBinsDeps(ctx) + j.deps(ctx) } @@ -948,17 +1049,40 @@ func (j *TestHost) AddExtraResource(p android.Path) { j.extraResources = append(j.extraResources, p) } +func (j *TestHost) dataDeviceBins() []string { + ret := make([]string, 0, + len(j.testHostProperties.Data_device_bins_first)+ + len(j.testHostProperties.Data_device_bins_both)+ + len(j.testHostProperties.Data_device_bins_prefer32)+ + len(j.testHostProperties.Data_device_bins_32)+ + len(j.testHostProperties.Data_device_bins_64), + ) + + ret = append(ret, j.testHostProperties.Data_device_bins_first...) + ret = append(ret, j.testHostProperties.Data_device_bins_both...) + ret = append(ret, j.testHostProperties.Data_device_bins_prefer32...) + ret = append(ret, j.testHostProperties.Data_device_bins_32...) + ret = append(ret, j.testHostProperties.Data_device_bins_64...) + + return ret +} + func (j *TestHost) GenerateAndroidBuildActions(ctx android.ModuleContext) { var configs []tradefed.Config - if len(j.testHostProperties.Data_device_bins) > 0 { + dataDeviceBins := j.dataDeviceBins() + if len(dataDeviceBins) > 0 { // add Tradefed configuration to push device bins to device for testing remoteDir := filepath.Join("/data/local/tests/unrestricted/", j.Name()) options := []tradefed.Option{{Name: "cleanup", Value: "true"}} - for _, bin := range j.testHostProperties.Data_device_bins { + for _, bin := range dataDeviceBins { fullPath := filepath.Join(remoteDir, bin) options = append(options, tradefed.Option{Name: "push-file", Key: bin, Value: fullPath}) } - configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.PushFilePreparer", options}) + configs = append(configs, tradefed.Object{ + Type: "target_preparer", + Class: "com.android.tradefed.targetprep.PushFilePreparer", + Options: options, + }) } j.Test.generateAndroidBuildActionsWithConfig(ctx, configs) @@ -1249,10 +1373,10 @@ func (j *Binary) GenerateAndroidBuildActions(ctx android.ModuleContext) { } func (j *Binary) DepsMutator(ctx android.BottomUpMutatorContext) { - if ctx.Arch().ArchType == android.Common || ctx.BazelConversionMode() { + if ctx.Arch().ArchType == android.Common { j.deps(ctx) } - if ctx.Arch().ArchType != android.Common || ctx.BazelConversionMode() { + if ctx.Arch().ArchType != android.Common { // These dependencies ensure the host installation rules will install the jar file and // the jni libraries when the wrapper is installed. ctx.AddVariationDependencies(nil, jniInstallTag, j.binaryProperties.Jni_libs...) @@ -1996,10 +2120,8 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, depTag := ctx.OtherModuleDependencyTag(depModule) if depTag == libTag { // Ok, propagate <uses-library> through non-static library dependencies. - } else if tag, ok := depTag.(usesLibraryDependencyTag); ok && - tag.sdkVersion == dexpreopt.AnySdkVersion && tag.implicit { - // Ok, propagate <uses-library> through non-compatibility implicit <uses-library> - // dependencies. + } else if tag, ok := depTag.(usesLibraryDependencyTag); ok && tag.sdkVersion == dexpreopt.AnySdkVersion { + // Ok, propagate <uses-library> through non-compatibility <uses-library> dependencies. } else if depTag == staticLibTag { // Propagate <uses-library> through static library dependencies, unless it is a component // library (such as stubs). Component libraries have a dependency on their SDK library, @@ -2017,14 +2139,56 @@ func addCLCFromDep(ctx android.ModuleContext, depModule android.Module, // <uses_library> and should not be added to CLC, but the transitive <uses-library> dependencies // from its CLC should be added to the current CLC. if sdkLib != nil { - clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, true, + clcMap.AddContext(ctx, dexpreopt.AnySdkVersion, *sdkLib, false, dep.DexJarBuildPath().PathOrNil(), dep.DexJarInstallPath(), dep.ClassLoaderContexts()) } else { clcMap.AddContextMap(dep.ClassLoaderContexts(), depName) } } +type javaResourcesAttributes struct { + Resources bazel.LabelListAttribute + Resource_strip_prefix *string +} + +func (m *Library) convertJavaResourcesAttributes(ctx android.TopDownMutatorContext) *javaResourcesAttributes { + var resources bazel.LabelList + var resourceStripPrefix *string + + if m.properties.Java_resources != nil { + resources.Append(android.BazelLabelForModuleSrc(ctx, m.properties.Java_resources)) + } + + //TODO(b/179889880) handle case where glob includes files outside package + resDeps := ResourceDirsToFiles( + ctx, + m.properties.Java_resource_dirs, + m.properties.Exclude_java_resource_dirs, + m.properties.Exclude_java_resources, + ) + + for i, resDep := range resDeps { + dir, files := resDep.dir, resDep.files + + resources.Append(bazel.MakeLabelList(android.RootToModuleRelativePaths(ctx, files))) + + // Bazel includes the relative path from the WORKSPACE root when placing the resource + // inside the JAR file, so we need to remove that prefix + resourceStripPrefix = proptools.StringPtr(dir.String()) + if i > 0 { + // TODO(b/226423379) allow multiple resource prefixes + ctx.ModuleErrorf("bp2build does not support more than one directory in java_resource_dirs (b/226423379)") + } + } + + return &javaResourcesAttributes{ + Resources: bazel.MakeLabelListAttribute(resources), + Resource_strip_prefix: resourceStripPrefix, + } +} + type javaCommonAttributes struct { + *javaResourcesAttributes Srcs bazel.LabelListAttribute Plugins bazel.LabelListAttribute Javacopts bazel.StringListAttribute @@ -2089,6 +2253,11 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) if m.properties.Javacflags != nil { javacopts = append(javacopts, m.properties.Javacflags...) } + if m.properties.Java_version != nil { + javaVersion := normalizeJavaVersion(ctx, *m.properties.Java_version).String() + javacopts = append(javacopts, fmt.Sprintf("-source %s -target %s", javaVersion, javaVersion)) + } + epEnabled := m.properties.Errorprone.Enabled //TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable if Bool(epEnabled) { @@ -2096,7 +2265,8 @@ func (m *Library) convertLibraryAttrsBp2Build(ctx android.TopDownMutatorContext) } commonAttrs := &javaCommonAttributes{ - Srcs: javaSrcs, + Srcs: javaSrcs, + javaResourcesAttributes: m.convertJavaResourcesAttributes(ctx), Plugins: bazel.MakeLabelListAttribute( android.BazelLabelForModuleDeps(ctx, m.properties.Plugins), ), diff --git a/java/java_resources.go b/java/java_resources.go index 787d74a0d..b0dc5a1cf 100644 --- a/java/java_resources.go +++ b/java/java_resources.go @@ -33,8 +33,13 @@ var resourceExcludes = []string{ "**/*~", } -func ResourceDirsToJarArgs(ctx android.ModuleContext, - resourceDirs, excludeResourceDirs, excludeResourceFiles []string) (args []string, deps android.Paths) { +type resourceDeps struct { + dir android.Path + files android.Paths +} + +func ResourceDirsToFiles(ctx android.BaseModuleContext, + resourceDirs, excludeResourceDirs, excludeResourceFiles []string) (deps []resourceDeps) { var excludeDirs []string var excludeFiles []string @@ -55,21 +60,36 @@ func ResourceDirsToJarArgs(ctx android.ModuleContext, dirs := ctx.Glob(android.PathForSource(ctx, ctx.ModuleDir()).Join(ctx, resourceDir).String(), excludeDirs) for _, dir := range dirs { files := ctx.GlobFiles(filepath.Join(dir.String(), "**/*"), excludeFiles) + deps = append(deps, resourceDeps{ + dir: dir, + files: files, + }) + } + } - deps = append(deps, files...) + return deps +} + +func ResourceDirsToJarArgs(ctx android.ModuleContext, + resourceDirs, excludeResourceDirs, excludeResourceFiles []string) (args []string, deps android.Paths) { + resDeps := ResourceDirsToFiles(ctx, resourceDirs, excludeResourceDirs, excludeResourceFiles) - if len(files) > 0 { - args = append(args, "-C", dir.String()) + for _, resDep := range resDeps { + dir, files := resDep.dir, resDep.files - for _, f := range files { - path := f.String() - if !strings.HasPrefix(path, dir.String()) { - panic(fmt.Errorf("path %q does not start with %q", path, dir)) - } - args = append(args, "-f", pathtools.MatchEscape(path)) + if len(files) > 0 { + args = append(args, "-C", dir.String()) + deps = append(deps, files...) + + for _, f := range files { + path := f.String() + if !strings.HasPrefix(path, dir.String()) { + panic(fmt.Errorf("path %q does not start with %q", path, dir)) } + args = append(args, "-f", pathtools.MatchEscape(path)) } } + } return args, deps diff --git a/java/java_test.go b/java/java_test.go index 4c9382413..32b0b0f68 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -723,9 +723,9 @@ func TestDefaults(t *testing.T) { t.Errorf("atestNoOptimize should not optimize APK") } - atestDefault := ctx.ModuleForTests("atestDefault", "android_common").MaybeRule("r8") + atestDefault := ctx.ModuleForTests("atestDefault", "android_common").MaybeRule("d8") if atestDefault.Output == nil { - t.Errorf("atestDefault should optimize APK") + t.Errorf("atestDefault should not optimize APK") } } @@ -1498,62 +1498,172 @@ func TestErrorproneEnabledOnlyByEnvironmentVariable(t *testing.T) { } func TestDataDeviceBinsBuildsDeviceBinary(t *testing.T) { - bp := ` + testCases := []struct { + dataDeviceBinType string + depCompileMultilib string + variants []string + expectedError string + }{ + { + dataDeviceBinType: "first", + depCompileMultilib: "first", + variants: []string{"android_arm64_armv8-a"}, + }, + { + dataDeviceBinType: "first", + depCompileMultilib: "both", + variants: []string{"android_arm64_armv8-a"}, + }, + { + // this is true because our testing framework is set up with + // Targets ~ [<64bit target>, <32bit target>], where 64bit is "first" + dataDeviceBinType: "first", + depCompileMultilib: "32", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "first", + depCompileMultilib: "64", + variants: []string{"android_arm64_armv8-a"}, + }, + { + dataDeviceBinType: "both", + depCompileMultilib: "both", + variants: []string{ + "android_arm_armv7-a-neon", + "android_arm64_armv8-a", + }, + }, + { + dataDeviceBinType: "both", + depCompileMultilib: "32", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "both", + depCompileMultilib: "64", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "both", + depCompileMultilib: "first", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "32", + depCompileMultilib: "32", + variants: []string{"android_arm_armv7-a-neon"}, + }, + { + dataDeviceBinType: "32", + depCompileMultilib: "first", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "32", + depCompileMultilib: "both", + variants: []string{"android_arm_armv7-a-neon"}, + }, + { + dataDeviceBinType: "32", + depCompileMultilib: "64", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "64", + depCompileMultilib: "64", + variants: []string{"android_arm64_armv8-a"}, + }, + { + dataDeviceBinType: "64", + depCompileMultilib: "both", + variants: []string{"android_arm64_armv8-a"}, + }, + { + dataDeviceBinType: "64", + depCompileMultilib: "first", + variants: []string{"android_arm64_armv8-a"}, + }, + { + dataDeviceBinType: "64", + depCompileMultilib: "32", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "prefer32", + depCompileMultilib: "32", + variants: []string{"android_arm_armv7-a-neon"}, + }, + { + dataDeviceBinType: "prefer32", + depCompileMultilib: "both", + variants: []string{"android_arm_armv7-a-neon"}, + }, + { + dataDeviceBinType: "prefer32", + depCompileMultilib: "first", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + { + dataDeviceBinType: "prefer32", + depCompileMultilib: "64", + expectedError: `Android.bp:2:3: dependency "bar" of "foo" missing variant`, + }, + } + + bpTemplate := ` java_test_host { name: "foo", srcs: ["test.java"], - data_device_bins: ["bar"], + data_device_bins_%s: ["bar"], } cc_binary { name: "bar", + compile_multilib: "%s", } ` - ctx := android.GroupFixturePreparers( - PrepareForIntegrationTestWithJava, - ).RunTestWithBp(t, bp) - - buildOS := ctx.Config.BuildOS.String() - fooVariant := ctx.ModuleForTests("foo", buildOS+"_common") - barVariant := ctx.ModuleForTests("bar", "android_arm64_armv8-a") - fooMod := fooVariant.Module().(*TestHost) + for _, tc := range testCases { + bp := fmt.Sprintf(bpTemplate, tc.dataDeviceBinType, tc.depCompileMultilib) - relocated := barVariant.Output("bar") - expectedInput := "out/soong/.intermediates/bar/android_arm64_armv8-a/unstripped/bar" - android.AssertPathRelativeToTopEquals(t, "relocation input", expectedInput, relocated.Input) + errorHandler := android.FixtureExpectsNoErrors + if tc.expectedError != "" { + errorHandler = android.FixtureExpectsAtLeastOneErrorMatchingPattern(tc.expectedError) + } - entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, fooMod)[0] - expectedData := []string{ - "out/soong/.intermediates/bar/android_arm64_armv8-a/bar:bar", - } - actualData := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"] - android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", ctx.Config, expectedData, actualData) -} + testName := fmt.Sprintf(`data_device_bins_%s with compile_multilib:"%s"`, tc.dataDeviceBinType, tc.depCompileMultilib) + t.Run(testName, func(t *testing.T) { + ctx := android.GroupFixturePreparers(PrepareForIntegrationTestWithJava). + ExtendWithErrorHandler(errorHandler). + RunTestWithBp(t, bp) + if tc.expectedError != "" { + return + } -func TestDataDeviceBinsAutogenTradefedConfig(t *testing.T) { - bp := ` - java_test_host { - name: "foo", - srcs: ["test.java"], - data_device_bins: ["bar"], - } + buildOS := ctx.Config.BuildOS.String() + fooVariant := ctx.ModuleForTests("foo", buildOS+"_common") + fooMod := fooVariant.Module().(*TestHost) + entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, fooMod)[0] - cc_binary { - name: "bar", - } - ` + expectedAutogenConfig := `<option name="push-file" key="bar" value="/data/local/tests/unrestricted/foo/bar" />` + autogen := fooVariant.Rule("autogen") + if !strings.Contains(autogen.Args["extraConfigs"], expectedAutogenConfig) { + t.Errorf("foo extraConfigs %v does not contain %q", autogen.Args["extraConfigs"], expectedAutogenConfig) + } - ctx := android.GroupFixturePreparers( - PrepareForIntegrationTestWithJava, - ).RunTestWithBp(t, bp) + expectedData := []string{} + for _, variant := range tc.variants { + barVariant := ctx.ModuleForTests("bar", variant) + relocated := barVariant.Output("bar") + expectedInput := fmt.Sprintf("out/soong/.intermediates/bar/%s/unstripped/bar", variant) + android.AssertPathRelativeToTopEquals(t, "relocation input", expectedInput, relocated.Input) - buildOS := ctx.Config.BuildOS.String() - fooModule := ctx.ModuleForTests("foo", buildOS+"_common") - expectedAutogenConfig := `<option name="push-file" key="bar" value="/data/local/tests/unrestricted/foo/bar" />` + expectedData = append(expectedData, fmt.Sprintf("out/soong/.intermediates/bar/%s/bar:bar", variant)) + } - autogen := fooModule.Rule("autogen") - if !strings.Contains(autogen.Args["extraConfigs"], expectedAutogenConfig) { - t.Errorf("foo extraConfigs %v does not contain %q", autogen.Args["extraConfigs"], expectedAutogenConfig) + actualData := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"] + android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_TEST_DATA", ctx.Config, expectedData, actualData) + }) } } diff --git a/java/kotlin.go b/java/kotlin.go index eff5bb53f..903c6249b 100644 --- a/java/kotlin.go +++ b/java/kotlin.go @@ -175,6 +175,7 @@ func kotlinKapt(ctx android.ModuleContext, srcJarOutputFile, resJarOutputFile an var deps android.Paths deps = append(deps, flags.kotlincClasspath...) + deps = append(deps, flags.kotlincDeps...) deps = append(deps, srcJars...) deps = append(deps, flags.processorPath...) deps = append(deps, commonSrcFiles...) diff --git a/java/kotlin_test.go b/java/kotlin_test.go index f9ff98229..491ce2939 100644 --- a/java/kotlin_test.go +++ b/java/kotlin_test.go @@ -42,6 +42,11 @@ func TestKotlin(t *testing.T) { } `) + kotlinStdlib := ctx.ModuleForTests("kotlin-stdlib", "android_common"). + Output("turbine-combined/kotlin-stdlib.jar").Output + kotlinAnnotations := ctx.ModuleForTests("kotlin-annotations", "android_common"). + Output("turbine-combined/kotlin-annotations.jar").Output + fooKotlinc := ctx.ModuleForTests("foo", "android_common").Rule("kotlinc") fooJavac := ctx.ModuleForTests("foo", "android_common").Rule("javac") fooJar := ctx.ModuleForTests("foo", "android_common").Output("combined/foo.jar") @@ -69,6 +74,16 @@ func TestKotlin(t *testing.T) { fooJar.Inputs.Strings(), fooKotlincClasses.String()) } + if !inList(kotlinStdlib.String(), fooJar.Inputs.Strings()) { + t.Errorf("foo jar inputs %v does not contain %v", + fooJar.Inputs.Strings(), kotlinStdlib.String()) + } + + if !inList(kotlinAnnotations.String(), fooJar.Inputs.Strings()) { + t.Errorf("foo jar inputs %v does not contain %v", + fooJar.Inputs.Strings(), kotlinAnnotations.String()) + } + if !inList(fooKotlincHeaderClasses.String(), fooHeaderJar.Inputs.Strings()) { t.Errorf("foo header jar inputs %v does not contain %q", fooHeaderJar.Inputs.Strings(), fooKotlincHeaderClasses.String()) @@ -325,6 +340,7 @@ func TestKotlinCompose(t *testing.T) { java_library { name: "withcompose", srcs: ["a.kt"], + plugins: ["plugin"], static_libs: ["androidx.compose.runtime_runtime"], } @@ -332,6 +348,10 @@ func TestKotlinCompose(t *testing.T) { name: "nocompose", srcs: ["a.kt"], } + + java_plugin { + name: "plugin", + } `) buildOS := result.Config.BuildOS.String() @@ -346,6 +366,9 @@ func TestKotlinCompose(t *testing.T) { android.AssertStringDoesContain(t, "missing compose compiler plugin", withCompose.VariablesForTestsRelativeToTop()["kotlincFlags"], "-Xplugin="+composeCompiler.String()) + android.AssertStringListContains(t, "missing kapt compose compiler dependency", + withCompose.Rule("kapt").Implicits.Strings(), composeCompiler.String()) + android.AssertStringListDoesNotContain(t, "unexpected compose compiler dependency", noCompose.Rule("kotlinc").Implicits.Strings(), composeCompiler.String()) diff --git a/java/lint.go b/java/lint.go index f09db955d..e276345eb 100644 --- a/java/lint.go +++ b/java/lint.go @@ -525,10 +525,18 @@ func (l *lintSingleton) copyLintDependencies(ctx android.SingletonContext) { return } - frameworkDocStubs := findModuleOrErr(ctx, "framework-doc-stubs") - if frameworkDocStubs == nil { + apiVersionsDb := findModuleOrErr(ctx, "api_versions_public") + if apiVersionsDb == nil { if !ctx.Config().AllowMissingDependencies() { - ctx.Errorf("lint: missing framework-doc-stubs") + ctx.Errorf("lint: missing module api_versions_public") + } + return + } + + sdkAnnotations := findModuleOrErr(ctx, "sdk-annotations.zip") + if sdkAnnotations == nil { + if !ctx.Config().AllowMissingDependencies() { + ctx.Errorf("lint: missing module sdk-annotations.zip") } return } @@ -543,13 +551,13 @@ func (l *lintSingleton) copyLintDependencies(ctx android.SingletonContext) { ctx.Build(pctx, android.BuildParams{ Rule: android.CpIfChanged, - Input: android.OutputFileForModule(ctx, frameworkDocStubs, ".annotations.zip"), + Input: android.OutputFileForModule(ctx, sdkAnnotations, ""), Output: copiedAnnotationsZipPath(ctx), }) ctx.Build(pctx, android.BuildParams{ Rule: android.CpIfChanged, - Input: android.OutputFileForModule(ctx, frameworkDocStubs, ".api_versions.xml"), + Input: android.OutputFileForModule(ctx, apiVersionsDb, ".api_versions.xml"), Output: copiedAPIVersionsXmlPath(ctx, "api_versions.xml"), }) diff --git a/java/platform_bootclasspath_test.go b/java/platform_bootclasspath_test.go index 1c2a3aee5..10c918715 100644 --- a/java/platform_bootclasspath_test.go +++ b/java/platform_bootclasspath_test.go @@ -51,6 +51,7 @@ func TestPlatformBootclasspath(t *testing.T) { var addSourceBootclassPathModule = android.FixtureAddTextFile("source/Android.bp", ` java_library { name: "foo", + host_supported: true, // verify that b/232106778 is fixed srcs: ["a.java"], system_modules: "none", sdk_version: "none", @@ -271,7 +272,9 @@ func TestPlatformBootclasspath_Dist(t *testing.T) { entries := android.AndroidMkEntriesForTest(t, result.TestContext, platformBootclasspath) goals := entries[0].GetDistForGoals(platformBootclasspath) android.AssertStringEquals(t, "platform dist goals phony", ".PHONY: droidcore\n", goals[0]) - android.AssertStringEquals(t, "platform dist goals call", "$(call dist-for-goals,droidcore,out/soong/hiddenapi/hiddenapi-flags.csv:hiddenapi-flags.csv)\n", android.StringRelativeToTop(result.Config, goals[1])) + android.AssertStringDoesContain(t, "platform dist goals meta check", goals[1], "$(if $(strip $(ALL_TARGETS.") + android.AssertStringDoesContain(t, "platform dist goals meta assign", goals[1], "),,$(eval ALL_TARGETS.") + android.AssertStringEquals(t, "platform dist goals call", "$(call dist-for-goals,droidcore,out/soong/hiddenapi/hiddenapi-flags.csv:hiddenapi-flags.csv)\n", android.StringRelativeToTop(result.Config, goals[2])) } func TestPlatformBootclasspath_HiddenAPIMonolithicFiles(t *testing.T) { diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go index 44650a6d0..944970783 100644 --- a/java/prebuilt_apis.go +++ b/java/prebuilt_apis.go @@ -212,6 +212,10 @@ func createSystemModules(mctx android.LoadHookContext, version, scope string) { mctx.CreateModule(systemModulesImportFactory, &props) } +func PrebuiltApiModuleName(module, scope, version string) string { + return module + ".api." + scope + "." + version +} + func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { // <apiver>/<scope>/api/<module>.txt apiLevelFiles := globApiDirs(mctx, p, "api/*.txt") @@ -220,12 +224,9 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { } // Create modules for all (<module>, <scope, <version>) triplets, - apiModuleName := func(module, scope, version string) string { - return module + ".api." + scope + "." + version - } for _, f := range apiLevelFiles { module, version, scope := parseFinalizedPrebuiltPath(mctx, f) - createApiModule(mctx, apiModuleName(module, scope, strconv.Itoa(version)), f) + createApiModule(mctx, PrebuiltApiModuleName(module, scope, strconv.Itoa(version)), f) } // Figure out the latest version of each module/scope @@ -266,7 +267,7 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { // Sort the keys in order to make build.ninja stable for _, k := range android.SortedStringKeys(latest) { info := latest[k] - name := apiModuleName(info.module, info.scope, "latest") + name := PrebuiltApiModuleName(info.module, info.scope, "latest") createApiModule(mctx, name, info.path) } @@ -278,7 +279,7 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { filename, _, scope := parsePrebuiltPath(mctx, f) referencedModule := strings.TrimSuffix(filename, "-incompatibilities") - createApiModule(mctx, apiModuleName(referencedModule+"-incompatibilities", scope, "latest"), f) + createApiModule(mctx, PrebuiltApiModuleName(referencedModule+"-incompatibilities", scope, "latest"), f) incompatibilities[referencedModule+"."+scope] = true } @@ -286,7 +287,7 @@ func prebuiltApiFiles(mctx android.LoadHookContext, p *prebuiltApis) { // Create empty incompatibilities files for remaining modules for _, k := range android.SortedStringKeys(latest) { if _, ok := incompatibilities[k]; !ok { - createEmptyFile(mctx, apiModuleName(latest[k].module+"-incompatibilities", latest[k].scope, "latest")) + createEmptyFile(mctx, PrebuiltApiModuleName(latest[k].module+"-incompatibilities", latest[k].scope, "latest")) } } } diff --git a/java/proto.go b/java/proto.go index 5ba486fd6..5280077f1 100644 --- a/java/proto.go +++ b/java/proto.go @@ -91,7 +91,7 @@ func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) { case "lite", unspecifiedProtobufPluginType: ctx.AddVariationDependencies(nil, staticLibTag, "libprotobuf-java-lite") case "full": - if ctx.Host() || ctx.BazelConversionMode() { + if ctx.Host() { ctx.AddVariationDependencies(nil, staticLibTag, "libprotobuf-java-full") } else { ctx.PropertyErrorf("proto.type", "full java protos only supported on the host") diff --git a/java/sdk.go b/java/sdk.go index 0dddd40aa..b0da5afba 100644 --- a/java/sdk.go +++ b/java/sdk.go @@ -57,6 +57,12 @@ func defaultJavaLanguageVersion(ctx android.EarlyModuleContext, s android.SdkSpe return JAVA_VERSION_8 } else if sdk.FinalOrFutureInt() <= 31 { return JAVA_VERSION_9 + } else if ctx.Config().TargetsJava17() { + // Temporary experimental flag to be able to try and build with + // java version 17 options. The flag, if used, just sets Java + // 17 as the default version, leaving any components that + // target an older version intact. + return JAVA_VERSION_17 } else { return JAVA_VERSION_11 } diff --git a/java/sdk_library.go b/java/sdk_library.go index c37ed1a27..f7e5d9d40 100644 --- a/java/sdk_library.go +++ b/java/sdk_library.go @@ -97,6 +97,13 @@ type apiScope struct { // The tag to use to depend on the stubs source and API module. stubsSourceAndApiTag scopeDependencyTag + // The tag to use to depend on the module that provides the latest version of the API .txt file. + latestApiModuleTag scopeDependencyTag + + // The tag to use to depend on the module that provides the latest version of the API removed.txt + // file. + latestRemovedApiModuleTag scopeDependencyTag + // The scope specific prefix to add to the api file base of "current.txt" or "removed.txt". apiFilePrefix string @@ -158,6 +165,16 @@ func initApiScope(scope *apiScope) *apiScope { apiScope: scope, depInfoExtractor: (*scopePaths).extractStubsSourceAndApiInfoFromApiStubsProvider, } + scope.latestApiModuleTag = scopeDependencyTag{ + name: name + "-latest-api", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractLatestApiPath, + } + scope.latestRemovedApiModuleTag = scopeDependencyTag{ + name: name + "-latest-removed-api", + apiScope: scope, + depInfoExtractor: (*scopePaths).extractLatestRemovedApiPath, + } // To get the args needed to generate the stubs source append all the args from // this scope and all the scopes it extends as each set of args adds additional @@ -203,6 +220,24 @@ func (scope *apiScope) String() string { return scope.name } +// snapshotRelativeDir returns the snapshot directory into which the files related to scopes will +// be stored. +func (scope *apiScope) snapshotRelativeDir() string { + return filepath.Join("sdk_library", scope.name) +} + +// snapshotRelativeCurrentApiTxtPath returns the snapshot path to the API .txt file for the named +// library. +func (scope *apiScope) snapshotRelativeCurrentApiTxtPath(name string) string { + return filepath.Join(scope.snapshotRelativeDir(), name+".txt") +} + +// snapshotRelativeRemovedApiTxtPath returns the snapshot path to the removed API .txt file for the +// named library. +func (scope *apiScope) snapshotRelativeRemovedApiTxtPath(name string) string { + return filepath.Join(scope.snapshotRelativeDir(), name+"-removed.txt") +} + type apiScopes []*apiScope func (scopes apiScopes) Strings(accessor func(*apiScope) string) []string { @@ -377,6 +412,9 @@ type sdkLibraryProperties struct { // List of Java libraries that will be in the classpath when building the implementation lib Impl_only_libs []string `android:"arch_variant"` + // List of Java libraries that will included in the implementation lib. + Impl_only_static_libs []string `android:"arch_variant"` + // List of Java libraries that will be in the classpath when building stubs Stub_only_libs []string `android:"arch_variant"` @@ -399,7 +437,7 @@ type sdkLibraryProperties struct { // Determines whether a runtime implementation library is built; defaults to false. // // If true then it also prevents the module from being used as a shared module, i.e. - // it is as is shared_library: false, was set. + // it is as if shared_library: false, was set. Api_only *bool // local files that are used within user customized droiddoc options. @@ -536,6 +574,12 @@ type scopePaths struct { // Extracted annotations. annotationsZip android.OptionalPath + + // The path to the latest API file. + latestApiPath android.OptionalPath + + // The path to the latest removed API file. + latestRemovedApiPath android.OptionalPath } func (paths *scopePaths) extractStubsLibraryInfoFromDependency(ctx android.ModuleContext, dep android.Module) error { @@ -599,6 +643,31 @@ func (paths *scopePaths) extractStubsSourceAndApiInfoFromApiStubsProvider(ctx an }) } +func extractSingleOptionalOutputPath(dep android.Module) (android.OptionalPath, error) { + var paths android.Paths + if sourceFileProducer, ok := dep.(android.SourceFileProducer); ok { + paths = sourceFileProducer.Srcs() + } else { + return android.OptionalPath{}, fmt.Errorf("module %q does not produce source files", dep) + } + if len(paths) != 1 { + return android.OptionalPath{}, fmt.Errorf("expected one path from %q, got %q", dep, paths) + } + return android.OptionalPathForPath(paths[0]), nil +} + +func (paths *scopePaths) extractLatestApiPath(ctx android.ModuleContext, dep android.Module) error { + outputPath, err := extractSingleOptionalOutputPath(dep) + paths.latestApiPath = outputPath + return err +} + +func (paths *scopePaths) extractLatestRemovedApiPath(ctx android.ModuleContext, dep android.Module) error { + outputPath, err := extractSingleOptionalOutputPath(dep) + paths.latestRemovedApiPath = outputPath + return err +} + type commonToSdkLibraryAndImportProperties struct { // The naming scheme to use for the components that this module creates. // @@ -1171,6 +1240,16 @@ func (module *SdkLibrary) ComponentDepsMutator(ctx android.BottomUpMutatorContex // Add a dependency on the stubs source in order to access both stubs source and api information. ctx.AddVariationDependencies(nil, apiScope.stubsSourceAndApiTag, module.stubsSourceModuleName(apiScope)) + + if module.compareAgainstLatestApi(apiScope) { + // Add dependencies on the latest finalized version of the API .txt file. + latestApiModuleName := module.latestApiModuleName(apiScope) + ctx.AddDependency(module, apiScope.latestApiModuleTag, latestApiModuleName) + + // Add dependencies on the latest finalized version of the remove API .txt file. + latestRemovedApiModuleName := module.latestRemovedApiModuleName(apiScope) + ctx.AddDependency(module, apiScope.latestRemovedApiModuleTag, latestRemovedApiModuleName) + } } if module.requiresRuntimeImplementationLibrary() { @@ -1191,13 +1270,13 @@ func (module *SdkLibrary) DepsMutator(ctx android.BottomUpMutatorContext) { if apiScope.unstable { continue } - if m := android.SrcIsModule(module.latestApiFilegroupName(apiScope)); !ctx.OtherModuleExists(m) { + if m := module.latestApiModuleName(apiScope); !ctx.OtherModuleExists(m) { missingApiModules = append(missingApiModules, m) } - if m := android.SrcIsModule(module.latestRemovedApiFilegroupName(apiScope)); !ctx.OtherModuleExists(m) { + if m := module.latestRemovedApiModuleName(apiScope); !ctx.OtherModuleExists(m) { missingApiModules = append(missingApiModules, m) } - if m := android.SrcIsModule(module.latestIncompatibilitiesFilegroupName(apiScope)); !ctx.OtherModuleExists(m) { + if m := module.latestIncompatibilitiesModuleName(apiScope); !ctx.OtherModuleExists(m) { missingApiModules = append(missingApiModules, m) } } @@ -1271,6 +1350,26 @@ func (module *SdkLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) // Make the set of components exported by this module available for use elsewhere. exportedComponentInfo := android.ExportedComponentsInfo{Components: android.SortedStringKeys(exportedComponents)} ctx.SetProvider(android.ExportedComponentsInfoProvider, exportedComponentInfo) + + // Provide additional information for inclusion in an sdk's generated .info file. + additionalSdkInfo := map[string]interface{}{} + additionalSdkInfo["dist_stem"] = module.distStem() + baseModuleName := module.BaseModuleName() + scopes := map[string]interface{}{} + additionalSdkInfo["scopes"] = scopes + for scope, scopePaths := range module.scopePaths { + scopeInfo := map[string]interface{}{} + scopes[scope.name] = scopeInfo + scopeInfo["current_api"] = scope.snapshotRelativeCurrentApiTxtPath(baseModuleName) + scopeInfo["removed_api"] = scope.snapshotRelativeRemovedApiTxtPath(baseModuleName) + if p := scopePaths.latestApiPath; p.Valid() { + scopeInfo["latest_api"] = p.Path().String() + } + if p := scopePaths.latestRemovedApiPath; p.Valid() { + scopeInfo["latest_removed_api"] = p.Path().String() + } + } + ctx.SetProvider(android.AdditionalSdkInfoProvider, android.AdditionalSdkInfo{additionalSdkInfo}) } func (module *SdkLibrary) AndroidMkEntries() []android.AndroidMkEntries { @@ -1316,16 +1415,32 @@ func (module *SdkLibrary) distGroup() string { return proptools.StringDefault(module.sdkLibraryProperties.Dist_group, "unknown") } +func latestPrebuiltApiModuleName(name string, apiScope *apiScope) string { + return PrebuiltApiModuleName(name, apiScope.name, "latest") +} + func (module *SdkLibrary) latestApiFilegroupName(apiScope *apiScope) string { - return ":" + module.distStem() + ".api." + apiScope.name + ".latest" + return ":" + module.latestApiModuleName(apiScope) +} + +func (module *SdkLibrary) latestApiModuleName(apiScope *apiScope) string { + return latestPrebuiltApiModuleName(module.distStem(), apiScope) } func (module *SdkLibrary) latestRemovedApiFilegroupName(apiScope *apiScope) string { - return ":" + module.distStem() + "-removed.api." + apiScope.name + ".latest" + return ":" + module.latestRemovedApiModuleName(apiScope) +} + +func (module *SdkLibrary) latestRemovedApiModuleName(apiScope *apiScope) string { + return latestPrebuiltApiModuleName(module.distStem()+"-removed", apiScope) } func (module *SdkLibrary) latestIncompatibilitiesFilegroupName(apiScope *apiScope) string { - return ":" + module.distStem() + "-incompatibilities.api." + apiScope.name + ".latest" + return ":" + module.latestIncompatibilitiesModuleName(apiScope) +} + +func (module *SdkLibrary) latestIncompatibilitiesModuleName(apiScope *apiScope) string { + return latestPrebuiltApiModuleName(module.distStem()+"-incompatibilities", apiScope) } func childModuleVisibility(childVisibility []string) []string { @@ -1346,10 +1461,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) visibility := childModuleVisibility(module.sdkLibraryProperties.Impl_library_visibility) props := struct { - Name *string - Visibility []string - Instrument bool - Libs []string + Name *string + Visibility []string + Instrument bool + Libs []string + Static_libs []string + Apex_available []string }{ Name: proptools.StringPtr(module.implLibraryModuleName()), Visibility: visibility, @@ -1358,6 +1475,12 @@ func (module *SdkLibrary) createImplLibrary(mctx android.DefaultableHookContext) // Set the impl_only libs. Note that the module's "Libs" get appended as well, via the // addition of &module.properties below. Libs: module.sdkLibraryProperties.Impl_only_libs, + // Set the impl_only static libs. Note that the module's "static_libs" get appended as well, via the + // addition of &module.properties below. + Static_libs: module.sdkLibraryProperties.Impl_only_static_libs, + // Pass the apex_available settings down so that the impl library can be statically + // embedded within a library that is added to an APEX. Needed for updatable-media. + Apex_available: module.ApexAvailable(), } properties := []interface{}{ @@ -1546,7 +1669,7 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC props.Check_api.Current.Api_file = proptools.StringPtr(currentApiFileName) props.Check_api.Current.Removed_api_file = proptools.StringPtr(removedApiFileName) - if !(apiScope.unstable || module.sdkLibraryProperties.Unsafe_ignore_missing_latest_api) { + if module.compareAgainstLatestApi(apiScope) { // check against the latest released API latestApiFilegroupName := proptools.StringPtr(module.latestApiFilegroupName(apiScope)) props.Previous_api = latestApiFilegroupName @@ -1598,6 +1721,10 @@ func (module *SdkLibrary) createStubsSourcesAndApi(mctx android.DefaultableHookC mctx.CreateModule(DroidstubsFactory, &props) } +func (module *SdkLibrary) compareAgainstLatestApi(apiScope *apiScope) bool { + return !(apiScope.unstable || module.sdkLibraryProperties.Unsafe_ignore_missing_latest_api) +} + // Implements android.ApexModule func (module *SdkLibrary) DepIsInSameApex(mctx android.BaseModuleContext, dep android.Module) bool { depTag := mctx.OtherModuleDependencyTag(dep) @@ -1814,8 +1941,9 @@ func (module *SdkLibrary) CreateInternalModules(mctx android.DefaultableHookCont *javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName()) } - // Add the impl_only_libs *after* we're done using the Libs prop in submodules. + // Add the impl_only_libs and impl_only_static_libs *after* we're done using them in submodules. module.properties.Libs = append(module.properties.Libs, module.sdkLibraryProperties.Impl_only_libs...) + module.properties.Static_libs = append(module.properties.Static_libs, module.sdkLibraryProperties.Impl_only_static_libs...) } func (module *SdkLibrary) InitSdkLibraryProperties() { @@ -2211,8 +2339,23 @@ func (module *SdkLibraryImport) UniqueApexVariations() bool { return module.uniqueApexVariations() } +// MinSdkVersion - Implements hiddenAPIModule +func (module *SdkLibraryImport) MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec { + return android.SdkSpecNone +} + +var _ hiddenAPIModule = (*SdkLibraryImport)(nil) + func (module *SdkLibraryImport) OutputFiles(tag string) (android.Paths, error) { - return module.commonOutputFiles(tag) + paths, err := module.commonOutputFiles(tag) + if paths != nil || err != nil { + return paths, err + } + if module.implLibraryModule != nil { + return module.implLibraryModule.OutputFiles(tag) + } else { + return nil, nil + } } func (module *SdkLibraryImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { @@ -2876,7 +3019,7 @@ func (s *sdkLibrarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberCo if properties, ok := s.Scopes[apiScope]; ok { scopeSet := propertySet.AddPropertySet(apiScope.propertyName) - scopeDir := filepath.Join("sdk_library", s.OsPrefix(), apiScope.name) + scopeDir := apiScope.snapshotRelativeDir() var jars []string for _, p := range properties.Jars { @@ -2900,13 +3043,13 @@ func (s *sdkLibrarySdkMemberProperties) AddToPropertySet(ctx android.SdkMemberCo } if properties.CurrentApiFile != nil { - currentApiSnapshotPath := filepath.Join(scopeDir, ctx.Name()+".txt") + currentApiSnapshotPath := apiScope.snapshotRelativeCurrentApiTxtPath(ctx.Name()) ctx.SnapshotBuilder().CopyToSnapshot(properties.CurrentApiFile, currentApiSnapshotPath) scopeSet.AddProperty("current_api", currentApiSnapshotPath) } if properties.RemovedApiFile != nil { - removedApiSnapshotPath := filepath.Join(scopeDir, ctx.Name()+"-removed.txt") + removedApiSnapshotPath := apiScope.snapshotRelativeRemovedApiTxtPath(ctx.Name()) ctx.SnapshotBuilder().CopyToSnapshot(properties.RemovedApiFile, removedApiSnapshotPath) scopeSet.AddProperty("removed_api", removedApiSnapshotPath) } diff --git a/java/sdk_library_test.go b/java/sdk_library_test.go index 3500c84d2..805bc226f 100644 --- a/java/sdk_library_test.go +++ b/java/sdk_library_test.go @@ -126,6 +126,10 @@ func TestJavaSdkLibrary(t *testing.T) { exportedComponentsInfo := result.ModuleProvider(foo.Module(), android.ExportedComponentsInfoProvider).(android.ExportedComponentsInfo) expectedFooExportedComponents := []string{ + "foo-removed.api.public.latest", + "foo-removed.api.system.latest", + "foo.api.public.latest", + "foo.api.system.latest", "foo.stubs", "foo.stubs.source", "foo.stubs.source.system", @@ -529,6 +533,8 @@ func TestJavaSdkLibrary_Deps(t *testing.T) { CheckModuleDependencies(t, result.TestContext, "sdklib", "android_common", []string{ `dex2oatd`, + `sdklib-removed.api.public.latest`, + `sdklib.api.public.latest`, `sdklib.impl`, `sdklib.stubs`, `sdklib.stubs.source`, @@ -851,6 +857,8 @@ func TestJavaSdkLibraryImport_WithSource(t *testing.T) { CheckModuleDependencies(t, result.TestContext, "sdklib", "android_common", []string{ `dex2oatd`, `prebuilt_sdklib`, + `sdklib-removed.api.public.latest`, + `sdklib.api.public.latest`, `sdklib.impl`, `sdklib.stubs`, `sdklib.stubs.source`, @@ -894,6 +902,8 @@ func TestJavaSdkLibraryImport_Preferred(t *testing.T) { CheckModuleDependencies(t, result.TestContext, "sdklib", "android_common", []string{ `prebuilt_sdklib`, + `sdklib-removed.api.public.latest`, + `sdklib.api.public.latest`, `sdklib.impl`, `sdklib.stubs`, `sdklib.stubs.source`, diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go index e84eacd86..cc83430a8 100644 --- a/mk2rbc/cmd/mk2rbc.go +++ b/mk2rbc/cmd/mk2rbc.go @@ -173,7 +173,7 @@ func main() { } ok := true for _, mkFile := range files { - ok = convertOne(mkFile) && ok + ok = convertOne(mkFile, []string{}) && ok } if *launcher != "" { @@ -183,7 +183,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the product launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the product launcher input variables file failed to convert")) } @@ -201,7 +201,7 @@ func main() { if *inputVariables == "" { quit(fmt.Errorf("the board launcher requires an input variables file")) } - if !convertOne(*inputVariables) { + if !convertOne(*inputVariables, []string{}) { quit(fmt.Errorf("the board launcher input variables file failed to convert")) } err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( @@ -310,9 +310,13 @@ const copyright = `# // the output hierarchy, or to the stdout. // Optionally, recursively convert the files this one includes by // $(call inherit-product) or an include statement. -func convertOne(mkFile string) (ok bool) { +func convertOne(mkFile string, loadStack []string) (ok bool) { if v, ok := converted[mkFile]; ok { - return v != nil + if v == nil { + fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile) + return false + } + return true } converted[mkFile] = nil defer func() { @@ -356,6 +360,7 @@ func convertOne(mkFile string) (ok bool) { return false } } + loadStack = append(loadStack, mkFile) ok = true if *recurse { for _, sub := range ss.SubConfigFiles() { @@ -363,7 +368,7 @@ func convertOne(mkFile string) (ok bool) { if _, err := os.Stat(sub); os.IsNotExist(err) { continue } - ok = convertOne(sub) && ok + ok = convertOne(sub, loadStack) && ok } } converted[mkFile] = ss diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go index 92665207d..6a6eb460a 100644 --- a/mk2rbc/expr.go +++ b/mk2rbc/expr.go @@ -741,8 +741,8 @@ func (_ *badExpr) typ() starlarkType { return starlarkTypeUnknown } -func (_ *badExpr) emitListVarCopy(_ *generationContext) { - panic("implement me") +func (b *badExpr) emitListVarCopy(gctx *generationContext) { + b.emit(gctx) } func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr { diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go index 02b3d0859..2707f0c5e 100644 --- a/mk2rbc/mk2rbc.go +++ b/mk2rbc/mk2rbc.go @@ -86,7 +86,7 @@ var knownFunctions = map[string]interface { "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList}, "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList}, "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList}, - "firstword": &firstOrLastwordCallParser{isLastWord: false}, + "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString}, "foreach": &foreachCallParser{}, "if": &ifCallParser{}, "info": &makeControlFuncParser{name: baseName + ".mkinfo"}, @@ -97,7 +97,7 @@ var knownFunctions = map[string]interface { "is-product-in-list": &isProductInListCallParser{}, "is-vendor-board-platform": &isVendorBoardPlatformCallParser{}, "is-vendor-board-qcom": &isVendorBoardQcomCallParser{}, - "lastword": &firstOrLastwordCallParser{isLastWord: true}, + "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString}, "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString}, "math_max": &mathMaxOrMinCallParser{function: "max"}, "math_min": &mathMaxOrMinCallParser{function: "min"}, @@ -116,6 +116,7 @@ var knownFunctions = map[string]interface { "subst": &substCallParser{fname: "subst"}, "warning": &makeControlFuncParser{name: baseName + ".mkwarning"}, "word": &wordCallParser{}, + "words": &wordsCallParser{}, "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList}, } @@ -130,6 +131,14 @@ var knownNodeFunctions = map[string]interface { "foreach": &foreachCallNodeParser{}, } +// These look like variables, but are actually functions, and would give +// undefined variable errors if we converted them as variables. Instead, +// emit an error instead of converting them. +var unsupportedFunctions = map[string]bool{ + "local-generated-sources-dir": true, + "local-intermediates-dir": true, +} + // These are functions that we don't implement conversions for, but // we allow seeing their definitions in the product config files. var ignoredDefines = map[string]bool{ @@ -459,6 +468,7 @@ func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { predefined := []struct{ name, value string }{ {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")}, {"LOCAL_PATH", filepath.Dir(ss.mkFile)}, + {"MAKEFILE_LIST", ss.mkFile}, {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk // TODO(asmundak): maybe read it from build/make/core/envsetup.mk? {"TARGET_COPY_OUT_SYSTEM", "system"}, @@ -562,9 +572,6 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeUnknown { // Try to divine variable type from the RHS asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{xBad}} - } inferred_type := asgn.value.typ() if inferred_type != starlarkTypeUnknown { lhs.setValueType(inferred_type) @@ -573,21 +580,19 @@ func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode if lhs.valueType() == starlarkTypeList { xConcat, xBad := ctx.buildConcatExpr(a) if xBad != nil { - return []starlarkNode{&exprNode{expr: xBad}} - } - switch len(xConcat.items) { - case 0: - asgn.value = &listExpr{} - case 1: - asgn.value = xConcat.items[0] - default: - asgn.value = xConcat + asgn.value = xBad + } else { + switch len(xConcat.items) { + case 0: + asgn.value = &listExpr{} + case 1: + asgn.value = xConcat.items[0] + default: + asgn.value = xConcat + } } } else { asgn.value = ctx.parseMakeString(a, a.Value) - if xBad, ok := asgn.value.(*badExpr); ok { - return []starlarkNode{&exprNode{expr: xBad}} - } } if asgn.lhs.valueType() == starlarkTypeString && @@ -817,35 +822,32 @@ func (ctx *parseContext) handleSubConfig( // rblf.inherit(handle, _e[0], _e[1]) // var matchingPaths []string - varPath, ok := pathExpr.(*interpolateExpr) - if !ok { - return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} - } - - pathPattern := []string{varPath.chunks[0]} - for _, chunk := range varPath.chunks[1:] { - if chunk != "" { - pathPattern = append(pathPattern, chunk) + var needsWarning = false + if interpolate, ok := pathExpr.(*interpolateExpr); ok { + pathPattern := []string{interpolate.chunks[0]} + for _, chunk := range interpolate.chunks[1:] { + if chunk != "" { + pathPattern = append(pathPattern, chunk) + } } - } - if pathPattern[0] == "" && len(ctx.includeTops) > 0 { - // If pattern starts from the top. restrict it to the directories where - // we know inherit-product uses dynamically calculated path. - for _, p := range ctx.includeTops { - pathPattern[0] = p - matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...) + if len(pathPattern) == 1 { + pathPattern = append(pathPattern, "") } - } else { matchingPaths = ctx.findMatchingPaths(pathPattern) + needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0 + } else if len(ctx.includeTops) > 0 { + matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{"", ""})...) + } else { + return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} } + // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) const maxMatchingFiles = 150 if len(matchingPaths) > maxMatchingFiles { return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} } - needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0 - res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} + res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} for _, p := range matchingPaths { // A product configuration files discovered dynamically may attempt to inherit // from another one which does not exist in this source tree. Prevent load errors @@ -862,17 +864,31 @@ func (ctx *parseContext) findMatchingPaths(pattern []string) []string { } // Create regular expression from the pattern - s_regexp := "^" + regexp.QuoteMeta(pattern[0]) + regexString := "^" + regexp.QuoteMeta(pattern[0]) for _, s := range pattern[1:] { - s_regexp += ".*" + regexp.QuoteMeta(s) + regexString += ".*" + regexp.QuoteMeta(s) } - s_regexp += "$" - rex := regexp.MustCompile(s_regexp) + regexString += "$" + rex := regexp.MustCompile(regexString) + + includeTopRegexString := "" + if len(ctx.includeTops) > 0 { + for i, top := range ctx.includeTops { + if i > 0 { + includeTopRegexString += "|" + } + includeTopRegexString += "^" + regexp.QuoteMeta(top) + } + } else { + includeTopRegexString = ".*" + } + + includeTopRegex := regexp.MustCompile(includeTopRegexString) // Now match var res []string for _, p := range files { - if rex.MatchString(p) { + if rex.MatchString(p) && includeTopRegex.MatchString(p) { res = append(res, p) } } @@ -895,8 +911,9 @@ func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, arg }) } -func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode { - return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode { +func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode { + loadAlways := v.Name[0] != '-' + return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode { return &includeNode{im, loadAlways} }) } @@ -1074,6 +1091,18 @@ func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { return otherOperand } } + if otherOperand.typ() == starlarkTypeList { + fields := strings.Fields(stringOperand) + elements := make([]starlarkExpr, len(fields)) + for i, s := range fields { + elements[i] = &stringLiteralExpr{literal: s} + } + return &eqExpr{ + left: otherOperand, + right: &listExpr{elements}, + isEq: isEq, + } + } if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt { return &eqExpr{ left: otherOperand, @@ -1119,8 +1148,6 @@ func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, switch call.name { case baseName + ".filter": return ctx.parseCompareFilterFuncResult(directive, call, value, isEq) - case baseName + ".expand_wildcard": - return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true case baseName + ".findstring": return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true case baseName + ".strip": @@ -1165,22 +1192,6 @@ func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, } } -func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive, - xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { - if !isEmptyString(xValue) { - return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue) - } - callFunc := baseName + ".file_wildcard_exists" - if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") { - callFunc = baseName + ".file_exists" - } - var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool} - if !negate { - cc = ¬Expr{cc} - } - return cc -} - func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive, xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { if isEmptyString(xValue) { @@ -1265,7 +1276,34 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt // Handle only the case where the first (or only) word is constant words := ref.SplitN(" ", 2) if !words[0].Const() { - return ctx.newBadExpr(node, "reference is too complex: %s", refDump) + if len(words) == 1 { + expr := ctx.parseMakeString(node, ref) + return &callExpr{ + object: &identifierExpr{"cfg"}, + name: "get", + args: []starlarkExpr{ + expr, + &callExpr{ + object: &identifierExpr{"g"}, + name: "get", + args: []starlarkExpr{ + expr, + &stringLiteralExpr{literal: ""}, + }, + returnType: starlarkTypeUnknown, + }, + }, + returnType: starlarkTypeUnknown, + } + } else { + return ctx.newBadExpr(node, "reference is too complex: %s", refDump) + } + } + + if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok { + if _, unsupported := unsupportedFunctions[name]; unsupported { + return ctx.newBadExpr(node, "%s is not supported", refDump) + } } // If it is a single word, it can be a simple variable @@ -1315,9 +1353,8 @@ func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeSt } else { return ctx.newBadExpr(node, "cannot handle invoking %s", name) } - } else { - return ctx.newBadExpr(node, "cannot handle %s", refDump) } + return ctx.newBadExpr(node, "cannot handle %s", refDump) } type simpleCallParser struct { @@ -1564,11 +1601,21 @@ func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *m } } - return &foreachExpr{ + var result starlarkExpr = &foreachExpr{ varName: loopVarName, list: list, action: action, } + + if action.typ() == starlarkTypeList { + result = &callExpr{ + name: baseName + ".flatten_2d_list", + args: []starlarkExpr{result}, + returnType: starlarkTypeList, + } + } + + return result } func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) { @@ -1593,6 +1640,16 @@ func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starla for _, n := range a.actions { transformNode(n, transformer) } + case *inheritNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } + case *includeNode: + if b, ok := a.module.(inheritedDynamicModule); ok { + b.path = b.path.transform(transformer) + a.module = b + } } } @@ -1643,9 +1700,11 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa if len(words) != 2 { return ctx.newBadExpr(node, "word function should have 2 arguments") } - var index uint64 = 0 + var index = 0 if words[0].Const() { - index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64) + if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil { + index = i + } } if index < 1 { return ctx.newBadExpr(node, "word index should be constant positive integer") @@ -1653,35 +1712,40 @@ func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa words[1].TrimLeftSpaces() words[1].TrimRightSpaces() array := ctx.parseMakeString(node, words[1]) - if xBad, ok := array.(*badExpr); ok { - return xBad + if bad, ok := array.(*badExpr); ok { + return bad } if array.typ() != starlarkTypeList { - array = &callExpr{object: array, name: "split", returnType: starlarkTypeList} + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, + } } - return &indexExpr{array, &intLiteralExpr{int(index - 1)}} + return &indexExpr{array, &intLiteralExpr{index - 1}} } -type firstOrLastwordCallParser struct { - isLastWord bool -} +type wordsCallParser struct{} -func (p *firstOrLastwordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { - arg := ctx.parseMakeString(node, args) - if bad, ok := arg.(*badExpr); ok { +func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { + args.TrimLeftSpaces() + args.TrimRightSpaces() + array := ctx.parseMakeString(node, args) + if bad, ok := array.(*badExpr); ok { return bad } - index := &intLiteralExpr{0} - if p.isLastWord { - if v, ok := arg.(*variableRefExpr); ok && v.ref.name() == "MAKEFILE_LIST" { - return &stringLiteralExpr{ctx.script.mkFile} + if array.typ() != starlarkTypeList { + array = &callExpr{ + name: baseName + ".words", + args: []starlarkExpr{array}, + returnType: starlarkTypeList, } - index.literal = -1 } - if arg.typ() == starlarkTypeList { - return &indexExpr{arg, index} + return &callExpr{ + name: "len", + args: []starlarkExpr{array}, + returnType: starlarkTypeInt, } - return &indexExpr{&callExpr{object: arg, name: "split", returnType: starlarkTypeList}, index} } func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) { @@ -1765,10 +1829,23 @@ func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkpa } case *mkparser.Comment: return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}} + case *mkparser.Directive: + if n.Name == "include" || n.Name == "-include" { + return ctx.handleInclude(n) + } + case *mkparser.Variable: + // Technically inherit-product(-if-exists) don't need to be put inside + // an eval, but some makefiles do it, presumably because they copy+pasted + // from a $(eval include ...) + if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok { + if name == "inherit-product" || name == "inherit-product-if-exists" { + return ctx.handleVariable(n) + } + } } } - return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")} + return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")} } func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr { @@ -1828,7 +1905,7 @@ func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNod result = []starlarkNode{res} } case "include", "-include": - result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-') + result = ctx.handleInclude(x) case "ifeq", "ifneq", "ifdef", "ifndef": result = []starlarkNode{ctx.handleIfBlock(x)} default: diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go index 9485a42fc..31555d3f5 100644 --- a/mk2rbc/mk2rbc_test.go +++ b/mk2rbc/mk2rbc_test.go @@ -117,8 +117,8 @@ PRODUCT_NAME := $(call foo0) def init(g, handle): cfg = rblf.cfg(handle) - rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") - rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1") + cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0") `, }, { @@ -568,14 +568,18 @@ ifeq (,$(wildcard foo.mk)) endif ifneq (,$(wildcard foo*.mk)) endif +ifeq (foo1.mk foo2.mk barxyz.mk,$(wildcard foo*.mk bar*.mk)) +endif `, expected: `load("//build/make/core:product_config.rbc", "rblf") def init(g, handle): cfg = rblf.cfg(handle) - if not rblf.file_exists("foo.mk"): + if not rblf.expand_wildcard("foo.mk"): + pass + if rblf.expand_wildcard("foo*.mk"): pass - if rblf.file_wildcard_exists("foo*.mk"): + if rblf.expand_wildcard("foo*.mk bar*.mk") == ["foo1.mk", "foo2.mk", "barxyz.mk"]: pass `, }, @@ -808,6 +812,10 @@ def init(g, handle): PRODUCT_COPY_FILES := $(addprefix pfx-,a b c) PRODUCT_COPY_FILES := $(addsuffix .sff, a b c) PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM))) +ifeq (1,$(words $(SOME_UNKNOWN_VARIABLE))) +endif +ifeq ($(words $(SOME_OTHER_VARIABLE)),$(SOME_INT_VARIABLE)) +endif $(info $(patsubst %.pub,$(PRODUCT_NAME)%,$(PRODUCT_ADB_KEYS))) $(info $$(dir foo/bar): $(dir foo/bar)) $(info $(firstword $(PRODUCT_COPY_FILES))) @@ -830,14 +838,18 @@ def init(g, handle): cfg = rblf.cfg(handle) cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c") cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c") - cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0] + cfg["PRODUCT_NAME"] = rblf.words((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " "))[0] + if len(rblf.words(g.get("SOME_UNKNOWN_VARIABLE", ""))) == 1: + pass + if ("%d" % (len(rblf.words(g.get("SOME_OTHER_VARIABLE", ""))))) == g.get("SOME_INT_VARIABLE", ""): + pass rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%s%%" % cfg["PRODUCT_NAME"], g.get("PRODUCT_ADB_KEYS", ""))) rblf.mkinfo("product.mk", "$(dir foo/bar): %s" % rblf.dir("foo/bar")) - rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0]) - rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][-1]) - rblf.mkinfo("product.mk", rblf.dir("product.mk")) - rblf.mkinfo("product.mk", rblf.dir(cfg["PRODUCT_COPY_FILES"][-1])) - rblf.mkinfo("product.mk", rblf.dir((_foobar).split()[-1])) + rblf.mkinfo("product.mk", rblf.first_word(cfg["PRODUCT_COPY_FILES"])) + rblf.mkinfo("product.mk", rblf.last_word(cfg["PRODUCT_COPY_FILES"])) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word("product.mk"))) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(cfg["PRODUCT_COPY_FILES"]))) + rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(_foobar))) rblf.mkinfo("product.mk", rblf.abspath("foo/bar")) rblf.mkinfo("product.mk", rblf.notdir("foo/bar")) rblf.soong_config_namespace(g, "snsconfig") @@ -975,7 +987,7 @@ def init(g, handle): rblf.soong_config_namespace(g, "cvd") rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json") rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg") - rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") + _x = rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config") `, }, { desc: "soong namespace accesses", @@ -1142,6 +1154,13 @@ def init(g, handle): MY_PATH:=foo #RBC# include_top vendor/foo1 $(call inherit-product,$(MY_PATH)/cfg.mk) +#RBC# include_top vendor/foo1 +$(call inherit-product,$(MY_OTHER_PATH)) +#RBC# include_top vendor/foo1 +$(call inherit-product,vendor/$(MY_OTHER_PATH)) +#RBC# include_top vendor/foo1 +$(foreach f,$(MY_MAKEFILES), \ + $(call inherit-product,$(f))) `, expected: `load("//build/make/core:product_config.rbc", "rblf") load("//vendor/foo1:cfg.star|init", _cfg_init = "init") @@ -1156,6 +1175,28 @@ def init(g, handle): if not _varmod_init: rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"])) rblf.inherit(handle, _varmod, _varmod_init) + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(g.get("MY_OTHER_PATH", "")) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (g.get("MY_OTHER_PATH", ""))) + rblf.inherit(handle, _varmod, _varmod_init) + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get("vendor/%s" % g.get("MY_OTHER_PATH", "")) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % ("vendor/%s" % g.get("MY_OTHER_PATH", ""))) + rblf.inherit(handle, _varmod, _varmod_init) + for f in rblf.words(g.get("MY_MAKEFILES", "")): + _entry = { + "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init), + }.get(f) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % (f)) + rblf.inherit(handle, _varmod, _varmod_init) `, }, { @@ -1271,6 +1312,7 @@ def init(g, handle): in: ` ifeq (,$(call foobar)) endif +my_sources := $(local-generated-sources-dir) `, expected: `load("//build/make/core:product_config.rbc", "rblf") @@ -1278,6 +1320,7 @@ def init(g, handle): cfg = rblf.cfg(handle) if rblf.mk2rbc_error("build/product.mk:2", "cannot handle invoking foobar"): pass + _my_sources = rblf.mk2rbc_error("build/product.mk:4", "local-generated-sources-dir is not supported") `, }, { @@ -1329,6 +1372,8 @@ BOOT_KERNEL_MODULES_FILTER := $(foreach m,$(BOOT_KERNEL_MODULES),%/$(m)) BOOT_KERNEL_MODULES_LIST := foo.ko BOOT_KERNEL_MODULES_LIST += bar.ko BOOT_KERNEL_MODULES_FILTER_2 := $(foreach m,$(BOOT_KERNEL_MODULES_LIST),%/$(m)) +NESTED_LISTS := $(foreach m,$(SOME_VAR),$(BOOT_KERNEL_MODULES_LIST)) +NESTED_LISTS_2 := $(foreach x,$(SOME_VAR),$(foreach y,$(x),prefix$(y))) FOREACH_WITH_IF := $(foreach module,\ $(BOOT_KERNEL_MODULES_LIST),\ @@ -1348,6 +1393,8 @@ def init(g, handle): g["BOOT_KERNEL_MODULES_LIST"] = ["foo.ko"] g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"] g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]] + g["NESTED_LISTS"] = rblf.flatten_2d_list([g["BOOT_KERNEL_MODULES_LIST"] for m in rblf.words(g.get("SOME_VAR", ""))]) + g["NESTED_LISTS_2"] = rblf.flatten_2d_list([["prefix%s" % y for y in rblf.words(x)] for x in rblf.words(g.get("SOME_VAR", ""))]) g["FOREACH_WITH_IF"] = [("" if rblf.filter(module, "foo.ko") else rblf.mkerror("product.mk", "module \"%s\" has an error!" % module)) for module in g["BOOT_KERNEL_MODULES_LIST"]] # Same as above, but not assigning it to a variable allows it to be converted to statements for module in g["BOOT_KERNEL_MODULES_LIST"]: @@ -1511,24 +1558,40 @@ $(eval) $(eval MY_VAR := foo) $(eval # This is a test of eval functions) $(eval $(TOO_COMPLICATED) := bar) +$(eval include foo/font.mk) +$(eval $(call inherit-product,vendor/foo1/cfg.mk)) + $(foreach x,$(MY_LIST_VAR), \ $(eval PRODUCT_COPY_FILES += foo/bar/$(x):$(TARGET_COPY_OUT_VENDOR)/etc/$(x)) \ - $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))) \ -) + $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x)))) +$(foreach x,$(MY_LIST_VAR), \ + $(eval include foo/$(x).mk)) `, expected: `load("//build/make/core:product_config.rbc", "rblf") +load("//foo:font.star", _font_init = "init") +load("//vendor/foo1:cfg.star", _cfg_init = "init") def init(g, handle): cfg = rblf.cfg(handle) g["MY_VAR"] = "foo" # This is a test of eval functions - rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments and comments are supported") + rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported") + _font_init(g, handle) + rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init) for x in rblf.words(g.get("MY_LIST_VAR", "")): rblf.setdefault(handle, "PRODUCT_COPY_FILES") cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split() if g.get("MY_OTHER_VAR", ""): cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split() + for x in rblf.words(g.get("MY_LIST_VAR", "")): + _entry = { + "foo/font.mk": ("foo/font", _font_init), + }.get("foo/%s.mk" % x) + (_varmod, _varmod_init) = _entry if _entry else (None, None) + if not _varmod_init: + rblf.mkerror("product.mk", "Cannot find %s" % ("foo/%s.mk" % x)) + _varmod_init(g, handle) `, }, { @@ -1545,6 +1608,27 @@ def init(g, handle): g["MY_VAR"] = "foo" `, }, + { + desc: "Complicated variable references", + mkname: "product.mk", + in: ` +MY_VAR := foo +MY_VAR_2 := MY_VAR +MY_VAR_3 := $($(MY_VAR_2)) +MY_VAR_4 := $(foo bar) +MY_VAR_5 := $($(MY_VAR_2) bar) +`, + expected: `load("//build/make/core:product_config.rbc", "rblf") + +def init(g, handle): + cfg = rblf.cfg(handle) + g["MY_VAR"] = "foo" + g["MY_VAR_2"] = "MY_VAR" + g["MY_VAR_3"] = (cfg).get(g["MY_VAR_2"], (g).get(g["MY_VAR_2"], "")) + g["MY_VAR_4"] = rblf.mk2rbc_error("product.mk:5", "cannot handle invoking foo") + g["MY_VAR_5"] = rblf.mk2rbc_error("product.mk:6", "reference is too complex: $(MY_VAR_2) bar") +`, + }, } var known_variables = []struct { diff --git a/mk2rbc/node.go b/mk2rbc/node.go index 7c39b9ead..a01abd8ac 100644 --- a/mk2rbc/node.go +++ b/mk2rbc/node.go @@ -83,7 +83,7 @@ func (im inheritedStaticModule) needsLoadCheck() bool { } type inheritedDynamicModule struct { - path interpolateExpr + path starlarkExpr candidateModules []*moduleInfo loadAlways bool location ErrorLocation @@ -120,7 +120,7 @@ func (i inheritedDynamicModule) emitSelect(gctx *generationContext) { } func (i inheritedDynamicModule) pathExpr() starlarkExpr { - return &i.path + return i.path } func (i inheritedDynamicModule) needsLoadCheck() bool { diff --git a/multitree/Android.bp b/multitree/Android.bp new file mode 100644 index 000000000..9b16d2021 --- /dev/null +++ b/multitree/Android.bp @@ -0,0 +1,19 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "soong-multitree", + pkgPath: "android/soong/multitree", + deps: [ + "blueprint", + "soong-android", + ], + srcs: [ + "api_surface.go", + "export.go", + "metadata.go", + "import.go", + ], + pluginFor: ["soong_build"], +} diff --git a/multitree/api_surface.go b/multitree/api_surface.go new file mode 100644 index 000000000..f739a2430 --- /dev/null +++ b/multitree/api_surface.go @@ -0,0 +1,119 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "fmt" + + "github.com/google/blueprint" +) + +var ( + pctx = android.NewPackageContext("android/soong/multitree") +) + +func init() { + RegisterApiSurfaceBuildComponents(android.InitRegistrationContext) +} + +var PrepareForTestWithApiSurface = android.FixtureRegisterWithContext(RegisterApiSurfaceBuildComponents) + +func RegisterApiSurfaceBuildComponents(ctx android.RegistrationContext) { + ctx.RegisterModuleType("api_surface", ApiSurfaceFactory) +} + +type ApiSurface struct { + android.ModuleBase + ExportableModuleBase + properties apiSurfaceProperties + + allOutputs android.Paths + taggedOutputs map[string]android.Paths +} + +type apiSurfaceProperties struct { + Contributions []string +} + +func ApiSurfaceFactory() android.Module { + module := &ApiSurface{} + module.AddProperties(&module.properties) + android.InitAndroidModule(module) + InitExportableModule(module) + return module +} + +func (surface *ApiSurface) DepsMutator(ctx android.BottomUpMutatorContext) { + if surface.properties.Contributions != nil { + ctx.AddVariationDependencies(nil, nil, surface.properties.Contributions...) + } + +} +func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext) { + contributionFiles := make(map[string]android.Paths) + var allOutputs android.Paths + ctx.WalkDeps(func(child, parent android.Module) bool { + if contribution, ok := child.(ApiContribution); ok { + copied := contribution.CopyFilesWithTag(ctx) + for tag, files := range copied { + contributionFiles[child.Name()+"#"+tag] = files + } + for _, paths := range copied { + allOutputs = append(allOutputs, paths...) + } + return false // no transitive dependencies + } + return false + }) + + // phony target + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, ctx.ModuleName()), + Inputs: allOutputs, + }) + + surface.allOutputs = allOutputs + surface.taggedOutputs = contributionFiles +} + +func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) { + if tag != "" { + return nil, fmt.Errorf("unknown tag: %q", tag) + } + return surface.allOutputs, nil +} + +func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths { + return surface.taggedOutputs +} + +func (surface *ApiSurface) Exportable() bool { + return true +} + +var _ android.OutputFileProducer = (*ApiSurface)(nil) +var _ Exportable = (*ApiSurface)(nil) + +type ApiContribution interface { + // copy files necessaryt to construct an API surface + // For C, it will be map.txt and .h files + // For Java, it will be api.txt + CopyFilesWithTag(ctx android.ModuleContext) map[string]android.Paths // output paths + + // Generate Android.bp in out/ to use the exported .txt files + // GenerateBuildFiles(ctx ModuleContext) Paths //output paths +} diff --git a/multitree/export.go b/multitree/export.go new file mode 100644 index 000000000..aecade58d --- /dev/null +++ b/multitree/export.go @@ -0,0 +1,67 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + + "github.com/google/blueprint/proptools" +) + +type moduleExportProperty struct { + // True if the module is exported to the other components in a multi-tree. + // Any components in the multi-tree can import this module to use. + Export *bool +} + +type ExportableModuleBase struct { + properties moduleExportProperty +} + +type Exportable interface { + // Properties for the exporable module. + exportableModuleProps() *moduleExportProperty + + // Check if this module can be exported. + // If this returns false, the module will not be exported regardless of the 'export' value. + Exportable() bool + + // Returns 'true' if this module has 'export: true' + // This module will not be exported if it returns 'false' to 'Exportable()' interface even if + // it has 'export: true'. + IsExported() bool + + // Map from tags to outputs. + // Each module can tag their outputs for convenience. + TaggedOutputs() map[string]android.Paths +} + +type ExportableModule interface { + android.Module + android.OutputFileProducer + Exportable +} + +func InitExportableModule(module ExportableModule) { + module.AddProperties(module.exportableModuleProps()) +} + +func (m *ExportableModuleBase) exportableModuleProps() *moduleExportProperty { + return &m.properties +} + +func (m *ExportableModuleBase) IsExported() bool { + return proptools.Bool(m.properties.Export) +} diff --git a/multitree/import.go b/multitree/import.go new file mode 100644 index 000000000..1e5c421bc --- /dev/null +++ b/multitree/import.go @@ -0,0 +1,96 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" +) + +var ( + nameSuffix = ".imported" +) + +type MultitreeImportedModuleInterface interface { + GetMultitreeImportedModuleName() string +} + +func init() { + android.RegisterModuleType("imported_filegroup", importedFileGroupFactory) + + android.PreArchMutators(RegisterMultitreePreArchMutators) +} + +type importedFileGroupProperties struct { + // Imported modules from the other components in a multi-tree + Imported []string +} + +type importedFileGroup struct { + android.ModuleBase + + properties importedFileGroupProperties + srcs android.Paths +} + +func (ifg *importedFileGroup) Name() string { + return ifg.BaseModuleName() + nameSuffix +} + +func importedFileGroupFactory() android.Module { + module := &importedFileGroup{} + module.AddProperties(&module.properties) + + android.InitAndroidModule(module) + return module +} + +var _ MultitreeImportedModuleInterface = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) GetMultitreeImportedModuleName() string { + // The base module name of the imported filegroup is used as the imported module name + return ifg.BaseModuleName() +} + +var _ android.SourceFileProducer = (*importedFileGroup)(nil) + +func (ifg *importedFileGroup) Srcs() android.Paths { + return ifg.srcs +} + +func (ifg *importedFileGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // srcs from this module must not be used. Adding a dot path to avoid the empty + // source failure. Still soong returns error when a module wants to build against + // this source, which is intended. + ifg.srcs = android.PathsForModuleSrc(ctx, []string{"."}) +} + +func RegisterMultitreePreArchMutators(ctx android.RegisterMutatorsContext) { + ctx.BottomUp("multitree_imported_rename", MultitreeImportedRenameMutator).Parallel() +} + +func MultitreeImportedRenameMutator(ctx android.BottomUpMutatorContext) { + if m, ok := ctx.Module().(MultitreeImportedModuleInterface); ok { + name := m.GetMultitreeImportedModuleName() + if !ctx.OtherModuleExists(name) { + // Provide an empty filegroup not to break the build while updating the metadata. + // In other cases, soong will report an error to guide users to run 'm update-meta' + // first. + if !ctx.Config().TargetMultitreeUpdateMeta() { + ctx.ModuleErrorf("\"%s\" filegroup must be imported.\nRun 'm update-meta' first to import the filegroup.", name) + } + ctx.Rename(name) + } + } +} diff --git a/multitree/metadata.go b/multitree/metadata.go new file mode 100644 index 000000000..3fd721599 --- /dev/null +++ b/multitree/metadata.go @@ -0,0 +1,74 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multitree + +import ( + "android/soong/android" + "encoding/json" +) + +func init() { + android.RegisterSingletonType("update-meta", UpdateMetaSingleton) +} + +func UpdateMetaSingleton() android.Singleton { + return &updateMetaSingleton{} +} + +type jsonImported struct { + FileGroups map[string][]string `json:",omitempty"` +} + +type metadataJsonFlags struct { + Imported jsonImported `json:",omitempty"` + Exported map[string][]string `json:",omitempty"` +} + +type updateMetaSingleton struct { + importedModules []string + generatedMetadataFile android.OutputPath +} + +func (s *updateMetaSingleton) GenerateBuildActions(ctx android.SingletonContext) { + metadata := metadataJsonFlags{ + Imported: jsonImported{ + FileGroups: make(map[string][]string), + }, + Exported: make(map[string][]string), + } + ctx.VisitAllModules(func(module android.Module) { + if ifg, ok := module.(*importedFileGroup); ok { + metadata.Imported.FileGroups[ifg.BaseModuleName()] = ifg.properties.Imported + } + if e, ok := module.(ExportableModule); ok { + if e.IsExported() && e.Exportable() { + for tag, files := range e.TaggedOutputs() { + // TODO(b/219846705): refactor this to a dictionary + metadata.Exported[e.Name()+":"+tag] = append(metadata.Exported[e.Name()+":"+tag], files.Strings()...) + } + } + } + }) + jsonStr, err := json.Marshal(metadata) + if err != nil { + ctx.Errorf(err.Error()) + } + s.generatedMetadataFile = android.PathForOutput(ctx, "multitree", "metadata.json") + android.WriteFileRule(ctx, s.generatedMetadataFile, string(jsonStr)) +} + +func (s *updateMetaSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("MULTITREE_METADATA", s.generatedMetadataFile.String()) +} diff --git a/provenance/provenance_singleton.go b/provenance/provenance_singleton.go index e49f3d4f3..fbb6212ff 100644 --- a/provenance/provenance_singleton.go +++ b/provenance/provenance_singleton.go @@ -35,9 +35,10 @@ var ( mergeProvenanceMetaData = pctx.AndroidStaticRule("mergeProvenanceMetaData", blueprint.RuleParams{ - Command: `rm -rf $out $out.temp && ` + - `echo -e "# proto-file: build/soong/provenance/proto/provenance_metadata.proto\n# proto-message: ProvenanceMetaDataList" > $out && ` + - `touch $out.temp && cat $out.temp $in | grep -v "^#.*" >> $out && rm -rf $out.temp`, + Command: `rm -rf $out && ` + + `echo "# proto-file: build/soong/provenance/proto/provenance_metadata.proto" > $out && ` + + `echo "# proto-message: ProvenanceMetaDataList" >> $out && ` + + `for file in $in; do echo '' >> $out; echo 'metadata {' | cat - $$file | grep -Ev "^#.*|^$$" >> $out; echo '}' >> $out; done`, }) ) diff --git a/provenance/tools/gen_provenance_metadata.py b/provenance/tools/gen_provenance_metadata.py index b33f9112b..f3f4d1f73 100644 --- a/provenance/tools/gen_provenance_metadata.py +++ b/provenance/tools/gen_provenance_metadata.py @@ -16,6 +16,7 @@ import argparse import hashlib +import os.path import sys import google.protobuf.text_format as text_format @@ -51,6 +52,11 @@ def main(argv): h.update(artifact_file.read()) provenance_metadata.artifact_sha256 = h.hexdigest() + Log("Check if there is attestation for the artifact") + attestation_file_name = args.artifact_path + ".intoto.jsonl" + if os.path.isfile(attestation_file_name): + provenance_metadata.attestation_path = attestation_file_name + text_proto = [ "# proto-file: build/soong/provenance/proto/provenance_metadata.proto", "# proto-message: ProvenanceMetaData", diff --git a/provenance/tools/gen_provenance_metadata_test.py b/provenance/tools/gen_provenance_metadata_test.py index 2fc04bf12..1f69b8f4b 100644 --- a/provenance/tools/gen_provenance_metadata_test.py +++ b/provenance/tools/gen_provenance_metadata_test.py @@ -100,6 +100,11 @@ class ProvenanceMetaDataToolTest(unittest.TestCase): artifact_file = tempfile.mktemp() with open(artifact_file,"wt") as f: f.write(artifact_content) + + attestation_file = artifact_file + ".intoto.jsonl" + with open(attestation_file, "wt") as af: + af.write("attestation file") + metadata_file = tempfile.mktemp() cmd = ["gen_provenance_metadata"] cmd.extend(["--module_name", "a"]) @@ -117,9 +122,11 @@ class ProvenanceMetaDataToolTest(unittest.TestCase): self.assertEqual(provenance_metadata.artifact_path, artifact_file) self.assertEqual(provenance_metadata.artifact_install_path, "b") self.assertEqual(provenance_metadata.artifact_sha256, sha256(artifact_content)) + self.assertEqual(provenance_metadata.attestation_path, attestation_file) os.remove(artifact_file) os.remove(metadata_file) + os.remove(attestation_file) if __name__ == '__main__': unittest.main(verbosity=2)
\ No newline at end of file diff --git a/python/binary.go b/python/binary.go index 99c625916..af29bb6b8 100644 --- a/python/binary.go +++ b/python/binary.go @@ -38,6 +38,7 @@ type bazelPythonBinaryAttributes struct { Srcs bazel.LabelListAttribute Deps bazel.LabelListAttribute Python_version *string + Imports bazel.StringListAttribute } func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *Module) { @@ -75,6 +76,7 @@ func pythonBinaryBp2Build(ctx android.TopDownMutatorContext, m *Module) { Srcs: baseAttrs.Srcs, Deps: baseAttrs.Deps, Python_version: python_version, + Imports: baseAttrs.Imports, } props := bazel.BazelTargetModuleProperties{ diff --git a/python/library.go b/python/library.go index d026c1323..df92df42b 100644 --- a/python/library.go +++ b/python/library.go @@ -43,9 +43,14 @@ func PythonLibraryHostFactory() android.Module { type bazelPythonLibraryAttributes struct { Srcs bazel.LabelListAttribute Deps bazel.LabelListAttribute + Imports bazel.StringListAttribute Srcs_version *string } +type bazelPythonProtoLibraryAttributes struct { + Deps bazel.LabelListAttribute +} + func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *Module) { // TODO(b/182306917): this doesn't fully handle all nested props versioned // by the python version, which would have been handled by the version split @@ -65,15 +70,17 @@ func pythonLibBp2Build(ctx android.TopDownMutatorContext, m *Module) { } baseAttrs := m.makeArchVariantBaseAttributes(ctx) + attrs := &bazelPythonLibraryAttributes{ Srcs: baseAttrs.Srcs, Deps: baseAttrs.Deps, Srcs_version: python_version, + Imports: baseAttrs.Imports, } props := bazel.BazelTargetModuleProperties{ - Rule_class: "py_library", - Bzl_load_location: "//build/bazel/rules/python:library.bzl", + // Use the native py_library rule. + Rule_class: "py_library", } ctx.CreateBazelTargetModule(props, android.CommonAttributes{ diff --git a/python/python.go b/python/python.go index b100cc318..eb0d3cad0 100644 --- a/python/python.go +++ b/python/python.go @@ -131,7 +131,8 @@ type baseAttributes struct { Srcs bazel.LabelListAttribute Deps bazel.LabelListAttribute // Combines Data and Java_data (invariant) - Data bazel.LabelListAttribute + Data bazel.LabelListAttribute + Imports bazel.StringListAttribute } // Used to store files of current module after expanding dependencies @@ -207,6 +208,56 @@ func (m *Module) makeArchVariantBaseAttributes(ctx android.TopDownMutatorContext } } } + + partitionedSrcs := bazel.PartitionLabelListAttribute(ctx, &attrs.Srcs, bazel.LabelPartitions{ + "proto": android.ProtoSrcLabelPartition, + "py": bazel.LabelPartition{Keep_remainder: true}, + }) + attrs.Srcs = partitionedSrcs["py"] + + if !partitionedSrcs["proto"].IsEmpty() { + protoInfo, _ := android.Bp2buildProtoProperties(ctx, &m.ModuleBase, partitionedSrcs["proto"]) + protoLabel := bazel.Label{Label: ":" + protoInfo.Name} + + pyProtoLibraryName := m.Name() + "_py_proto" + ctx.CreateBazelTargetModule(bazel.BazelTargetModuleProperties{ + Rule_class: "py_proto_library", + Bzl_load_location: "//build/bazel/rules/python:py_proto.bzl", + }, android.CommonAttributes{ + Name: pyProtoLibraryName, + }, &bazelPythonProtoLibraryAttributes{ + Deps: bazel.MakeSingleLabelListAttribute(protoLabel), + }) + + attrs.Deps.Add(bazel.MakeLabelAttribute(":" + pyProtoLibraryName)) + } + + // Bazel normally requires `import path.from.top.of.tree` statements in + // python code, but with soong you can directly import modules from libraries. + // Add "imports" attributes to the bazel library so it matches soong's behavior. + imports := "." + if m.properties.Pkg_path != nil { + // TODO(b/215119317) This is a hack to handle the fact that we don't convert + // pkg_path properly right now. If the folder structure that contains this + // Android.bp file matches pkg_path, we can set imports to an appropriate + // number of ../..s to emulate moving the files under a pkg_path folder. + pkg_path := filepath.Clean(*m.properties.Pkg_path) + if strings.HasPrefix(pkg_path, "/") { + ctx.ModuleErrorf("pkg_path cannot start with a /: %s", pkg_path) + } + + if !strings.HasSuffix(ctx.ModuleDir(), "/"+pkg_path) && ctx.ModuleDir() != pkg_path { + ctx.ModuleErrorf("Currently, bp2build only supports pkg_paths that are the same as the folders the Android.bp file is in. pkg_path: %s, module directory: %s", pkg_path, ctx.ModuleDir()) + } + numFolders := strings.Count(pkg_path, "/") + 1 + dots := make([]string, numFolders) + for i := 0; i < numFolders; i++ { + dots[i] = ".." + } + imports = strings.Join(dots, "/") + } + attrs.Imports = bazel.MakeStringListAttribute([]string{imports}) + return attrs } @@ -631,7 +682,8 @@ func (p *Module) createSrcsZip(ctx android.ModuleContext, pkgPath string) androi // in order to keep stable order of soong_zip params, we sort the keys here. roots := android.SortedStringKeys(relativeRootMap) - parArgs := []string{} + // Use -symlinks=false so that the symlinks in the bazel output directory are followed + parArgs := []string{"-symlinks=false"} if pkgPath != "" { // use package path as path prefix parArgs = append(parArgs, `-P `+pkgPath) diff --git a/rust/binary.go b/rust/binary.go index 0dc320e5f..41110f92f 100644 --- a/rust/binary.go +++ b/rust/binary.go @@ -128,11 +128,11 @@ func (binary *binaryDecorator) preferRlib() bool { return Bool(binary.baseCompiler.Properties.Prefer_rlib) || Bool(binary.Properties.Static_executable) } -func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix() srcPath, _ := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs) outputFile := android.PathForModuleOut(ctx, fileName) - ret := outputFile + ret := buildOutput{outputFile: outputFile} flags.RustFlags = append(flags.RustFlags, deps.depFlags...) flags.LinkFlags = append(flags.LinkFlags, deps.depLinkFlags...) @@ -147,8 +147,7 @@ func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps Path } binary.baseCompiler.unstrippedOutputFile = outputFile - TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile) - + ret.kytheFile = TransformSrcToBinary(ctx, srcPath, deps, flags, outputFile).kytheFile return ret } diff --git a/rust/bindgen.go b/rust/bindgen.go index c2b05129a..4d723d6a6 100644 --- a/rust/bindgen.go +++ b/rust/bindgen.go @@ -30,7 +30,7 @@ var ( defaultBindgenFlags = []string{""} // bindgen should specify its own Clang revision so updating Clang isn't potentially blocked on bindgen failures. - bindgenClangVersion = "clang-r445002" + bindgenClangVersion = "clang-r450784d" _ = pctx.VariableFunc("bindgenClangVersion", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("LLVM_BINDGEN_PREBUILTS_VERSION"); override != "" { @@ -41,10 +41,25 @@ var ( //TODO(b/160803703) Use a prebuilt bindgen instead of the built bindgen. _ = pctx.HostBinToolVariable("bindgenCmd", "bindgen") + _ = pctx.VariableFunc("bindgenHostPrebuiltTag", func(ctx android.PackageVarContext) string { + if ctx.Config().UseHostMusl() { + // This is a hack to use the glibc bindgen binary until we have a musl version checked in. + return "linux-x86" + } else { + return "${config.HostPrebuiltTag}" + } + }) + _ = pctx.VariableFunc("bindgenClangLibdir", func(ctx android.PackageVarContext) string { + if ctx.Config().UseHostMusl() { + return "musl/lib64/" + } else { + return "lib64/" + } + }) _ = pctx.SourcePathVariable("bindgenClang", - "${cc_config.ClangBase}/${config.HostPrebuiltTag}/${bindgenClangVersion}/bin/clang") + "${cc_config.ClangBase}/${bindgenHostPrebuiltTag}/${bindgenClangVersion}/bin/clang") _ = pctx.SourcePathVariable("bindgenLibClang", - "${cc_config.ClangBase}/${config.HostPrebuiltTag}/${bindgenClangVersion}/lib64/") + "${cc_config.ClangBase}/${bindgenHostPrebuiltTag}/${bindgenClangVersion}/${bindgenClangLibdir}") //TODO(ivanlozano) Switch this to RuleBuilder bindgen = pctx.AndroidStaticRule("bindgen", diff --git a/rust/builder.go b/rust/builder.go index 20ca5dbee..7dd9dd276 100644 --- a/rust/builder.go +++ b/rust/builder.go @@ -83,10 +83,37 @@ var ( RspfileContent: "$in", }, "outDir") + + // Cross-referencing: + _ = pctx.SourcePathVariable("rustExtractor", + "prebuilts/build-tools/${config.HostPrebuiltTag}/bin/rust_extractor") + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + _ = pctx.VariableFunc("kytheCuEncoding", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCuEncoding() }) + _ = pctx.SourcePathVariable("kytheVnames", "build/soong/vnames.json") + kytheExtract = pctx.AndroidStaticRule("kythe", + blueprint.RuleParams{ + Command: `KYTHE_CORPUS=${kytheCorpus} ` + + `KYTHE_OUTPUT_FILE=$out ` + + `KYTHE_VNAMES=$kytheVnames ` + + `KYTHE_KZIP_ENCODING=${kytheCuEncoding} ` + + `KYTHE_CANONICALIZE_VNAME_PATHS=prefer-relative ` + + `$rustExtractor $envVars ` + + `$rustcCmd ` + + `-C linker=${config.RustLinker} ` + + `-C link-args="${crtBegin} ${config.RustLinkerArgs} ${linkFlags} ${crtEnd}" ` + + `$in ${libFlags} $rustcFlags`, + CommandDeps: []string{"$rustExtractor", "$kytheVnames"}, + Rspfile: "${out}.rsp", + RspfileContent: "$in", + }, + "rustcFlags", "linkFlags", "libFlags", "crtBegin", "crtEnd", "envVars") ) type buildOutput struct { outputFile android.Path + kytheFile android.Path } func init() { @@ -324,6 +351,25 @@ func transformSrctoCrate(ctx ModuleContext, main android.Path, deps PathDeps, fl }, }) + if flags.EmitXrefs { + kytheFile := android.PathForModuleOut(ctx, outputFile.Base()+".kzip") + ctx.Build(pctx, android.BuildParams{ + Rule: kytheExtract, + Description: "Xref Rust extractor " + main.Rel(), + Output: kytheFile, + Inputs: inputs, + Implicits: implicits, + Args: map[string]string{ + "rustcFlags": strings.Join(rustcFlags, " "), + "linkFlags": strings.Join(linkFlags, " "), + "libFlags": strings.Join(libFlags, " "), + "crtBegin": strings.Join(deps.CrtBegin.Strings(), " "), + "crtEnd": strings.Join(deps.CrtEnd.Strings(), " "), + "envVars": strings.Join(envVars, " "), + }, + }) + output.kytheFile = kytheFile + } return output } diff --git a/rust/compiler.go b/rust/compiler.go index 19499fa36..bf6a48844 100644 --- a/rust/compiler.go +++ b/rust/compiler.go @@ -304,6 +304,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag flags.GlobalRustFlags = append(flags.GlobalRustFlags, config.GlobalRustFlags...) flags.GlobalRustFlags = append(flags.GlobalRustFlags, ctx.toolchain().ToolchainRustFlags()) flags.GlobalLinkFlags = append(flags.GlobalLinkFlags, ctx.toolchain().ToolchainLinkFlags()) + flags.EmitXrefs = ctx.Config().EmitXrefRules() if ctx.Host() && !ctx.Windows() { rpathPrefix := `\$$ORIGIN/` @@ -324,7 +325,7 @@ func (compiler *baseCompiler) compilerFlags(ctx ModuleContext, flags Flags) Flag return flags } -func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (compiler *baseCompiler) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { panic(fmt.Errorf("baseCrater doesn't know how to crate things!")) } @@ -370,8 +371,9 @@ func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps { if !Bool(compiler.Properties.No_stdlibs) { for _, stdlib := range config.Stdlibs { - // If we're building for the build host, use the prebuilt stdlibs - if ctx.Target().Os == android.Linux || ctx.Target().Os == android.Darwin { + // If we're building for the build host, use the prebuilt stdlibs, unless the host + // is linux_bionic which doesn't have prebuilts. + if ctx.Host() && !ctx.Target().HostCross && ctx.Target().Os != android.LinuxBionic { stdlib = "prebuilt_" + stdlib } deps.Stdlibs = append(deps.Stdlibs, stdlib) diff --git a/rust/config/Android.bp b/rust/config/Android.bp index 7757c79fc..ba40cb0a6 100644 --- a/rust/config/Android.bp +++ b/rust/config/Android.bp @@ -11,6 +11,7 @@ bootstrap_go_package { ], srcs: [ "arm_device.go", + "arm_linux_host.go", "arm64_device.go", "global.go", "lints.go", diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go index 802e1da7f..9129b0e10 100644 --- a/rust/config/allowed_list.go +++ b/rust/config/allowed_list.go @@ -8,6 +8,7 @@ var ( RustAllowedPaths = []string{ "device/google/cuttlefish", "external/adhd", + "external/boringssl", "external/crosvm", "external/libchromeos-rs", "external/minijail", @@ -29,9 +30,11 @@ var ( "system/core/debuggerd/rust", "system/core/libstats/pull_rust", "system/core/trusty/libtrusty-rs", + "system/core/trusty/keymint", "system/extras/profcollectd", "system/extras/simpleperf", "system/hardware/interfaces/keystore2", + "system/keymint", "system/librustutils", "system/logging/liblog", "system/logging/rust", diff --git a/rust/config/arm_linux_host.go b/rust/config/arm_linux_host.go new file mode 100644 index 000000000..22bdaee3e --- /dev/null +++ b/rust/config/arm_linux_host.go @@ -0,0 +1,147 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + + "android/soong/android" +) + +var ( + linuxArmRustflags = []string{} + linuxArmLinkflags = []string{} + linuxArm64Rustflags = []string{} + linuxArm64Linkflags = []string{} +) + +func init() { + registerToolchainFactory(android.LinuxMusl, android.Arm64, linuxMuslArm64ToolchainFactory) + registerToolchainFactory(android.LinuxMusl, android.Arm, linuxMuslArmToolchainFactory) + + pctx.StaticVariable("LinuxToolchainArmRustFlags", strings.Join(linuxArmRustflags, " ")) + pctx.StaticVariable("LinuxToolchainArmLinkFlags", strings.Join(linuxArmLinkflags, " ")) + pctx.StaticVariable("LinuxToolchainArm64RustFlags", strings.Join(linuxArm64Rustflags, " ")) + pctx.StaticVariable("LinuxToolchainArm64LinkFlags", strings.Join(linuxArm64Linkflags, " ")) +} + +// Base 64-bit linux rust toolchain +type toolchainLinuxArm64 struct { + toolchain64Bit +} + +func (toolchainLinuxArm64) Supported() bool { + return true +} + +func (toolchainLinuxArm64) Bionic() bool { + return false +} + +func (t *toolchainLinuxArm64) Name() string { + return "arm64" +} + +func (t *toolchainLinuxArm64) ToolchainLinkFlags() string { + // Prepend the lld flags from cc_config so we stay in sync with cc + return "${cc_config.LinuxLldflags} ${cc_config.LinuxArm64Lldflags} " + + "${config.LinuxToolchainLinkFlags} ${config.LinuxToolchainArm64LinkFlags}" +} + +func (t *toolchainLinuxArm64) ToolchainRustFlags() string { + return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainArm64RustFlags}" +} + +// Specialization of the 64-bit linux rust toolchain for musl. Adds the musl rust triple and +// linker flags to avoid using the host sysroot. +type toolchainLinuxMuslArm64 struct { + toolchainLinuxArm64 +} + +func (t *toolchainLinuxMuslArm64) RustTriple() string { + return "aarch64-unknown-linux-musl" +} + +func (t *toolchainLinuxMuslArm64) ToolchainLinkFlags() string { + return t.toolchainLinuxArm64.ToolchainLinkFlags() + " " + "${config.LinuxMuslToolchainLinkFlags}" +} + +func (t *toolchainLinuxMuslArm64) ToolchainRustFlags() string { + return t.toolchainLinuxArm64.ToolchainRustFlags() + " " + "${config.LinuxMuslToolchainRustFlags}" +} + +func linuxMuslArm64ToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslArm64Singleton +} + +// Base 32-bit linux rust toolchain +type toolchainLinuxArm struct { + toolchain32Bit +} + +func (toolchainLinuxArm) Supported() bool { + return true +} + +func (toolchainLinuxArm) Bionic() bool { + return false +} + +func (t *toolchainLinuxArm) Name() string { + return "arm" +} + +func (toolchainLinuxArm) LibclangRuntimeLibraryArch() string { + return "arm" +} + +func (toolchainLinuxArm64) LibclangRuntimeLibraryArch() string { + return "arm64" +} + +func (t *toolchainLinuxArm) ToolchainLinkFlags() string { + // Prepend the lld flags from cc_config so we stay in sync with cc + return "${cc_config.LinuxLldflags} ${cc_config.LinuxArmLldflags} " + + "${config.LinuxToolchainLinkFlags} ${config.LinuxToolchainArmLinkFlags}" +} + +func (t *toolchainLinuxArm) ToolchainRustFlags() string { + return "${config.LinuxToolchainRustFlags} ${config.LinuxToolchainArmRustFlags}" +} + +// Specialization of the 32-bit linux rust toolchain for musl. Adds the musl rust triple and +// linker flags to avoid using the host sysroot. +type toolchainLinuxMuslArm struct { + toolchainLinuxArm +} + +func (t *toolchainLinuxMuslArm) RustTriple() string { + return "arm-unknown-linux-musleabihf" +} + +func (t *toolchainLinuxMuslArm) ToolchainLinkFlags() string { + return t.toolchainLinuxArm.ToolchainLinkFlags() + " " + "${config.LinuxMuslToolchainLinkFlags}" +} + +func (t *toolchainLinuxMuslArm) ToolchainRustFlags() string { + return t.toolchainLinuxArm.ToolchainRustFlags() + " " + "${config.LinuxMuslToolchainRustFlags}" +} + +func linuxMuslArmToolchainFactory(arch android.Arch) Toolchain { + return toolchainLinuxMuslArmSingleton +} + +var toolchainLinuxMuslArm64Singleton Toolchain = &toolchainLinuxMuslArm64{} +var toolchainLinuxMuslArmSingleton Toolchain = &toolchainLinuxMuslArm{} diff --git a/rust/config/global.go b/rust/config/global.go index 6bfa9cf94..3ef0ecb18 100644 --- a/rust/config/global.go +++ b/rust/config/global.go @@ -24,7 +24,7 @@ import ( var pctx = android.NewPackageContext("android/soong/rust/config") var ( - RustDefaultVersion = "1.60.0" + RustDefaultVersion = "1.61.0.p2" RustDefaultBase = "prebuilts/rust/" DefaultEdition = "2021" Stdlibs = []string{ @@ -50,6 +50,7 @@ var ( "-C force-unwind-tables=yes", // Use v0 mangling to distinguish from C++ symbols "-C symbol-mangling-version=v0", + "--color always", } deviceGlobalRustFlags = []string{ @@ -77,7 +78,13 @@ var ( func init() { pctx.SourcePathVariable("RustDefaultBase", RustDefaultBase) - pctx.VariableConfigMethod("HostPrebuiltTag", android.Config.PrebuiltOS) + pctx.VariableConfigMethod("HostPrebuiltTag", func(config android.Config) string { + if config.UseHostMusl() { + return "linux-musl-x86" + } else { + return config.PrebuiltOS() + } + }) pctx.VariableFunc("RustBase", func(ctx android.PackageVarContext) string { if override := ctx.Config().Getenv("RUST_PREBUILTS_BASE"); override != "" { diff --git a/rust/coverage.go b/rust/coverage.go index 651ce6e16..5ea481fcf 100644 --- a/rust/coverage.go +++ b/rust/coverage.go @@ -56,7 +56,7 @@ func (cov *coverage) flags(ctx ModuleContext, flags Flags, deps PathDeps) (Flags flags.Coverage = true coverage := ctx.GetDirectDepWithTag(CovLibraryName, cc.CoverageDepTag).(cc.LinkableInterface) flags.RustFlags = append(flags.RustFlags, - "-Z instrument-coverage", "-g") + "-C instrument-coverage", "-g") flags.LinkFlags = append(flags.LinkFlags, profileInstrFlag, "-g", coverage.OutputFile().Path().String(), "-Wl,--wrap,open") deps.StaticLibs = append(deps.StaticLibs, coverage.OutputFile().Path()) diff --git a/rust/coverage_test.go b/rust/coverage_test.go index f3cd3758c..0f599d745 100644 --- a/rust/coverage_test.go +++ b/rust/coverage_test.go @@ -56,7 +56,7 @@ func TestCoverageFlags(t *testing.T) { fizzCov := ctx.ModuleForTests("fizz_cov", "android_arm64_armv8-a_cov").Rule("rustc") buzzNoCov := ctx.ModuleForTests("buzzNoCov", "android_arm64_armv8-a").Rule("rustc") - rustcCoverageFlags := []string{"-Z instrument-coverage", " -g "} + rustcCoverageFlags := []string{"-C instrument-coverage", " -g "} for _, flag := range rustcCoverageFlags { missingErrorStr := "missing rustc flag '%s' for '%s' module with coverage enabled; rustcFlags: %#v" containsErrorStr := "contains rustc flag '%s' for '%s' module with coverage disabled; rustcFlags: %#v" diff --git a/rust/library.go b/rust/library.go index 62eaefd68..c2ce9de6b 100644 --- a/rust/library.go +++ b/rust/library.go @@ -246,10 +246,6 @@ func (library *libraryDecorator) autoDep(ctx android.BottomUpMutatorContext) aut return rlibAutoDep } else if library.dylib() || library.shared() { return dylibAutoDep - } else if ctx.BazelConversionMode() { - // In Bazel conversion mode, we are currently ignoring the deptag, so we just need to supply a - // compatible tag in order to add the dependency. - return rlibAutoDep } else { panic(fmt.Errorf("autoDep called on library %q that has no enabled variants.", ctx.ModuleName())) } @@ -474,8 +470,9 @@ func (library *libraryDecorator) compilerFlags(ctx ModuleContext, flags Flags) F return flags } -func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { - var outputFile, ret android.ModuleOutPath +func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { + var outputFile android.ModuleOutPath + var ret buildOutput var fileName string srcPath := library.srcPath(ctx, deps) @@ -487,19 +484,19 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa if library.rlib() { fileName = library.getStem(ctx) + ctx.toolchain().RlibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.dylib() { fileName = library.getStem(ctx) + ctx.toolchain().DylibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.static() { fileName = library.getStem(ctx) + ctx.toolchain().StaticLibSuffix() outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } else if library.shared() { fileName = library.sharedLibFilename(ctx) outputFile = android.PathForModuleOut(ctx, fileName) - ret = outputFile + ret.outputFile = outputFile } if !library.rlib() && !library.static() && library.stripper.NeedsStrip(ctx) { @@ -524,13 +521,13 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa // Call the appropriate builder for this library type if library.rlib() { - TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoRlib(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.dylib() { - TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoDylib(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.static() { - TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoStatic(ctx, srcPath, deps, flags, outputFile).kytheFile } else if library.shared() { - TransformSrctoShared(ctx, srcPath, deps, flags, outputFile) + ret.kytheFile = TransformSrctoShared(ctx, srcPath, deps, flags, outputFile).kytheFile } if library.rlib() || library.dylib() { @@ -572,7 +569,7 @@ func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps Pa return ret } -func (library *libraryDecorator) srcPath(ctx ModuleContext, deps PathDeps) android.Path { +func (library *libraryDecorator) srcPath(ctx ModuleContext, _ PathDeps) android.Path { if library.sourceProvider != nil { // Assume the first source from the source provider is the library entry point. return library.sourceProvider.Srcs()[0] diff --git a/rust/prebuilt.go b/rust/prebuilt.go index 6cdd07de9..fe9d0b5dd 100644 --- a/rust/prebuilt.go +++ b/rust/prebuilt.go @@ -145,7 +145,7 @@ func (prebuilt *prebuiltLibraryDecorator) compilerProps() []interface{} { &prebuilt.Properties) } -func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...) prebuilt.flagExporter.setProvider(ctx) @@ -154,7 +154,7 @@ func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)") } prebuilt.baseCompiler.unstrippedOutputFile = srcPath - return srcPath + return buildOutput{outputFile: srcPath} } func (prebuilt *prebuiltLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, @@ -202,7 +202,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compilerProps() []interface{} { &prebuilt.Properties) } -func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { prebuilt.flagExporter.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...) prebuilt.flagExporter.setProvider(ctx) @@ -211,7 +211,7 @@ func (prebuilt *prebuiltProcMacroDecorator) compile(ctx ModuleContext, flags Fla ctx.PropertyErrorf("srcs", "prebuilt libraries can only have one entry in srcs (the prebuilt path)") } prebuilt.baseCompiler.unstrippedOutputFile = srcPath - return srcPath + return buildOutput{outputFile: srcPath} } func (prebuilt *prebuiltProcMacroDecorator) rustdoc(ctx ModuleContext, flags Flags, diff --git a/rust/proc_macro.go b/rust/proc_macro.go index f8a4bbded..832b62c36 100644 --- a/rust/proc_macro.go +++ b/rust/proc_macro.go @@ -70,14 +70,14 @@ func (procMacro *procMacroDecorator) compilerFlags(ctx ModuleContext, flags Flag return flags } -func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (procMacro *procMacroDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { fileName := procMacro.getStem(ctx) + ctx.toolchain().ProcMacroSuffix() outputFile := android.PathForModuleOut(ctx, fileName) srcPath, _ := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs) - TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile) + ret := TransformSrctoProcMacro(ctx, srcPath, deps, flags, outputFile) procMacro.baseCompiler.unstrippedOutputFile = outputFile - return outputFile + return ret } func (procMacro *procMacroDecorator) getStem(ctx ModuleContext) string { diff --git a/rust/rust.go b/rust/rust.go index c4fd14859..d5d492971 100644 --- a/rust/rust.go +++ b/rust/rust.go @@ -15,6 +15,7 @@ package rust import ( + "android/soong/bloaty" "fmt" "strings" @@ -22,7 +23,6 @@ import ( "github.com/google/blueprint/proptools" "android/soong/android" - "android/soong/bloaty" "android/soong/cc" cc_config "android/soong/cc/config" "android/soong/fuzz" @@ -52,6 +52,7 @@ func init() { }) pctx.Import("android/soong/rust/config") pctx.ImportAs("cc_config", "android/soong/cc/config") + android.InitRegistrationContext.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) } type Flags struct { @@ -64,6 +65,7 @@ type Flags struct { Toolchain config.Toolchain Coverage bool Clippy bool + EmitXrefs bool // If true, emit rules to aid cross-referencing } type BaseProperties struct { @@ -161,6 +163,9 @@ type Module struct { // Output file to be installed, may be stripped or unstripped. outputFile android.OptionalPath + // Cross-reference input file + kytheFiles android.Paths + docTimestampFile android.OptionalPath hideApexVariantFromMake bool @@ -394,6 +399,10 @@ func (mod *Module) SplitPerApiLevel() bool { return false } +func (mod *Module) XrefRustFiles() android.Paths { + return mod.kytheFiles +} + type Deps struct { Dylibs []string Rlibs []string @@ -457,7 +466,7 @@ type compiler interface { cfgFlags(ctx ModuleContext, flags Flags) Flags featureFlags(ctx ModuleContext, flags Flags) Flags compilerProps() []interface{} - compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path + compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput compilerDeps(ctx DepsContext, deps Deps) Deps crateName() string rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath @@ -493,6 +502,10 @@ type exportedFlagsProducer interface { exportLinkObjects(...string) } +type xref interface { + XrefRustFiles() android.Paths +} + type flagExporter struct { linkDirs []string linkObjects []string @@ -682,6 +695,19 @@ func (mod *Module) CoverageFiles() android.Paths { panic(fmt.Errorf("CoverageFiles called on non-library module: %q", mod.BaseModuleName())) } +// Rust does not produce gcno files, and therefore does not produce a coverage archive. +func (mod *Module) CoverageOutputFile() android.OptionalPath { + return android.OptionalPath{} +} + +func (mod *Module) IsNdk(config android.Config) bool { + return false +} + +func (mod *Module) IsStubs() bool { + return false +} + func (mod *Module) installable(apexInfo android.ApexInfo) bool { if !proptools.BoolDefault(mod.Installable(), mod.EverInstallable()) { return false @@ -904,11 +930,14 @@ func (mod *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if mod.compiler != nil && !mod.compiler.Disabled() { mod.compiler.initialize(ctx) - outputFile := mod.compiler.compile(ctx, flags, deps) + buildOutput := mod.compiler.compile(ctx, flags, deps) if ctx.Failed() { return } - mod.outputFile = android.OptionalPathForPath(outputFile) + mod.outputFile = android.OptionalPathForPath(buildOutput.outputFile) + if buildOutput.kytheFile != nil { + mod.kytheFiles = append(mod.kytheFiles, buildOutput.kytheFile) + } bloaty.MeasureSizeForPaths(ctx, mod.compiler.strippedOutputFilePath(), android.OptionalPathForPath(mod.compiler.unstrippedOutputFilePath())) mod.docTimestampFile = mod.compiler.rustdoc(ctx, flags, deps) @@ -1618,6 +1647,25 @@ func libNameFromFilePath(filepath android.Path) (string, bool) { return "", false } +func kytheExtractRustFactory() android.Singleton { + return &kytheExtractRustSingleton{} +} + +type kytheExtractRustSingleton struct { +} + +func (k kytheExtractRustSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if rustModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, rustModule.XrefRustFiles()...) + } + }) + if len(xrefTargets) > 0 { + ctx.Phony("xref_rust", xrefTargets...) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String diff --git a/rust/sanitize.go b/rust/sanitize.go index 39aaf33c2..a3c5cb583 100644 --- a/rust/sanitize.go +++ b/rust/sanitize.go @@ -50,7 +50,6 @@ type SanitizeProperties struct { } } SanitizerEnabled bool `blueprint:"mutated"` - SanitizeDep bool `blueprint:"mutated"` // Used when we need to place libraries in their own directory, such as ASAN. InSanitizerDir bool `blueprint:"mutated"` @@ -175,7 +174,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } // Enable Memtag for all components in the include paths (for Aarch64 only) - if ctx.Arch().ArchType == android.Arm64 { + if ctx.Arch().ArchType == android.Arm64 && ctx.Os().Bionic() { if ctx.Config().MemtagHeapSyncEnabledForPath(ctx.ModuleDir()) { if s.Memtag_heap == nil { s.Memtag_heap = proptools.BoolPtr(true) @@ -201,7 +200,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } // HWASan requires AArch64 hardware feature (top-byte-ignore). - if ctx.Arch().ArchType != android.Arm64 { + if ctx.Arch().ArchType != android.Arm64 || !ctx.Os().Bionic() { s.Hwaddress = nil } @@ -216,7 +215,7 @@ func (sanitize *sanitize) begin(ctx BaseModuleContext) { } // Memtag_heap is only implemented on AArch64. - if ctx.Arch().ArchType != android.Arm64 { + if ctx.Arch().ArchType != android.Arm64 || !ctx.Os().Bionic() { s.Memtag_heap = nil } @@ -235,7 +234,7 @@ func (sanitize *sanitize) flags(ctx ModuleContext, flags Flags, deps PathDeps) ( } if Bool(sanitize.Properties.Sanitize.Fuzzer) { flags.RustFlags = append(flags.RustFlags, fuzzerFlags...) - if ctx.Arch().ArchType == android.Arm64 { + if ctx.Arch().ArchType == android.Arm64 && ctx.Os().Bionic() { flags.RustFlags = append(flags.RustFlags, hwasanFlags...) } else { flags.RustFlags = append(flags.RustFlags, asanFlags...) @@ -283,13 +282,13 @@ func rustSanitizerRuntimeMutator(mctx android.BottomUpMutatorContext) { var deps []string if mod.IsSanitizerEnabled(cc.Asan) || - (mod.IsSanitizerEnabled(cc.Fuzzer) && mctx.Arch().ArchType != android.Arm64) { + (mod.IsSanitizerEnabled(cc.Fuzzer) && (mctx.Arch().ArchType != android.Arm64 || !mctx.Os().Bionic())) { variations = append(variations, blueprint.Variation{Mutator: "link", Variation: "shared"}) depTag = cc.SharedDepTag() deps = []string{config.LibclangRuntimeLibrary(mod.toolchain(mctx), "asan")} } else if mod.IsSanitizerEnabled(cc.Hwasan) || - (mod.IsSanitizerEnabled(cc.Fuzzer) && mctx.Arch().ArchType == android.Arm64) { + (mod.IsSanitizerEnabled(cc.Fuzzer) && mctx.Arch().ArchType == android.Arm64 && mctx.Os().Bionic()) { // TODO(b/204776996): HWASan for static Rust binaries isn't supported yet. if binary, ok := mod.compiler.(binaryInterface); ok { if binary.staticallyLinked() { @@ -444,20 +443,12 @@ func (mod *Module) IsSanitizerExplicitlyDisabled(t cc.SanitizerType) bool { return mod.sanitize.isSanitizerExplicitlyDisabled(t) } -func (mod *Module) SanitizeDep() bool { - return mod.sanitize.Properties.SanitizeDep -} - func (mod *Module) SetSanitizer(t cc.SanitizerType, b bool) { if !Bool(mod.sanitize.Properties.Sanitize.Never) { mod.sanitize.SetSanitizer(t, b) } } -func (mod *Module) SetSanitizeDep(b bool) { - mod.sanitize.Properties.SanitizeDep = b -} - func (mod *Module) StaticallyLinked() bool { if lib, ok := mod.compiler.(libraryInterface); ok { return lib.rlib() || lib.static() diff --git a/rust/snapshot_prebuilt.go b/rust/snapshot_prebuilt.go index dfbc1d1e7..2f79cc5c3 100644 --- a/rust/snapshot_prebuilt.go +++ b/rust/snapshot_prebuilt.go @@ -69,7 +69,7 @@ func snapshotLibraryFactory(image cc.SnapshotImage, moduleSuffix string) (*Modul return module, prebuilt } -func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path { +func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) buildOutput { var variant string if library.static() { variant = cc.SnapshotStaticSuffix @@ -85,11 +85,11 @@ func (library *snapshotLibraryDecorator) compile(ctx ModuleContext, flags Flags, } if !library.MatchesWithDevice(ctx.DeviceConfig()) { - return nil + return buildOutput{} } outputFile := android.PathForModuleSrc(ctx, *library.properties.Src) library.unstrippedOutputFile = outputFile - return outputFile + return buildOutput{outputFile: outputFile} } func (library *snapshotLibraryDecorator) rustdoc(ctx ModuleContext, flags Flags, deps PathDeps) android.OptionalPath { diff --git a/rust/test.go b/rust/test.go index 250b7657e..6e5393595 100644 --- a/rust/test.go +++ b/rust/test.go @@ -15,6 +15,8 @@ package rust import ( + "path/filepath" + "github.com/google/blueprint/proptools" "android/soong/android" @@ -151,9 +153,15 @@ func (test *testDecorator) install(ctx ModuleContext) { ctx.ModuleErrorf("data_lib %q is not a linkable module", depName) } if linkableDep.OutputFile().Valid() { + // Copy the output in "lib[64]" so that it's compatible with + // the default rpath values. + libDir := "lib" + if linkableDep.Target().Arch.ArchType.Multilib == "lib64" { + libDir = "lib64" + } test.data = append(test.data, android.DataPath{SrcPath: linkableDep.OutputFile().Path(), - RelativeInstallPath: linkableDep.RelativeInstallPath()}) + RelativeInstallPath: filepath.Join(libDir, linkableDep.RelativeInstallPath())}) } }) diff --git a/rust/test_test.go b/rust/test_test.go index 112417673..8906f1cb0 100644 --- a/rust/test_test.go +++ b/rust/test_test.go @@ -187,12 +187,12 @@ func TestDataLibsRelativeInstallPath(t *testing.T) { t.Errorf("expected test output file to be 'main_test', but was '%s'", outputPath) } entries := android.AndroidMkEntriesForTest(t, ctx, module)[0] - if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:foo/bar/baz") { - t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:foo/bar/baz`,"+ + if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:lib64/foo/bar/baz") { + t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:lib64/foo/bar/baz`,"+ " but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][0]) } - if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:foo/bar/baz") { - t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:foo/bar/baz`,"+ + if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:lib64/foo/bar/baz") { + t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:lib64/foo/bar/baz`,"+ " but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][1]) } if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][2], ":rusty:foo/bar/baz") { diff --git a/rust/testing.go b/rust/testing.go index cb98bed65..4796f691a 100644 --- a/rust/testing.go +++ b/rust/testing.go @@ -194,6 +194,7 @@ func registerRequiredBuildComponentsForTest(ctx android.RegistrationContext) { ctx.BottomUp("rust_begin", BeginMutator).Parallel() }) ctx.RegisterSingletonType("rust_project_generator", rustProjectGeneratorSingleton) + ctx.RegisterSingletonType("kythe_rust_extract", kytheExtractRustFactory) ctx.PostDepsMutators(func(ctx android.RegisterMutatorsContext) { ctx.BottomUp("rust_sanitizers", rustSanitizerRuntimeMutator).Parallel() }) diff --git a/scripts/hiddenapi/generate_hiddenapi_lists.py b/scripts/hiddenapi/generate_hiddenapi_lists.py index 35e0948f0..6546c7f80 100755 --- a/scripts/hiddenapi/generate_hiddenapi_lists.py +++ b/scripts/hiddenapi/generate_hiddenapi_lists.py @@ -27,6 +27,7 @@ FLAG_MAX_TARGET_O = 'max-target-o' FLAG_MAX_TARGET_P = 'max-target-p' FLAG_MAX_TARGET_Q = 'max-target-q' FLAG_MAX_TARGET_R = 'max-target-r' +FLAG_MAX_TARGET_S = 'max-target-s' FLAG_CORE_PLATFORM_API = 'core-platform-api' FLAG_PUBLIC_API = 'public-api' FLAG_SYSTEM_API = 'system-api' @@ -41,6 +42,7 @@ FLAGS_API_LIST = [ FLAG_MAX_TARGET_P, FLAG_MAX_TARGET_Q, FLAG_MAX_TARGET_R, + FLAG_MAX_TARGET_S, ] ALL_FLAGS = FLAGS_API_LIST + [ FLAG_CORE_PLATFORM_API, diff --git a/sdk/bootclasspath_fragment_sdk_test.go b/sdk/bootclasspath_fragment_sdk_test.go index 2dacdb55c..93ad172ae 100644 --- a/sdk/bootclasspath_fragment_sdk_test.go +++ b/sdk/bootclasspath_fragment_sdk_test.go @@ -101,6 +101,9 @@ func TestSnapshotWithBootclasspathFragment_ImageName(t *testing.T) { image_name: "art", contents: ["mybootlib"], apex_available: ["com.android.art"], + hidden_api: { + split_packages: ["*"], + }, } apex_key { @@ -124,7 +127,7 @@ func TestSnapshotWithBootclasspathFragment_ImageName(t *testing.T) { preparerForSnapshot := fixtureAddPrebuiltApexForBootclasspathFragment("com.android.art", "mybootclasspathfragment") CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_bootclasspath_fragment { @@ -152,41 +155,6 @@ java_import { jars: ["java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -prebuilt_bootclasspath_fragment { - name: "mysdk_mybootclasspathfragment@current", - sdk_member_name: "mybootclasspathfragment", - visibility: ["//visibility:public"], - apex_available: ["com.android.art"], - image_name: "art", - contents: ["mysdk_mybootlib@current"], - hidden_api: { - annotation_flags: "hiddenapi/annotation-flags.csv", - metadata: "hiddenapi/metadata.csv", - index: "hiddenapi/index.csv", - signature_patterns: "hiddenapi/signature-patterns.csv", - filtered_stub_flags: "hiddenapi/filtered-stub-flags.csv", - filtered_flags: "hiddenapi/filtered-flags.csv", - }, -} - -java_import { - name: "mysdk_mybootlib@current", - sdk_member_name: "mybootlib", - visibility: ["//visibility:public"], - apex_available: ["com.android.art"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - bootclasspath_fragments: ["mysdk_mybootclasspathfragment@current"], - java_boot_libs: ["mysdk_mybootlib@current"], -} -`), checkAllCopyRules(` .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/annotation-flags.csv -> hiddenapi/annotation-flags.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/metadata.csv -> hiddenapi/metadata.csv @@ -270,6 +238,9 @@ func TestSnapshotWithBootClasspathFragment_Contents(t *testing.T) { // This should be automatically added to the sdk_snapshot as a java_sdk_libs module. stub_libs: ["mycoreplatform"], }, + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -317,7 +288,7 @@ func TestSnapshotWithBootClasspathFragment_Contents(t *testing.T) { preparerForSnapshot := fixtureAddPrebuiltApexForBootclasspathFragment("myapex", "mybootclasspathfragment") CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_bootclasspath_fragment { @@ -402,103 +373,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -prebuilt_bootclasspath_fragment { - name: "mysdk_mybootclasspathfragment@current", - sdk_member_name: "mybootclasspathfragment", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - contents: [ - "mysdk_mybootlib@current", - "mysdk_myothersdklibrary@current", - ], - api: { - stub_libs: ["mysdk_mysdklibrary@current"], - }, - core_platform_api: { - stub_libs: ["mysdk_mycoreplatform@current"], - }, - hidden_api: { - annotation_flags: "hiddenapi/annotation-flags.csv", - metadata: "hiddenapi/metadata.csv", - index: "hiddenapi/index.csv", - signature_patterns: "hiddenapi/signature-patterns.csv", - filtered_stub_flags: "hiddenapi/filtered-stub-flags.csv", - filtered_flags: "hiddenapi/filtered-flags.csv", - }, -} - -java_import { - name: "mysdk_mybootlib@current", - sdk_member_name: "mybootlib", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/mybootlib.jar"], - permitted_packages: ["mybootlib"], -} - -java_sdk_library_import { - name: "mysdk_myothersdklibrary@current", - sdk_member_name: "myothersdklibrary", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - shared_library: true, - compile_dex: true, - permitted_packages: ["myothersdklibrary"], - public: { - jars: ["sdk_library/public/myothersdklibrary-stubs.jar"], - stub_srcs: ["sdk_library/public/myothersdklibrary_stub_sources"], - current_api: "sdk_library/public/myothersdklibrary.txt", - removed_api: "sdk_library/public/myothersdklibrary-removed.txt", - sdk_version: "current", - }, -} - -java_sdk_library_import { - name: "mysdk_mysdklibrary@current", - sdk_member_name: "mysdklibrary", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - shared_library: false, - public: { - jars: ["sdk_library/public/mysdklibrary-stubs.jar"], - stub_srcs: ["sdk_library/public/mysdklibrary_stub_sources"], - current_api: "sdk_library/public/mysdklibrary.txt", - removed_api: "sdk_library/public/mysdklibrary-removed.txt", - sdk_version: "current", - }, -} - -java_sdk_library_import { - name: "mysdk_mycoreplatform@current", - sdk_member_name: "mycoreplatform", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - shared_library: true, - compile_dex: true, - public: { - jars: ["sdk_library/public/mycoreplatform-stubs.jar"], - stub_srcs: ["sdk_library/public/mycoreplatform_stub_sources"], - current_api: "sdk_library/public/mycoreplatform.txt", - removed_api: "sdk_library/public/mycoreplatform-removed.txt", - sdk_version: "current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - bootclasspath_fragments: ["mysdk_mybootclasspathfragment@current"], - java_boot_libs: ["mysdk_mybootlib@current"], - java_sdk_libs: [ - "mysdk_myothersdklibrary@current", - "mysdk_mysdklibrary@current", - "mysdk_mycoreplatform@current", - ], -} - `), checkAllCopyRules(` .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/annotation-flags.csv -> hiddenapi/annotation-flags.csv .intermediates/mybootclasspathfragment/android_common/modular-hiddenapi/metadata.csv -> hiddenapi/metadata.csv @@ -585,6 +459,9 @@ func TestSnapshotWithBootClasspathFragment_Fragments(t *testing.T) { contents: [ "myotherlib", ], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -614,6 +491,9 @@ func TestSnapshotWithBootClasspathFragment_Fragments(t *testing.T) { module: "myotherbootclasspathfragment" }, ], + hidden_api: { + split_packages: ["*"], + }, } java_sdk_library { @@ -630,7 +510,7 @@ func TestSnapshotWithBootClasspathFragment_Fragments(t *testing.T) { preparerForSnapshot := fixtureAddPrebuiltApexForBootclasspathFragment("myapex", "mybootclasspathfragment") CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_bootclasspath_fragment { @@ -693,6 +573,9 @@ func TestBasicSdkWithBootclasspathFragment(t *testing.T) { image_name: "art", contents: ["mybootlib"], apex_available: ["myapex"], + hidden_api: { + split_packages: ["*"], + }, } java_library { @@ -800,6 +683,7 @@ func TestSnapshotWithBootclasspathFragment_HiddenAPI(t *testing.T) { unsupported_packages: [ "my-unsupported-packages.txt", ], + split_packages: ["*"], }, } @@ -828,7 +712,7 @@ func TestSnapshotWithBootclasspathFragment_HiddenAPI(t *testing.T) { preparerForSnapshot := fixtureAddPrebuiltApexForBootclasspathFragment("myapex", "mybootclasspathfragment") CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_bootclasspath_fragment { @@ -298,15 +298,15 @@ func (t identityTransformation) transformModule(module *bpModule) *bpModule { return module } -func (t identityTransformation) transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { +func (t identityTransformation) transformPropertySetBeforeContents(_ string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { return propertySet, tag } -func (t identityTransformation) transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { +func (t identityTransformation) transformPropertySetAfterContents(_ string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { return propertySet, tag } -func (t identityTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { +func (t identityTransformation) transformProperty(_ string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { return value, tag } @@ -332,7 +332,7 @@ func (t deepCopyTransformation) transformModule(module *bpModule) *bpModule { return &moduleCopy } -func (t deepCopyTransformation) transformPropertySetBeforeContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { +func (t deepCopyTransformation) transformPropertySetBeforeContents(_ string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { // Create a shallow copy of the properties map. Any mutable property values will be copied by the // transformer. propertiesCopy := make(map[string]interface{}) @@ -354,7 +354,7 @@ func (t deepCopyTransformation) transformPropertySetBeforeContents(name string, }, tag } -func (t deepCopyTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { +func (t deepCopyTransformation) transformProperty(_ string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { // Copy string slice, otherwise return value. if values, ok := value.([]string); ok { valuesCopy := make([]string, len(values)) @@ -372,7 +372,7 @@ type bpFile struct { order []*bpModule } -// Add a module. +// AddModule adds a module to this. // // The module must have had its "name" property set to a string value that // is unique within this file. diff --git a/sdk/cc_sdk_test.go b/sdk/cc_sdk_test.go index cd63dac8c..265579aa1 100644 --- a/sdk/cc_sdk_test.go +++ b/sdk/cc_sdk_test.go @@ -120,7 +120,7 @@ func TestSdkCompileMultilibOverride(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -145,162 +145,12 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_sdkmember@current", - sdk_member_name: "sdkmember", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - installable: false, - stl: "none", - compile_multilib: "64", - target: { - host: { - enabled: false, - }, - android_arm64: { - srcs: ["android/arm64/lib/sdkmember.so"], - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/lib/sdkmember.so"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - compile_multilib: "64", - native_shared_libs: ["mysdk_sdkmember@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/sdkmember/android_arm64_armv8-a_shared/sdkmember.so -> android/arm64/lib/sdkmember.so .intermediates/sdkmember/linux_glibc_x86_64_shared/sdkmember.so -> linux_glibc/x86_64/lib/sdkmember.so `)) } -func TestBasicSdkWithCc(t *testing.T) { - result := testSdkWithCc(t, ` - sdk { - name: "mysdk", - native_shared_libs: ["sdkmember"], - } - - cc_library_shared { - name: "sdkmember", - system_shared_libs: [], - stl: "none", - apex_available: ["mysdkapex"], - } - - sdk_snapshot { - name: "mysdk@1", - native_shared_libs: ["sdkmember_mysdk@1"], - } - - sdk_snapshot { - name: "mysdk@2", - native_shared_libs: ["sdkmember_mysdk@2"], - } - - cc_prebuilt_library_shared { - name: "sdkmember", - srcs: ["libfoo.so"], - prefer: false, - system_shared_libs: [], - stl: "none", - } - - cc_prebuilt_library_shared { - name: "sdkmember_mysdk@1", - sdk_member_name: "sdkmember", - srcs: ["libfoo.so"], - system_shared_libs: [], - stl: "none", - // TODO: remove //apex_available:platform - apex_available: [ - "//apex_available:platform", - "myapex", - ], - } - - cc_prebuilt_library_shared { - name: "sdkmember_mysdk@2", - sdk_member_name: "sdkmember", - srcs: ["libfoo.so"], - system_shared_libs: [], - stl: "none", - // TODO: remove //apex_available:platform - apex_available: [ - "//apex_available:platform", - "myapex2", - ], - } - - cc_library_shared { - name: "mycpplib", - srcs: ["Test.cpp"], - shared_libs: ["sdkmember"], - system_shared_libs: [], - stl: "none", - apex_available: [ - "myapex", - "myapex2", - ], - } - - apex { - name: "myapex", - native_shared_libs: ["mycpplib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "myapex2", - native_shared_libs: ["mycpplib"], - uses_sdks: ["mysdk@2"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "mysdkapex", - native_shared_libs: ["sdkmember"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - `) - - sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk@1", "android_arm64_armv8-a_shared_apex10000_mysdk_1").Rule("toc").Output - sdkMemberV2 := result.ModuleForTests("sdkmember_mysdk@2", "android_arm64_armv8-a_shared_apex10000_mysdk_2").Rule("toc").Output - - cpplibForMyApex := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_apex10000_mysdk_1") - cpplibForMyApex2 := result.ModuleForTests("mycpplib", "android_arm64_armv8-a_shared_apex10000_mysdk_2") - - // Depending on the uses_sdks value, different libs are linked - ensureListContains(t, pathsToStrings(cpplibForMyApex.Rule("ld").Implicits), sdkMemberV1.String()) - ensureListContains(t, pathsToStrings(cpplibForMyApex2.Rule("ld").Implicits), sdkMemberV2.String()) -} - // Make sure the sdk can use host specific cc libraries static/shared and both. func TestHostSdkWithCc(t *testing.T) { testSdkWithCc(t, ` @@ -373,7 +223,7 @@ func TestSnapshotWithObject(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_object { @@ -397,37 +247,6 @@ cc_prebuilt_object { }, } `), - // Make sure that the generated sdk_snapshot uses the native_objects property. - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_object { - name: "mysdk_crtobj@current", - sdk_member_name: "crtobj", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - stl: "none", - compile_multilib: "both", - system_shared_libs: [], - sanitize: { - never: true, - }, - arch: { - arm64: { - srcs: ["arm64/lib/crtobj.o"], - }, - arm: { - srcs: ["arm/lib/crtobj.o"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - native_objects: ["mysdk_crtobj@current"], -} -`), checkAllCopyRules(` .intermediates/crtobj/android_arm64_armv8-a/crtobj.o -> arm64/lib/crtobj.o .intermediates/crtobj/android_arm_armv7-a-neon/crtobj.o -> arm/lib/crtobj.o @@ -511,7 +330,7 @@ func TestSnapshotWithCcExportGeneratedHeaders(t *testing.T) { errorHandler := android.FixtureExpectsAtLeastOneErrorMatchingPattern(`module source path "snapshot/include_gen/generated_foo/gen/protos" does not exist`) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -584,7 +403,7 @@ func TestSnapshotWithCcSharedLibraryCommonProperties(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -643,7 +462,7 @@ func TestSnapshotWithCcBinary(t *testing.T) { `) CheckSnapshot(t, result, "mymodule_exports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_binary { @@ -662,33 +481,6 @@ cc_prebuilt_binary { }, } `), - // Make sure that the generated sdk_snapshot uses the native_binaries property. - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_binary { - name: "mymodule_exports_mynativebinary@current", - sdk_member_name: "mynativebinary", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - installable: false, - compile_multilib: "both", - arch: { - arm64: { - srcs: ["arm64/bin/mynativebinary"], - }, - arm: { - srcs: ["arm/bin/mynativebinary"], - }, - }, -} - -module_exports_snapshot { - name: "mymodule_exports@current", - visibility: ["//visibility:public"], - native_binaries: ["mymodule_exports_mynativebinary@current"], -} -`), checkAllCopyRules(` .intermediates/mynativebinary/android_arm64_armv8-a/mynativebinary -> arm64/bin/mynativebinary .intermediates/mynativebinary/android_arm_armv7-a-neon/mynativebinary -> arm/bin/mynativebinary @@ -728,7 +520,7 @@ func TestMultipleHostOsTypesSnapshotWithCcBinary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_binary { @@ -764,68 +556,6 @@ cc_prebuilt_binary { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_binary { - name: "myexports_mynativebinary@current", - sdk_member_name: "mynativebinary", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - target: { - host: { - enabled: false, - }, - linux_glibc: { - compile_multilib: "both", - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/bin/mynativebinary"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["linux_glibc/x86/bin/mynativebinary"], - }, - windows: { - compile_multilib: "64", - }, - windows_x86_64: { - enabled: true, - srcs: ["windows/x86_64/bin/mynativebinary.exe"], - }, - }, -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_binaries: ["myexports_mynativebinary@current"], - target: { - windows: { - compile_multilib: "64", - }, - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - windows_x86_64: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/mynativebinary/linux_glibc_x86_64/mynativebinary -> linux_glibc/x86_64/bin/mynativebinary .intermediates/mynativebinary/linux_glibc_x86/mynativebinary -> linux_glibc/x86/bin/mynativebinary @@ -888,7 +618,7 @@ func TestSnapshotWithSingleHostOsType(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_binary { @@ -931,69 +661,6 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_binary { - name: "myexports_mynativebinary@current", - sdk_member_name: "mynativebinary", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - compile_multilib: "64", - target: { - host: { - enabled: false, - }, - linux_bionic_x86_64: { - enabled: true, - srcs: ["x86_64/bin/mynativebinary"], - }, - }, -} - -cc_prebuilt_library_shared { - name: "myexports_mynativelib@current", - sdk_member_name: "mynativelib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - compile_multilib: "64", - target: { - host: { - enabled: false, - }, - linux_bionic_x86_64: { - enabled: true, - srcs: ["x86_64/lib/mynativelib.so"], - }, - }, -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - compile_multilib: "64", - native_binaries: ["myexports_mynativebinary@current"], - native_shared_libs: ["myexports_mynativelib@current"], - target: { - host: { - enabled: false, - }, - linux_bionic_x86_64: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/mynativebinary/linux_bionic_x86_64/mynativebinary -> x86_64/bin/mynativebinary .intermediates/mynativelib/linux_bionic_x86_64_shared/mynativelib.so -> x86_64/lib/mynativelib.so @@ -1026,7 +693,7 @@ func TestSnapshotWithCcStaticNocrtBinary(t *testing.T) { `) CheckSnapshot(t, result, "mymodule_exports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_binary { @@ -1055,55 +722,6 @@ cc_prebuilt_binary { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_binary { - name: "mymodule_exports_linker@current", - sdk_member_name: "linker", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - compile_multilib: "both", - static_executable: true, - nocrt: true, - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["x86_64/bin/linker"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["x86/bin/linker"], - }, - }, -} - -module_exports_snapshot { - name: "mymodule_exports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_binaries: ["mymodule_exports_linker@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/linker/linux_glibc_x86_64/linker -> x86_64/bin/linker .intermediates/linker/linux_glibc_x86/linker -> x86/bin/linker @@ -1134,7 +752,7 @@ func TestSnapshotWithCcSharedLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -1235,7 +853,7 @@ func TestSnapshotWithCcSharedLibrarySharedLibs(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -1332,7 +950,7 @@ func TestHostSnapshotWithCcSharedLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -1363,57 +981,6 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_mynativelib@current", - sdk_member_name: "mynativelib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - sdk_version: "minimum", - stl: "none", - compile_multilib: "both", - export_include_dirs: ["include/myinclude"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["x86_64/lib/mynativelib.so"], - export_include_dirs: ["x86_64/include_gen/mynativelib/linux_glibc_x86_64_shared/gen/aidl"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["x86/lib/mynativelib.so"], - export_include_dirs: ["x86/include_gen/mynativelib/linux_glibc_x86_shared/gen/aidl"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_shared_libs: ["mysdk_mynativelib@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` myinclude/Test.h -> include/myinclude/Test.h .intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> x86_64/lib/mynativelib.so @@ -1459,7 +1026,7 @@ func TestMultipleHostOsTypesSnapshotWithCcSharedLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -1495,68 +1062,6 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_mynativelib@current", - sdk_member_name: "mynativelib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - target: { - host: { - enabled: false, - }, - linux_glibc: { - compile_multilib: "both", - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/lib/mynativelib.so"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["linux_glibc/x86/lib/mynativelib.so"], - }, - windows: { - compile_multilib: "64", - }, - windows_x86_64: { - enabled: true, - srcs: ["windows/x86_64/lib/mynativelib.dll"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_shared_libs: ["mysdk_mynativelib@current"], - target: { - windows: { - compile_multilib: "64", - }, - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - windows_x86_64: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> linux_glibc/x86_64/lib/mynativelib.so .intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> linux_glibc/x86/lib/mynativelib.so @@ -1587,7 +1092,7 @@ func TestSnapshotWithCcStaticLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_static { @@ -1650,7 +1155,7 @@ func TestHostSnapshotWithCcStaticLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_static { @@ -1680,56 +1185,6 @@ cc_prebuilt_library_static { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_static { - name: "myexports_mynativelib@current", - sdk_member_name: "mynativelib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - installable: false, - stl: "none", - compile_multilib: "both", - export_include_dirs: ["include/myinclude"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["x86_64/lib/mynativelib.a"], - export_include_dirs: ["x86_64/include_gen/mynativelib/linux_glibc_x86_64_static/gen/aidl"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["x86/lib/mynativelib.a"], - export_include_dirs: ["x86/include_gen/mynativelib/linux_glibc_x86_static/gen/aidl"], - }, - }, -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_static_libs: ["myexports_mynativelib@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` myinclude/Test.h -> include/myinclude/Test.h .intermediates/mynativelib/linux_glibc_x86_64_static/mynativelib.a -> x86_64/lib/mynativelib.a @@ -1764,7 +1219,7 @@ func TestSnapshotWithCcLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library { @@ -1796,127 +1251,282 @@ cc_prebuilt_library { }, } `), - // Make sure that the generated sdk_snapshot uses the native_libs property. - checkVersionedAndroidBpContents(` + checkAllCopyRules(` +myinclude/Test.h -> include/myinclude/Test.h +.intermediates/mynativelib/android_arm64_armv8-a_static/mynativelib.a -> arm64/lib/mynativelib.a +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_static/mynativelib.a -> arm/lib/mynativelib.a +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so +`), + // TODO(b/183315522): Remove this and fix the issue. + snapshotTestErrorHandler(checkSnapshotPreferredWithSource, android.FixtureExpectsAtLeastOneErrorMatchingPattern(`\Qunrecognized property "arch.arm.shared.export_include_dirs"\E`)), + ) +} + +func TestSnapshotSameLibraryWithNativeLibsAndNativeSharedLib(t *testing.T) { + result := testSdkWithCc(t, ` + module_exports { + host_supported: true, + name: "myexports", + target: { + android: { + native_shared_libs: [ + "mynativelib", + ], + }, + not_windows: { + native_libs: [ + "mynativelib", + ], + }, + }, + } + + cc_library { + name: "mynativelib", + host_supported: true, + srcs: [ + "Test.cpp", + ], + stl: "none", + recovery_available: true, + vendor_available: true, + } + `) + + CheckSnapshot(t, result, "myexports", "", + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library { - name: "myexports_mynativelib@current", - sdk_member_name: "mynativelib", + name: "mynativelib", + prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], - installable: false, + host_supported: true, vendor_available: true, stl: "none", compile_multilib: "both", - export_include_dirs: ["include/myinclude"], - arch: { - arm64: { + target: { + host: { + enabled: false, + }, + android_arm64: { + shared: { + srcs: ["android/arm64/lib/mynativelib.so"], + }, static: { - srcs: ["arm64/lib/mynativelib.a"], + enabled: false, }, + }, + android_arm: { shared: { - srcs: ["arm64/lib/mynativelib.so"], + srcs: ["android/arm/lib/mynativelib.so"], + }, + static: { + enabled: false, }, }, - arm: { + linux_glibc_x86_64: { + enabled: true, static: { - srcs: ["arm/lib/mynativelib.a"], + srcs: ["linux_glibc/x86_64/lib/mynativelib.a"], }, shared: { - srcs: ["arm/lib/mynativelib.so"], + srcs: ["linux_glibc/x86_64/lib/mynativelib.so"], + }, + }, + linux_glibc_x86: { + enabled: true, + static: { + srcs: ["linux_glibc/x86/lib/mynativelib.a"], + }, + shared: { + srcs: ["linux_glibc/x86/lib/mynativelib.so"], }, }, }, } - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - native_libs: ["myexports_mynativelib@current"], -} `), checkAllCopyRules(` -myinclude/Test.h -> include/myinclude/Test.h -.intermediates/mynativelib/android_arm64_armv8-a_static/mynativelib.a -> arm64/lib/mynativelib.a -.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> arm64/lib/mynativelib.so -.intermediates/mynativelib/android_arm_armv7-a-neon_static/mynativelib.a -> arm/lib/mynativelib.a -.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> android/arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> android/arm/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_64_static/mynativelib.a -> linux_glibc/x86_64/lib/mynativelib.a +.intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> linux_glibc/x86_64/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_static/mynativelib.a -> linux_glibc/x86/lib/mynativelib.a +.intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> linux_glibc/x86/lib/mynativelib.so `), - // TODO(b/183315522): Remove this and fix the issue. - snapshotTestErrorHandler(checkSnapshotPreferredWithSource, android.FixtureExpectsAtLeastOneErrorMatchingPattern(`\Qunrecognized property "arch.arm.shared.export_include_dirs"\E`)), ) } -func TestHostSnapshotWithMultiLib64(t *testing.T) { +func TestSnapshotSameLibraryWithAndroidNativeLibsAndHostNativeSharedLib(t *testing.T) { result := testSdkWithCc(t, ` module_exports { - name: "myexports", - device_supported: false, host_supported: true, + name: "myexports", target: { - host: { - compile_multilib: "64", + android: { + native_libs: [ + "mynativelib", + ], + }, + not_windows: { + native_shared_libs: [ + "mynativelib", + ], }, }, - native_static_libs: ["mynativelib"], } - cc_library_static { + cc_library { name: "mynativelib", - device_supported: false, host_supported: true, srcs: [ "Test.cpp", - "aidl/foo/bar/Test.aidl", ], - export_include_dirs: ["myinclude"], - aidl: { - export_aidl_headers: true, - }, stl: "none", + recovery_available: true, + vendor_available: true, } `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. -cc_prebuilt_library_static { +cc_prebuilt_library { name: "mynativelib", prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], - device_supported: false, host_supported: true, + vendor_available: true, stl: "none", - compile_multilib: "64", - export_include_dirs: [ - "include/myinclude", - "include_gen/mynativelib/linux_glibc_x86_64_static/gen/aidl", - ], + compile_multilib: "both", target: { host: { enabled: false, }, + android_arm64: { + static: { + srcs: ["android/arm64/lib/mynativelib.a"], + }, + shared: { + srcs: ["android/arm64/lib/mynativelib.so"], + }, + }, + android_arm: { + static: { + srcs: ["android/arm/lib/mynativelib.a"], + }, + shared: { + srcs: ["android/arm/lib/mynativelib.so"], + }, + }, linux_glibc_x86_64: { enabled: true, - srcs: ["x86_64/lib/mynativelib.a"], + shared: { + srcs: ["linux_glibc/x86_64/lib/mynativelib.so"], + }, + static: { + enabled: false, + }, + }, + linux_glibc_x86: { + enabled: true, + shared: { + srcs: ["linux_glibc/x86/lib/mynativelib.so"], + }, + static: { + enabled: false, + }, }, }, } `), - checkVersionedAndroidBpContents(` + checkAllCopyRules(` +.intermediates/mynativelib/android_arm64_armv8-a_static/mynativelib.a -> android/arm64/lib/mynativelib.a +.intermediates/mynativelib/android_arm64_armv8-a_shared/mynativelib.so -> android/arm64/lib/mynativelib.so +.intermediates/mynativelib/android_arm_armv7-a-neon_static/mynativelib.a -> android/arm/lib/mynativelib.a +.intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> android/arm/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> linux_glibc/x86_64/lib/mynativelib.so +.intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> linux_glibc/x86/lib/mynativelib.so +`), + ) +} + +func TestSnapshotSameLibraryWithNativeStaticLibsAndNativeSharedLib(t *testing.T) { + testSdkError(t, "Incompatible member types", ` + module_exports { + host_supported: true, + name: "myexports", + target: { + android: { + native_shared_libs: [ + "mynativelib", + ], + }, + not_windows: { + native_static_libs: [ + "mynativelib", + ], + }, + }, + } + + cc_library { + name: "mynativelib", + host_supported: true, + srcs: [ + ], + stl: "none", + recovery_available: true, + vendor_available: true, + } + `) +} + +func TestHostSnapshotWithMultiLib64(t *testing.T) { + result := testSdkWithCc(t, ` + module_exports { + name: "myexports", + device_supported: false, + host_supported: true, + target: { + host: { + compile_multilib: "64", + }, + }, + native_static_libs: ["mynativelib"], + } + + cc_library_static { + name: "mynativelib", + device_supported: false, + host_supported: true, + srcs: [ + "Test.cpp", + "aidl/foo/bar/Test.aidl", + ], + export_include_dirs: ["myinclude"], + aidl: { + export_aidl_headers: true, + }, + stl: "none", + } + `) + + CheckSnapshot(t, result, "myexports", "", + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_static { - name: "myexports_mynativelib@current", - sdk_member_name: "mynativelib", + name: "mynativelib", + prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], device_supported: false, host_supported: true, - installable: false, stl: "none", compile_multilib: "64", export_include_dirs: [ @@ -1933,23 +1543,6 @@ cc_prebuilt_library_static { }, }, } - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - compile_multilib: "64", - native_static_libs: ["myexports_mynativelib@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - }, -} `), checkAllCopyRules(` myinclude/Test.h -> include/myinclude/Test.h @@ -1976,7 +1569,7 @@ func TestSnapshotWithCcHeadersLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_headers { @@ -2020,7 +1613,7 @@ func TestSnapshotWithCcHeadersLibraryAndNativeBridgeSupport(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_headers { @@ -2105,7 +1698,7 @@ func TestSnapshotWithCcHeadersLibraryAndImageVariants(t *testing.T) { `, trait, property)) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(fmt.Sprintf(` + checkAndroidBpContents(fmt.Sprintf(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_headers { @@ -2154,7 +1747,7 @@ func TestHostSnapshotWithCcHeadersLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_headers { @@ -2180,51 +1773,6 @@ cc_prebuilt_library_headers { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_headers { - name: "mysdk_mynativeheaders@current", - sdk_member_name: "mynativeheaders", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - stl: "none", - compile_multilib: "both", - export_include_dirs: ["include/myinclude"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - native_header_libs: ["mysdk_mynativeheaders@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` myinclude/Test.h -> include/myinclude/Test.h `), @@ -2256,7 +1804,7 @@ func TestDeviceAndHostSnapshotWithCcHeadersLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_headers { @@ -2287,55 +1835,6 @@ cc_prebuilt_library_headers { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_headers { - name: "mysdk_mynativeheaders@current", - sdk_member_name: "mynativeheaders", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - stl: "none", - compile_multilib: "both", - export_system_include_dirs: ["common_os/include/myinclude"], - target: { - host: { - enabled: false, - }, - android: { - export_include_dirs: ["android/include/myinclude-android"], - }, - linux_glibc: { - export_include_dirs: ["linux_glibc/include/myinclude-host"], - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - native_header_libs: ["mysdk_mynativeheaders@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` myinclude/Test.h -> common_os/include/myinclude/Test.h myinclude-android/AndroidTest.h -> android/include/myinclude-android/AndroidTest.h @@ -2368,7 +1867,7 @@ func TestSystemSharedLibPropagation(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2441,7 +1940,7 @@ cc_prebuilt_library_shared { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2475,59 +1974,7 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_sslvariants@current", - sdk_member_name: "sslvariants", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - installable: false, - compile_multilib: "both", - target: { - host: { - enabled: false, - }, - android: { - system_shared_libs: [], - }, - android_arm64: { - srcs: ["android/arm64/lib/sslvariants.so"], - }, - android_arm: { - srcs: ["android/arm/lib/sslvariants.so"], - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/lib/sslvariants.so"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["linux_glibc/x86/lib/sslvariants.so"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - native_shared_libs: ["mysdk_sslvariants@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`)) + ) } func TestStubsLibrary(t *testing.T) { @@ -2552,7 +1999,7 @@ func TestStubsLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2606,7 +2053,7 @@ func TestDeviceAndHostSnapshotWithStubsLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2645,64 +2092,7 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_stubslib@current", - sdk_member_name: "stubslib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - installable: false, - compile_multilib: "both", - stubs: { - versions: [ - "1", - "2", - "3", - "current", - ], - }, - target: { - host: { - enabled: false, - }, - android_arm64: { - srcs: ["android/arm64/lib/stubslib.so"], - }, - android_arm: { - srcs: ["android/arm/lib/stubslib.so"], - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/lib/stubslib.so"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["linux_glibc/x86/lib/stubslib.so"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - native_shared_libs: ["mysdk_stubslib@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`)) + ) } func TestUniqueHostSoname(t *testing.T) { @@ -2721,7 +2111,7 @@ func TestUniqueHostSoname(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2753,57 +2143,6 @@ cc_prebuilt_library_shared { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -cc_prebuilt_library_shared { - name: "mysdk_mylib@current", - sdk_member_name: "mylib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - installable: false, - unique_host_soname: true, - compile_multilib: "both", - target: { - host: { - enabled: false, - }, - android_arm64: { - srcs: ["android/arm64/lib/mylib.so"], - }, - android_arm: { - srcs: ["android/arm/lib/mylib.so"], - }, - linux_glibc_x86_64: { - enabled: true, - srcs: ["linux_glibc/x86_64/lib/mylib-host.so"], - }, - linux_glibc_x86: { - enabled: true, - srcs: ["linux_glibc/x86/lib/mylib-host.so"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - native_shared_libs: ["mysdk_mylib@current"], - target: { - host: { - enabled: false, - }, - linux_glibc_x86_64: { - enabled: true, - }, - linux_glibc_x86: { - enabled: true, - }, - }, -} -`), checkAllCopyRules(` .intermediates/mylib/android_arm64_armv8-a_shared/mylib.so -> android/arm64/lib/mylib.so .intermediates/mylib/android_arm_armv7-a-neon_shared/mylib.so -> android/arm/lib/mylib.so @@ -2835,13 +2174,8 @@ func TestNoSanitizerMembers(t *testing.T) { } `) - // Mixing the snapshot with the source (irrespective of which one is preferred) causes a problem - // due to missing variants. - // TODO(b/183204176): Remove this and fix the cause. - snapshotWithSourceErrorHandler := android.FixtureExpectsAtLeastOneErrorMatchingPattern(`\QReplaceDependencies could not find identical variant {os:android,image:,arch:arm64_armv8-a,sdk:,link:shared,version:} for module mynativelib\E`) - CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. cc_prebuilt_library_shared { @@ -2866,7 +2200,5 @@ myinclude/Test.h -> include/myinclude/Test.h arm64/include/Arm64Test.h -> arm64/include/arm64/include/Arm64Test.h .intermediates/mynativelib/android_arm_armv7-a-neon_shared/mynativelib.so -> arm/lib/mynativelib.so `), - snapshotTestErrorHandler(checkSnapshotWithSourcePreferred, snapshotWithSourceErrorHandler), - snapshotTestErrorHandler(checkSnapshotPreferredWithSource, snapshotWithSourceErrorHandler), ) } diff --git a/sdk/compat_config_sdk_test.go b/sdk/compat_config_sdk_test.go index 00073c29d..d166add00 100644 --- a/sdk/compat_config_sdk_test.go +++ b/sdk/compat_config_sdk_test.go @@ -37,23 +37,7 @@ func TestSnapshotWithCompatConfig(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -prebuilt_platform_compat_config { - name: "mysdk_myconfig@current", - sdk_member_name: "myconfig", - visibility: ["//visibility:public"], - metadata: "compat_configs/myconfig/myconfig_meta.xml", -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - compat_configs: ["mysdk_myconfig@current"], -} -`), - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_platform_compat_config { diff --git a/sdk/exports.go b/sdk/exports.go index 9a0ba4e32..7645d3fa1 100644 --- a/sdk/exports.go +++ b/sdk/exports.go @@ -31,7 +31,7 @@ func ModuleExportsFactory() android.Module { return newSdkModule(true) } -// module_exports_snapshot is a versioned snapshot of prebuilt versions of all the exports +// module_exports_snapshot is a snapshot of prebuilt versions of all the exports // of a mainline module. func ModuleExportsSnapshotsFactory() android.Module { s := newSdkModule(true) diff --git a/sdk/exports_test.go b/sdk/exports_test.go index 17ddf1772..2605fd141 100644 --- a/sdk/exports_test.go +++ b/sdk/exports_test.go @@ -43,7 +43,7 @@ func TestModuleExportsSnapshot(t *testing.T) { }) CheckSnapshot(t, result, "myexports", "package", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -54,22 +54,5 @@ java_import { jars: ["java/myjavalib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - java_libs: ["myexports_myjavalib@current"], -} -`), ) } diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go index f0d3b35d7..d25138f5b 100644 --- a/sdk/java_sdk_test.go +++ b/sdk/java_sdk_test.go @@ -71,90 +71,6 @@ func TestSdkDependsOnSourceEvenWhenPrebuiltPreferred(t *testing.T) { ) } -func TestBasicSdkWithJavaLibrary(t *testing.T) { - result := android.GroupFixturePreparers( - prepareForSdkTestWithJava, - prepareForSdkTestWithApex, - ).RunTestWithBp(t, ` - sdk { - name: "mysdk", - java_header_libs: ["sdkmember"], - } - - sdk_snapshot { - name: "mysdk@1", - java_header_libs: ["sdkmember_mysdk@1"], - } - - sdk_snapshot { - name: "mysdk@2", - java_header_libs: ["sdkmember_mysdk@2"], - } - - java_library { - name: "sdkmember", - srcs: ["Test.java"], - system_modules: "none", - sdk_version: "none", - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk@1", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk@2", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_library { - name: "myjavalib", - srcs: ["Test.java"], - libs: ["sdkmember"], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - apex_available: [ - "myapex", - "myapex2", - ], - } - - apex { - name: "myapex", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - - apex { - name: "myapex2", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@2"], - key: "myapex.key", - certificate: ":myapex.cert", - updatable: false, - } - `) - - sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk@1", "android_common").Rule("combineJar").Output - sdkMemberV2 := result.ModuleForTests("sdkmember_mysdk@2", "android_common").Rule("combineJar").Output - - javalibForMyApex := result.ModuleForTests("myjavalib", "android_common_apex10000_mysdk_1") - javalibForMyApex2 := result.ModuleForTests("myjavalib", "android_common_apex10000_mysdk_2") - - // Depending on the uses_sdks value, different libs are linked - ensureListContains(t, pathsToStrings(javalibForMyApex.Rule("javac").Implicits), sdkMemberV1.String()) - ensureListContains(t, pathsToStrings(javalibForMyApex2.Rule("javac").Implicits), sdkMemberV2.String()) -} - func TestSnapshotWithJavaHeaderLibrary(t *testing.T) { result := android.GroupFixturePreparers( prepareForSdkTestWithJava, @@ -180,7 +96,7 @@ func TestSnapshotWithJavaHeaderLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -192,24 +108,6 @@ java_import { permitted_packages: ["pkg.myjavalib"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], - permitted_packages: ["pkg.myjavalib"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl @@ -244,7 +142,7 @@ func TestHostSnapshotWithJavaHeaderLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -257,27 +155,6 @@ java_import { jars: ["java/myjavalib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - jars: ["java/myjavalib.jar"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - java_header_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl @@ -304,7 +181,7 @@ func TestDeviceAndHostSnapshotWithJavaHeaderLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -323,32 +200,6 @@ java_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - target: { - android: { - jars: ["java/android/myjavalib.jar"], - }, - linux_glibc: { - jars: ["java/linux_glibc/myjavalib.jar"], - }, - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - host_supported: true, - java_header_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar @@ -382,7 +233,7 @@ func TestSnapshotWithJavaImplLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -393,23 +244,6 @@ java_import { jars: ["java/myjavalib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - java_libs: ["myexports_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib/android_common/withres/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl @@ -445,7 +279,7 @@ func TestSnapshotWithJavaBootLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -457,25 +291,6 @@ java_import { permitted_packages: ["pkg.myjavalib"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java_boot_libs/snapshot/jars/are/invalid/myjavalib.jar"], - permitted_packages: ["pkg.myjavalib"], -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - java_boot_libs: ["myexports_myjavalib@current"], -} - -`), checkAllCopyRules(` .intermediates/myexports/common_os/empty -> java_boot_libs/snapshot/jars/are/invalid/myjavalib.jar `), @@ -511,7 +326,7 @@ func TestSnapshotWithJavaSystemserverLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -523,24 +338,6 @@ java_import { permitted_packages: ["pkg.myjavalib"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java_systemserver_libs/snapshot/jars/are/invalid/myjavalib.jar"], - permitted_packages: ["pkg.myjavalib"], -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - java_systemserver_libs: ["myexports_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myexports/common_os/empty -> java_systemserver_libs/snapshot/jars/are/invalid/myjavalib.jar `), @@ -574,7 +371,7 @@ func TestHostSnapshotWithJavaImplLibrary(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -587,27 +384,6 @@ java_import { jars: ["java/myjavalib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - jars: ["java/myjavalib.jar"], -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - java_libs: ["myexports_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/myjavalib.jar aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl @@ -633,7 +409,7 @@ func TestSnapshotWithJavaTest(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_test_import { @@ -645,24 +421,6 @@ java_test_import { test_config: "java/myjavatests-AndroidTest.xml", } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_test_import { - name: "myexports_myjavatests@current", - sdk_member_name: "myjavatests", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavatests.jar"], - test_config: "java/myjavatests-AndroidTest.xml", -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - java_tests: ["myexports_myjavatests@current"], -} -`), checkAllCopyRules(` .intermediates/myjavatests/android_common/javac/myjavatests.jar -> java/myjavatests.jar .intermediates/myjavatests/android_common/myjavatests.config -> java/myjavatests-AndroidTest.xml @@ -691,7 +449,7 @@ func TestHostSnapshotWithJavaTest(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_test_import { @@ -705,28 +463,6 @@ java_test_import { test_config: "java/myjavatests-AndroidTest.xml", } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_test_import { - name: "myexports_myjavatests@current", - sdk_member_name: "myjavatests", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - jars: ["java/myjavatests.jar"], - test_config: "java/myjavatests-AndroidTest.xml", -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - java_tests: ["myexports_myjavatests@current"], -} -`), checkAllCopyRules(` .intermediates/myjavatests/linux_glibc_common/javac/myjavatests.jar -> java/myjavatests.jar .intermediates/myjavatests/linux_glibc_common/myjavatests.config -> java/myjavatests-AndroidTest.xml @@ -735,7 +471,19 @@ module_exports_snapshot { } func TestSnapshotWithJavaSystemModules(t *testing.T) { - result := android.GroupFixturePreparers(prepareForSdkTestWithJavaSdkLibrary).RunTestWithBp(t, ` + result := android.GroupFixturePreparers( + prepareForSdkTestWithJava, + java.PrepareForTestWithJavaDefaultModules, + java.PrepareForTestWithJavaSdkLibraryFiles, + java.FixtureWithPrebuiltApisAndExtensions(map[string][]string{ + "31": {"myjavalib"}, + "32": {"myjavalib"}, + "current": {"myjavalib"}, + }, map[string][]string{ + "1": {"myjavalib"}, + "2": {"myjavalib"}, + }), + ).RunTestWithBp(t, ` sdk { name: "mysdk", java_header_libs: ["exported-system-module"], @@ -775,7 +523,7 @@ func TestSnapshotWithJavaSystemModules(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -820,59 +568,6 @@ java_system_modules_import { ], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_exported-system-module@current", - sdk_member_name: "exported-system-module", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/exported-system-module.jar"], -} - -java_import { - name: "mysdk_system-module@current", - sdk_member_name: "system-module", - visibility: ["//visibility:private"], - apex_available: ["//apex_available:platform"], - jars: ["java/system-module.jar"], -} - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - shared_library: false, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, -} - -java_system_modules_import { - name: "mysdk_my-system-modules@current", - sdk_member_name: "my-system-modules", - visibility: ["//visibility:public"], - libs: [ - "mysdk_system-module@current", - "mysdk_exported-system-module@current", - "mysdk_myjavalib.stubs@current", - ], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_exported-system-module@current"], - java_sdk_libs: ["mysdk_myjavalib@current"], - java_system_modules: ["mysdk_my-system-modules@current"], -} -`), checkAllCopyRules(` .intermediates/exported-system-module/android_common/turbine-combined/exported-system-module.jar -> java/exported-system-module.jar .intermediates/system-module/android_common/turbine-combined/system-module.jar -> java/system-module.jar @@ -880,6 +575,53 @@ sdk_snapshot { .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_removed.txt -> sdk_library/public/myjavalib-removed.txt `), + checkInfoContents(result.Config, ` +[ + { + "@type": "sdk", + "@name": "mysdk", + "java_header_libs": [ + "exported-system-module", + "system-module" + ], + "java_sdk_libs": [ + "myjavalib" + ], + "java_system_modules": [ + "my-system-modules" + ] + }, + { + "@type": "java_library", + "@name": "exported-system-module" + }, + { + "@type": "java_system_modules", + "@name": "my-system-modules", + "@deps": [ + "exported-system-module", + "system-module" + ] + }, + { + "@type": "java_sdk_library", + "@name": "myjavalib", + "dist_stem": "myjavalib", + "scopes": { + "public": { + "current_api": "sdk_library/public/myjavalib.txt", + "latest_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib.api.public.latest/gen/myjavalib.api.public.latest", + "latest_removed_api": "out/soong/.intermediates/prebuilts/sdk/myjavalib-removed.api.public.latest/gen/myjavalib-removed.api.public.latest", + "removed_api": "sdk_library/public/myjavalib-removed.txt" + } + } + }, + { + "@type": "java_library", + "@name": "system-module" + } +] +`), ) } @@ -910,7 +652,7 @@ func TestHostSnapshotWithJavaSystemModules(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -932,36 +674,6 @@ java_system_modules_import { libs: ["mysdk_system-module"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_system-module@current", - sdk_member_name: "system-module", - visibility: ["//visibility:private"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - jars: ["java/system-module.jar"], -} - -java_system_modules_import { - name: "mysdk_my-system-modules@current", - sdk_member_name: "my-system-modules", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - libs: ["mysdk_system-module@current"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - device_supported: false, - host_supported: true, - java_system_modules: ["mysdk_my-system-modules@current"], -} -`), checkAllCopyRules(".intermediates/system-module/linux_glibc_common/javac/system-module.jar -> java/system-module.jar"), ) } @@ -1004,7 +716,7 @@ func TestDeviceAndHostSnapshotWithOsSpecificMembers(t *testing.T) { `) CheckSnapshot(t, result, "myexports", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -1041,58 +753,6 @@ java_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myexports_hostjavalib@current", - sdk_member_name: "hostjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - device_supported: false, - host_supported: true, - jars: ["java/hostjavalib.jar"], -} - -java_import { - name: "myexports_androidjavalib@current", - sdk_member_name: "androidjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/androidjavalib.jar"], -} - -java_import { - name: "myexports_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - host_supported: true, - target: { - android: { - jars: ["java/android/myjavalib.jar"], - }, - linux_glibc: { - jars: ["java/linux_glibc/myjavalib.jar"], - }, - }, -} - -module_exports_snapshot { - name: "myexports@current", - visibility: ["//visibility:public"], - host_supported: true, - java_libs: ["myexports_myjavalib@current"], - target: { - android: { - java_header_libs: ["myexports_androidjavalib@current"], - }, - linux_glibc: { - java_header_libs: ["myexports_hostjavalib@current"], - }, - }, -} -`), checkAllCopyRules(` .intermediates/hostjavalib/linux_glibc_common/javac/hostjavalib.jar -> java/hostjavalib.jar .intermediates/androidjavalib/android_common/turbine-combined/androidjavalib.jar -> java/androidjavalib.jar @@ -1122,7 +782,7 @@ func TestSnapshotWithJavaSdkLibrary(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1155,45 +815,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - shared_library: false, - permitted_packages: ["pkg.myjavalib"], - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, - system: { - jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], - current_api: "sdk_library/system/myjavalib.txt", - removed_api: "sdk_library/system/myjavalib-removed.txt", - sdk_version: "system_current", - }, - test: { - jars: ["sdk_library/test/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/test/myjavalib_stub_sources"], - current_api: "sdk_library/test/myjavalib.txt", - removed_api: "sdk_library/test/myjavalib-removed.txt", - sdk_version: "test_current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1210,12 +831,6 @@ sdk_snapshot { ".intermediates/mysdk/common_os/tmp/sdk_library/system/myjavalib_stub_sources.zip", ".intermediates/mysdk/common_os/tmp/sdk_library/test/myjavalib_stub_sources.zip", ), - snapshotTestChecker(checkSnapshotWithoutSource, func(t *testing.T, result *android.TestResult) { - // Make sure that the name of the child modules created by a versioned java_sdk_library_import - // module is correct, i.e. the suffix is added before the version and not after. - result.Module("mysdk_myjavalib.stubs@current", "android_common") - result.Module("mysdk_myjavalib.stubs.source@current", "android_common") - }), ) } @@ -1243,7 +858,7 @@ func TestSnapshotWithJavaSdkLibrary_UseSrcJar(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1290,7 +905,7 @@ func TestSnapshotWithJavaSdkLibrary_AnnotationsZip(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1344,7 +959,7 @@ func TestSnapshotWithJavaSdkLibrary_AnnotationsZip_PreT(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1394,7 +1009,7 @@ func TestSnapshotWithJavaSdkLibrary_CompileDex(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1461,7 +1076,7 @@ func TestSnapshotWithJavaSdkLibrary_SdkVersion_None(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1479,30 +1094,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "none", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1533,7 +1124,7 @@ func TestSnapshotWithJavaSdkLibrary_SdkVersion_ForScope(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1551,30 +1142,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "module_current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1608,7 +1175,7 @@ func TestSnapshotWithJavaSdkLibrary_ApiScopes(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1633,37 +1200,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, - system: { - jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], - current_api: "sdk_library/system/myjavalib.txt", - removed_api: "sdk_library/system/myjavalib-removed.txt", - sdk_version: "system_current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1704,7 +1240,7 @@ func TestSnapshotWithJavaSdkLibrary_ModuleLib(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1736,44 +1272,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, - system: { - jars: ["sdk_library/system/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system/myjavalib_stub_sources"], - current_api: "sdk_library/system/myjavalib.txt", - removed_api: "sdk_library/system/myjavalib-removed.txt", - sdk_version: "system_current", - }, - module_lib: { - jars: ["sdk_library/module-lib/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/module-lib/myjavalib_stub_sources"], - current_api: "sdk_library/module-lib/myjavalib.txt", - removed_api: "sdk_library/module-lib/myjavalib-removed.txt", - sdk_version: "module_current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1815,7 +1313,7 @@ func TestSnapshotWithJavaSdkLibrary_SystemServer(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1840,37 +1338,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, - system_server: { - jars: ["sdk_library/system-server/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/system-server/myjavalib_stub_sources"], - current_api: "sdk_library/system-server/myjavalib.txt", - removed_api: "sdk_library/system-server/myjavalib-removed.txt", - sdk_version: "system_server_current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1906,7 +1373,7 @@ func TestSnapshotWithJavaSdkLibrary_NamingScheme(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -1925,31 +1392,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:anyapex"], - naming_scheme: "default", - shared_library: true, - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt @@ -1988,7 +1430,7 @@ func TestSnapshotWithJavaSdkLibrary_DoctagFiles(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -2007,31 +1449,6 @@ java_sdk_library_import { }, } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - shared_library: true, - doctag_files: ["doctags/docs/known_doctags"], - public: { - jars: ["sdk_library/public/myjavalib-stubs.jar"], - stub_srcs: ["sdk_library/public/myjavalib_stub_sources"], - current_api: "sdk_library/public/myjavalib.txt", - removed_api: "sdk_library/public/myjavalib-removed.txt", - sdk_version: "current", - }, -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_myjavalib@current"], -} -`), checkAllCopyRules(` .intermediates/myjavalib.stubs/android_common/javac/myjavalib.stubs.jar -> sdk_library/public/myjavalib-stubs.jar .intermediates/myjavalib.stubs.source/android_common/metalava/myjavalib.stubs.source_api.txt -> sdk_library/public/myjavalib.txt diff --git a/sdk/license_sdk_test.go b/sdk/license_sdk_test.go index 1ef6fe684..829edf117 100644 --- a/sdk/license_sdk_test.go +++ b/sdk/license_sdk_test.go @@ -60,7 +60,7 @@ func TestSnapshotWithPackageDefaultLicense(t *testing.T) { `) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. package { @@ -91,44 +91,6 @@ license { ], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -package { - // A default list here prevents the license LSC from adding its own list which would - // be unnecessary as every module in the sdk already has its own licenses property. - default_applicable_licenses: ["Android-Apache-2.0"], -} - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - licenses: ["mysdk_mylicense@current"], - jars: ["java/myjavalib.jar"], -} - -license { - name: "mysdk_mylicense@current", - sdk_member_name: "mylicense", - visibility: ["//visibility:private"], - license_kinds: [ - "SPDX-license-identifier-Apache-2.0", - "legacy_unencumbered", - ], - license_text: [ - "licenses/NOTICE1", - "licenses/NOTICE2", - ], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} - `), checkAllCopyRules(` .intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/myjavalib.jar NOTICE1 -> licenses/NOTICE1 diff --git a/sdk/member_trait_test.go b/sdk/member_trait_test.go index a3db189b2..99caf13e3 100644 --- a/sdk/member_trait_test.go +++ b/sdk/member_trait_test.go @@ -134,7 +134,7 @@ func TestBasicTrait_WithoutTrait(t *testing.T) { ).RunTest(t) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_import { @@ -145,23 +145,6 @@ java_import { jars: ["javalibs/myjavalib.jar"], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["javalibs/myjavalib.jar"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - fake_members: ["mysdk_myjavalib@current"], -} -`), ) } @@ -216,7 +199,7 @@ func TestBasicTrait_MultipleTraits(t *testing.T) { ).RunTest(t) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_test_import { diff --git a/sdk/sdk.go b/sdk/sdk.go index 84c9a96e4..aeeedb428 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -39,7 +39,6 @@ func registerSdkBuildComponents(ctx android.RegistrationContext) { ctx.RegisterModuleType("sdk", SdkModuleFactory) ctx.RegisterModuleType("sdk_snapshot", SnapshotModuleFactory) ctx.PreDepsMutators(RegisterPreDepsMutators) - ctx.PostDepsMutators(RegisterPostDepsMutators) } type sdk struct { @@ -76,6 +75,8 @@ type sdk struct { snapshotFile android.OptionalPath + infoFile android.OptionalPath + // The builder, preserved for testing. builderForTests *snapshotBuilder } @@ -145,7 +146,7 @@ func newSdkModule(moduleExports bool) *sdk { return s } -// sdk_snapshot is a versioned snapshot of an SDK. This is an auto-generated module. +// sdk_snapshot is a snapshot of an SDK. This is an auto-generated module. func SnapshotModuleFactory() android.Module { s := newSdkModule(false) s.properties.Snapshot = true @@ -192,27 +193,32 @@ func (s *sdk) GenerateAndroidBuildActions(ctx android.ModuleContext) { } // Generate the snapshot from the member info. - p := s.buildSnapshot(ctx, sdkVariants) - zip := ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), p.Base(), p) - s.snapshotFile = android.OptionalPathForPath(zip) + s.buildSnapshot(ctx, sdkVariants) } } func (s *sdk) AndroidMkEntries() []android.AndroidMkEntries { - if !s.snapshotFile.Valid() { + if !s.snapshotFile.Valid() != !s.infoFile.Valid() { + panic("Snapshot (%q) and info file (%q) should both be set or neither should be set.") + } else if !s.snapshotFile.Valid() { return []android.AndroidMkEntries{} } return []android.AndroidMkEntries{android.AndroidMkEntries{ Class: "FAKE", OutputFile: s.snapshotFile, - DistFiles: android.MakeDefaultDistFiles(s.snapshotFile.Path()), + DistFiles: android.MakeDefaultDistFiles(s.snapshotFile.Path(), s.infoFile.Path()), Include: "$(BUILD_PHONY_PACKAGE)", ExtraFooters: []android.AndroidMkExtraFootersFunc{ func(w io.Writer, name, prefix, moduleDir string) { // Allow the sdk to be built by simply passing its name on the command line. fmt.Fprintln(w, ".PHONY:", s.Name()) fmt.Fprintln(w, s.Name()+":", s.snapshotFile.String()) + + // Allow the sdk info to be built by simply passing its name on the command line. + infoTarget := s.Name() + ".info" + fmt.Fprintln(w, ".PHONY:", infoTarget) + fmt.Fprintln(w, infoTarget+":", s.infoFile.String()) }, }, }} @@ -275,21 +281,6 @@ var _ android.SdkDependencyContext = (*dependencyContext)(nil) func RegisterPreDepsMutators(ctx android.RegisterMutatorsContext) { ctx.BottomUp("SdkMember", memberMutator).Parallel() ctx.TopDown("SdkMember_deps", memberDepsMutator).Parallel() - ctx.BottomUp("SdkMemberInterVersion", memberInterVersionMutator).Parallel() -} - -// RegisterPostDepsMutators registers post-deps mutators to support modules implementing SdkAware -// interface and the sdk module type. This function has been made public to be called by tests -// outside of the sdk package -func RegisterPostDepsMutators(ctx android.RegisterMutatorsContext) { - // These must run AFTER apexMutator. Note that the apex package is imported even though there is - // no direct dependency to the package here. sdkDepsMutator sets the SDK requirements from an - // APEX to its dependents. Since different versions of the same SDK can be used by different - // APEXes, the apex and its dependents (which includes the dependencies to the sdk members) - // should have been mutated for the apex before the SDK requirements are set. - ctx.TopDown("SdkDepsMutator", sdkDepsMutator).Parallel() - ctx.BottomUp("SdkDepsReplaceMutator", sdkDepsReplaceMutator).Parallel() - ctx.TopDown("SdkRequirementCheck", sdkRequirementsMutator).Parallel() } type dependencyTag struct { @@ -301,38 +292,6 @@ func (t dependencyTag) ExcludeFromApexContents() {} var _ android.ExcludeFromApexContentsTag = dependencyTag{} -// For dependencies from an in-development version of an SDK member to frozen versions of the same member -// e.g. libfoo -> libfoo.mysdk.11 and libfoo.mysdk.12 -// -// The dependency represented by this tag requires that for every APEX variant created for the -// `from` module that an equivalent APEX variant is created for the 'to' module. This is because an -// APEX that requires a specific version of an sdk (via the `uses_sdks` property will replace -// dependencies on the unversioned sdk member with a dependency on the appropriate versioned sdk -// member. In order for that to work the versioned sdk member needs to have a variant for that APEX. -// As it is not known at the time that the APEX variants are created which specific APEX variants of -// a versioned sdk members will be required it is necessary for the versioned sdk members to have -// variants for any APEX that it could be used within. -// -// If the APEX selects a versioned sdk member then it will not have a dependency on the `from` -// module at all so any dependencies of that module will not affect the APEX. However, if the APEX -// selects the unversioned sdk member then it must exclude all the versioned sdk members. In no -// situation would this dependency cause the `to` module to be added to the APEX hence why this tag -// also excludes the `to` module from being added to the APEX contents. -type sdkMemberVersionedDepTag struct { - dependencyTag - member string - version string -} - -func (t sdkMemberVersionedDepTag) AlwaysRequireApexVariant() bool { - return true -} - -// Mark this tag so dependencies that use it are excluded from visibility enforcement. -func (t sdkMemberVersionedDepTag) ExcludeFromVisibilityEnforcement() {} - -var _ android.AlwaysRequireApexVariantTag = sdkMemberVersionedDepTag{} - // Step 1: create dependencies from an SDK module to its members. func memberMutator(mctx android.BottomUpMutatorContext) { if s, ok := mctx.Module().(*sdk); ok { @@ -391,125 +350,10 @@ func memberDepsMutator(mctx android.TopDownMutatorContext) { } } -// Step 3: create dependencies from the unversioned SDK member to snapshot versions -// of the same member. By having these dependencies, they are mutated for multiple Mainline modules -// (apex and apk), each of which might want different sdks to be built with. For example, if both -// apex A and B are referencing libfoo which is a member of sdk 'mysdk', the two APEXes can be -// built with libfoo.mysdk.11 and libfoo.mysdk.12, respectively depending on which sdk they are -// using. -func memberInterVersionMutator(mctx android.BottomUpMutatorContext) { - if m, ok := mctx.Module().(android.SdkAware); ok && m.IsInAnySdk() && m.IsVersioned() { - if !m.ContainingSdk().Unversioned() { - memberName := m.MemberName() - tag := sdkMemberVersionedDepTag{member: memberName, version: m.ContainingSdk().Version} - mctx.AddReverseDependency(mctx.Module(), tag, memberName) - } - } -} - // An interface that encapsulates all the functionality needed to manage the sdk dependencies. // // It is a mixture of apex and sdk module functionality. type sdkAndApexModule interface { android.Module android.DepIsInSameApex - android.RequiredSdks -} - -// Step 4: transitively ripple down the SDK requirements from the root modules like APEX to its -// descendants -func sdkDepsMutator(mctx android.TopDownMutatorContext) { - if parent, ok := mctx.Module().(sdkAndApexModule); ok { - // Module types for Mainline modules (e.g. APEX) are expected to implement RequiredSdks() - // by reading its own properties like `uses_sdks`. - requiredSdks := parent.RequiredSdks() - if len(requiredSdks) > 0 { - mctx.VisitDirectDeps(func(m android.Module) { - // Only propagate required sdks from the apex onto its contents. - if dep, ok := m.(android.SdkAware); ok && android.IsDepInSameApex(mctx, parent, dep) { - dep.BuildWithSdks(requiredSdks) - } - }) - } - } -} - -// Step 5: if libfoo.mysdk.11 is in the context where version 11 of mysdk is requested, the -// versioned module is used instead of the un-versioned (in-development) module libfoo -func sdkDepsReplaceMutator(mctx android.BottomUpMutatorContext) { - if versionedSdkMember, ok := mctx.Module().(android.SdkAware); ok && versionedSdkMember.IsInAnySdk() && versionedSdkMember.IsVersioned() { - if sdk := versionedSdkMember.ContainingSdk(); !sdk.Unversioned() { - // Only replace dependencies to <sdkmember> with <sdkmember@required-version> - // if the depending module requires it. e.g. - // foo -> sdkmember - // will be transformed to: - // foo -> sdkmember@1 - // if and only if foo is a member of an APEX that requires version 1 of the - // sdk containing sdkmember. - memberName := versionedSdkMember.MemberName() - - // Convert a panic into a normal error to allow it to be more easily tested for. This is a - // temporary workaround, once http://b/183204176 has been fixed this can be removed. - // TODO(b/183204176): Remove this after fixing. - defer func() { - if r := recover(); r != nil { - mctx.ModuleErrorf("sdkDepsReplaceMutator %s", r) - } - }() - - // Replace dependencies on sdkmember with a dependency on the current module which - // is a versioned prebuilt of the sdkmember if required. - mctx.ReplaceDependenciesIf(memberName, func(from blueprint.Module, tag blueprint.DependencyTag, to blueprint.Module) bool { - // from - foo - // to - sdkmember - replace := false - if parent, ok := from.(android.RequiredSdks); ok { - replace = parent.RequiredSdks().Contains(sdk) - } - return replace - }) - } - } -} - -// Step 6: ensure that the dependencies outside of the APEX are all from the required SDKs -func sdkRequirementsMutator(mctx android.TopDownMutatorContext) { - if m, ok := mctx.Module().(sdkAndApexModule); ok { - requiredSdks := m.RequiredSdks() - if len(requiredSdks) == 0 { - return - } - mctx.VisitDirectDeps(func(dep android.Module) { - tag := mctx.OtherModuleDependencyTag(dep) - if tag == android.DefaultsDepTag { - // dependency to defaults is always okay - return - } - - // Ignore the dependency from the unversioned member to any versioned members as an - // apex that depends on the unversioned member will not also be depending on a versioned - // member. - if _, ok := tag.(sdkMemberVersionedDepTag); ok { - return - } - - // If the dep is outside of the APEX, but is not in any of the required SDKs, we know that the - // dep is a violation. - if sa, ok := dep.(android.SdkAware); ok { - // It is not an error if a dependency that is excluded from the apex due to the tag is not - // in one of the required SDKs. That is because all of the existing tags that implement it - // do not depend on modules which can or should belong to an sdk_snapshot. - if _, ok := tag.(android.ExcludeFromApexContentsTag); ok { - // The tag defines a dependency that never requires the child module to be part of the - // same apex. - return - } - - if !m.DepIsInSameApex(mctx, dep) && !requiredSdks.Contains(sa.ContainingSdk()) { - mctx.ModuleErrorf("depends on %q (in SDK %q) that isn't part of the required SDKs: %v", - sa.Name(), sa.ContainingSdk(), requiredSdks) - } - } - }) - } } diff --git a/sdk/sdk_test.go b/sdk/sdk_test.go index 83294f6ab..1ec12c346 100644 --- a/sdk/sdk_test.go +++ b/sdk/sdk_test.go @@ -37,64 +37,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestDepNotInRequiredSdks(t *testing.T) { - testSdkError(t, `module "myjavalib".*depends on "otherlib".*that isn't part of the required SDKs:.*`, ` - sdk { - name: "mysdk", - java_header_libs: ["sdkmember"], - } - - sdk_snapshot { - name: "mysdk@1", - java_header_libs: ["sdkmember_mysdk_1"], - } - - java_import { - name: "sdkmember", - prefer: false, - host_supported: true, - } - - java_import { - name: "sdkmember_mysdk_1", - sdk_member_name: "sdkmember", - host_supported: true, - } - - java_library { - name: "myjavalib", - srcs: ["Test.java"], - libs: [ - "sdkmember", - "otherlib", - ], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - apex_available: ["myapex"], - } - - // this lib is no in mysdk - java_library { - name: "otherlib", - srcs: ["Test.java"], - system_modules: "none", - sdk_version: "none", - compile_dex: true, - host_supported: true, - } - - apex { - name: "myapex", - java_libs: ["myjavalib"], - uses_sdks: ["mysdk@1"], - key: "myapex.key", - certificate: ":myapex.cert", - } - `) -} - // Ensure that prebuilt modules have the same effective visibility as the source // modules. func TestSnapshotVisibility(t *testing.T) { @@ -177,18 +119,6 @@ func TestSnapshotVisibility(t *testing.T) { // This is auto-generated. DO NOT EDIT. java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: [ - "//other/foo", - "//package", - "//prebuilts/mysdk", - ], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -java_import { name: "myjavalib", prefer: false, visibility: [ @@ -201,14 +131,6 @@ java_import { } java_import { - name: "mysdk_mypublicjavalib@current", - sdk_member_name: "mypublicjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/mypublicjavalib.jar"], -} - -java_import { name: "mypublicjavalib", prefer: false, visibility: ["//visibility:public"], @@ -217,18 +139,6 @@ java_import { } java_import { - name: "mysdk_mydefaultedjavalib@current", - sdk_member_name: "mydefaultedjavalib", - visibility: [ - "//other/bar", - "//package", - "//prebuilts/mysdk", - ], - apex_available: ["//apex_available:platform"], - jars: ["java/mydefaultedjavalib.jar"], -} - -java_import { name: "mydefaultedjavalib", prefer: false, visibility: [ @@ -241,17 +151,6 @@ java_import { } java_import { - name: "mysdk_myprivatejavalib@current", - sdk_member_name: "myprivatejavalib", - visibility: [ - "//package", - "//prebuilts/mysdk", - ], - apex_available: ["//apex_available:platform"], - jars: ["java/myprivatejavalib.jar"], -} - -java_import { name: "myprivatejavalib", prefer: false, visibility: [ @@ -261,20 +160,6 @@ java_import { apex_available: ["//apex_available:platform"], jars: ["java/myprivatejavalib.jar"], } - -sdk_snapshot { - name: "mysdk@current", - visibility: [ - "//other/foo", - "//package:__subpackages__", - ], - java_header_libs: [ - "mysdk_myjavalib@current", - "mysdk_mypublicjavalib@current", - "mysdk_mydefaultedjavalib@current", - "mysdk_myprivatejavalib@current", - ], -} `)) } @@ -321,7 +206,10 @@ func TestSdkInstall(t *testing.T) { result := testSdkWithFs(t, sdk, nil) CheckSnapshot(t, result, "mysdk", "", - checkAllOtherCopyRules(`.intermediates/mysdk/common_os/mysdk-current.zip -> mysdk-current.zip`)) + checkAllOtherCopyRules(` +.intermediates/mysdk/common_os/mysdk-current.info -> mysdk-current.info +.intermediates/mysdk/common_os/mysdk-current.zip -> mysdk-current.zip +`)) } type EmbeddedPropertiesStruct struct { @@ -505,26 +393,12 @@ func TestSnapshot_EnvConfiguration(t *testing.T) { // This is auto-generated. DO NOT EDIT. java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -java_import { name: "myjavalib", prefer: false, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], jars: ["java/myjavalib.jar"], } - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} `), ) }) @@ -544,26 +418,12 @@ sdk_snapshot { // This is auto-generated. DO NOT EDIT. java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -java_import { name: "myjavalib", prefer: true, visibility: ["//visibility:public"], apex_available: ["//apex_available:platform"], jars: ["java/myjavalib.jar"], } - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} `), ) }) @@ -583,14 +443,6 @@ sdk_snapshot { // This is auto-generated. DO NOT EDIT. java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -java_import { name: "myjavalib", prefer: false, use_source_config_var: { @@ -601,113 +453,10 @@ java_import { apex_available: ["//apex_available:platform"], jars: ["java/myjavalib.jar"], } - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} - `), - ) - }) - - t.Run("SOONG_SDK_SNAPSHOT_VERSION=unversioned", func(t *testing.T) { - result := android.GroupFixturePreparers( - preparer, - android.FixtureMergeEnv(map[string]string{ - "SOONG_SDK_SNAPSHOT_VERSION": "unversioned", - }), - ).RunTest(t) - - checkZipFile(t, result, "out/soong/.intermediates/mysdk/common_os/mysdk.zip") - - CheckSnapshot(t, result, "mysdk", "", - checkAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "myjavalib", - prefer: false, - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - `), - ) - }) - - t.Run("SOONG_SDK_SNAPSHOT_VERSION=current", func(t *testing.T) { - result := android.GroupFixturePreparers( - preparer, - android.FixtureMergeEnv(map[string]string{ - "SOONG_SDK_SNAPSHOT_VERSION": "current", - }), - ).RunTest(t) - - checkZipFile(t, result, "out/soong/.intermediates/mysdk/common_os/mysdk-current.zip") - - CheckSnapshot(t, result, "mysdk", "", - checkAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@current", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -java_import { - name: "myjavalib", - prefer: false, - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@current"], -} `), ) }) - t.Run("SOONG_SDK_SNAPSHOT_VERSION=2", func(t *testing.T) { - result := android.GroupFixturePreparers( - preparer, - android.FixtureMergeEnv(map[string]string{ - "SOONG_SDK_SNAPSHOT_VERSION": "2", - }), - ).RunTest(t) - - checkZipFile(t, result, "out/soong/.intermediates/mysdk/common_os/mysdk-2.zip") - - CheckSnapshot(t, result, "mysdk", "", - checkAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_import { - name: "mysdk_myjavalib@2", - sdk_member_name: "myjavalib", - visibility: ["//visibility:public"], - apex_available: ["//apex_available:platform"], - jars: ["java/myjavalib.jar"], -} - -sdk_snapshot { - name: "mysdk@2", - visibility: ["//visibility:public"], - java_header_libs: ["mysdk_myjavalib@2"], -} - `), - // A versioned snapshot cannot be used on its own so add the source back in. - snapshotTestPreparer(checkSnapshotWithoutSource, android.FixtureWithRootAndroidBp(bp)), - ) - }) - t.Run("SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE=S", func(t *testing.T) { result := android.GroupFixturePreparers( prepareForSdkTestWithJava, @@ -724,6 +473,9 @@ sdk_snapshot { name: "mybootclasspathfragment", apex_available: ["myapex"], contents: ["mysdklibrary"], + hidden_api: { + split_packages: ["*"], + }, } java_sdk_library { @@ -740,7 +492,7 @@ sdk_snapshot { ).RunTest(t) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. prebuilt_bootclasspath_fragment { diff --git a/sdk/systemserverclasspath_fragment_sdk_test.go b/sdk/systemserverclasspath_fragment_sdk_test.go index 16e3e7fa9..01692a35b 100644 --- a/sdk/systemserverclasspath_fragment_sdk_test.go +++ b/sdk/systemserverclasspath_fragment_sdk_test.go @@ -83,7 +83,7 @@ func TestSnapshotWithSystemServerClasspathFragment(t *testing.T) { ).RunTest(t) CheckSnapshot(t, result, "mysdk", "", - checkUnversionedAndroidBpContents(` + checkAndroidBpContents(` // This is auto-generated. DO NOT EDIT. java_sdk_library_import { @@ -121,51 +121,5 @@ prebuilt_systemserverclasspath_fragment { ], } `), - checkVersionedAndroidBpContents(` -// This is auto-generated. DO NOT EDIT. - -java_sdk_library_import { - name: "mysdk_mysdklibrary@current", - sdk_member_name: "mysdklibrary", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - shared_library: false, - public: { - jars: ["sdk_library/public/mysdklibrary-stubs.jar"], - stub_srcs: ["sdk_library/public/mysdklibrary_stub_sources"], - current_api: "sdk_library/public/mysdklibrary.txt", - removed_api: "sdk_library/public/mysdklibrary-removed.txt", - sdk_version: "current", - }, -} - -java_import { - name: "mysdk_mylib@current", - sdk_member_name: "mylib", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - jars: ["java_systemserver_libs/snapshot/jars/are/invalid/mylib.jar"], - permitted_packages: ["mylib"], -} - -prebuilt_systemserverclasspath_fragment { - name: "mysdk_mysystemserverclasspathfragment@current", - sdk_member_name: "mysystemserverclasspathfragment", - visibility: ["//visibility:public"], - apex_available: ["myapex"], - contents: [ - "mysdk_mylib@current", - "mysdk_mysdklibrary@current", - ], -} - -sdk_snapshot { - name: "mysdk@current", - visibility: ["//visibility:public"], - java_sdk_libs: ["mysdk_mysdklibrary@current"], - java_systemserver_libs: ["mysdk_mylib@current"], - systemserverclasspath_fragments: ["mysdk_mysystemserverclasspathfragment@current"], -} -`), ) } diff --git a/sdk/testing.go b/sdk/testing.go index 062f2000e..bed11b27d 100644 --- a/sdk/testing.go +++ b/sdk/testing.go @@ -122,28 +122,18 @@ func ensureListContains(t *testing.T, result []string, expected string) { } } -func pathsToStrings(paths android.Paths) []string { - var ret []string - for _, p := range paths { - ret = append(ret, p.String()) - } - return ret -} - // Analyse the sdk build rules to extract information about what it is doing. // // e.g. find the src/dest pairs from each cp command, the various zip files // generated, etc. func getSdkSnapshotBuildInfo(t *testing.T, result *android.TestResult, sdk *sdk) *snapshotBuildInfo { info := &snapshotBuildInfo{ - t: t, - r: result, - version: sdk.builderForTests.version, - androidBpContents: sdk.GetAndroidBpContentsForTests(), - androidUnversionedBpContents: sdk.GetUnversionedAndroidBpContentsForTests(), - androidVersionedBpContents: sdk.GetVersionedAndroidBpContentsForTests(), - snapshotTestCustomizations: map[snapshotTest]*snapshotTestCustomization{}, - targetBuildRelease: sdk.builderForTests.targetBuildRelease, + t: t, + r: result, + androidBpContents: sdk.GetAndroidBpContentsForTests(), + infoContents: sdk.GetInfoContentsForTests(), + snapshotTestCustomizations: map[snapshotTest]*snapshotTestCustomization{}, + targetBuildRelease: sdk.builderForTests.targetBuildRelease, } buildParams := sdk.BuildParamsForTests() @@ -257,10 +247,7 @@ func CheckSnapshot(t *testing.T, result *android.TestResult, name string, dir st if dir != "" { dir = filepath.Clean(dir) + "/" } - suffix := "" - if snapshotBuildInfo.version != soongSdkSnapshotVersionUnversioned { - suffix = "-" + snapshotBuildInfo.version - } + suffix := "-" + soongSdkSnapshotVersionCurrent expectedZipPath := fmt.Sprintf(".intermediates/%s%s/%s/%s%s.zip", dir, name, variant, name, suffix) android.AssertStringEquals(t, "Snapshot zip file in wrong place", expectedZipPath, actual) @@ -344,33 +331,6 @@ func checkAndroidBpContents(expected string) snapshotBuildInfoChecker { } } -// Check that the snapshot's unversioned generated Android.bp is correct. -// -// This func should be used to check the general snapshot generation code. -// -// Both the expected and actual string are both trimmed before comparing. -func checkUnversionedAndroidBpContents(expected string) snapshotBuildInfoChecker { - return func(info *snapshotBuildInfo) { - info.t.Helper() - android.AssertTrimmedStringEquals(info.t, "unversioned Android.bp contents do not match", expected, info.androidUnversionedBpContents) - } -} - -// Check that the snapshot's versioned generated Android.bp is correct. -// -// This func should only be used to check the version specific snapshot generation code, -// i.e. the encoding of version into module names and the generation of the _snapshot module. The -// general snapshot generation code should be checked using the checkUnversionedAndroidBpContents() -// func. -// -// Both the expected and actual string are both trimmed before comparing. -func checkVersionedAndroidBpContents(expected string) snapshotBuildInfoChecker { - return func(info *snapshotBuildInfo) { - info.t.Helper() - android.AssertTrimmedStringEquals(info.t, "versioned Android.bp contents do not match", expected, info.androidVersionedBpContents) - } -} - // Check that the snapshot's copy rules are correct. // // The copy rules are formatted as <src> -> <dest>, one per line and then compared @@ -402,6 +362,17 @@ func checkMergeZips(expected ...string) snapshotBuildInfoChecker { } } +// Check that the snapshot's info contents are ciorrect. +// +// Both the expected and actual string are both trimmed before comparing. +func checkInfoContents(config android.Config, expected string) snapshotBuildInfoChecker { + return func(info *snapshotBuildInfo) { + info.t.Helper() + android.AssertTrimmedStringEquals(info.t, "info contents do not match", + expected, android.StringRelativeToTop(config, info.infoContents)) + } +} + type resultChecker func(t *testing.T, result *android.TestResult) // snapshotTestPreparer registers a preparer that will be used to customize the specified @@ -465,19 +436,11 @@ type snapshotBuildInfo struct { // The result from RunTest() r *android.TestResult - // The version of the generated snapshot. - // - // See snapshotBuilder.version for more information about this field. - version string - // The contents of the generated Android.bp file androidBpContents string - // The contents of the unversioned Android.bp file - androidUnversionedBpContents string - - // The contents of the versioned Android.bp file - androidVersionedBpContents string + // The contents of the info file. + infoContents string // The paths, relative to the snapshot root, of all files and directories copied into the // snapshot. diff --git a/sdk/update.go b/sdk/update.go index 5db604b7c..457828bdd 100644 --- a/sdk/update.go +++ b/sdk/update.go @@ -15,6 +15,8 @@ package sdk import ( + "bytes" + "encoding/json" "fmt" "reflect" "sort" @@ -33,7 +35,7 @@ import ( // ======================================================== // // SOONG_SDK_SNAPSHOT_PREFER -// By default every unversioned module in the generated snapshot has prefer: false. Building it +// By default every module in the generated snapshot has prefer: false. Building it // with SOONG_SDK_SNAPSHOT_PREFER=true will force them to use prefer: true. // // SOONG_SDK_SNAPSHOT_USE_SOURCE_CONFIG_VAR @@ -67,20 +69,6 @@ import ( // maintainable solution has been implemented. // TODO(b/174997203): Remove when no longer necessary. // -// SOONG_SDK_SNAPSHOT_VERSION -// This provides control over the version of the generated snapshot. -// -// SOONG_SDK_SNAPSHOT_VERSION=current will generate unversioned and versioned prebuilts and a -// versioned snapshot module. This is the default behavior. The zip file containing the -// generated snapshot will be <sdk-name>-current.zip. -// -// SOONG_SDK_SNAPSHOT_VERSION=unversioned will generate unversioned prebuilts only and the zip -// file containing the generated snapshot will be <sdk-name>.zip. -// -// SOONG_SDK_SNAPSHOT_VERSION=<number> will generate versioned prebuilts and a versioned -// snapshot module only. The zip file containing the generated snapshot will be -// <sdk-name>-<number>.zip. -// // SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE // This allows the target build release (i.e. the release version of the build within which // the snapshot will be used) of the snapshot to be specified. If unspecified then it defaults @@ -128,8 +116,7 @@ var ( ) const ( - soongSdkSnapshotVersionUnversioned = "unversioned" - soongSdkSnapshotVersionCurrent = "current" + soongSdkSnapshotVersionCurrent = "current" ) type generatedContents struct { @@ -161,13 +148,13 @@ func (gc *generatedContents) Dedent() { // IndentedPrintf will add spaces to indent the line to the appropriate level before printing the // arguments. func (gc *generatedContents) IndentedPrintf(format string, args ...interface{}) { - fmt.Fprintf(&(gc.content), strings.Repeat(" ", gc.indentLevel)+format, args...) + _, _ = fmt.Fprintf(&(gc.content), strings.Repeat(" ", gc.indentLevel)+format, args...) } // UnindentedPrintf does not add spaces to indent the line to the appropriate level before printing // the arguments. func (gc *generatedContents) UnindentedPrintf(format string, args ...interface{}) { - fmt.Fprintf(&(gc.content), format, args...) + _, _ = fmt.Fprintf(&(gc.content), format, args...) } func (gf *generatedFile) build(pctx android.PackageContext, ctx android.BuilderContext, implicits android.Paths) { @@ -219,9 +206,19 @@ func (s *sdk) collectMembers(ctx android.ModuleContext) { exportedComponentsInfo = ctx.OtherModuleProvider(child, android.ExportedComponentsInfoProvider).(android.ExportedComponentsInfo) } + var container android.SdkAware + if parent != ctx.Module() { + container = parent.(android.SdkAware) + } + export := memberTag.ExportMember() s.memberVariantDeps = append(s.memberVariantDeps, sdkMemberVariantDep{ - s, memberType, child.(android.SdkAware), export, exportedComponentsInfo, + sdkVariant: s, + memberType: memberType, + variant: child.(android.SdkAware), + container: container, + export: export, + exportedComponentsInfo: exportedComponentsInfo, }) // Recurse down into the member's dependencies as it may have dependencies that need to be @@ -256,13 +253,19 @@ func (s *sdk) groupMemberVariantsByMemberThenType(ctx android.ModuleContext, mem member = &sdkMember{memberType: memberType, name: name} byName[name] = member byType[memberType] = append(byType[memberType], member) + } else if member.memberType != memberType { + // validate whether this is the same member type or and overriding member type + if memberType.Overrides(member.memberType) { + member.memberType = memberType + } else if !member.memberType.Overrides(memberType) { + ctx.ModuleErrorf("Incompatible member types %q %q", member.memberType, memberType) + } } // Only append new variants to the list. This is needed because a member can be both // exported by the sdk and also be a transitive sdk member. member.variants = appendUniqueVariants(member.variants, variant) } - var members []*sdkMember for _, memberListProperty := range s.memberTypeListProperties() { membersOfType := byType[memberListProperty.memberType] @@ -303,15 +306,9 @@ const BUILD_NUMBER_FILE = "snapshot-creation-build-number.txt" // <arch>/lib/ // libFoo.so : a stub library -// A name that uniquely identifies a prebuilt SDK member for a version of SDK snapshot -// This isn't visible to users, so could be changed in future. -func versionedSdkMemberName(ctx android.ModuleContext, memberName string, version string) string { - return ctx.ModuleName() + "_" + memberName + string(android.SdkVersionSeparator) + version -} - // buildSnapshot is the main function in this source file. It creates rules to copy // the contents (header files, stub libraries, etc) into the zip file. -func (s *sdk) buildSnapshot(ctx android.ModuleContext, sdkVariants []*sdk) android.OutputPath { +func (s *sdk) buildSnapshot(ctx android.ModuleContext, sdkVariants []*sdk) { // Aggregate all the sdkMemberVariantDep instances from all the sdk variants. hasLicenses := false @@ -360,20 +357,9 @@ func (s *sdk) buildSnapshot(ctx android.ModuleContext, sdkVariants []*sdk) andro } config := ctx.Config() - version := config.GetenvWithDefault("SOONG_SDK_SNAPSHOT_VERSION", "current") - // Generate versioned modules in the snapshot unless an unversioned snapshot has been requested. - generateVersioned := version != soongSdkSnapshotVersionUnversioned - - // Generate unversioned modules in the snapshot unless a numbered snapshot has been requested. - // - // Unversioned modules are not required in that case because the numbered version will be a - // finalized version of the snapshot that is intended to be kept separate from the - generateUnversioned := version == soongSdkSnapshotVersionUnversioned || version == soongSdkSnapshotVersionCurrent - snapshotZipFileSuffix := "" - if generateVersioned { - snapshotZipFileSuffix = "-" + version - } + // Always add -current to the end + snapshotFileSuffix := "-current" currentBuildRelease := latestBuildRelease() targetBuildReleaseEnv := config.GetenvWithDefault("SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE", currentBuildRelease.name) @@ -386,7 +372,6 @@ func (s *sdk) buildSnapshot(ctx android.ModuleContext, sdkVariants []*sdk) andro builder := &snapshotBuilder{ ctx: ctx, sdk: s, - version: version, snapshotDir: snapshotDir.OutputPath, copies: make(map[string]string), filesToZip: []android.Path{bp.path}, @@ -436,38 +421,19 @@ be unnecessary as every module in the sdk already has its own licenses property. s.createMemberSnapshot(memberCtx, member, prebuiltModule.(*bpModule)) } - // Create a transformer that will transform an unversioned module into a versioned module. - unversionedToVersionedTransformer := unversionedToVersionedTransformation{builder: builder} - - // Create a transformer that will transform an unversioned module by replacing any references + // Create a transformer that will transform a module by replacing any references // to internal members with a unique module name and setting prefer: false. - unversionedTransformer := unversionedTransformation{ + snapshotTransformer := snapshotTransformation{ builder: builder, } - for _, unversioned := range builder.prebuiltOrder { + for _, module := range builder.prebuiltOrder { // Prune any empty property sets. - unversioned = unversioned.transform(pruneEmptySetTransformer{}) - - if generateVersioned { - // Copy the unversioned module so it can be modified to make it versioned. - versioned := unversioned.deepCopy() - - // Transform the unversioned module into a versioned one. - versioned.transform(unversionedToVersionedTransformer) - bpFile.AddModule(versioned) - } - - if generateUnversioned { - // Transform the unversioned module to make it suitable for use in the snapshot. - unversioned.transform(unversionedTransformer) - bpFile.AddModule(unversioned) - } - } + module = module.transform(pruneEmptySetTransformer{}) - if generateVersioned { - // Add the sdk/module_exports_snapshot module to the bp file. - s.addSnapshotModule(ctx, builder, sdkVariants, memberVariantDeps) + // Transform the module module to make it suitable for use in the snapshot. + module.transform(snapshotTransformer) + bpFile.AddModule(module) } // generate Android.bp @@ -489,7 +455,7 @@ be unnecessary as every module in the sdk already has its own licenses property. filesToZip := builder.filesToZip // zip them all - zipPath := fmt.Sprintf("%s%s.zip", ctx.ModuleName(), snapshotZipFileSuffix) + zipPath := fmt.Sprintf("%s%s.zip", ctx.ModuleName(), snapshotFileSuffix) outputZipFile := android.PathForModuleOut(ctx, zipPath).OutputPath outputDesc := "Building snapshot for " + ctx.ModuleName() @@ -502,7 +468,7 @@ be unnecessary as every module in the sdk already has its own licenses property. zipFile = outputZipFile desc = outputDesc } else { - intermediatePath := fmt.Sprintf("%s%s.unmerged.zip", ctx.ModuleName(), snapshotZipFileSuffix) + intermediatePath := fmt.Sprintf("%s%s.unmerged.zip", ctx.ModuleName(), snapshotFileSuffix) zipFile = android.PathForModuleOut(ctx, intermediatePath).OutputPath desc = "Building intermediate snapshot for " + ctx.ModuleName() } @@ -527,7 +493,125 @@ be unnecessary as every module in the sdk already has its own licenses property. }) } - return outputZipFile + modules := s.generateInfoData(ctx, memberVariantDeps) + + // Output the modules information as pretty printed JSON. + info := newGeneratedFile(ctx, fmt.Sprintf("%s%s.info", ctx.ModuleName(), snapshotFileSuffix)) + output, err := json.MarshalIndent(modules, "", " ") + if err != nil { + ctx.ModuleErrorf("error generating %q: %s", info, err) + } + builder.infoContents = string(output) + info.generatedContents.UnindentedPrintf("%s", output) + info.build(pctx, ctx, nil) + infoPath := info.path + installedInfo := ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), infoPath.Base(), infoPath) + s.infoFile = android.OptionalPathForPath(installedInfo) + + // Install the zip, making sure that the info file has been installed as well. + installedZip := ctx.InstallFile(android.PathForMainlineSdksInstall(ctx), outputZipFile.Base(), outputZipFile, installedInfo) + s.snapshotFile = android.OptionalPathForPath(installedZip) +} + +type moduleInfo struct { + // The type of the module, e.g. java_sdk_library + moduleType string + // The name of the module. + name string + // A list of additional dependencies of the module. + deps []string + // Additional member specific properties. + // These will be added into the generated JSON alongside the above properties. + memberSpecific map[string]interface{} +} + +func (m *moduleInfo) MarshalJSON() ([]byte, error) { + buffer := bytes.Buffer{} + + separator := "" + writeObjectPair := func(key string, value interface{}) { + buffer.WriteString(fmt.Sprintf("%s%q: ", separator, key)) + b, err := json.Marshal(value) + if err != nil { + panic(err) + } + buffer.Write(b) + separator = "," + } + + buffer.WriteString("{") + writeObjectPair("@type", m.moduleType) + writeObjectPair("@name", m.name) + if m.deps != nil { + writeObjectPair("@deps", m.deps) + } + for _, k := range android.SortedStringKeys(m.memberSpecific) { + v := m.memberSpecific[k] + writeObjectPair(k, v) + } + buffer.WriteString("}") + return buffer.Bytes(), nil +} + +var _ json.Marshaler = (*moduleInfo)(nil) + +// generateInfoData creates a list of moduleInfo structures that will be marshalled into JSON. +func (s *sdk) generateInfoData(ctx android.ModuleContext, memberVariantDeps []sdkMemberVariantDep) interface{} { + modules := []*moduleInfo{} + sdkInfo := moduleInfo{ + moduleType: "sdk", + name: ctx.ModuleName(), + memberSpecific: map[string]interface{}{}, + } + modules = append(modules, &sdkInfo) + + name2Info := map[string]*moduleInfo{} + getModuleInfo := func(module android.Module) *moduleInfo { + name := module.Name() + info := name2Info[name] + if info == nil { + moduleType := ctx.OtherModuleType(module) + // Remove any suffix added when creating modules dynamically. + moduleType = strings.Split(moduleType, "__")[0] + info = &moduleInfo{ + moduleType: moduleType, + name: name, + } + + additionalSdkInfo := ctx.OtherModuleProvider(module, android.AdditionalSdkInfoProvider).(android.AdditionalSdkInfo) + info.memberSpecific = additionalSdkInfo.Properties + + name2Info[name] = info + } + return info + } + + for _, memberVariantDep := range memberVariantDeps { + propertyName := memberVariantDep.memberType.SdkPropertyName() + var list []string + if v, ok := sdkInfo.memberSpecific[propertyName]; ok { + list = v.([]string) + } + + memberName := memberVariantDep.variant.Name() + list = append(list, memberName) + sdkInfo.memberSpecific[propertyName] = android.SortedUniqueStrings(list) + + if memberVariantDep.container != nil { + containerInfo := getModuleInfo(memberVariantDep.container) + containerInfo.deps = android.SortedUniqueStrings(append(containerInfo.deps, memberName)) + } + + // Make sure that the module info is created for each module. + getModuleInfo(memberVariantDep.variant) + } + + for _, memberName := range android.SortedStringKeys(name2Info) { + info := name2Info[memberName] + modules = append(modules, info) + } + + return modules } // filterOutComponents removes any item from the deps list that is a component of another item in @@ -536,7 +620,7 @@ be unnecessary as every module in the sdk already has its own licenses property. func filterOutComponents(ctx android.ModuleContext, deps []sdkMemberVariantDep) []sdkMemberVariantDep { // Collate the set of components that all the modules added to the sdk provide. components := map[string]*sdkMemberVariantDep{} - for i, _ := range deps { + for i := range deps { dep := &deps[i] for _, c := range dep.exportedComponentsInfo.Components { components[c] = dep @@ -571,81 +655,6 @@ func filterOutComponents(ctx android.ModuleContext, deps []sdkMemberVariantDep) return filtered } -// addSnapshotModule adds the sdk_snapshot/module_exports_snapshot module to the builder. -func (s *sdk) addSnapshotModule(ctx android.ModuleContext, builder *snapshotBuilder, sdkVariants []*sdk, memberVariantDeps []sdkMemberVariantDep) { - bpFile := builder.bpFile - - snapshotName := ctx.ModuleName() + string(android.SdkVersionSeparator) + builder.version - var snapshotModuleType string - if s.properties.Module_exports { - snapshotModuleType = "module_exports_snapshot" - } else { - snapshotModuleType = "sdk_snapshot" - } - snapshotModule := bpFile.newModule(snapshotModuleType) - snapshotModule.AddProperty("name", snapshotName) - - // Make sure that the snapshot has the same visibility as the sdk. - visibility := android.EffectiveVisibilityRules(ctx, s).Strings() - if len(visibility) != 0 { - snapshotModule.AddProperty("visibility", visibility) - } - - addHostDeviceSupportedProperties(s.ModuleBase.DeviceSupported(), s.ModuleBase.HostSupported(), snapshotModule) - - combinedPropertiesList := s.collateSnapshotModuleInfo(ctx, sdkVariants, memberVariantDeps) - commonCombinedProperties := s.optimizeSnapshotModuleProperties(ctx, combinedPropertiesList) - - s.addSnapshotPropertiesToPropertySet(builder, snapshotModule, commonCombinedProperties) - - targetPropertySet := snapshotModule.AddPropertySet("target") - - // Create a mapping from osType to combined properties. - osTypeToCombinedProperties := map[android.OsType]*combinedSnapshotModuleProperties{} - for _, combined := range combinedPropertiesList { - osTypeToCombinedProperties[combined.sdkVariant.Os()] = combined - } - - // Iterate over the os types in a fixed order. - for _, osType := range s.getPossibleOsTypes() { - if combined, ok := osTypeToCombinedProperties[osType]; ok { - osPropertySet := targetPropertySet.AddPropertySet(osType.Name) - - s.addSnapshotPropertiesToPropertySet(builder, osPropertySet, combined) - } - } - - // If host is supported and any member is host OS dependent then disable host - // by default, so that we can enable each host OS variant explicitly. This - // avoids problems with implicitly enabled OS variants when the snapshot is - // used, which might be different from this run (e.g. different build OS). - if s.HostSupported() { - var supportedHostTargets []string - for _, memberVariantDep := range memberVariantDeps { - if memberVariantDep.memberType.IsHostOsDependent() && memberVariantDep.variant.Target().Os.Class == android.Host { - targetString := memberVariantDep.variant.Target().Os.String() + "_" + memberVariantDep.variant.Target().Arch.ArchType.String() - if !android.InList(targetString, supportedHostTargets) { - supportedHostTargets = append(supportedHostTargets, targetString) - } - } - } - if len(supportedHostTargets) > 0 { - hostPropertySet := targetPropertySet.AddPropertySet("host") - hostPropertySet.AddProperty("enabled", false) - } - // Enable the <os>_<arch> variant explicitly when we've disabled it by default on host. - for _, hostTarget := range supportedHostTargets { - propertySet := targetPropertySet.AddPropertySet(hostTarget) - propertySet.AddProperty("enabled", true) - } - } - - // Prune any empty property sets. - snapshotModule.transform(pruneEmptySetTransformer{}) - - bpFile.AddModule(snapshotModule) -} - // Check the syntax of the generated Android.bp file contents and if they are // invalid then log an error with the contents (tagged with line numbers) and the // errors that were found so that it is easy to see where the problem lies. @@ -782,92 +791,34 @@ func (s *sdk) optimizeSnapshotModuleProperties(ctx android.ModuleContext, list [ } } -func (s *sdk) addSnapshotPropertiesToPropertySet(builder *snapshotBuilder, propertySet android.BpPropertySet, combined *combinedSnapshotModuleProperties) { - staticProperties := combined.staticProperties - multilib := staticProperties.Compile_multilib - if multilib != "" && multilib != "both" { - // Compile_multilib defaults to both so only needs to be set when it's specified and not both. - propertySet.AddProperty("compile_multilib", multilib) - } - - dynamicMemberTypeListProperties := combined.dynamicProperties - for _, memberListProperty := range s.memberTypeListProperties() { - if memberListProperty.getter == nil { - continue - } - names := memberListProperty.getter(dynamicMemberTypeListProperties) - if len(names) > 0 { - propertySet.AddProperty(memberListProperty.propertyName(), builder.versionedSdkMemberNames(names, false)) - } - } -} - type propertyTag struct { name string } var _ android.BpPropertyTag = propertyTag{} -// A BpPropertyTag to add to a property that contains references to other sdk members. +// BpPropertyTag instances to add to a property that contains references to other sdk members. // -// This will cause the references to be rewritten to a versioned reference in the version -// specific instance of a snapshot module. +// These will ensure that the referenced modules are available, if required. var requiredSdkMemberReferencePropertyTag = propertyTag{"requiredSdkMemberReferencePropertyTag"} var optionalSdkMemberReferencePropertyTag = propertyTag{"optionalSdkMemberReferencePropertyTag"} -// A BpPropertyTag that indicates the property should only be present in the versioned -// module. -// -// This will cause the property to be removed from the unversioned instance of a -// snapshot module. -var sdkVersionedOnlyPropertyTag = propertyTag{"sdkVersionedOnlyPropertyTag"} - -type unversionedToVersionedTransformation struct { - identityTransformation - builder *snapshotBuilder -} - -func (t unversionedToVersionedTransformation) transformModule(module *bpModule) *bpModule { - // Use a versioned name for the module but remember the original name for the - // snapshot. - name := module.Name() - module.setProperty("name", t.builder.versionedSdkMemberName(name, true)) - module.insertAfter("name", "sdk_member_name", name) - // Remove the prefer property if present as versioned modules never need marking with prefer. - module.removeProperty("prefer") - // Ditto for use_source_config_var - module.removeProperty("use_source_config_var") - return module -} - -func (t unversionedToVersionedTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { - if tag == requiredSdkMemberReferencePropertyTag || tag == optionalSdkMemberReferencePropertyTag { - required := tag == requiredSdkMemberReferencePropertyTag - return t.builder.versionedSdkMemberNames(value.([]string), required), tag - } else { - return value, tag - } -} - -type unversionedTransformation struct { +type snapshotTransformation struct { identityTransformation builder *snapshotBuilder } -func (t unversionedTransformation) transformModule(module *bpModule) *bpModule { +func (t snapshotTransformation) transformModule(module *bpModule) *bpModule { // If the module is an internal member then use a unique name for it. name := module.Name() - module.setProperty("name", t.builder.unversionedSdkMemberName(name, true)) + module.setProperty("name", t.builder.snapshotSdkMemberName(name, true)) return module } -func (t unversionedTransformation) transformProperty(name string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { +func (t snapshotTransformation) transformProperty(_ string, value interface{}, tag android.BpPropertyTag) (interface{}, android.BpPropertyTag) { if tag == requiredSdkMemberReferencePropertyTag || tag == optionalSdkMemberReferencePropertyTag { required := tag == requiredSdkMemberReferencePropertyTag - return t.builder.unversionedSdkMemberNames(value.([]string), required), tag - } else if tag == sdkVersionedOnlyPropertyTag { - // The property is not allowed in the unversioned module so remove it. - return nil, nil + return t.builder.snapshotSdkMemberNames(value.([]string), required), tag } else { return value, tag } @@ -879,7 +830,7 @@ type pruneEmptySetTransformer struct { var _ bpTransformer = (*pruneEmptySetTransformer)(nil) -func (t pruneEmptySetTransformer) transformPropertySetAfterContents(name string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { +func (t pruneEmptySetTransformer) transformPropertySetAfterContents(_ string, propertySet *bpPropertySet, tag android.BpPropertyTag) (*bpPropertySet, android.BpPropertyTag) { if len(propertySet.properties) == 0 { return nil, nil } else { @@ -888,20 +839,12 @@ func (t pruneEmptySetTransformer) transformPropertySetAfterContents(name string, } func generateBpContents(contents *generatedContents, bpFile *bpFile) { - generateFilteredBpContents(contents, bpFile, func(*bpModule) bool { - return true - }) -} - -func generateFilteredBpContents(contents *generatedContents, bpFile *bpFile, moduleFilter func(module *bpModule) bool) { contents.IndentedPrintf("// This is auto-generated. DO NOT EDIT.\n") for _, bpModule := range bpFile.order { - if moduleFilter(bpModule) { - contents.IndentedPrintf("\n") - contents.IndentedPrintf("%s {\n", bpModule.moduleType) - outputPropertySet(contents, bpModule.bpPropertySet) - contents.IndentedPrintf("}\n") - } + contents.IndentedPrintf("\n") + contents.IndentedPrintf("%s {\n", bpModule.moduleType) + outputPropertySet(contents, bpModule.bpPropertySet) + contents.IndentedPrintf("}\n") } } @@ -1033,36 +976,14 @@ func (s *sdk) GetAndroidBpContentsForTests() string { return contents.content.String() } -func (s *sdk) GetUnversionedAndroidBpContentsForTests() string { - contents := &generatedContents{} - generateFilteredBpContents(contents, s.builderForTests.bpFile, func(module *bpModule) bool { - name := module.Name() - // Include modules that are either unversioned or have no name. - return !strings.Contains(name, "@") - }) - return contents.content.String() -} - -func (s *sdk) GetVersionedAndroidBpContentsForTests() string { - contents := &generatedContents{} - generateFilteredBpContents(contents, s.builderForTests.bpFile, func(module *bpModule) bool { - name := module.Name() - // Include modules that are either versioned or have no name. - return name == "" || strings.Contains(name, "@") - }) - return contents.content.String() +func (s *sdk) GetInfoContentsForTests() string { + return s.builderForTests.infoContents } type snapshotBuilder struct { ctx android.ModuleContext sdk *sdk - // The version of the generated snapshot. - // - // See the documentation of SOONG_SDK_SNAPSHOT_VERSION above for details of the valid values of - // this field. - version string - snapshotDir android.OutputPath bpFile *bpFile @@ -1087,6 +1008,9 @@ type snapshotBuilder struct { // The target build release for which the snapshot is to be generated. targetBuildRelease *buildRelease + + // The contents of the .info file that describes the sdk contents. + infoContents string } func (s *snapshotBuilder) CopyToSnapshot(src android.Path, dest string) { @@ -1213,13 +1137,6 @@ func (s *snapshotBuilder) AddPrebuiltModule(member android.SdkMember, moduleType addHostDeviceSupportedProperties(deviceSupported, hostSupported, m) - // Disable installation in the versioned module of those modules that are ever installable. - if installable, ok := variant.(interface{ EverInstallable() bool }); ok { - if installable.EverInstallable() { - m.AddPropertyWithTag("installable", false, sdkVersionedOnlyPropertyTag) - } - } - s.prebuiltModules[name] = m s.prebuiltOrder = append(s.prebuiltOrder, m) return m @@ -1252,45 +1169,28 @@ func (s *snapshotBuilder) OptionalSdkMemberReferencePropertyTag() android.BpProp return optionalSdkMemberReferencePropertyTag } -// Get a versioned name appropriate for the SDK snapshot version being taken. -func (s *snapshotBuilder) versionedSdkMemberName(unversionedName string, required bool) string { - if _, ok := s.allMembersByName[unversionedName]; !ok { +// Get a name for sdk snapshot member. If the member is private then generate a snapshot specific +// name. As part of the processing this checks to make sure that any required members are part of +// the snapshot. +func (s *snapshotBuilder) snapshotSdkMemberName(name string, required bool) string { + if _, ok := s.allMembersByName[name]; !ok { if required { - s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", unversionedName) + s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", name) } - return unversionedName - } - return versionedSdkMemberName(s.ctx, unversionedName, s.version) -} - -func (s *snapshotBuilder) versionedSdkMemberNames(members []string, required bool) []string { - var references []string = nil - for _, m := range members { - references = append(references, s.versionedSdkMemberName(m, required)) + return name } - return references -} -// Get an internal name unique to the sdk. -func (s *snapshotBuilder) unversionedSdkMemberName(unversionedName string, required bool) string { - if _, ok := s.allMembersByName[unversionedName]; !ok { - if required { - s.ctx.ModuleErrorf("Required member reference %s is not a member of the sdk", unversionedName) - } - return unversionedName - } - - if s.isInternalMember(unversionedName) { - return s.ctx.ModuleName() + "_" + unversionedName + if s.isInternalMember(name) { + return s.ctx.ModuleName() + "_" + name } else { - return unversionedName + return name } } -func (s *snapshotBuilder) unversionedSdkMemberNames(members []string, required bool) []string { +func (s *snapshotBuilder) snapshotSdkMemberNames(members []string, required bool) []string { var references []string = nil for _, m := range members { - references = append(references, s.unversionedSdkMemberName(m, required)) + references = append(references, s.snapshotSdkMemberName(m, required)) } return references } @@ -1322,6 +1222,11 @@ type sdkMemberVariantDep struct { // The variant that is added to the sdk. variant android.SdkAware + // The optional container of this member, i.e. the module that is depended upon by the sdk + // (possibly transitively) and whose dependency on this module is why it was added to the sdk. + // Is nil if this a direct dependency of the sdk. + container android.SdkAware + // True if the member should be exported, i.e. accessible, from outside the sdk. export bool @@ -1641,7 +1546,9 @@ func newArchSpecificInfo(ctx android.SdkMemberContext, archId archId, osType and // added. archInfo.Properties = variantPropertiesFactory() - if len(archVariants) == 1 { + // if there are multiple supported link variants, we want to nest based on linkage even if there + // is only one variant, otherwise, if there is only one variant we can populate based on the arch + if len(archVariants) == 1 && len(ctx.MemberType().SupportedLinkages()) <= 1 { archInfo.Properties.PopulateFromVariant(ctx, archVariants[0]) } else { // Group the variants by image type. @@ -1768,11 +1675,13 @@ func newImageVariantSpecificInfo(ctx android.SdkMemberContext, imageVariant stri // Create the properties into which the image variant specific properties will be added. imageInfo.Properties = variantPropertiesFactory() - if len(imageVariants) == 1 { + // if there are multiple supported link variants, we want to nest even if there is only one + // variant, otherwise, if there is only one variant we can populate based on the image + if len(imageVariants) == 1 && len(ctx.MemberType().SupportedLinkages()) <= 1 { imageInfo.Properties.PopulateFromVariant(ctx, imageVariants[0]) } else { // There is more than one variant for this image variant which must be differentiated by link - // type. + // type. Or there are multiple supported linkages and we need to nest based on link type. for _, linkVariant := range imageVariants { linkType := getLinkType(linkVariant) if linkType == "" { @@ -1816,10 +1725,22 @@ func (imageInfo *imageVariantSpecificInfo) addToPropertySet(ctx *memberContext, addSdkMemberPropertiesToSet(ctx, imageInfo.Properties, propertySet) + usedLinkages := make(map[string]bool, len(imageInfo.linkInfos)) for _, linkInfo := range imageInfo.linkInfos { + usedLinkages[linkInfo.linkType] = true linkInfo.addToPropertySet(ctx, propertySet) } + // If not all supported linkages had existing variants, we need to disable the unsupported variant + if len(imageInfo.linkInfos) < len(ctx.MemberType().SupportedLinkages()) { + for _, l := range ctx.MemberType().SupportedLinkages() { + if _, ok := usedLinkages[l]; !ok { + otherLinkagePropertySet := propertySet.AddPropertySet(l) + otherLinkagePropertySet.AddProperty("enabled", false) + } + } + } + // If this is for a non-core image variant then make sure that the property set does not contain // any properties as providing non-core image variant specific properties for prebuilts is not // currently supported. diff --git a/sh/sh_binary.go b/sh/sh_binary.go index d1beaba41..4de01443d 100644 --- a/sh/sh_binary.go +++ b/sh/sh_binary.go @@ -323,7 +323,7 @@ func (s *ShTest) DepsMutator(ctx android.BottomUpMutatorContext) { ctx.AddFarVariationDependencies(ctx.Target().Variations(), shTestDataBinsTag, s.testProperties.Data_bins...) ctx.AddFarVariationDependencies(append(ctx.Target().Variations(), sharedLibVariations...), shTestDataLibsTag, s.testProperties.Data_libs...) - if (ctx.Target().Os.Class == android.Host || ctx.BazelConversionMode()) && len(ctx.Config().Targets[android.Android]) > 0 { + if ctx.Target().Os.Class == android.Host && len(ctx.Config().Targets[android.Android]) > 0 { deviceVariations := ctx.Config().AndroidFirstDeviceTarget.Variations() ctx.AddFarVariationDependencies(deviceVariations, shTestDataDeviceBinsTag, s.testProperties.Data_device_bins...) ctx.AddFarVariationDependencies(append(deviceVariations, sharedLibVariations...), diff --git a/tests/androidmk_test.sh b/tests/androidmk_test.sh index 331dc7770..d0d382b4f 100755 --- a/tests/androidmk_test.sh +++ b/tests/androidmk_test.sh @@ -5,7 +5,7 @@ set -o pipefail # How to run: bash path-to-script/androidmk_test.sh # Tests of converting license functionality of the androidmk tool REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)" -$REAL_TOP/build/soong/soong_ui.bash --make-mode androidmk +"$REAL_TOP/build/soong/soong_ui.bash" --make-mode androidmk source "$(dirname "$0")/lib.sh" @@ -113,11 +113,14 @@ EOF run_androidmk_test "a/b/c/d/Android.mk" "a/b/c/d/Android.bp" } -run_androidmk_test () { +function run_androidmk_test { export ANDROID_BUILD_TOP="$MOCK_TOP" - - local out=$($REAL_TOP/*/host/*/bin/androidmk "$1") - local expected=$(<"$2") + local -r androidmk=("$REAL_TOP"/*/host/*/bin/androidmk) + if [[ ${#androidmk[@]} -ne 1 ]]; then + fail "Multiple androidmk binaries found: ${androidmk[*]}" + fi + local -r out=$("${androidmk[0]}" "$1") + local -r expected=$(<"$2") if [[ "$out" != "$expected" ]]; then ANDROID_BUILD_TOP="$REAL_TOP" diff --git a/tests/apex_comparison_tests.sh b/tests/apex_comparison_tests.sh new file mode 100755 index 000000000..4b2f795de --- /dev/null +++ b/tests/apex_comparison_tests.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Soong/Bazel integration test for building unbundled apexes in the real source tree. +# +# These tests build artifacts from head and compares their contents. + +if [ ! -e "build/make/core/Makefile" ]; then + echo "$0 must be run from the top of the Android source tree." + exit 1 +fi + +############ +# Test Setup +############ + +OUTPUT_DIR="$(mktemp -d)" +SOONG_OUTPUT_DIR="$OUTPUT_DIR/soong" +BAZEL_OUTPUT_DIR="$OUTPUT_DIR/bazel" + +function cleanup { + # call bazel clean because some bazel outputs don't have w bits. + call_bazel clean + rm -rf "${OUTPUT_DIR}" +} +trap cleanup EXIT + +########### +# Run Soong +########### +export UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true # don't rely on prebuilts +export TARGET_BUILD_APPS="com.android.adbd com.android.tzdata build.bazel.examples.apex.minimal" +packages/modules/common/build/build_unbundled_mainline_module.sh \ + --product module_arm \ + --dist_dir "$SOONG_OUTPUT_DIR" + +###################### +# Run bp2build / Bazel +###################### +build/soong/soong_ui.bash --make-mode BP2BUILD_VERBOSE=1 --skip-soong-tests bp2build + +function call_bazel() { + tools/bazel --output_base="$BAZEL_OUTPUT_DIR" $@ +} +BAZEL_OUT="$(call_bazel info output_path)" + +call_bazel build --config=bp2build --config=ci --config=android_arm \ + //packages/modules/adb/apex:com.android.adbd \ + //system/timezone/apex:com.android.tzdata \ + //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex + +# Build debugfs separately, as it's not a dep of apexer, but needs to be an explicit arg. +call_bazel build --config=bp2build --config=linux_x86_64 //external/e2fsprogs/debugfs +DEBUGFS_PATH="$BAZEL_OUT/linux_x86_64-fastbuild/bin/external/e2fsprogs/debugfs/debugfs" + +function run_deapexer() { + call_bazel run --config=bp2build --config=linux_x86_64 //system/apex/tools:deapexer \ + -- \ + --debugfs_path="$DEBUGFS_PATH" \ + $@ +} + +####### +# Tests +####### + +function compare_deapexer_list() { + local APEX_DIR=$1; shift + local APEX=$1; shift + + # Compare the outputs of `deapexer list`, which lists the contents of the apex filesystem image. + local SOONG_APEX="$SOONG_OUTPUT_DIR/$APEX" + local BAZEL_APEX="$BAZEL_OUT/android_arm-fastbuild/bin/$APEX_DIR/$APEX" + + local SOONG_LIST="$OUTPUT_DIR/soong.list" + local BAZEL_LIST="$OUTPUT_DIR/bazel.list" + + run_deapexer list "$SOONG_APEX" > "$SOONG_LIST" + run_deapexer list "$BAZEL_APEX" > "$BAZEL_LIST" + + if cmp -s "$SOONG_LIST" "$BAZEL_LIST" + then + echo "ok: $APEX" + else + echo "contents of $APEX are different between Soong and Bazel:" + echo + echo expected + echo + cat "$SOONG_LIST" + echo + echo got + echo + cat "$BAZEL_LIST" + exit 1 + fi +} + +compare_deapexer_list packages/modules/adb/apex com.android.adbd.apex +compare_deapexer_list system/timezone/apex com.android.tzdata.apex +compare_deapexer_list build/bazel/examples/apex/minimal build.bazel.examples.apex.minimal.apex diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh index 4f37c2bff..3cdf6aa36 100755 --- a/tests/bp2build_bazel_test.sh +++ b/tests/bp2build_bazel_test.sh @@ -11,10 +11,10 @@ readonly GENERATED_BUILD_FILE_NAME="BUILD.bazel" function test_bp2build_null_build() { setup run_soong bp2build - local output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) + local -r output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong bp2build - local output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) + local -r output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then fail "Output bp2build marker file changed on null build" @@ -36,10 +36,10 @@ EOF touch foo/bar/a.txt foo/bar/b.txt run_soong bp2build - local output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) + local -r output_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker) run_soong bp2build - local output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) + local -r output_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker) if [[ "$output_mtime1" != "$output_mtime2" ]]; then fail "Output bp2build marker file changed on null build" @@ -115,3 +115,83 @@ EOF } test_bp2build_generates_all_buildfiles + +function test_cc_correctness { + setup + create_mock_bazel + + mkdir -p a + cat > a/Android.bp <<EOF +cc_object { + name: "qq", + srcs: ["qq.cc"], + bazel_module: { + bp2build_available: true, + }, + stl: "none", + system_shared_libs: [], +} +EOF + + cat > a/qq.cc <<EOF +#include "qq.h" +int qq() { + return QQ; +} +EOF + + cat > a/qq.h <<EOF +#define QQ 1 +EOF + + run_soong bp2build + + run_bazel build --package_path=out/soong/workspace //a:qq + local -r output_mtime1=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + run_bazel build --package_path=out/soong/workspace //a:qq + local -r output_mtime2=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + if [[ "$output_mtime1" != "$output_mtime2" ]]; then + fail "output changed on null build" + fi + + cat > a/qq.h <<EOF +#define QQ 2 +EOF + + run_bazel build --package_path=out/soong/workspace //a:qq + local -r output_mtime3=$(stat -c "%y" bazel-bin/a/_objs/qq/qq.o) + + if [[ "$output_mtime1" == "$output_mtime3" ]]; then + fail "output not changed when included header changed" + fi +} + +test_cc_correctness + +# Regression test for the following failure during symlink forest creation: +# +# Cannot stat '/tmp/st.rr054/foo/bar/unresolved_symlink': stat /tmp/st.rr054/foo/bar/unresolved_symlink: no such file or directory +# +function test_bp2build_null_build_with_unresolved_symlink_in_source() { + setup + + mkdir -p foo/bar + ln -s /tmp/non-existent foo/bar/unresolved_symlink + cat > foo/bar/Android.bp <<'EOF' +filegroup { + name: "fg", + srcs: ["unresolved_symlink/non-existent-file.txt"], + } +EOF + + run_soong bp2build + + dest=$(readlink -f out/soong/workspace/foo/bar/unresolved_symlink) + if [[ "$dest" != "/tmp/non-existent" ]]; then + fail "expected to plant an unresolved symlink out/soong/workspace/foo/bar/unresolved_symlink that resolves to /tmp/non-existent" + fi +} + +test_bp2build_null_build_with_unresolved_symlink_in_source diff --git a/tests/lib.sh b/tests/lib.sh index 1bb2df915..69ad2013d 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -8,7 +8,7 @@ HARDWIRED_MOCK_TOP= REAL_TOP="$(readlink -f "$(dirname "$0")"/../../..)" -if [[ ! -z "$HARDWIRED_MOCK_TOP" ]]; then +if [[ -n "$HARDWIRED_MOCK_TOP" ]]; then MOCK_TOP="$HARDWIRED_MOCK_TOP" else MOCK_TOP=$(mktemp -t -d st.XXXXX) @@ -36,37 +36,38 @@ function cleanup_mock_top { } function info { - echo -e "\e[92;1m[TEST HARNESS INFO]\e[0m" $* + echo -e "\e[92;1m[TEST HARNESS INFO]\e[0m" "$*" } function fail { - echo -e "\e[91;1mFAILED:\e[0m" $* + echo -e "\e[91;1mFAILED:\e[0m" "$*" exit 1 } -function copy_directory() { +function copy_directory { local dir="$1" - local parent="$(dirname "$dir")" + local -r parent="$(dirname "$dir")" mkdir -p "$MOCK_TOP/$parent" cp -R "$REAL_TOP/$dir" "$MOCK_TOP/$parent" } -function symlink_file() { +function symlink_file { local file="$1" mkdir -p "$MOCK_TOP/$(dirname "$file")" ln -s "$REAL_TOP/$file" "$MOCK_TOP/$file" } -function symlink_directory() { +function symlink_directory { local dir="$1" mkdir -p "$MOCK_TOP/$dir" # We need to symlink the contents of the directory individually instead of # using one symlink for the whole directory because finder.go doesn't follow # symlinks when looking for Android.bp files - for i in $(ls "$REAL_TOP/$dir"); do + for i in "$REAL_TOP/$dir"/*; do + i=$(basename "$i") local target="$MOCK_TOP/$dir/$i" local source="$REAL_TOP/$dir/$i" @@ -85,6 +86,7 @@ function create_mock_soong { copy_directory build/soong copy_directory build/make/tools/rbcrun + symlink_directory prebuilts/sdk symlink_directory prebuilts/go symlink_directory prebuilts/build-tools symlink_directory prebuilts/clang/host @@ -95,7 +97,7 @@ function create_mock_soong { touch "$MOCK_TOP/Android.bp" } -function setup() { +function setup { cleanup_mock_top mkdir -p "$MOCK_TOP" @@ -107,33 +109,42 @@ function setup() { tar xzf "$WARMED_UP_MOCK_TOP" } -function run_soong() { +# shellcheck disable=SC2120 +function run_soong { build/soong/soong_ui.bash --make-mode --skip-ninja --skip-config --soong-only --skip-soong-tests "$@" } -function create_mock_bazel() { +function create_mock_bazel { copy_directory build/bazel + copy_directory build/bazel_common_rules symlink_directory prebuilts/bazel + symlink_directory prebuilts/clang symlink_directory prebuilts/jdk symlink_directory external/bazel-skylib + symlink_directory external/bazelbuild-rules_android symlink_file WORKSPACE symlink_file BUILD symlink_file tools/bazel } -run_bazel() { +function run_bazel { + # Remove the ninja_build output marker file to communicate to buildbot that this is not a regular Ninja build, and its + # output should not be parsed as such. + rm -rf out/ninja_build + tools/bazel "$@" } -run_ninja() { +function run_ninja { build/soong/soong_ui.bash --make-mode --skip-config --soong-only --skip-soong-tests "$@" } -info "Starting Soong integration test suite $(basename $0)" +info "Starting Soong integration test suite $(basename "$0")" info "Mock top: $MOCK_TOP" export ALLOW_MISSING_DEPENDENCIES=true +export ALLOW_BP_UNDER_SYMLINKS=true warmup_mock_top diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 76a918be4..1e0772797 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -9,3 +9,7 @@ TOP="$(readlink -f "$(dirname "$0")"/../../..)" "$TOP/build/soong/tests/bp2build_bazel_test.sh" "$TOP/build/soong/tests/soong_test.sh" "$TOP/build/bazel/ci/rbc_regression_test.sh" aosp_arm64-userdebug + +# The following tests build against the full source tree and don't rely on the +# mock client. +"$TOP/build/soong/tests/apex_comparison_tests.sh" diff --git a/ui/build/build.go b/ui/build/build.go index aadf4af4e..ec42b7004 100644 --- a/ui/build/build.go +++ b/ui/build/build.go @@ -266,6 +266,7 @@ func Build(ctx Context, config Config) { } if config.StartRBE() { + cleanupRBELogsDir(ctx, config) startRBE(ctx, config) defer DumpRBEMetrics(ctx, config, filepath.Join(config.LogsDir(), "rbe_metrics.pb")) } diff --git a/ui/build/config.go b/ui/build/config.go index e271bfca2..887420904 100644 --- a/ui/build/config.go +++ b/ui/build/config.go @@ -19,12 +19,14 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/rand" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" + "syscall" "time" "android/soong/shared" @@ -42,6 +44,15 @@ const ( envConfigFetchTimeout = 10 * time.Second ) +var ( + rbeRandPrefix int +) + +func init() { + rand.Seed(time.Now().UnixNano()) + rbeRandPrefix = rand.Intn(1000) +} + type Config struct{ *configImpl } type configImpl struct { @@ -200,7 +211,7 @@ func loadEnvConfig(ctx Context, config *configImpl) error { } if err := fetchEnvConfig(ctx, config, bc); err != nil { - fmt.Fprintf(os.Stderr, "Failed to fetch config file: %v\n", err) + ctx.Verbosef("Failed to fetch config file: %v\n", err) } configDirs := []string{ @@ -1144,34 +1155,25 @@ func (c *configImpl) StartRBE() bool { return true } -func (c *configImpl) rbeLogDir() string { - for _, f := range []string{"RBE_log_dir", "FLAG_log_dir"} { +func (c *configImpl) rbeProxyLogsDir() string { + for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { if v, ok := c.environ.Get(f); ok { return v } } - if c.Dist() { - return c.LogsDir() - } - return c.OutDir() + buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir()) + return filepath.Join(buildTmpDir, "rbe") } -func (c *configImpl) rbeStatsOutputDir() string { - for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { - if v, ok := c.environ.Get(f); ok { - return v - } - } - return c.rbeLogDir() -} - -func (c *configImpl) rbeLogPath() string { - for _, f := range []string{"RBE_log_path", "FLAG_log_path"} { - if v, ok := c.environ.Get(f); ok { - return v +func (c *configImpl) shouldCleanupRBELogsDir() bool { + // Perform a log directory cleanup only when the log directory + // is auto created by the build rather than user-specified. + for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { + if _, ok := c.environ.Get(f); ok { + return false } } - return fmt.Sprintf("text://%v/reproxy_log.txt", c.rbeLogDir()) + return true } func (c *configImpl) rbeExecRoot() string { @@ -1223,10 +1225,57 @@ func (c *configImpl) rbeAuth() (string, string) { return "RBE_use_application_default_credentials", "true" } +func (c *configImpl) rbeSockAddr(dir string) (string, error) { + maxNameLen := len(syscall.RawSockaddrUnix{}.Path) + base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) + + name := filepath.Join(dir, base) + if len(name) < maxNameLen { + return name, nil + } + + name = filepath.Join("/tmp", base) + if len(name) < maxNameLen { + return name, nil + } + + return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) +} + +// IsGooglerEnvironment returns true if the current build is running +// on a Google developer machine and false otherwise. +func (c *configImpl) IsGooglerEnvironment() bool { + cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" + if v, ok := c.environ.Get(cf); ok { + return v == "googler" + } + return false +} + +// GoogleProdCredsExist determine whether credentials exist on the +// Googler machine to use remote execution. +func (c *configImpl) GoogleProdCredsExist() bool { + if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil { + return false + } + return true +} + +// UseRemoteBuild indicates whether to use a remote build acceleration system +// to speed up the build. func (c *configImpl) UseRemoteBuild() bool { return c.UseGoma() || c.UseRBE() } +// StubbyExists checks whether the stubby binary exists on the machine running +// the build. +func (c *configImpl) StubbyExists() bool { + if _, err := exec.LookPath("stubby"); err != nil { + return false + } + return true +} + // RemoteParallel controls how many remote jobs (i.e., commands which contain // gomacc) are run in parallel. Note the parallelism of all other jobs is // still limited by Parallel() diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go index 285f1569b..f56964c80 100644 --- a/ui/build/dumpvars.go +++ b/ui/build/dumpvars.go @@ -245,8 +245,6 @@ func runMakeProductConfig(ctx Context, config Config) { "BUILD_BROKEN_USES_BUILD_EXECUTABLE", "BUILD_BROKEN_USES_BUILD_FUZZ_TEST", "BUILD_BROKEN_USES_BUILD_HEADER_LIBRARY", - "BUILD_BROKEN_USES_BUILD_HOST_DALVIK_JAVA_LIBRARY", - "BUILD_BROKEN_USES_BUILD_HOST_DALVIK_STATIC_JAVA_LIBRARY", "BUILD_BROKEN_USES_BUILD_HOST_EXECUTABLE", "BUILD_BROKEN_USES_BUILD_HOST_JAVA_LIBRARY", "BUILD_BROKEN_USES_BUILD_HOST_PREBUILT", diff --git a/ui/build/finder.go b/ui/build/finder.go index 262de3de7..4d6ad426f 100644 --- a/ui/build/finder.go +++ b/ui/build/finder.go @@ -64,6 +64,7 @@ func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { cacheParams := finder.CacheParams{ WorkingDirectory: dir, RootDirs: []string{"."}, + FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"), ExcludeDirs: []string{".git", ".repo"}, PruneFiles: pruneFiles, IncludeFiles: []string{ diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go index 831a80f97..b3092eaf1 100644 --- a/ui/build/paths/config.go +++ b/ui/build/paths/config.go @@ -31,18 +31,25 @@ type PathConfig struct { LinuxOnlyPrebuilt bool } +// These binaries can be run from $PATH, nonhermetically. There should be as +// few as possible of these, since this means that the build depends on tools +// that are not shipped in the source tree and whose behavior is therefore +// unpredictable. var Allowed = PathConfig{ Symlink: true, Log: false, Error: false, } +// This tool is specifically disallowed and calling it will result in an +// "executable no found" error. var Forbidden = PathConfig{ Symlink: false, Log: true, Error: true, } +// This tool is allowed, but access to it will be logged. var Log = PathConfig{ Symlink: true, Log: true, @@ -52,13 +59,16 @@ var Log = PathConfig{ // The configuration used if the tool is not listed in the config below. // Currently this will create the symlink, but log and error when it's used. In // the future, I expect the symlink to be removed, and this will be equivalent -// to Forbidden. +// to Forbidden. This applies to every tool not specifically mentioned in the +// configuration. var Missing = PathConfig{ Symlink: true, Log: true, Error: true, } +// This is used for binaries for which we have prebuilt versions, but only for +// Linux. Thus, their execution from $PATH is only allowed on Mac OS. var LinuxOnlyPrebuilt = PathConfig{ Symlink: false, Log: true, @@ -73,6 +83,8 @@ func GetConfig(name string) PathConfig { return Missing } +// This list specifies whether a particular binary from $PATH is allowed to be +// run during the build. For more documentation, see path_interposer.go . var Configuration = map[string]PathConfig{ "bash": Allowed, "dd": Allowed, diff --git a/ui/build/rbe.go b/ui/build/rbe.go index 8f9a69991..82fc15f91 100644 --- a/ui/build/rbe.go +++ b/ui/build/rbe.go @@ -16,12 +16,10 @@ package build import ( "fmt" - "math/rand" "os" "path/filepath" "runtime" - "syscall" - "time" + "strings" "android/soong/ui/metrics" ) @@ -54,34 +52,16 @@ func rbeCommand(ctx Context, config Config, rbeCmd string) string { return cmdPath } -func sockAddr(dir string) (string, error) { - maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - rand.Seed(time.Now().UnixNano()) - base := fmt.Sprintf("reproxy_%v.sock", rand.Intn(1000)) - - name := filepath.Join(dir, base) - if len(name) < maxNameLen { - return name, nil - } - - name = filepath.Join("/tmp", base) - if len(name) < maxNameLen { - return name, nil - } - - return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) -} - func getRBEVars(ctx Context, config Config) map[string]string { vars := map[string]string{ - "RBE_log_path": config.rbeLogPath(), - "RBE_log_dir": config.rbeLogDir(), + "RBE_log_dir": config.rbeProxyLogsDir(), "RBE_re_proxy": config.rbeReproxy(), "RBE_exec_root": config.rbeExecRoot(), - "RBE_output_dir": config.rbeStatsOutputDir(), + "RBE_output_dir": config.rbeProxyLogsDir(), + "RBE_proxy_log_dir": config.rbeProxyLogsDir(), } if config.StartRBE() { - name, err := sockAddr(absPath(ctx, config.TempDir())) + name, err := config.rbeSockAddr(absPath(ctx, config.TempDir())) if err != nil { ctx.Fatalf("Error retrieving socket address: %v", err) return nil @@ -100,6 +80,17 @@ func getRBEVars(ctx Context, config Config) map[string]string { return vars } +func cleanupRBELogsDir(ctx Context, config Config) { + if !config.shouldCleanupRBELogsDir() { + return + } + + rbeTmpDir := config.rbeProxyLogsDir() + if err := os.RemoveAll(rbeTmpDir); err != nil { + fmt.Fprintln(ctx.Writer, "\033[33mUnable to remove RBE log directory: ", err, "\033[0m") + } +} + func startRBE(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap") defer ctx.EndTrace() @@ -110,6 +101,11 @@ func startRBE(ctx Context, config Config) { if n := ulimitOrFatal(ctx, config, "-n"); n < rbeLeastNFiles { ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles) } + if _, err := os.Stat(config.rbeProxyLogsDir()); os.IsNotExist(err) { + if err := os.MkdirAll(config.rbeProxyLogsDir(), 0744); err != nil { + ctx.Fatalf("Unable to create logs dir (%v) for RBE: %v", config.rbeProxyLogsDir, err) + } + } cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd)) @@ -131,6 +127,34 @@ func stopRBE(ctx Context, config Config) { } } +func prodCredsAuthType(config Config) bool { + authVar, val := config.rbeAuth() + if strings.Contains(authVar, "use_google_prod_creds") && val != "" && val != "false" { + return true + } + return false +} + +// Check whether proper auth exists for RBE builds run within a +// Google dev environment. +func CheckProdCreds(ctx Context, config Config) { + if !config.IsGooglerEnvironment() { + return + } + if !config.StubbyExists() && prodCredsAuthType(config) { + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, fmt.Sprintf("\033[33mWARNING: %q binary not found in $PATH, follow go/build-fast#opting-out-of-loas-credentials instead for authenticating with RBE.\033[0m", "stubby")) + fmt.Fprintln(ctx.Writer, "") + return + } + if config.GoogleProdCredsExist() { + return + } + fmt.Fprintln(ctx.Writer, "") + fmt.Fprintln(ctx.Writer, "\033[33mWARNING: Missing LOAS credentials, please run `gcert`. This will result in failing builds in the future, see go/rbe-android-default-announcement.\033[0m") + fmt.Fprintln(ctx.Writer, "") +} + // DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics. // The protobuf file is created if RBE is enabled and the proxy service has // started. The proxy service is shutdown in order to dump the RBE metrics to the @@ -151,7 +175,7 @@ func DumpRBEMetrics(ctx Context, config Config, filename string) { return } - outputDir := config.rbeStatsOutputDir() + outputDir := config.rbeProxyLogsDir() if outputDir == "" { ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.") } diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go index 8ff96bcbb..266f76b35 100644 --- a/ui/build/rbe_test.go +++ b/ui/build/rbe_test.go @@ -56,7 +56,8 @@ func TestDumpRBEMetrics(t *testing.T) { env := Environment(tt.env) env.Set("OUT_DIR", tmpDir) env.Set("RBE_DIR", tmpDir) - env.Set("RBE_output_dir", t.TempDir()) + env.Set("RBE_output_dir", tmpDir) + env.Set("RBE_proxy_log_dir", tmpDir) config := Config{&configImpl{ environ: &env, }} diff --git a/ui/metrics/metrics_proto/metrics.pb.go b/ui/metrics/metrics_proto/metrics.pb.go index 69f5689b9..4bc713b84 100644 --- a/ui/metrics/metrics_proto/metrics.pb.go +++ b/ui/metrics/metrics_proto/metrics.pb.go @@ -14,7 +14,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 +// protoc-gen-go v1.28.0 // protoc v3.9.1 // source: metrics.proto @@ -954,9 +954,9 @@ type ModuleTypeInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The build system, eg. Soong or Make. + // The build system, e.g. Soong or Make. BuildSystem *ModuleTypeInfo_BuildSystem `protobuf:"varint,1,opt,name=build_system,json=buildSystem,enum=soong_build_metrics.ModuleTypeInfo_BuildSystem,def=0" json:"build_system,omitempty"` - // The module type, eg. java_library, cc_binary, and etc. + // The module type, e.g. java_library, cc_binary, and etc. ModuleType *string `protobuf:"bytes,2,opt,name=module_type,json=moduleType" json:"module_type,omitempty"` // The number of logical modules. NumOfModules *uint32 `protobuf:"varint,3,opt,name=num_of_modules,json=numOfModules" json:"num_of_modules,omitempty"` @@ -1142,6 +1142,8 @@ type SoongBuildMetrics struct { MaxHeapSize *uint64 `protobuf:"varint,5,opt,name=max_heap_size,json=maxHeapSize" json:"max_heap_size,omitempty"` // Runtime metrics for soong_build execution. Events []*PerfInfo `protobuf:"bytes,6,rep,name=events" json:"events,omitempty"` + // Mixed Builds information + MixedBuildsInfo *MixedBuildsInfo `protobuf:"bytes,7,opt,name=mixed_builds_info,json=mixedBuildsInfo" json:"mixed_builds_info,omitempty"` } func (x *SoongBuildMetrics) Reset() { @@ -1218,6 +1220,13 @@ func (x *SoongBuildMetrics) GetEvents() []*PerfInfo { return nil } +func (x *SoongBuildMetrics) GetMixedBuildsInfo() *MixedBuildsInfo { + if x != nil { + return x.MixedBuildsInfo + } + return nil +} + type ExpConfigFetcher struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1287,6 +1296,63 @@ func (x *ExpConfigFetcher) GetMicros() uint64 { return 0 } +type MixedBuildsInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Modules that are enabled for Mixed Builds. + MixedBuildEnabledModules []string `protobuf:"bytes,1,rep,name=mixed_build_enabled_modules,json=mixedBuildEnabledModules" json:"mixed_build_enabled_modules,omitempty"` + // Modules that are not enabled for MixedBuilds + MixedBuildDisabledModules []string `protobuf:"bytes,2,rep,name=mixed_build_disabled_modules,json=mixedBuildDisabledModules" json:"mixed_build_disabled_modules,omitempty"` +} + +func (x *MixedBuildsInfo) Reset() { + *x = MixedBuildsInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_metrics_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MixedBuildsInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MixedBuildsInfo) ProtoMessage() {} + +func (x *MixedBuildsInfo) ProtoReflect() protoreflect.Message { + mi := &file_metrics_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MixedBuildsInfo.ProtoReflect.Descriptor instead. +func (*MixedBuildsInfo) Descriptor() ([]byte, []int) { + return file_metrics_proto_rawDescGZIP(), []int{10} +} + +func (x *MixedBuildsInfo) GetMixedBuildEnabledModules() []string { + if x != nil { + return x.MixedBuildEnabledModules + } + return nil +} + +func (x *MixedBuildsInfo) GetMixedBuildDisabledModules() []string { + if x != nil { + return x.MixedBuildDisabledModules + } + return nil +} + var File_metrics_proto protoreflect.FileDescriptor var file_metrics_proto_rawDesc = []byte{ @@ -1491,7 +1557,7 @@ var file_metrics_proto_rawDesc = []byte{ 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, 0x72, 0x69, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x55, 0x73, 0x65, 0x72, 0x4a, 0x6f, 0x75, 0x72, 0x6e, 0x65, 0x79, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, - 0x04, 0x63, 0x75, 0x6a, 0x73, 0x22, 0xfa, 0x01, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, + 0x04, 0x63, 0x75, 0x6a, 0x73, 0x22, 0xcc, 0x02, 0x0a, 0x11, 0x53, 0x6f, 0x6f, 0x6e, 0x67, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, @@ -1507,22 +1573,36 @@ var file_metrics_proto_rawDesc = []byte{ 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x50, 0x65, 0x72, 0x66, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x73, 0x22, 0xc8, 0x01, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x45, 0x78, - 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x22, 0x34, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x4f, - 0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, - 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x42, 0x28, 0x5a, - 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, - 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x74, 0x73, 0x12, 0x50, 0x0a, 0x11, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, + 0x73, 0x6f, 0x6f, 0x6e, 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x2e, 0x4d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, + 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xc8, 0x01, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x6f, 0x6f, 0x6e, + 0x67, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, + 0x45, 0x78, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x65, 0x74, 0x63, 0x68, 0x65, 0x72, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x06, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x22, 0x34, 0x0a, 0x0c, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, + 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4e, 0x46, + 0x49, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x22, + 0x91, 0x01, 0x0a, 0x0f, 0x4d, 0x69, 0x78, 0x65, 0x64, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x73, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x3d, 0x0a, 0x1b, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x1c, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x19, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x4d, 0x6f, 0x64, 0x75, + 0x6c, 0x65, 0x73, 0x42, 0x28, 0x5a, 0x26, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x2f, 0x73, + 0x6f, 0x6f, 0x6e, 0x67, 0x2f, 0x75, 0x69, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -1538,7 +1618,7 @@ func file_metrics_proto_rawDescGZIP() []byte { } var file_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_metrics_proto_goTypes = []interface{}{ (MetricsBase_BuildVariant)(0), // 0: soong_build_metrics.MetricsBase.BuildVariant (MetricsBase_Arch)(0), // 1: soong_build_metrics.MetricsBase.Arch @@ -1554,6 +1634,7 @@ var file_metrics_proto_goTypes = []interface{}{ (*CriticalUserJourneysMetrics)(nil), // 11: soong_build_metrics.CriticalUserJourneysMetrics (*SoongBuildMetrics)(nil), // 12: soong_build_metrics.SoongBuildMetrics (*ExpConfigFetcher)(nil), // 13: soong_build_metrics.ExpConfigFetcher + (*MixedBuildsInfo)(nil), // 14: soong_build_metrics.MixedBuildsInfo } var file_metrics_proto_depIdxs = []int32{ 0, // 0: soong_build_metrics.MetricsBase.target_build_variant:type_name -> soong_build_metrics.MetricsBase.BuildVariant @@ -1575,12 +1656,13 @@ var file_metrics_proto_depIdxs = []int32{ 4, // 16: soong_build_metrics.CriticalUserJourneyMetrics.metrics:type_name -> soong_build_metrics.MetricsBase 10, // 17: soong_build_metrics.CriticalUserJourneysMetrics.cujs:type_name -> soong_build_metrics.CriticalUserJourneyMetrics 7, // 18: soong_build_metrics.SoongBuildMetrics.events:type_name -> soong_build_metrics.PerfInfo - 3, // 19: soong_build_metrics.ExpConfigFetcher.status:type_name -> soong_build_metrics.ExpConfigFetcher.ConfigStatus - 20, // [20:20] is the sub-list for method output_type - 20, // [20:20] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 14, // 19: soong_build_metrics.SoongBuildMetrics.mixed_builds_info:type_name -> soong_build_metrics.MixedBuildsInfo + 3, // 20: soong_build_metrics.ExpConfigFetcher.status:type_name -> soong_build_metrics.ExpConfigFetcher.ConfigStatus + 21, // [21:21] is the sub-list for method output_type + 21, // [21:21] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_metrics_proto_init() } @@ -1709,6 +1791,18 @@ func file_metrics_proto_init() { return nil } } + file_metrics_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MixedBuildsInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1716,7 +1810,7 @@ func file_metrics_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_metrics_proto_rawDesc, NumEnums: 4, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 0, }, diff --git a/ui/metrics/metrics_proto/metrics.proto b/ui/metrics/metrics_proto/metrics.proto index 814eb67c8..51dd523d2 100644 --- a/ui/metrics/metrics_proto/metrics.proto +++ b/ui/metrics/metrics_proto/metrics.proto @@ -200,10 +200,10 @@ message ModuleTypeInfo { SOONG = 1; MAKE = 2; } - // The build system, eg. Soong or Make. + // The build system, e.g. Soong or Make. optional BuildSystem build_system = 1 [default = UNKNOWN]; - // The module type, eg. java_library, cc_binary, and etc. + // The module type, e.g. java_library, cc_binary, and etc. optional string module_type = 2; // The number of logical modules. @@ -241,6 +241,9 @@ message SoongBuildMetrics { // Runtime metrics for soong_build execution. repeated PerfInfo events = 6; + + // Mixed Builds information + optional MixedBuildsInfo mixed_builds_info = 7; } message ExpConfigFetcher { @@ -261,3 +264,25 @@ message ExpConfigFetcher { // Time, in microseconds, taken by the expconfigfetcher optional uint64 micros = 3; } + +message MixedBuildsInfo{ + // Modules may be listed below as both enabled for Mixed Builds + // and disabled for Mixed Builds. This implies that some variants + // of the module are handled by Bazel in a Mixed Build, and other + // variants of the same module are handled by Soong. + + // Modules that are enabled for Mixed Builds. + repeated string mixed_build_enabled_modules = 1; + + // Modules that are not currently eligible to be handled + // by Bazel in a Mixed Build. + // Note that not all modules exempt from Bazel handling are + // listed. This list includes only modules which are of a + // Mixed-Build supported module type but are nevertheless not + // handled by Bazel. This may occur due to being present in + // the mixed build denylist, or as part of an unsupported + // mixed build variant type such as Windows. + + // Modules that are not enabled for MixedBuilds + repeated string mixed_build_disabled_modules = 2; +} diff --git a/ui/terminal/simple_status.go b/ui/terminal/simple_status.go index 31578136c..cef3b5d5c 100644 --- a/ui/terminal/simple_status.go +++ b/ui/terminal/simple_status.go @@ -46,7 +46,11 @@ func NewSimpleStatusOutput(w io.Writer, formatter formatter, keepANSI bool, quie func (s *simpleStatusOutput) Message(level status.MsgLevel, message string) { if level >= s.outputLevel { - fmt.Fprintln(s.writer, s.formatter.message(level, message)) + output := s.formatter.message(level, message) + if !s.keepANSI { + output = string(stripAnsiEscapes([]byte(output))) + } + fmt.Fprintln(s.writer, output) } } diff --git a/ui/terminal/status_test.go b/ui/terminal/status_test.go index 810e31d1b..b9057d298 100644 --- a/ui/terminal/status_test.go +++ b/ui/terminal/status_test.go @@ -81,9 +81,9 @@ func TestStatusOutput(t *testing.T) { }, { name: "action with output with ansi codes", - calls: actionWithOuptutWithAnsiCodes, - smart: "\r\x1b[1m[ 0% 0/1] action1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] action1\x1b[0m\x1b[K\n\x1b[31mcolor\x1b[0m\n", - simple: "[100% 1/1] action1\ncolor\n", + calls: actionWithOutputWithAnsiCodes, + smart: "\r\x1b[1m[ 0% 0/1] action1\x1b[0m\x1b[K\r\x1b[1m[100% 1/1] action1\x1b[0m\x1b[K\n\x1b[31mcolor\x1b[0m\n\x1b[31mcolor message\x1b[0m\n", + simple: "[100% 1/1] action1\ncolor\ncolor message\n", }, } @@ -257,12 +257,14 @@ func actionWithLongDescription(stat status.StatusOutput) { runner.finishAction(result1) } -func actionWithOuptutWithAnsiCodes(stat status.StatusOutput) { +func actionWithOutputWithAnsiCodes(stat status.StatusOutput) { result1WithOutputWithAnsiCodes := status.ActionResult{Action: action1, Output: "\x1b[31mcolor\x1b[0m"} runner := newRunner(stat, 1) runner.startAction(action1) runner.finishAction(result1WithOutputWithAnsiCodes) + + stat.Message(status.PrintLvl, "\x1b[31mcolor message\x1b[0m") } func TestSmartStatusOutputWidthChange(t *testing.T) { |