diff options
364 files changed, 14607 insertions, 5646 deletions
diff --git a/Changes.md b/Changes.md index 9f2449c2c3..eddec04a6c 100644 --- a/Changes.md +++ b/Changes.md @@ -40,14 +40,8 @@ within a product configuration .mk file, board config .mk file, or buildspec.mk. ## Python 2 to 3 migration -The path set when running builds now makes the `python` executable point to python 3, -whereas on previous versions it pointed to python 2. If you still have python 2 scripts, -you can change the shebang line to use `python2` explicitly. This only applies for -scripts run directly from makefiles, or from soong genrules. - -In addition, `python_*` soong modules no longer allow python 2. - -Python 2 is slated for complete removal in V. +Python 2 has been completely removed from the build. Please migrate any remaining usages to +Python 3, and remove any version-specific properties from bp files. ## Stop referencing sysprop_library directly from cc modules diff --git a/backported_fixes/Android.bp b/backported_fixes/Android.bp new file mode 100644 index 0000000000..0caea56a57 --- /dev/null +++ b/backported_fixes/Android.bp @@ -0,0 +1,144 @@ +// Copyright 2024 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], + default_team: "trendy_team_android_media_reliability", +} + +genrule { + name: "applied_backported_fixes", + tools: ["applied_backported_fixes_property_writer"], + srcs: [":applied_backported_fix_binpbs"], + out: ["applied_backported_fixes.prop"], + cmd: "$(location applied_backported_fixes_property_writer)" + + " -p $(location applied_backported_fixes.prop)" + + " $(in)", +} + +filegroup { + name: "backported_fixes_proto_file", + srcs: [ + "backported_fixes.proto", + ], +} + +java_library { + name: "backported_fixes_proto", + srcs: ["backported_fixes.proto"], + host_supported: true, + sdk_version: "current", +} + +java_library { + name: "backported_fixes_common", + srcs: ["src/java/com/android/build/backportedfixes/common/*.java"], + static_libs: [ + "backported_fixes_proto", + "guava", + ], + host_supported: true, +} + +java_test_host { + name: "backported_fixes_common_test", + srcs: ["tests/java/com/android/build/backportedfixes/common/*.java"], + static_libs: [ + "backported_fixes_common", + "backported_fixes_proto", + "junit", + "truth", + "truth-liteproto-extension", + "truth-proto-extension", + ], + test_options: { + unit_test: true, + }, + test_suites: ["general-tests"], +} + +java_library { + name: "backported_fixes_main_lib", + srcs: ["src/java/com/android/build/backportedfixes/*.java"], + static_libs: [ + "backported_fixes_common", + "backported_fixes_proto", + "jcommander", + "guava", + ], + host_supported: true, +} + +java_binary_host { + name: "applied_backported_fixes_property_writer", + main_class: "com.android.build.backportedfixes.WriteBackportedFixesPropFile", + static_libs: [ + "backported_fixes_main_lib", + ], +} + +java_binary_host { + name: "backported_fixes_combiner", + main_class: "com.android.build.backportedfixes.CombineBackportedFixes", + static_libs: [ + "backported_fixes_main_lib", + ], +} + +// Combines BackportedFix binary proto files into a single BackportedFixes binary proto file. +genrule_defaults { + name: "default_backported_fixes_combiner", + tools: ["backported_fixes_combiner"], + cmd: "$(location backported_fixes_combiner)" + + " -o $(out)" + + " $(in)", +} + +java_test_host { + name: "backported_fixes_main_lib_test", + srcs: ["tests/java/com/android/build/backportedfixes/*.java"], + static_libs: [ + "backported_fixes_main_lib", + "backported_fixes_proto", + "junit", + "truth", + ], + test_options: { + unit_test: true, + }, + test_suites: ["general-tests"], +} + +// Converts BackprotedFix text protos to binary protos +genrule_defaults { + name: "default_backported_fix_binpbs", + tools: ["aprotoc"], + tool_files: [ + ":backported_fixes_proto_file", + ], + cmd: "$(location aprotoc) " + + " --encode=com.android.build.backportedfixes.BackportedFix" + + " $(location :backported_fixes_proto_file)" + + " < $(in)" + + " > $(out); echo $(out)", +} + +gensrcs { + name: "applied_backported_fix_binpbs", + defaults: ["default_backported_fix_binpbs"], + output_extension: "binpb", + srcs: [ + "applied_fixes/*.txtpb", + ], +} diff --git a/backported_fixes/OWNERS b/backported_fixes/OWNERS new file mode 100644 index 0000000000..ac176bf0b4 --- /dev/null +++ b/backported_fixes/OWNERS @@ -0,0 +1,3 @@ +essick@google.com +nchalko@google.com +portmannc@google.com diff --git a/core/tasks/tools/update_bootloader_radio_image.mk b/backported_fixes/applied_fixes/ki350037023.txtpb index 0ebf247213..456a7aec35 100644 --- a/core/tasks/tools/update_bootloader_radio_image.mk +++ b/backported_fixes/applied_fixes/ki350037023.txtpb @@ -4,14 +4,16 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http:#www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# proto-file: ../backported_fixes.proto +# proto-message: BackportedFix -ifeq ($(USES_DEVICE_GOOGLE_ZUMA),true) - -include vendor/google_devices/zuma/prebuilts/misc_bins/update_bootloader_radio_image.mk -endif +known_issue: 350037023 +alias: 1 diff --git a/backported_fixes/backported_fixes.proto b/backported_fixes/backported_fixes.proto new file mode 100644 index 0000000000..91618eebd9 --- /dev/null +++ b/backported_fixes/backported_fixes.proto @@ -0,0 +1,37 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package com.android.build.backportedfixes; + +option java_multiple_files = true; + +// A list of backported fixes. +message BackportedFixes { + repeated BackportedFix fixes = 1; +} + +// A known issue approved for reporting Build.getBackportedFixStatus +message BackportedFix { + + // The issue id from the public bug tracker + // https://issuetracker.google.com/issues/{known_issue} + optional int64 known_issue = 1; + // The alias for the known issue. + // 1 - 1023 are valid aliases + // Must be unique across all backported fixes. + optional int32 alias = 2; +} + diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/CombineBackportedFixes.java b/backported_fixes/src/java/com/android/build/backportedfixes/CombineBackportedFixes.java new file mode 100644 index 0000000000..0592cc187b --- /dev/null +++ b/backported_fixes/src/java/com/android/build/backportedfixes/CombineBackportedFixes.java @@ -0,0 +1,65 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes; + +import com.android.build.backportedfixes.common.Parser; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.converters.FileConverter; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + + +/** Creates a BackportedFixes binary proto file from a list of BackportedFix proto binary files. */ +public final class CombineBackportedFixes { + + @Parameter(description = "BackportedFix proto binary files", + converter = FileConverter.class, + required = true) + List<File> fixFiles; + @Parameter(description = "Write the BackportedFixes proto binary to this file", + names = {"--out","-o"}, + converter = FileConverter.class, + required = true) + File outFile; + + public static void main(String... argv) throws Exception { + CombineBackportedFixes main = new CombineBackportedFixes(); + JCommander.newBuilder().addObject(main).build().parse(argv); + main.run(); + } + + CombineBackportedFixes() { + } + + private void run() throws Exception { + try (var out = new FileOutputStream(outFile)) { + var fixes = Parser.parseBackportedFixFiles(fixFiles); + writeBackportedFixes(fixes, out); + } + } + + static void writeBackportedFixes(BackportedFixes fixes, OutputStream out) + throws IOException { + fixes.writeTo(out); + } +} diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/WriteBackportedFixesPropFile.java b/backported_fixes/src/java/com/android/build/backportedfixes/WriteBackportedFixesPropFile.java new file mode 100644 index 0000000000..0ffb4ac904 --- /dev/null +++ b/backported_fixes/src/java/com/android/build/backportedfixes/WriteBackportedFixesPropFile.java @@ -0,0 +1,89 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.android.build.backportedfixes.common.Parser; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.converters.FileConverter; +import com.google.common.io.Files; + +import java.io.File; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + + +/** + * Creates backported fix properties file. + * + * <p>Writes BitSet of backported fix aliases from a list of BackportedFix proto binary files and + * writes the property {@value PROPERTY_NAME} to a file. + */ +public final class WriteBackportedFixesPropFile { + + private static final String PROPERTY_NAME = "ro.build.backported_fixes.alias_bitset.long_list"; + @Parameter(description = "BackportedFix proto binary files", + converter = FileConverter.class, + required = true) + List<File> fixFiles; + @Parameter(description = "The file to write the property value to.", + names = {"--property_file", "-p"}, + converter = FileConverter.class, + required = true) + File propertyFile; + + public static void main(String... argv) throws Exception { + WriteBackportedFixesPropFile main = new WriteBackportedFixesPropFile(); + JCommander.newBuilder().addObject(main).build().parse(argv); + main.run(); + } + + WriteBackportedFixesPropFile() { + } + + private void run() throws Exception { + try (var out = Files.newWriter(propertyFile, UTF_8)) { + var fixes = Parser.parseBackportedFixFiles(fixFiles); + writeFixesAsAliasBitSet(fixes, out); + } + } + + static void writeFixesAsAliasBitSet(BackportedFixes fixes, Writer out) { + PrintWriter printWriter = new PrintWriter(out); + printWriter.println("# The following backported fixes have been applied"); + for (var f : fixes.getFixesList()) { + printWriter.printf("# https://issuetracker.google.com/issues/%d with alias %d", + f.getKnownIssue(), f.getAlias()); + printWriter.println(); + } + var bsArray = Parser.getBitSetArray( + fixes.getFixesList().stream().mapToInt(BackportedFix::getAlias).toArray()); + String bsString = Arrays.stream(bsArray).mapToObj(Long::toString).collect( + Collectors.joining(",")); + printWriter.printf("%s=%s", PROPERTY_NAME, bsString); + printWriter.println(); + if (printWriter.checkError()) { + throw new RuntimeException("There was an error writing to " + out.toString()); + } + } +} diff --git a/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java b/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java new file mode 100644 index 0000000000..6180fdc3da --- /dev/null +++ b/backported_fixes/src/java/com/android/build/backportedfixes/common/Parser.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes.common; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.android.build.backportedfixes.BackportedFix; +import com.android.build.backportedfixes.BackportedFixes; + +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + + +/** Static utilities for working with {@link BackportedFixes}. */ +public final class Parser { + + /** Creates list of FileInputStreams for a list of files. */ + public static ImmutableList<FileInputStream> getFileInputStreams(List<File> fixFiles) throws + FileNotFoundException { + var streams = ImmutableList.<FileInputStream>builder(); + for (var f : fixFiles) { + streams.add(new FileInputStream(f)); + } + return streams.build(); + } + + /** Converts a list of backported fix aliases into a long array representing a {@link BitSet} */ + public static long[] getBitSetArray(int[] aliases) { + BitSet bs = new BitSet(); + for (int a : aliases) { + bs.set(a); + } + return bs.toLongArray(); + } + + /** + * Creates a {@link BackportedFixes} from a list of {@link BackportedFix} binary proto streams. + */ + public static BackportedFixes parseBackportedFixFiles(List<File> fixFiles) + throws IOException { + try { + return fixFiles.stream().map(Parser::tunelFileInputStream) + .map(Parser::tunnelParse) + .sorted(Comparator.comparing(BackportedFix::getKnownIssue)) + .collect(fixCollector()); + + } catch (TunnelException e) { + throw e.rethrow(FileNotFoundException.class, IOException.class); + } + } + + + private static Collector<BackportedFix, ?, BackportedFixes> fixCollector() { + return Collectors.collectingAndThen(Collectors.toList(), fixList -> { + var result = BackportedFixes.newBuilder(); + result.addAllFixes(fixList); + return result.build(); + }); + } + + private static FileInputStream tunelFileInputStream(File file) throws TunnelException { + try { + return new FileInputStream(file); + } catch (FileNotFoundException e) { + throw new TunnelException(e); + } + } + + private static BackportedFix tunnelParse(InputStream s) throws TunnelException { + try { + var fix = BackportedFix.parseFrom(s); + s.close(); + return fix; + } catch (IOException e) { + throw new TunnelException(e); + } + } + + private static class TunnelException extends RuntimeException { + TunnelException(Exception cause) { + super("If you see this TunnelException something went wrong. It should always be rethrown as the cause.", cause); + } + + <X extends Exception> RuntimeException rethrow(Class<X> exceptionClazz) throws X { + checkNotNull(exceptionClazz); + Throwables.throwIfInstanceOf(getCause(), exceptionClazz); + throw exception( + getCause(), + "rethrow(%s) doesn't match underlying exception", exceptionClazz); + } + + public <X1 extends Exception, X2 extends Exception> RuntimeException rethrow( + Class<X1> exceptionClazz1, Class<X2> exceptionClazz2) throws X1, X2 { + checkNotNull(exceptionClazz1); + checkNotNull(exceptionClazz2); + Throwables.throwIfInstanceOf(getCause(), exceptionClazz1); + Throwables.throwIfInstanceOf(getCause(), exceptionClazz2); + throw exception( + getCause(), + "rethrow(%s, %s) doesn't match underlying exception", + exceptionClazz1, + exceptionClazz2); + } + + private static ClassCastException exception( + Throwable cause, String message, Object... formatArgs) { + ClassCastException result = new ClassCastException(String.format(message, formatArgs)); + result.initCause(cause); + return result; + } + + } + + private Parser() { + } +} diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/CombineBackportedFixesTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/CombineBackportedFixesTest.java new file mode 100644 index 0000000000..21d5f1e676 --- /dev/null +++ b/backported_fixes/tests/java/com/android/build/backportedfixes/CombineBackportedFixesTest.java @@ -0,0 +1,41 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes; + +import com.google.common.truth.Truth; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** Tests for {@link CombineBackportedFixes}. */ +public class CombineBackportedFixesTest { + + + @Test + public void writeBackportedFixes_default() throws IOException { + // Not much of a test, but there is not much to test. + BackportedFixes fixes = BackportedFixes.newBuilder() + .addFixes(BackportedFix.newBuilder().setKnownIssue(123).build()) + .build(); + var result = new ByteArrayOutputStream(); + CombineBackportedFixes.writeBackportedFixes(fixes, result); + Truth.assertThat(BackportedFixes.parseFrom(result.toByteArray())) + .isEqualTo(fixes); + } +} diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/WriteBackportedFixesPropFileTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/WriteBackportedFixesPropFileTest.java new file mode 100644 index 0000000000..3209c15911 --- /dev/null +++ b/backported_fixes/tests/java/com/android/build/backportedfixes/WriteBackportedFixesPropFileTest.java @@ -0,0 +1,64 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes; + +import com.google.common.truth.Truth; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** Tests for {@link WriteBackportedFixesPropFile}. */ +public class WriteBackportedFixesPropFileTest { + + + @Test + public void writeFixesAsAliasBitSet_default() { + BackportedFixes fixes = BackportedFixes.newBuilder().build(); + var result = new StringWriter(); + + WriteBackportedFixesPropFile.writeFixesAsAliasBitSet(fixes, new PrintWriter(result)); + + Truth.assertThat(result.toString()) + .isEqualTo(""" + # The following backported fixes have been applied + ro.build.backported_fixes.alias_bitset.long_list= + """); + } + + @Test + public void writeFixesAsAliasBitSet_some() { + BackportedFixes fixes = BackportedFixes.newBuilder() + .addFixes(BackportedFix.newBuilder().setKnownIssue(1234L).setAlias(1)) + .addFixes(BackportedFix.newBuilder().setKnownIssue(3L).setAlias(65)) + .addFixes(BackportedFix.newBuilder().setKnownIssue(4L).setAlias(67)) + .build(); + var result = new StringWriter(); + + WriteBackportedFixesPropFile.writeFixesAsAliasBitSet(fixes, new PrintWriter(result)); + + Truth.assertThat(result.toString()) + .isEqualTo(""" + # The following backported fixes have been applied + # https://issuetracker.google.com/issues/1234 with alias 1 + # https://issuetracker.google.com/issues/3 with alias 65 + # https://issuetracker.google.com/issues/4 with alias 67 + ro.build.backported_fixes.alias_bitset.long_list=2,10 + """); + } +} diff --git a/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java b/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java new file mode 100644 index 0000000000..57a0a40b90 --- /dev/null +++ b/backported_fixes/tests/java/com/android/build/backportedfixes/common/ParserTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.build.backportedfixes.common; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; + +import com.android.build.backportedfixes.BackportedFix; +import com.android.build.backportedfixes.BackportedFixes; + +import com.google.common.collect.ImmutableList; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; + +/** Tests for {@link Parser}.*/ +public class ParserTest { + + @Rule + public TemporaryFolder mTempFolder = new TemporaryFolder(); + + @Test + public void getFileInputStreams() throws IOException { + var results = Parser.getFileInputStreams( + ImmutableList.of(Files.createTempFile("test", null).toFile())); + assertThat(results).isNotEmpty(); + } + + + @Test + public void getBitSetArray_empty() { + var results = Parser.getBitSetArray(new int[]{}); + assertThat(results).isEmpty(); + } + + @Test + public void getBitSetArray_2_3_64() { + var results = Parser.getBitSetArray(new int[]{2,3,64}); + assertThat(results).asList().containsExactly(12L,1L).inOrder(); + } + + @Test + public void parseBackportedFixFiles_empty() throws IOException { + var result = Parser.parseBackportedFixFiles(ImmutableList.of()); + assertThat(result).isEqualTo(BackportedFixes.getDefaultInstance()); + } + + + @Test + public void parseBackportedFixFiles_oneBlank() throws IOException { + var result = Parser.parseBackportedFixFiles(ImmutableList.of(mTempFolder.newFile())); + + assertThat(result).isEqualTo( + BackportedFixes.newBuilder() + .addFixes(BackportedFix.getDefaultInstance()) + .build()); + } + + @Test + public void parseBackportedFixFiles_two() throws IOException { + BackportedFix ki123 = BackportedFix.newBuilder() + .setKnownIssue(123) + .setAlias(1) + .build(); + BackportedFix ki456 = BackportedFix.newBuilder() + .setKnownIssue(456) + .setAlias(2) + .build(); + var result = Parser.parseBackportedFixFiles( + ImmutableList.of(tempFile(ki456), tempFile(ki123))); + assertThat(result).isEqualTo( + BackportedFixes.newBuilder() + .addFixes(ki123) + .addFixes(ki456) + .build()); + } + + private File tempFile(BackportedFix fix) throws IOException { + File f = mTempFolder.newFile(); + try (FileOutputStream out = new FileOutputStream(f)) { + fix.writeTo(out); + return f; + } + } +} diff --git a/ci/Android.bp b/ci/Android.bp index 6d4ac35517..3f28be4494 100644 --- a/ci/Android.bp +++ b/ci/Android.bp @@ -25,7 +25,7 @@ python_test_host { "build_test_suites_test.py", ], libs: [ - "build_test_suites", + "build_test_suites_lib", "pyfakefs", "ci_test_lib", ], @@ -56,7 +56,7 @@ python_test_host { "build_test_suites_local_test.py", ], libs: [ - "build_test_suites", + "build_test_suites_lib", "pyfakefs", "ci_test_lib", ], @@ -79,7 +79,7 @@ python_test_host { "optimized_targets_test.py", ], libs: [ - "build_test_suites", + "build_test_suites_lib", "pyfakefs", ], test_options: { @@ -95,13 +95,36 @@ python_test_host { }, } -python_library_host { +python_binary_host { name: "build_test_suites", srcs: [ "build_test_suites.py", "optimized_targets.py", "test_mapping_module_retriever.py", "build_context.py", + "test_discovery_agent.py", + "metrics_agent.py", + "buildbot.py", + ], + main: "build_test_suites.py", + libs: [ + "soong-metrics-proto-py", + ], +} + +python_library_host { + name: "build_test_suites_lib", + srcs: [ + "build_test_suites.py", + "optimized_targets.py", + "test_mapping_module_retriever.py", + "build_context.py", + "test_discovery_agent.py", + "metrics_agent.py", + "buildbot.py", + ], + libs: [ + "soong-metrics-proto-py", ], } diff --git a/ci/build_context.py b/ci/build_context.py index cc48d53992..c7a1defb57 100644 --- a/ci/build_context.py +++ b/ci/build_context.py @@ -47,6 +47,9 @@ class BuildContext: self.is_test_mapping = False self.test_mapping_test_groups = set() self.file_download_options = set() + self.name = test_info_dict.get('name') + self.command = test_info_dict.get('command') + self.extra_options = test_info_dict.get('extraOptions') for opt in test_info_dict.get('extraOptions', []): key = opt.get('key') if key == 'test-mapping-test-group': diff --git a/ci/build_device_and_tests b/ci/build_device_and_tests new file mode 100755 index 0000000000..63d3ce3519 --- /dev/null +++ b/ci/build_device_and_tests @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail + +build/soong/soong_ui.bash --make-mode build_test_suites +$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites --device-build $@ diff --git a/ci/build_metadata b/ci/build_metadata index 8136702d94..3e9218f200 100755 --- a/ci/build_metadata +++ b/ci/build_metadata @@ -14,12 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex +set -x + +source build/make/shell_utils.sh export TARGET_PRODUCT=aosp_arm64 export TARGET_RELEASE=trunk_staging export TARGET_BUILD_VARIANT=eng -build/soong/bin/m dist \ - code_metadata +import_build_vars \ + OUT_DIR \ + DIST_DIR \ + HOST_OUT_EXECUTABLES \ + || exit $? + +TARGETS=( + all_teams + source_tree_size + release_config_metadata +) + +# Build modules +build/soong/bin/m dist ${TARGETS[@]} || exit $? +# List all source files in the tree +( \ + $HOST_OUT_EXECUTABLES/source_tree_size -o $DIST_DIR/all_source_tree_files.pb \ + && gzip -fn $DIST_DIR/all_source_tree_files.pb \ +) || exit $? diff --git a/ci/build_test_suites b/ci/build_test_suites index 5aaf2f49b7..74470a8e16 100755 --- a/ci/build_test_suites +++ b/ci/build_test_suites @@ -1,4 +1,4 @@ -#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B +#!/usr/bin/env bash # # Copyright 2024, The Android Open Source Project # @@ -13,8 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +set -euo pipefail -import build_test_suites -import sys - -build_test_suites.main(sys.argv[1:]) +build/soong/soong_ui.bash --make-mode build_test_suites +$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py index 933e43e387..d81248b496 100644 --- a/ci/build_test_suites.py +++ b/ci/build_test_suites.py @@ -20,15 +20,22 @@ import json import logging import os import pathlib +import re import subprocess import sys +from typing import Callable from build_context import BuildContext import optimized_targets +import metrics_agent +import test_discovery_agent -REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP']) +REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP', 'DIST_DIR']) SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash' LOG_PATH = 'logs/build_test_suites.log' +# Currently, this prevents the removal of those tags when they exist. In the future we likely +# want the script to supply 'dist directly +REQUIRED_BUILD_TARGETS = frozenset(['dist', 'droid', 'checkbuild']) class Error(Exception): @@ -67,14 +74,28 @@ class BuildPlanner: if 'optimized_build' not in self.build_context.enabled_build_features: return BuildPlan(set(self.args.extra_targets), set()) + if not self.build_context.test_infos: + logging.warning('Build context has no test infos, skipping optimizations.') + for target in self.args.extra_targets: + get_metrics_agent().report_unoptimized_target(target, 'BUILD_CONTEXT has no test infos.') + return BuildPlan(set(self.args.extra_targets), set()) + build_targets = set() - packaging_commands = [] - for target in self.args.extra_targets: - if self._unused_target_exclusion_enabled( - target - ) and not self.build_context.build_target_used(target): - continue + packaging_commands_getters = [] + # In order to roll optimizations out differently between test suites and + # device builds, we have separate flags. + enable_discovery = (('test_suites_zip_test_discovery' + in self.build_context.enabled_build_features + and not self.args.device_build + ) or ( + 'device_zip_test_discovery' + in self.build_context.enabled_build_features + and self.args.device_build + )) and not self.args.test_discovery_info_mode + logging.info(f'Discovery mode is enabled= {enable_discovery}') + preliminary_build_targets = self._collect_preliminary_build_targets(enable_discovery) + for target in preliminary_build_targets: target_optimizer_getter = self.target_optimizations.get(target, None) if not target_optimizer_getter: build_targets.add(target) @@ -84,9 +105,66 @@ class BuildPlanner: target, self.build_context, self.args ) build_targets.update(target_optimizer.get_build_targets()) - packaging_commands.extend(target_optimizer.get_package_outputs_commands()) + packaging_commands_getters.append( + target_optimizer.get_package_outputs_commands + ) + + return BuildPlan(build_targets, packaging_commands_getters) + + def _collect_preliminary_build_targets(self, enable_discovery: bool): + build_targets = set() + try: + test_discovery_zip_regexes = self._get_test_discovery_zip_regexes() + logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}') + except test_discovery_agent.TestDiscoveryError as e: + optimization_rationale = e.message + logging.warning(f'Unable to perform test discovery: {optimization_rationale}') + + for target in self.args.extra_targets: + get_metrics_agent().report_unoptimized_target(target, optimization_rationale) + return self._legacy_collect_preliminary_build_targets() + + for target in self.args.extra_targets: + if target in REQUIRED_BUILD_TARGETS: + build_targets.add(target) + get_metrics_agent().report_unoptimized_target(target, 'Required build target.') + continue + # If nothing is discovered without error, that means nothing is needed. + if not test_discovery_zip_regexes: + get_metrics_agent().report_optimized_target(target) + continue + + regex = r'\b(%s.*)\b' % re.escape(target) + for opt in test_discovery_zip_regexes: + try: + if re.search(regex, opt): + get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.') + build_targets.add(target) + # proceed to next target evaluation + break + get_metrics_agent().report_optimized_target(target) + except Exception as e: + # In case of exception report as unoptimized + build_targets.add(target) + get_metrics_agent().report_unoptimized_target(target, f'Error in parsing test discovery output for {target}: {repr(e)}') + logging.error(f'unable to parse test discovery output: {repr(e)}') + break + # If discovery is not enabled, return the original list + if not enable_discovery: + return self._legacy_collect_preliminary_build_targets() + + return build_targets + + def _legacy_collect_preliminary_build_targets(self): + build_targets = set() + for target in self.args.extra_targets: + if self._unused_target_exclusion_enabled( + target + ) and not self.build_context.build_target_used(target): + continue - return BuildPlan(build_targets, packaging_commands) + build_targets.add(target) + return build_targets def _unused_target_exclusion_enabled(self, target: str) -> bool: return ( @@ -94,11 +172,39 @@ class BuildPlanner: in self.build_context.enabled_build_features ) + def _get_test_discovery_zip_regexes(self) -> set[str]: + build_target_regexes = set() + for test_info in self.build_context.test_infos: + tf_command = self._build_tf_command(test_info) + discovery_agent = test_discovery_agent.TestDiscoveryAgent(tradefed_args=tf_command) + for regex in discovery_agent.discover_test_zip_regexes(): + build_target_regexes.add(regex) + return build_target_regexes + + + def _build_tf_command(self, test_info) -> list[str]: + command = [test_info.command] + for extra_option in test_info.extra_options: + if not extra_option.get('key'): + continue + arg_key = '--' + extra_option.get('key') + if arg_key == '--build-id': + command.append(arg_key) + command.append(os.environ.get('BUILD_NUMBER')) + continue + if extra_option.get('values'): + for value in extra_option.get('values'): + command.append(arg_key) + command.append(value) + else: + command.append(arg_key) + + return command @dataclass(frozen=True) class BuildPlan: build_targets: set[str] - packaging_commands: list[list[str]] + packaging_commands_getters: list[Callable[[], list[list[str]]]] def build_test_suites(argv: list[str]) -> int: @@ -110,19 +216,27 @@ def build_test_suites(argv: list[str]) -> int: Returns: The exit code of the build. """ - args = parse_args(argv) - check_required_env() - build_context = BuildContext(load_build_context()) - build_planner = BuildPlanner( - build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS - ) - build_plan = build_planner.create_build_plan() + get_metrics_agent().analysis_start() + try: + args = parse_args(argv) + check_required_env() + build_context = BuildContext(load_build_context()) + build_planner = BuildPlanner( + build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS + ) + build_plan = build_planner.create_build_plan() + except: + raise + finally: + get_metrics_agent().analysis_end() try: execute_build_plan(build_plan) except BuildFailureError as e: logging.error('Build command failed! Check build_log for details.') return e.return_code + finally: + get_metrics_agent().end_reporting() return 0 @@ -133,6 +247,16 @@ def parse_args(argv: list[str]) -> argparse.Namespace: argparser.add_argument( 'extra_targets', nargs='*', help='Extra test suites to build.' ) + argparser.add_argument( + '--device-build', + action='store_true', + help='Flag to indicate running a device build.', + ) + argparser.add_argument( + '--test_discovery_info_mode', + action='store_true', + help='Flag to enable running test discovery in info only mode.', + ) return argparser.parse_args(argv) @@ -174,17 +298,21 @@ def execute_build_plan(build_plan: BuildPlan): build_command.append(get_top().joinpath(SOONG_UI_EXE_REL_PATH)) build_command.append('--make-mode') build_command.extend(build_plan.build_targets) - + logging.info(f'Running build command: {build_command}') try: run_command(build_command) except subprocess.CalledProcessError as e: raise BuildFailureError(e.returncode) from e - for packaging_command in build_plan.packaging_commands: - try: - run_command(packaging_command) - except subprocess.CalledProcessError as e: - raise BuildFailureError(e.returncode) from e + get_metrics_agent().packaging_start() + try: + for packaging_commands_getter in build_plan.packaging_commands_getters: + for packaging_command in packaging_commands_getter(): + run_command(packaging_command) + except subprocess.CalledProcessError as e: + raise BuildFailureError(e.returncode) from e + finally: + get_metrics_agent().packaging_end() def get_top() -> pathlib.Path: @@ -195,6 +323,10 @@ def run_command(args: list[str], stdout=None): subprocess.run(args=args, check=True, stdout=stdout) +def get_metrics_agent(): + return metrics_agent.MetricsAgent.instance() + + def main(argv): dist_dir = os.environ.get('DIST_DIR') if dist_dir: @@ -205,3 +337,7 @@ def main(argv): filename=log_file, ) sys.exit(build_test_suites(argv)) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py index fd06a3ae1e..190740f811 100644 --- a/ci/build_test_suites_test.py +++ b/ci/build_test_suites_test.py @@ -37,6 +37,8 @@ import build_test_suites import ci_test_lib import optimized_targets from pyfakefs import fake_filesystem_unittest +import metrics_agent +import test_discovery_agent class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): @@ -52,6 +54,10 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): self.addCleanup(subprocess_run_patcher.stop) self.mock_subprocess_run = subprocess_run_patcher.start() + metrics_agent_finalize_patcher = mock.patch('metrics_agent.MetricsAgent.end_reporting') + self.addCleanup(metrics_agent_finalize_patcher.stop) + self.mock_metrics_agent_end = metrics_agent_finalize_patcher.start() + self._setup_working_build_env() def test_missing_target_release_env_var_raises(self): @@ -72,6 +78,12 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): with self.assert_raises_word(build_test_suites.Error, 'TOP'): build_test_suites.main([]) + def test_missing_dist_dir_env_var_raises(self): + del os.environ['DIST_DIR'] + + with self.assert_raises_word(build_test_suites.Error, 'DIST_DIR'): + build_test_suites.main([]) + def test_invalid_arg_raises(self): invalid_args = ['--invalid_arg'] @@ -108,6 +120,9 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): self.soong_ui_dir = self.fake_top.joinpath('build/soong') self.soong_ui_dir.mkdir(parents=True, exist_ok=True) + self.logs_dir = self.fake_top.joinpath('dist/logs') + self.logs_dir.mkdir(parents=True, exist_ok=True) + self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash') self.soong_ui.touch() @@ -115,6 +130,7 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): 'TARGET_RELEASE': 'release', 'TARGET_PRODUCT': 'product', 'TOP': str(self.fake_top), + 'DIST_DIR': str(self.fake_top.joinpath('dist')), }) self.mock_subprocess_run.return_value = 0 @@ -256,6 +272,12 @@ class BuildPlannerTest(unittest.TestCase): def get_enabled_flag(self): return f'{self.target}_enabled' + def setUp(self): + test_discovery_agent_patcher = mock.patch('test_discovery_agent.TestDiscoveryAgent.discover_test_zip_regexes') + self.addCleanup(test_discovery_agent_patcher.stop) + self.mock_test_discovery_agent_end = test_discovery_agent_patcher.start() + + def test_build_optimization_off_builds_everything(self): build_targets = {'target_1', 'target_2'} build_planner = self.create_build_planner( @@ -276,14 +298,16 @@ class BuildPlannerTest(unittest.TestCase): build_plan = build_planner.create_build_plan() - self.assertEqual(len(build_plan.packaging_commands), 0) + for packaging_command in self.run_packaging_commands(build_plan): + self.assertEqual(len(packaging_command), 0) def test_build_optimization_on_optimizes_target(self): build_targets = {'target_1', 'target_2'} build_planner = self.create_build_planner( build_targets=build_targets, build_context=self.create_build_context( - enabled_build_features=[{'name': self.get_target_flag('target_1')}] + enabled_build_features=[{'name': self.get_target_flag('target_1')}], + test_context=self.get_test_context('target_1'), ), ) @@ -299,14 +323,15 @@ class BuildPlannerTest(unittest.TestCase): build_planner = self.create_build_planner( build_targets=build_targets, build_context=self.create_build_context( - enabled_build_features=[{'name': self.get_target_flag('target_1')}] + enabled_build_features=[{'name': self.get_target_flag('target_1')}], + test_context=self.get_test_context('target_1'), ), packaging_commands=packaging_commands, ) build_plan = build_planner.create_build_plan() - self.assertIn([f'packaging {optimized_target_name}'], build_plan.packaging_commands) + self.assertIn(packaging_commands, self.run_packaging_commands(build_plan)) def test_individual_build_optimization_off_doesnt_optimize(self): build_targets = {'target_1', 'target_2'} @@ -328,7 +353,8 @@ class BuildPlannerTest(unittest.TestCase): build_plan = build_planner.create_build_plan() - self.assertFalse(build_plan.packaging_commands) + for packaging_command in self.run_packaging_commands(build_plan): + self.assertEqual(len(packaging_command), 0) def test_target_output_used_target_built(self): build_target = 'test_target' @@ -485,6 +511,12 @@ class BuildPlannerTest(unittest.TestCase): ], } + def run_packaging_commands(self, build_plan: build_test_suites.BuildPlan): + return [ + packaging_command_getter() + for packaging_command_getter in build_plan.packaging_commands_getters + ] + def wait_until( condition_function: Callable[[], bool], diff --git a/ci/metrics_agent.py b/ci/metrics_agent.py new file mode 100644 index 0000000000..bc2479eab6 --- /dev/null +++ b/ci/metrics_agent.py @@ -0,0 +1,116 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MetricsAgent is a singleton class that collects metrics for optimized build.""" + +from enum import Enum +import time +import metrics_pb2 +import os +import logging + + +class MetricsAgent: + _SOONG_METRICS_PATH = 'logs/soong_metrics' + _DIST_DIR = 'DIST_DIR' + _instance = None + + def __init__(self): + raise RuntimeError( + 'MetricsAgent cannot be instantialized, use instance() instead' + ) + + @classmethod + def instance(cls): + if not cls._instance: + cls._instance = cls.__new__(cls) + cls._instance._proto = metrics_pb2.OptimizedBuildMetrics() + cls._instance._init_proto() + cls._instance._target_results = dict() + + return cls._instance + + def _init_proto(self): + self._proto.analysis_perf.name = 'Optimized build analysis time.' + self._proto.packaging_perf.name = 'Optimized build total packaging time.' + + def analysis_start(self): + self._proto.analysis_perf.start_time = time.time_ns() + + def analysis_end(self): + self._proto.analysis_perf.real_time = ( + time.time_ns() - self._proto.analysis_perf.start_time + ) + + def packaging_start(self): + self._proto.packaging_perf.start_time = time.time_ns() + + def packaging_end(self): + self._proto.packaging_perf.real_time = ( + time.time_ns() - self._proto.packaging_perf.start_time + ) + + def report_optimized_target(self, name: str): + target_result = metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult() + target_result.name = name + target_result.optimized = True + self._target_results[name] = target_result + + def report_unoptimized_target(self, name: str, optimization_rationale: str): + target_result = metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult() + target_result.name = name + target_result.optimization_rationale = optimization_rationale + target_result.optimized = False + self._target_results[name] = target_result + + def target_packaging_start(self, name: str): + target_result = self._target_results.get(name) + target_result.packaging_perf.start_time = time.time_ns() + self._target_results[name] = target_result + + def target_packaging_end(self, name: str): + target_result = self._target_results.get(name) + target_result.packaging_perf.real_time = ( + time.time_ns() - target_result.packaging_perf.start_time + ) + + def add_target_artifact( + self, + target_name: str, + artifact_name: str, + size: int, + included_modules: set[str], + ): + target_result = self.target_results.get(target_name) + artifact = ( + metrics_pb2.OptimizedBuildMetrics.TargetOptimizationResult.OutputArtifact() + ) + artifact.name = artifact_name + artifact.size = size + for module in included_modules: + artifact.included_modules.add(module) + target_result.output_artifacts.add(artifact) + + def end_reporting(self): + for target_result in self._target_results.values(): + self._proto.target_result.append(target_result) + soong_metrics_proto = metrics_pb2.MetricsBase() + # Read in existing metrics that should have been written out by the soong + # build command so that we don't overwrite them. + with open(os.path.join(os.environ[self._DIST_DIR], self._SOONG_METRICS_PATH), 'rb') as f: + soong_metrics_proto.ParseFromString(f.read()) + soong_metrics_proto.optimized_build_metrics.CopyFrom(self._proto) + logging.info(soong_metrics_proto) + with open(os.path.join(os.environ[self._DIST_DIR], self._SOONG_METRICS_PATH), 'wb') as f: + f.write(soong_metrics_proto.SerializeToString()) diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py index 9143cbf2f1..688bdd8370 100644 --- a/ci/optimized_targets.py +++ b/ci/optimized_targets.py @@ -16,12 +16,13 @@ from abc import ABC import argparse import functools -from build_context import BuildContext import json import logging import os -from typing import Self +import pathlib +import subprocess +from build_context import BuildContext import test_mapping_module_retriever @@ -33,6 +34,9 @@ class OptimizedBuildTarget(ABC): build. """ + _SOONG_UI_BASH_PATH = 'build/soong/soong_ui.bash' + _PREBUILT_SOONG_ZIP_PATH = 'prebuilts/build-tools/linux-x86/bin/soong_zip' + def __init__( self, target: str, @@ -75,6 +79,88 @@ class OptimizedBuildTarget(ABC): f'get_build_targets_impl not implemented in {type(self).__name__}' ) + def _generate_zip_options_for_items( + self, + prefix: str = '', + relative_root: str = '', + list_files: list[str] | None = None, + files: list[str] | None = None, + directories: list[str] | None = None, + ) -> list[str]: + if not list_files and not files and not directories: + raise RuntimeError( + f'No items specified to be added to zip! Prefix: {prefix}, Relative' + f' root: {relative_root}' + ) + command_segment = [] + # These are all soong_zip options so consult soong_zip --help for specifics. + if prefix: + command_segment.append('-P') + command_segment.append(prefix) + if relative_root: + command_segment.append('-C') + command_segment.append(relative_root) + if list_files: + for list_file in list_files: + command_segment.append('-l') + command_segment.append(list_file) + if files: + for file in files: + command_segment.append('-f') + command_segment.append(file) + if directories: + for directory in directories: + command_segment.append('-D') + command_segment.append(directory) + + return command_segment + + def _query_soong_vars( + self, src_top: pathlib.Path, soong_vars: list[str] + ) -> dict[str, str]: + process_result = subprocess.run( + args=[ + f'{src_top / self._SOONG_UI_BASH_PATH}', + '--dumpvars-mode', + f'--abs-vars={" ".join(soong_vars)}', + ], + env=os.environ, + check=False, + capture_output=True, + text=True, + ) + if not process_result.returncode == 0: + logging.error('soong dumpvars command failed! stderr:') + logging.error(process_result.stderr) + raise RuntimeError('Soong dumpvars failed! See log for stderr.') + + if not process_result.stdout: + raise RuntimeError( + 'Necessary soong variables ' + soong_vars + ' not found.' + ) + + try: + return { + line.split('=')[0]: line.split('=')[1].strip("'") + for line in process_result.stdout.strip().split('\n') + } + except IndexError as e: + raise RuntimeError( + 'Error parsing soong dumpvars output! See output here:' + f' {process_result.stdout}', + e, + ) + + def _base_zip_command( + self, src_top: pathlib.Path, dist_dir: pathlib.Path, name: str + ) -> list[str]: + return [ + f'{src_top / self._PREBUILT_SOONG_ZIP_PATH }', + '-d', + '-o', + f'{dist_dir / name}', + ] + class NullOptimizer(OptimizedBuildTarget): """No-op target optimizer. @@ -121,8 +207,6 @@ class ChangeInfo: class GeneralTestsOptimizer(OptimizedBuildTarget): """general-tests optimizer - TODO(b/358215235): Implement - This optimizer reads in the list of changed files from the file located in env[CHANGE_INFO] and uses this list alongside the normal TEST MAPPING logic to determine what test mapping modules will run for the given changes. It then @@ -130,10 +214,13 @@ class GeneralTestsOptimizer(OptimizedBuildTarget): normally built. """ - # List of modules that are always required to be in general-tests.zip. - _REQUIRED_MODULES = frozenset( - ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util'] - ) + # List of modules that are built alongside general-tests as dependencies. + _REQUIRED_MODULES = frozenset([ + 'cts-tradefed', + 'vts-tradefed', + 'compatibility-host-util', + 'general-tests-shared-libs', + ]) def get_build_targets_impl(self) -> set[str]: change_info_file_path = os.environ.get('CHANGE_INFO') @@ -177,6 +264,212 @@ class GeneralTestsOptimizer(OptimizedBuildTarget): return modules_to_build + def get_package_outputs_commands_impl(self): + src_top = pathlib.Path(os.environ.get('TOP', os.getcwd())) + dist_dir = pathlib.Path(os.environ.get('DIST_DIR')) + + soong_vars = self._query_soong_vars( + src_top, + [ + 'HOST_OUT_TESTCASES', + 'TARGET_OUT_TESTCASES', + 'PRODUCT_OUT', + 'SOONG_HOST_OUT', + 'HOST_OUT', + ], + ) + host_out_testcases = pathlib.Path(soong_vars.get('HOST_OUT_TESTCASES')) + target_out_testcases = pathlib.Path(soong_vars.get('TARGET_OUT_TESTCASES')) + product_out = pathlib.Path(soong_vars.get('PRODUCT_OUT')) + soong_host_out = pathlib.Path(soong_vars.get('SOONG_HOST_OUT')) + host_out = pathlib.Path(soong_vars.get('HOST_OUT')) + + host_paths = [] + target_paths = [] + host_config_files = [] + target_config_files = [] + for module in self.modules_to_build: + # The required modules are handled separately, no need to package. + if module in self._REQUIRED_MODULES: + continue + + host_path = host_out_testcases / module + if os.path.exists(host_path): + host_paths.append(host_path) + self._collect_config_files(src_top, host_path, host_config_files) + + target_path = target_out_testcases / module + if os.path.exists(target_path): + target_paths.append(target_path) + self._collect_config_files(src_top, target_path, target_config_files) + + if not os.path.exists(host_path) and not os.path.exists(target_path): + logging.info(f'No host or target build outputs found for {module}.') + + zip_commands = [] + + zip_commands.extend( + self._get_zip_test_configs_zips_commands( + src_top, + dist_dir, + host_out, + product_out, + host_config_files, + target_config_files, + ) + ) + + zip_command = self._base_zip_command(src_top, dist_dir, 'general-tests.zip') + + # Add host testcases. + if host_paths: + zip_command.extend( + self._generate_zip_options_for_items( + prefix='host', + relative_root=f'{src_top / soong_host_out}', + directories=host_paths, + ) + ) + + # Add target testcases. + if target_paths: + zip_command.extend( + self._generate_zip_options_for_items( + prefix='target', + relative_root=f'{src_top / product_out}', + directories=target_paths, + ) + ) + + # TODO(lucafarsi): Push this logic into a general-tests-minimal build command + # Add necessary tools. These are also hardcoded in general-tests.mk. + framework_path = soong_host_out / 'framework' + + zip_command.extend( + self._generate_zip_options_for_items( + prefix='host/tools', + relative_root=str(framework_path), + files=[ + f"{framework_path / 'cts-tradefed.jar'}", + f"{framework_path / 'compatibility-host-util.jar'}", + f"{framework_path / 'vts-tradefed.jar'}", + ], + ) + ) + + zip_commands.append(zip_command) + return zip_commands + + def _collect_config_files( + self, + src_top: pathlib.Path, + root_dir: pathlib.Path, + config_files: list[str], + ): + for root, dirs, files in os.walk(src_top / root_dir): + for file in files: + if file.endswith('.config'): + config_files.append(root_dir / file) + + def _get_zip_test_configs_zips_commands( + self, + src_top: pathlib.Path, + dist_dir: pathlib.Path, + host_out: pathlib.Path, + product_out: pathlib.Path, + host_config_files: list[str], + target_config_files: list[str], + ) -> tuple[list[str], list[str]]: + """Generate general-tests_configs.zip and general-tests_list.zip. + + general-tests_configs.zip contains all of the .config files that were + built and general-tests_list.zip contains a text file which lists + all of the .config files that are in general-tests_configs.zip. + + general-tests_configs.zip is organized as follows: + / + host/ + testcases/ + test_1.config + test_2.config + ... + target/ + testcases/ + test_1.config + test_2.config + ... + + So the process is we write out the paths to all the host config files into + one + file and all the paths to the target config files in another. We also write + the paths to all the config files into a third file to use for + general-tests_list.zip. + + Args: + dist_dir: dist directory. + host_out: host out directory. + product_out: product out directory. + host_config_files: list of all host config files. + target_config_files: list of all target config files. + + Returns: + The commands to generate general-tests_configs.zip and + general-tests_list.zip + """ + with open( + f"{host_out / 'host_general-tests_list'}", 'w' + ) as host_list_file, open( + f"{product_out / 'target_general-tests_list'}", 'w' + ) as target_list_file, open( + f"{host_out / 'general-tests_list'}", 'w' + ) as list_file: + + for config_file in host_config_files: + host_list_file.write(f'{config_file}' + '\n') + list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n') + + for config_file in target_config_files: + target_list_file.write(f'{config_file}' + '\n') + list_file.write( + 'target/' + os.path.relpath(config_file, product_out) + '\n' + ) + + zip_commands = [] + + tests_config_zip_command = self._base_zip_command( + src_top, dist_dir, 'general-tests_configs.zip' + ) + tests_config_zip_command.extend( + self._generate_zip_options_for_items( + prefix='host', + relative_root=str(host_out), + list_files=[f"{host_out / 'host_general-tests_list'}"], + ) + ) + + tests_config_zip_command.extend( + self._generate_zip_options_for_items( + prefix='target', + relative_root=str(product_out), + list_files=[f"{product_out / 'target_general-tests_list'}"], + ), + ) + + zip_commands.append(tests_config_zip_command) + + tests_list_zip_command = self._base_zip_command( + src_top, dist_dir, 'general-tests_list.zip' + ) + tests_list_zip_command.extend( + self._generate_zip_options_for_items( + relative_root=str(host_out), + files=[f"{host_out / 'general-tests_list'}"], + ) + ) + zip_commands.append(tests_list_zip_command) + + return zip_commands + def get_enabled_flag(self): return 'general_tests_optimized' diff --git a/ci/optimized_targets_test.py b/ci/optimized_targets_test.py index 919c193955..0b0c0ec087 100644 --- a/ci/optimized_targets_test.py +++ b/ci/optimized_targets_test.py @@ -19,10 +19,12 @@ import logging import os import pathlib import re +import subprocess +import textwrap import unittest from unittest import mock -import optimized_targets from build_context import BuildContext +import optimized_targets from pyfakefs import fake_filesystem_unittest @@ -43,11 +45,68 @@ class GeneralTestsOptimizerTest(fake_filesystem_unittest.TestCase): def _setup_working_build_env(self): self.change_info_file = pathlib.Path('/tmp/change_info') + self._write_soong_ui_file() + self._host_out_testcases = pathlib.Path('/tmp/top/host_out_testcases') + self._host_out_testcases.mkdir(parents=True) + self._target_out_testcases = pathlib.Path('/tmp/top/target_out_testcases') + self._target_out_testcases.mkdir(parents=True) + self._product_out = pathlib.Path('/tmp/top/product_out') + self._product_out.mkdir(parents=True) + self._soong_host_out = pathlib.Path('/tmp/top/soong_host_out') + self._soong_host_out.mkdir(parents=True) + self._host_out = pathlib.Path('/tmp/top/host_out') + self._host_out.mkdir(parents=True) + + self._dist_dir = pathlib.Path('/tmp/top/out/dist') + self._dist_dir.mkdir(parents=True) self.mock_os_environ.update({ 'CHANGE_INFO': str(self.change_info_file), + 'TOP': '/tmp/top', + 'DIST_DIR': '/tmp/top/out/dist', }) + def _write_soong_ui_file(self): + soong_path = pathlib.Path('/tmp/top/build/soong') + soong_path.mkdir(parents=True) + with open(os.path.join(soong_path, 'soong_ui.bash'), 'w') as f: + f.write(""" + #/bin/bash + echo HOST_OUT_TESTCASES='/tmp/top/host_out_testcases' + echo TARGET_OUT_TESTCASES='/tmp/top/target_out_testcases' + echo PRODUCT_OUT='/tmp/top/product_out' + echo SOONG_HOST_OUT='/tmp/top/soong_host_out' + echo HOST_OUT='/tmp/top/host_out' + """) + os.chmod(os.path.join(soong_path, 'soong_ui.bash'), 0o666) + + def _write_change_info_file(self): + change_info_contents = { + 'changes': [{ + 'projectPath': '/project/path', + 'revisions': [{ + 'fileInfos': [{ + 'path': 'file/path/file_name', + }], + }], + }] + } + + with open(self.change_info_file, 'w') as f: + json.dump(change_info_contents, f) + + def _write_test_mapping_file(self): + test_mapping_contents = { + 'test-mapping-group': [ + { + 'name': 'test_mapping_module', + }, + ], + } + + with open('/project/path/file/path/TEST_MAPPING', 'w') as f: + json.dump(test_mapping_contents, f) + def test_general_tests_optimized(self): optimizer = self._create_general_tests_optimizer() @@ -124,36 +183,44 @@ class GeneralTestsOptimizerTest(fake_filesystem_unittest.TestCase): with self.assertRaises(json.decoder.JSONDecodeError): build_targets = optimizer.get_build_targets() - def _write_change_info_file(self): - change_info_contents = { - 'changes': [{ - 'projectPath': '/project/path', - 'revisions': [{ - 'fileInfos': [{ - 'path': 'file/path/file_name', - }], - }], - }] - } + @mock.patch('subprocess.run') + def test_packaging_outputs_success(self, subprocess_run): + subprocess_run.return_value = self._get_soong_vars_output() + optimizer = self._create_general_tests_optimizer() + self._set_up_build_outputs(['test_mapping_module']) - with open(self.change_info_file, 'w') as f: - json.dump(change_info_contents, f) + targets = optimizer.get_build_targets() + package_commands = optimizer.get_package_outputs_commands() - def _write_test_mapping_file(self): - test_mapping_contents = { - 'test-mapping-group': [ - { - 'name': 'test_mapping_module', - }, - ], - } + self._verify_soong_zip_commands(package_commands, ['test_mapping_module']) - with open('/project/path/file/path/TEST_MAPPING', 'w') as f: - json.dump(test_mapping_contents, f) + @mock.patch('subprocess.run') + def test_get_soong_dumpvars_fails_raises(self, subprocess_run): + subprocess_run.return_value = self._get_soong_vars_output(return_code=-1) + optimizer = self._create_general_tests_optimizer() + self._set_up_build_outputs(['test_mapping_module']) + + targets = optimizer.get_build_targets() - def _create_general_tests_optimizer( - self, build_context: BuildContext = None - ): + with self.assertRaisesRegex(RuntimeError, 'Soong dumpvars failed!'): + package_commands = optimizer.get_package_outputs_commands() + + @mock.patch('subprocess.run') + def test_get_soong_dumpvars_bad_output_raises(self, subprocess_run): + subprocess_run.return_value = self._get_soong_vars_output( + stdout='This output is bad' + ) + optimizer = self._create_general_tests_optimizer() + self._set_up_build_outputs(['test_mapping_module']) + + targets = optimizer.get_build_targets() + + with self.assertRaisesRegex( + RuntimeError, 'Error parsing soong dumpvars output' + ): + package_commands = optimizer.get_package_outputs_commands() + + def _create_general_tests_optimizer(self, build_context: BuildContext = None): if not build_context: build_context = self._create_build_context() return optimized_targets.GeneralTestsOptimizer( @@ -170,7 +237,9 @@ class GeneralTestsOptimizerTest(fake_filesystem_unittest.TestCase): build_context_dict = {} build_context_dict['enabledBuildFeatures'] = [{'name': 'optimized_build'}] if general_tests_optimized: - build_context_dict['enabledBuildFeatures'].append({'name': 'general_tests_optimized'}) + build_context_dict['enabledBuildFeatures'].append( + {'name': 'general_tests_optimized'} + ) build_context_dict['testContext'] = test_context return BuildContext(build_context_dict) @@ -199,6 +268,81 @@ class GeneralTestsOptimizerTest(fake_filesystem_unittest.TestCase): ], } + def _get_soong_vars_output( + self, return_code: int = 0, stdout: str = '' + ) -> subprocess.CompletedProcess: + return_value = subprocess.CompletedProcess(args=[], returncode=return_code) + if not stdout: + stdout = textwrap.dedent(f"""\ + HOST_OUT_TESTCASES='{self._host_out_testcases}' + TARGET_OUT_TESTCASES='{self._target_out_testcases}' + PRODUCT_OUT='{self._product_out}' + SOONG_HOST_OUT='{self._soong_host_out}' + HOST_OUT='{self._host_out}'""") + + return_value.stdout = stdout + return return_value + + def _set_up_build_outputs(self, targets: list[str]): + for target in targets: + host_dir = self._host_out_testcases / target + host_dir.mkdir() + (host_dir / f'{target}.config').touch() + (host_dir / f'test_file').touch() + + target_dir = self._target_out_testcases / target + target_dir.mkdir() + (target_dir / f'{target}.config').touch() + (target_dir / f'test_file').touch() + + def _verify_soong_zip_commands(self, commands: list[str], targets: list[str]): + """Verify the structure of the zip commands. + + Zip commands have to start with the soong_zip binary path, then are followed + by a couple of options and the name of the file being zipped. Depending on + which zip we are creating look for a few essential items being added in + those zips. + + Args: + commands: list of command lists + targets: list of targets expected to be in general-tests.zip + """ + for command in commands: + self.assertEqual( + '/tmp/top/prebuilts/build-tools/linux-x86/bin/soong_zip', + command[0], + ) + self.assertEqual('-d', command[1]) + self.assertEqual('-o', command[2]) + match (command[3]): + case '/tmp/top/out/dist/general-tests_configs.zip': + self.assertIn(f'{self._host_out}/host_general-tests_list', command) + self.assertIn( + f'{self._product_out}/target_general-tests_list', command + ) + return + case '/tmp/top/out/dist/general-tests_list.zip': + self.assertIn('-f', command) + self.assertIn(f'{self._host_out}/general-tests_list', command) + return + case '/tmp/top/out/dist/general-tests.zip': + for target in targets: + self.assertIn(f'{self._host_out_testcases}/{target}', command) + self.assertIn(f'{self._target_out_testcases}/{target}', command) + self.assertIn( + f'{self._soong_host_out}/framework/cts-tradefed.jar', command + ) + self.assertIn( + f'{self._soong_host_out}/framework/compatibility-host-util.jar', + command, + ) + self.assertIn( + f'{self._soong_host_out}/framework/vts-tradefed.jar', command + ) + return + case _: + self.fail(f'malformed command: {command}') + if __name__ == '__main__': # Setup logging to be silent so unit tests can pass through TF. diff --git a/ci/test_discovery_agent.py b/ci/test_discovery_agent.py new file mode 100644 index 0000000000..008ee47f8e --- /dev/null +++ b/ci/test_discovery_agent.py @@ -0,0 +1,120 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test discovery agent that uses TradeFed to discover test artifacts.""" +import glob +import json +import logging +import os +import subprocess + + +class TestDiscoveryAgent: + """Test discovery agent.""" + + _TRADEFED_PREBUILT_JAR_RELATIVE_PATH = ( + "vendor/google_tradefederation/prebuilts/filegroups/google-tradefed/" + ) + + _TRADEFED_NO_POSSIBLE_TEST_DISCOVERY_KEY = "NoPossibleTestDiscovery" + + _TRADEFED_TEST_ZIP_REGEXES_LIST_KEY = "TestZipRegexes" + + _TRADEFED_DISCOVERY_OUTPUT_FILE_NAME = "test_discovery_agent.txt" + + def __init__( + self, + tradefed_args: list[str], + test_mapping_zip_path: str = "", + tradefed_jar_revelant_files_path: str = _TRADEFED_PREBUILT_JAR_RELATIVE_PATH, + ): + self.tradefed_args = tradefed_args + self.test_mapping_zip_path = test_mapping_zip_path + self.tradefed_jar_relevant_files_path = tradefed_jar_revelant_files_path + + def discover_test_zip_regexes(self) -> list[str]: + """Discover test zip regexes from TradeFed. + + Returns: + A list of test zip regexes that TF is going to try to pull files from. + """ + test_discovery_output_file_name = os.path.join( + os.environ.get('TOP'), 'out', self._TRADEFED_DISCOVERY_OUTPUT_FILE_NAME + ) + with open( + test_discovery_output_file_name, mode="w+t" + ) as test_discovery_output_file: + java_args = [] + java_args.append("prebuilts/jdk/jdk21/linux-x86/bin/java") + java_args.append("-cp") + java_args.append( + self.create_classpath(self.tradefed_jar_relevant_files_path) + ) + java_args.append( + "com.android.tradefed.observatory.TestZipDiscoveryExecutor" + ) + java_args.extend(self.tradefed_args) + env = os.environ.copy() + env.update({"DISCOVERY_OUTPUT_FILE": test_discovery_output_file.name}) + logging.info(f"Calling test discovery with args: {java_args}") + try: + result = subprocess.run(args=java_args, env=env, text=True, check=True) + logging.info(f"Test zip discovery output: {result.stdout}") + except subprocess.CalledProcessError as e: + raise TestDiscoveryError( + f"Failed to run test discovery, strout: {e.stdout}, strerr:" + f" {e.stderr}, returncode: {e.returncode}" + ) + data = json.loads(test_discovery_output_file.read()) + logging.info(f"Test discovery result file content: {data}") + if ( + self._TRADEFED_NO_POSSIBLE_TEST_DISCOVERY_KEY in data + and data[self._TRADEFED_NO_POSSIBLE_TEST_DISCOVERY_KEY] + ): + raise TestDiscoveryError("No possible test discovery") + if ( + data[self._TRADEFED_TEST_ZIP_REGEXES_LIST_KEY] is None + or data[self._TRADEFED_TEST_ZIP_REGEXES_LIST_KEY] is [] + ): + raise TestDiscoveryError("No test zip regexes returned") + return data[self._TRADEFED_TEST_ZIP_REGEXES_LIST_KEY] + + def discover_test_modules(self) -> list[str]: + """Discover test modules from TradeFed. + + Returns: + A list of test modules that TradeFed is going to execute based on the + TradeFed test args. + """ + return [] + + def create_classpath(self, directory): + """Creates a classpath string from all .jar files in the given directory. + + Args: + directory: The directory to search for .jar files. + + Returns: + A string representing the classpath, with jar files separated by the + OS-specific path separator (e.g., ':' on Linux/macOS, ';' on Windows). + """ + jar_files = glob.glob(os.path.join(directory, "*.jar")) + return os.pathsep.join(jar_files) + + +class TestDiscoveryError(Exception): + """A TestDiscoveryErrorclass.""" + + def __init__(self, message): + super().__init__(message) + self.message = message diff --git a/cogsetup.sh b/cogsetup.sh deleted file mode 100644 index 5c64a068e0..0000000000 --- a/cogsetup.sh +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (C) 2023 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file is executed by build/envsetup.sh, and can use anything -# defined in envsetup.sh. -function _create_out_symlink_for_cog() { - if [[ "${OUT_DIR}" == "" ]]; then - OUT_DIR="out" - fi - - # getoutdir ensures paths are absolute. envsetup could be called from a - # directory other than the root of the source tree - local outdir=$(getoutdir) - if [[ -L "${outdir}" ]]; then - return - fi - if [ -d "${outdir}" ]; then - echo -e "\tOutput directory ${outdir} cannot be present in a Cog workspace." - echo -e "\tDelete \"${outdir}\" or create a symlink from \"${outdir}\" to a directory outside your workspace." - return 1 - fi - - DEFAULT_OUTPUT_DIR="${HOME}/.cog/android-build-out" - mkdir -p ${DEFAULT_OUTPUT_DIR} - ln -s ${DEFAULT_OUTPUT_DIR} ${outdir} -} - -# This function sets up the build environment to be appropriate for Cog. -function _setup_cog_env() { - _create_out_symlink_for_cog - if [ "$?" -eq "1" ]; then - echo -e "\e[0;33mWARNING:\e[00m Cog environment setup failed!" - return 1 - fi - - export ANDROID_BUILD_ENVIRONMENT_CONFIG="googler-cog" - - # Running repo command within Cog workspaces is not supported, so override - # it with this function. If the user is running repo within a Cog workspace, - # we'll fail with an error, otherwise, we run the original repo command with - # the given args. - if ! ORIG_REPO_PATH=`which repo`; then - return 0 - fi - function repo { - if [[ "${PWD}" == /google/cog/* ]]; then - echo -e "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces." - return 1 - fi - ${ORIG_REPO_PATH} "$@" - } -} - -if [[ "${PWD}" != /google/cog/* ]]; then - echo -e "\e[01;31mERROR:\e[0m This script must be run from a Cog workspace." -fi - -_setup_cog_env diff --git a/core/Makefile b/core/Makefile index f376014312..0d6b175b54 100644 --- a/core/Makefile +++ b/core/Makefile @@ -169,7 +169,7 @@ $(foreach cf,$(unique_product_copy_files_pairs), \ $(eval $(call copy-xml-file-checked,$(_src),$(_fulldest))),\ $(if $(and $(filter %.jar,$(_dest)),$(filter $(basename $(notdir $(_dest))),$(PRODUCT_LOADED_BY_PRIVILEGED_MODULES))),\ $(eval $(call copy-and-uncompress-dexs,$(_src),$(_fulldest))), \ - $(if $(filter init%rc,$(notdir $(_dest)))$(filter %/etc/init,$(dir $(_dest))),\ + $(if $(filter init%rc,$(notdir $(_dest)))$(filter %/etc/init/,$(dir $(_dest))),\ $(eval $(call copy-init-script-file-checked,$(_src),$(_fulldest))),\ $(if $(and $(filter true,$(check_elf_prebuilt_product_copy_files)), \ $(filter bin lib lib64,$(subst /,$(space),$(_dest)))), \ @@ -192,6 +192,34 @@ product_copy_files_ignored := unique_product_copy_files_pairs := unique_product_copy_files_destinations := + +# Returns a list of EXTRA_INSTALL_ZIPS trios whose primary file is contained within $(1) +# The trios will contain the primary installed file : the directory to unzip the zip to : the zip +define relevant-extra-install-zips +$(strip $(foreach p,$(EXTRA_INSTALL_ZIPS), \ + $(if $(filter $(call word-colon,1,$(p)),$(1)), \ + $(p)))) +endef + +# Writes a text file that contains all of the files that will be inside a partition. +# All the file paths will be relative to the partition's staging directory. +# It will also take into account files inside zips listed in EXTRA_INSTALL_ZIPS. +# +# Arguments: +# $(1): Output file +# $(2): The partition's staging directory +# $(3): Files to include in the partition +define write-partition-file-list +$(1): PRIVATE_FILES := $(subst $(2)/,,$(filter $(2)/%,$(3))) +$(1): PRIVATE_EXTRA_INSTALL_ZIPS := $(call relevant-extra-install-zips,$(filter $(2)/%,$(3))) +$(1): $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(foreach p,$(call relevant-extra-install-zips,$(filter $(2)/%,$(3))),$(call word-colon,3,$(p))) + @echo Writing $$@ + rm -f $$@ + echo -n > $$@ + $$(foreach f,$$(PRIVATE_FILES),echo "$$(f)" >> $$@$$(newline)) + $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(2) $$(PRIVATE_EXTRA_INSTALL_ZIPS) >> $$@ +endef + # ----------------------------------------------------------------- # Returns the max allowed size for an image suitable for hash verification # (e.g., boot.img, recovery.img, etc). @@ -267,11 +295,6 @@ ndk-docs: $(ndk_doxygen_out)/index.html .PHONY: ndk-docs endif -ifeq ($(HOST_OS),linux) -$(call dist-for-goals,sdk,$(API_FINGERPRINT)) -$(call dist-for-goals,droidcore,$(API_FINGERPRINT)) -endif - INSTALLED_RECOVERYIMAGE_TARGET := # Build recovery image if # BUILDING_RECOVERY_IMAGE && !BOARD_USES_RECOVERY_AS_BOOT && !BOARD_MOVE_RECOVERY_RESOURCES_TO_VENDOR_BOOT. @@ -692,7 +715,7 @@ endif BOARD_KERNEL_MODULE_DIRS += top -# Default to not generating modules.dep for kernel modules on system +# Default to not generating modules.load for kernel modules on system # side. We should only load these modules if they are depended by vendor # side modules. ifeq ($(BOARD_SYSTEM_KERNEL_MODULES_LOAD),) @@ -745,10 +768,7 @@ endif # $5 partition tag # $6 output file define _apkcerts_write_line -$(hide) echo -n 'name="$(1).apk" certificate="$2" private_key="$3"' >> $6 -$(if $(4), $(hide) echo -n ' compressed="$4"' >> $6) -$(if $(5), $(hide) echo -n ' partition="$5"' >> $6) -$(hide) echo '' >> $6 +$(hide) echo 'name="$(1).apk" certificate="$2" private_key="$3"$(if $(4), compressed="$4")$(if $(5), partition="$5")' >> $6 endef @@ -819,16 +839,6 @@ endif # ----------------------------------------------------------------- -# build system stats -BUILD_SYSTEM_STATS := $(PRODUCT_OUT)/build_system_stats.txt -$(BUILD_SYSTEM_STATS): - @rm -f $@ - @$(foreach s,$(STATS.MODULE_TYPE),echo "modules_type_make,$(s),$(words $(STATS.MODULE_TYPE.$(s)))" >>$@;) - @$(foreach s,$(STATS.SOONG_MODULE_TYPE),echo "modules_type_soong,$(s),$(STATS.SOONG_MODULE_TYPE.$(s))" >>$@;) -$(call declare-1p-target,$(BUILD_SYSTEM_STATS),build) -$(call dist-for-goals,droidcore-unbundled,$(BUILD_SYSTEM_STATS)) - -# ----------------------------------------------------------------- # build /product/etc/security/avb/system_other.avbpubkey if needed ifdef BUILDING_SYSTEM_OTHER_IMAGE ifeq ($(BOARD_AVB_ENABLE),true) @@ -844,6 +854,7 @@ SOONG_CONV := $(sort $(SOONG_CONV)) SOONG_CONV_DATA := $(call intermediates-dir-for,PACKAGING,soong_conversion)/soong_conv_data $(SOONG_CONV_DATA): @rm -f $@ + @touch $@ # This file must be present even if SOONG_CONV is empty. @$(foreach s,$(SOONG_CONV),echo "$(s),$(SOONG_CONV.$(s).TYPE),$(sort $(SOONG_CONV.$(s).PROBLEMS)),$(sort $(filter-out $(SOONG_ALREADY_CONV),$(SOONG_CONV.$(s).DEPS))),$(sort $(SOONG_CONV.$(s).MAKEFILES)),$(sort $(SOONG_CONV.$(s).INSTALLED))" >>$@;) $(call declare-1p-target,$(SOONG_CONV_DATA),build) @@ -856,11 +867,6 @@ $(SOONG_TO_CONVERT): $(SOONG_CONV_DATA) $(SOONG_TO_CONVERT_SCRIPT) $(call declare-1p-target,$(SOONG_TO_CONVERT),build) $(call dist-for-goals,droidcore-unbundled,$(SOONG_TO_CONVERT)) -$(PRODUCT_OUT)/product_packages.txt: - @rm -f $@ - echo "" > $@ - $(foreach x,$(PRODUCT_PACKAGES),echo $(x) >> $@$(newline)) - MK2BP_CATALOG_SCRIPT := build/make/tools/mk2bp_catalog.py PRODUCT_PACKAGES_TXT := $(PRODUCT_OUT)/product_packages.txt MK2BP_REMAINING_HTML := $(PRODUCT_OUT)/mk2bp_remaining.html @@ -907,18 +913,6 @@ $(call declare-0p-target,$(WALL_WERROR)) $(call dist-for-goals,droidcore-unbundled,$(WALL_WERROR)) -# ----------------------------------------------------------------- -# Modules missing profile files -PGO_PROFILE_MISSING := $(PRODUCT_OUT)/pgo_profile_file_missing.txt -$(PGO_PROFILE_MISSING): - @rm -f $@ - echo "# Modules missing PGO profile files" >> $@ - for m in $(SOONG_MODULES_MISSING_PGO_PROFILE_FILE); do echo $$m >> $@; done - -$(call declare-0p-target,$(PGO_PROFILE_MISSING)) - -$(call dist-for-goals,droidcore,$(PGO_PROFILE_MISSING)) - CERTIFICATE_VIOLATION_MODULES_FILENAME := $(PRODUCT_OUT)/certificate_violation_modules.txt $(CERTIFICATE_VIOLATION_MODULES_FILENAME): rm -f $@ @@ -941,27 +935,12 @@ systemimage: # ----------------------------------------------------------------- -.PHONY: event-log-tags - -# Produce an event logs tag file for everything we know about, in order -# to properly allocate numbers. Then produce a file that's filtered -# for what's going to be installed. - -all_event_log_tags_file := $(TARGET_OUT_COMMON_INTERMEDIATES)/all-event-log-tags.txt - event_log_tags_file := $(TARGET_OUT)/etc/event-log-tags # Include tags from all packages that we know about all_event_log_tags_src := \ $(sort $(foreach m, $(ALL_MODULES), $(ALL_MODULES.$(m).EVENT_LOG_TAGS))) -$(all_event_log_tags_file): PRIVATE_SRC_FILES := $(all_event_log_tags_src) -$(all_event_log_tags_file): $(all_event_log_tags_src) $(MERGETAGS) build/make/tools/event_log_tags.py - $(hide) mkdir -p $(dir $@) - $(hide) $(MERGETAGS) -o $@ $(PRIVATE_SRC_FILES) - -$(call declare-0p-target,$(all_event_log_tags_file)) - # Include tags from all packages included in this product, plus all # tags that are part of the system (ie, not in a vendor/ or device/ # directory). @@ -973,13 +952,13 @@ event_log_tags_src := \ $(filter-out vendor/% device/% out/%,$(all_event_log_tags_src))) $(event_log_tags_file): PRIVATE_SRC_FILES := $(event_log_tags_src) -$(event_log_tags_file): PRIVATE_MERGED_FILE := $(all_event_log_tags_file) -$(event_log_tags_file): $(event_log_tags_src) $(all_event_log_tags_file) $(MERGETAGS) build/make/tools/event_log_tags.py +$(event_log_tags_file): $(event_log_tags_src) $(MERGETAGS) $(hide) mkdir -p $(dir $@) - $(hide) $(MERGETAGS) -o $@ -m $(PRIVATE_MERGED_FILE) $(PRIVATE_SRC_FILES) + $(hide) $(MERGETAGS) -o $@ $(PRIVATE_SRC_FILES) $(eval $(call declare-0p-target,$(event_log_tags_file))) +.PHONY: event-log-tags event-log-tags: $(event_log_tags_file) ALL_DEFAULT_INSTALLED_MODULES += $(event_log_tags_file) @@ -1219,53 +1198,8 @@ endif endif # BOARD_PREBUILT_DTBOIMAGE_16KB -ifneq ($(BOARD_KERNEL_PATH_16K),) -BUILT_KERNEL_16K_TARGET := $(PRODUCT_OUT)/kernel_16k - -$(eval $(call copy-one-file,$(BOARD_KERNEL_PATH_16K),$(BUILT_KERNEL_16K_TARGET))) - -# Copies BOARD_KERNEL_PATH_16K to output directory as is -kernel_16k: $(BUILT_KERNEL_16K_TARGET) -.PHONY: kernel_16k - -BUILT_BOOTIMAGE_16K_TARGET := $(PRODUCT_OUT)/boot_16k.img - -BOARD_KERNEL_16K_BOOTIMAGE_PARTITION_SIZE := $(BOARD_BOOTIMAGE_PARTITION_SIZE) - -$(BUILT_BOOTIMAGE_16K_TARGET): $(MKBOOTIMG) $(AVBTOOL) $(INTERNAL_BOOTIMAGE_FILES) $(BOARD_AVB_BOOT_KEY_PATH) $(BUILT_KERNEL_16K_TARGET) - $(call pretty,"Target boot 16k image: $@") - $(call build_boot_from_kernel_avb_enabled,$@,$(BUILT_KERNEL_16K_TARGET)) - - -bootimage_16k: $(BUILT_BOOTIMAGE_16K_TARGET) -.PHONY: bootimage_16k - -BUILT_BOOT_OTA_PACKAGE_16K := $(PRODUCT_OUT)/boot_ota_16k.zip -$(BUILT_BOOT_OTA_PACKAGE_16K): $(OTA_FROM_RAW_IMG) \ - $(BUILT_BOOTIMAGE_16K_TARGET) \ - $(INSTALLED_BOOTIMAGE_TARGET) \ - $(DEFAULT_SYSTEM_DEV_CERTIFICATE).pk8 \ - $(INSTALLED_DTBOIMAGE_16KB_TARGET) \ - $(INSTALLED_DTBOIMAGE_TARGET) - $(OTA_FROM_RAW_IMG) --package_key $(DEFAULT_SYSTEM_DEV_CERTIFICATE) \ - --max_timestamp `cat $(BUILD_DATETIME_FILE)` \ - --path $(HOST_OUT) \ - --partition_name $(if $(and $(INSTALLED_DTBOIMAGE_TARGET),\ - $(INSTALLED_DTBOIMAGE_16KB_TARGET)),\ - boot$(comma)dtbo,\ - boot) \ - --output $@ \ - $(if $(BOARD_16K_OTA_USE_INCREMENTAL),\ - $(INSTALLED_BOOTIMAGE_TARGET):$(BUILT_BOOTIMAGE_16K_TARGET),\ - $(BUILT_BOOTIMAGE_16K_TARGET)\ - )\ - $(if $(and $(INSTALLED_DTBOIMAGE_TARGET),$(INSTALLED_DTBOIMAGE_16KB_TARGET)),\ - $(INSTALLED_DTBOIMAGE_16KB_TARGET)) - -boototapackage_16k: $(BUILT_BOOT_OTA_PACKAGE_16K) -.PHONY: boototapackage_16k - -endif +ramdisk_intermediates :=$= $(call intermediates-dir-for,PACKAGING,ramdisk) +$(eval $(call write-partition-file-list,$(ramdisk_intermediates)/file_list.txt,$(TARGET_RAMDISK_OUT),$(INTERNAL_RAMDISK_FILES))) # The value of RAMDISK_NODE_LIST is defined in system/core/rootdir/Android.bp. # This file contains /dev nodes description added to the generic ramdisk @@ -1500,6 +1434,55 @@ endif # BOARD_PREBUILT_BOOTIMAGE endif # my_installed_prebuilt_gki_apex not defined ifneq ($(BOARD_KERNEL_PATH_16K),) + +BUILT_KERNEL_16K_TARGET := $(PRODUCT_OUT)/kernel_16k + +$(eval $(call copy-one-file,$(BOARD_KERNEL_PATH_16K),$(BUILT_KERNEL_16K_TARGET))) + +# Copies BOARD_KERNEL_PATH_16K to output directory as is +kernel_16k: $(BUILT_KERNEL_16K_TARGET) +.PHONY: kernel_16k + +BUILT_BOOTIMAGE_16K_TARGET := $(PRODUCT_OUT)/boot_16k.img + +BOARD_KERNEL_16K_BOOTIMAGE_PARTITION_SIZE := $(BOARD_BOOTIMAGE_PARTITION_SIZE) + +$(BUILT_BOOTIMAGE_16K_TARGET): $(MKBOOTIMG) $(AVBTOOL) $(INTERNAL_BOOTIMAGE_FILES) $(BOARD_AVB_BOOT_KEY_PATH) $(BUILT_KERNEL_16K_TARGET) + $(call pretty,"Target boot 16k image: $@") + $(call build_boot_from_kernel_avb_enabled,$@,$(BUILT_KERNEL_16K_TARGET)) + + +bootimage_16k: $(BUILT_BOOTIMAGE_16K_TARGET) +.PHONY: bootimage_16k + +BUILT_BOOT_OTA_PACKAGE_16K := $(PRODUCT_OUT)/boot_ota_16k.zip +$(BUILT_BOOT_OTA_PACKAGE_16K): PRIVATE_BOOTIMAGE_TARGET := $(INSTALLED_BOOTIMAGE_TARGET) +$(BUILT_BOOT_OTA_PACKAGE_16K): PRIVATE_BOOTIMAGE_16KB_TARGET := $(BUILT_BOOTIMAGE_16K_TARGET) +$(BUILT_BOOT_OTA_PACKAGE_16K): $(OTA_FROM_RAW_IMG) \ + $(DEFAULT_SYSTEM_DEV_CERTIFICATE).pk8 \ + $(INSTALLED_BOOTIMAGE_TARGET) \ + $(BUILT_BOOTIMAGE_16K_TARGET) \ + $(INSTALLED_DTBOIMAGE_16KB_TARGET) \ + $(INSTALLED_DTBOIMAGE_TARGET) + $(OTA_FROM_RAW_IMG) --package_key $(DEFAULT_SYSTEM_DEV_CERTIFICATE) \ + --max_timestamp `cat $(BUILD_DATETIME_FILE)` \ + --path $(HOST_OUT) \ + --partition_name $(if $(and $(INSTALLED_DTBOIMAGE_TARGET),\ + $(INSTALLED_DTBOIMAGE_16KB_TARGET)),\ + boot$(comma)dtbo,\ + boot) \ + --output $@ \ + $(if $(BOARD_16K_OTA_USE_INCREMENTAL),\ + $(PRIVATE_BOOTIMAGE_TARGET):$(PRIVATE_BOOTIMAGE_16KB_TARGET),\ + $(PRIVATE_BOOTIMAGE_16KB_TARGET)\ + )\ + $(if $(and $(INSTALLED_DTBOIMAGE_TARGET),$(INSTALLED_DTBOIMAGE_16KB_TARGET)),\ + $(INSTALLED_DTBOIMAGE_16KB_TARGET)) + +boototapackage_16k: $(BUILT_BOOT_OTA_PACKAGE_16K) +.PHONY: boototapackage_16k + + BUILT_BOOT_OTA_PACKAGE_4K := $(PRODUCT_OUT)/boot_ota_4k.zip $(BUILT_BOOT_OTA_PACKAGE_4K): $(OTA_FROM_RAW_IMG) \ $(INSTALLED_BOOTIMAGE_TARGET) \ @@ -1528,11 +1511,26 @@ boototapackage_4k: $(BUILT_BOOT_OTA_PACKAGE_4K) ifeq ($(BOARD_16K_OTA_MOVE_VENDOR),true) $(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_4K),$(TARGET_OUT_VENDOR)/boot_otas/boot_ota_4k.zip)) $(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_16K),$(TARGET_OUT_VENDOR)/boot_otas/boot_ota_16k.zip)) + ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT_VENDOR)/boot_otas/boot_ota_4k.zip ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT_VENDOR)/boot_otas/boot_ota_16k.zip + +ifneq ($(BOARD_VENDOR_KERNEL_MODULES_2ND_STAGE_16KB_MODE),) +# Add the modules that need to be loaded in the Second Boot Stage +# to /vendor_dlkm/lib/modules/16k-mode +VENDOR_DLKM_16K_MODE_DIR := lib/modules/16k-mode +$(foreach module,$(BOARD_VENDOR_KERNEL_MODULES_2ND_STAGE_16KB_MODE), \ + $(eval $(call copy-one-file,$(TARGET_KERNEL_DIR_16K)/$(module),\ + $(TARGET_OUT_VENDOR_DLKM)/$(VENDOR_DLKM_16K_MODE_DIR)/$(module)))) + +ALL_DEFAULT_INSTALLED_MODULES += $(foreach module,$(BOARD_VENDOR_KERNEL_MODULES_2ND_STAGE_16KB_MODE),\ + $(TARGET_OUT_VENDOR_DLKM)/$(VENDOR_DLKM_16K_MODE_DIR)/$(module)) +endif # BOARD_VENDOR_KERNEL_MODULES_2ND_STAGE_16KB_MODE not empty + else $(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_4K),$(TARGET_OUT)/boot_otas/boot_ota_4k.zip)) $(eval $(call copy-one-file,$(BUILT_BOOT_OTA_PACKAGE_16K),$(TARGET_OUT)/boot_otas/boot_ota_16k.zip)) + ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT)/boot_otas/boot_ota_4k.zip ALL_DEFAULT_INSTALLED_MODULES += $(TARGET_OUT)/boot_otas/boot_ota_16k.zip endif # BOARD_16K_OTA_MOVE_VENDOR == true @@ -1564,6 +1562,7 @@ $(INSTALLED_INIT_BOOT_IMAGE_TARGET): $(AVBTOOL) $(BOARD_AVB_INIT_BOOT_KEY_PATH) $(AVBTOOL) add_hash_footer \ --image $@ \ $(call get-partition-size-argument,$(BOARD_INIT_BOOT_IMAGE_PARTITION_SIZE)) \ + --salt $$(sha256sum $(BUILD_NUMBER_FILE) $(BUILD_DATETIME_FILE) | cut -d " " -f 1 | tr -d '\n') \ --partition_name init_boot $(INTERNAL_AVB_INIT_BOOT_SIGNING_ARGS) \ $(BOARD_AVB_INIT_BOOT_ADD_HASH_FOOTER_ARGS) @@ -1622,6 +1621,8 @@ INTERNAL_VENDOR_RAMDISK_FILES := $(filter $(TARGET_VENDOR_RAMDISK_OUT)/%, \ $(ALL_DEFAULT_INSTALLED_MODULES)) INTERNAL_VENDOR_RAMDISK_TARGET := $(call intermediates-dir-for,PACKAGING,vendor_boot)/vendor_ramdisk.cpio$(RAMDISK_EXT) +vendor_ramdisk_intermediates :=$= $(call intermediates-dir-for,PACKAGING,vendor_ramdisk) +$(eval $(call write-partition-file-list,$(vendor_ramdisk_intermediates)/file_list.txt,$(TARGET_VENDOR_RAMDISK_OUT),$(INTERNAL_VENDOR_RAMDISK_FILES))) # Exclude recovery files in the default vendor ramdisk if including a standalone # recovery ramdisk in vendor_boot. @@ -1676,12 +1677,13 @@ ifdef INTERNAL_KERNEL_CMDLINE INTERNAL_VENDOR_BOOTIMAGE_ARGS += --vendor_cmdline "$(INTERNAL_KERNEL_CMDLINE)" endif -ifdef INTERNAL_BOOTCONFIG +ifneq (, $(INTERNAL_BOOTCONFIG)$(INTERNAL_BOOTCONFIG_FILE)) INTERNAL_VENDOR_BOOTCONFIG_TARGET := $(PRODUCT_OUT)/vendor-bootconfig.img $(INTERNAL_VENDOR_BOOTCONFIG_TARGET): rm -f $@ $(foreach param,$(INTERNAL_BOOTCONFIG), \ printf "%s\n" $(param) >> $@;) + cat $(INTERNAL_BOOTCONFIG_FILE) >> $@ INTERNAL_VENDOR_BOOTIMAGE_ARGS += --vendor_bootconfig $(INTERNAL_VENDOR_BOOTCONFIG_TARGET) endif @@ -1736,6 +1738,7 @@ $(INSTALLED_VENDOR_BOOTIMAGE_TARGET): $(AVBTOOL) $(BOARD_AVB_VENDOR_BOOTIMAGE_KE $(AVBTOOL) add_hash_footer \ --image $@ \ $(call get-partition-size-argument,$(BOARD_VENDOR_BOOTIMAGE_PARTITION_SIZE)) \ + --salt $$(sha256sum $(BUILD_NUMBER_FILE) $(BUILD_DATETIME_FILE) | cut -d " " -f 1 | tr -d '\n') \ --partition_name vendor_boot $(INTERNAL_AVB_VENDOR_BOOT_SIGNING_ARGS) \ $(BOARD_AVB_VENDOR_BOOT_ADD_HASH_FOOTER_ARGS) else @@ -1784,6 +1787,7 @@ INTERNAL_VENDOR_KERNEL_RAMDISK_FILES := $(filter $(TARGET_VENDOR_KERNEL_RAMDISK_ INTERNAL_VENDOR_KERNEL_RAMDISK_TARGET := $(call intermediates-dir-for,PACKAGING,vendor_kernel_boot)/vendor_kernel_ramdisk.cpio$(RAMDISK_EXT) $(INTERNAL_VENDOR_KERNEL_RAMDISK_TARGET): $(MKBOOTFS) $(INTERNAL_VENDOR_KERNEL_RAMDISK_FILES) | $(COMPRESSION_COMMAND_DEPS) + $(hide) : $(words $(INTERNAL_VENDOR_KERNEL_RAMDISK_FILES)) $(MKBOOTFS) -d $(TARGET_OUT) $(TARGET_VENDOR_KERNEL_RAMDISK_OUT) | $(COMPRESSION_COMMAND) > $@ INSTALLED_VENDOR_KERNEL_RAMDISK_TARGET := $(PRODUCT_OUT)/vendor_kernel_ramdisk.img @@ -1975,7 +1979,8 @@ ALL_INSTALLED_NOTICE_FILES := \ # $1 installed file path, e.g. out/target/product/vsoc_x86_64/system_ext/etc/NOTICE.xml.gz define is-notice-file -$(if $(findstring $1,$(ALL_INSTALLED_NOTICE_FILES)),Y) +$(if $(filter true,$(PRODUCT_USE_SOONG_NOTICE_XML)),, \ + $(if $(findstring $1,$(ALL_INSTALLED_NOTICE_FILES)),Y)) endef # Notice files are copied to TARGET_OUT_NOTICE_FILES as a side-effect of their module @@ -2378,7 +2383,7 @@ $(if $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)),\ $(hide) echo "root_dir=$(TARGET_ROOT_OUT)" >> $(1) $(if $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITION_SIZE)),\ $(hide) echo "use_dynamic_partition_size=true" >> $(1)) -$(if $(COPY_IMAGES_FOR_TARGET_FILES_ZIP),\ +$(if $(USE_FIXED_TIMESTAMP_IMG_FILES)$(COPY_IMAGES_FOR_TARGET_FILES_ZIP),\ $(hide) echo "use_fixed_timestamp=true" >> $(1)) $(if $(3),$(hide) $(foreach kv,$(3),echo "$(kv)" >> $(1);)) $(hide) sort -o $(1) $(1) @@ -2624,7 +2629,7 @@ ifndef TARGET_PRIVATE_RES_DIRS TARGET_PRIVATE_RES_DIRS := $(wildcard $(TARGET_DEVICE_DIR)/recovery/res) endif recovery_resource_deps := $(shell find $(recovery_resources_common) \ - $(TARGET_PRIVATE_RES_DIRS) -type f) + $(TARGET_PRIVATE_RES_DIRS) -type f -not -name "*.bp") recovery_resource_deps += $(generated_recovery_text_files) @@ -2897,6 +2902,9 @@ ifneq ($(BOARD_NAND_SPARE_SIZE),) $(error MTD device is no longer supported and thus BOARD_NAND_SPARE_SIZE is deprecated.) endif +recovery_intermediates := $(call intermediates-dir-for,PACKAGING,recovery) +$(eval $(call write-partition-file-list,$(recovery_intermediates)/file_list.txt,$(TARGET_RECOVERY_OUT),$(INTERNAL_RECOVERYIMAGE_FILES))) + # ----------------------------------------------------------------- # Build debug ramdisk and debug boot image. @@ -3469,31 +3477,6 @@ endif FULL_SYSTEMIMAGE_DEPS += $(INTERNAL_ROOT_FILES) $(INSTALLED_FILES_FILE_ROOT) -# Returns a list of EXTRA_INSTALL_ZIPS trios whose primary file is contained within $(1) -# The trios will contain the primary installed file : the directory to unzip the zip to : the zip -define relevant-extra-install-zips -$(strip $(foreach p,$(EXTRA_INSTALL_ZIPS), \ - $(if $(filter $(call word-colon,1,$(p)),$(1)), \ - $(p)))) -endef - -# Writes a text file that contains all of the files that will be inside a partition. -# All the file paths will be relative to the partition's staging directory. -# It will also take into account files inside zips listed in EXTRA_INSTALL_ZIPS. -# -# Arguments: -# $(1): Output file -# $(2): The partition's staging directory -# $(3): Files to include in the partition -define write-partition-file-list -$(1): $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(foreach p,$(call relevant-extra-install-zips,$(filter $(2)/%,$(3))),$(call word-colon,3,$(p))) - @echo Writing $$@ - rm -f $$@ - echo -n > $$@ - $$(foreach f,$(subst $(2)/,,$(filter $(2)/%,$(3))),echo "$$(f)" >> $$@$$(newline)) - $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(2) $(call relevant-extra-install-zips,$(filter $(2)/%,$(3))) >> $$@ -endef - # ----------------------------------------------------------------- ifdef BUILDING_SYSTEM_IMAGE @@ -3501,16 +3484,20 @@ ifdef BUILDING_SYSTEM_IMAGE # Collect all available stub libraries installed in system and install with predefined linker configuration # Also append LLNDK libraries in the APEX as required libs SYSTEM_LINKER_CONFIG := $(TARGET_OUT)/etc/linker.config.pb -SYSTEM_LINKER_CONFIG_SOURCE := $(call intermediates-dir-for,ETC,system_linker_config)/system_linker_config +SYSTEM_LINKER_CONFIG_SOURCE := system/core/rootdir/etc/linker.config.json $(SYSTEM_LINKER_CONFIG): PRIVATE_SYSTEM_LINKER_CONFIG_SOURCE := $(SYSTEM_LINKER_CONFIG_SOURCE) $(SYSTEM_LINKER_CONFIG): $(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE) | conv_linker_config @echo Creating linker config: $@ @mkdir -p $(dir $@) - @rm -f $@ - $(HOST_OUT_EXECUTABLES)/conv_linker_config systemprovide --source $(PRIVATE_SYSTEM_LINKER_CONFIG_SOURCE) \ + @rm -f $@ $@.step1 + $(HOST_OUT_EXECUTABLES)/conv_linker_config proto --force -s $(PRIVATE_SYSTEM_LINKER_CONFIG_SOURCE) -o $@.step1 + $(HOST_OUT_EXECUTABLES)/conv_linker_config systemprovide --source $@.step1 \ --output $@ --value "$(STUB_LIBRARIES)" --system "$(TARGET_OUT)" $(HOST_OUT_EXECUTABLES)/conv_linker_config append --source $@ --output $@ --key requireLibs \ --value "$(foreach lib,$(LLNDK_MOVED_TO_APEX_LIBRARIES), $(lib).so)" + $(HOST_OUT_EXECUTABLES)/conv_linker_config append --source $@ --output $@ --key provideLibs \ + --value "$(foreach lib,$(PRODUCT_EXTRA_STUB_LIBRARIES), $(lib).so)" + rm -f $@.step1 $(call declare-1p-target,$(SYSTEM_LINKER_CONFIG),) $(call declare-license-deps,$(SYSTEM_LINKER_CONFIG),$(INTERNAL_SYSTEMIMAGE_FILES) $(SYSTEM_LINKER_CONFIG_SOURCE)) @@ -3563,14 +3550,24 @@ ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),) file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX) system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp +# The build configuration to build the REL version may have more files to allow. +# Use allowlist_next in addition to the allowlist in this case. +system_file_diff_allowlist_next := +ifeq (REL,$(PLATFORM_VERSION_CODENAME)) +system_file_diff_allowlist_next := $(ALL_MODULES.system_image_diff_allowlist_next.INSTALLED) +$(system_file_diff_timestamp): PRIVATE_ALLOWLIST_NEXT := $(system_file_diff_allowlist_next) +endif $(system_file_diff_timestamp): \ $(systemimage_intermediates)/file_list.txt \ $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \ + $(system_file_diff_allowlist_next) \ $(file_list_diff) $(file_list_diff) $(systemimage_intermediates)/file_list.txt \ $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \ - $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) + $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) \ + --allowlists $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \ + $(PRIVATE_ALLOWLIST_NEXT) touch $@ $(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp) @@ -3588,10 +3585,9 @@ ifeq ($(USE_SOONG_DEFINED_SYSTEM_IMAGE),true) ifeq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),) $(error PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE must be set if USE_SOONG_DEFINED_SYSTEM_IMAGE is true) endif -soong_defined_system_image := $(call intermediates-dir-for,ETC,$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE))/$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) -$(BUILT_SYSTEMIMAGE): $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt $(soong_defined_system_image) -$(eval $(call copy-one-file, $(soong_defined_system_image), $(BUILT_SYSTEMIMAGE))) -soong_defined_system_image := +SOONG_DEFINED_SYSTEM_IMAGE_BASE := $(dir $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST)) +$(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt $(SOONG_DEFINED_SYSTEM_IMAGE_PATH) +$(eval $(call copy-one-file, $(SOONG_DEFINED_SYSTEM_IMAGE_PATH), $(BUILT_SYSTEMIMAGE))) else $(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt $(call build-systemimage-target,$@) @@ -3996,6 +3992,21 @@ INTERNAL_PRODUCTIMAGE_FILES := \ $(filter $(TARGET_OUT_PRODUCT)/%,\ $(ALL_DEFAULT_INSTALLED_MODULES)) +# Install product/etc/linker.config.pb with PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS +product_linker_config_file := $(TARGET_OUT_PRODUCT)/etc/linker.config.pb +$(product_linker_config_file): private_linker_config_fragments := $(PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS) +$(product_linker_config_file): $(INTERNAL_PRODUCTIMAGE_FILES) | $(HOST_OUT_EXECUTABLES)/conv_linker_config + @echo Creating linker config: $@ + @mkdir -p $(dir $@) + @rm -f $@ + $(HOST_OUT_EXECUTABLES)/conv_linker_config proto \ + --source $(call normalize-path-list,$(private_linker_config_fragments)) \ + --output $@ +$(call define declare-1p-target,$(product_linker_config_file),) +INTERNAL_PRODUCTIMAGE_FILES += $(product_linker_config_file) +ALL_DEFAULT_INSTALLED_MODULES += $(product_linker_config_file) + + INSTALLED_FILES_FILE_PRODUCT := $(PRODUCT_OUT)/installed-files-product.txt INSTALLED_FILES_JSON_PRODUCT := $(INSTALLED_FILES_FILE_PRODUCT:.txt=.json) $(INSTALLED_FILES_FILE_PRODUCT): .KATI_IMPLICIT_OUTPUTS := $(INSTALLED_FILES_JSON_PRODUCT) @@ -4437,6 +4448,25 @@ INTERNAL_PVMFWIMAGE_FILES := $(call module-target-built-files,pvmfw_img) INTERNAL_PVMFW_EMBEDDED_AVBKEY := $(call module-target-built-files,pvmfw_embedded_key_pub_bin) INTERNAL_PVMFW_SYMBOL := $(TARGET_OUT_EXECUTABLES_UNSTRIPPED)/pvmfw +# If pvmfw target is not available and there is a prebuilt available use prebuilt +# NOTE: This is only a temporary feature for x86_64 and is not meant to be supported for long. +# TODO(b/391333413): Don't allow use of pvmfw prebuilts as soon as it is possible +ifeq ($(INTERNAL_PVMFWIMAGE_FILES),) +ifneq ($(PRODUCT_PVMFW_IMAGE_PREBUILT),) +INTERNAL_PVMFWIMAGE_FILES := $(call module-target-built-files,$(PRODUCT_PVMFW_IMAGE_PREBUILT)) +INTERNAL_PVMFW_SYMBOL := + +ifneq ($(PRODUCT_PVMFW_BIN_PREBUILT),) +INSTALLED_PVMFW_BINARY_TARGET := $(call module-target-built-files,$(PRODUCT_PVMFW_BIN_PREBUILT)) +endif # PRODUCT_PVMFW_BIN_PREBUILT + +ifneq ($(PRODUCT_PVMFW_EMBEDDED_AVBKEY_PREBUILT),) +INTERNAL_PVMFW_EMBEDDED_AVBKEY := $(call module-target-built-files,$(PRODUCT_PVMFW_EMBEDDED_AVBKEY_PREBUILT)) +endif # PRODUCT_PVMFW_EMBEDDED_AVBKEY_PREBUILT + +endif # PRODUCT_PVMFW_IMAGE_PREBUILT +endif # INTERNAL_PVMFWIMAGE_FILES + $(call declare-1p-container,$(INSTALLED_PVMFWIMAGE_TARGET),) $(call declare-container-license-deps,$(INSTALLED_PVMFWIMAGE_TARGET),$(INTERNAL_PVMFWIMAGE_FILES),$(PRODUCT_OUT)/:/) @@ -5007,6 +5037,10 @@ define build-chained-vbmeta-image $(foreach image,$(BOARD_AVB_$(call to-upper,$(1))), \ --include_descriptors_from_image $(call images-for-partitions,$(image))) \ --output $@ + # libavb expects to be able to read the maximum vbmeta size, so we must provide a partition + # which matches this or the read will fail. + # See external/avb/libavb/avb_slot_verify.c#VBMETA_MAX_SIZE + truncate -s 65536 $@ endef ifdef BUILDING_SYSTEM_IMAGE @@ -5065,6 +5099,10 @@ define build-vbmetaimage-target $(PRIVATE_AVB_VBMETA_SIGNING_ARGS) \ $(BOARD_AVB_MAKE_VBMETA_IMAGE_ARGS) \ --output $@ + # libavb expects to be able to read the maximum vbmeta size, so we must provide a partition + # which matches this or the read will fail. + # See external/avb/libavb/avb_slot_verify.c#VBMETA_MAX_SIZE + truncate -s 65536 $@ $(hide) rm -rf $(AVB_CHAIN_KEY_DIR) endef @@ -5132,6 +5170,7 @@ INTERNAL_ALLIMAGES_FILES := \ # Run apex_sepolicy_tests for all installed APEXes ifeq (,$(TARGET_BUILD_UNBUNDLED)) +ifneq (,$(filter ext4 erofs,$(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE))) intermediate := $(call intermediates-dir-for,PACKAGING,apex_sepolicy_tests) apex_dirs := \ $(TARGET_OUT)/apex/% \ @@ -5148,11 +5187,10 @@ apex_dirs := define _run_apex_sepolicy_tests $2: $1 \ $(HOST_OUT_EXECUTABLES)/apex_sepolicy_tests \ - $(HOST_OUT_EXECUTABLES)/deapexer \ - $(HOST_OUT_EXECUTABLES)/debugfs_static + $(HOST_OUT_EXECUTABLES)/apex-ls @rm -rf $$@ @mkdir -p $(dir $$@) - $(HOST_OUT_EXECUTABLES)/apex_sepolicy_tests --all -f <($(HOST_OUT_EXECUTABLES)/deapexer --debugfs_path $(HOST_OUT_EXECUTABLES)/debugfs_static list -Z $$<) + $(HOST_OUT_EXECUTABLES)/apex_sepolicy_tests --all -f <($(HOST_OUT_EXECUTABLES)/apex-ls -Z $$<) @touch $$@ endef @@ -5171,6 +5209,7 @@ droid_targets: run_apex_sepolicy_tests apex_files := intermediate := +endif # PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE endif # TARGET_BUILD_UNBUNDLED # ----------------------------------------------------------------- @@ -5266,7 +5305,7 @@ $(vintffm_log): $(HOST_OUT_EXECUTABLES)/vintffm $(check_vintf_system_deps) $(APE --dirmap /system_ext:$(TARGET_OUT_SYSTEM_EXT) \ --dirmap /product:$(TARGET_OUT_PRODUCT) \ --dirmap /apex:$(APEX_OUT) \ - $(VINTF_FRAMEWORK_MANIFEST_FROZEN_DIR) > $@ 2>&1 ) || ( cat $@ && exit 1 ) + system/libhidl/vintfdata/frozen > $@ 2>&1 ) || ( cat $@ && exit 1 ) $(call declare-1p-target,$(vintffm_log)) @@ -5405,7 +5444,8 @@ ifeq (default,$(ENABLE_UFFD_GC)) ifneq (,$(BUILT_KERNEL_VERSION_FILE)) $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC): $(BUILT_KERNEL_VERSION_FILE) $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC): - cp $(BUILT_KERNEL_VERSION_FILE) $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC) + if ! cmp -s $(BUILT_KERNEL_VERSION_FILE) $@ ; then cp $(BUILT_KERNEL_VERSION_FILE) $@; fi +.KATI_RESTAT: $(BUILT_KERNEL_VERSION_FILE_FOR_UFFD_GC) else # We make this a warning rather than an error to avoid breaking too many builds. When it happens, # we use a placeholder as the kernel version, which is consumed by uffd_gc_utils.py. @@ -5614,7 +5654,9 @@ else endif endif # INSTALLED_BOOTIMAGE_TARGET == "" ifeq ($(recovery_fstab),) - build_ota_package := false + ifeq ($(filter $(TARGET_RECOVERY_ROOT_OUT)/system/etc/recovery.fstab,$(INTERNAL_RECOVERYIMAGE_FILES)),) + build_ota_package := false + endif endif endif # PRODUCT_BUILD_GENERIC_OTA_PACKAGE @@ -5639,6 +5681,7 @@ INTERNAL_OTATOOLS_MODULES := \ brotli \ bsdiff \ build_image \ + build_mixed_kernels_ramdisk_host \ build_super_image \ build_verity_metadata \ build_verity_tree \ @@ -6134,6 +6177,9 @@ $(BUILT_TARGET_FILES_PACKAGE): zip_root := $(intermediates)/$(name) $(BUILT_TARGET_FILES_DIR): zip_root := $(intermediates)/$(name) $(BUILT_TARGET_FILES_DIR): intermediates := $(intermediates) +ifneq ($(SOONG_DEFINED_SYSTEM_IMAGE_PATH),) + $(BUILT_TARGET_FILES_DIR): $(SOONG_DEFINED_SYSTEM_IMAGE_PATH) +endif # $(1): Directory to copy # $(2): Location to copy it to @@ -6147,11 +6193,14 @@ endef built_ota_tools := + # We can't build static executables when SANITIZE_TARGET=address ifeq (,$(filter address, $(SANITIZE_TARGET))) +ifeq (false,$(AB_OTA_UPDATER)) built_ota_tools += \ $(call intermediates-dir-for,EXECUTABLES,updater)/updater endif +endif $(BUILT_TARGET_FILES_DIR): PRIVATE_OTA_TOOLS := $(built_ota_tools) @@ -6348,6 +6397,10 @@ ifdef BUILDING_VENDOR_BOOT_IMAGE endif endif +ifdef BUILDING_VENDOR_KERNEL_BOOT_IMAGE + $(BUILT_TARGET_FILES_DIR): $(INTERNAL_VENDOR_KERNEL_RAMDISK_FILES) +endif + ifdef BUILDING_RECOVERY_IMAGE # TODO(b/30414428): Can't depend on INTERNAL_RECOVERYIMAGE_FILES alone like other # BUILT_TARGET_FILES_PACKAGE dependencies because currently there're cp/rsync/rm @@ -6620,8 +6673,13 @@ endif endif # INSTALLED_VENDOR_BOOTIMAGE_TARGET ifdef BUILDING_SYSTEM_IMAGE @# Contents of the system image +ifneq ($(SOONG_DEFINED_SYSTEM_IMAGE_PATH),) + $(hide) $(call package_files-copy-root, \ + $(SOONG_DEFINED_SYSTEM_IMAGE_BASE)/system/system,$(zip_root)/SYSTEM) +else $(hide) $(call package_files-copy-root, \ $(SYSTEMIMAGE_SOURCE_DIR),$(zip_root)/SYSTEM) +endif else ifdef INSTALLED_BUILD_PROP_TARGET @# Copy the system build.prop even if not building a system image @# because add_img_to_target_files may need it to build other partition @@ -7251,29 +7309,6 @@ ifeq (true,$(CLANG_COVERAGE)) $(call dist-for-goals,droidcore-unbundled apps_only,$(LLVM_COVERAGE_TOOLS_ZIP)) endif -# ----------------------------------------------------------------- -# A zip of the Android Apps. Not keeping full path so that we don't -# include product names when distributing -# -name := $(TARGET_PRODUCT) -ifeq ($(TARGET_BUILD_TYPE),debug) - name := $(name)_debug -endif -name := $(name)-apps - -APPS_ZIP := $(PRODUCT_OUT)/$(name).zip -$(APPS_ZIP): $(FULL_SYSTEMIMAGE_DEPS) - @echo "Package apps: $@" - $(hide) rm -rf $@ - $(hide) mkdir -p $(dir $@) - $(hide) apps_to_zip=`find $(TARGET_OUT_APPS) $(TARGET_OUT_APPS_PRIVILEGED) -mindepth 2 -maxdepth 3 -name "*.apk"`; \ - if [ -z "$$apps_to_zip" ]; then \ - echo "No apps to zip up. Generating empty apps archive." ; \ - a=$$(mktemp /tmp/XXXXXXX) && touch $$a && zip $@ $$a && zip -d $@ $$a; \ - else \ - zip -qjX $@ $$apps_to_zip; \ - fi - ifeq (true,$(EMMA_INSTRUMENT)) #------------------------------------------------------------------ # An archive of classes for use in generating code-coverage reports @@ -7915,6 +7950,18 @@ IMAGES := $(INSTALLED_BOOTIMAGE_TARGET) \ $(INSTALLED_USERDATAIMAGE_TARGET) # ----------------------------------------------------------------- +# Desktop generated firmware filesystem. +TARGET_PRODUCT_FW_IMAGE_PACKAGE := prebuilt-$(TARGET_PRODUCT)-firmware-image +GENERATED_FW_IMAGE := $(PRODUCT_OUT)/product/etc/$(TARGET_PRODUCT)-firmware.img + +generated_fw_image_found := $(strip $(foreach pp,$(PRODUCT_PACKAGES),\ + $(if $(findstring $(TARGET_PRODUCT_FW_IMAGE_PACKAGE),$(pp)),$(pp)))) + +ifneq (,$(generated_fw_image_found)) +$(call dist-for-goals,dist_files,$(GENERATED_FW_IMAGE)) +endif + +# ----------------------------------------------------------------- # Desktop pack image hook. ifneq (,$(strip $(PACK_DESKTOP_FILESYSTEM_IMAGES))) PACK_IMAGE_TARGET := $(PRODUCT_OUT)/android-desktop_image.bin @@ -7936,11 +7983,16 @@ endif # PACK_DESKTOP_FILESYSTEM_IMAGES # ----------------------------------------------------------------- # Desktop pack recovery image hook. -ifneq (,$(strip $(PACK_DESKTOP_RECOVERY_IMAGE))) +ifeq ($(BOARD_USES_DESKTOP_RECOVERY_IMAGE),true) PACK_RECOVERY_IMAGE_TARGET := $(PRODUCT_OUT)/android-desktop_recovery_image.bin +PACK_RECOVERY_IMAGE_ARGS := --noarchive --recovery + +ifneq (,$(strip $(PACK_RECOVERY_IMAGE_EXPERIMENTAL))) +PACK_RECOVERY_IMAGE_ARGS += --experimental +endif # PACK_RECOVERY_IMAGE_EXPERIMENTAL $(PACK_RECOVERY_IMAGE_TARGET): $(IMAGES) $(PACK_IMAGE_SCRIPT) - $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) --noarchive --recovery + $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) $(PACK_RECOVERY_IMAGE_ARGS) PACKED_RECOVERY_IMAGE_ARCHIVE_TARGET := $(PACK_RECOVERY_IMAGE_TARGET).gz @@ -7952,15 +8004,20 @@ $(call dist-for-goals,dist_files,$(PACKED_RECOVERY_IMAGE_ARCHIVE_TARGET)) .PHONY: pack-recovery-image pack-recovery-image: $(PACK_RECOVERY_IMAGE_TARGET) -endif # PACK_DESKTOP_RECOVERY_IMAGE +endif # BOARD_USES_DESKTOP_RECOVERY_IMAGE # ----------------------------------------------------------------- # Desktop pack update image hook. -ifneq (,$(strip $(PACK_DESKTOP_UPDATE_IMAGE))) +ifeq ($(BOARD_USES_DESKTOP_UPDATE_IMAGE),true) PACK_UPDATE_IMAGE_TARGET := $(PRODUCT_OUT)/android-desktop_update_image.bin +PACK_UPDATE_IMAGE_ARGS := --noarchive --update + +ifneq (,$(strip $(PACK_UPDATE_IMAGE_EXPERIMENTAL))) +PACK_UPDATE_IMAGE_ARGS += --experimental +endif # PACK_UPDATE_IMAGE_EXPERIMENTAL $(PACK_UPDATE_IMAGE_TARGET): $(IMAGES) $(PACK_IMAGE_SCRIPT) - $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) --noarchive --update + $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) $(PACK_UPDATE_IMAGE_ARGS) PACKED_UPDATE_IMAGE_ARCHIVE_TARGET := $(PACK_UPDATE_IMAGE_TARGET).gz @@ -7972,7 +8029,69 @@ $(call dist-for-goals,dist_files,$(PACKED_UPDATE_IMAGE_ARCHIVE_TARGET)) .PHONY: pack-update-image pack-update-image: $(PACK_UPDATE_IMAGE_TARGET) -endif # PACK_DESKTOP_UPDATE_IMAGE +endif # BOARD_USES_DESKTOP_UPDATE_IMAGE + +PACK_MIGRATION_IMAGE_SCRIPT := $(HOST_OUT_EXECUTABLES)/pack_migration_image + +# ----------------------------------------------------------------- +# Desktop pack migration image hook. +ifeq ($(ANDROID_DESKTOP_MIGRATION_IMAGE),true) +PACK_MIGRATION_IMAGE_TARGET := $(PRODUCT_OUT)/android-desktop_migration_image.bin + +$(PACK_MIGRATION_IMAGE_TARGET): $(IMAGES) $(PACK_MIGRATION_IMAGE_SCRIPT) + $(PACK_MIGRATION_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) --noarchive + +PACKED_MIGRATION_IMAGE_ARCHIVE_TARGET := $(PACK_MIGRATION_IMAGE_TARGET).gz + +$(PACKED_MIGRATION_IMAGE_ARCHIVE_TARGET): $(PACK_MIGRATION_IMAGE_TARGET) | $(GZIP) + $(GZIP) -fk $(PACK_MIGRATION_IMAGE_TARGET) + +$(call dist-for-goals,dist_files,$(PACKED_MIGRATION_IMAGE_ARCHIVE_TARGET)) + +.PHONY: pack-migration-image +pack-migration-image: $(PACK_MIGRATION_IMAGE_TARGET) + +endif # ANDROID_DESKTOP_MIGRATION_IMAGE + +ifdef SOONG_ONLY_ALL_IMAGES_ZIP + +allimages_soong_zip_args := +allimages_deps := + +define include_image +$(if $(1), \ + $(eval allimages_soong_zip_args += -e $(notdir $(1)) -f $(1)) \ + $(eval allimages_deps += $(1))) +endef + +$(call include_image,$(INSTALLED_SUPERIMAGE_TARGET)) +$(call include_image,$(INSTALLED_BOOTIMAGE_TARGET)) +$(call include_image,$(INSTALLED_INIT_BOOT_IMAGE_TARGET)) +$(call include_image,$(INSTALLED_VENDOR_BOOTIMAGE_TARGET)) +$(call include_image,$(INSTALLED_USERDATAIMAGE_TARGET)) +$(call include_image,$(INSTALLED_RECOVERYIMAGE_TARGET)) +$(call include_image,$(INSTALLED_VBMETAIMAGE_TARGET)) +$(call include_image,$(INSTALLED_VBMETA_SYSTEMIMAGE_TARGET)) +$(call include_image,$(INSTALLED_VBMETA_VENDORIMAGE_TARGET)) +$(foreach partition,$(call to-upper,$(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS)), \ + $(call include_image,$(INSTALLED_VBMETA_$(partition)IMAGE_TARGET))) + +allimages_zip := $(PRODUCT_OUT)/all_images.zip +$(allimages_zip): PRIVATE_SOONG_ZIP_ARGUMENTS := $(allimages_soong_zip_args) +$(allimages_zip): $(SOONG_ZIP) $(allimages_deps) + $(SOONG_ZIP) -o $@ $(PRIVATE_SOONG_ZIP_ARGUMENTS) + +.PHONY: soong_only_diff_test +soong_only_diff_test: PRIVATE_ALLIMAGES_ZIP := $(allimages_zip) +soong_only_diff_test: $(allimages_zip) $(SOONG_ONLY_ALL_IMAGES_ZIP) + diff $(PRIVATE_ALLIMAGES_ZIP) $(SOONG_ONLY_ALL_IMAGES_ZIP) + +allimages_soong_zip_args := +allimages_deps := +allimages_zip := +include_image := + +endif # ifdef SOONG_ONLY_ALL_IMAGES_ZIP # ----------------------------------------------------------------- # OS Licensing diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk index 48667ac509..6aea68000b 100644 --- a/core/android_soong_config_vars.mk +++ b/core/android_soong_config_vars.mk @@ -30,12 +30,36 @@ $(call add_soong_config_var,ANDROID,BOARD_USES_ODMIMAGE) $(call soong_config_set_bool,ANDROID,BOARD_USES_RECOVERY_AS_BOOT,$(BOARD_USES_RECOVERY_AS_BOOT)) $(call soong_config_set_bool,ANDROID,BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT,$(BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT)) $(call add_soong_config_var,ANDROID,CHECK_DEV_TYPE_VIOLATIONS) +$(call soong_config_set_bool,ANDROID,HAS_BOARD_SYSTEM_EXT_PREBUILT_DIR,$(if $(BOARD_SYSTEM_EXT_PREBUILT_DIR),true,false)) +$(call soong_config_set_bool,ANDROID,HAS_BOARD_PRODUCT_PREBUILT_DIR,$(if $(BOARD_PRODUCT_PREBUILT_DIR),true,false)) $(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_VERSION) $(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_COMPAT_VERSIONS) $(call add_soong_config_var,ANDROID,PRODUCT_INSTALL_DEBUG_POLICY_TO_SYSTEM_EXT) +$(call soong_config_set_bool,ANDROID,RELEASE_BOARD_API_LEVEL_FROZEN,$(RELEASE_BOARD_API_LEVEL_FROZEN)) $(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_DRMSERVER) $(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64) $(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_MEDIASERVER) +$(call soong_config_set_bool,ANDROID,TARGET_SUPPORTS_32_BIT_APPS,$(if $(filter true,$(TARGET_SUPPORTS_32_BIT_APPS)),true,false)) +$(call soong_config_set_bool,ANDROID,TARGET_SUPPORTS_64_BIT_APPS,$(if $(filter true,$(TARGET_SUPPORTS_64_BIT_APPS)),true,false)) +$(call add_soong_config_var,ANDROID,BOARD_GENFS_LABELS_VERSION) +$(call soong_config_set_bool,ANDROID,PRODUCT_FSVERITY_GENERATE_METADATA,$(if $(filter true,$(PRODUCT_FSVERITY_GENERATE_METADATA)),true,false)) + +$(call add_soong_config_var,ANDROID,ADDITIONAL_M4DEFS,$(if $(BOARD_SEPOLICY_M4DEFS),$(addprefix -D,$(BOARD_SEPOLICY_M4DEFS)))) +$(call add_soong_config_var,ANDROID,TARGET_ADD_ROOT_EXTRA_VENDOR_SYMLINKS) + +# For BUILDING_GSI +$(call soong_config_set_bool,gsi,building_gsi,$(if $(filter true,$(BUILDING_GSI)),true,false)) + +# For bootable/recovery +RECOVERY_API_VERSION := 3 +RECOVERY_FSTAB_VERSION := 2 +$(call soong_config_set, recovery, recovery_api_version, $(RECOVERY_API_VERSION)) +$(call soong_config_set, recovery, recovery_fstab_version, $(RECOVERY_FSTAB_VERSION)) +$(call soong_config_set_bool, recovery ,target_userimages_use_f2fs ,$(if $(TARGET_USERIMAGES_USE_F2FS),true,false)) +$(call soong_config_set_bool, recovery ,has_board_cacheimage_partition_size ,$(if $(BOARD_CACHEIMAGE_PARTITION_SIZE),true,false)) +ifdef TARGET_RECOVERY_UI_LIB + $(call soong_config_set_string_list, recovery, target_recovery_ui_lib, $(TARGET_RECOVERY_UI_LIB)) +endif # For Sanitizers $(call soong_config_set_bool,ANDROID,ASAN_ENABLED,$(if $(filter address,$(SANITIZE_TARGET)),true,false)) @@ -60,6 +84,13 @@ endif # The default value of ART_BUILD_HOST_DEBUG is true $(call soong_config_set_bool,art_module,art_build_host_debug,$(if $(filter false,$(ART_BUILD_HOST_DEBUG)),false,true)) +# For chre +$(call soong_config_set_bool,chre,chre_daemon_lpma_enabled,$(if $(filter true,$(CHRE_DAEMON_LPMA_ENABLED)),true,false)) +$(call soong_config_set_bool,chre,chre_dedicated_transport_channel_enabled,$(if $(filter true,$(CHRE_DEDICATED_TRANSPORT_CHANNEL_ENABLED)),true,false)) +$(call soong_config_set_bool,chre,chre_log_atom_extension_enabled,$(if $(filter true,$(CHRE_LOG_ATOM_EXTENSION_ENABLED)),true,false)) +$(call soong_config_set_bool,chre,building_vendor_image,$(if $(filter true,$(BUILDING_VENDOR_IMAGE)),true,false)) +$(call soong_config_set_bool,chre,chre_usf_daemon_enabled,$(if $(filter true,$(CHRE_USF_DAEMON_ENABLED)),true,false)) + ifdef TARGET_BOARD_AUTO $(call add_soong_config_var_value, ANDROID, target_board_auto, $(TARGET_BOARD_AUTO)) endif @@ -77,6 +108,9 @@ endif SYSTEMUI_OPTIMIZE_JAVA ?= true $(call add_soong_config_var,ANDROID,SYSTEMUI_OPTIMIZE_JAVA) +# Flag for enabling compose for Launcher. +$(call soong_config_set,ANDROID,release_enable_compose_in_launcher,$(RELEASE_ENABLE_COMPOSE_IN_LAUNCHER)) + ifdef PRODUCT_AVF_ENABLED $(call add_soong_config_var_value,ANDROID,avf_enabled,$(PRODUCT_AVF_ENABLED)) endif @@ -91,12 +125,14 @@ ifdef PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION $(call add_soong_config_var_value,ANDROID,avf_microdroid_guest_gki_version,$(PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION)) endif -ifdef PRODUCT_MEMCG_V2_FORCE_ENABLED -$(call add_soong_config_var_value,ANDROID,memcg_v2_force_enabled,$(PRODUCT_MEMCG_V2_FORCE_ENABLED)) +ifdef TARGET_BOOTS_16K +$(call soong_config_set_bool,ANDROID,target_boots_16k,$(filter true,$(TARGET_BOOTS_16K))) endif ifdef PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED $(call add_soong_config_var_value,ANDROID,cgroup_v2_sys_app_isolation,$(PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED)) +else +$(call add_soong_config_var_value,ANDROID,cgroup_v2_sys_app_isolation,true) endif $(call add_soong_config_var_value,ANDROID,release_avf_allow_preinstalled_apps,$(RELEASE_AVF_ALLOW_PREINSTALLED_APPS)) @@ -159,6 +195,14 @@ else $(call add_soong_config_var_value,ANDROID,include_nonpublic_framework_api,true) endif +# Add nfc build flag to soong +ifneq ($(RELEASE_PACKAGE_NFC_STACK),NfcNci) + $(call soong_config_set,bootclasspath,nfc_apex_bootclasspath_fragment,true) +endif + +# Add uwb build flag to soong +$(call soong_config_set,bootclasspath,release_ranging_stack,$(RELEASE_RANGING_STACK)) + # Add crashrecovery build flag to soong $(call soong_config_set,ANDROID,release_crashrecovery_module,$(RELEASE_CRASHRECOVERY_MODULE)) # Add crashrecovery file move flags to soong, for both platform and module @@ -172,10 +216,24 @@ endif # Required as platform_bootclasspath is using this namespace $(call soong_config_set,bootclasspath,release_crashrecovery_module,$(RELEASE_CRASHRECOVERY_MODULE)) +# Add uprobestats build flag to soong +$(call soong_config_set,ANDROID,release_uprobestats_module,$(RELEASE_UPROBESTATS_MODULE)) +# Add uprobestats file move flags to soong, for both platform and module +ifeq (true,$(RELEASE_UPROBESTATS_FILE_MOVE)) + $(call soong_config_set,ANDROID,uprobestats_files_in_module,true) + $(call soong_config_set,ANDROID,uprobestats_files_in_platform,false) +else + $(call soong_config_set,ANDROID,uprobestats_files_in_module,false) + $(call soong_config_set,ANDROID,uprobestats_files_in_platform,true) +endif + # Enable Profiling module. Also used by platform_bootclasspath. $(call soong_config_set,ANDROID,release_package_profiling_module,$(RELEASE_PACKAGE_PROFILING_MODULE)) $(call soong_config_set,bootclasspath,release_package_profiling_module,$(RELEASE_PACKAGE_PROFILING_MODULE)) +# Move VCN from platform to the Tethering module; used by both platform and module +$(call soong_config_set,ANDROID,is_vcn_in_mainline,$(RELEASE_MOVE_VCN_TO_MAINLINE)) + # Add perf-setup build flag to soong # Note: BOARD_PERFSETUP_SCRIPT location must be under platform_testing/scripts/perf-setup/. ifdef BOARD_PERFSETUP_SCRIPT @@ -184,3 +242,91 @@ endif # Add target_use_pan_display flag for hardware/libhardware:gralloc.default $(call soong_config_set_bool,gralloc,target_use_pan_display,$(if $(filter true,$(TARGET_USE_PAN_DISPLAY)),true,false)) + +# Add use_camera_v4l2_hal flag for hardware/libhardware/modules/camera/3_4:camera.v4l2 +$(call soong_config_set_bool,camera,use_camera_v4l2_hal,$(if $(filter true,$(USE_CAMERA_V4L2_HAL)),true,false)) + +# Add audioserver_multilib flag for hardware/interfaces/soundtrigger/2.0/default:android.hardware.soundtrigger@2.0-impl +ifneq ($(strip $(AUDIOSERVER_MULTILIB)),) + $(call soong_config_set,soundtrigger,audioserver_multilib,$(AUDIOSERVER_MULTILIB)) +endif + +# Add sim_count, disable_rild_oem_hook, and use_aosp_rild flag for ril related modules +$(call soong_config_set,ril,sim_count,$(SIM_COUNT)) +ifneq ($(DISABLE_RILD_OEM_HOOK), false) + $(call soong_config_set_bool,ril,disable_rild_oem_hook,true) +endif +ifneq ($(ENABLE_VENDOR_RIL_SERVICE), true) + $(call soong_config_set_bool,ril,use_aosp_rild,true) +endif + +# Export target_board_platform to soong for hardware/google/graphics/common/libmemtrack:memtrack.$(TARGET_BOARD_PLATFORM) +$(call soong_config_set,ANDROID,target_board_platform,$(TARGET_BOARD_PLATFORM)) + +# Export board_uses_scaler_m2m1shot and board_uses_align_restriction to soong for hardware/google/graphics/common/libscaler:libexynosscaler +$(call soong_config_set_bool,google_graphics,board_uses_scaler_m2m1shot,$(if $(filter true,$(BOARD_USES_SCALER_M2M1SHOT)),true,false)) +$(call soong_config_set_bool,google_graphics,board_uses_align_restriction,$(if $(filter true,$(BOARD_USES_ALIGN_RESTRICTION)),true,false)) + +# Export related variables to soong for hardware/google/graphics/common/libacryl:libacryl +ifdef BOARD_LIBACRYL_DEFAULT_COMPOSITOR + $(call soong_config_set,acryl,libacryl_default_compositor,$(BOARD_LIBACRYL_DEFAULT_COMPOSITOR)) +endif +ifdef BOARD_LIBACRYL_DEFAULT_SCALER + $(call soong_config_set,acryl,libacryl_default_scaler,$(BOARD_LIBACRYL_DEFAULT_SCALER)) +endif +ifdef BOARD_LIBACRYL_DEFAULT_BLTER + $(call soong_config_set,acryl,libacryl_default_blter,$(BOARD_LIBACRYL_DEFAULT_BLTER)) +endif +ifdef BOARD_LIBACRYL_G2D_HDR_PLUGIN + #BOARD_LIBACRYL_G2D_HDR_PLUGIN is set in each board config + $(call soong_config_set_bool,acryl,libacryl_use_g2d_hdr_plugin,true) +endif + +# Export related variables to soong for hardware/google/graphics/common/BoardConfigCFlags.mk +$(call soong_config_set_bool,google_graphics,hwc_no_support_skip_validate,$(if $(filter true,$(HWC_NO_SUPPORT_SKIP_VALIDATE)),true,false)) +$(call soong_config_set_bool,google_graphics,hwc_support_color_transform,$(if $(filter true,$(HWC_SUPPORT_COLOR_TRANSFORM)),true,false)) +$(call soong_config_set_bool,google_graphics,hwc_support_render_intent,$(if $(filter true,$(HWC_SUPPORT_RENDER_INTENT)),true,false)) +$(call soong_config_set_bool,google_graphics,board_uses_virtual_display,$(if $(filter true,$(BOARD_USES_VIRTUAL_DISPLAY)),true,false)) +$(call soong_config_set_bool,google_graphics,board_uses_dt,$(if $(filter true,$(BOARD_USES_DT)),true,false)) +$(call soong_config_set_bool,google_graphics,board_uses_decon_64bit_address,$(if $(filter true,$(BOARD_USES_DECON_64BIT_ADDRESS)),true,false)) +$(call soong_config_set_bool,google_graphics,board_uses_hdrui_gles_conversion,$(if $(filter true,$(BOARD_USES_HDRUI_GLES_CONVERSION)),true,false)) +$(call soong_config_set_bool,google_graphics,uses_idisplay_intf_sec,$(if $(filter true,$(USES_IDISPLAY_INTF_SEC)),true,false)) + +# Variables for fs_config +$(call soong_config_set_bool,fs_config,vendor,$(if $(BOARD_USES_VENDORIMAGE)$(BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE),true,false)) +$(call soong_config_set_bool,fs_config,oem,$(if $(BOARD_USES_OEMIMAGE)$(BOARD_OEMIMAGE_FILE_SYSTEM_TYPE),true,false)) +$(call soong_config_set_bool,fs_config,odm,$(if $(BOARD_USES_ODMIMAGE)$(BOARD_ODMIMAGE_FILE_SYSTEM_TYPE),true,false)) +$(call soong_config_set_bool,fs_config,vendor_dlkm,$(if $(BOARD_USES_VENDOR_DLKMIMAGE)$(BOARD_VENDOR_DLKMIMAGE_FILE_SYSTEM_TYPE),true,false)) +$(call soong_config_set_bool,fs_config,odm_dlkm,$(if $(BOARD_USES_ODM_DLKMIMAGE)$(BOARD_ODM_DLKMIMAGE_FILE_SYSTEM_TYPE),true,false)) +$(call soong_config_set_bool,fs_config,system_dlkm,$(if $(BOARD_USES_SYSTEM_DLKMIMAGE)$(BOARD_SYSTEM_DLKMIMAGE_FILE_SYSTEM_TYPE),true,false)) + +# Variables for telephony +$(call soong_config_set_bool,telephony,sec_cp_secure_boot,$(if $(filter true,$(SEC_CP_SECURE_BOOT)),true,false)) +$(call soong_config_set_bool,telephony,cbd_protocol_sit,$(if $(filter true,$(CBD_PROTOCOL_SIT)),true,false)) +$(call soong_config_set_bool,telephony,use_radioexternal_hal_aidl,$(if $(filter true,$(USE_RADIOEXTERNAL_HAL_AIDL)),true,false)) + +# Variables for hwcomposer.$(TARGET_BOARD_PLATFORM) +$(call soong_config_set_bool,google_graphics,board_uses_hwc_services,$(if $(filter true,$(BOARD_USES_HWC_SERVICES)),true,false)) + +# Variables for controlling android.hardware.composer.hwc3-service.pixel +$(call soong_config_set,google_graphics,board_hwc_version,$(BOARD_HWC_VERSION)) + +# Flag ExcludeExtractApk is to support "extract_apk" property for the following conditions. +ifneq ($(WITH_DEXPREOPT),true) + $(call soong_config_set_bool,PrebuiltGmsCore,ExcludeExtractApk,true) +endif +ifeq ($(DONT_DEXPREOPT_PREBUILTS),true) + $(call soong_config_set_bool,PrebuiltGmsCore,ExcludeExtractApk,true) +endif +ifeq ($(WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY),true) + $(call soong_config_set_bool,PrebuiltGmsCore,ExcludeExtractApk,true) +endif + +# Variables for extra branches +# TODO(b/383238397): Use bootstrap_go_package to enable extra flags. +-include vendor/google/build/extra_soong_config_vars.mk + +# Variable for CI test packages +ifneq ($(filter arm x86 true,$(TARGET_ARCH) $(TARGET_2ND_ARCH) $(TARGET_ENABLE_MEDIADRM_64)),) + $(call soong_config_set_bool,ci_tests,uses_widevine_tests, true) +endif diff --git a/core/base_rules.mk b/core/base_rules.mk index 5363e0fbf9..9ffe51830a 100644 --- a/core/base_rules.mk +++ b/core/base_rules.mk @@ -861,13 +861,6 @@ else $(eval my_compat_dist_config_$(suite) += $(foreach dir, $(call compatibility_suite_dirs,$(suite)), \ $(LOCAL_PATH)/DynamicConfig.xml:$(dir)/$(LOCAL_MODULE).dynamic))) endif - - ifneq (,$(wildcard $(LOCAL_PATH)/$(LOCAL_MODULE)_*.config)) - $(foreach extra_config, $(wildcard $(LOCAL_PATH)/$(LOCAL_MODULE)_*.config), \ - $(foreach suite, $(LOCAL_COMPATIBILITY_SUITE), \ - $(eval my_compat_dist_config_$(suite) += $(foreach dir, $(call compatibility_suite_dirs,$(suite)), \ - $(extra_config):$(dir)/$(notdir $(extra_config)))))) - endif endif # $(my_prefix)$(LOCAL_MODULE_CLASS)_$(LOCAL_MODULE)_compat_files @@ -938,12 +931,6 @@ else my_supported_variant := DEVICE endif endif -########################################################### -## Add test module to ALL_DISABLED_PRESUBMIT_TESTS if LOCAL_PRESUBMIT_DISABLED is set to true. -########################################################### -ifeq ($(LOCAL_PRESUBMIT_DISABLED),true) - ALL_DISABLED_PRESUBMIT_TESTS += $(LOCAL_MODULE) -endif # LOCAL_PRESUBMIT_DISABLED ########################################################### ## Register with ALL_MODULES diff --git a/core/binary.mk b/core/binary.mk index 1e98bc08fb..ea862be6b4 100644 --- a/core/binary.mk +++ b/core/binary.mk @@ -174,7 +174,7 @@ my_allow_undefined_symbols := true endif endif -my_ndk_sysroot_include := +my_ndk_sysroot := my_ndk_sysroot_lib := my_api_level := 10000 @@ -207,11 +207,9 @@ ifneq ($(LOCAL_SDK_VERSION),) my_built_ndk := $(SOONG_OUT_DIR)/ndk my_ndk_triple := $($(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_NDK_TRIPLE) - my_ndk_sysroot_include := \ - $(my_built_ndk)/sysroot/usr/include \ - $(my_built_ndk)/sysroot/usr/include/$(my_ndk_triple) \ + my_ndk_sysroot := $(my_built_ndk)/sysroot - my_ndk_sysroot_lib := $(my_built_ndk)/sysroot/usr/lib/$(my_ndk_triple)/$(my_ndk_api) + my_ndk_sysroot_lib := $(my_ndk_sysroot)/usr/lib/$(my_ndk_triple)/$(my_ndk_api) # The bionic linker now has support for packed relocations and gnu style # hashes (which are much faster!), but shipping to older devices requires @@ -330,18 +328,20 @@ ifneq ($(call module-in-vendor-or-product),) ifneq ($(LOCAL_IN_VENDOR),) # Vendor modules have LOCAL_IN_VENDOR my_cflags += -D__ANDROID_VENDOR__ - - ifeq ($(BOARD_API_LEVEL),) - # TODO(b/314036847): This is a fallback for UDC targets. - # This must be a build failure when UDC is no longer built from this source tree. - my_cflags += -D__ANDROID_VENDOR_API__=$(PLATFORM_SDK_VERSION) - else - my_cflags += -D__ANDROID_VENDOR_API__=$(BOARD_API_LEVEL) - endif else ifneq ($(LOCAL_IN_PRODUCT),) # Product modules have LOCAL_IN_PRODUCT my_cflags += -D__ANDROID_PRODUCT__ endif + + # Define __ANDROID_VENDOR_API__ for both product and vendor variants because + # they both use the same LLNDK libraries. + ifeq ($(BOARD_API_LEVEL),) + # TODO(b/314036847): This is a fallback for UDC targets. + # This must be a build failure when UDC is no longer built from this source tree. + my_cflags += -D__ANDROID_VENDOR_API__=$(PLATFORM_SDK_VERSION) + else + my_cflags += -D__ANDROID_VENDOR_API__=$(BOARD_API_LEVEL) + endif endif ifndef LOCAL_IS_HOST_MODULE @@ -1626,19 +1626,6 @@ my_ldlibs += $(my_cxx_ldlibs) ########################################################### ifndef LOCAL_IS_HOST_MODULE -ifeq ($(call module-in-vendor-or-product),true) - my_target_global_c_includes := - my_target_global_c_system_includes := $(TARGET_OUT_HEADERS) -else ifdef LOCAL_SDK_VERSION - my_target_global_c_includes := - my_target_global_c_system_includes := $(my_ndk_stl_include_path) $(my_ndk_sysroot_include) -else - my_target_global_c_includes := $(SRC_HEADERS) \ - $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES) - my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \ - $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES) -endif - my_target_global_cflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CFLAGS) my_target_global_conlyflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CONLYFLAGS) $(my_c_std_conlyflags) my_target_global_cppflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CPPFLAGS) $(my_cpp_std_cppflags) @@ -1654,6 +1641,22 @@ else my_target_global_ldflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_LDFLAGS) endif # my_use_clang_lld +ifeq ($(call module-in-vendor-or-product),true) + my_target_global_c_includes := + my_target_global_c_system_includes := $(TARGET_OUT_HEADERS) + my_target_global_cflags += -nostdlibinc +else ifdef LOCAL_SDK_VERSION + my_target_global_c_includes := + my_target_global_c_system_includes := $(my_ndk_stl_include_path) + my_target_global_cflags += --sysroot $(my_ndk_sysroot) +else + my_target_global_c_includes := $(SRC_HEADERS) \ + $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES) + my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \ + $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES) + my_target_global_cflags += -nostdlibinc +endif + my_target_triple := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)TRIPLE) ifndef LOCAL_IS_HOST_MODULE my_target_triple_flag := -target $(my_target_triple)$(my_api_level) diff --git a/core/board_config.mk b/core/board_config.mk index 5606964950..ad89c0313b 100644 --- a/core/board_config.mk +++ b/core/board_config.mk @@ -27,6 +27,7 @@ _board_strip_readonly_list += BOARD_INSTALLER_CMDLINE _board_strip_readonly_list += BOARD_KERNEL_CMDLINE _board_strip_readonly_list += BOARD_BOOT_HEADER_VERSION _board_strip_readonly_list += BOARD_BOOTCONFIG +_board_strip_readonly_list += BOARD_BOOTCONFIG_FILE _board_strip_readonly_list += BOARD_KERNEL_BASE _board_strip_readonly_list += BOARD_USES_GENERIC_AUDIO _board_strip_readonly_list += BOARD_USES_RECOVERY_AS_BOOT @@ -290,6 +291,7 @@ $(foreach var,$(_board_true_false_vars), \ include $(BUILD_SYSTEM)/board_config_wifi.mk # Set up soong config for "soong_config_value_variable". +-include hardware/interfaces/configstore/1.1/default/surfaceflinger.mk -include vendor/google/build/soong/soong_config_namespace/camera.mk # Default *_CPU_VARIANT_RUNTIME to CPU_VARIANT if unspecified. @@ -311,9 +313,10 @@ endif .KATI_READONLY := $(_board_strip_readonly_list) INTERNAL_KERNEL_CMDLINE := $(BOARD_KERNEL_CMDLINE) -ifneq (,$(BOARD_BOOTCONFIG)) +ifneq (,$(BOARD_BOOTCONFIG)$(BOARD_BOOTCONFIG_FILE)) INTERNAL_KERNEL_CMDLINE += bootconfig INTERNAL_BOOTCONFIG := $(BOARD_BOOTCONFIG) + INTERNAL_BOOTCONFIG_FILE := $(BOARD_BOOTCONFIG_FILE) endif ifneq ($(filter %64,$(TARGET_ARCH)),) @@ -922,6 +925,18 @@ ifeq ($(PRODUCT_BUILD_PVMFW_IMAGE),true) endif .KATI_READONLY := BOARD_USES_PVMFWIMAGE +BOARD_USES_DESKTOP_RECOVERY_IMAGE := +ifeq ($(PRODUCT_BUILD_DESKTOP_RECOVERY_IMAGE),true) + BOARD_USES_DESKTOP_RECOVERY_IMAGE := true +endif +.KATI_READONLY := BOARD_USES_DESKTOP_RECOVERY_IMAGE + +BOARD_USES_DESKTOP_UPDATE_IMAGE := +ifeq ($(PRODUCT_BUILD_DESKTOP_UPDATE_IMAGE),true) + BOARD_USES_DESKTOP_UPDATE_IMAGE := true +endif +.KATI_READONLY := BOARD_USES_DESKTOP_UPDATE_IMAGE + ########################################### # Ensure consistency among TARGET_RECOVERY_UPDATER_LIBS, AB_OTA_UPDATER, and PRODUCT_OTA_FORCE_NON_AB_PACKAGE. TARGET_RECOVERY_UPDATER_LIBS ?= diff --git a/core/clear_vars.mk b/core/clear_vars.mk index fed19e6d45..2e67aff20d 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -204,7 +204,6 @@ LOCAL_PREBUILT_OBJ_FILES:= LOCAL_PREBUILT_STATIC_JAVA_LIBRARIES:= LOCAL_USE_EMBEDDED_DEX:= LOCAL_USE_EMBEDDED_NATIVE_LIBS:= -LOCAL_PRESUBMIT_DISABLED:= LOCAL_PRIVATE_PLATFORM_APIS:= LOCAL_PRIVILEGED_MODULE:= LOCAL_PROC_MACRO_LIBRARIES:= diff --git a/core/BUILD.bazel b/core/combo/arch/arm64/armv9-2a.mk index f4869d4833..69ffde014b 100644 --- a/core/BUILD.bazel +++ b/core/combo/arch/arm64/armv9-2a.mk @@ -1,28 +1,18 @@ +# # Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License - -# Export tradefed templates for tests. -exports_files( - glob(["*.xml"]), -) +# limitations under the License. +# -# Export proguard flag files for r8. -filegroup( - name = "global_proguard_flags", - srcs = [ - "proguard.flags", - "proguard_basic_keeps.flags", - ], - visibility = ["//visibility:public"], -) +# .mk file required to support build for the ARMv9.2-A arch variant. +# The file just needs to be present, it does not need to contain anything. diff --git a/core/combo/arch/x86/alderlake.mk b/core/combo/arch/x86/alderlake.mk new file mode 100644 index 0000000000..a7ae6ed679 --- /dev/null +++ b/core/combo/arch/x86/alderlake.mk @@ -0,0 +1,6 @@ +# Configuration for Linux on x86. +# Generating binaries for processors +# that have AVX2 feature flag +# + +ARCH_X86_HAVE_SSE4_1 := true diff --git a/core/combo/arch/x86_64/alderlake.mk b/core/combo/arch/x86_64/alderlake.mk new file mode 100644 index 0000000000..a7ae6ed679 --- /dev/null +++ b/core/combo/arch/x86_64/alderlake.mk @@ -0,0 +1,6 @@ +# Configuration for Linux on x86. +# Generating binaries for processors +# that have AVX2 feature flag +# + +ARCH_X86_HAVE_SSE4_1 := true diff --git a/core/config.mk b/core/config.mk index 192c8b28c8..b89292400b 100644 --- a/core/config.mk +++ b/core/config.mk @@ -330,6 +330,18 @@ $(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(filter true,$3)) $(eval SOONG_CONFIG_TYPE_$(strip $1)_$(strip $2):=bool) endef +# soong_config_set_string_list is the same as soong_config_set, but it will +# also type the variable as a list of strings, so that when using select() expressions +# in blueprint files they can use list values instead of strings. +# The values of the list must be space-separated. +# $1 is the namespace. $2 is the variable name. $3 is the variable value. +# Ex: $(call soong_config_set_string_list,acme,COOL_LIBS,a b) +define soong_config_set_string_list +$(call soong_config_define_internal,$1,$2) \ +$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(strip $3)) +$(eval SOONG_CONFIG_TYPE_$(strip $1)_$(strip $2):=string_list) +endef + # soong_config_append appends to the value of the variable in the given Soong # config namespace. If the variable does not exist, it will be defined. If the # namespace does not exist, it will be defined. @@ -432,13 +444,6 @@ else endif .KATI_READONLY := TARGET_MAX_PAGE_SIZE_SUPPORTED -ifdef PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE - TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := $(PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE) -else - TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false -endif -.KATI_READONLY := TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE - # Boolean variable determining if AOSP relies on bionic's PAGE_SIZE macro. ifdef PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO TARGET_NO_BIONIC_PAGE_SIZE_MACRO := $(PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO) @@ -725,8 +730,8 @@ SYMBOLS_MAP := $(HOST_OUT_EXECUTABLES)/symbols_map PROGUARD_HOME := external/proguard PROGUARD := $(PROGUARD_HOME)/bin/proguard.sh PROGUARD_DEPS := $(PROGUARD) $(PROGUARD_HOME)/lib/proguard.jar -JAVATAGS := build/make/tools/java-event-log-tags.py -MERGETAGS := build/make/tools/merge-event-log-tags.py +JAVATAGS := $(HOST_OUT_EXECUTABLES)/java-event-log-tags +MERGETAGS := $(HOST_OUT_EXECUTABLES)/merge-event-log-tags APPEND2SIMG := $(HOST_OUT_EXECUTABLES)/append2simg VERITY_SIGNER := $(HOST_OUT_EXECUTABLES)/verity_signer BUILD_VERITY_METADATA := $(HOST_OUT_EXECUTABLES)/build_verity_metadata @@ -758,50 +763,23 @@ endif .KATI_READONLY := \ PRODUCT_COMPATIBLE_PROPERTY -# Boolean variable determining if Treble is fully enabled -PRODUCT_FULL_TREBLE := false -ifneq ($(PRODUCT_FULL_TREBLE_OVERRIDE),) - PRODUCT_FULL_TREBLE := $(PRODUCT_FULL_TREBLE_OVERRIDE) -else ifeq ($(PRODUCT_SHIPPING_API_LEVEL),) - #$(warning no product shipping level defined) -else ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),26),) - PRODUCT_FULL_TREBLE := true -endif - -requirements := \ - PRODUCT_TREBLE_LINKER_NAMESPACES \ - PRODUCT_ENFORCE_VINTF_MANIFEST - -# If it is overriden, then the requirement override is taken, otherwise it's -# PRODUCT_FULL_TREBLE -$(foreach req,$(requirements),$(eval \ - $(req) := $(if $($(req)_OVERRIDE),$($(req)_OVERRIDE),$(PRODUCT_FULL_TREBLE)))) -# If the requirement is false for any reason, then it's not PRODUCT_FULL_TREBLE -$(foreach req,$(requirements),$(eval \ - PRODUCT_FULL_TREBLE := $(if $(filter false,$($(req))),false,$(PRODUCT_FULL_TREBLE)))) - -PRODUCT_FULL_TREBLE_OVERRIDE ?= -$(foreach req,$(requirements),$(eval $(req)_OVERRIDE ?=)) - -# used to be a part of PRODUCT_FULL_TREBLE, but now always set it +# TODO: remove all code referencing these, and remove override variables +PRODUCT_FULL_TREBLE := true PRODUCT_NOTICE_SPLIT := true +PRODUCT_TREBLE_LINKER_NAMESPACES := true +PRODUCT_ENFORCE_VINTF_MANIFEST := true # TODO(b/114488870): disallow PRODUCT_FULL_TREBLE_OVERRIDE from being used. .KATI_READONLY := \ - PRODUCT_FULL_TREBLE_OVERRIDE \ - $(foreach req,$(requirements),$(req)_OVERRIDE) \ - $(requirements) \ PRODUCT_FULL_TREBLE \ + PRODUCT_TREBLE_LINKER_NAMESPACES \ + PRODUCT_ENFORCE_VINTF_MANIFEST \ PRODUCT_NOTICE_SPLIT \ -ifneq ($(PRODUCT_FULL_TREBLE),true) - $(warning This device does not have Treble enabled. This is unsafe.) -endif - -$(KATI_obsolete_var $(foreach req,$(requirements),$(req)_OVERRIDE) \ - ,This should be referenced without the _OVERRIDE suffix.) - -requirements := +# TODO(b/114488870): remove all sets of these everwhere, and disallow them to be used +$(KATI_obsolete_var PRODUCT_TREBLE_LINKER_NAMESPACES_OVERRIDE,Deprecated.) +$(KATI_obsolete_var PRODUCT_ENFORCE_VINTF_MANIFEST_OVERRIDE,Deprecated.) +$(KATI_obsolete_var PRODUCT_FULL_TREBLE_OVERRIDE,Deprecated.) # BOARD_PROPERTY_OVERRIDES_SPLIT_ENABLED can be true only if early-mount of # partitions is supported. But the early-mount must be supported for full @@ -817,6 +795,18 @@ ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),36),) endif endif +ifdef PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE + TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := $(PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE) +else ifeq (true,$(TARGET_BUILD_UNBUNDLED)) + # unbundled builds may not have updated build sources + TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false +else ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),36),) + TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := true +else + TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false +endif +.KATI_READONLY := TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE + # Set BOARD_SYSTEMSDK_VERSIONS to the latest SystemSDK version starting from P-launching # devices if unset. ifndef BOARD_SYSTEMSDK_VERSIONS @@ -839,12 +829,6 @@ endif .KATI_READONLY := BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES ifdef PRODUCT_SHIPPING_API_LEVEL - board_api_level := $(firstword $(BOARD_API_LEVEL) $(BOARD_SHIPPING_API_LEVEL)) - ifneq (,$(board_api_level)) - min_systemsdk_version := $(call math_min,$(board_api_level),$(PRODUCT_SHIPPING_API_LEVEL)) - else - min_systemsdk_version := $(PRODUCT_SHIPPING_API_LEVEL) - endif ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),29),) ifneq ($(BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE),) $(error When PRODUCT_SHIPPING_API_LEVEL >= 29, BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE cannot be set) @@ -881,20 +865,28 @@ BOARD_SEPOLICY_VERS := $(PLATFORM_SEPOLICY_VERSION) .KATI_READONLY := PLATFORM_SEPOLICY_VERSION BOARD_SEPOLICY_VERS # A list of SEPolicy versions, besides PLATFORM_SEPOLICY_VERSION, that the framework supports. -PLATFORM_SEPOLICY_COMPAT_VERSIONS := $(filter-out $(PLATFORM_SEPOLICY_VERSION), \ +PLATFORM_SEPOLICY_COMPAT_VERSIONS := \ 29.0 \ 30.0 \ 31.0 \ 32.0 \ 33.0 \ 34.0 \ + +PLATFORM_SEPOLICY_COMPAT_VERSIONS += $(foreach ver,\ 202404 \ - ) + 202504 \ + ,$(if $(filter true,$(call math_gt,$(PLATFORM_SEPOLICY_VERSION),$(ver))),$(ver))) .KATI_READONLY := \ PLATFORM_SEPOLICY_COMPAT_VERSIONS \ PLATFORM_SEPOLICY_VERSION \ +BOARD_GENFS_LABELS_VERSION ?= $(BOARD_API_LEVEL) +ifeq ($(call math_gt,$(BOARD_API_LEVEL),$(BOARD_GENFS_LABELS_VERSION)),true) + $(error BOARD_GENFS_LABELS_VERSION ($(BOARD_GENFS_LABELS_VERSION)) must be greater than or equal to BOARD_API_LEVEL ($(BOARD_API_LEVEL))) +endif + ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true) ifneq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true) $(error PRODUCT_USE_DYNAMIC_PARTITIONS must be true when PRODUCT_RETROFIT_DYNAMIC_PARTITIONS \ @@ -1303,10 +1295,6 @@ endif SOONG_VARIABLES := SOONG_EXTRA_VARIABLES := --include external/ltp/android/ltp_package_list.mk -DEFAULT_DATA_OUT_MODULES := ltp $(ltp_packages) -.KATI_READONLY := DEFAULT_DATA_OUT_MODULES - include $(BUILD_SYSTEM)/dumpvar.mk ifdef BOARD_VNDK_VERSION @@ -1321,3 +1309,58 @@ ifeq (false,$(SYSTEM_OPTIMIZE_JAVA)) $(error SYSTEM_OPTIMIZE_JAVA must be enabled when FULL_SYSTEM_OPTIMIZE_JAVA is enabled) endif endif + +# ----------------------------------------------------------------- +# Define fingerprint, thumbprint, and version tags for the current build +# +# BUILD_VERSION_TAGS is a comma-separated list of tags chosen by the device +# implementer that further distinguishes the build. It's basically defined +# by the device implementer. Here, we are adding a mandatory tag that +# identifies the signing config of the build. +BUILD_VERSION_TAGS := $(BUILD_VERSION_TAGS) +ifeq ($(TARGET_BUILD_TYPE),debug) + BUILD_VERSION_TAGS += debug +endif +# The "test-keys" tag marks builds signed with the old test keys, +# which are available in the SDK. "dev-keys" marks builds signed with +# non-default dev keys (usually private keys from a vendor directory). +# Both of these tags will be removed and replaced with "release-keys" +# when the target-files is signed in a post-build step. +ifeq ($(DEFAULT_SYSTEM_DEV_CERTIFICATE),build/make/target/product/security/testkey) +BUILD_KEYS := test-keys +else +BUILD_KEYS := dev-keys +endif +BUILD_VERSION_TAGS += $(BUILD_KEYS) +BUILD_VERSION_TAGS := $(subst $(space),$(comma),$(sort $(BUILD_VERSION_TAGS))) + +# BUILD_FINGERPRINT is used used to uniquely identify the combined build and +# product; used by the OTA server. +ifeq (,$(strip $(BUILD_FINGERPRINT))) + BUILD_FINGERPRINT := $(PRODUCT_BRAND)/$(TARGET_PRODUCT)/$(TARGET_DEVICE):$(PLATFORM_VERSION)/$(BUILD_ID)/$(BUILD_NUMBER_FROM_FILE):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) +endif + +BUILD_FINGERPRINT_FILE := $(PRODUCT_OUT)/build_fingerprint.txt +ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_FINGERPRINT) >$(BUILD_FINGERPRINT_FILE).tmp && (if ! cmp -s $(BUILD_FINGERPRINT_FILE).tmp $(BUILD_FINGERPRINT_FILE); then mv $(BUILD_FINGERPRINT_FILE).tmp $(BUILD_FINGERPRINT_FILE); else rm $(BUILD_FINGERPRINT_FILE).tmp; fi) && grep " " $(BUILD_FINGERPRINT_FILE))) + $(error BUILD_FINGERPRINT cannot contain spaces: "$(file <$(BUILD_FINGERPRINT_FILE))") +endif +BUILD_FINGERPRINT_FROM_FILE := $$(cat $(BUILD_FINGERPRINT_FILE)) +# unset it for safety. +BUILD_FINGERPRINT := + +# BUILD_THUMBPRINT is used to uniquely identify the system build; used by the +# OTA server. This purposefully excludes any product-specific variables. +ifeq (,$(strip $(BUILD_THUMBPRINT))) + BUILD_THUMBPRINT := $(PLATFORM_VERSION)/$(BUILD_ID)/$(BUILD_NUMBER_FROM_FILE):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) +endif + +BUILD_THUMBPRINT_FILE := $(PRODUCT_OUT)/build_thumbprint.txt +ifeq ($(strip $(HAS_BUILD_NUMBER)),true) +$(BUILD_THUMBPRINT_FILE): $(BUILD_NUMBER_FILE) +endif +ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_THUMBPRINT) >$(BUILD_THUMBPRINT_FILE) && grep " " $(BUILD_THUMBPRINT_FILE))) + $(error BUILD_THUMBPRINT cannot contain spaces: "$(file <$(BUILD_THUMBPRINT_FILE))") +endif +# unset it for safety. +BUILD_THUMBPRINT_FILE := +BUILD_THUMBPRINT := diff --git a/core/config_sanitizers.mk b/core/config_sanitizers.mk index ab2d5c1ddf..c0f2c6893f 100644 --- a/core/config_sanitizers.mk +++ b/core/config_sanitizers.mk @@ -284,9 +284,9 @@ endif ifneq ($(filter memtag_stack,$(my_sanitize)),) my_cflags += -fsanitize=memtag-stack my_ldflags += -fsanitize=memtag-stack - my_cflags += -march=armv8a+memtag - my_ldflags += -march=armv8a+memtag - my_asflags += -march=armv8a+memtag + my_cflags += -Xclang -target-feature -Xclang +mte + my_ldflags += -Xclang -target-feature -Xclang +mte + my_asflags += -Xclang -target-feature -Xclang +mte my_sanitize := $(filter-out memtag_stack,$(my_sanitize)) endif diff --git a/core/definitions.mk b/core/definitions.mk index cd1b36e4c7..40f5af0f64 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -90,9 +90,6 @@ ALL_INIT_RC_INSTALLED_PAIRS := # All installed vintf manifest fragments for a partition at ALL_VINTF_MANIFEST_FRAGMENTS_LIST:= -# All tests that should be skipped in presubmit check. -ALL_DISABLED_PRESUBMIT_TESTS := - # All compatibility suites mentioned in LOCAL_COMPATIBILITY_SUITE ALL_COMPATIBILITY_SUITES := @@ -1555,7 +1552,7 @@ endef define transform-logtags-to-java @mkdir -p $(dir $@) @echo "logtags: $@ <= $<" -$(hide) $(JAVATAGS) -o $@ $< $(PRIVATE_MERGED_TAG) +$(hide) $(JAVATAGS) -o $@ $< endef @@ -2605,7 +2602,87 @@ define dump-words-to-file @$(call emit-line,$(wordlist 108501,109000,$(1)),$(2)) @$(call emit-line,$(wordlist 109001,109500,$(1)),$(2)) @$(call emit-line,$(wordlist 109501,110000,$(1)),$(2)) - @$(if $(wordlist 110001,110002,$(1)),$(error dump-words-to-file: Too many words ($(words $(1))))) + @$(call emit-line,$(wordlist 110001,110500,$(1)),$(2)) + @$(call emit-line,$(wordlist 110501,111000,$(1)),$(2)) + @$(call emit-line,$(wordlist 111001,111500,$(1)),$(2)) + @$(call emit-line,$(wordlist 111501,112000,$(1)),$(2)) + @$(call emit-line,$(wordlist 112001,112500,$(1)),$(2)) + @$(call emit-line,$(wordlist 112501,113000,$(1)),$(2)) + @$(call emit-line,$(wordlist 113001,113500,$(1)),$(2)) + @$(call emit-line,$(wordlist 113501,114000,$(1)),$(2)) + @$(call emit-line,$(wordlist 114001,114500,$(1)),$(2)) + @$(call emit-line,$(wordlist 114501,115000,$(1)),$(2)) + @$(call emit-line,$(wordlist 115001,115500,$(1)),$(2)) + @$(call emit-line,$(wordlist 115501,116000,$(1)),$(2)) + @$(call emit-line,$(wordlist 116001,116500,$(1)),$(2)) + @$(call emit-line,$(wordlist 116501,117000,$(1)),$(2)) + @$(call emit-line,$(wordlist 117001,117500,$(1)),$(2)) + @$(call emit-line,$(wordlist 117501,118000,$(1)),$(2)) + @$(call emit-line,$(wordlist 118001,118500,$(1)),$(2)) + @$(call emit-line,$(wordlist 118501,119000,$(1)),$(2)) + @$(call emit-line,$(wordlist 119001,119500,$(1)),$(2)) + @$(call emit-line,$(wordlist 119501,120000,$(1)),$(2)) + @$(call emit-line,$(wordlist 120001,120500,$(1)),$(2)) + @$(call emit-line,$(wordlist 120501,121000,$(1)),$(2)) + @$(call emit-line,$(wordlist 121001,121500,$(1)),$(2)) + @$(call emit-line,$(wordlist 121501,122000,$(1)),$(2)) + @$(call emit-line,$(wordlist 122001,122500,$(1)),$(2)) + @$(call emit-line,$(wordlist 122501,123000,$(1)),$(2)) + @$(call emit-line,$(wordlist 123001,123500,$(1)),$(2)) + @$(call emit-line,$(wordlist 123501,124000,$(1)),$(2)) + @$(call emit-line,$(wordlist 124001,124500,$(1)),$(2)) + @$(call emit-line,$(wordlist 124501,125000,$(1)),$(2)) + @$(call emit-line,$(wordlist 125001,125500,$(1)),$(2)) + @$(call emit-line,$(wordlist 125501,126000,$(1)),$(2)) + @$(call emit-line,$(wordlist 126001,126500,$(1)),$(2)) + @$(call emit-line,$(wordlist 126501,127000,$(1)),$(2)) + @$(call emit-line,$(wordlist 127001,127500,$(1)),$(2)) + @$(call emit-line,$(wordlist 127501,128000,$(1)),$(2)) + @$(call emit-line,$(wordlist 128001,128500,$(1)),$(2)) + @$(call emit-line,$(wordlist 128501,129000,$(1)),$(2)) + @$(call emit-line,$(wordlist 129001,129500,$(1)),$(2)) + @$(call emit-line,$(wordlist 129501,130000,$(1)),$(2)) + @$(call emit-line,$(wordlist 130001,130500,$(1)),$(2)) + @$(call emit-line,$(wordlist 130501,131000,$(1)),$(2)) + @$(call emit-line,$(wordlist 131001,131500,$(1)),$(2)) + @$(call emit-line,$(wordlist 131501,132000,$(1)),$(2)) + @$(call emit-line,$(wordlist 132001,132500,$(1)),$(2)) + @$(call emit-line,$(wordlist 132501,133000,$(1)),$(2)) + @$(call emit-line,$(wordlist 133001,133500,$(1)),$(2)) + @$(call emit-line,$(wordlist 133501,134000,$(1)),$(2)) + @$(call emit-line,$(wordlist 134001,134500,$(1)),$(2)) + @$(call emit-line,$(wordlist 134501,135000,$(1)),$(2)) + @$(call emit-line,$(wordlist 135001,135500,$(1)),$(2)) + @$(call emit-line,$(wordlist 135501,136000,$(1)),$(2)) + @$(call emit-line,$(wordlist 136001,136500,$(1)),$(2)) + @$(call emit-line,$(wordlist 136501,137000,$(1)),$(2)) + @$(call emit-line,$(wordlist 137001,137500,$(1)),$(2)) + @$(call emit-line,$(wordlist 137501,138000,$(1)),$(2)) + @$(call emit-line,$(wordlist 138001,138500,$(1)),$(2)) + @$(call emit-line,$(wordlist 138501,139000,$(1)),$(2)) + @$(call emit-line,$(wordlist 139001,139500,$(1)),$(2)) + @$(call emit-line,$(wordlist 139501,140000,$(1)),$(2)) + @$(call emit-line,$(wordlist 140001,140500,$(1)),$(2)) + @$(call emit-line,$(wordlist 140501,141000,$(1)),$(2)) + @$(call emit-line,$(wordlist 141001,141500,$(1)),$(2)) + @$(call emit-line,$(wordlist 141501,142000,$(1)),$(2)) + @$(call emit-line,$(wordlist 142001,142500,$(1)),$(2)) + @$(call emit-line,$(wordlist 142501,143000,$(1)),$(2)) + @$(call emit-line,$(wordlist 143001,143500,$(1)),$(2)) + @$(call emit-line,$(wordlist 143501,144000,$(1)),$(2)) + @$(call emit-line,$(wordlist 144001,144500,$(1)),$(2)) + @$(call emit-line,$(wordlist 144501,145000,$(1)),$(2)) + @$(call emit-line,$(wordlist 145001,145500,$(1)),$(2)) + @$(call emit-line,$(wordlist 145501,146000,$(1)),$(2)) + @$(call emit-line,$(wordlist 146001,146500,$(1)),$(2)) + @$(call emit-line,$(wordlist 146501,147000,$(1)),$(2)) + @$(call emit-line,$(wordlist 147001,147500,$(1)),$(2)) + @$(call emit-line,$(wordlist 147501,148000,$(1)),$(2)) + @$(call emit-line,$(wordlist 148001,148500,$(1)),$(2)) + @$(call emit-line,$(wordlist 148501,149000,$(1)),$(2)) + @$(call emit-line,$(wordlist 149001,149500,$(1)),$(2)) + @$(call emit-line,$(wordlist 149501,150000,$(1)),$(2)) + @$(if $(wordlist 150001,150002,$(1)),$(error dump-words-to-file: Too many words ($(words $(1))))) endef # Return jar arguments to compress files in a given directory # $(1): directory @@ -3206,7 +3283,7 @@ $(check_non_elf_file_timestamp): $(1) $(LLVM_READOBJ) $(hide) mkdir -p "$$(dir $$@)" $(hide) rm -f "$$@" $(hide) \ - if $(LLVM_READOBJ) -h "$$<" >/dev/null 2>&1; then \ + if $(LLVM_READOBJ) -h "$$<" 2>/dev/null | grep -q "^Format: elf"; then \ $(call echo-error,$(2),$(3)); \ $(call echo-error,$(2),found ELF file: $$<); \ false; \ diff --git a/core/dex_preopt.mk b/core/dex_preopt.mk index 906d7f0163..9b5e1dbeeb 100644 --- a/core/dex_preopt.mk +++ b/core/dex_preopt.mk @@ -13,28 +13,6 @@ else install-on-system-other = $(filter-out $(PRODUCT_DEXPREOPT_SPEED_APPS) $(PRODUCT_SYSTEM_SERVER_APPS),$(basename $(notdir $(filter $(foreach f,$(SYSTEM_OTHER_ODEX_FILTER),$(TARGET_OUT)/$(f)),$(1))))) endif -# Install boot images for testing on host. We exclude framework image as it is not part of art manifest. -my_boot_image_arch := HOST_ARCH -my_boot_image_out := $(HOST_OUT) -my_boot_image_syms := $(HOST_OUT)/symbols -HOST_BOOT_IMAGE_MODULE := \ - $(foreach my_boot_image_name,art_host,$(strip \ - $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \ - $(my_boot_image_module))) -HOST_BOOT_IMAGE := $(call module-installed-files,$(HOST_BOOT_IMAGE_MODULE)) -ifdef HOST_2ND_ARCH - my_boot_image_arch := HOST_2ND_ARCH - 2ND_HOST_BOOT_IMAGE_MODULE := \ - $(foreach my_boot_image_name,art_host,$(strip \ - $(eval include $(BUILD_SYSTEM)/dex_preopt_libart.mk) \ - $(my_boot_image_module))) - 2ND_HOST_BOOT_IMAGE := $(call module-installed-files,$(2ND_HOST_BOOT_IMAGE_MODULE)) -endif -my_boot_image_arch := -my_boot_image_out := -my_boot_image_syms := -my_boot_image_module := - # Build the boot.zip which contains the boot jars and their compilation output # We can do this only if preopt is enabled and if the product uses libart config (which sets the # default properties for preopting). @@ -104,42 +82,6 @@ $(boot_zip): $(bootclasspath_jars) $(system_server_jars) $(SOONG_ZIP) $(MERGE_ZI $(call dist-for-goals, droidcore, $(boot_zip)) -# Build the system_server.zip which contains the Apex system server jars and standalone system server jars -system_server_dex2oat_dir := $(SOONG_OUT_DIR)/system_server_dexjars -system_server_zip := $(PRODUCT_OUT)/system_server.zip -# non_updatable_system_server_jars contains jars in /system and /system_ext that are not part of an apex. -non_updatable_system_server_jars := \ - $(foreach m,$(PRODUCT_SYSTEM_SERVER_JARS),\ - $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) - -apex_system_server_jars := \ - $(foreach m,$(PRODUCT_APEX_SYSTEM_SERVER_JARS),\ - $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) - -apex_standalone_system_server_jars := \ - $(foreach m,$(PRODUCT_APEX_STANDALONE_SYSTEM_SERVER_JARS),\ - $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) - -standalone_system_server_jars := \ - $(foreach m,$(PRODUCT_STANDALONE_SYSTEM_SERVER_JARS),\ - $(system_server_dex2oat_dir)/$(call word-colon,2,$(m)).jar) - -$(system_server_zip): PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR := $(system_server_dex2oat_dir) -$(system_server_zip): PRIVATE_SYSTEM_SERVER_JARS := $(non_updatable_system_server_jars) -$(system_server_zip): PRIVATE_APEX_SYSTEM_SERVER_JARS := $(apex_system_server_jars) -$(system_server_zip): PRIVATE_APEX_STANDALONE_SYSTEM_SERVER_JARS := $(apex_standalone_system_server_jars) -$(system_server_zip): PRIVATE_STANDALONE_SYSTEM_SERVER_JARS := $(standalone_system_server_jars) -$(system_server_zip): $(system_server_jars) $(apex_system_server_jars) $(apex_standalone_system_server_jars) $(standalone_system_server_jars) $(SOONG_ZIP) - @echo "Create system server package: $@" - rm -f $@ - $(SOONG_ZIP) -o $@ \ - -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_SYSTEM_SERVER_JARS)) \ - -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_APEX_SYSTEM_SERVER_JARS)) \ - -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_APEX_STANDALONE_SYSTEM_SERVER_JARS)) \ - -C $(PRIVATE_SYSTEM_SERVER_DEX2OAT_DIR) $(addprefix -f ,$(PRIVATE_STANDALONE_SYSTEM_SERVER_JARS)) - -$(call dist-for-goals, droidcore, $(system_server_zip)) - endif #PRODUCT_USES_DEFAULT_ART_CONFIG endif #WITH_DEXPREOPT_ART_BOOT_IMG_ONLY endif #WITH_DEXPREOPT diff --git a/core/dex_preopt_libart.mk b/core/dex_preopt_libart.mk deleted file mode 100644 index a2c9942a41..0000000000 --- a/core/dex_preopt_libart.mk +++ /dev/null @@ -1,109 +0,0 @@ -#################################### -# ART boot image installation -# Input variables: -# my_boot_image_name: the boot image to install -# my_boot_image_arch: the architecture to install (e.g. TARGET_ARCH, not expanded) -# my_boot_image_out: the install directory (e.g. $(PRODUCT_OUT)) -# my_boot_image_syms: the symbols director (e.g. $(TARGET_OUT_UNSTRIPPED)) -# -# Output variables: -# my_boot_image_module: the created module name. Empty if no module is created. -# -# Install the boot images compiled by Soong. -# Create a module named dexpreopt_bootjar.$(my_boot_image_name)_$($(my_boot_image_arch)) -# that installs all of boot image files. -# If there is no file to install for $(my_boot_image_name), for example when -# building an unbundled build, then no module is created. -# -#################################### - -# Takes a list of src:dest install pairs and returns a new list with a path -# prefixed to each dest value. -# $(1): list of src:dest install pairs -# $(2): path to prefix to each dest value -define prefix-copy-many-files-dest -$(foreach v,$(1),$(call word-colon,1,$(v)):$(2)$(call word-colon,2,$(v))) -endef - -# Converts an architecture-specific vdex path into a location that can be shared -# between architectures. -define vdex-shared-install-path -$(dir $(patsubst %/,%,$(dir $(1))))$(notdir $(1)) -endef - -# Takes a list of src:dest install pairs of vdex files and returns a new list -# where each dest has been rewritten to the shared location for vdex files. -define vdex-copy-many-files-shared-dest -$(foreach v,$(1),$(call word-colon,1,$(v)):$(call vdex-shared-install-path,$(call word-colon,2,$(v)))) -endef - -# Creates a rule to symlink an architecture specific vdex file to the shared -# location for that vdex file. -define symlink-vdex-file -$(strip \ - $(call symlink-file,\ - $(call vdex-shared-install-path,$(1)),\ - ../$(notdir $(1)),\ - $(1))\ - $(1)) -endef - -# Takes a list of src:dest install pairs of vdex files and creates rules to -# symlink each dest to the shared location for that vdex file. -define symlink-vdex-files -$(foreach v,$(1),$(call symlink-vdex-file,$(call word-colon,2,$(v)))) -endef - -my_boot_image_module := - -my_suffix := $(my_boot_image_name)_$($(my_boot_image_arch)) -my_copy_pairs := $(call prefix-copy-many-files-dest,$(DEXPREOPT_IMAGE_BUILT_INSTALLED_$(my_suffix)),$(my_boot_image_out)) -my_vdex_copy_pairs := $(call prefix-copy-many-files-dest,$(DEXPREOPT_IMAGE_VDEX_BUILT_INSTALLED_$(my_suffix)),$(my_boot_image_out)) -my_vdex_copy_shared_pairs := $(call vdex-copy-many-files-shared-dest,$(my_vdex_copy_pairs)) -ifeq (,$(filter %_2ND_ARCH,$(my_boot_image_arch))) - # Only install the vdex to the shared location for the primary architecture. - my_copy_pairs += $(my_vdex_copy_shared_pairs) -endif - -my_unstripped_copy_pairs := $(call prefix-copy-many-files-dest,$(DEXPREOPT_IMAGE_UNSTRIPPED_BUILT_INSTALLED_$(my_suffix)),$(my_boot_image_syms)) - -# Generate the boot image module only if there is any file to install. -ifneq (,$(strip $(my_copy_pairs))) - my_first_pair := $(firstword $(my_copy_pairs)) - my_rest_pairs := $(wordlist 2,$(words $(my_copy_pairs)),$(my_copy_pairs)) - - my_first_src := $(call word-colon,1,$(my_first_pair)) - my_first_dest := $(call word-colon,2,$(my_first_pair)) - - my_installed := $(call copy-many-files,$(my_copy_pairs)) - my_unstripped_installed := $(call copy-many-files,$(my_unstripped_copy_pairs)) - - my_symlinks := $(call symlink-vdex-files,$(my_vdex_copy_pairs)) - - # We don't have a LOCAL_PATH for the auto-generated modules, so let it be the $(BUILD_SYSTEM). - LOCAL_PATH := $(BUILD_SYSTEM) - # Hack to let these pseudo-modules wrapped around Soong modules use LOCAL_SOONG_INSTALLED_MODULE. - LOCAL_MODULE_MAKEFILE := $(SOONG_ANDROID_MK) - - include $(CLEAR_VARS) - LOCAL_MODULE := dexpreopt_bootjar.$(my_suffix) - LOCAL_PREBUILT_MODULE_FILE := $(my_first_src) - LOCAL_MODULE_PATH := $(dir $(my_first_dest)) - LOCAL_MODULE_STEM := $(notdir $(my_first_dest)) - LOCAL_SOONG_INSTALL_PAIRS := $(my_copy_pairs) - LOCAL_SOONG_INSTALL_SYMLINKS := $(my_symlinks) - LOCAL_SOONG_INSTALLED_MODULE := $(my_first_dest) - LOCAL_SOONG_LICENSE_METADATA := $(DEXPREOPT_IMAGE_LICENSE_METADATA_$(my_suffix)) - ifneq (,$(strip $(filter HOST_%,$(my_boot_image_arch)))) - LOCAL_IS_HOST_MODULE := true - endif - LOCAL_MODULE_CLASS := ETC - include $(BUILD_PREBUILT) - $(LOCAL_BUILT_MODULE): | $(my_unstripped_installed) - # Installing boot.art causes all boot image bits to be installed. - # Keep this old behavior in case anyone still needs it. - $(LOCAL_INSTALLED_MODULE): $(wordlist 2,$(words $(my_installed)),$(my_installed)) $(my_symlinks) - $(my_all_targets): $(my_installed) $(my_symlinks) - - my_boot_image_module := $(LOCAL_MODULE) -endif # my_copy_pairs != empty diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk index e7086b7e4e..6fe9d38a36 100644 --- a/core/dex_preopt_odex_install.mk +++ b/core/dex_preopt_odex_install.mk @@ -152,7 +152,7 @@ my_dexpreopt_libs_all := $(sort $(my_dexpreopt_libs) $(my_dexpreopt_libs_compat) # this dexpreopt.config is generated. So it's necessary to add file-level # dependencies between dexpreopt.config files. my_dexpreopt_dep_configs := $(foreach lib, \ - $(filter-out $(my_dexpreopt_libs_compat),$(LOCAL_USES_LIBRARIES) $(my_filtered_optional_uses_libraries)), \ + $(filter-out $(my_dexpreopt_libs_compat) $(FRAMEWORK_LIBRARIES),$(LOCAL_USES_LIBRARIES) $(my_filtered_optional_uses_libraries)), \ $(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,)/dexpreopt.config) # 1: SDK version diff --git a/core/envsetup.mk b/core/envsetup.mk index c063f60a15..f82e861abf 100644 --- a/core/envsetup.mk +++ b/core/envsetup.mk @@ -417,6 +417,7 @@ HOST_OUT_SDK_ADDON := $(HOST_OUT)/sdk_addon HOST_OUT_NATIVE_TESTS := $(HOST_OUT)/nativetest64 HOST_OUT_COVERAGE := $(HOST_OUT)/coverage HOST_OUT_TESTCASES := $(HOST_OUT)/testcases +HOST_OUT_ETC := $(HOST_OUT)/etc .KATI_READONLY := \ HOST_OUT_EXECUTABLES \ HOST_OUT_SHARED_LIBRARIES \ @@ -425,7 +426,8 @@ HOST_OUT_TESTCASES := $(HOST_OUT)/testcases HOST_OUT_SDK_ADDON \ HOST_OUT_NATIVE_TESTS \ HOST_OUT_COVERAGE \ - HOST_OUT_TESTCASES + HOST_OUT_TESTCASES \ + HOST_OUT_ETC HOST_CROSS_OUT_EXECUTABLES := $(HOST_CROSS_OUT)/bin HOST_CROSS_OUT_SHARED_LIBRARIES := $(HOST_CROSS_OUT)/lib diff --git a/core/java.mk b/core/java.mk index 5fbc916859..41a1b1ba84 100644 --- a/core/java.mk +++ b/core/java.mk @@ -140,8 +140,7 @@ ifneq ($(strip $(logtags_sources)),) logtags_java_sources := $(patsubst %.logtags,%.java,$(addprefix $(intermediates.COMMON)/logtags/, $(logtags_sources))) logtags_sources := $(addprefix $(LOCAL_PATH)/, $(logtags_sources)) -$(logtags_java_sources): PRIVATE_MERGED_TAG := $(TARGET_OUT_COMMON_INTERMEDIATES)/all-event-log-tags.txt -$(logtags_java_sources): $(intermediates.COMMON)/logtags/%.java: $(LOCAL_PATH)/%.logtags $(TARGET_OUT_COMMON_INTERMEDIATES)/all-event-log-tags.txt $(JAVATAGS) build/make/tools/event_log_tags.py +$(logtags_java_sources): $(intermediates.COMMON)/logtags/%.java: $(LOCAL_PATH)/%.logtags $(JAVATAGS) $(transform-logtags-to-java) else diff --git a/core/java_common.mk b/core/java_common.mk index a21f062029..f574b7623e 100644 --- a/core/java_common.mk +++ b/core/java_common.mk @@ -32,7 +32,7 @@ ifeq (,$(LOCAL_JAVA_LANGUAGE_VERSION)) else ifneq (,$(LOCAL_SDK_VERSION)$(TARGET_BUILD_USE_PREBUILT_SDKS)) # TODO(ccross): allow 1.9 for current and unbundled once we have SDK system modules LOCAL_JAVA_LANGUAGE_VERSION := 1.8 - else ifeq ($(EXPERIMENTAL_TARGET_JAVA_VERSION_21),true) + else ifeq ($(RELEASE_TARGET_JAVA_21),true) LOCAL_JAVA_LANGUAGE_VERSION := 21 else LOCAL_JAVA_LANGUAGE_VERSION := 17 diff --git a/core/layoutlib_data.mk b/core/layoutlib_data.mk index e420a004de..dabcfb20c9 100644 --- a/core/layoutlib_data.mk +++ b/core/layoutlib_data.mk @@ -31,8 +31,18 @@ $(keyboards): $(KEYBOARD_TEMP)/%.kcm: frameworks/base/data/keyboards/%.kcm $(hide) mkdir -p $(dir $@) $(hide) cp -vf $< $@ -# List of all data files - font files, font configuration files, key character map files -LAYOUTLIB_FILES := $(fonts_device) $(font_config) $(keyboards) +HYPHEN_TEMP := $(call intermediates-dir-for,PACKAGING,hyphen,HOST,COMMON) + +# The hyphenation pattern files needed to support text hyphenation +hyphen := $(filter $(TARGET_OUT)/usr/hyphen-data/%.hyb, $(INTERNAL_SYSTEMIMAGE_FILES)) +hyphen := $(addprefix $(HYPHEN_TEMP)/, $(notdir $(hyphen))) + +$(hyphen): $(HYPHEN_TEMP)/%: $(TARGET_OUT)/usr/hyphen-data/% + $(hide) mkdir -p $(dir $@) + $(hide) cp -vf $< $@ + +# List of all data files - font files, font configuration files, key character map files, hyphenation pattern files +LAYOUTLIB_FILES := $(fonts_device) $(font_config) $(keyboards) $(hyphen) .PHONY: layoutlib layoutlib-tests layoutlib layoutlib-tests: $(LAYOUTLIB_FILES) @@ -40,6 +50,7 @@ layoutlib layoutlib-tests: $(LAYOUTLIB_FILES) $(call dist-for-goals, layoutlib, $(foreach m,$(fonts_device), $(m):layoutlib_native/fonts/$(notdir $(m)))) $(call dist-for-goals, layoutlib, $(foreach m,$(font_config), $(m):layoutlib_native/fonts/$(notdir $(m)))) $(call dist-for-goals, layoutlib, $(foreach m,$(keyboards), $(m):layoutlib_native/keyboards/$(notdir $(m)))) +$(call dist-for-goals, layoutlib, $(foreach m,$(hyphen), $(m):layoutlib_native/hyphen-data/$(notdir $(m)))) FONT_TEMP := font_config := @@ -95,6 +106,7 @@ LAYOUTLIB_SBOM := $(call intermediates-dir-for,PACKAGING,layoutlib-sbom,HOST) _layoutlib_font_config_files := $(sort $(wildcard frameworks/base/data/fonts/*.xml)) _layoutlib_fonts_files := $(filter $(TARGET_OUT)/fonts/%.ttf $(TARGET_OUT)/fonts/%.ttc $(TARGET_OUT)/fonts/%.otf, $(INTERNAL_SYSTEMIMAGE_FILES)) _layoutlib_keyboard_files := $(sort $(wildcard frameworks/base/data/keyboards/*.kcm)) +_layoutlib_hyphen_files := $(filter $(TARGET_OUT)/usr/hyphen-data/%.hyb, $(INTERNAL_SYSTEMIMAGE_FILES)) # Find out files disted with layoutlib in Soong. ### Filter out static libraries for Windows and files already handled in make. @@ -124,6 +136,13 @@ $(LAYOUTLIB_SBOM)/sbom-metadata.csv: echo data/keyboards/$(notdir $f),frameworks/base/data/keyboards,prebuilt_etc,,,,,$f,,, >> $@; \ ) + $(foreach f,$(_layoutlib_hyphen_files), \ + $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \ + $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \ + $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \ + echo data/hyphen-data/$(notdir $f),$(_module_path),$(_soong_module_type),,,,,$f,,, >> $@; \ + ) + $(foreach f,$(_layoutlib_files_disted_by_soong), \ $(eval _prebuilt_module_file := $(call word-colon,1,$f)) \ $(eval _dist_file := $(call word-colon,2,$f)) \ @@ -152,7 +171,7 @@ $(LAYOUTLIB_SBOM)/sbom-metadata.csv: .PHONY: layoutlib-sbom layoutlib-sbom: $(LAYOUTLIB_SBOM)/layoutlib.spdx.json -$(LAYOUTLIB_SBOM)/layoutlib.spdx.json: $(PRODUCT_OUT)/always_dirty_file.txt $(GEN_SBOM) $(LAYOUTLIB_SBOM)/sbom-metadata.csv $(_layoutlib_font_config_files) $(_layoutlib_fonts_files) $(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop $(_layoutlib_keyboard_files) $(LAYOUTLIB_RES_FILES) $(EMULATED_OVERLAYS_FILES) $(DEVICE_OVERLAYS_FILES) +$(LAYOUTLIB_SBOM)/layoutlib.spdx.json: $(PRODUCT_OUT)/always_dirty_file.txt $(GEN_SBOM) $(LAYOUTLIB_SBOM)/sbom-metadata.csv $(_layoutlib_font_config_files) $(_layoutlib_fonts_files) $(LAYOUTLIB_BUILD_PROP)/layoutlib-build.prop $(_layoutlib_keyboard_files) $(_layoutlib_hyphen_files) $(LAYOUTLIB_RES_FILES) $(EMULATED_OVERLAYS_FILES) $(DEVICE_OVERLAYS_FILES) rm -rf $@ $(GEN_SBOM) --output_file $@ --metadata $(LAYOUTLIB_SBOM)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --module_name "layoutlib" --json diff --git a/core/main.mk b/core/main.mk index 80ffec43b7..9710dc86c4 100644 --- a/core/main.mk +++ b/core/main.mk @@ -45,11 +45,6 @@ BUILD_HOSTNAME_FILE := $(SOONG_OUT_DIR)/build_hostname.txt $(KATI_obsolete_var BUILD_HOSTNAME,Use BUILD_HOSTNAME_FROM_FILE instead) $(KATI_obsolete_var FILE_NAME_TAG,https://android.googlesource.com/platform/build/+/master/Changes.md#FILE_NAME_TAG) -$(BUILD_NUMBER_FILE): - # empty rule to prevent dangling rule error for a file that is written by soong_ui -$(BUILD_HOSTNAME_FILE): - # empty rule to prevent dangling rule error for a file that is written by soong_ui - .KATI_RESTAT: $(BUILD_NUMBER_FILE) .KATI_RESTAT: $(BUILD_HOSTNAME_FILE) @@ -83,6 +78,8 @@ $(shell mkdir -p $(EMPTY_DIRECTORY) && rm -rf $(EMPTY_DIRECTORY)/*) -include test/cts-root/tools/build/config.mk # WVTS-specific config. -include test/wvts/tools/build/config.mk +# DTS-specific config. +-include test/dts/tools/build/config.mk # Clean rules @@ -275,11 +272,24 @@ FULL_BUILD := true # Include all of the makefiles in the system # -subdir_makefiles := $(SOONG_OUT_DIR)/installs-$(TARGET_PRODUCT)$(COVERAGE_SUFFIX).mk $(SOONG_ANDROID_MK) +subdir_makefiles := \ + $(SOONG_OUT_DIR)/installs-$(TARGET_PRODUCT)$(COVERAGE_SUFFIX).mk \ + $(SOONG_ANDROID_MK) \ + build/make/target/board/android-info.mk # Android.mk files are only used on Linux builds, Mac only supports Android.bp ifeq ($(HOST_OS),linux) - subdir_makefiles += $(file <$(OUT_DIR)/.module_paths/Android.mk.list) + ifeq ($(PRODUCT_IGNORE_ALL_ANDROIDMK),true) + allowed_androidmk_files := + ifdef PRODUCT_ANDROIDMK_ALLOWLIST_FILE + -include $(PRODUCT_ANDROIDMK_ALLOWLIST_FILE) + endif + allowed_androidmk_files += $(PRODUCT_ALLOWED_ANDROIDMK_FILES) + subdir_makefiles += $(filter $(allowed_androidmk_files),$(file <$(OUT_DIR)/.module_paths/Android.mk.list)) + allowed_androidmk_files := + else + subdir_makefiles += $(file <$(OUT_DIR)/.module_paths/Android.mk.list) + endif endif subdir_makefiles += $(SOONG_OUT_DIR)/late-$(TARGET_PRODUCT)$(COVERAGE_SUFFIX).mk @@ -290,7 +300,7 @@ subdir_makefiles_total := $(words int $(subdir_makefiles) post finish) $(foreach mk,$(subdir_makefiles),$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] including $(mk) ...)$(eval include $(mk))) # Build bootloader.img/radio.img, and unpack the partitions. -include $(BUILD_SYSTEM)/tasks/tools/update_bootloader_radio_image.mk +-include vendor/google_devices/$(TARGET_SOC)/prebuilts/misc_bins/update_bootloader_radio_image.mk # For an unbundled image, we can skip blueprint_tools because unbundled image # aims to remove a large number framework projects from the manifest, the @@ -687,12 +697,12 @@ endef # Scan all modules in general-tests, device-tests and other selected suites and # flatten the shared library dependencies. define update-host-shared-libs-deps-for-suites -$(foreach suite,general-tests device-tests vts tvts art-host-tests host-unit-tests,\ +$(foreach suite,general-tests device-tests vts tvts art-host-tests host-unit-tests camera-hal-tests,\ $(foreach m,$(COMPATIBILITY.$(suite).MODULES),\ $(eval my_deps := $(call get-all-shared-libs-deps,$(m)))\ $(foreach dep,$(my_deps),\ $(foreach f,$(ALL_MODULES.$(dep).HOST_SHARED_LIBRARY_FILES),\ - $(if $(filter $(suite),device-tests general-tests art-host-tests host-unit-tests),\ + $(if $(filter $(suite),device-tests general-tests art-host-tests host-unit-tests camera-hal-tests),\ $(eval my_testcases := $(HOST_OUT_TESTCASES)),\ $(eval my_testcases := $$(COMPATIBILITY_TESTCASES_OUT_$(suite))))\ $(eval target := $(my_testcases)/$(lastword $(subst /, ,$(dir $(f))))/$(notdir $(f)))\ @@ -981,8 +991,8 @@ endef # Returns modules included automatically as a result of certain BoardConfig # variables being set. define auto-included-modules - llndk_in_system \ - $(if $(DEVICE_MANIFEST_FILE),vendor_manifest.xml) \ + $(foreach vndk_ver,$(PRODUCT_EXTRA_VNDK_VERSIONS),com.android.vndk.v$(vndk_ver)) \ + llndk.libraries.txt \ $(if $(DEVICE_MANIFEST_SKUS),$(foreach sku, $(DEVICE_MANIFEST_SKUS),vendor_manifest_$(sku).xml)) \ $(if $(ODM_MANIFEST_FILES),odm_manifest.xml) \ $(if $(ODM_MANIFEST_SKUS),$(foreach sku, $(ODM_MANIFEST_SKUS),odm_manifest_$(sku).xml)) \ @@ -1185,8 +1195,9 @@ endif ifneq ($(TARGET_BUILD_APPS),) # If this build is just for apps, only build apps and not the full system by default. ifneq ($(filter all,$(TARGET_BUILD_APPS)),) - # If they used the magic goal "all" then build all apps in the source tree. - unbundled_build_modules := $(foreach m,$(sort $(ALL_MODULES)),$(if $(filter APPS,$(ALL_MODULES.$(m).CLASS)),$(m))) + # The magic goal "all" used to build all apps in the source tree. This was deprecated + # so that we can know all TARGET_BUILD_APPS apps are built with soong for soong-only builds. + $(error TARGET_BUILD_APPS=all is deprecated) else unbundled_build_modules := $(sort $(TARGET_BUILD_APPS)) endif @@ -1384,6 +1395,7 @@ droidcore-unbundled: $(filter $(HOST_OUT_ROOT)/%,$(modules_to_install)) \ $(INSTALLED_RAMDISK_TARGET) \ $(INSTALLED_BOOTIMAGE_TARGET) \ $(INSTALLED_INIT_BOOT_IMAGE_TARGET) \ + $(INSTALLED_DTBOIMAGE_TARGET) \ $(INSTALLED_RADIOIMAGE_TARGET) \ $(INSTALLED_DEBUG_RAMDISK_TARGET) \ $(INSTALLED_DEBUG_BOOTIMAGE_TARGET) \ @@ -1455,7 +1467,6 @@ droidcore: droidcore-unbundled # dist_files only for putting your library into the dist directory with a full build. .PHONY: dist_files -$(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json) $(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json) .PHONY: apps_only @@ -1609,7 +1620,6 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) ifneq ($(ANDROID_BUILD_EMBEDDED),true) $(call dist-for-goals-with-filenametag, droidcore, \ - $(APPS_ZIP) \ $(INTERNAL_EMULATOR_PACKAGE_TARGET) \ ) endif @@ -1656,24 +1666,6 @@ else ifeq ($(TARGET_BUILD_UNBUNDLED),$(TARGET_BUILD_UNBUNDLED_IMAGE)) $(call dist-for-goals, dist_files, $(JACOCO_REPORT_CLASSES_ALL)) endif - # Put XML formatted API files in the dist dir. - $(TARGET_OUT_COMMON_INTERMEDIATES)/api.xml: $(call java-lib-files,$(ANDROID_PUBLIC_STUBS)) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/system-api.xml: $(call java-lib-files,$(ANDROID_SYSTEM_STUBS)) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/module-lib-api.xml: $(call java-lib-files,$(ANDROID_MODULE_LIB_STUBS)) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/system-server-api.xml: $(call java-lib-files,$(ANDROID_SYSTEM_SERVER_STUBS)) $(APICHECK) - $(TARGET_OUT_COMMON_INTERMEDIATES)/test-api.xml: $(call java-lib-files,$(ANDROID_TEST_STUBS)) $(APICHECK) - - api_xmls := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/,api.xml system-api.xml module-lib-api.xml system-server-api.xml test-api.xml) - $(api_xmls): - $(hide) echo "Converting API file to XML: $@" - $(hide) mkdir -p $(dir $@) - $(hide) $(APICHECK_COMMAND) jar-to-jdiff $< $@ - - $(foreach xml,$(sort $(api_xmls)),$(call declare-1p-target,$(xml),)) - - $(call dist-for-goals, dist_files, $(api_xmls)) - api_xmls := - ifdef CLANG_COVERAGE $(foreach f,$(SOONG_NDK_API_XML), \ $(call dist-for-goals,droidcore,$(f):ndk_apis/$(notdir $(f)))) @@ -1743,10 +1735,6 @@ dump-files: @echo $(sort $(patsubst $(PRODUCT_OUT)/%,%,$(filter $(PRODUCT_OUT)/%,$(modules_to_install)))) | tr -s ' ' '\n' @echo Successfully dumped product target file list. -.PHONY: nothing -nothing: - @echo Successfully read the makefiles. - .PHONY: tidy_only tidy_only: @echo Successfully make tidy_only. @@ -1899,20 +1887,20 @@ $(SOONG_OUT_DIR)/compliance-metadata/$(TARGET_PRODUCT)/make-metadata.csv: $(eval _kernel_module_copy_files := $(sort $(filter %$(_path_on_device),$(KERNEL_MODULE_COPY_FILES)))) \ $(eval _is_build_prop := $(call is-build-prop,$f)) \ $(eval _is_notice_file := $(call is-notice-file,$f)) \ - $(eval _is_dexpreopt_image_profile := $(if $(filter %:/$(_path_on_device),$(DEXPREOPT_IMAGE_PROFILE_BUILT_INSTALLED)),Y)) \ $(eval _is_product_system_other_avbkey := $(if $(findstring $f,$(INSTALLED_PRODUCT_SYSTEM_OTHER_AVBKEY_TARGET)),Y)) \ $(eval _is_event_log_tags_file := $(if $(findstring $f,$(event_log_tags_file)),Y)) \ $(eval _is_system_other_odex_marker := $(if $(findstring $f,$(INSTALLED_SYSTEM_OTHER_ODEX_MARKER)),Y)) \ $(eval _is_kernel_modules_blocklist := $(if $(findstring $f,$(ALL_KERNEL_MODULES_BLOCKLIST)),Y)) \ $(eval _is_fsverity_build_manifest_apk := $(if $(findstring $f,$(ALL_FSVERITY_BUILD_MANIFEST_APK)),Y)) \ - $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \ + $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file) $(product_linker_config_file)),Y)) \ $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \ $(eval _is_flags_file := $(if $(findstring $f, $(ALL_FLAGS_FILES)),Y)) \ $(eval _is_rootdir_symlink := $(if $(findstring $f, $(ALL_ROOTDIR_SYMLINKS)),Y)) \ - $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)$(_is_flags_file)$(_is_rootdir_symlink)) \ + $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)$(_is_flags_file)$(_is_rootdir_symlink)) \ $(eval _static_libs := $(if $(_is_soong_module),,$(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES))) \ $(eval _whole_static_libs := $(if $(_is_soong_module),,$(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES))) \ - $(eval _license_text := $(if $(filter $(_build_output_path),$(ALL_NON_MODULES)),$(ALL_NON_MODULES.$(_build_output_path).NOTICES))) \ + $(eval _license_text := $(if $(filter $(_build_output_path),$(ALL_NON_MODULES)),$(ALL_NON_MODULES.$(_build_output_path).NOTICES),\ + $(if $(_is_partition_compat_symlink),build/soong/licenses/LICENSE))) \ echo '$(_build_output_path),$(_module_path),$(_is_soong_module),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_static_libs),$(_whole_static_libs),$(_license_text)' >> $@; \ ) diff --git a/core/misc_prebuilt_internal.mk b/core/misc_prebuilt_internal.mk index a56220772c..b14b9ce032 100644 --- a/core/misc_prebuilt_internal.mk +++ b/core/misc_prebuilt_internal.mk @@ -25,7 +25,7 @@ endif include $(BUILD_SYSTEM)/base_rules.mk -ifneq ($(filter init%rc,$(notdir $(LOCAL_INSTALLED_MODULE)))$(filter %/etc/init,$(dir $(LOCAL_INSTALLED_MODULE))),) +ifneq ($(filter init%rc,$(notdir $(LOCAL_INSTALLED_MODULE)))$(filter %/etc/init/,$(dir $(LOCAL_INSTALLED_MODULE))),) $(eval $(call copy-init-script-file-checked,$(my_prebuilt_src_file),$(LOCAL_BUILT_MODULE))) else $(LOCAL_BUILT_MODULE) : $(my_prebuilt_src_file) diff --git a/core/ninja_config.mk b/core/ninja_config.mk index d4b7c6df11..27b4190145 100644 --- a/core/ninja_config.mk +++ b/core/ninja_config.mk @@ -19,9 +19,6 @@ PARSE_TIME_MAKE_GOALS := \ build-art% \ build_kernel-nodeps \ clean-oat% \ - continuous_instrumentation_tests \ - continuous_native_tests \ - cts \ custom_images \ dicttool_aosp \ docs \ diff --git a/core/os_licensing.mk b/core/os_licensing.mk index 1e1b7df7a9..97e55a7685 100644 --- a/core/os_licensing.mk +++ b/core/os_licensing.mk @@ -17,13 +17,17 @@ $(eval $(call xml-notice-rule,$(target_notice_file_xml_gz),"System image",$(syst $(eval $(call text-notice-rule,$(target_notice_file_txt),"System image",$(system_notice_file_message),$(SYSTEM_NOTICE_DEPS),$(SYSTEM_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_notice_html_or_xml_gz): $(target_notice_file_xml_gz) $(copy-file-to-target) endif +endif $(call declare-1p-target,$(target_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_notice_html_or_xml_gz)) endif +endif .PHONY: vendorlicense vendorlicense: $(call corresponding-license-metadata, $(VENDOR_NOTICE_DEPS)) reportmissinglicenses @@ -40,12 +44,16 @@ $(eval $(call xml-notice-rule,$(target_vendor_notice_file_xml_gz),"Vendor image" "Notices for files contained in all filesystem images except system/system_ext/product/odm/vendor_dlkm/odm_dlkm in this directory:", \ $(VENDOR_NOTICE_DEPS),$(VENDOR_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_vendor_notice_xml_gz): $(target_vendor_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_vendor_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_vendor_notice_xml_gz)) endif +endif .PHONY: odmlicense odmlicense: $(call corresponding-license-metadata, $(ODM_NOTICE_DEPS)) reportmissinglicenses @@ -59,12 +67,16 @@ $(eval $(call xml-notice-rule,$(target_odm_notice_file_xml_gz),"ODM filesystem i "Notices for files contained in the odm filesystem image in this directory:", \ $(ODM_NOTICE_DEPS),$(ODM_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_odm_notice_xml_gz): $(target_odm_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_odm_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_odm_notice_xml_gz)) endif +endif .PHONY: oemlicense oemlicense: $(call corresponding-license-metadata, $(OEM_NOTICE_DEPS)) reportmissinglicenses @@ -81,12 +93,16 @@ $(eval $(call xml-notice-rule,$(target_product_notice_file_xml_gz),"Product imag "Notices for files contained in the product filesystem image in this directory:", \ $(PRODUCT_NOTICE_DEPS),$(PRODUCT_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_product_notice_xml_gz): $(target_product_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_product_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_product_notice_xml_gz)) endif +endif .PHONY: systemextlicense systemextlicense: $(call corresponding-license-metadata, $(SYSTEM_EXT_NOTICE_DEPS)) reportmissinglicenses @@ -100,12 +116,16 @@ $(eval $(call xml-notice-rule,$(target_system_ext_notice_file_xml_gz),"System_ex "Notices for files contained in the system_ext filesystem image in this directory:", \ $(SYSTEM_EXT_NOTICE_DEPS),$(SYSTEM_EXT_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_system_ext_notice_xml_gz): $(target_system_ext_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_system_ext_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_system_ext_notice_xml_gz)) endif +endif .PHONY: vendor_dlkmlicense vendor_dlkmlicense: $(call corresponding-license-metadata, $(VENDOR_DLKM_NOTICE_DEPS)) reportmissinglicenses @@ -119,12 +139,16 @@ $(eval $(call xml-notice-rule,$(target_vendor_dlkm_notice_file_xml_gz),"Vendor_d "Notices for files contained in the vendor_dlkm filesystem image in this directory:", \ $(VENDOR_DLKM_NOTICE_DEPS),$(VENDOR_DLKM_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_vendor_dlkm_notice_xml_gz): $(target_vendor_dlkm_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_vendor_dlkm_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_vendor_dlkm_notice_xml_gz)) endif +endif .PHONY: odm_dlkmlicense odm_dlkmlicense: $(call corresponding-license-metadata, $(ODM_DLKM_NOTICE_DEPS)) reportmissinglicenses @@ -138,12 +162,16 @@ $(eval $(call xml-notice-rule,$(target_odm_dlkm_notice_file_xml_gz),"ODM_dlkm fi "Notices for files contained in the odm_dlkm filesystem image in this directory:", \ $(ODM_DLKM_NOTICE_DEPS),$(ODM_DLKM_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_odm_dlkm_notice_xml_gz): $(target_odm_dlkm_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_odm_dlkm_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_odm_dlkm_notice_xml_gz)) endif +endif .PHONY: system_dlkmlicense system_dlkmlicense: $(call corresponding-license-metadata, $(SYSTEM_DLKM_NOTICE_DEPS)) reportmissinglicenses @@ -157,11 +185,15 @@ $(eval $(call xml-notice-rule,$(target_system_dlkm_notice_file_xml_gz),"System_d "Notices for files contained in the system_dlkm filesystem image in this directory:", \ $(SYSTEM_DLKM_NOTICE_DEPS),$(SYSTEM_DLKM_NOTICE_DEPS))) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(installed_system_dlkm_notice_xml_gz): $(target_system_dlkm_notice_file_xml_gz) $(copy-file-to-target) +endif $(call declare-1p-target,$(target_system_dlkm_notice_file_xml_gz)) +ifneq ($(PRODUCT_USE_SOONG_NOTICE_XML),true) $(call declare-1p-target,$(installed_sysetm_dlkm_notice_xml_gz)) endif +endif endif # not TARGET_BUILD_APPS diff --git a/core/packaging/flags.mk b/core/packaging/flags.mk index a77956bdea..a371a00461 100644 --- a/core/packaging/flags.mk +++ b/core/packaging/flags.mk @@ -18,13 +18,13 @@ # # TODO: Should we do all of the images in $(IMAGES_TO_BUILD)? -_FLAG_PARTITIONS := product system system_ext vendor +_FLAG_PARTITIONS := product system vendor # ----------------------------------------------------------------- # Aconfig Flags -# Create a summary file of build flags for each partition +# Create a summary file of build flags for a single partition # $(1): built aconfig flags file (out) # $(2): installed aconfig flags file (out) # $(3): the partition (in) @@ -36,12 +36,15 @@ $(strip $(1)): $(ACONFIG) $(strip $(4)) mkdir -p $$(dir $$(PRIVATE_OUT)) $$(if $$(PRIVATE_IN), \ $$(ACONFIG) dump --dedup --format protobuf --out $$(PRIVATE_OUT) \ - --filter container:$$(strip $(3)) $$(addprefix --cache ,$$(PRIVATE_IN)), \ + --filter container:$(strip $(3))+state:ENABLED \ + --filter container:$(strip $(3))+permission:READ_WRITE \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ echo -n > $$(PRIVATE_OUT) \ ) $(call copy-one-file, $(1), $(2)) endef + # Create a summary file of build flags for each partition # $(1): built aconfig flags file (out) # $(2): installed aconfig flags file (out) @@ -59,16 +62,22 @@ $(strip $(1)): $(ACONFIG) $(strip $(3)) $(call copy-one-file, $(1), $(2)) endef - $(foreach partition, $(_FLAG_PARTITIONS), \ $(eval aconfig_flag_summaries_protobuf.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig_flags.pb) \ $(eval $(call generate-partition-aconfig-flag-file, \ - $(TARGET_OUT_FLAGS)/$(partition)/aconfig_flags.pb, \ - $(aconfig_flag_summaries_protobuf.$(partition)), \ - $(partition), \ - $(sort $(foreach m,$(call register-names-for-partition, $(partition)), \ + $(TARGET_OUT_FLAGS)/$(partition)/aconfig_flags.pb, \ + $(aconfig_flag_summaries_protobuf.$(partition)), \ + $(partition), \ + $(sort \ + $(foreach m, $(call register-names-for-partition, $(partition)), \ $(ALL_MODULES.$(m).ACONFIG_FILES) \ - )), \ + ) \ + $(if $(filter system, $(partition)), \ + $(foreach m, $(call register-names-for-partition, system_ext), \ + $(ALL_MODULES.$(m).ACONFIG_FILES) \ + ) \ + ) \ + ) \ )) \ ) @@ -90,42 +99,61 @@ $(eval $(call generate-global-aconfig-flag-file, \ # $(1): built aconfig flags storage package map file (out) # $(2): built aconfig flags storage flag map file (out) # $(3): built aconfig flags storage flag val file (out) -# $(4): installed aconfig flags storage package map file (out) -# $(5): installed aconfig flags storage flag map file (out) -# $(6): installed aconfig flags storage flag value file (out) -# $(7): input aconfig files for the partition (in) -# $(8): partition name +# $(4): built aconfig flags storage flag info file (out) +# $(5): installed aconfig flags storage package map file (out) +# $(6): installed aconfig flags storage flag map file (out) +# $(7): installed aconfig flags storage flag value file (out) +# $(8): installed aconfig flags storage flag info file (out) +# $(9): input aconfig files for the partition (in) +# $(10): partition name define generate-partition-aconfig-storage-file $(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1))) -$(eval $(strip $(1)): PRIVATE_IN := $(strip $(7))) -$(strip $(1)): $(ACONFIG) $(strip $(7)) +$(eval $(strip $(1)): PRIVATE_IN := $(strip $(9))) + +ifneq (,$(RELEASE_FINGERPRINT_ACONFIG_PACKAGES)) +STORAGE_FILE_VERSION := 2 +else +STORAGE_FILE_VERSION := 1 +endif + +$(strip $(1)): $(ACONFIG) $(strip $(9)) mkdir -p $$(dir $$(PRIVATE_OUT)) $$(if $$(PRIVATE_IN), \ - $$(ACONFIG) create-storage --container $(8) --file package_map --out $$(PRIVATE_OUT) \ + $$(ACONFIG) create-storage --container $(10) --file package_map --out $$(PRIVATE_OUT) --version $$(STORAGE_FILE_VERSION) \ $$(addprefix --cache ,$$(PRIVATE_IN)), \ ) touch $$(PRIVATE_OUT) $(eval $(strip $(2)): PRIVATE_OUT := $(strip $(2))) -$(eval $(strip $(2)): PRIVATE_IN := $(strip $(7))) -$(strip $(2)): $(ACONFIG) $(strip $(7)) +$(eval $(strip $(2)): PRIVATE_IN := $(strip $(9))) +$(strip $(2)): $(ACONFIG) $(strip $(9)) mkdir -p $$(dir $$(PRIVATE_OUT)) $$(if $$(PRIVATE_IN), \ - $$(ACONFIG) create-storage --container $(8) --file flag_map --out $$(PRIVATE_OUT) \ + $$(ACONFIG) create-storage --container $(10) --file flag_map --out $$(PRIVATE_OUT) --version $$(STORAGE_FILE_VERSION) \ $$(addprefix --cache ,$$(PRIVATE_IN)), \ ) touch $$(PRIVATE_OUT) $(eval $(strip $(3)): PRIVATE_OUT := $(strip $(3))) -$(eval $(strip $(3)): PRIVATE_IN := $(strip $(7))) -$(strip $(3)): $(ACONFIG) $(strip $(7)) +$(eval $(strip $(3)): PRIVATE_IN := $(strip $(9))) +$(strip $(3)): $(ACONFIG) $(strip $(9)) mkdir -p $$(dir $$(PRIVATE_OUT)) $$(if $$(PRIVATE_IN), \ - $$(ACONFIG) create-storage --container $(8) --file flag_val --out $$(PRIVATE_OUT) \ + $$(ACONFIG) create-storage --container $(10) --file flag_val --out $$(PRIVATE_OUT) --version $$(STORAGE_FILE_VERSION) \ $$(addprefix --cache ,$$(PRIVATE_IN)), \ ) touch $$(PRIVATE_OUT) -$(call copy-one-file, $(strip $(1)), $(4)) -$(call copy-one-file, $(strip $(2)), $(5)) -$(call copy-one-file, $(strip $(3)), $(6)) +$(eval $(strip $(4)): PRIVATE_OUT := $(strip $(4))) +$(eval $(strip $(4)): PRIVATE_IN := $(strip $(9))) +$(strip $(4)): $(ACONFIG) $(strip $(9)) + mkdir -p $$(dir $$(PRIVATE_OUT)) + $$(if $$(PRIVATE_IN), \ + $$(ACONFIG) create-storage --container $(10) --file flag_info --out $$(PRIVATE_OUT) --version $$(STORAGE_FILE_VERSION) \ + $$(addprefix --cache ,$$(PRIVATE_IN)), \ + ) + touch $$(PRIVATE_OUT) +$(call copy-one-file, $(strip $(1)), $(5)) +$(call copy-one-file, $(strip $(2)), $(6)) +$(call copy-one-file, $(strip $(3)), $(7)) +$(call copy-one-file, $(strip $(4)), $(8)) endef ifeq ($(RELEASE_CREATE_ACONFIG_STORAGE_FILE),true) @@ -133,13 +161,16 @@ $(foreach partition, $(_FLAG_PARTITIONS), \ $(eval aconfig_storage_package_map.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/package.map) \ $(eval aconfig_storage_flag_map.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/flag.map) \ $(eval aconfig_storage_flag_val.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/flag.val) \ + $(eval aconfig_storage_flag_info.$(partition) := $(PRODUCT_OUT)/$(partition)/etc/aconfig/flag.info) \ $(eval $(call generate-partition-aconfig-storage-file, \ $(TARGET_OUT_FLAGS)/$(partition)/package.map, \ $(TARGET_OUT_FLAGS)/$(partition)/flag.map, \ $(TARGET_OUT_FLAGS)/$(partition)/flag.val, \ + $(TARGET_OUT_FLAGS)/$(partition)/flag.info, \ $(aconfig_storage_package_map.$(partition)), \ $(aconfig_storage_flag_map.$(partition)), \ $(aconfig_storage_flag_val.$(partition)), \ + $(aconfig_storage_flag_info.$(partition)), \ $(aconfig_flag_summaries_protobuf.$(partition)), \ $(partition), \ )) \ @@ -155,6 +186,7 @@ required_flags_files := \ $(aconfig_storage_package_map.$(partition)) \ $(aconfig_storage_flag_map.$(partition)) \ $(aconfig_storage_flag_val.$(partition)) \ + $(aconfig_storage_flag_info.$(partition)) \ )) ALL_DEFAULT_INSTALLED_MODULES += $(required_flags_files) @@ -174,5 +206,5 @@ $(foreach partition, $(_FLAG_PARTITIONS), \ $(eval aconfig_storage_package_map.$(partition):=) \ $(eval aconfig_storage_flag_map.$(partition):=) \ $(eval aconfig_storage_flag_val.$(partition):=) \ + $(eval aconfig_storage_flag_info.$(partition):=) \ ) - diff --git a/core/product.mk b/core/product.mk index 4c23e5dfdd..1fbc3eef51 100644 --- a/core/product.mk +++ b/core/product.mk @@ -284,6 +284,9 @@ _product_list_vars += PRODUCT_EXTRA_VNDK_VERSIONS # Whether APEX should be compressed or not _product_single_value_vars += PRODUCT_COMPRESSED_APEX +# Default fs type for APEX payload image (apex_payload.img) +_product_single_value_vars += PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE + # VNDK version of product partition. It can be 'current' if the product # partitions uses PLATFORM_VNDK_VERSION. _product_single_value_vars += PRODUCT_PRODUCT_VNDK_VERSION @@ -366,6 +369,8 @@ _product_single_value_vars += PRODUCT_BUILD_DEBUG_VENDOR_BOOT_IMAGE _product_single_value_vars += PRODUCT_BUILD_VBMETA_IMAGE _product_single_value_vars += PRODUCT_BUILD_SUPER_EMPTY_IMAGE _product_single_value_vars += PRODUCT_BUILD_PVMFW_IMAGE +_product_single_value_vars += PRODUCT_BUILD_DESKTOP_RECOVERY_IMAGE +_product_single_value_vars += PRODUCT_BUILD_DESKTOP_UPDATE_IMAGE # List of boot jars delivered via updatable APEXes, following the same format as # PRODUCT_BOOT_JARS. @@ -390,20 +395,6 @@ _product_single_value_vars += PRODUCT_OTA_FORCE_NON_AB_PACKAGE # If set, Java module in product partition cannot use hidden APIs. _product_single_value_vars += PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE -# If set, only java_sdk_library can be used at inter-partition dependency. -# Note: Build error if BOARD_VNDK_VERSION is not set while -# PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY is true, because -# PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY has no meaning if -# BOARD_VNDK_VERSION is not set. -# Note: When PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE is not set, there are -# no restrictions at dependency between system and product partition. -_product_single_value_vars += PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY - -# Allowlist for PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY option. -# Listed modules are allowed at inter-partition dependency even if it isn't -# a java_sdk_library module. -_product_list_vars += PRODUCT_INTER_PARTITION_JAVA_LIBRARY_ALLOWLIST - # Install a copy of the debug policy to the system_ext partition, and allow # init-second-stage to load debug policy from system_ext. # This option is only meant to be set by compliance GSI targets. @@ -436,8 +427,9 @@ _product_single_value_vars += PRODUCT_MEMCG_V2_FORCE_ENABLED # If true, the cgroup v2 hierarchy will be split into apps/system subtrees _product_single_value_vars += PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED -# List of .json files to be merged/compiled into vendor/etc/linker.config.pb +# List of .json files to be merged/compiled into vendor/etc/linker.config.pb and product/etc/linker.config.pb _product_list_vars += PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS +_product_list_vars += PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS # Whether to use userfaultfd GC. # Possible values are: @@ -499,6 +491,23 @@ _product_single_value_vars += PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER # If set, build would generate system image from Soong-defined module. _product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE +# List of stub libraries specific to the product that are already present in the system image and +# should be included in the system_linker_config. +_product_list_vars += PRODUCT_EXTRA_STUB_LIBRARIES + +# If set to true, all Android.mk files will be ignored. +_product_single_value_vars += PRODUCT_IGNORE_ALL_ANDROIDMK +# When PRODUCT_IGNORE_ALL_ANDROIDMK is set to true, this variable will be used to allow some Android.mk files. +_product_list_vars += PRODUCT_ALLOWED_ANDROIDMK_FILES +# When PRODUCT_IGNORE_ALL_ANDROIDMK is set to true, path of file that contains a list of allowed Android.mk files +_product_single_value_vars += PRODUCT_ANDROIDMK_ALLOWLIST_FILE +# Setting PRODUCT_SOONG_ONLY will cause the build to default to --soong-only mode, and the main +# kati invocation will not be run. +_product_single_value_vars += PRODUCT_SOONG_ONLY + +# If set to true, use NOTICE.xml.gz generated by soong +_product_single_value_vars += PRODUCT_USE_SOONG_NOTICE_XML + .KATI_READONLY := _product_single_value_vars _product_list_vars _product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars) diff --git a/core/product_config.mk b/core/product_config.mk index 738d4cff58..019d711403 100644 --- a/core/product_config.mk +++ b/core/product_config.mk @@ -424,10 +424,12 @@ ifdef PRODUCT_DEFAULT_DEV_CERTIFICATE endif endif -$(foreach pair,$(PRODUCT_APEX_BOOT_JARS), \ - $(eval jar := $(call word-colon,2,$(pair))) \ - $(if $(findstring $(jar), $(PRODUCT_BOOT_JARS)), \ - $(error A jar in PRODUCT_APEX_BOOT_JARS must not be in PRODUCT_BOOT_JARS, but $(jar) is))) +$(foreach apexpair,$(PRODUCT_APEX_BOOT_JARS), \ + $(foreach platformpair,$(PRODUCT_BOOT_JARS), \ + $(eval apexjar := $(call word-colon,2,$(apexpair))) \ + $(eval platformjar := $(call word-colon,2,$(platformpair))) \ + $(if $(filter $(apexjar), $(platformjar)), \ + $(error A jar in PRODUCT_APEX_BOOT_JARS must not be in PRODUCT_BOOT_JARS, but $(apexjar) is)))) ENFORCE_SYSTEM_CERTIFICATE := $(PRODUCT_ENFORCE_ARTIFACT_SYSTEM_CERTIFICATE_REQUIREMENT) ENFORCE_SYSTEM_CERTIFICATE_ALLOW_LIST := $(PRODUCT_ARTIFACT_SYSTEM_CERTIFICATE_REQUIREMENT_ALLOW_LIST) @@ -466,10 +468,17 @@ $(foreach c,$(PRODUCT_SANITIZER_MODULE_CONFIGS),\ $(eval SANITIZER.$(TARGET_PRODUCT).$(m).CONFIG := $(cf)))) _psmc_modules := -# Reset ADB keys for non-debuggable builds -ifeq (,$(filter eng userdebug,$(TARGET_BUILD_VARIANT))) +# Reset ADB keys. If RELEASE_BUILD_USE_VARIANT_FLAGS is set look for +# the value of a dedicated flag. Otherwise check if build variant is +# non-debuggable. +ifneq (,$(RELEASE_BUILD_USE_VARIANT_FLAGS)) +ifneq (,$(RELEASE_BUILD_PURGE_PRODUCT_ADB_KEYS)) PRODUCT_ADB_KEYS := endif +else ifeq (,$(filter eng userdebug,$(TARGET_BUILD_VARIANT))) + PRODUCT_ADB_KEYS := +endif + ifneq ($(filter-out 0 1,$(words $(PRODUCT_ADB_KEYS))),) $(error Only one file may be in PRODUCT_ADB_KEYS: $(PRODUCT_ADB_KEYS)) endif @@ -532,6 +541,17 @@ ifdef OVERRIDE_PRODUCT_COMPRESSED_APEX PRODUCT_COMPRESSED_APEX := $(OVERRIDE_PRODUCT_COMPRESSED_APEX) endif +ifdef OVERRIDE_PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE + PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE := $(OVERRIDE_PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE) +else ifeq ($(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE),) + # Use ext4 as a default payload fs type + PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE := ext4 +endif +ifeq ($(filter ext4 erofs,$(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE)),) + $(error PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE should be either erofs or ext4,\ + not $(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE).) +endif + $(KATI_obsolete_var OVERRIDE_PRODUCT_EXTRA_VNDK_VERSIONS \ ,Use PRODUCT_EXTRA_VNDK_VERSIONS instead) @@ -602,7 +622,12 @@ else # Vendors with GRF must define BOARD_SHIPPING_API_LEVEL for the vendor API level. # In this case, the VSR API level is the minimum of the PRODUCT_SHIPPING_API_LEVEL # and RELEASE_BOARD_API_LEVEL - VSR_VENDOR_API_LEVEL := $(call math_min,$(VSR_VENDOR_API_LEVEL),$(RELEASE_BOARD_API_LEVEL)) + board_api_level := $(RELEASE_BOARD_API_LEVEL) + ifdef BOARD_API_LEVEL_PROP_OVERRIDE + board_api_level := $(BOARD_API_LEVEL_PROP_OVERRIDE) + endif + VSR_VENDOR_API_LEVEL := $(call math_min,$(VSR_VENDOR_API_LEVEL),$(board_api_level)) + board_api_level := endif endif .KATI_READONLY := VSR_VENDOR_API_LEVEL @@ -674,4 +699,12 @@ $(foreach image, \ product-build-image-config := +ifdef PRODUCT_SOONG_ONLY + ifneq ($(PRODUCT_SOONG_ONLY),true) + ifneq ($(PRODUCT_SOONG_ONLY),false) + $(error PRODUCT_SOONG_ONLY can only be true, false or unset) + endif + endif +endif + $(call readonly-product-vars) diff --git a/core/proguard.flags b/core/proguard.flags index aa406b983e..76655ca6aa 100644 --- a/core/proguard.flags +++ b/core/proguard.flags @@ -1,14 +1,3 @@ -# We have moved -dontobfuscate and -dontoptimize to the makefiles. -# dex does not like code run through proguard optimize and preverify steps. -# -dontoptimize --dontpreverify - -# Don't obfuscate. We only need dead code striping. -# -dontobfuscate - -# Add this flag in your package's own configuration if it's needed. -#-flattenpackagehierarchy - # Keep classes and members with the platform-defined @VisibleForTesting annotation. -keep @com.android.internal.annotations.VisibleForTesting class * -keepclassmembers class * { @@ -38,6 +27,17 @@ @com.android.internal.annotations.KeepForWeakReference <fields>; } +# Needed to ensure callback field references are kept in their respective +# owning classes when the downstream callback registrars only store weak refs. +-if @com.android.internal.annotations.WeaklyReferencedCallback class * +-keepclassmembers,allowaccessmodification,allowobfuscation,allowshrinking class * { + !synthetic <1> *; +} +-if class * extends @com.android.internal.annotations.WeaklyReferencedCallback ** +-keepclassmembers,allowaccessmodification,allowobfuscation,allowshrinking class * { + !synthetic <1> *; +} + # Understand the common @Keep annotation from various Android packages: # * android.support.annotation # * androidx.annotation diff --git a/core/proguard/checknotnull.flags b/core/proguard/checknotnull.flags new file mode 100644 index 0000000000..1e1e5ce46c --- /dev/null +++ b/core/proguard/checknotnull.flags @@ -0,0 +1,25 @@ +# Tell R8 that the following methods are check not null methods, and to +# replace invocations to them with a more concise nullness check that produces +# (slightly) less informative error messages + +-convertchecknotnull class com.google.common.base.Preconditions { + ** checkNotNull(...); +} + +-convertchecknotnull class java.util.Objects { + ** requireNonNull(...); +} + +-convertchecknotnull class kotlin.jvm.internal.Intrinsics { + void checkNotNull(...); + void checkExpressionValueIsNotNull(...); + void checkNotNullExpressionValue(...); + void checkReturnedValueIsNotNull(...); + void checkFieldIsNotNull(...); + void checkParameterIsNotNull(...); + void checkNotNullParameter(...); +} + +-convertchecknotnull class dagger.internal.Preconditions { + ** checkNotNull*(...); +} diff --git a/core/proguard_basic_keeps.flags b/core/proguard_basic_keeps.flags index f6b34b8217..a9416d5df0 100644 --- a/core/proguard_basic_keeps.flags +++ b/core/proguard_basic_keeps.flags @@ -1,7 +1,3 @@ -# Some classes in the libraries extend package private classes to chare common functionality -# that isn't explicitly part of the API --dontskipnonpubliclibraryclasses -dontskipnonpubliclibraryclassmembers - # Preserve line number information for debugging stack traces. -keepattributes SourceFile,LineNumberTable diff --git a/core/ravenwood_test_config_template.xml b/core/ravenwood_test_config_template.xml index 2f21baedf7..9e9dd762ff 100644 --- a/core/ravenwood_test_config_template.xml +++ b/core/ravenwood_test_config_template.xml @@ -22,6 +22,7 @@ <option name="use-ravenwood-resources" value="true" /> <option name="exclude-paths" value="java" /> <option name="null-device" value="true" /> + <option name="do-not-swallow-runner-errors" value="true" /> {EXTRA_CONFIGS} diff --git a/core/release_config.mk b/core/release_config.mk index fe2170ede4..c6986c704e 100644 --- a/core/release_config.mk +++ b/core/release_config.mk @@ -146,6 +146,9 @@ ifneq (,$(_use_protobuf)) # This will also set ALL_RELEASE_CONFIGS_FOR_PRODUCT and _used_files for us. $(eval include $(_flags_file)) $(KATI_extra_file_deps $(OUT_DIR)/release-config $(protobuf_map_files) $(_flags_file)) + ifneq (,$(_disallow_lunch_use)) + $(error Release config ${TARGET_RELEASE} is disallowed for build. Please use one of: $(ALL_RELEASE_CONFIGS_FOR_PRODUCT)) + endif else # This is the first pass of product config. $(eval include $(_flags_varmk)) @@ -153,20 +156,6 @@ ifneq (,$(_use_protobuf)) _used_files := ifeq (,$(_must_protobuf)$(RELEASE_BUILD_FLAGS_IN_PROTOBUF)) _use_protobuf := - else - _base_all_release := all_release_configs-$(TARGET_PRODUCT) - $(call dist-for-goals,droid,\ - $(_flags_dir)/$(_base_all_release).pb:build_flags/all_release_configs.pb \ - $(_flags_dir)/$(_base_all_release).textproto:build_flags/all_release_configs.textproto \ - $(_flags_dir)/$(_base_all_release).json:build_flags/all_release_configs.json \ - $(_flags_dir)/inheritance_graph-$(TARGET_PRODUCT).dot:build_flags/inheritance_graph-$(TARGET_PRODUCT).dot \ - ) -# These are always created, add an empty rule for them to keep ninja happy. -$(_flags_dir)/inheritance_graph-$(TARGET_PRODUCT).dot: - : created by $(OUT_DIR)/release-config -$(_flags_dir)/$(_base_all_release).pb $(_flags_dir)/$(_base_all_release).textproto $(_flags_dir)/$(_base_all_release).json: - : created by $(OUT_DIR)/release-config - _base_all_release := endif _flags_dir:= _flags_file:= diff --git a/core/robolectric_test_config_template.xml b/core/robolectric_test_config_template.xml index b1d0c2f4fa..509ac7bfba 100644 --- a/core/robolectric_test_config_template.xml +++ b/core/robolectric_test_config_template.xml @@ -18,13 +18,22 @@ <option name="test-suite-tag" value="robolectric" /> <option name="test-suite-tag" value="robolectric-tests" /> - <option name="java-folder" value="prebuilts/jdk/jdk21/linux-x86/" /> <option name="exclude-paths" value="java" /> <option name="use-robolectric-resources" value="true" /> + <!-- attempt to always show Tradefed errors --> + <option name="do-not-swallow-runner-errors" value="true" /> + + <!-- prevent Tradefed from hanging indefinitely in CI --> + <option name="socket-timeout" value="600000" /> + <option name="test-case-timeout" value="2m" /> + {EXTRA_CONFIGS} <test class="com.android.tradefed.testtype.IsolatedHostTest" > + + {EXTRA_TEST_RUNNER_CONFIGS} + <option name="jar" value="{MODULE}.jar" /> <option name="java-flags" value="--add-modules=jdk.compiler"/> <option name="java-flags" value="--add-opens=java.base/java.lang=ALL-UNNAMED"/> @@ -33,5 +42,15 @@ <option name="java-flags" value="--add-opens=java.base/jdk.internal.util.random=ALL-UNNAMED"/> <!-- b/251387255 --> <option name="java-flags" value="--add-opens=java.base/java.io=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.base/java.net=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.base/java.nio=ALL-UNNAMED"/> <!-- required for ShadowVMRuntime --> + <option name="java-flags" value="--add-opens=java.base/java.security=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.base/java.text=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.base/java.util=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.base/jdk.internal.access=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=java.desktop/java.awt.font=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED"/> + <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"/> </test> </configuration> diff --git a/core/soong_app_prebuilt.mk b/core/soong_app_prebuilt.mk index df1cf2d369..ab9227f676 100644 --- a/core/soong_app_prebuilt.mk +++ b/core/soong_app_prebuilt.mk @@ -224,30 +224,6 @@ my_common := COMMON include $(BUILD_SYSTEM)/link_type.mk endif # !LOCAL_IS_HOST_MODULE -ifeq (,$(filter tests,$(LOCAL_MODULE_TAGS))) - ifdef LOCAL_SOONG_DEVICE_RRO_DIRS - $(call append_enforce_rro_sources, \ - $(my_register_name), \ - false, \ - $(LOCAL_FULL_MANIFEST_FILE), \ - $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ - $(LOCAL_SOONG_DEVICE_RRO_DIRS), \ - vendor \ - ) - endif - - ifdef LOCAL_SOONG_PRODUCT_RRO_DIRS - $(call append_enforce_rro_sources, \ - $(my_register_name), \ - false, \ - $(LOCAL_FULL_MANIFEST_FILE), \ - $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \ - $(LOCAL_SOONG_PRODUCT_RRO_DIRS), \ - product \ - ) - endif -endif - ifdef LOCAL_PREBUILT_COVERAGE_ARCHIVE my_coverage_dir := $(TARGET_OUT_COVERAGE)/$(patsubst $(PRODUCT_OUT)/%,%,$(my_module_path)) my_coverage_copy_pairs := $(foreach f,$(LOCAL_PREBUILT_COVERAGE_ARCHIVE),$(f):$(my_coverage_dir)/$(notdir $(f))) diff --git a/core/soong_config.mk b/core/soong_config.mk index 1e6388a5ba..bd52d88492 100644 --- a/core/soong_config.mk +++ b/core/soong_config.mk @@ -150,6 +150,7 @@ $(call add_json_bool, ArtUseReadBarrier, $(call invert_bool,$(fi $(call add_json_str, BtConfigIncludeDir, $(BOARD_BLUETOOTH_BDROID_BUILDCFG_INCLUDE_DIR)) $(call add_json_list, DeviceKernelHeaders, $(TARGET_DEVICE_KERNEL_HEADERS) $(TARGET_BOARD_KERNEL_HEADERS) $(TARGET_PRODUCT_KERNEL_HEADERS)) $(call add_json_str, VendorApiLevel, $(BOARD_API_LEVEL)) +$(call add_json_str, VendorApiLevelPropOverride, $(BOARD_API_LEVEL_PROP_OVERRIDE)) $(call add_json_list, ExtraVndkVersions, $(PRODUCT_EXTRA_VNDK_VERSIONS)) $(call add_json_list, DeviceSystemSdkVersions, $(BOARD_SYSTEMSDK_VERSIONS)) $(call add_json_list, Platform_systemsdk_versions, $(PLATFORM_SYSTEMSDK_VERSIONS)) @@ -182,10 +183,21 @@ $(call add_json_bool, Enforce_vintf_manifest, $(filter true,$(PRODUCT $(call add_json_bool, Uml, $(filter true,$(TARGET_USER_MODE_LINUX))) $(call add_json_str, VendorPath, $(TARGET_COPY_OUT_VENDOR)) +$(call add_json_str, VendorDlkmPath, $(TARGET_COPY_OUT_VENDOR_DLKM)) +$(call add_json_bool, BuildingVendorImage, $(BUILDING_VENDOR_IMAGE)) $(call add_json_str, OdmPath, $(TARGET_COPY_OUT_ODM)) +$(call add_json_bool, BuildingOdmImage, $(BUILDING_ODM_IMAGE)) +$(call add_json_str, OdmDlkmPath, $(TARGET_COPY_OUT_ODM_DLKM)) $(call add_json_str, ProductPath, $(TARGET_COPY_OUT_PRODUCT)) +$(call add_json_bool, BuildingProductImage, $(BUILDING_PRODUCT_IMAGE)) $(call add_json_str, SystemExtPath, $(TARGET_COPY_OUT_SYSTEM_EXT)) +$(call add_json_str, SystemDlkmPath, $(TARGET_COPY_OUT_SYSTEM_DLKM)) +$(call add_json_str, OemPath, $(TARGET_COPY_OUT_OEM)) $(call add_json_bool, MinimizeJavaDebugInfo, $(filter true,$(PRODUCT_MINIMIZE_JAVA_DEBUG_INFO))) +$(call add_json_str, RecoveryPath, $(TARGET_COPY_OUT_RECOVERY)) +$(call add_json_bool, BuildingRecoveryImage, $(BUILDING_RECOVERY_IMAGE)) +$(call add_json_str, UserdataPath, $(TARGET_COPY_OUT_DATA)) +$(call add_json_bool, BuildingUserdataImage, $(BUILDING_USERDATA_IMAGE)) $(call add_json_bool, UseGoma, $(filter-out false,$(USE_GOMA))) $(call add_json_bool, UseRBE, $(filter-out false,$(USE_RBE))) @@ -242,6 +254,8 @@ $(call add_json_list, TargetFSConfigGen, $(TARGET_FS_CONFIG_GEN) $(call add_json_bool, UseSoongSystemImage, $(filter true,$(USE_SOONG_DEFINED_SYSTEM_IMAGE))) $(call add_json_str, ProductSoongDefinedSystemImage, $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)) +$(call add_json_bool, UseSoongNoticeXML, $(filter true,$(PRODUCT_USE_SOONG_NOTICE_XML))) + $(call add_json_map, VendorVars) $(foreach namespace,$(sort $(SOONG_CONFIG_NAMESPACES)),\ $(call add_json_map, $(namespace))\ @@ -265,14 +279,8 @@ $(call end_json_map) $(call add_json_bool, EnforceProductPartitionInterface, $(filter true,$(PRODUCT_ENFORCE_PRODUCT_PARTITION_INTERFACE))) $(call add_json_str, DeviceCurrentApiLevelForVendorModules, $(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES)) -$(call add_json_bool, EnforceInterPartitionJavaSdkLibrary, $(filter true,$(PRODUCT_ENFORCE_INTER_PARTITION_JAVA_SDK_LIBRARY))) -$(call add_json_list, InterPartitionJavaLibraryAllowList, $(PRODUCT_INTER_PARTITION_JAVA_LIBRARY_ALLOWLIST)) - $(call add_json_bool, CompressedApex, $(filter true,$(PRODUCT_COMPRESSED_APEX))) - -ifndef APEX_BUILD_FOR_PRE_S_DEVICES -$(call add_json_bool, TrimmedApex, $(filter true,$(PRODUCT_TRIMMED_APEX))) -endif +$(call add_json_str, DefaultApexPayloadType, $(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE)) $(call add_json_bool, BoardUsesRecoveryAsBoot, $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT))) @@ -319,6 +327,8 @@ $(call add_json_list, AfdoProfiles, $(ALL_AFDO_PROFILES)) $(call add_json_str, ProductManufacturer, $(PRODUCT_MANUFACTURER)) $(call add_json_str, ProductBrand, $(PRODUCT_BRAND)) +$(call add_json_str, ProductDevice, $(PRODUCT_DEVICE)) +$(call add_json_str, ProductModel, $(PRODUCT_MODEL)) $(call add_json_str, ReleaseVersion, $(_RELEASE_VERSION)) $(call add_json_list, ReleaseAconfigValueSets, $(RELEASE_ACONFIG_VALUE_SETS)) @@ -350,6 +360,7 @@ $(call add_json_list, SystemPropFiles, $(TARGET_SYSTEM_PROP)) $(call add_json_list, SystemExtPropFiles, $(TARGET_SYSTEM_EXT_PROP)) $(call add_json_list, ProductPropFiles, $(TARGET_PRODUCT_PROP)) $(call add_json_list, OdmPropFiles, $(TARGET_ODM_PROP)) +$(call add_json_list, VendorPropFiles, $(TARGET_VENDOR_PROP)) # Do not set ArtTargetIncludeDebugBuild into any value if PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD is not set, # to have the same behavior from runtime_libart.mk. @@ -361,12 +372,226 @@ _config_enable_uffd_gc := \ $(firstword $(OVERRIDE_ENABLE_UFFD_GC) $(PRODUCT_ENABLE_UFFD_GC) default) $(call add_json_str, EnableUffdGc, $(_config_enable_uffd_gc)) _config_enable_uffd_gc := +$(call add_json_str, BoardKernelVersion, $(BOARD_KERNEL_VERSION)) $(call add_json_list, DeviceFrameworkCompatibilityMatrixFile, $(DEVICE_FRAMEWORK_COMPATIBILITY_MATRIX_FILE)) $(call add_json_list, DeviceProductCompatibilityMatrixFile, $(DEVICE_PRODUCT_COMPATIBILITY_MATRIX_FILE)) $(call add_json_list, BoardAvbSystemAddHashtreeFooterArgs, $(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS)) $(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE))) +$(call add_json_str, AdbKeys, $(PRODUCT_ADB_KEYS)) + +$(call add_json_map, PartitionVarsForSoongMigrationOnlyDoNotUse) + $(call add_json_str, ProductDirectory, $(dir $(INTERNAL_PRODUCT))) + + $(call add_json_map,PartitionQualifiedVariables) + $(foreach image_type,INIT_BOOT BOOT VENDOR_BOOT SYSTEM VENDOR CACHE USERDATA PRODUCT SYSTEM_EXT OEM ODM VENDOR_DLKM ODM_DLKM SYSTEM_DLKM VBMETA VBMETA_SYSTEM VBMETA_SYSTEM_DLKM VBMETA_VENDOR_DLKM, \ + $(call add_json_map,$(call to-lower,$(image_type))) \ + $(call add_json_bool, BuildingImage, $(filter true,$(BUILDING_$(image_type)_IMAGE))) \ + $(call add_json_bool, PrebuiltImage, $(filter true,$(BOARD_PREBUILT_$(image_type)IMAGE))) \ + $(call add_json_str, BoardErofsCompressor, $(BOARD_$(image_type)IMAGE_EROFS_COMPRESSOR)) \ + $(call add_json_str, BoardErofsCompressHints, $(BOARD_$(image_type)IMAGE_EROFS_COMPRESS_HINTS)) \ + $(call add_json_str, BoardErofsPclusterSize, $(BOARD_$(image_type)IMAGE_EROFS_PCLUSTER_SIZE)) \ + $(call add_json_str, BoardExtfsInodeCount, $(BOARD_$(image_type)IMAGE_EXTFS_INODE_COUNT)) \ + $(call add_json_str, BoardExtfsRsvPct, $(BOARD_$(image_type)IMAGE_EXTFS_RSV_PCT)) \ + $(call add_json_str, BoardF2fsSloadCompressFlags, $(BOARD_$(image_type)IMAGE_F2FS_SLOAD_COMPRESS_FLAGS)) \ + $(call add_json_str, BoardFileSystemCompress, $(BOARD_$(image_type)IMAGE_FILE_SYSTEM_COMPRESS)) \ + $(call add_json_str, BoardFileSystemType, $(BOARD_$(image_type)IMAGE_FILE_SYSTEM_TYPE)) \ + $(call add_json_str, BoardJournalSize, $(BOARD_$(image_type)IMAGE_JOURNAL_SIZE)) \ + $(call add_json_str, BoardPartitionReservedSize, $(BOARD_$(image_type)IMAGE_PARTITION_RESERVED_SIZE)) \ + $(call add_json_str, BoardPartitionSize, $(BOARD_$(image_type)IMAGE_PARTITION_SIZE)) \ + $(call add_json_str, BoardSquashfsBlockSize, $(BOARD_$(image_type)IMAGE_SQUASHFS_BLOCK_SIZE)) \ + $(call add_json_str, BoardSquashfsCompressor, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR)) \ + $(call add_json_str, BoardSquashfsCompressorOpt, $(BOARD_$(image_type)IMAGE_SQUASHFS_COMPRESSOR_OPT)) \ + $(call add_json_str, BoardSquashfsDisable4kAlign, $(BOARD_$(image_type)IMAGE_SQUASHFS_DISABLE_4K_ALIGN)) \ + $(call add_json_str, BoardAvbKeyPath, $(BOARD_AVB_$(image_type)_KEY_PATH)) \ + $(call add_json_str, BoardAvbAlgorithm, $(BOARD_AVB_$(image_type)_ALGORITHM)) \ + $(call add_json_str, BoardAvbRollbackIndex, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX)) \ + $(call add_json_str, BoardAvbRollbackIndexLocation, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX_LOCATION)) \ + $(call add_json_str, BoardAvbAddHashtreeFooterArgs, $(BOARD_AVB_$(image_type)_ADD_HASHTREE_FOOTER_ARGS)) \ + $(call add_json_str, ProductBaseFsPath, $(PRODUCT_$(image_type)_BASE_FS_PATH)) \ + $(call add_json_str, ProductHeadroom, $(PRODUCT_$(image_type)_HEADROOM)) \ + $(call add_json_str, ProductVerityPartition, $(PRODUCT_$(image_type)_VERITY_PARTITION)) \ + $(call end_json_map) \ + ) + $(call end_json_map) + + $(call add_json_bool, TargetUserimagesUseExt2, $(filter true,$(TARGET_USERIMAGES_USE_EXT2))) + $(call add_json_bool, TargetUserimagesUseExt3, $(filter true,$(TARGET_USERIMAGES_USE_EXT3))) + $(call add_json_bool, TargetUserimagesUseExt4, $(filter true,$(TARGET_USERIMAGES_USE_EXT4))) + + $(call add_json_bool, TargetUserimagesSparseExtDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_EXT_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseErofsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_EROFS_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseSquashfsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_SQUASHFS_DISABLED))) + $(call add_json_bool, TargetUserimagesSparseF2fsDisabled, $(filter true,$(TARGET_USERIMAGES_SPARSE_F2FS_DISABLED))) + + $(call add_json_str, BoardErofsCompressor, $(BOARD_EROFS_COMPRESSOR)) + $(call add_json_str, BoardErofsCompressorHints, $(BOARD_EROFS_COMPRESS_HINTS)) + $(call add_json_str, BoardErofsPclusterSize, $(BOARD_EROFS_PCLUSTER_SIZE)) + $(call add_json_str, BoardErofsShareDupBlocks, $(BOARD_EROFS_SHARE_DUP_BLOCKS)) + $(call add_json_str, BoardErofsUseLegacyCompression, $(BOARD_EROFS_USE_LEGACY_COMPRESSION)) + $(call add_json_str, BoardExt4ShareDupBlocks, $(BOARD_EXT4_SHARE_DUP_BLOCKS)) + $(call add_json_str, BoardFlashLogicalBlockSize, $(BOARD_FLASH_LOGICAL_BLOCK_SIZE)) + $(call add_json_str, BoardFlashEraseBlockSize, $(BOARD_FLASH_ERASE_BLOCK_SIZE)) + $(call add_json_bool, BuildingVbmetaImage, $(BUILDING_VBMETA_IMAGE)) + + # boot image stuff + $(call add_json_bool, BuildingRamdiskImage, $(filter true,$(BUILDING_RAMDISK_IMAGE))) + $(call add_json_bool, ProductBuildBootImage, $(filter true,$(PRODUCT_BUILD_BOOT_IMAGE))) + $(call add_json_str, ProductBuildVendorBootImage, $(PRODUCT_BUILD_VENDOR_BOOT_IMAGE)) + $(call add_json_bool, ProductBuildInitBootImage, $(filter true,$(PRODUCT_BUILD_INIT_BOOT_IMAGE))) + $(call add_json_bool, BoardUsesRecoveryAsBoot, $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT))) + $(call add_json_str, BoardPrebuiltBootimage, $(BOARD_PREBUILT_BOOT_IMAGE)) + $(call add_json_str, BoardPrebuiltInitBootimage, $(BOARD_PREBUILT_INIT_BOOT_IMAGE)) + $(call add_json_str, BoardBootimagePartitionSize, $(BOARD_BOOTIMAGE_PARTITION_SIZE)) + $(call add_json_str, BoardVendorBootimagePartitionSize, $(BOARD_VENDOR_BOOTIMAGE_PARTITION_SIZE)) + $(call add_json_str, BoardInitBootimagePartitionSize, $(BOARD_INIT_BOOT_IMAGE_PARTITION_SIZE)) + $(call add_json_str, BoardBootHeaderVersion, $(BOARD_BOOT_HEADER_VERSION)) + $(call add_json_str, TargetKernelPath, $(TARGET_KERNEL_PATH)) + $(call add_json_bool, BoardUsesGenericKernelImage, $(BOARD_USES_GENERIC_KERNEL_IMAGE)) + $(call add_json_str, BootSecurityPatch, $(BOOT_SECURITY_PATCH)) + $(call add_json_str, InitBootSecurityPatch, $(INIT_BOOT_SECURITY_PATCH)) + $(call add_json_str, VendorSecurityPatch, $(VENDOR_SECURITY_PATCH)) + $(call add_json_str, OdmSecurityPatch, $(ODM_SECURITY_PATCH)) + $(call add_json_str, SystemDlkmSecurityPatch, $(SYSTEM_DLKM_SECURITY_PATCH)) + $(call add_json_str, VendorDlkmSecurityPatch, $(VENDOR_DLKM_SECURITY_PATCH)) + $(call add_json_str, OdmDlkmSecurityPatch, $(ODM_DLKM_SECURITY_PATCH)) + $(call add_json_bool, BoardIncludeDtbInBootimg, $(BOARD_INCLUDE_DTB_IN_BOOTIMG)) + $(call add_json_list, InternalKernelCmdline, $(INTERNAL_KERNEL_CMDLINE)) + $(call add_json_list, InternalBootconfig, $(INTERNAL_BOOTCONFIG)) + $(call add_json_str, InternalBootconfigFile, $(INTERNAL_BOOTCONFIG_FILE)) + + $(call add_json_bool, BuildingSystemOtherImage, $(BUILDING_SYSTEM_OTHER_IMAGE)) + + # super image stuff + $(call add_json_bool, ProductUseDynamicPartitions, $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITIONS))) + $(call add_json_bool, ProductRetrofitDynamicPartitions, $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS))) + $(call add_json_bool, ProductBuildSuperPartition, $(filter true,$(PRODUCT_BUILD_SUPER_PARTITION))) + $(call add_json_str, BoardSuperPartitionSize, $(BOARD_SUPER_PARTITION_SIZE)) + $(call add_json_str, BoardSuperPartitionMetadataDevice, $(BOARD_SUPER_PARTITION_METADATA_DEVICE)) + $(call add_json_list, BoardSuperPartitionBlockDevices, $(BOARD_SUPER_PARTITION_BLOCK_DEVICES)) + $(call add_json_map, BoardSuperPartitionGroups) + $(foreach group, $(BOARD_SUPER_PARTITION_GROUPS), \ + $(call add_json_map, $(group)) \ + $(call add_json_str, GroupSize, $(BOARD_$(call to-upper,$(group))_SIZE)) \ + $(if $(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \ + $(call add_json_list, PartitionList, $(BOARD_$(call to-upper,$(group))_PARTITION_LIST))) \ + $(call end_json_map)) + $(call end_json_map) + $(call add_json_bool, ProductVirtualAbOta, $(filter true,$(PRODUCT_VIRTUAL_AB_OTA))) + $(call add_json_bool, ProductVirtualAbOtaRetrofit, $(filter true,$(PRODUCT_VIRTUAL_AB_OTA_RETROFIT))) + $(call add_json_bool, ProductVirtualAbCompression, $(filter true,$(PRODUCT_VIRTUAL_AB_COMPRESSION))) + $(call add_json_str, ProductVirtualAbCompressionMethod, $(PRODUCT_VIRTUAL_AB_COMPRESSION_METHOD)) + $(call add_json_str, ProductVirtualAbCompressionFactor, $(PRODUCT_VIRTUAL_AB_COMPRESSION_FACTOR)) + $(call add_json_str, ProductVirtualAbCowVersion, $(PRODUCT_VIRTUAL_AB_COW_VERSION)) + $(call add_json_bool, AbOtaUpdater, $(filter true,$(AB_OTA_UPDATER))) + $(call add_json_list, AbOtaPartitions, $(AB_OTA_PARTITIONS)) + $(call add_json_list, AbOtaKeys, $(PRODUCT_OTA_PUBLIC_KEYS)) + $(call add_json_list, AbOtaPostInstallConfig, $(AB_OTA_POSTINSTALL_CONFIG)) + + # Avb (android verified boot) stuff + $(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE))) + $(call add_json_str, BoardAvbAlgorithm, $(BOARD_AVB_ALGORITHM)) + $(call add_json_str, BoardAvbKeyPath, $(BOARD_AVB_KEY_PATH)) + $(call add_json_str, BoardAvbRollbackIndex, $(BOARD_AVB_ROLLBACK_INDEX)) + $(call add_json_map, ChainedVbmetaPartitions) + $(foreach partition,system vendor $(BOARD_AVB_VBMETA_CUSTOM_PARTITIONS),\ + $(call add_json_map, $(partition)) \ + $(call add_json_list,Partitions,$(BOARD_AVB_VBMETA_$(call to-upper,$(partition)))) \ + $(call add_json_str,Key,$(BOARD_AVB_VBMETA_$(call to-upper,$(partition))_KEY_PATH)) \ + $(call add_json_str,Algorithm,$(BOARD_AVB_VBMETA_$(call to-upper,$(partition))_ALGORITHM)) \ + $(call add_json_str,RollbackIndex,$(BOARD_AVB_VBMETA_$(call to-upper,$(partition))_ROLLBACK_INDEX)) \ + $(call add_json_str,RollbackIndexLocation,$(BOARD_AVB_VBMETA_$(call to-upper,$(partition))_ROLLBACK_INDEX_LOCATION)) \ + $(call end_json_map)) + $(call end_json_map) + + $(call add_json_bool, ProductUseDynamicPartitionSize, $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITION_SIZE))) + $(call add_json_bool, CopyImagesForTargetFilesZip, $(filter true,$(COPY_IMAGES_FOR_TARGET_FILES_ZIP))) + + $(call add_json_list, ProductPackages, $(PRODUCT_PACKAGES)) + $(call add_json_list, ProductPackagesDebug, $(PRODUCT_PACKAGES_DEBUG)) + + # Used to generate /vendor/linker.config.pb + $(call add_json_list, VendorLinkerConfigSrcs, $(PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS)) + $(call add_json_list, ProductLinkerConfigSrcs, $(PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS)) + + # Used to generate _dlkm partitions + $(call add_json_bool, BuildingSystemDlkmImage, $(BUILDING_SYSTEM_DLKM_IMAGE)) + $(call add_json_list, SystemKernelModules, $(BOARD_SYSTEM_KERNEL_MODULES)) + $(call add_json_str, SystemKernelBlocklistFile, $(BOARD_SYSTEM_KERNEL_MODULES_BLOCKLIST_FILE)) + $(call add_json_list, SystemKernelLoadModules, $(BOARD_SYSTEM_KERNEL_MODULES_LOAD)) + $(call add_json_bool, BuildingVendorDlkmImage, $(BUILDING_VENDOR_DLKM_IMAGE)) + $(call add_json_list, VendorKernelModules, $(BOARD_VENDOR_KERNEL_MODULES)) + $(call add_json_str, VendorKernelBlocklistFile, $(BOARD_VENDOR_KERNEL_MODULES_BLOCKLIST_FILE)) + $(call add_json_bool, BuildingOdmDlkmImage, $(BUILDING_ODM_DLKM_IMAGE)) + $(call add_json_list, OdmKernelModules, $(BOARD_ODM_KERNEL_MODULES)) + $(call add_json_str, OdmKernelBlocklistFile, $(BOARD_ODM_KERNEL_MODULES_BLOCKLIST_FILE)) + $(call add_json_list, VendorRamdiskKernelModules, $(BOARD_VENDOR_RAMDISK_KERNEL_MODULES)) + $(call add_json_str, VendorRamdiskKernelBlocklistFile, $(BOARD_VENDOR_RAMDISK_KERNEL_MODULES_BLOCKLIST_FILE)) + $(call add_json_list, VendorRamdiskKernelLoadModules, $(BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD)) + $(call add_json_str, VendorRamdiskKernelOptionsFile, $(BOARD_VENDOR_RAMDISK_KERNEL_MODULES_OPTIONS_FILE)) + + # Used to generate /vendor/build.prop + $(call add_json_list, BoardInfoFiles, $(if $(TARGET_BOARD_INFO_FILES),$(TARGET_BOARD_INFO_FILES),$(firstword $(TARGET_BOARD_INFO_FILE) $(wildcard $(TARGET_DEVICE_DIR)/board-info.txt)))) + $(call add_json_str, BootLoaderBoardName, $(TARGET_BOOTLOADER_BOARD_NAME)) + + $(call add_json_list, ProductCopyFiles, $(PRODUCT_COPY_FILES)) + + # Used to generate fsv meta + $(call add_json_bool, ProductFsverityGenerateMetadata, $(PRODUCT_FSVERITY_GENERATE_METADATA)) + + # Used to generate recovery partition + $(call add_json_str, TargetScreenDensity, $(TARGET_SCREEN_DENSITY)) + + # Used to generate /recovery/root/build.prop + $(call add_json_map, PrivateRecoveryUiProperties) + $(call add_json_str, animation_fps, $(TARGET_RECOVERY_UI_ANIMATION_FPS)) + $(call add_json_str, margin_height, $(TARGET_RECOVERY_UI_MARGIN_HEIGHT)) + $(call add_json_str, margin_width, $(TARGET_RECOVERY_UI_MARGIN_WIDTH)) + $(call add_json_str, menu_unusable_rows, $(TARGET_RECOVERY_UI_MENU_UNUSABLE_ROWS)) + $(call add_json_str, progress_bar_baseline, $(TARGET_RECOVERY_UI_PROGRESS_BAR_BASELINE)) + $(call add_json_str, touch_low_threshold, $(TARGET_RECOVERY_UI_TOUCH_LOW_THRESHOLD)) + $(call add_json_str, touch_high_threshold, $(TARGET_RECOVERY_UI_TOUCH_HIGH_THRESHOLD)) + $(call add_json_str, vr_stereo_offset, $(TARGET_RECOVERY_UI_VR_STEREO_OFFSET)) + $(call add_json_str, brightness_file, $(TARGET_RECOVERY_UI_BRIGHTNESS_FILE)) + $(call add_json_str, max_brightness_file, $(TARGET_RECOVERY_UI_MAX_BRIGHTNESS_FILE)) + $(call add_json_str, brightness_normal_percent, $(TARGET_RECOVERY_UI_BRIGHTNESS_NORMAL)) + $(call add_json_str, brightness_dimmed_percent, $(TARGET_RECOVERY_UI_BRIGHTNESS_DIMMED)) + $(call end_json_map) + + $(call add_json_str, PrebuiltBootloader, $(BOARD_PREBUILT_BOOTLOADER)) + + # Used to generate userdata partition + $(call add_json_str, ProductFsCasefold, $(PRODUCT_FS_CASEFOLD)) + $(call add_json_str, ProductQuotaProjid, $(PRODUCT_QUOTA_PROJID)) + $(call add_json_str, ProductFsCompression, $(PRODUCT_FS_COMPRESSION)) + + $(call add_json_str, ReleaseToolsExtensionDir, $(firstword $(TARGET_RELEASETOOLS_EXTENSIONS) $($(TARGET_DEVICE_DIR)/../common))) + +$(call end_json_map) + +# For converting vintf_data +$(call add_json_list, DeviceMatrixFile, $(DEVICE_MATRIX_FILE)) +$(call add_json_list, ProductManifestFiles, $(PRODUCT_MANIFEST_FILES)) +$(call add_json_list, SystemManifestFile, $(DEVICE_FRAMEWORK_MANIFEST_FILE)) +SYSTEM_EXT_HWSERVICE_FILES := +ifeq ($(PRODUCT_HIDL_ENABLED),true) + ifneq ($(filter hwservicemanager,$(PRODUCT_PACKAGES)),) + SYSTEM_EXT_HWSERVICE_FILES += system/hwservicemanager/hwservicemanager_no_max.xml + else + $(error If PRODUCT_HIDL_ENABLED is set, hwservicemanager must be added to PRODUCT_PACKAGES explicitly) + endif +else + ifneq ($(filter hwservicemanager,$(PRODUCT_PACKAGES)),) + SYSTEM_EXT_HWSERVICE_FILES += system/hwservicemanager/hwservicemanager.xml + else ifneq ($(filter hwservicemanager,$(PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34)),) + SYSTEM_EXT_HWSERVICE_FILES += system/hwservicemanager/hwservicemanager.xml + endif +endif +$(call add_json_list, SystemExtManifestFiles, $(SYSTEM_EXT_MANIFEST_FILES) $(SYSTEM_EXT_HWSERVICE_FILES)) +$(call add_json_list, DeviceManifestFiles, $(DEVICE_MANIFEST_FILE)) +$(call add_json_list, OdmManifestFiles, $(ODM_MANIFEST_FILES)) + $(call json_end) $(file >$(SOONG_VARIABLES).tmp,$(json_contents)) diff --git a/core/soong_extra_config.mk b/core/soong_extra_config.mk index 00b5c0fd63..8eee50ae00 100644 --- a/core/soong_extra_config.mk +++ b/core/soong_extra_config.mk @@ -43,6 +43,7 @@ $(call add_json_list, PRODUCT_VENDOR_PROPERTIES, $(call collapse-prop-pa $(call add_json_list, PRODUCT_PRODUCT_PROPERTIES, $(call collapse-prop-pairs,PRODUCT_PRODUCT_PROPERTIES)) $(call add_json_list, PRODUCT_ODM_PROPERTIES, $(call collapse-prop-pairs,PRODUCT_ODM_PROPERTIES)) $(call add_json_list, PRODUCT_PROPERTY_OVERRIDES, $(call collapse-prop-pairs,PRODUCT_PROPERTY_OVERRIDES)) +$(call add_json_list, PRODUCT_DEFAULT_PROPERTY_OVERRIDES, $(call collapse-prop-pairs,PRODUCT_DEFAULT_PROPERTY_OVERRIDES)) $(call add_json_str, BootloaderBoardName, $(TARGET_BOOTLOADER_BOARD_NAME)) @@ -79,7 +80,7 @@ $(call add_json_bool, PropertySplitEnabled, $(filter true,$(BOARD_PROPERTY_OVERR $(call add_json_str, ScreenDensity, $(TARGET_SCREEN_DENSITY)) -$(call add_json_bool, UsesVulkan, $(filter true,$(TARGET_USES_VULKAN))) +$(call add_json_str, UsesVulkan, $(TARGET_USES_VULKAN)) $(call add_json_bool, ZygoteForce64, $(filter true,$(ZYGOTE_FORCE_64))) diff --git a/core/sysprop.mk b/core/sysprop.mk index dc6f2c4ac6..9a9f509333 100644 --- a/core/sysprop.mk +++ b/core/sysprop.mk @@ -122,11 +122,19 @@ $(2): $(POST_PROCESS_PROPS) $(INTERNAL_BUILD_ID_MAKEFILE) $(3) $(6) $(BUILT_KERN ifneq ($(strip $(7)), true) $(hide) $$(call generate-common-build-props,$(call to-lower,$(strip $(1))),$$@) endif + # Make and Soong use different intermediate files to build vendor/build.prop. + # Although the sysprop contents are same, the absolute paths of android-info.prop are different. + # Print the filename for the intermediate files (files in OUT_DIR). + # This helps with validating mk->soong migration of android partitions. $(hide) $(foreach file,$(strip $(3)),\ if [ -f "$(file)" ]; then\ echo "" >> $$@;\ echo "####################################" >> $$@;\ - echo "# from $(file)" >> $$@;\ + $(if $(filter $(OUT_DIR)/%,$(file)), \ + echo "# from $(notdir $(file))" >> $$@;\ + ,\ + echo "# from $(file)" >> $$@;\ + )\ echo "####################################" >> $$@;\ cat $(file) >> $$@;\ fi;) @@ -152,61 +160,6 @@ endif $(call declare-1p-target,$(2)) endef -# ----------------------------------------------------------------- -# Define fingerprint, thumbprint, and version tags for the current build -# -# BUILD_VERSION_TAGS is a comma-separated list of tags chosen by the device -# implementer that further distinguishes the build. It's basically defined -# by the device implementer. Here, we are adding a mandatory tag that -# identifies the signing config of the build. -BUILD_VERSION_TAGS := $(BUILD_VERSION_TAGS) -ifeq ($(TARGET_BUILD_TYPE),debug) - BUILD_VERSION_TAGS += debug -endif -# The "test-keys" tag marks builds signed with the old test keys, -# which are available in the SDK. "dev-keys" marks builds signed with -# non-default dev keys (usually private keys from a vendor directory). -# Both of these tags will be removed and replaced with "release-keys" -# when the target-files is signed in a post-build step. -ifeq ($(DEFAULT_SYSTEM_DEV_CERTIFICATE),build/make/target/product/security/testkey) -BUILD_KEYS := test-keys -else -BUILD_KEYS := dev-keys -endif -BUILD_VERSION_TAGS += $(BUILD_KEYS) -BUILD_VERSION_TAGS := $(subst $(space),$(comma),$(sort $(BUILD_VERSION_TAGS))) - -# BUILD_FINGERPRINT is used used to uniquely identify the combined build and -# product; used by the OTA server. -ifeq (,$(strip $(BUILD_FINGERPRINT))) - BUILD_FINGERPRINT := $(PRODUCT_BRAND)/$(TARGET_PRODUCT)/$(TARGET_DEVICE):$(PLATFORM_VERSION)/$(BUILD_ID)/$(BUILD_NUMBER_FROM_FILE):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) -endif - -BUILD_FINGERPRINT_FILE := $(PRODUCT_OUT)/build_fingerprint.txt -ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_FINGERPRINT) >$(BUILD_FINGERPRINT_FILE) && grep " " $(BUILD_FINGERPRINT_FILE))) - $(error BUILD_FINGERPRINT cannot contain spaces: "$(file <$(BUILD_FINGERPRINT_FILE))") -endif -BUILD_FINGERPRINT_FROM_FILE := $$(cat $(BUILD_FINGERPRINT_FILE)) -# unset it for safety. -BUILD_FINGERPRINT := - -# BUILD_THUMBPRINT is used to uniquely identify the system build; used by the -# OTA server. This purposefully excludes any product-specific variables. -ifeq (,$(strip $(BUILD_THUMBPRINT))) - BUILD_THUMBPRINT := $(PLATFORM_VERSION)/$(BUILD_ID)/$(BUILD_NUMBER_FROM_FILE):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS) -endif - -BUILD_THUMBPRINT_FILE := $(PRODUCT_OUT)/build_thumbprint.txt -ifeq ($(strip $(HAS_BUILD_NUMBER)),true) -$(BUILD_THUMBPRINT_FILE): $(BUILD_NUMBER_FILE) -endif -ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_THUMBPRINT) >$(BUILD_THUMBPRINT_FILE) && grep " " $(BUILD_THUMBPRINT_FILE))) - $(error BUILD_THUMBPRINT cannot contain spaces: "$(file <$(BUILD_THUMBPRINT_FILE))") -endif -# unset it for safety. -BUILD_THUMBPRINT_FILE := -BUILD_THUMBPRINT := - KNOWN_OEM_THUMBPRINT_PROPERTIES := \ ro.product.brand \ ro.product.name \ @@ -230,7 +183,7 @@ _prop_files_ := $(if $(TARGET_VENDOR_PROP),\ $(TARGET_VENDOR_PROP),\ $(wildcard $(TARGET_DEVICE_DIR)/vendor.prop)) -android_info_prop := $(call intermediates-dir-for,ETC,android_info_prop)/android_info.prop +android_info_prop := $(call intermediates-dir-for,ETC,android_info_prop)/android-info.prop $(android_info_prop): $(INSTALLED_ANDROID_INFO_TXT_TARGET) cat $< | grep 'require version-' | sed -e 's/require version-/ro.build.expect./g' > $@ @@ -281,51 +234,17 @@ INSTALLED_ODM_BUILD_PROP_TARGET := $(TARGET_OUT_ODM)/etc/build.prop # ---------------------------------------------------------------- # vendor_dlkm/etc/build.prop -# - -INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET := $(TARGET_OUT_VENDOR_DLKM)/etc/build.prop -$(eval $(call build-properties,\ - vendor_dlkm,\ - $(INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty))) - -$(eval $(call declare-1p-target,$(INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET))) - -# ---------------------------------------------------------------- # odm_dlkm/etc/build.prop -# - -INSTALLED_ODM_DLKM_BUILD_PROP_TARGET := $(TARGET_OUT_ODM_DLKM)/etc/build.prop -$(eval $(call build-properties,\ - odm_dlkm,\ - $(INSTALLED_ODM_DLKM_BUILD_PROP_TARGET),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty))) - -$(eval $(call declare-1p-target,$(INSTALLED_ODM_DLKM_BUILD_PROP_TARGET))) - -# ---------------------------------------------------------------- # system_dlkm/build.prop -# +# These are built by Soong. See build/soong/Android.bp +INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET := $(TARGET_OUT_VENDOR_DLKM)/etc/build.prop +INSTALLED_ODM_DLKM_BUILD_PROP_TARGET := $(TARGET_OUT_ODM_DLKM)/etc/build.prop INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET := $(TARGET_OUT_SYSTEM_DLKM)/etc/build.prop -$(eval $(call build-properties,\ - system_dlkm,\ - $(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty))) - -$(eval $(call declare-1p-target,$(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET))) +ALL_DEFAULT_INSTALLED_MODULES += \ + $(INSTALLED_VENDOR_DLKM_BUILD_PROP_TARGET) \ + $(INSTALLED_ODM_DLKM_BUILD_PROP_TARGET) \ + $(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET) \ # ----------------------------------------------------------------- # system_ext/etc/build.prop @@ -335,22 +254,12 @@ $(eval $(call declare-1p-target,$(INSTALLED_SYSTEM_DLKM_BUILD_PROP_TARGET))) INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET := $(TARGET_OUT_SYSTEM_EXT)/etc/build.prop -# ---------------------------------------------------------------- -# ramdisk/boot/etc/build.prop -# - RAMDISK_BUILD_PROP_REL_PATH := system/etc/ramdisk/build.prop +ifeq (true,$(BOARD_USES_RECOVERY_AS_BOOT)) +INSTALLED_RAMDISK_BUILD_PROP_TARGET := $(TARGET_RECOVERY_ROOT_OUT)/first_stage_ramdisk/$(RAMDISK_BUILD_PROP_REL_PATH) +else INSTALLED_RAMDISK_BUILD_PROP_TARGET := $(TARGET_RAMDISK_OUT)/$(RAMDISK_BUILD_PROP_REL_PATH) -$(eval $(call build-properties,\ - bootimage,\ - $(INSTALLED_RAMDISK_BUILD_PROP_TARGET),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty),\ - $(empty))) - -$(eval $(call declare-1p-target,$(INSTALLED_RAMDISK_BUILD_PROP_TARGET))) +endif ALL_INSTALLED_BUILD_PROP_FILES := \ $(INSTALLED_BUILD_PROP_TARGET) \ diff --git a/core/sysprop_config.mk b/core/sysprop_config.mk index 69066117a3..199150347c 100644 --- a/core/sysprop_config.mk +++ b/core/sysprop_config.mk @@ -91,8 +91,12 @@ endif # Build system set BOARD_API_LEVEL to show the api level of the vendor API surface. # This must not be altered outside of build system. ifdef BOARD_API_LEVEL -ADDITIONAL_VENDOR_PROPERTIES += \ - ro.board.api_level=$(BOARD_API_LEVEL) + ADDITIONAL_VENDOR_PROPERTIES += \ + ro.board.api_level?=$(BOARD_API_LEVEL) + ifdef BOARD_API_LEVEL_PROP_OVERRIDE + ADDITIONAL_VENDOR_PROPERTIES += \ + ro.board.api_level=$(BOARD_API_LEVEL_PROP_OVERRIDE) + endif endif # RELEASE_BOARD_API_LEVEL_FROZEN is true when the vendor API surface is frozen. ifdef RELEASE_BOARD_API_LEVEL_FROZEN diff --git a/core/tasks/autorepro.mk b/core/tasks/autorepro.mk new file mode 100644 index 0000000000..2f81f9bf85 --- /dev/null +++ b/core/tasks/autorepro.mk @@ -0,0 +1,39 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifneq ($(wildcard test/sts/README-autorepro.md),) +test_suite_name := autorepro +test_suite_tradefed := sts-tradefed +test_suite_readme := test/sts/README-autorepro.md +autorepro_zip := $(HOST_OUT)/$(test_suite_name)/autorepro.zip + +include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk + +autorepro_plugin_skel := $(call intermediates-dir-for,ETC,autorepro-plugin-skel.zip)/autorepro-plugin-skel.zip + +$(autorepro_zip): AUTOREPRO_ZIP := $(compatibility_zip) +$(autorepro_zip): AUTOREPRO_PLUGIN_SKEL := $(autorepro_plugin_skel) +$(autorepro_zip): $(MERGE_ZIPS) $(ZIP2ZIP) $(compatibility_zip) $(autorepro_plugin_skel) + rm -f $@ $(AUTOREPRO_ZIP)_filtered + $(ZIP2ZIP) -i $(AUTOREPRO_ZIP) -o $(AUTOREPRO_ZIP)_filtered \ + -x android-autorepro/tools/sts-tradefed-tests.jar \ + 'android-autorepro/tools/*:autorepro/src/main/resources/sts-tradefed-tools/' + $(MERGE_ZIPS) $@ $(AUTOREPRO_ZIP)_filtered $(AUTOREPRO_PLUGIN_SKEL) + rm -f $(AUTOREPRO_ZIP)_filtered + +.PHONY: autorepro +autorepro: $(autorepro_zip) +$(call dist-for-goals, autorepro, $(autorepro_zip)) + +endif diff --git a/target/product/gsi/Android.mk b/core/tasks/check-abi-dump-list.mk index 36897fef8e..81d549e46f 100644 --- a/target/product/gsi/Android.mk +++ b/core/tasks/check-abi-dump-list.mk @@ -1,4 +1,16 @@ -LOCAL_PATH:= $(call my-dir) +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ##################################################################### # Check the generate list against the latest list stored in the @@ -109,60 +121,3 @@ $(check-abi-dump-list-timestamp): $(if $(added_vndk_abi_dumps)$(added_platform_abi_dumps),exit 1) $(hide) mkdir -p $(dir $@) $(hide) touch $@ - -##################################################################### -# VNDK package and snapshot. - -include $(CLEAR_VARS) - -LOCAL_MODULE := vndk_apex_snapshot_package -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach vndk_ver,$(PRODUCT_EXTRA_VNDK_VERSIONS),com.android.vndk.v$(vndk_ver)) -include $(BUILD_PHONY_PACKAGE) - -##################################################################### -# Define Phony module to install LLNDK modules which are installed in -# the system image -include $(CLEAR_VARS) -LOCAL_MODULE := llndk_in_system -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE - -# Filter LLNDK libs moved to APEX to avoid pulling them into /system/LIB -LOCAL_REQUIRED_MODULES := \ - $(filter-out $(LLNDK_MOVED_TO_APEX_LIBRARIES),$(LLNDK_LIBRARIES)) \ - llndk.libraries.txt - - -include $(BUILD_PHONY_PACKAGE) - -##################################################################### -# init.gsi.rc, GSI-specific init script. - -include $(CLEAR_VARS) -LOCAL_MODULE := init.gsi.rc -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_SRC_FILES := $(LOCAL_MODULE) -LOCAL_MODULE_CLASS := ETC -LOCAL_SYSTEM_EXT_MODULE := true -LOCAL_MODULE_RELATIVE_PATH := init - -include $(BUILD_PREBUILT) - - -include $(CLEAR_VARS) -LOCAL_MODULE := init.vndk-nodef.rc -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_SRC_FILES := $(LOCAL_MODULE) -LOCAL_MODULE_CLASS := ETC -LOCAL_SYSTEM_EXT_MODULE := true -LOCAL_MODULE_RELATIVE_PATH := gsi - -include $(BUILD_PREBUILT) diff --git a/core/tasks/cts.mk b/core/tasks/cts.mk index 294cb577e2..1a55bf480b 100644 --- a/core/tasks/cts.mk +++ b/core/tasks/cts.mk @@ -97,12 +97,28 @@ cts_api_map_exe := $(HOST_OUT_EXECUTABLES)/cts-api-map coverage_out := $(HOST_OUT)/cts-api-coverage api_map_out := $(HOST_OUT)/cts-api-map -cts_jar_files := $(api_map_out)/api_map_files.txt +cts_jar_files := $(api_map_out)/cts_jar_files.txt +cts_v_host_jar_files := $(api_map_out)/cts_v_host_jar_files.txt +cts_all_jar_files := $(api_map_out)/cts_all_jar_files.txt + $(cts_jar_files): PRIVATE_API_MAP_FILES := $(sort $(COMPATIBILITY.cts.API_MAP_FILES)) $(cts_jar_files): mkdir -p $(dir $@) echo $(PRIVATE_API_MAP_FILES) > $@ +$(cts_v_host_jar_files): PRIVATE_API_MAP_FILES := $(sort $(COMPATIBILITY.cts-v-host.API_MAP_FILES)) +$(cts_v_host_jar_files): $(SOONG_ANDROID_CTS_VERIFIER_APP_LIST) + mkdir -p $(dir $@) + cp $< $@ + echo $(PRIVATE_API_MAP_FILES) >> $@ + +$(cts_all_jar_files): PRIVATE_API_MAP_FILES := $(sort $(COMPATIBILITY.cts.API_MAP_FILES) \ + $(COMPATIBILITY.cts-v-host.API_MAP_FILES)) +$(cts_all_jar_files): $(SOONG_ANDROID_CTS_VERIFIER_APP_LIST) + mkdir -p $(dir $@) + cp $< $@ + echo $(PRIVATE_API_MAP_FILES) >> $@ + api_xml_description := $(TARGET_OUT_COMMON_INTERMEDIATES)/api.xml napi_text_description := cts/tools/cts-api-coverage/etc/ndk-api.xml @@ -113,6 +129,13 @@ $(napi_xml_description) : $(napi_text_description) $(ACP) $(hide) $(ACP) $< $@ system_api_xml_description := $(TARGET_OUT_COMMON_INTERMEDIATES)/system-api.xml +module_lib_api_xml_description := $(TARGET_OUT_COMMON_INTERMEDIATES)/module-lib-api.xml +system_service_api_description := $(TARGET_OUT_COMMON_INTERMEDIATES)/system-server-api.xml + +combined_api_xml_description := $(api_xml_description) \ + $(system_api_xml_description) \ + $(module_lib_api_xml_description) \ + $(system_service_api_description) cts-test-coverage-report := $(coverage_out)/test-coverage.html cts-system-api-coverage-report := $(coverage_out)/system-api-coverage.html @@ -124,13 +147,14 @@ cts-combined-xml-coverage-report := $(coverage_out)/combined-coverage.xml cts_api_coverage_dependencies := $(cts_api_coverage_exe) $(dexdeps_exe) $(api_xml_description) $(napi_xml_description) cts_system_api_coverage_dependencies := $(cts_api_coverage_exe) $(dexdeps_exe) $(system_api_xml_description) -cts-api-xml-api-map-report := $(api_map_out)/api-map.xml -cts-api-html-api-map-report := $(api_map_out)/api-map.html -cts-system-api-xml-api-map-report := $(api_map_out)/system-api-map.xml -cts-system-api-html-api-map-report := $(api_map_out)/system-api-map.html +cts-api-map-xml-report := $(api_map_out)/cts-api-map.xml +cts-v-host-api-map-xml-report := $(api_map_out)/cts-v-host-api-map.xml +cts-combined-api-map-xml-report := $(api_map_out)/cts-combined-api-map.xml +cts-combined-api-map-html-report := $(api_map_out)/cts-combined-api-map.html -cts_system_api_map_dependencies := $(cts_api_map_exe) $(system_api_xml_description) $(cts_jar_files) -cts_api_map_dependencies := $(cts_api_map_exe) $(api_xml_description) $(cts_jar_files) +cts_api_map_dependencies := $(cts_api_map_exe) $(combined_api_xml_description) $(cts_jar_files) +cts_v_host_api_map_dependencies := $(cts_api_map_exe) $(combined_api_xml_description) $(cts_v_host_jar_files) +cts_combined_api_map_dependencies := $(cts_api_map_exe) $(combined_api_xml_description) $(cts_all_jar_files) android_cts_zip := $(HOST_OUT)/cts/android-cts.zip cts_verifier_apk := $(call intermediates-dir-for,APPS,CtsVerifier)/package.apk @@ -210,45 +234,42 @@ cts-combined-xml-coverage : $(cts-combined-xml-coverage-report) .PHONY: cts-coverage-report-all cts-api-coverage cts-coverage-report-all: cts-test-coverage cts-verifier-coverage cts-combined-coverage cts-combined-xml-coverage -$(cts-system-api-xml-api-map-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) -$(cts-system-api-xml-api-map-report): PRIVATE_API_XML_DESC := $(system_api_xml_description) -$(cts-system-api-xml-api-map-report): PRIVATE_JAR_FILES := $(cts_jar_files) -$(cts-system-api-xml-api-map-report) : $(android_cts_zip) $(cts_system_api_map_dependencies) | $(ACP) - $(call generate-api-map-report-cts,"CTS System API MAP Report - XML",\ +$(cts-api-map-xml-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) +$(cts-api-map-xml-report): PRIVATE_API_XML_DESC := $(combined_api_xml_description) +$(cts-api-map-xml-report): PRIVATE_JAR_FILES := $(cts_jar_files) +$(cts-api-map-xml-report) : $(android_cts_zip) $(cts_api_map_dependencies) | $(ACP) + $(call generate-api-map-report-cts,"CTS API MAP Report - XML",\ $(PRIVATE_JAR_FILES),xml) -$(cts-system-api-html-api-map-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) -$(cts-system-api-html-api-map-report): PRIVATE_API_XML_DESC := $(system_api_xml_description) -$(cts-system-api-html-api-map-report): PRIVATE_JAR_FILES := $(cts_jar_files) -$(cts-system-api-html-api-map-report) : $(android_cts_zip) $(cts_system_api_map_dependencies) | $(ACP) - $(call generate-api-map-report-cts,"CTS System API MAP Report - HTML",\ - $(PRIVATE_JAR_FILES),html) +$(cts-v-host-api-map-xml-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) +$(cts-v-host-api-map-xml-report): PRIVATE_API_XML_DESC := $(combined_api_xml_description) +$(cts-v-host-api-map-xml-report): PRIVATE_JAR_FILES := $(cts_v_host_jar_files) +$(cts-v-host-api-map-xml-report) : $(verifier_zip) $(cts_v_host_api_map_dependencies) | $(ACP) + $(call generate-api-map-report-cts,"CTS-V-HOST API MAP Report - XML",\ + $(PRIVATE_JAR_FILES),xml) -$(cts-api-xml-api-map-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) -$(cts-api-xml-api-map-report): PRIVATE_API_XML_DESC := $(api_xml_description) -$(cts-api-xml-api-map-report): PRIVATE_JAR_FILES := $(cts_jar_files) -$(cts-api-xml-api-map-report) : $(android_cts_zip) $(cts_api_map_dependencies) | $(ACP) - $(call generate-api-map-report-cts,"CTS API MAP Report - XML",\ +$(cts-combined-api-map-xml-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) +$(cts-combined-api-map-xml-report): PRIVATE_API_XML_DESC := $(combined_api_xml_description) +$(cts-combined-api-map-xml-report): PRIVATE_JAR_FILES := $(cts_all_jar_files) +$(cts-combined-api-map-xml-report) : $(verifier_zip) $(android_cts_zip) $(cts_combined_api_map_dependencies) | $(ACP) + $(call generate-api-map-report-cts,"CTS Combined API MAP Report - XML",\ $(PRIVATE_JAR_FILES),xml) -$(cts-api-html-api-map-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) -$(cts-api-html-api-map-report): PRIVATE_API_XML_DESC := $(api_xml_description) -$(cts-api-html-api-map-report): PRIVATE_JAR_FILES := $(cts_jar_files) -$(cts-api-html-api-map-report) : $(android_cts_zip) $(cts_api_map_dependencies) | $(ACP) - $(call generate-api-map-report-cts,"CTS API MAP Report - HTML",\ +$(cts-combined-api-map-html-report): PRIVATE_CTS_API_MAP_EXE := $(cts_api_map_exe) +$(cts-combined-api-map-html-report): PRIVATE_API_XML_DESC := $(combined_api_xml_description) +$(cts-combined-api-map-html-report): PRIVATE_JAR_FILES := $(cts_all_jar_files) +$(cts-combined-api-map-html-report) : $(verifier_zip) $(android_cts_zip) $(cts_combined_api_map_dependencies) | $(ACP) + $(call generate-api-map-report-cts,"CTS Combined API MAP Report - HTML",\ $(PRIVATE_JAR_FILES),html) -.PHONY: cts-system-api-xml-api-map -cts-system-api-xml-api-map : $(cts-system-api-xml-api-map-report) - -.PHONY: cts-system-api-html-api-map -cts-system-api-html-api-map : $(cts-system-api-html-api-map-report) +.PHONY: cts-api-map-xml +cts-api-map-xml : $(cts-api-map-xml-report) -.PHONY: cts-api-xml-api-map -cts-api-xml-api-map : $(cts-api-xml-api-map-report) +.PHONY: cts-v-host-api-map-xml +cts-v-host-api-map-xml: $(cts-v-host-api-map-xml-report) -.PHONY: cts-api-html-api-map -cts-api-html-api-map : $(cts-api-html-api-map-report) +.PHONY: cts-combined-api-map-xml +cts-combined-api-map-xml : $(cts-combined-api-map-xml-report) .PHONY: cts-api-map-all @@ -268,15 +289,13 @@ ALL_TARGETS.$(cts-combined-coverage-report).META_LIC:=$(module_license_metadata) ALL_TARGETS.$(cts-combined-xml-coverage-report).META_LIC:=$(module_license_metadata) # Put the test api map report in the dist dir if "cts-api-map-all" is among the build goals. -$(call dist-for-goals, cts-api-map-all, $(cts-system-api-xml-api-map-report):cts-system-api-xml-api-map-report.xml) -$(call dist-for-goals, cts-api-map-all, $(cts-system-api-html-api-map-report):cts-system-api-html-api-map-report.html) -$(call dist-for-goals, cts-api-map-all, $(cts-api-xml-api-map-report):cts-api-xml-api-map-report.xml) -$(call dist-for-goals, cts-api-map-all, $(cts-api-html-api-map-report):cts-api-html-api-map-report.html) +$(call dist-for-goals, cts-api-map-all, $(cts-combined-api-map-xml-report):cts-api-map-report.xml) +$(call dist-for-goals, cts-api-map-all, $(cts-combined-api-map-html-report):cts-api-map-report.html) -ALL_TARGETS.$(cts-system-api-xml-api-map-report).META_LIC:=$(module_license_metadata) -ALL_TARGETS.$(cts-system-api-html-api-map-report).META_LIC:=$(module_license_metadata) -ALL_TARGETS.$(cts-api-xml-api-map-report).META_LIC:=$(module_license_metadata) -ALL_TARGETS.$(cts-api-html-api-map-report).META_LIC:=$(module_license_metadata) +ALL_TARGETS.$(cts-api-map-xml-report).META_LIC:=$(module_license_metadata) +ALL_TARGETS.$(cts-v-host-api-map-xml-report).META_LIC:=$(module_license_metadata) +ALL_TARGETS.$(cts-combined-api-map-xml-report).META_LIC:=$(module_license_metadata) +ALL_TARGETS.$(cts-combined-api-map-html-report).META_LIC:=$(module_license_metadata) # Arguments; # 1 - Name of the report printed out on the screen @@ -294,7 +313,7 @@ endef # 3 - Format of the report define generate-api-map-report-cts $(hide) mkdir -p $(dir $@) - $(hide) $(PRIVATE_CTS_API_MAP_EXE) -j 8 -a $(PRIVATE_API_XML_DESC) -i $(2) -f $(3) -o $@ + $(hide) $(PRIVATE_CTS_API_MAP_EXE) -j 8 -a $(shell echo "$(PRIVATE_API_XML_DESC)" | tr ' ' ',') -i $(2) -f $(3) -o $@ @ echo $(1): file://$$(cd $(dir $@); pwd)/$(notdir $@) endef @@ -302,20 +321,22 @@ endef cts_api_coverage_dependencies := cts_system_api_coverage_dependencies := cts_api_map_dependencies := -cts_system_api_map_dependencies := +cts_v_host_api_map_dependencies := +cts_combined_api_map_dependencies := cts-combined-coverage-report := cts-combined-xml-coverage-report := cts-verifier-coverage-report := cts-test-coverage-report := cts-system-api-coverage-report := cts-system-api-xml-coverage-report := -cts-api-xml-api-map-report := -cts-api-html-api-map-report := -cts-system-api-xml-api-map-report := -cts-system-api-html-api-map-report := +cts-api-map-xml-report := +cts-v-host-api-map-xml-report := +cts-combined-api-map-xml-report := +cts-combined-api-map-html-report := api_xml_description := api_text_description := system_api_xml_description := +combined_api_xml_description := napi_xml_description := napi_text_description := coverage_out := diff --git a/core/tasks/device-tests.mk b/core/tasks/device-tests.mk index 5850c4ed73..062ac23bbe 100644 --- a/core/tasks/device-tests.mk +++ b/core/tasks/device-tests.mk @@ -21,39 +21,33 @@ device-tests-list-zip := $(PRODUCT_OUT)/device-tests_list.zip # Create an artifact to include all test config files in device-tests. device-tests-configs-zip := $(PRODUCT_OUT)/device-tests_configs.zip my_host_shared_lib_for_device_tests := $(call copy-many-files,$(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES)) -device_tests_host_shared_libs_zip := $(PRODUCT_OUT)/device-tests_host-shared-libs.zip -$(device-tests-zip) : .KATI_IMPLICIT_OUTPUTS := $(device-tests-list-zip) $(device-tests-configs-zip) $(device_tests_host_shared_libs_zip) +$(device-tests-zip) : .KATI_IMPLICIT_OUTPUTS := $(device-tests-list-zip) $(device-tests-configs-zip) $(device-tests-zip) : PRIVATE_device_tests_list := $(PRODUCT_OUT)/device-tests_list $(device-tests-zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_device_tests) -$(device-tests-zip) : PRIVATE_device_host_shared_libs_zip := $(device_tests_host_shared_libs_zip) $(device-tests-zip) : $(COMPATIBILITY.device-tests.FILES) $(COMPATIBILITY.device-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES) $(my_host_shared_lib_for_device_tests) $(SOONG_ZIP) - rm -f $@-shared-libs.list echo $(sort $(COMPATIBILITY.device-tests.FILES) $(COMPATIBILITY.device-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES)) | tr " " "\n" > $@.list grep $(HOST_OUT_TESTCASES) $@.list > $@-host.list || true grep -e .*\\.config$$ $@-host.list > $@-host-test-configs.list || true $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ echo $$shared_lib >> $@-host.list; \ - echo $$shared_lib >> $@-shared-libs.list; \ done - grep $(HOST_OUT_TESTCASES) $@-shared-libs.list > $@-host-shared-libs.list || true grep $(TARGET_OUT_TESTCASES) $@.list > $@-target.list || true grep -e .*\\.config$$ $@-target.list > $@-target-test-configs.list || true $(hide) $(SOONG_ZIP) -d -o $@ -P host -C $(HOST_OUT) -l $@-host.list -P target -C $(PRODUCT_OUT) -l $@-target.list -sha256 $(hide) $(SOONG_ZIP) -d -o $(device-tests-configs-zip) \ -P host -C $(HOST_OUT) -l $@-host-test-configs.list \ -P target -C $(PRODUCT_OUT) -l $@-target-test-configs.list - $(SOONG_ZIP) -d -o $(PRIVATE_device_host_shared_libs_zip) \ - -P host -C $(HOST_OUT) -l $@-host-shared-libs.list rm -f $(PRIVATE_device_tests_list) $(hide) grep -e .*\\.config$$ $@-host.list | sed s%$(HOST_OUT)%host%g > $(PRIVATE_device_tests_list) $(hide) grep -e .*\\.config$$ $@-target.list | sed s%$(PRODUCT_OUT)%target%g >> $(PRIVATE_device_tests_list) $(hide) $(SOONG_ZIP) -d -o $(device-tests-list-zip) -C $(dir $@) -f $(PRIVATE_device_tests_list) rm -f $@.list $@-host.list $@-target.list $@-host-test-configs.list $@-target-test-configs.list \ - $@-shared-libs.list $@-host-shared-libs.list $(PRIVATE_device_tests_list) + $(PRIVATE_device_tests_list) device-tests: $(device-tests-zip) -$(call dist-for-goals, device-tests, $(device-tests-zip) $(device-tests-list-zip) $(device-tests-configs-zip) $(device_tests_host_shared_libs_zip)) + +$(call dist-for-goals, device-tests, $(device-tests-zip) $(device-tests-list-zip) $(device-tests-configs-zip)) $(call declare-1p-container,$(device-tests-zip),) $(call declare-container-license-deps,$(device-tests-zip),$(COMPATIBILITY.device-tests.FILES) $(my_host_shared_lib_for_device_tests),$(PRODUCT_OUT)/:/) diff --git a/core/tasks/dts.mk b/core/tasks/dts.mk new file mode 100644 index 0000000000..8f090828d9 --- /dev/null +++ b/core/tasks/dts.mk @@ -0,0 +1,28 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Desktop test suite +ifneq ($(wildcard test/dts/tools/dts-tradefed/README),) +test_suite_name := dts +test_suite_tradefed := dts-tradefed +test_suite_readme := test/dts/tools/dts-tradefed/README +test_suite_tools := $(HOST_OUT_JAVA_LIBRARIES)/ats_console_deploy.jar \ + $(HOST_OUT_JAVA_LIBRARIES)/ats_olc_server_local_mode_deploy.jar + +include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk + +.PHONY: dts +dts: $(compatibility_zip) $(compatibility_tests_list_zip) +$(call dist-for-goals, dts, $(compatibility_zip) $(compatibility_tests_list_zip)) +endif diff --git a/core/tasks/general-tests-shared-libs.mk b/core/tasks/general-tests-shared-libs.mk deleted file mode 100644 index 240514073e..0000000000 --- a/core/tasks/general-tests-shared-libs.mk +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2024 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.PHONY: general-tests-shared-libs - -intermediates_dir := $(call intermediates-dir-for,PACKAGING,general-tests-shared-libs) - -general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip - -# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES, -# to avoid warning about overriding commands. -my_host_shared_lib_for_general_tests := \ - $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ - $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m))) -my_general_tests_shared_lib_files := \ - $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ - $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)) - -my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files)) - -$(general_tests_shared_libs_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) -$(general_tests_shared_libs_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests) -$(general_tests_shared_libs_zip) : PRIVATE_general_host_shared_libs_zip := $(general_tests_shared_libs_zip) -$(general_tests_shared_libs_zip) : $(my_host_shared_lib_for_general_tests) $(SOONG_ZIP) - rm -rf $(PRIVATE_INTERMEDIATES_DIR) - mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools - $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ - echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ - done - grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true - $(SOONG_ZIP) -d -o $(PRIVATE_general_host_shared_libs_zip) \ - -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list - -general-tests-shared-libs: $(general_tests_shared_libs_zip) -$(call dist-for-goals, general-tests-shared-libs, $(general_tests_shared_libs_zip)) - -$(call declare-1p-container,$(general_tests_shared_libs_zip),) -$(call declare-container-license-deps,$(general_tests_shared_libs_zip),$(my_host_shared_lib_for_general_tests),$(PRODUCT_OUT)/:/) - -intermediates_dir := -general_tests_shared_libs_zip := diff --git a/core/tasks/general-tests.mk b/core/tasks/general-tests.mk index d6fc0722ef..dcfcfad62f 100644 --- a/core/tasks/general-tests.mk +++ b/core/tasks/general-tests.mk @@ -27,32 +27,61 @@ general_tests_list_zip := $(PRODUCT_OUT)/general-tests_list.zip # Create an artifact to include all test config files in general-tests. general_tests_configs_zip := $(PRODUCT_OUT)/general-tests_configs.zip -# Copy kernel test modules to testcases directories -include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk -ltp_copy_pairs := \ - $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_host_out)) -copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs)) +# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES, +# to avoid warning about overriding commands. +my_host_shared_lib_for_general_tests := \ + $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m))) +my_general_tests_shared_lib_files := \ + $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\ + $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)) -# PHONY target to be used to build and test `vts_ltp_tests` without building full vts -.PHONY: vts_kernel_ltp_tests -vts_kernel_ltp_tests: $(copy_ltp_tests) +my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files)) -general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip +my_host_shared_lib_symlinks := \ + $(filter $(COMPATIBILITY.host-unit-tests.SYMLINKS),\ + $(COMPATIBILITY.general-tests.SYMLINKS)) + +my_general_tests_symlinks := \ + $(filter-out $(COMPATIBILITY.camera-hal-tests.SYMLINKS),\ + $(filter-out $(COMPATIBILITY.host-unit-tests.SYMLINKS),\ + $(COMPATIBILITY.general-tests.SYMLINKS))) + +my_symlinks_for_general_tests := $(foreach f,$(my_general_tests_symlinks),\ + $(strip $(eval _cmf_tuple := $(subst :, ,$(f))) \ + $(eval _cmf_dep := $(word 1,$(_cmf_tuple))) \ + $(eval _cmf_src := $(word 2,$(_cmf_tuple))) \ + $(eval _cmf_dest := $(word 3,$(_cmf_tuple))) \ + $(call symlink-file,$(_cmf_dep),$(_cmf_src),$(_cmf_dest)) \ + $(_cmf_dest))) + +# In this one directly take the overlap into the zip since we can't rewrite rules +my_symlinks_for_general_tests += $(foreach f,$(my_host_shared_lib_symlinks),\ + $(strip $(eval _cmf_tuple := $(subst :, ,$(f))) \ + $(eval _cmf_dep := $(word 1,$(_cmf_tuple))) \ + $(eval _cmf_src := $(word 2,$(_cmf_tuple))) \ + $(eval _cmf_dest := $(word 3,$(_cmf_tuple))) \ + $(_cmf_dest))) -$(general_tests_zip) : $(general_tests_shared_libs_zip) -$(general_tests_zip) : $(copy_ltp_tests) -$(general_tests_zip) : PRIVATE_KERNEL_LTP_HOST_OUT := $(kernel_ltp_host_out) $(general_tests_zip) : PRIVATE_general_tests_list_zip := $(general_tests_list_zip) $(general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(general_tests_list_zip) $(general_tests_configs_zip) $(general_tests_zip) : PRIVATE_TOOLS := $(general_tests_tools) $(general_tests_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir) +$(general_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests) +$(general_tests_zip) : PRIVATE_SYMLINKS := $(my_symlinks_for_general_tests) $(general_tests_zip) : PRIVATE_general_tests_configs_zip := $(general_tests_configs_zip) -$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES) $(general_tests_tools) $(SOONG_ZIP) +$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(my_host_shared_lib_for_general_tests) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES) $(general_tests_tools) $(my_symlinks_for_general_tests) $(SOONG_ZIP) rm -rf $(PRIVATE_INTERMEDIATES_DIR) rm -f $@ $(PRIVATE_general_tests_list_zip) mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools echo $(sort $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list - find $(PRIVATE_KERNEL_LTP_HOST_OUT) >> $(PRIVATE_INTERMEDIATES_DIR)/list + for symlink in $(PRIVATE_SYMLINKS); do \ + echo $$symlink >> $(PRIVATE_INTERMEDIATES_DIR)/list; \ + done + $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \ + echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \ + done + grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true @@ -62,6 +91,7 @@ $(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.gene -P host -C $(PRIVATE_INTERMEDIATES_DIR) -D $(PRIVATE_INTERMEDIATES_DIR)/tools \ -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host.list \ -P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target.list \ + -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list \ -sha256 $(SOONG_ZIP) -d -o $(PRIVATE_general_tests_configs_zip) \ -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list \ @@ -82,3 +112,8 @@ general_tests_zip := general_tests_list_zip := general_tests_configs_zip := general_tests_shared_libs_zip := +my_host_shared_lib_for_general_tests := +my_symlinks_for_general_tests := +my_general_tests_shared_lib_files := +my_general_tests_symlinks := +my_host_shared_lib_symlinks := diff --git a/core/tasks/mcts.mk b/core/tasks/mcts.mk index 09a41915eb..02e916a38d 100644 --- a/core/tasks/mcts.mk +++ b/core/tasks/mcts.mk @@ -15,7 +15,8 @@ ifneq ($(wildcard test/mts/README.md),) mcts_test_suites := -mcts_test_suites += mcts +mcts_all_test_suites := +mcts_all_test_suites += mcts $(foreach module, $(mts_modules), $(eval mcts_test_suites += mcts-$(module))) @@ -29,4 +30,14 @@ $(foreach suite, $(mcts_test_suites), \ $(eval $(call dist-for-goals, $(suite), $(compatibility_zip))) \ ) +$(foreach suite, $(mcts_all_test_suites), \ + $(eval test_suite_name := $(suite)) \ + $(eval test_suite_tradefed := mcts-tradefed) \ + $(eval test_suite_readme := test/mts/README.md) \ + $(eval include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk) \ + $(eval .PHONY: $(suite)) \ + $(eval $(suite): $(compatibility_zip)) \ + $(eval $(call dist-for-goals, $(suite), $(compatibility_zip))) \ +) + endif diff --git a/core/tasks/module-info.mk b/core/tasks/module-info.mk index 0ca27d8222..dd01f9667c 100644 --- a/core/tasks/module-info.mk +++ b/core/tasks/module-info.mk @@ -50,6 +50,8 @@ $(MODULE_INFO_JSON): $(SOONG_MODULE_INFO) $(call write-optional-json-list, "host_dependencies", $(sort $(ALL_MODULES.$(m).HOST_REQUIRED_FROM_TARGET))) \ $(call write-optional-json-list, "target_dependencies", $(sort $(ALL_MODULES.$(m).TARGET_REQUIRED_FROM_HOST))) \ $(call write-optional-json-bool, "test_module_config_base", $(ALL_MODULES.$(m).TEST_MODULE_CONFIG_BASE)) \ + $(call write-optional-json-bool, "make", $(if $(ALL_MODULES.$(m).IS_SOONG_MODULE),,true)) \ + $(call write-optional-json-bool, "make_generated_module_info", true) \ '}')'\n}\n' >> $@.tmp $(PRIVATE_MERGE_JSON_OBJECTS) -o $@ $(PRIVATE_SOONG_MODULE_INFO) $@.tmp rm $@.tmp diff --git a/core/tasks/tools/vts-kernel-tests.mk b/core/tasks/prebuilt_tradefed.mk index e727dc1f55..96c57d5633 100644 --- a/core/tasks/tools/vts-kernel-tests.mk +++ b/core/tasks/prebuilt_tradefed.mk @@ -1,4 +1,4 @@ -# Copyright (C) 2022 The Android Open Source Project +# Copyright (C) 2020 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. --include external/ltp/android/ltp_package_list.mk +ifeq (,$(wildcard tools/tradefederation/core)) +.PHONY: tradefed-core +tradefed-core: tradefed atest_tradefed.sh +.PHONY: tradefed-all +tradefed-all: tradefed atest_tradefed.sh -include $(BUILD_SYSTEM)/tasks/tools/vts_package_utils.mk - -# Copy kernel test modules to testcases directories -kernel_ltp_host_out := $(HOST_OUT_TESTCASES)/vts_kernel_ltp_tests -kernel_ltp_vts_out := $(HOST_OUT)/$(test_suite_name)/android-$(test_suite_name)/testcases/vts_kernel_ltp_tests -kernel_ltp_modules := \ - ltp \ - $(ltp_packages) +$(call dist-for-goals, tradefed, $(HOST_OUT)/etc/tradefed.zip) +endif diff --git a/core/tasks/sts-sdk.mk b/core/tasks/sts-sdk.mk deleted file mode 100644 index 4abbc29c5e..0000000000 --- a/core/tasks/sts-sdk.mk +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2022 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ifneq ($(wildcard test/sts/README-sts-sdk.md),) -test_suite_name := sts-sdk -test_suite_tradefed := sts-tradefed -test_suite_readme := test/sts/README-sts-sdk.md -sts_sdk_zip := $(HOST_OUT)/$(test_suite_name)/sts-sdk.zip - -include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk - -sts_sdk_plugin_skel := $(call intermediates-dir-for,ETC,sts-sdk-plugin-skel.zip)/sts-sdk-plugin-skel.zip - -$(sts_sdk_zip): STS_SDK_ZIP := $(compatibility_zip) -$(sts_sdk_zip): STS_SDK_PLUGIN_SKEL := $(sts_sdk_plugin_skel) -$(sts_sdk_zip): $(MERGE_ZIPS) $(ZIP2ZIP) $(compatibility_zip) $(sts_sdk_plugin_skel) - rm -f $@ $(STS_SDK_ZIP)_filtered - $(ZIP2ZIP) -i $(STS_SDK_ZIP) -o $(STS_SDK_ZIP)_filtered \ - -x android-sts-sdk/tools/sts-tradefed-tests.jar \ - 'android-sts-sdk/tools/*:sts-sdk/src/main/resources/sts-tradefed-tools/' - $(MERGE_ZIPS) $@ $(STS_SDK_ZIP)_filtered $(STS_SDK_PLUGIN_SKEL) - rm -f $(STS_SDK_ZIP)_filtered - -.PHONY: sts-sdk -sts-sdk: $(sts_sdk_zip) -$(call dist-for-goals, sts-sdk, $(sts_sdk_zip)) - -endif diff --git a/core/tasks/test_mapping.mk b/core/tasks/test_mapping.mk deleted file mode 100644 index eb2a585880..0000000000 --- a/core/tasks/test_mapping.mk +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create an artifact to include TEST_MAPPING files in source tree. Also include -# a file (out/disabled-presubmit-tests) containing the tests that should be -# skipped in presubmit check. - -.PHONY: test_mapping - -intermediates := $(call intermediates-dir-for,PACKAGING,test_mapping) -test_mappings_zip := $(intermediates)/test_mappings.zip -test_mapping_list := $(OUT_DIR)/.module_paths/TEST_MAPPING.list -$(test_mappings_zip) : PRIVATE_all_disabled_presubmit_tests := $(ALL_DISABLED_PRESUBMIT_TESTS) -$(test_mappings_zip) : PRIVATE_test_mapping_list := $(test_mapping_list) - -$(test_mappings_zip) : .KATI_DEPFILE := $(test_mappings_zip).d -$(test_mappings_zip) : $(test_mapping_list) $(SOONG_ZIP) - @echo "Building artifact to include TEST_MAPPING files and tests to skip in presubmit check." - rm -rf $@ $(dir $@)/disabled-presubmit-tests - echo $(sort $(PRIVATE_all_disabled_presubmit_tests)) | tr " " "\n" > $(dir $@)/disabled-presubmit-tests - $(SOONG_ZIP) -o $@ -C . -l $(PRIVATE_test_mapping_list) -C $(dir $@) -f $(dir $@)/disabled-presubmit-tests - echo "$@ : " $$(cat $(PRIVATE_test_mapping_list)) > $@.d - rm -f $(dir $@)/disabled-presubmit-tests - -test_mapping : $(test_mappings_zip) - -$(call dist-for-goals, dist_files test_mapping,$(test_mappings_zip)) - -$(call declare-1p-target,$(test_mappings_zip),) diff --git a/core/tasks/tools/compatibility.mk b/core/tasks/tools/compatibility.mk index 9189c2d58e..f205cea156 100644 --- a/core/tasks/tools/compatibility.mk +++ b/core/tasks/tools/compatibility.mk @@ -27,7 +27,6 @@ # compatibility_zip: the path to the output zip file. special_mts_test_suites := -special_mts_test_suites += mcts special_mts_test_suites += $(mts_modules) ifneq ($(filter $(special_mts_test_suites),$(patsubst mcts-%,%,$(test_suite_name))),) test_suite_subdir := android-mts diff --git a/core/tasks/tools/vts_package_utils.mk b/core/tasks/tools/vts_package_utils.mk deleted file mode 100644 index 1a819f2172..0000000000 --- a/core/tasks/tools/vts_package_utils.mk +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# $(1): List of target native files to copy. -# $(2): Copy destination directory. -# Evaluates to a list of ":"-separated pairs src:dst. -define target-native-copy-pairs -$(foreach m,$(1),\ - $(eval _built_files := $(strip $(ALL_MODULES.$(m).BUILT_INSTALLED)\ - $(ALL_MODULES.$(m)$(TARGET_2ND_ARCH_MODULE_SUFFIX).BUILT_INSTALLED)))\ - $(foreach i, $(sort $(_built_files)),\ - $(eval bui_ins := $(subst :,$(space),$(i)))\ - $(eval ins := $(word 2,$(bui_ins)))\ - $(if $(filter $(TARGET_OUT_ROOT)/%,$(ins)),\ - $(eval bui := $(word 1,$(bui_ins)))\ - $(eval my_copy_dest := $(patsubst data/%,DATA/%,\ - $(patsubst system/%,DATA/%,\ - $(patsubst $(PRODUCT_OUT)/%,%,$(ins)))))\ - $(call declare-copy-target-license-metadata,$(2)/$(my_copy_dest),$(bui))\ - $(bui):$(2)/$(my_copy_dest)))) -endef diff --git a/core/tasks/tradefed-tests-list.mk b/core/tasks/tradefed-tests-list.mk index 47c360de52..e437f894dc 100644 --- a/core/tasks/tradefed-tests-list.mk +++ b/core/tasks/tradefed-tests-list.mk @@ -18,11 +18,19 @@ COMPATIBILITY.tradefed_tests_dir := \ $(COMPATIBILITY.tradefed_tests_dir) \ tools/tradefederation/core/res/config \ - tools/tradefederation/core/javatests/res/config + tools/tradefederation/core/javatests/res/config \ + vendor/google_tradefederation/contrib/res/config \ + vendor/google_tradefederation/core/res/config \ + vendor/google_tradefederation/core/javatests/res/config \ + vendor/google_tradefederation/core/prod_tests/res/config tradefed_tests := $(foreach dir, $(COMPATIBILITY.tradefed_tests_dir), \ - $(eval tradefed_tests += $(shell find $(dir) -type f -name "*.xml"))) + $(if $(wildcard $(dir)/*), \ + $(eval tradefed_tests += $(shell find $(dir) -type f -name "*.xml")) \ + ) \ +) + tradefed_tests_list_intermediates := $(call intermediates-dir-for,PACKAGING,tradefed_tests_list,HOST,COMMON) tradefed_tests_list_zip := $(tradefed_tests_list_intermediates)/tradefed-tests_list.zip all_tests := diff --git a/core/tasks/vts-core-tests.mk b/core/tasks/vts-core-tests.mk index 1eeb0789ec..11bb932f7d 100644 --- a/core/tasks/vts-core-tests.mk +++ b/core/tasks/vts-core-tests.mk @@ -16,15 +16,6 @@ test_suite_name := vts test_suite_tradefed := vts-tradefed test_suite_readme := test/vts/tools/vts-core-tradefed/README -include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk - -ltp_copy_pairs := \ - $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_vts_out)) - -copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs)) - -test_suite_extra_deps := $(copy_ltp_tests) - include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk .PHONY: vts diff --git a/core/version_util.mk b/core/version_util.mk index 0e346347bb..2bf41ecceb 100644 --- a/core/version_util.mk +++ b/core/version_util.mk @@ -23,6 +23,7 @@ # PLATFORM_DISPLAY_VERSION # PLATFORM_SDK_VERSION # PLATFORM_SDK_EXTENSION_VERSION +# PLATFORM_BASE_SDK_EXTENSION_VERSION # PLATFORM_VERSION_CODENAME # DEFAULT_APP_TARGET_SDK # BUILD_ID @@ -67,8 +68,16 @@ endif PLATFORM_SDK_EXTENSION_VERSION := $(RELEASE_PLATFORM_SDK_EXTENSION_VERSION) .KATI_READONLY := PLATFORM_SDK_EXTENSION_VERSION -# This is the sdk extension version that PLATFORM_SDK_VERSION ships with. -PLATFORM_BASE_SDK_EXTENSION_VERSION := $(PLATFORM_SDK_EXTENSION_VERSION) +ifdef PLATFORM_BASE_SDK_EXTENSION_VERSION + $(error Do not set PLATFORM_BASE_SDK_EXTENSION_VERSION directly. Use RELEASE_PLATFORM_BASE_SDK_EXTENSION_VERSION. value: $(PLATFORM_BASE_SDK_EXTENSION_VERSION)) +endif +ifdef RELEASE_PLATFORM_BASE_SDK_EXTENSION_VERSION + # This is the sdk extension version that PLATFORM_SDK_VERSION ships with. + PLATFORM_BASE_SDK_EXTENSION_VERSION := $(RELEASE_PLATFORM_BASE_SDK_EXTENSION_VERSION) +else + # Fallback to PLATFORM_SDK_EXTENSION_VERSION if RELEASE_PLATFORM_BASE_SDK_EXTENSION_VERSION is undefined. + PLATFORM_BASE_SDK_EXTENSION_VERSION := $(PLATFORM_SDK_EXTENSION_VERSION) +endif .KATI_READONLY := PLATFORM_BASE_SDK_EXTENSION_VERSION ifdef PLATFORM_VERSION_CODENAME diff --git a/envsetup.sh b/envsetup.sh index 06dadd3f38..c04031186e 100644 --- a/envsetup.sh +++ b/envsetup.sh @@ -362,7 +362,6 @@ function addcompletions() packages/modules/adb/adb.bash system/core/fastboot/fastboot.bash tools/asuite/asuite.sh - prebuilts/bazel/common/bazel-complete.bash ) # Completion can be disabled selectively to allow users to use non-standard completion. # e.g. @@ -439,67 +438,6 @@ function print_lunch_menu() echo } -function lunch() -{ - local answer - - if [[ $# -gt 1 ]]; then - echo "usage: lunch [target]" >&2 - return 1 - fi - - local used_lunch_menu=0 - - if [ "$1" ]; then - answer=$1 - else - print_lunch_menu - echo "Which would you like? [aosp_cf_x86_64_phone-trunk_staging-eng]" - echo -n "Pick from common choices above (e.g. 13) or specify your own (e.g. aosp_barbet-trunk_staging-eng): " - read answer - used_lunch_menu=1 - fi - - local selection= - - if [ -z "$answer" ] - then - selection=aosp_cf_x86_64_phone-trunk_staging-eng - elif (echo -n $answer | grep -q -e "^[0-9][0-9]*$") - then - local choices=($(TARGET_BUILD_APPS= TARGET_PRODUCT= TARGET_RELEASE= TARGET_BUILD_VARIANT= _get_build_var_cached COMMON_LUNCH_CHOICES 2>/dev/null)) - if [ $answer -le ${#choices[@]} ] - then - # array in zsh starts from 1 instead of 0. - if [ -n "$ZSH_VERSION" ] - then - selection=${choices[$(($answer))]} - else - selection=${choices[$(($answer-1))]} - fi - fi - else - selection=$answer - fi - - export TARGET_BUILD_APPS= - - # This must be <product>-<release>-<variant> - local product release variant - # Split string on the '-' character. - IFS="-" read -r product release variant <<< "$selection" - - if [[ -z "$product" ]] || [[ -z "$release" ]] || [[ -z "$variant" ]] - then - echo - echo "Invalid lunch combo: $selection" - echo "Valid combos must be of the form <product>-<release>-<variant>" - return 1 - fi - - _lunch_meat $product $release $variant -} - function _lunch_meat() { local product=$1 @@ -582,13 +520,13 @@ function _lunch_usage() echo "Note that the previous interactive menu and list of hard-coded" echo "list of curated targets has been removed. If you would like the" echo "list of products, release configs for a particular product, or" - echo "variants, run list_products, list_release_configs, list_variants" + echo "variants, run list_products list_releases or list_variants" echo "respectively." echo ) 1>&2 } -function lunch2() +function lunch() { if [[ $# -eq 1 && $1 = "--help" ]]; then _lunch_usage @@ -1079,10 +1017,7 @@ function source_vendorsetup() { done done - if [[ "${PWD}" == /google/cog/* ]]; then - f="build/make/cogsetup.sh" - echo "including $f"; . "$T/$f" - fi + setup_cog_env_if_needed } function showcommands() { diff --git a/packaging/distdir.mk b/packaging/distdir.mk index 153ecf65b1..97ed95a569 100644 --- a/packaging/distdir.mk +++ b/packaging/distdir.mk @@ -45,5 +45,3 @@ ifeq ($(DIST),true) endif copy-one-dist-file := -DIST_GOAL_OUTPUT_PAIRS := -DIST_SRC_DST_PAIRS := diff --git a/packaging/main_soong_only.mk b/packaging/main_soong_only.mk new file mode 100644 index 0000000000..f29e5f6f0d --- /dev/null +++ b/packaging/main_soong_only.mk @@ -0,0 +1,60 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifndef KATI +$(error Only Kati is supported.) +endif + +$(info [1/4] initializing packaging system ...) + +.KATI_READONLY := KATI_PACKAGE_MK_DIR + +include build/make/common/core.mk +include build/make/common/strings.mk + +# Define well-known goals and their dependency graph that they've +# traditionally had in make builds. Also it's important to define +# droid first so that it's built by default. + +.PHONY: droid +droid: droid_targets + +.PHONY: droid_targets +droid_targets: droidcore dist_files + +.PHONY: dist_files +dist_files: + +.PHONY: droidcore +droidcore: droidcore-unbundled + +.PHONY: droidcore-unbundled +droidcore-unbundled: + +$(info [2/4] including distdir.mk ...) + +include build/make/packaging/distdir.mk + +$(info [3/4] defining phony modules ...) + +include $(OUT_DIR)/soong/soong_phony_targets.mk + +goals := $(sort $(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS),$(call word-colon,1,$(pair)))) +$(foreach goal,$(goals), \ + $(eval .PHONY: $$(goal)) \ + $(eval $$(goal):) \ + $(if $(call streq,$(DIST),true),\ + $(eval $$(goal): _dist_$$(goal)))) + +$(info [4/4] writing packaging rules ...) diff --git a/shell_utils.sh b/shell_utils.sh index 86f3f49f50..3124db598c 100644 --- a/shell_utils.sh +++ b/shell_utils.sh @@ -63,6 +63,70 @@ function require_lunch } fi +# This function sets up the build environment to be appropriate for Cog. +function setup_cog_env_if_needed() { + local top=$(gettop) + + # return early if not in a cog workspace + if [[ ! "$top" =~ ^/google/cog ]]; then + return 0 + fi + + setup_cog_symlink + + export ANDROID_BUILD_ENVIRONMENT_CONFIG="googler-cog" + + # Running repo command within Cog workspaces is not supported, so override + # it with this function. If the user is running repo within a Cog workspace, + # we'll fail with an error, otherwise, we run the original repo command with + # the given args. + if ! ORIG_REPO_PATH=`which repo`; then + return 0 + fi + function repo { + if [[ "${PWD}" == /google/cog/* ]]; then + echo -e "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces." + kill -INT $$ # exits the script without exiting the user's shell + fi + ${ORIG_REPO_PATH} "$@" + } +} + +# creates a symlink for the out/ dir when inside a cog workspace. +function setup_cog_symlink() { + local out_dir=$(getoutdir) + local top=$(gettop) + + # return early if out dir is already a symlink + if [[ -L "$out_dir" ]]; then + return 0 + fi + + # return early if out dir is not in the workspace + if [[ ! "$out_dir" =~ ^$top/ ]]; then + return 0 + fi + + local link_destination="${HOME}/.cog/android-build-out" + + # remove existing out/ dir if it exists + if [[ -d "$out_dir" ]]; then + echo "Detected existing out/ directory in the Cog workspace which is not supported. Repairing workspace by removing it and creating the symlink to ~/.cog/android-build-out" + if ! rm -rf "$out_dir"; then + echo "Failed to remove existing out/ directory: $out_dir" >&2 + kill -INT $$ # exits the script without exiting the user's shell + fi + fi + + # create symlink + echo "Creating symlink: $out_dir -> $link_destination" + mkdir -p ${link_destination} + if ! ln -s "$link_destination" "$out_dir"; then + echo "Failed to create cog symlink: $out_dir -> $link_destination" >&2 + kill -INT $$ # exits the script without exiting the user's shell + fi +} + function getoutdir { local top=$(gettop) @@ -114,11 +178,11 @@ function _wrap_build() echo -n "${color_failed}#### failed to build some targets " fi if [ $hours -gt 0 ] ; then - printf "(%02g:%02g:%02g (hh:mm:ss))" $hours $mins $secs + printf "(%02d:%02d:%02d (hh:mm:ss))" $hours $mins $secs elif [ $mins -gt 0 ] ; then - printf "(%02g:%02g (mm:ss))" $mins $secs + printf "(%02d:%02d (mm:ss))" $mins $secs elif [ $secs -gt 0 ] ; then - printf "(%s seconds)" $secs + printf "(%d seconds)" $secs fi echo " ####${color_reset}" echo @@ -150,3 +214,19 @@ function log_tool_invocation() ' SIGINT SIGTERM SIGQUIT EXIT } +# Import the build variables supplied as arguments into this shell's environment. +# For absolute variables, prefix the variable name with a '/'. For example: +# import_build_vars OUT_DIR DIST_DIR /HOST_OUT_EXECUTABLES +# Returns nonzero if the build command failed. Stderr is passed through. +function import_build_vars() +{ + require_top + local script + script=$(cd $TOP && build/soong/bin/get_build_vars "$@") + local ret=$? + if [ $ret -ne 0 ] ; then + return $ret + fi + eval "$script" + return $? +} diff --git a/target/board/BoardConfigGsiCommon.mk b/target/board/BoardConfigGsiCommon.mk index 67e31dfa5f..8a62796e51 100644 --- a/target/board/BoardConfigGsiCommon.mk +++ b/target/board/BoardConfigGsiCommon.mk @@ -69,6 +69,11 @@ BOARD_SUPER_PARTITION_SIZE := 3229614080 BOARD_SUPER_PARTITION_GROUPS := gsi_dynamic_partitions BOARD_GSI_DYNAMIC_PARTITIONS_PARTITION_LIST := system BOARD_GSI_DYNAMIC_PARTITIONS_SIZE := 3221225472 + +# Build pvmfw with GSI: b/376363989 +ifeq (true,$(PRODUCT_BUILD_PVMFW_IMAGE)) +BOARD_PVMFWIMAGE_PARTITION_SIZE := 0x00100000 +endif endif # TODO(b/123695868, b/146149698): diff --git a/target/board/Android.mk b/target/board/android-info.mk index 8133af9a7f..36be0025ad 100644 --- a/target/board/Android.mk +++ b/target/board/android-info.mk @@ -51,29 +51,6 @@ $(call declare-0p-target,$(INSTALLED_ANDROID_INFO_TXT_TARGET)) # Copy compatibility metadata to the device. -# Device Manifest -ifdef DEVICE_MANIFEST_FILE -# $(DEVICE_MANIFEST_FILE) can be a list of files -include $(CLEAR_VARS) -LOCAL_MODULE := vendor_manifest.xml -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 legacy_not_a_contribution -LOCAL_LICENSE_CONDITIONS := by_exception_only not_allowed notice -LOCAL_MODULE_STEM := manifest.xml -LOCAL_MODULE_CLASS := ETC -LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR)/etc/vintf - -GEN := $(local-generated-sources-dir)/manifest.xml -$(GEN): PRIVATE_DEVICE_MANIFEST_FILE := $(DEVICE_MANIFEST_FILE) -$(GEN): $(DEVICE_MANIFEST_FILE) $(HOST_OUT_EXECUTABLES)/assemble_vintf - BOARD_SEPOLICY_VERS=$(BOARD_SEPOLICY_VERS) \ - PRODUCT_ENFORCE_VINTF_MANIFEST=$(PRODUCT_ENFORCE_VINTF_MANIFEST) \ - $(HOST_OUT_EXECUTABLES)/assemble_vintf -o $@ \ - -i $(call normalize-path-list,$(PRIVATE_DEVICE_MANIFEST_FILE)) - -LOCAL_PREBUILT_MODULE_FILE := $(GEN) -include $(BUILD_PREBUILT) -endif - # DEVICE_MANIFEST_SKUS: a list of SKUS where DEVICE_MANIFEST_<sku>_FILES is defined. ifdef DEVICE_MANIFEST_SKUS @@ -112,30 +89,6 @@ _add_device_sku_manifest := endif # DEVICE_MANIFEST_SKUS -# ODM manifest -ifdef ODM_MANIFEST_FILES -# ODM_MANIFEST_FILES is a list of files that is combined and installed as the default ODM manifest. -include $(CLEAR_VARS) -LOCAL_MODULE := odm_manifest.xml -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 legacy_not_a_contribution -LOCAL_LICENSE_CONDITIONS := by_exception_only not_allowed notice -LOCAL_MODULE_STEM := manifest.xml -LOCAL_MODULE_CLASS := ETC -LOCAL_MODULE_RELATIVE_PATH := vintf -LOCAL_ODM_MODULE := true - -GEN := $(local-generated-sources-dir)/manifest.xml -$(GEN): PRIVATE_SRC_FILES := $(ODM_MANIFEST_FILES) -$(GEN): $(ODM_MANIFEST_FILES) $(HOST_OUT_EXECUTABLES)/assemble_vintf - # Set VINTF_IGNORE_TARGET_FCM_VERSION to true because it should only be in device manifest. - VINTF_IGNORE_TARGET_FCM_VERSION=true \ - $(HOST_OUT_EXECUTABLES)/assemble_vintf -o $@ \ - -i $(call normalize-path-list,$(PRIVATE_SRC_FILES)) - -LOCAL_PREBUILT_MODULE_FILE := $(GEN) -include $(BUILD_PREBUILT) -endif # ODM_MANIFEST_FILES - # ODM_MANIFEST_SKUS: a list of SKUS where ODM_MANIFEST_<sku>_FILES are defined. ifdef ODM_MANIFEST_SKUS diff --git a/target/board/generic_64bitonly_x86_64/device.mk b/target/board/generic_64bitonly_x86_64/device.mk index bb49057abf..5edf5e0822 100644 --- a/target/board/generic_64bitonly_x86_64/device.mk +++ b/target/board/generic_64bitonly_x86_64/device.mk @@ -17,8 +17,3 @@ ifdef NET_ETH0_STARTONBOOT PRODUCT_PROPERTY_OVERRIDES += net.eth0.startonboot=1 endif - -# Ensure we package the BIOS files too. -PRODUCT_HOST_PACKAGES += \ - bios.bin \ - vgabios-cirrus.bin \ diff --git a/target/board/generic_arm64/BoardConfig.mk b/target/board/generic_arm64/BoardConfig.mk index e2d5fb4df8..1a05549193 100644 --- a/target/board/generic_arm64/BoardConfig.mk +++ b/target/board/generic_arm64/BoardConfig.mk @@ -23,14 +23,14 @@ TARGET_2ND_ARCH := arm TARGET_2ND_CPU_ABI := armeabi-v7a TARGET_2ND_CPU_ABI2 := armeabi -ifneq ($(TARGET_BUILD_APPS)$(filter cts sdk,$(MAKECMDGOALS)),) +ifneq ($(TARGET_BUILD_APPS)$(filter sdk,$(MAKECMDGOALS)),) # DO NOT USE # DO NOT USE # # This architecture / CPU variant must NOT be used for any 64 bit # platform builds. It is the lowest common denominator required # to build an unbundled application or cts for all supported 32 and 64 bit -# platforms. +# platforms. It now recommended to use generic_arm64_plus_armv7 to achieve this. # # If you're building a 64 bit platform (and not an application) the # ARM-v8 specification allows you to assume all the features available in an @@ -66,6 +66,8 @@ include build/make/target/board/BoardConfigGsiCommon.mk BOARD_ROOT_EXTRA_SYMLINKS += /vendor/lib/dsp:/dsp BOARD_ROOT_EXTRA_SYMLINKS += /mnt/vendor/persist:/persist BOARD_ROOT_EXTRA_SYMLINKS += /vendor/firmware_mnt:/firmware +# for Android.bp +TARGET_ADD_ROOT_EXTRA_VENDOR_SYMLINKS := true # TODO(b/36764215): remove this setting when the generic system image # no longer has QCOM-specific directories under /. diff --git a/target/board/generic_arm64_plus_armv7/BoardConfig.mk b/target/board/generic_arm64_plus_armv7/BoardConfig.mk new file mode 100644 index 0000000000..2dca04f707 --- /dev/null +++ b/target/board/generic_arm64_plus_armv7/BoardConfig.mk @@ -0,0 +1,55 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# arm64 emulator specific definitions +TARGET_ARCH := arm64 +TARGET_ARCH_VARIANT := armv8-a +TARGET_CPU_VARIANT := generic +TARGET_CPU_ABI := arm64-v8a + +TARGET_2ND_ARCH := arm +TARGET_2ND_CPU_ABI := armeabi-v7a +TARGET_2ND_CPU_ABI2 := armeabi + +# DO NOT USE +# DO NOT USE +# +# This architecture / CPU variant must NOT be used for any 64 bit +# platform builds. It is the lowest common denominator required +# to build an unbundled application or cts for all supported 32 and 64 bit +# platforms. +# +# If you're building a 64 bit platform (and not an application) the +# ARM-v8 specification allows you to assume all the features available in an +# armv7-a-neon CPU. You should set the following as 2nd arch/cpu variant: +# +# TARGET_2ND_ARCH_VARIANT := armv8-a +# TARGET_2ND_CPU_VARIANT := generic +# +# DO NOT USE +# DO NOT USE +TARGET_2ND_ARCH_VARIANT := armv7-a-neon +# DO NOT USE +# DO NOT USE +TARGET_2ND_CPU_VARIANT := generic +# DO NOT USE +# DO NOT USE + +# Include 64-bit mediaserver to support 64-bit only devices +TARGET_DYNAMIC_64_32_MEDIASERVER := true +# Include 64-bit drmserver to support 64-bit only devices +TARGET_DYNAMIC_64_32_DRMSERVER := true + +include build/make/target/board/BoardConfigGsiCommon.mk diff --git a/target/board/generic_arm64_plus_armv7/README.txt b/target/board/generic_arm64_plus_armv7/README.txt new file mode 100644 index 0000000000..284bdc254c --- /dev/null +++ b/target/board/generic_arm64_plus_armv7/README.txt @@ -0,0 +1,7 @@ +The "generic_arm64_plus_armv7" product defines a non-hardware-specific arm64 +target with armv7 compatible arm32. It is used for building CTS and other +test suites for which the 32-bit binaries may be run on older devices with +armv7 CPUs. + +It is not a product "base class"; no other products inherit +from it or use it in any way. diff --git a/target/board/generic_arm64_plus_armv7/device.mk b/target/board/generic_arm64_plus_armv7/device.mk new file mode 100644 index 0000000000..a9586f3c16 --- /dev/null +++ b/target/board/generic_arm64_plus_armv7/device.mk @@ -0,0 +1,15 @@ +# +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/target/board/generic_x86/device.mk b/target/board/generic_x86/device.mk index 60f0cc33f1..27fb310c2c 100644 --- a/target/board/generic_x86/device.mk +++ b/target/board/generic_x86/device.mk @@ -17,8 +17,3 @@ ifdef NET_ETH0_STARTONBOOT PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 endif - -# Ensure we package the BIOS files too. -PRODUCT_HOST_PACKAGES += \ - bios.bin \ - vgabios-cirrus.bin \ diff --git a/target/product/AndroidProducts.mk b/target/product/AndroidProducts.mk index 07eb96db2a..5a7414e49f 100644 --- a/target/product/AndroidProducts.mk +++ b/target/product/AndroidProducts.mk @@ -36,6 +36,7 @@ ifneq ($(TARGET_BUILD_APPS),) PRODUCT_MAKEFILES := \ $(LOCAL_DIR)/aosp_arm64.mk \ $(LOCAL_DIR)/aosp_arm64_fullmte.mk \ + $(LOCAL_DIR)/aosp_arm64_plus_armv7.mk \ $(LOCAL_DIR)/aosp_arm.mk \ $(LOCAL_DIR)/aosp_riscv64.mk \ $(LOCAL_DIR)/aosp_x86_64.mk \ @@ -48,6 +49,7 @@ PRODUCT_MAKEFILES := \ $(LOCAL_DIR)/aosp_64bitonly_x86_64.mk \ $(LOCAL_DIR)/aosp_arm64.mk \ $(LOCAL_DIR)/aosp_arm64_fullmte.mk \ + $(LOCAL_DIR)/aosp_arm64_plus_armv7.mk \ $(LOCAL_DIR)/aosp_arm.mk \ $(LOCAL_DIR)/aosp_riscv64.mk \ $(LOCAL_DIR)/aosp_x86_64.mk \ diff --git a/target/product/OWNERS b/target/product/OWNERS index 48d3f2a33c..276c885280 100644 --- a/target/product/OWNERS +++ b/target/product/OWNERS @@ -8,3 +8,6 @@ per-file developer_gsi_keys.mk = file:/target/product/gsi/OWNERS per-file go_defaults.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com per-file go_defaults_512.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com per-file go_defaults_common.mk = gkaiser@google.com, kushg@google.com, rajekumar@google.com + +# Translation +per-file languages_default.mk = aapple@google.com diff --git a/target/product/aosp_arm.mk b/target/product/aosp_arm.mk index d9c362eb56..595c3dbb0f 100644 --- a/target/product/aosp_arm.mk +++ b/target/product/aosp_arm.mk @@ -60,8 +60,12 @@ ifeq (aosp_arm,$(TARGET_PRODUCT)) MODULE_BUILD_FROM_SOURCE ?= true $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) -endif +PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE := aosp_system_image +USE_SOONG_DEFINED_SYSTEM_IMAGE := true +PRODUCT_USE_SOONG_NOTICE_XML := true + +endif PRODUCT_NAME := aosp_arm PRODUCT_DEVICE := generic diff --git a/target/product/aosp_arm64.mk b/target/product/aosp_arm64.mk index 7a9325dae3..cd3de51bd8 100644 --- a/target/product/aosp_arm64.mk +++ b/target/product/aosp_arm64.mk @@ -66,8 +66,12 @@ ifeq (aosp_arm64,$(TARGET_PRODUCT)) MODULE_BUILD_FROM_SOURCE ?= true $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) -endif +PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE := aosp_system_image +USE_SOONG_DEFINED_SYSTEM_IMAGE := true +PRODUCT_USE_SOONG_NOTICE_XML := true + +endif PRODUCT_NAME := aosp_arm64 PRODUCT_DEVICE := generic_arm64 diff --git a/target/product/aosp_arm64_plus_armv7.mk b/target/product/aosp_arm64_plus_armv7.mk new file mode 100644 index 0000000000..7322629ee5 --- /dev/null +++ b/target/product/aosp_arm64_plus_armv7.mk @@ -0,0 +1,64 @@ +# +# Copyright (C) 2025 The Android Open-Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# aosp_arm64_plus_armv7 is for building CTS and other test suites with +# arm64 as the primary architecture and armv7 arm32 as the secondary +# architecture. + +# +# All components inherited here go to system image +# +$(call inherit-product, $(SRC_TARGET_DIR)/product/core_64_bit.mk) +$(call inherit-product, $(SRC_TARGET_DIR)/product/generic_system.mk) + +PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS := relaxed + +# +# All components inherited here go to system_ext image +# +$(call inherit-product, $(SRC_TARGET_DIR)/product/handheld_system_ext.mk) +$(call inherit-product, $(SRC_TARGET_DIR)/product/telephony_system_ext.mk) + +# pKVM +$(call inherit-product-if-exists, packages/modules/Virtualization/apex/product_packages.mk) + +# +# All components inherited here go to product image +# +$(call inherit-product, $(SRC_TARGET_DIR)/product/aosp_product.mk) + +# +# All components inherited here go to vendor or vendor_boot image +# +$(call inherit-product, $(SRC_TARGET_DIR)/board/generic_arm64/device.mk) +AB_OTA_UPDATER := true +AB_OTA_PARTITIONS ?= system + +# +# Special settings for GSI releasing +# +# Build modules from source if this has not been pre-configured +MODULE_BUILD_FROM_SOURCE ?= true + +$(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) + + +PRODUCT_NAME := aosp_arm64_plus_armv7 +PRODUCT_DEVICE := generic_arm64_plus_armv7 +PRODUCT_BRAND := Android +PRODUCT_MODEL := AOSP on ARM64 with ARMV7 + +PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO := true diff --git a/target/product/aosp_x86.mk b/target/product/aosp_x86.mk index c26a8bf45c..d14abc26df 100644 --- a/target/product/aosp_x86.mk +++ b/target/product/aosp_x86.mk @@ -58,8 +58,12 @@ ifeq (aosp_x86,$(TARGET_PRODUCT)) MODULE_BUILD_FROM_SOURCE ?= true $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) -endif +PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE := aosp_system_image +USE_SOONG_DEFINED_SYSTEM_IMAGE := true +PRODUCT_USE_SOONG_NOTICE_XML := true + +endif PRODUCT_NAME := aosp_x86 PRODUCT_DEVICE := generic_x86 diff --git a/target/product/aosp_x86_64.mk b/target/product/aosp_x86_64.mk index 595940d9d1..bd121e3712 100644 --- a/target/product/aosp_x86_64.mk +++ b/target/product/aosp_x86_64.mk @@ -68,8 +68,12 @@ ifeq (aosp_x86_64,$(TARGET_PRODUCT)) MODULE_BUILD_FROM_SOURCE ?= true $(call inherit-product, $(SRC_TARGET_DIR)/product/gsi_release.mk) -endif +PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE := aosp_system_image +USE_SOONG_DEFINED_SYSTEM_IMAGE := true +PRODUCT_USE_SOONG_NOTICE_XML := true + +endif PRODUCT_NAME := aosp_x86_64 PRODUCT_DEVICE := generic_x86_64 diff --git a/target/product/app_function_extensions.mk b/target/product/app_function_extensions.mk new file mode 100644 index 0000000000..a61afdc5ab --- /dev/null +++ b/target/product/app_function_extensions.mk @@ -0,0 +1,22 @@ +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The app function sidecar extensions + +# /system_ext packages +PRODUCT_PACKAGES += \ + com.google.android.appfunctions.sidecar \ + appfunctions.sidecar.xml diff --git a/target/product/base_system.mk b/target/product/base_system.mk index d806c061b6..40e2aa14a8 100644 --- a/target/product/base_system.mk +++ b/target/product/base_system.mk @@ -17,7 +17,7 @@ # Base modules and settings for the system partition. PRODUCT_PACKAGES += \ abx \ - aconfigd \ + aconfigd-system \ adbd_system_api \ aflags \ am \ @@ -52,7 +52,7 @@ PRODUCT_PACKAGES += \ com.android.adbd \ com.android.adservices \ com.android.appsearch \ - com.android.btservices \ + com.android.bt \ com.android.configinfrastructure \ com.android.conscrypt \ com.android.devicelock \ @@ -178,6 +178,7 @@ PRODUCT_PACKAGES += \ libmedia \ libmedia_jni \ libmediandk \ + libmonkey_jni \ libmtp \ libnetd_client \ libnetlink \ @@ -204,6 +205,7 @@ PRODUCT_PACKAGES += \ libstdc++ \ libsysutils \ libui \ + libuprobestats_client \ libusbhost \ libutils \ libvintf_jni \ @@ -211,6 +213,7 @@ PRODUCT_PACKAGES += \ libwilhelm \ linker \ llkd \ + llndk_libs \ lmkd \ LocalTransport \ locksettings \ @@ -239,6 +242,7 @@ PRODUCT_PACKAGES += \ PackageInstaller \ package-shareduid-allowlist.xml \ passwd_system \ + pbtombstone \ perfetto \ perfetto-extras \ ping \ @@ -246,6 +250,7 @@ PRODUCT_PACKAGES += \ pintool \ platform.xml \ pm \ + prefetch \ preinstalled-packages-asl-files.xml \ preinstalled-packages-platform.xml \ preinstalled-packages-strict-signature.xml \ @@ -274,7 +279,6 @@ PRODUCT_PACKAGES += \ Shell \ shell_and_utilities_system \ sm \ - snapshotctl \ snapuserd \ storaged \ surfaceflinger \ @@ -287,11 +291,11 @@ PRODUCT_PACKAGES += \ tombstoned \ traced \ traced_probes \ + tradeinmode \ tune2fs \ uiautomator \ uinput \ uncrypt \ - uprobestats \ usbd \ vdc \ vintf \ @@ -309,6 +313,17 @@ ifeq ($(RELEASE_CRASHRECOVERY_MODULE),true) endif +# When we release uprobestats module +ifeq ($(RELEASE_UPROBESTATS_MODULE),true) + PRODUCT_PACKAGES += \ + com.android.uprobestats \ + +else + PRODUCT_PACKAGES += \ + uprobestats \ + +endif + # These packages are not used on Android TV ifneq ($(PRODUCT_IS_ATV),true) PRODUCT_PACKAGES += \ @@ -345,6 +360,23 @@ ifeq ($(RELEASE_USE_WEBVIEW_BOOTSTRAP_MODULE),true) com.android.webview.bootstrap endif +# Only add the jar when it is not in the Tethering module. Otherwise, +# it will be added via com.android.tethering +ifneq ($(RELEASE_MOVE_VCN_TO_MAINLINE),true) + PRODUCT_PACKAGES += \ + framework-connectivity-b +endif + +ifneq (,$(RELEASE_RANGING_STACK)) + PRODUCT_PACKAGES += \ + com.android.ranging +endif + +ifeq ($(RELEASE_MEMORY_MANAGEMENT_DAEMON),true) + PRODUCT_PACKAGES += \ + mm_daemon +endif + # VINTF data for system image PRODUCT_PACKAGES += \ system_manifest.xml \ @@ -419,6 +451,7 @@ PRODUCT_HOST_PACKAGES += \ lpdump \ mke2fs \ mkfs.erofs \ + pbtombstone \ resize2fs \ sgdisk \ sqlite3 \ @@ -457,6 +490,7 @@ PRODUCT_VENDOR_PROPERTIES += ro.zygote?=zygote32 PRODUCT_SYSTEM_PROPERTIES += debug.atrace.tags.enableflags=0 PRODUCT_SYSTEM_PROPERTIES += persist.traced.enable=1 +PRODUCT_SYSTEM_PROPERTIES += ro.surface_flinger.game_default_frame_rate_override=60 # Include kernel configs. PRODUCT_PACKAGES += \ @@ -480,12 +514,14 @@ PRODUCT_PACKAGES_DEBUG := \ logtagd.rc \ ot-cli-ftd \ ot-ctl \ + overlay_remounter \ procrank \ profcollectd \ profcollectctl \ record_binder \ servicedispatcher \ showmap \ + snapshotctl \ sqlite3 \ ss \ start_with_lockagent \ diff --git a/target/product/base_vendor.mk b/target/product/base_vendor.mk index a80e0b31b6..16fc7fd906 100644 --- a/target/product/base_vendor.mk +++ b/target/product/base_vendor.mk @@ -17,7 +17,6 @@ # Base modules and settings for recovery. PRODUCT_PACKAGES += \ adbd.recovery \ - android.hardware.health@2.0-impl-default.recovery \ build_flag_vendor \ cgroups.recovery.json \ charger.recovery \ @@ -72,6 +71,8 @@ PRODUCT_PACKAGES += \ passwd_odm \ passwd_vendor \ selinux_policy_nonsystem \ + selinux_policy_vendor \ + selinux_policy_odm \ shell_and_utilities_vendor \ odm-build.prop \ @@ -105,6 +106,7 @@ PRODUCT_PACKAGES_SHIPPING_API_LEVEL_29 += \ # VINTF data for vendor image PRODUCT_PACKAGES += \ vendor_compatibility_matrix.xml \ + vendor_manifest.xml \ # Base modules and settings for the debug ramdisk, which is then packed # into a boot-debug.img and a vendor_boot-debug.img. diff --git a/target/product/build_variables.mk b/target/product/build_variables.mk index 9fc9ff9dc1..e99ab0686a 100644 --- a/target/product/build_variables.mk +++ b/target/product/build_variables.mk @@ -20,5 +20,14 @@ # Control libbinder client caching $(call soong_config_set, libbinder, release_libbinder_client_cache, $(RELEASE_LIBBINDER_CLIENT_CACHE)) +# Control caching while adding service in libbinder cache +$(call soong_config_set, libbinder, release_libbinder_addservice_cache, $(RELEASE_LIBBINDER_ADDSERVICE_CACHE)) + +# Remove static list in libbinder cache +$(call soong_config_set, libbinder, release_libbinder_remove_cache_static_list, $(RELEASE_LIBBINDER_REMOVE_CACHE_STATIC_LIST)) + # Use the configured release of sqlite $(call soong_config_set, libsqlite3, release_package_libsqlite3, $(RELEASE_PACKAGE_LIBSQLITE3)) + +# Use the configured MessageQueue implementation +$(call soong_config_set, messagequeue, release_package_messagequeue_implementation, $(RELEASE_PACKAGE_MESSAGEQUEUE_IMPLEMENTATION)) diff --git a/target/product/default_art_config.mk b/target/product/default_art_config.mk index 61d7235aac..e543ccf035 100644 --- a/target/product/default_art_config.mk +++ b/target/product/default_art_config.mk @@ -65,7 +65,7 @@ PRODUCT_APEX_BOOT_JARS := \ com.android.adservices:framework-adservices \ com.android.adservices:framework-sdksandbox \ com.android.appsearch:framework-appsearch \ - com.android.btservices:framework-bluetooth \ + com.android.bt:framework-bluetooth \ com.android.configinfrastructure:framework-configinfrastructure \ com.android.conscrypt:conscrypt \ com.android.devicelock:framework-devicelock \ @@ -114,6 +114,23 @@ ifeq ($(RELEASE_PACKAGE_PROFILING_MODULE),true) endif +ifneq (,$(RELEASE_RANGING_STACK)) + PRODUCT_APEX_BOOT_JARS += \ + com.android.uwb:framework-ranging \ + $(call soong_config_set,bootclasspath,release_ranging_stack,true) +endif + +# Check if VCN should be built into the tethering module or not +ifeq ($(RELEASE_MOVE_VCN_TO_MAINLINE),true) + PRODUCT_APEX_BOOT_JARS += \ + com.android.tethering:framework-connectivity-b \ + +else + PRODUCT_BOOT_JARS += \ + framework-connectivity-b \ + +endif + # List of system_server classpath jars delivered via apex. # Keep the list sorted by module names and then library names. # Note: For modules available in Q, DO NOT add new entries here. @@ -154,7 +171,7 @@ PRODUCT_STANDALONE_SYSTEM_SERVER_JARS := \ # Keep the list sorted by module names and then library names. # Note: For modules available in Q, DO NOT add new entries here. PRODUCT_APEX_STANDALONE_SYSTEM_SERVER_JARS := \ - com.android.btservices:service-bluetooth \ + com.android.bt:service-bluetooth \ com.android.devicelock:service-devicelock \ com.android.os.statsd:service-statsd \ com.android.scheduling:service-scheduling \ @@ -169,6 +186,11 @@ ifeq ($(RELEASE_PACKAGE_PROFILING_MODULE),true) endif +ifneq (,$(RELEASE_RANGING_STACK)) + PRODUCT_APEX_STANDALONE_SYSTEM_SERVER_JARS += \ + com.android.uwb:service-ranging +endif + # Overrides the (apex, jar) pairs above when determining the on-device location. The format is: # <old_apex>:<old_jar>:<new_apex>:<new_jar> PRODUCT_CONFIGURED_JAR_LOCATION_OVERRIDES := \ diff --git a/target/product/full_x86.mk b/target/product/full_x86.mk index 07f6472844..a1b71caaed 100644 --- a/target/product/full_x86.mk +++ b/target/product/full_x86.mk @@ -32,11 +32,6 @@ ifdef NET_ETH0_STARTONBOOT PRODUCT_VENDOR_PROPERTIES += net.eth0.startonboot=1 endif -# Ensure we package the BIOS files too. -PRODUCT_HOST_PACKAGES += \ - bios.bin \ - vgabios-cirrus.bin \ - # Enable dynamic partition size PRODUCT_USE_DYNAMIC_PARTITION_SIZE := true diff --git a/target/product/fullmte.mk b/target/product/fullmte.mk index b62249601e..fed66e7ef2 100644 --- a/target/product/fullmte.mk +++ b/target/product/fullmte.mk @@ -20,7 +20,7 @@ # For more details, see: # https://source.android.com/docs/security/test/memory-safety/arm-mte ifeq ($(filter memtag_heap,$(SANITIZE_TARGET)),) - SANITIZE_TARGET := $(strip $(SANITIZE_TARGET) memtag_heap memtag_stack) + SANITIZE_TARGET := $(strip $(SANITIZE_TARGET) memtag_heap memtag_stack memtag_globals) SANITIZE_TARGET_DIAG := $(strip $(SANITIZE_TARGET_DIAG) memtag_heap) endif PRODUCT_PRODUCT_PROPERTIES += persist.arm64.memtag.default=sync diff --git a/target/product/generic/Android.bp b/target/product/generic/Android.bp new file mode 100644 index 0000000000..5bfff66c32 --- /dev/null +++ b/target/product/generic/Android.bp @@ -0,0 +1,1052 @@ +generic_rootdirs = [ + "apex", + "bootstrap-apex", + "config", + "data", + "data_mirror", + "debug_ramdisk", + "dev", + "linkerconfig", + "metadata", + "mnt", + "odm", + "odm_dlkm", + "oem", + "postinstall", + "proc", + "second_stage_resources", + "storage", + "sys", + "system", + "system_dlkm", + "tmp", + "vendor", + "vendor_dlkm", +] + +android_rootdirs = [ + "system_ext", + "product", +] + +generic_symlinks = [ + { + target: "/system/bin/init", + name: "init", + }, + { + target: "/system/etc", + name: "etc", + }, + { + target: "/system/bin", + name: "bin", + }, + { + target: "/vendor", + name: "system/vendor", + }, + { + target: "/system_dlkm/lib/modules", + name: "system/lib/modules", + }, + { + target: "/data/user_de/0/com.android.shell/files/bugreports", + name: "bugreports", + }, + { + target: "/sys/kernel/debug", + name: "d", + }, + { + target: "/storage/self/primary", + name: "sdcard", + }, + { + target: "/product/etc/security/adb_keys", + name: "adb_keys", + }, + // For Treble Generic System Image (GSI), system-as-root GSI needs to work on both devices with + // and without /odm partition. Those symlinks are for devices without /odm partition. For + // devices with /odm partition, mount odm.img under /odm will hide those symlinks. + { + target: "/vendor/odm/app", + name: "odm/app", + }, + { + target: "/vendor/odm/bin", + name: "odm/bin", + }, + { + target: "/vendor/odm/etc", + name: "odm/etc", + }, + { + target: "/vendor/odm/firmware", + name: "odm/firmware", + }, + { + target: "/vendor/odm/framework", + name: "odm/framework", + }, + { + target: "/vendor/odm/lib", + name: "odm/lib", + }, + { + target: "/vendor/odm/lib64", + name: "odm/lib64", + }, + { + target: "/vendor/odm/overlay", + name: "odm/overlay", + }, + { + target: "/vendor/odm/priv-app", + name: "odm/priv-app", + }, + { + target: "/vendor/odm/usr", + name: "odm/usr", + }, +] + +android_symlinks = [ + { + target: "/product", + name: "system/product", + }, + { + target: "/system_ext", + name: "system/system_ext", + }, + { + target: "/data/cache", + name: "cache", + }, +] + +extra_vendor_symlinks = [ + // Some vendors still haven't cleaned up all device specific directories under root! + // TODO(b/111434759, b/111287060) SoC specific hacks + { + target: "/vendor/lib/dsp", + name: "dsp", + }, + { + target: "/mnt/vendor/persist", + name: "persist", + }, + { + target: "/vendor/firmware_mnt", + name: "firmware", + }, +] + +filegroup { + name: "generic_system_sign_key", + srcs: [":avb_testkey_rsa4096"], +} + +phony { + name: "generic_system_fonts", + required: [ + "AndroidClock.ttf", + "CarroisGothicSC-Regular.ttf", + "ComingSoon.ttf", + "CutiveMono.ttf", + "DancingScript-Regular.ttf", + "DroidSansMono.ttf", + "NotoColorEmoji.ttf", + "NotoColorEmojiFlags.ttf", + "NotoNaskhArabic-Bold.ttf", + "NotoNaskhArabic-Regular.ttf", + "NotoNaskhArabicUI-Bold.ttf", + "NotoNaskhArabicUI-Regular.ttf", + "NotoSansAdlam-VF.ttf", + "NotoSansAhom-Regular.otf", + "NotoSansAnatolianHieroglyphs-Regular.otf", + "NotoSansArmenian-VF.ttf", + "NotoSansAvestan-Regular.ttf", + "NotoSansBalinese-Regular.ttf", + "NotoSansBamum-Regular.ttf", + "NotoSansBassaVah-Regular.otf", + "NotoSansBatak-Regular.ttf", + "NotoSansBengali-VF.ttf", + "NotoSansBengaliUI-VF.ttf", + "NotoSansBhaiksuki-Regular.otf", + "NotoSansBrahmi-Regular.ttf", + "NotoSansBuginese-Regular.ttf", + "NotoSansBuhid-Regular.ttf", + "NotoSansCJK-Regular.ttc", + "NotoSansCanadianAboriginal-Regular.ttf", + "NotoSansCarian-Regular.ttf", + "NotoSansChakma-Regular.otf", + "NotoSansCham-Bold.ttf", + "NotoSansCham-Regular.ttf", + "NotoSansCherokee-Regular.ttf", + "NotoSansCoptic-Regular.ttf", + "NotoSansCuneiform-Regular.ttf", + "NotoSansCypriot-Regular.ttf", + "NotoSansDeseret-Regular.ttf", + "NotoSansDevanagari-VF.ttf", + "NotoSansDevanagariUI-VF.ttf", + "NotoSansEgyptianHieroglyphs-Regular.ttf", + "NotoSansElbasan-Regular.otf", + "NotoSansEthiopic-VF.ttf", + "NotoSansGeorgian-VF.ttf", + "NotoSansGlagolitic-Regular.ttf", + "NotoSansGothic-Regular.ttf", + "NotoSansGrantha-Regular.ttf", + "NotoSansGujarati-Bold.ttf", + "NotoSansGujarati-Regular.ttf", + "NotoSansGujaratiUI-Bold.ttf", + "NotoSansGujaratiUI-Regular.ttf", + "NotoSansGunjalaGondi-Regular.otf", + "NotoSansGurmukhi-VF.ttf", + "NotoSansGurmukhiUI-VF.ttf", + "NotoSansHanifiRohingya-Regular.otf", + "NotoSansHanunoo-Regular.ttf", + "NotoSansHatran-Regular.otf", + "NotoSansHebrew-Bold.ttf", + "NotoSansHebrew-Regular.ttf", + "NotoSansImperialAramaic-Regular.ttf", + "NotoSansInscriptionalPahlavi-Regular.ttf", + "NotoSansInscriptionalParthian-Regular.ttf", + "NotoSansJavanese-Regular.otf", + "NotoSansKaithi-Regular.ttf", + "NotoSansKannada-VF.ttf", + "NotoSansKannadaUI-VF.ttf", + "NotoSansKayahLi-Regular.ttf", + "NotoSansKharoshthi-Regular.ttf", + "NotoSansKhmer-VF.ttf", + "NotoSansKhmerUI-Bold.ttf", + "NotoSansKhmerUI-Regular.ttf", + "NotoSansKhojki-Regular.otf", + "NotoSansLao-Bold.ttf", + "NotoSansLao-Regular.ttf", + "NotoSansLaoUI-Bold.ttf", + "NotoSansLaoUI-Regular.ttf", + "NotoSansLepcha-Regular.ttf", + "NotoSansLimbu-Regular.ttf", + "NotoSansLinearA-Regular.otf", + "NotoSansLinearB-Regular.ttf", + "NotoSansLisu-Regular.ttf", + "NotoSansLycian-Regular.ttf", + "NotoSansLydian-Regular.ttf", + "NotoSansMalayalam-VF.ttf", + "NotoSansMalayalamUI-VF.ttf", + "NotoSansMandaic-Regular.ttf", + "NotoSansManichaean-Regular.otf", + "NotoSansMarchen-Regular.otf", + "NotoSansMasaramGondi-Regular.otf", + "NotoSansMedefaidrin-VF.ttf", + "NotoSansMeeteiMayek-Regular.ttf", + "NotoSansMeroitic-Regular.otf", + "NotoSansMiao-Regular.otf", + "NotoSansModi-Regular.ttf", + "NotoSansMongolian-Regular.ttf", + "NotoSansMro-Regular.otf", + "NotoSansMultani-Regular.otf", + "NotoSansMyanmar-Bold.otf", + "NotoSansMyanmar-Medium.otf", + "NotoSansMyanmar-Regular.otf", + "NotoSansMyanmarUI-Bold.otf", + "NotoSansMyanmarUI-Medium.otf", + "NotoSansMyanmarUI-Regular.otf", + "NotoSansNKo-Regular.ttf", + "NotoSansNabataean-Regular.otf", + "NotoSansNewTaiLue-Regular.ttf", + "NotoSansNewa-Regular.otf", + "NotoSansOgham-Regular.ttf", + "NotoSansOlChiki-Regular.ttf", + "NotoSansOldItalic-Regular.ttf", + "NotoSansOldNorthArabian-Regular.otf", + "NotoSansOldPermic-Regular.otf", + "NotoSansOldPersian-Regular.ttf", + "NotoSansOldSouthArabian-Regular.ttf", + "NotoSansOldTurkic-Regular.ttf", + "NotoSansOriya-Bold.ttf", + "NotoSansOriya-Regular.ttf", + "NotoSansOriyaUI-Bold.ttf", + "NotoSansOriyaUI-Regular.ttf", + "NotoSansOsage-Regular.ttf", + "NotoSansOsmanya-Regular.ttf", + "NotoSansPahawhHmong-Regular.otf", + "NotoSansPalmyrene-Regular.otf", + "NotoSansPauCinHau-Regular.otf", + "NotoSansPhagsPa-Regular.ttf", + "NotoSansPhoenician-Regular.ttf", + "NotoSansRejang-Regular.ttf", + "NotoSansRunic-Regular.ttf", + "NotoSansSamaritan-Regular.ttf", + "NotoSansSaurashtra-Regular.ttf", + "NotoSansSharada-Regular.otf", + "NotoSansShavian-Regular.ttf", + "NotoSansSinhala-VF.ttf", + "NotoSansSinhalaUI-VF.ttf", + "NotoSansSoraSompeng-Regular.otf", + "NotoSansSoyombo-VF.ttf", + "NotoSansSundanese-Regular.ttf", + "NotoSansSylotiNagri-Regular.ttf", + "NotoSansSymbols-Regular-Subsetted.ttf", + "NotoSansSymbols-Regular-Subsetted2.ttf", + "NotoSansSyriacEastern-Regular.ttf", + "NotoSansSyriacEstrangela-Regular.ttf", + "NotoSansSyriacWestern-Regular.ttf", + "NotoSansTagalog-Regular.ttf", + "NotoSansTagbanwa-Regular.ttf", + "NotoSansTaiLe-Regular.ttf", + "NotoSansTaiTham-Regular.ttf", + "NotoSansTaiViet-Regular.ttf", + "NotoSansTakri-VF.ttf", + "NotoSansTamil-VF.ttf", + "NotoSansTamilUI-VF.ttf", + "NotoSansTelugu-VF.ttf", + "NotoSansTeluguUI-VF.ttf", + "NotoSansThaana-Bold.ttf", + "NotoSansThaana-Regular.ttf", + "NotoSansThai-Bold.ttf", + "NotoSansThai-Regular.ttf", + "NotoSansThaiUI-Bold.ttf", + "NotoSansThaiUI-Regular.ttf", + "NotoSansTifinagh-Regular.otf", + "NotoSansUgaritic-Regular.ttf", + "NotoSansVai-Regular.ttf", + "NotoSansWancho-Regular.otf", + "NotoSansWarangCiti-Regular.otf", + "NotoSansYi-Regular.ttf", + "NotoSerif-Bold.ttf", + "NotoSerif-BoldItalic.ttf", + "NotoSerif-Italic.ttf", + "NotoSerif-Regular.ttf", + "NotoSerifArmenian-VF.ttf", + "NotoSerifBengali-VF.ttf", + "NotoSerifCJK-Regular.ttc", + "NotoSerifDevanagari-VF.ttf", + "NotoSerifDogra-Regular.ttf", + "NotoSerifEthiopic-VF.ttf", + "NotoSerifGeorgian-VF.ttf", + "NotoSerifGujarati-VF.ttf", + "NotoSerifGurmukhi-VF.ttf", + "NotoSerifHebrew-Bold.ttf", + "NotoSerifHebrew-Regular.ttf", + "NotoSerifHentaigana.ttf", + "NotoSerifKannada-VF.ttf", + "NotoSerifKhmer-Bold.otf", + "NotoSerifKhmer-Regular.otf", + "NotoSerifLao-Bold.ttf", + "NotoSerifLao-Regular.ttf", + "NotoSerifMalayalam-VF.ttf", + "NotoSerifMyanmar-Bold.otf", + "NotoSerifMyanmar-Regular.otf", + "NotoSerifNyiakengPuachueHmong-VF.ttf", + "NotoSerifSinhala-VF.ttf", + "NotoSerifTamil-VF.ttf", + "NotoSerifTelugu-VF.ttf", + "NotoSerifThai-Bold.ttf", + "NotoSerifThai-Regular.ttf", + "NotoSerifTibetan-VF.ttf", + "NotoSerifYezidi-VF.ttf", + "Roboto-Regular.ttf", + "RobotoFlex-Regular.ttf", + "RobotoStatic-Regular.ttf", + "SourceSansPro-Bold.ttf", + "SourceSansPro-BoldItalic.ttf", + "SourceSansPro-Italic.ttf", + "SourceSansPro-Regular.ttf", + "SourceSansPro-SemiBold.ttf", + "SourceSansPro-SemiBoldItalic.ttf", + "font_fallback.xml", + "fonts.xml", + ], +} + +android_filesystem_defaults { + name: "system_ext_image_defaults", + deps: [ + /////////////////////////////////////////// + // base_system_ext + /////////////////////////////////////////// + "build_flag_system_ext", + "fs_config_dirs_system_ext", + "fs_config_files_system_ext", + "group_system_ext", + "passwd_system_ext", + "SatelliteClient", + "selinux_policy_system_ext", + "system_ext_manifest.xml", + "system_ext-build.prop", + // Base modules when shipping api level is less than or equal to 34 + "hwservicemanager", + "android.hidl.allocator@1.0-service", + + /////////////////////////////////////////// + // window_extensions_base + /////////////////////////////////////////// + "androidx.window.extensions", + "androidx.window.sidecar", + + /////////////////////////////////////////// + // base_system + /////////////////////////////////////////// + "charger", + ] + select(release_flag("RELEASE_APPFUNCTION_SIDECAR"), { + true: [ + "com.google.android.appfunctions.sidecar", + "appfunctions.sidecar.xml", + ], + default: [], + }), +} + +android_filesystem_defaults { + name: "product_image_defaults", + deps: [ + /////////////////////////////////////////// + // media_product + /////////////////////////////////////////// + "webview", + + /////////////////////////////////////////// + // base_product + /////////////////////////////////////////// + + // Base modules and settings for the product partition. + "build_flag_product", + "fs_config_dirs_product", + "fs_config_files_product", + "group_product", + "ModuleMetadata", + "passwd_product", + "product_compatibility_matrix.xml", + "product_manifest.xml", + "selinux_policy_product", + "product-build.prop", + + // AUDIO + "frameworks_sounds", + ] + select(product_variable("debuggable"), { + // Packages included only for eng or userdebug builds, previously debug tagged + true: ["adb_keys"], + default: [], + }), +} + +system_image_fsverity_default = { + inputs: [ + "etc/boot-image.prof", + "etc/classpaths/*.pb", + "etc/dirty-image-objects", + "etc/preloaded-classes", + "framework/*", + "framework/*/*", // framework/{arch} + "framework/oat/*/*", // framework/oat/{arch} + ], + libs: [":framework-res{.export-package.apk}"], +} + +soong_config_module_type { + name: "system_image_defaults", + module_type: "android_filesystem_defaults", + config_namespace: "ANDROID", + bool_variables: ["TARGET_ADD_ROOT_EXTRA_VENDOR_SYMLINKS"], + properties: ["symlinks"], +} + +genrule { + name: "plat_and_vendor_file_contexts", + device_common_srcs: [ + ":plat_file_contexts", + ":vendor_file_contexts", + ], + out: ["file_contexts"], + cmd: "cat $(in) > $(out)", +} + +system_image_defaults { + name: "system_image_defaults", + partition_name: "system", + base_dir: "system", + stem: "system.img", + no_full_install: true, + dirs: generic_rootdirs, + soong_config_variables: { + TARGET_ADD_ROOT_EXTRA_VENDOR_SYMLINKS: { + symlinks: generic_symlinks + extra_vendor_symlinks, + conditions_default: { + symlinks: generic_symlinks, + }, + }, + }, + file_contexts: ":plat_and_vendor_file_contexts", + linker_config: { + gen_linker_config: true, + linker_config_srcs: [":system_linker_config_json_file"], + }, + fsverity: { + inputs: select(soong_config_variable("ANDROID", "PRODUCT_FSVERITY_GENERATE_METADATA"), { + true: [ + "etc/boot-image.prof", + "etc/classpaths/*.pb", + "etc/dirty-image-objects", + "etc/preloaded-classes", + "framework/*", + "framework/*/*", // framework/{arch} + "framework/oat/*/*", // framework/oat/{arch} + ], + default: [], + }), + libs: select(soong_config_variable("ANDROID", "PRODUCT_FSVERITY_GENERATE_METADATA"), { + true: [":framework-res{.export-package.apk}"], + default: [], + }), + }, + build_logtags: true, + gen_aconfig_flags_pb: true, + + compile_multilib: "both", + + use_avb: true, + avb_private_key: ":generic_system_sign_key", + avb_algorithm: "SHA256_RSA4096", + avb_hash_algorithm: "sha256", + rollback_index_location: 1, + + deps: [ + "abx", + "aconfigd-system", + "aflags", + "am", + "android.software.credentials.prebuilt.xml", // generic_system + "android.software.webview.prebuilt.xml", // media_system + "android.software.window_magnification.prebuilt.xml", // handheld_system + "android.system.suspend-service", + "apexd", + "appops", + "approved-ogki-builds.xml", // base_system + "appwidget", + "atrace", + "audioserver", + "bcc", + "blank_screen", + "blkid", + "bmgr", + "bootanimation", + "bootstat", + "bpfloader", + "bu", + "bugreport", + "bugreportz", + "cameraserver", + "cgroups.json", + "cmd", + "content", + "cppreopts.sh", // generic_system + "credstore", + "debuggerd", + "device_config", + "dirty-image-objects", + "dmctl", + "dmesgd", + "dnsmasq", + "dpm", + "dump.erofs", + "dumpstate", + "dumpsys", + "e2fsck", + "enhanced-confirmation.xml", // base_system + "etc_hosts", + "flags_health_check", + "framework-audio_effects.xml", // for handheld // handheld_system + "framework-sysconfig.xml", + "fs_config_dirs_system", + "fs_config_files_system", + "fsck.erofs", + "fsck.f2fs", // for media_system + "fsck_msdos", + "fsverity-release-cert-der", + "gatekeeperd", + "gpu_counter_producer", + "gpuservice", + "group_system", + "gsi_tool", + "gsid", + "heapprofd", + "hid", + "hiddenapi-package-whitelist.xml", // from runtime_libart + "idc_data", + "idmap2", + "idmap2d", + "ime", + "incident", + "incident-helper-cmd", + "incident_helper", + "incidentd", + "init.environ.rc-soong", + "init.usb.configfs.rc", + "init.usb.rc", + "init.zygote32.rc", + "init.zygote64.rc", + "init.zygote64_32.rc", + "initial-package-stopped-states.xml", + "input", + "installd", + "ip", // base_system + "iptables", + "kcmdlinectrl", + "kernel-lifetimes.xml", // base_system + "keychars_data", + "keylayout_data", + "keystore2", + "ld.mc", + "llkd", // base_system + "lmkd", // base_system + "locksettings", // base_system + "logcat", // base_system + "logd", // base_system + "lpdump", // base_system + "lshal", // base_system + "make_f2fs", // media_system + "mdnsd", // base_system + "media_profiles_V1_0.dtd", // base_system + "mediacodec.policy", // base_system + "mediaextractor", // base_system + "mediametrics", // base_system + "misctrl", // from base_system + "mke2fs", // base_system + "mkfs.erofs", // base_system + "monkey", // base_system + "mtectrl", // base_system + "ndc", // base_system + "netd", // base_system + "netutils-wrapper-1.0", // full_base + "notice_xml_system", + "odsign", // base_system + "otapreopt_script", // generic_system + "package-shareduid-allowlist.xml", // base_system + "passwd_system", // base_system + "pbtombstone", // base_system + "perfetto", // base_system + "ping", // base_system + "ping6", // base_system + "pintool", // base_system + "platform.xml", // base_system + "pm", // base_system + "prefetch", //base_system + "preinstalled-packages-asl-files.xml", // base_system + "preinstalled-packages-platform-generic-system.xml", // generic_system + "preinstalled-packages-platform-handheld-system.xml", // handheld_system + "preinstalled-packages-platform.xml", // base_system + "preinstalled-packages-strict-signature.xml", // base_system + "preloaded-classes", // ok + "printflags", // base_system + "privapp-permissions-platform.xml", // base_system + "prng_seeder", // base_system + "public.libraries.android.txt", + "recovery-persist", // base_system + "recovery-refresh", // generic_system + "requestsync", // media_system + "resize2fs", // base_system + "rss_hwm_reset", // base_system + "run-as", // base_system + "schedtest", // base_system + "screencap", // base_system + "screenrecord", // handheld_system + "sdcard", // base_system + "secdiscard", // base_system + "sensorservice", // base_system + "service", // base_system + "servicemanager", // base_system + "settings", // base_system + "sfdo", // base_system + "sgdisk", // base_system + "sm", // base_system + "snapuserd", // base_system + "storaged", // base_system + "surfaceflinger", // base_system + "svc", // base_system + "system_manifest.xml", // base_system + "task_profiles.json", // base_system + "tc", // base_system + "telecom", // base_system + "tombstoned", // base_system + "traced", // base_system + "traced_probes", // base_system + "tradeinmode", // base_system + "tune2fs", // base_system + "uiautomator", // base_system + "uinput", // base_system + "uncrypt", // base_system + "update_engine", // generic_system + "update_engine_sideload", // recovery + "update_verifier", // generic_system + "usbd", // base_system + "vdc", // base_system + "virtual_camera", // handheld_system // release_package_virtual_camera + "vold", // base_system + "vr", // handheld_system + "watchdogd", // base_system + "wifi.rc", // base_system + "wificond", // base_system + "wm", // base_system + ] + select(release_flag("RELEASE_PLATFORM_VERSION_CODENAME"), { + "REL": [], + default: [ + "android.software.preview_sdk.prebuilt.xml", // media_system + ], + }) + select(soong_config_variable("ANDROID", "release_package_profiling_module"), { + "true": [ + "trace_redactor", // base_system (RELEASE_PACKAGE_PROFILING_MODULE) + ], + default: [], + }) + select(release_flag("RELEASE_MEMORY_MANAGEMENT_DAEMON"), { + true: [ + "mm_daemon", // base_system (RELEASE_MEMORY_MANAGEMENT_DAEMON) + ], + default: [], + }) + select(product_variable("debuggable"), { + true: [ + "alloctop", + "adevice_fingerprint", + "arping", + "avbctl", + "bootctl", + "dmuserd", + "evemu-record", + "idlcli", + "init-debug.rc", + "iotop", + "iperf3", + "iw", + "layertracegenerator", + "logpersist.start", + "logtagd.rc", + "ot-cli-ftd", + "ot-ctl", + "overlay_remounter", + "procrank", + "profcollectctl", + "profcollectd", + "record_binder", + "sanitizer-status", + "servicedispatcher", + "showmap", + "snapshotctl", + "sqlite3", + "ss", + "start_with_lockagent", + "strace", + "su", + "tinycap", + "tinyhostless", + "tinymix", + "tinypcminfo", + "tinyplay", // host + "tracepath", + "tracepath6", + "traceroute6", + "unwind_info", + "unwind_reg_info", + "unwind_symbols", + "update_engine_client", + ], + default: [], + }) + select(release_flag("RELEASE_UPROBESTATS_MODULE"), { + true: [], + default: [ + "uprobestats", // base_system internal + ], + }), + multilib: { + common: { + deps: [ + "BackupRestoreConfirmation", // base_system + "BasicDreams", // handheld_system + "BlockedNumberProvider", // handheld_system + "BluetoothMidiService", // handheld_system + "BookmarkProvider", // handheld_system + "BuiltInPrintService", // handheld_system + "CalendarProvider", // handheld_system + "CallLogBackup", // telephony_system + "CameraExtensionsProxy", // handheld_system + "CaptivePortalLogin", // handheld_system + "CarrierDefaultApp", // telephony_system + "CellBroadcastLegacyApp", // telephony_system + "CertInstaller", // handheld_system + "CompanionDeviceManager", // media_system + "ContactsProvider", // base_system + "CredentialManager", // handheld_system + "DeviceAsWebcam", // handheld_system + "DeviceDiagnostics", // handheld_system - internal + "DocumentsUI", // handheld_system + "DownloadProvider", // base_system + "DownloadProviderUi", // handheld_system + "DynamicSystemInstallationService", // base_system + "E2eeContactKeysProvider", // base_system + "EasterEgg", // handheld_system + "ExtShared", // base_system + "ExternalStorageProvider", // handheld_system + "FusedLocation", // handheld_system + "HTMLViewer", // media_system + "InputDevices", // handheld_system + "IntentResolver", // base_system + "KeyChain", // handheld_system + "LiveWallpapersPicker", // generic_system, full_base + "LocalTransport", // base_system + "ManagedProvisioning", // handheld_system + "MediaProviderLegacy", // base_system + "MmsService", // handheld_system + "MtpService", // handheld_system + "MusicFX", // handheld_system + "NetworkStack", // base_system + "ONS", // telephony_system + "PacProcessor", // handheld_system + "PackageInstaller", // base_system + "PartnerBookmarksProvider", // generic_system + "PrintRecommendationService", // handheld_system + "PrintSpooler", // handheld_system + "ProxyHandler", // handheld_system + "SecureElement", // handheld_system + "SettingsProvider", // base_system + "SharedStorageBackup", // handheld_system + "Shell", // base_system + "SimAppDialog", // handheld_system + "SoundPicker", // not installed by anyone + "StatementService", // media_system + "Stk", // generic_system + "Tag", // generic_system + "TeleService", // handheld_system + "Telecom", // handheld_system + "TelephonyProvider", // handheld_system + "Traceur", // handheld_system + "UserDictionaryProvider", // handheld_system + "VpnDialogs", // handheld_system + "WallpaperBackup", // base_system + "adbd_system_api", // base_system + "android.hidl.base-V1.0-java", // base_system + "android.hidl.manager-V1.0-java", // base_system + "android.test.base", // from runtime_libart + "android.test.mock", // base_system + "android.test.runner", // base_system + "aosp_mainline_modules", // ok + "build_flag_system", // base_system + "charger_res_images", // generic_system + "com.android.apex.cts.shim.v1_prebuilt", // ok + "com.android.cellbroadcast", // telephony_system + "com.android.future.usb.accessory", // media_system + "com.android.location.provider", // base_system + "com.android.media.remotedisplay", // media_system + "com.android.media.remotedisplay.xml", // media_system + "com.android.mediadrm.signer", // media_system + "com.android.nfc_extras", // ok + "com.android.nfcservices", // base_system (RELEASE_PACKAGE_NFC_STACK != NfcNci) + "com.android.runtime", // ok + "dex_bootjars", + "ext", // from runtime_libart + "framework-graphics", // base_system + "framework-location", // base_system + "framework-minus-apex-install-dependencies", // base_system + "framework_compatibility_matrix.device.xml", + "generic_system_fonts", // ok + "hwservicemanager_compat_symlink_module", // base_system + "hyph-data", + "ims-common", // base_system + "init_system", // base_system + "javax.obex", // base_system + "llndk.libraries.txt", //ok + "org.apache.http.legacy", // base_system + "perfetto-extras", // system + "sanitizer.libraries.txt", // base_system + "selinux_policy_system_soong", // ok + "services", // base_system + "shell_and_utilities_system", // ok + "system-build.prop", + "system_compatibility_matrix.xml", //base_system + "telephony-common", // libs from TeleService + "voip-common", // base_system + ] + select(soong_config_variable("ANDROID", "release_crashrecovery_module"), { + "true": [ + "com.android.crashrecovery", // base_system (RELEASE_CRASHRECOVERY_MODULE) + ], + default: [], + }) + select(soong_config_variable("ANDROID", "release_package_profiling_module"), { + "true": [ + "com.android.profiling", // base_system (RELEASE_PACKAGE_PROFILING_MODULE) + ], + default: [], + }) + select(release_flag("RELEASE_MOVE_VCN_TO_MAINLINE"), { + true: [], + default: [ + "framework-connectivity-b", // base_system + ], + }) + select(release_flag("RELEASE_UPROBESTATS_MODULE"), { + true: [ + "com.android.uprobestats", // base_system (RELEASE_UPROBESTATS_MODULE) + ], + default: [], + }), + }, + prefer32: { + deps: [ + "drmserver", // media_system + "mediaserver", // base_system + ], + }, + lib64: { + deps: [ + "android.system.virtualizationcommon-ndk", + "android.system.virtualizationservice-ndk", + "libgsi", + "servicemanager", + ], + }, + both: { + deps: [ + "android.hardware.biometrics.fingerprint@2.1", // generic_system + "android.hardware.radio.config@1.0", // generic_system + "android.hardware.radio.deprecated@1.0", // generic_system + "android.hardware.radio@1.0", // generic_system + "android.hardware.radio@1.1", // generic_system + "android.hardware.radio@1.2", // generic_system + "android.hardware.radio@1.3", // generic_system + "android.hardware.radio@1.4", // generic_system + "android.hardware.secure_element@1.0", // generic_system + "app_process", // base_system + "boringssl_self_test", // base_system + "heapprofd_client", // base_system + "libEGL", // base_system + "libEGL_angle", // base_system + "libETC1", // base_system + "libFFTEm", // base_system + "libGLESv1_CM", // base_system + "libGLESv1_CM_angle", // base_system + "libGLESv2", // base_system + "libGLESv2_angle", // base_system + "libGLESv3", // base_system + "libOpenMAXAL", // base_system + "libOpenSLES", // base_system + "libaaudio", // base_system + "libalarm_jni", // base_system + "libamidi", // base_system + "libandroid", + "libandroid_runtime", + "libandroid_servers", + "libandroidfw", + "libartpalette-system", + "libaudio-resampler", // generic-system + "libaudioeffect_jni", + "libaudiohal", // generic-system + "libaudiopolicyengineconfigurable", // generic-system + "libbinder", + "libbinder_ndk", + "libbinder_rpc_unstable", + "libcamera2ndk", + "libcgrouprc", // llndk library + "libclang_rt.asan", + "libcompiler_rt", + "libcutils", // used by many libs + "libdmabufheap", // used by many libs + "libdrm", // used by many libs // generic_system + "libdrmframework", // base_system + "libdrmframework_jni", // base_system + "libfdtrack", // base_system + "libfilterfw", // base_system + "libfilterpack_imageproc", // media_system + "libfwdlockengine", // generic_system + "libgatekeeper", // base_system + "libgui", // base_system + "libhardware", // base_system + "libhardware_legacy", // base_system + "libhidltransport", // generic_system + "libhwbinder", // generic_system + "libinput", // base_system + "libinputflinger", // base_system + "libiprouteutil", // base_system + "libjnigraphics", // base_system + "libjpeg", // base_system + "liblog", // base_system + "liblogwrap", // generic_system + "liblz4", // generic_system + "libmedia", // base_system + "libmedia_jni", // base_system + "libmediandk", // base_system + "libminui", // generic_system + "libmonkey_jni", // base_system - internal + "libmtp", // base_system + "libnetd_client", // base_system + "libnetlink", // base_system + "libnetutils", // base_system + "libneuralnetworks_packageinfo", // base_system + "libnl", // generic_system + "libpdfium", // base_system + "libpolicy-subsystem", // generic_system + "libpower", // base_system + "libpowermanager", // base_system + "libprotobuf-cpp-full", // generic_system + "libradio_metadata", // base_system + "librs_jni", // handheld_system + "librtp_jni", // base_system + "libsensorservice", // base_system + "libsfplugin_ccodec", // base_system + "libskia", // base_system + "libsonic", // base_system + "libsonivox", // base_system + "libsoundpool", // base_system + "libspeexresampler", // base_system + "libsqlite", // base_system + "libstagefright", // base_system + "libstagefright_foundation", // base_system + "libstagefright_omx", // base_system + "libstdc++", // base_system + "libsysutils", // base_system + "libui", // base_system + "libusbhost", // base_system + "libutils", // base_system + "libvendorsupport", // llndk library + "libvintf_jni", // base_system + "libvulkan", // base_system + "libwebviewchromium_loader", // media_system + "libwebviewchromium_plat_support", // media_system + "libwilhelm", // base_system + "linker", // base_system + ] + select(soong_config_variable("ANDROID", "TARGET_DYNAMIC_64_32_DRMSERVER"), { + "true": ["drmserver"], + default: [], + }) + select(soong_config_variable("ANDROID", "TARGET_DYNAMIC_64_32_MEDIASERVER"), { + "true": ["mediaserver"], + default: [], + }) + select(release_flag("RELEASE_UPROBESTATS_MODULE"), { + true: [], + default: [ + "libuprobestats_client", // base_system internal + ], + }), + }, + }, + arch: { + arm64: { + deps: [ + "libclang_rt.hwasan", + "libc_hwasan", + ], + }, + }, +} + +android_system_image { + name: "aosp_shared_system_image", + defaults: ["system_image_defaults"], + dirs: android_rootdirs, + symlinks: android_symlinks, + type: "erofs", + erofs: { + compressor: "lz4hc,9", + compress_hints: "erofs_compress_hints.txt", + }, + deps: [ + // DO NOT update this list. Instead, update the system_image_defaults to + // sync with the base_system.mk + "logpersist.start", // cf only + ], +} diff --git a/target/product/generic/OWNERS b/target/product/generic/OWNERS new file mode 100644 index 0000000000..6d1446f099 --- /dev/null +++ b/target/product/generic/OWNERS @@ -0,0 +1,6 @@ +# Bug component: 1322713 +inseob@google.com +jeongik@google.com +jiyong@google.com +justinyun@google.com +kiyoungkim@google.com diff --git a/target/product/generic/erofs_compress_hints.txt b/target/product/generic/erofs_compress_hints.txt new file mode 100644 index 0000000000..8b2a711b8f --- /dev/null +++ b/target/product/generic/erofs_compress_hints.txt @@ -0,0 +1 @@ +0 .*\.apex$
\ No newline at end of file diff --git a/target/product/generic_ramdisk.mk b/target/product/generic_ramdisk.mk index ebac62fd6b..32277ece03 100644 --- a/target/product/generic_ramdisk.mk +++ b/target/product/generic_ramdisk.mk @@ -23,6 +23,8 @@ PRODUCT_PACKAGES += \ init_first_stage \ snapuserd_ramdisk \ + ramdisk-build.prop \ + toolbox_ramdisk \ # Debug ramdisk PRODUCT_PACKAGES += \ @@ -35,8 +37,6 @@ PRODUCT_PACKAGES += \ _my_paths := \ $(TARGET_COPY_OUT_RAMDISK)/ \ $(TARGET_COPY_OUT_DEBUG_RAMDISK)/ \ - system/usr/share/zoneinfo/tz_version \ - system/usr/share/zoneinfo/tzdata \ $(TARGET_COPY_OUT_RECOVERY)/root/first_stage_ramdisk/system \ diff --git a/target/product/generic_system.mk b/target/product/generic_system.mk index 0a09eb11d4..2482afccc6 100644 --- a/target/product/generic_system.mk +++ b/target/product/generic_system.mk @@ -36,11 +36,6 @@ PRODUCT_PACKAGES += \ Stk \ Tag \ -ifeq ($(RELEASE_AVATAR_PICKER_APP),true) - PRODUCT_PACKAGES += \ - AvatarPicker -endif - # OTA support PRODUCT_PACKAGES += \ recovery-refresh \ @@ -152,4 +147,5 @@ _my_paths := \ $(call require-artifacts-in-path, $(_my_paths), $(_my_allowed_list)) # Product config map to toggle between sources and prebuilts of required mainline modules +PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard build/release/gms_mainline/required/release_config_map.textproto) PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/gms_mainline/required/release_config_map.textproto) diff --git a/target/product/go_defaults.mk b/target/product/go_defaults.mk index c9285307ab..ccc4f365e7 100644 --- a/target/product/go_defaults.mk +++ b/target/product/go_defaults.mk @@ -18,6 +18,7 @@ $(call inherit-product, build/make/target/product/go_defaults_common.mk) # Product config map to toggle between sources and prebuilts of required mainline modules +PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard build/release/gms_mainline_go/required/release_config_map.textproto) PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/gms_mainline_go/required/release_config_map.textproto) # Add the system properties. diff --git a/target/product/go_defaults_common.mk b/target/product/go_defaults_common.mk index fd4047a65b..0fcf16b753 100644 --- a/target/product/go_defaults_common.mk +++ b/target/product/go_defaults_common.mk @@ -24,11 +24,6 @@ PRODUCT_VENDOR_PROPERTIES += \ # Speed profile services and wifi-service to reduce RAM and storage. PRODUCT_SYSTEM_SERVER_COMPILER_FILTER := speed-profile -# Use a profile based boot image for this device. Note that this is currently a -# generic profile and not Android Go optimized. -PRODUCT_USE_PROFILE_FOR_BOOT_IMAGE := true -PRODUCT_DEX_PREOPT_BOOT_IMAGE_PROFILE_LOCATION := frameworks/base/config/boot-image-profile.txt - # Do not generate libartd. PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD := false diff --git a/target/product/gsi/Android.bp b/target/product/gsi/Android.bp index 45ba14331b..8c200a1dcb 100644 --- a/target/product/gsi/Android.bp +++ b/target/product/gsi/Android.bp @@ -46,3 +46,177 @@ install_symlink { installed_location: "etc/init/config", symlink_target: "/system/system_ext/etc/init/config", } + +// init.gsi.rc, GSI-specific init script. +prebuilt_etc { + name: "init.gsi.rc", + src: "init.gsi.rc", + system_ext_specific: true, + relative_install_path: "init", +} + +prebuilt_etc { + name: "init.vndk-nodef.rc", + src: "init.vndk-nodef.rc", + system_ext_specific: true, + relative_install_path: "gsi", +} + +gsi_symlinks = [ + { + target: "/system/system_ext", + name: "system_ext", + }, + { + target: "/system/product", + name: "product", + }, + { + target: "/odm/odm_dlkm/etc", + name: "odm_dlkm/etc", + }, + { + target: "/vendor/vendor_dlkm/etc", + name: "vendor_dlkm/etc", + }, +] + +android_filesystem_defaults { + name: "android_gsi_defaults", + defaults: [ + "system_image_defaults", + "system_ext_image_defaults", + "product_image_defaults", + ], + symlinks: gsi_symlinks, + dirs: ["cache"], + deps: [ + /////////////////////////////////////////// + // gsi_system_ext + /////////////////////////////////////////// + + // handheld packages + "Launcher3QuickStep", + "Provision", + "Settings", + "StorageManager", + "SystemUI", + + // telephony packages + "CarrierConfig", + + /////////////////////////////////////////// + // gsi_release + /////////////////////////////////////////// + "gsi_skip_mount.cfg", + "init.gsi.rc", + "init.vndk-nodef.rc", + // Overlay the GSI specific setting for framework and SystemUI + "gsi_overlay_framework", + "gsi_overlay_systemui", + + /////////////////////////////////////////// + // VNDK + /////////////////////////////////////////// + "com.android.vndk.v30", + "com.android.vndk.v31", + "com.android.vndk.v32", + "com.android.vndk.v33", + "com.android.vndk.v34", + + /////////////////////////////////////////// + // gsi_product + /////////////////////////////////////////// + "Browser2", + "Camera2", + "Dialer", + "LatinIME", + "apns-full-conf.xml", + "frameworks-base-overlays", + ], + multilib: { + lib64: { + deps: [ + /////////////////////////////////////////// + // AVF + /////////////////////////////////////////// + "com.android.compos", + "features_com.android.virt.xml", + ], + }, + both: { + // PRODUCT_PACKAGES_SHIPPING_API_LEVEL_34 + deps: ["android.hidl.memory@1.0-impl"], + }, + }, + type: "ext4", +} + +// system.img for gsi_{arch} targets +android_system_image { + name: "android_gsi", + defaults: ["android_gsi_defaults"], + enabled: select(soong_config_variable("ANDROID", "PRODUCT_INSTALL_DEBUG_POLICY_TO_SYSTEM_EXT"), { + "true": true, + default: false, + }), + deps: [ + // Install a copy of the debug policy to the system_ext partition, and allow + // init-second-stage to load debug policy from system_ext. + // This option is only meant to be set by compliance GSI targets. + "system_ext_userdebug_plat_sepolicy.cil", + ], +} + +// system.img for aosp_{arch} targets +android_system_image { + name: "aosp_system_image", + defaults: ["android_gsi_defaults"], + deps: [ + // handheld_system_ext + "AccessibilityMenu", + "WallpaperCropper", + + // telephony_system_ext + "EmergencyInfo", + + // handheld_product + "Calendar", + "Contacts", + "DeskClock", + "Gallery2", + "Music", + "preinstalled-packages-platform-handheld-product.xml", + "QuickSearchBox", + "SettingsIntelligence", + "frameworks-base-overlays", + + // telephony_product + "ImsServiceEntitlement", + "preinstalled-packages-platform-telephony-product.xml", + + // more AOSP packages + "initial-package-stopped-states-aosp.xml", + "messaging", + "PhotoTable", + "preinstalled-packages-platform-aosp-product.xml", + "ThemePicker", + ] + select(product_variable("debuggable"), { + true: ["frameworks-base-overlays-debug"], + default: [], + }), + enabled: select(soong_config_variable("gsi", "building_gsi"), { + true: true, + default: false, + }), + multilib: { + common: { + deps: select(release_flag("RELEASE_AVATAR_PICKER_APP"), { + true: [ + "AvatarPicker", // handheld_system_ext (RELEASE_AVATAR_PICKER_APP) + ], + default: [], + }), + }, + }, +} diff --git a/target/product/gsi/current.txt b/target/product/gsi/current.txt index f771916f7a..cbb8a0e8cd 100644 --- a/target/product/gsi/current.txt +++ b/target/product/gsi/current.txt @@ -24,7 +24,7 @@ LLNDK: libvulkan.so VNDK-SP: android.hardware.common-V2-ndk.so VNDK-SP: android.hardware.common.fmq-V1-ndk.so VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so -VNDK-SP: android.hardware.graphics.common-V5-ndk.so +VNDK-SP: android.hardware.graphics.common-V6-ndk.so VNDK-SP: android.hardware.graphics.common@1.0.so VNDK-SP: android.hardware.graphics.common@1.1.so VNDK-SP: android.hardware.graphics.common@1.2.so diff --git a/target/product/gsi_release.mk b/target/product/gsi_release.mk index 39428d2cfe..115b355920 100644 --- a/target/product/gsi_release.mk +++ b/target/product/gsi_release.mk @@ -79,9 +79,15 @@ PRODUCT_BUILD_SUPER_EMPTY_IMAGE := false PRODUCT_BUILD_SYSTEM_DLKM_IMAGE := false PRODUCT_EXPORT_BOOT_IMAGE_TO_DIST := true +# Build pvmfw with GSI: b/376363989, pvmfw currently only supports AArch64 +ifneq (,$(filter %_arm64,$(TARGET_PRODUCT))) +PRODUCT_BUILD_PVMFW_IMAGE := true +endif + # Additional settings used in all GSI builds PRODUCT_PRODUCT_PROPERTIES += \ ro.crypto.metadata_init_delete_all_keys.enabled=false \ + debug.codec2.bqpool_dealloc_after_stop=1 \ # Window Extensions ifneq ($(PRODUCT_IS_ATV),true) diff --git a/target/product/handheld_system.mk b/target/product/handheld_system.mk index 546bbe7a4a..2b055c7ed0 100644 --- a/target/product/handheld_system.mk +++ b/target/product/handheld_system.mk @@ -69,13 +69,19 @@ PRODUCT_PACKAGES += \ SharedStorageBackup \ SimAppDialog \ Telecom \ - TelephonyProvider \ TeleService \ Traceur \ UserDictionaryProvider \ VpnDialogs \ vr \ +# Choose the correct products based on HSUM status +ifeq ($(PRODUCT_USE_HSUM),true) + PRODUCT_PACKAGES += TelephonyProviderHsum +else + PRODUCT_PACKAGES += TelephonyProvider +endif + PRODUCT_PACKAGES += $(RELEASE_PACKAGE_VIRTUAL_CAMERA) # Set virtual_camera_service_enabled soong config variable based on the # RELEASE_PACKAGE_VIRTUAL_CAMERA build. virtual_camera_service_enabled soong config diff --git a/target/product/handheld_system_ext.mk b/target/product/handheld_system_ext.mk index 187b6275bb..6d686c554f 100644 --- a/target/product/handheld_system_ext.mk +++ b/target/product/handheld_system_ext.mk @@ -23,6 +23,7 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/media_system_ext.mk) # /system_ext packages PRODUCT_PACKAGES += \ AccessibilityMenu \ + $(if $(RELEASE_AVATAR_PICKER_APP), AvatarPicker,) \ Launcher3QuickStep \ Provision \ Settings \ diff --git a/target/product/hsum_common.mk b/target/product/hsum_common.mk new file mode 100644 index 0000000000..b19bc65c90 --- /dev/null +++ b/target/product/hsum_common.mk @@ -0,0 +1,29 @@ +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Contains common default elements for devices running in Headless System User Mode. + +# Should generally be inherited first as using an HSUM configuration can affect downstream choices +# (such as ensuring that the HSUM-variants of packages are selected). + +PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \ + ro.fw.mu.headless_system_user=true + +# Variable for elsewhere choosing the appropriate products based on HSUM status. +PRODUCT_USE_HSUM := true + +PRODUCT_PACKAGES += \ + HsumDefaultConfigOverlay diff --git a/target/product/media_system_ext.mk b/target/product/media_system_ext.mk index 30dd2e2711..1179966777 100644 --- a/target/product/media_system_ext.mk +++ b/target/product/media_system_ext.mk @@ -20,9 +20,10 @@ # base_system_ext.mk. $(call inherit-product, $(SRC_TARGET_DIR)/product/base_system_ext.mk) -# /system_ext packages -PRODUCT_PACKAGES += \ - vndk_apex_snapshot_package \ - # Window Extensions $(call inherit-product, $(SRC_TARGET_DIR)/product/window_extensions_base.mk) + +# AppFunction Extensions +ifneq (,$(RELEASE_APPFUNCTION_SIDECAR)) + $(call inherit-product, $(SRC_TARGET_DIR)/product/app_function_extensions.mk) +endif diff --git a/target/product/runtime_libart.mk b/target/product/runtime_libart.mk index 9e8afa85a4..71138ac560 100644 --- a/target/product/runtime_libart.mk +++ b/target/product/runtime_libart.mk @@ -142,6 +142,7 @@ ifneq (,$(filter true,$(OVERRIDE_DISABLE_DEXOPT_ALL))) # be too much of a problem for platform developers because a change to framework code should not # trigger dexpreopt for the ART boot image. WITH_DEXPREOPT_ART_BOOT_IMG_ONLY := true + $(call soong_config_set_bool,PrebuiltGmsCore,ExcludeExtractApk,true) endif # Enable resolution of startup const strings. @@ -157,15 +158,14 @@ PRODUCT_SYSTEM_PROPERTIES += \ dalvik.vm.minidebuginfo=true \ dalvik.vm.dex2oat-minidebuginfo=true -# Enable Madvising of the whole art, odex and vdex files to MADV_WILLNEED. +# Enable Madvising of the whole odex and vdex files to MADV_WILLNEED. # The size specified here is the size limit of how much of the file # (in bytes) is madvised. -# We madvise the whole .art file to MADV_WILLNEED with UINT_MAX limit. # For odex and vdex files, we limit madvising to 100MB. +# For art files, we defer to the runtime for default behavior. PRODUCT_SYSTEM_PROPERTIES += \ dalvik.vm.madvise.vdexfile.size=104857600 \ - dalvik.vm.madvise.odexfile.size=104857600 \ - dalvik.vm.madvise.artfile.size=4294967295 + dalvik.vm.madvise.odexfile.size=104857600 # Properties for the Unspecialized App Process Pool PRODUCT_SYSTEM_PROPERTIES += \ diff --git a/target/product/security/Android.bp b/target/product/security/Android.bp index 0d7b35e1c9..214c009ec8 100644 --- a/target/product/security/Android.bp +++ b/target/product/security/Android.bp @@ -33,7 +33,18 @@ prebuilt_etc { // image otacerts_zip { name: "otacerts", - recovery_available: true, relative_install_path: "security", filename: "otacerts.zip", } + +otacerts_zip { + name: "otacerts.recovery", + recovery: true, + relative_install_path: "security", + filename: "otacerts.zip", +} + +adb_keys { + name: "adb_keys", + product_specific: true, +} diff --git a/target/product/security/Android.mk b/target/product/security/Android.mk deleted file mode 100644 index 138e5bbe31..0000000000 --- a/target/product/security/Android.mk +++ /dev/null @@ -1,17 +0,0 @@ -LOCAL_PATH:= $(call my-dir) - -####################################### -# adb key, if configured via PRODUCT_ADB_KEYS -ifdef PRODUCT_ADB_KEYS - ifneq ($(filter eng userdebug,$(TARGET_BUILD_VARIANT)),) - include $(CLEAR_VARS) - LOCAL_MODULE := adb_keys - LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 - LOCAL_LICENSE_CONDITIONS := notice - LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE - LOCAL_MODULE_CLASS := ETC - LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT_ETC)/security - LOCAL_PREBUILT_MODULE_FILE := $(PRODUCT_ADB_KEYS) - include $(BUILD_PREBUILT) - endif -endif diff --git a/target/product/security/BUILD.bazel b/target/product/security/BUILD.bazel deleted file mode 100644 index c12be79833..0000000000 --- a/target/product/security/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -filegroup( - name = "android_certificate_directory", - srcs = glob([ - "*.pk8", - "*.pem", - ]), - visibility = ["//visibility:public"], -) diff --git a/target/product/virtual_ab_ota/compression.mk b/target/product/virtual_ab_ota/compression.mk index dc1ee3e028..e77c36fb78 100644 --- a/target/product/virtual_ab_ota/compression.mk +++ b/target/product/virtual_ab_ota/compression.mk @@ -18,9 +18,12 @@ $(call inherit-product, $(SRC_TARGET_DIR)/product/virtual_ab_ota/launch_with_ven PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.enabled=true PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.userspace.snapshots.enabled=true -PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.io_uring.enabled=true PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.batch_writes=true +# Optional assignment. On low memory devices, disabling io_uring can relieve cpu and memory +# pressure during an OTA. +PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.io_uring.enabled?=true + # Enabling this property, will improve OTA install time # but will use an additional CPU core # PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.threads=true diff --git a/target/product/virtual_ab_ota/vabc_features.mk b/target/product/virtual_ab_ota/vabc_features.mk index e2745a1356..d092699a47 100644 --- a/target/product/virtual_ab_ota/vabc_features.mk +++ b/target/product/virtual_ab_ota/vabc_features.mk @@ -31,14 +31,15 @@ PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.enabled=true PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.enabled=true PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.userspace.snapshots.enabled=true -PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.io_uring.enabled=true -PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.xor.enabled=true PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.batch_writes=true + +# Optional assignments, low memory devices may benefit from overriding these. +PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.io_uring.enabled?=true +PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.compression.xor.enabled?=true + # Low memory device configurations. If memory usage and cpu utilization is # a bottleneck during OTA, the below configurations can be added to a -# device's .mk file improve performance for low mem devices. Disabling -# ro.virtual_ab.compression.xor.enabled and ro.virtual_ab.io_uring.enabled -# is also recommended +# device's .mk file improve performance for low mem devices. # # PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.read_ahead_size=16 # PRODUCT_VENDOR_PROPERTIES += ro.virtual_ab.o_direct.enabled=true diff --git a/teams/Android.bp b/teams/Android.bp index 94585fcb62..8689267461 100644 --- a/teams/Android.bp +++ b/teams/Android.bp @@ -13,6 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// DON'T ADD NEW RULES HERE. For more details refer to +// go/new-android-ownership-model + package { default_applicable_licenses: ["Android-Apache-2.0"], } @@ -67,13 +70,6 @@ team { } team { - name: "trendy_team_wear_wear_cloud_platform", - - // go/trendy/manage/engineers/5917762526281728 - trendy_team_id: "5917762526281728", -} - -team { name: "trendy_team_pixel_system_software", // go/trendy/manage/engineers/4856005120622592 @@ -515,13 +511,6 @@ team { } team { - name: "trendy_team_wear_wear_notifications_alerts_attention_management", - - // go/trendy/manage/engineers/6267643681996800 - trendy_team_id: "6267643681996800", -} - -team { name: "trendy_team_fwk_nfc", // go/trendy/manage/engineers/5962312512864256 @@ -529,13 +518,6 @@ team { } team { - name: "trendy_team_wear_personalization_developer_surfaces", - - // go/trendy/manage/engineers/4819890988810240 - trendy_team_id: "4819890988810240", -} - -team { name: "trendy_team_srajkumar_team", // go/trendy/manage/engineers/5170053894012928 @@ -690,13 +672,6 @@ team { } team { - name: "trendy_team_test_eng_android_wear", - - // go/trendy/manage/engineers/4979150422933504 - trendy_team_id: "4979150422933504", -} - -team { name: "trendy_team_mesch_team", // go/trendy/manage/engineers/5205465899368448 @@ -718,13 +693,6 @@ team { } team { - name: "trendy_team_wear_wear_developer_devx", - - // go/trendy/manage/engineers/4894890764697600 - trendy_team_id: "4894890764697600", -} - -team { name: "trendy_team_android_rust", // go/trendy/manage/engineers/4844600586305536 @@ -928,13 +896,6 @@ team { } team { - name: "trendy_team_wear_wallet_on_wear", - - // go/trendy/manage/engineers/5724960437731328 - trendy_team_id: "5724960437731328", -} - -team { name: "trendy_team_glanceables", // go/trendy/manage/engineers/4658222004600832 @@ -970,7 +931,7 @@ team { } team { - name: "trendy_team_camerax", + name: "trendy_team_android_camera_innovation_team", // go/trendy/manage/engineers/5272590669479936 trendy_team_id: "5272590669479936", @@ -1068,13 +1029,6 @@ team { } team { - name: "trendy_team_wear_3xp", - - // go/trendy/manage/engineers/5692317612539904 - trendy_team_id: "5692317612539904", -} - -team { name: "trendy_team_clockwork", // go/trendy/manage/engineers/4908781678755840 @@ -1208,13 +1162,6 @@ team { } team { - name: "trendy_team_wear_software_nti", - - // go/trendy/manage/engineers/5164973558759424 - trendy_team_id: "5164973558759424", -} - -team { name: "trendy_team_machine_learning", // go/trendy/manage/engineers/5276568318246912 @@ -1306,13 +1253,6 @@ team { } team { - name: "trendy_team_wear_wear_power_emulator", - - // go/trendy/manage/engineers/5160338936725504 - trendy_team_id: "5160338936725504", -} - -team { name: "trendy_team_deprecated_framework_svetoslavganov", // go/trendy/manage/engineers/6404117492531200 @@ -1327,13 +1267,6 @@ team { } team { - name: "trendy_team_wear_opus", - - // go/trendy/manage/engineers/5098351636676608 - trendy_team_id: "5098351636676608", -} - -team { name: "trendy_team_text_to_speech", // go/trendy/manage/engineers/6368933120442368 @@ -1439,13 +1372,6 @@ team { } team { - name: "trendy_team_wear_developer_foundation", - - // go/trendy/manage/engineers/5239127108648960 - trendy_team_id: "5239127108648960", -} - -team { name: "trendy_team_tpm_tvc", // go/trendy/manage/engineers/5390683333230592 @@ -1453,13 +1379,6 @@ team { } team { - name: "trendy_team_wear_wear_ux", - - // go/trendy/manage/engineers/5782097411080192 - trendy_team_id: "5782097411080192", -} - -team { name: "trendy_team_lse_desktop_os_experience", // go/trendy/manage/engineers/5125234900434944 @@ -1670,13 +1589,6 @@ team { } team { - name: "trendy_team_wear_wear_assistant", - - // go/trendy/manage/engineers/5848075306172416 - trendy_team_id: "5848075306172416", -} - -team { name: "trendy_team_android_power_and_comms_infra", // go/trendy/manage/engineers/5325547653332992 @@ -3455,6 +3367,13 @@ team { } team { + name: "trendy_team_aaos_display_safety_triage", + + // go/trendy/manage/engineers/6522093663780864 + trendy_team_id: "6522093663780864", +} + +team { name: "trendy_team_camera_htc_lg_qualcomm", // go/trendy/manage/engineers/6332099480911872 @@ -4440,3 +4359,55 @@ team { // go/trendy/manage/engineers/5097003746426880 trendy_team_id: "5097003746426880", } + +team { + name: "trendy_team_desktop_firmware", + + // go/trendy/manage/engineers/5787938454863872 + trendy_team_id: "5787938454863872", +} + +team { + name: "trendy_team_art_cloud", + + // go/trendy/manage/engineers/5121440647577600 + trendy_team_id: "5121440647577600", +} + +team { + name: "trendy_team_ravenwood", + + // go/trendy/manage/engineers/6027181500497920 + trendy_team_id: "6027181500497920", +} + +team { + name: "trendy_team_automotive_cast", + + // go/trendy/manage/engineers/5293683026264064 + trendy_team_id: "5293683026264064", +} + +team { + name: "trendy_team_wear_standalone_kids", + + // go/trendy/manage/engineers/6303298703949824 + trendy_team_id: "6303298703949824", +} + +team { + name: "trendy_team_desktop_stats", + + // go/trendy/manage/engineers/5440764114206720 + trendy_team_id: "5440764114206720", +} + +team { + name: "trendy_team_desktop_wifi", + + // go/trendy/manage/engineers/6463689697099776 + trendy_team_id: "6463689697099776", +} + +// DON'T ADD NEW RULES HERE. For more details refer to +// go/new-android-ownership-model diff --git a/teams/OWNERS b/teams/OWNERS index 85e69f356b..02846eb16a 100644 --- a/teams/OWNERS +++ b/teams/OWNERS @@ -1,3 +1,2 @@ dariofreni@google.com ronish@google.com -caditya@google.com diff --git a/tools/Android.bp b/tools/Android.bp index 59831a61ec..243cb5647b 100644 --- a/tools/Android.bp +++ b/tools/Android.bp @@ -123,3 +123,11 @@ python_binary_host { "merge-event-log-tags.py", ], } + +python_binary_host { + name: "java-event-log-tags", + srcs: [ + "event_log_tags.py", + "java-event-log-tags.py", + ], +} diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel deleted file mode 100644 index 9ec0dcef85..0000000000 --- a/tools/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -py_library( - name = "event_log_tags", - srcs = ["event_log_tags.py"], - imports = ["."], -) - -py_binary( - name = "java-event-log-tags", - srcs = ["java-event-log-tags.py"], - python_version = "PY3", - visibility = ["//visibility:public"], - deps = [":event_log_tags"], -) - -py_binary( - name = "merge-event-log-tags", - srcs = ["merge-event-log-tags.py"], - python_version = "PY3", - visibility = ["//visibility:public"], - deps = [":event_log_tags"], -) - -py_binary( - name = "check_elf_file", - srcs = ["check_elf_file.py"], - python_version = "PY3", - visibility = ["//visibility:public"], -) - -py_binary( - name = "auto_gen_test_config", - srcs = ["auto_gen_test_config.py"], - python_version = "PY3", - visibility = ["//visibility:public"], -) diff --git a/tools/aconfig/Cargo.toml b/tools/aconfig/Cargo.toml index bf5e1a9bc4..a031b7f928 100644 --- a/tools/aconfig/Cargo.toml +++ b/tools/aconfig/Cargo.toml @@ -8,7 +8,8 @@ members = [ "aconfig_storage_read_api", "aconfig_storage_write_api", "aflags", - "printflags" + "printflags", + "convert_finalized_flags" ] resolver = "2" diff --git a/tools/aconfig/OWNERS b/tools/aconfig/OWNERS index 9a76279cce..c92fc7cda3 100644 --- a/tools/aconfig/OWNERS +++ b/tools/aconfig/OWNERS @@ -1,7 +1,8 @@ -amhk@google.com dzshen@google.com -jham@google.com -joeo@google.com opg@google.com tedbauer@google.com zhidou@google.com + +amhk@google.com #{LAST_RESORT_SUGGESTION} +jham@google.com #{LAST_RESORT_SUGGESTION} +joeo@google.com #{LAST_RESORT_SUGGESTION} diff --git a/tools/aconfig/TEST_MAPPING b/tools/aconfig/TEST_MAPPING index 15e41876cf..6e53018a41 100644 --- a/tools/aconfig/TEST_MAPPING +++ b/tools/aconfig/TEST_MAPPING @@ -102,12 +102,10 @@ { // aconfig_storage file java integration tests "name": "aconfig_storage_file.test.java" - } - ], - "postsubmit": [ + }, { - // aconfig_storage read api java integration tests - "name": "aconfig_storage_read_api.test.java" + // aconfig_storage read functional test + "name": "aconfig_storage_read_functional" } ] } diff --git a/tools/aconfig/aconfig/Android.bp b/tools/aconfig/aconfig/Android.bp index 68521af91f..7bdec58004 100644 --- a/tools/aconfig/aconfig/Android.bp +++ b/tools/aconfig/aconfig/Android.bp @@ -7,7 +7,10 @@ rust_defaults { edition: "2021", clippy_lints: "android", lints: "android", - srcs: ["src/main.rs"], + srcs: [ + "src/main.rs", + ":finalized_flags_record.json", + ], rustlibs: [ "libaconfig_protos", "libaconfig_storage_file", @@ -18,6 +21,7 @@ rust_defaults { "libserde", "libserde_json", "libtinytemplate", + "libconvert_finalized_flags", ], } @@ -68,6 +72,14 @@ aconfig_values { ], } +aconfig_values { + name: "aconfig.test.flag.second_values", + package: "com.android.aconfig.test", + srcs: [ + "tests/third.values", + ], +} + aconfig_value_set { name: "aconfig.test.flag.value_set", values: [ @@ -234,6 +246,12 @@ rust_aconfig_library { name: "libaconfig_test_rust_library", crate_name: "aconfig_test_rust_library", aconfig_declarations: "aconfig.test.flags", + host_supported: true, + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } rust_test { diff --git a/tools/aconfig/aconfig/Cargo.toml b/tools/aconfig/aconfig/Cargo.toml index abd3ee01e8..7e4bdf2f7d 100644 --- a/tools/aconfig/aconfig/Cargo.toml +++ b/tools/aconfig/aconfig/Cargo.toml @@ -17,3 +17,11 @@ serde_json = "1.0.93" tinytemplate = "1.2.1" aconfig_protos = { path = "../aconfig_protos" } aconfig_storage_file = { path = "../aconfig_storage_file" } +convert_finalized_flags = { path = "../convert_finalized_flags" } + +[build-dependencies] +anyhow = "1.0.69" +itertools = "0.10.5" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +convert_finalized_flags = { path = "../convert_finalized_flags" } diff --git a/tools/aconfig/aconfig/build.rs b/tools/aconfig/aconfig/build.rs new file mode 100644 index 0000000000..8aaec3c43b --- /dev/null +++ b/tools/aconfig/aconfig/build.rs @@ -0,0 +1,93 @@ +use anyhow::{anyhow, Result}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use convert_finalized_flags::read_files_to_map_using_path; +use convert_finalized_flags::FinalizedFlagMap; + +// This fn makes assumptions about the working directory which we should not rely +// on for actual (Soong) builds. It is reasonable to assume that this is being +// called from the aconfig directory as cargo is used for local development and +// the cargo workspace for our project is build/make/tools/aconfig. +// This is meant to get the list of finalized flag +// files provided by the filegroup + "locations" in soong. +// Cargo-only usage is asserted via implementation of +// read_files_to_map_using_env, the only public cargo-only fn. +fn read_files_to_map_using_env() -> Result<FinalizedFlagMap> { + let mut current_dir = std::env::current_dir()?; + + // Path of aconfig from the top of tree. + let aconfig_path = PathBuf::from("build/make/tools/aconfig"); + + // Path of SDK files from the top of tree. + let sdk_dir_path = PathBuf::from("prebuilts/sdk"); + + // Iterate up the directory structure until we have the base aconfig dir. + while !current_dir.canonicalize()?.ends_with(&aconfig_path) { + if let Some(parent) = current_dir.parent() { + current_dir = parent.to_path_buf(); + } else { + return Err(anyhow!("Cannot execute outside of aconfig.")); + } + } + + // Remove the aconfig path, leaving the top of the tree. + for _ in 0..aconfig_path.components().count() { + current_dir.pop(); + } + + // Get the absolute path of the sdk files. + current_dir.push(sdk_dir_path); + + let mut flag_files = Vec::new(); + + // Search all sub-dirs in prebuilts/sdk for finalized-flags.txt files. + // The files are in prebuilts/sdk/<api level>/finalized-flags.txt. + let api_level_dirs = fs::read_dir(current_dir)?; + for api_level_dir in api_level_dirs { + if api_level_dir.is_err() { + eprintln!("Error opening directory: {}", api_level_dir.err().unwrap()); + continue; + } + + // Skip non-directories. + let api_level_dir_path = api_level_dir.unwrap().path(); + if !api_level_dir_path.is_dir() { + continue; + } + + // Some directories were created before trunk stable and don't have + // flags, or aren't api level directories at all. + let flag_file_path = api_level_dir_path.join("finalized-flags.txt"); + if !flag_file_path.exists() { + continue; + } + + if let Some(path) = flag_file_path.to_str() { + flag_files.push(path.to_string()); + } else { + eprintln!("Error converting path to string: {:?}", flag_file_path); + } + } + + read_files_to_map_using_path(flag_files) +} + +fn main() { + let out_dir = env::var_os("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("finalized_flags_record.json"); + + let finalized_flags_map: Result<FinalizedFlagMap> = read_files_to_map_using_env(); + if finalized_flags_map.is_err() { + return; + } + let json_str = serde_json::to_string(&finalized_flags_map.unwrap()).unwrap(); + + let mut f = File::create(&dest_path).unwrap(); + f.write_all(json_str.as_bytes()).unwrap(); + + //println!("cargo:rerun-if-changed=input.txt"); +} diff --git a/tools/aconfig/aconfig/src/codegen/cpp.rs b/tools/aconfig/aconfig/src/codegen/cpp.rs index 2c569da8f6..b855d78602 100644 --- a/tools/aconfig/aconfig/src/codegen/cpp.rs +++ b/tools/aconfig/aconfig/src/codegen/cpp.rs @@ -24,14 +24,13 @@ use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag}; use crate::codegen; use crate::codegen::CodegenMode; -use crate::commands::OutputFile; +use crate::commands::{should_include_flag, OutputFile}; pub fn generate_cpp_code<I>( package: &str, parsed_flags_iter: I, codegen_mode: CodegenMode, flag_ids: HashMap<String, u16>, - allow_instrumentation: bool, ) -> Result<Vec<OutputFile>> where I: Iterator<Item = ProtoParsedFlag>, @@ -59,7 +58,6 @@ where is_test_mode: codegen_mode == CodegenMode::Test, class_elements, container, - allow_instrumentation, }; let files = [ @@ -104,7 +102,6 @@ pub struct Context<'a> { pub is_test_mode: bool, pub class_elements: Vec<ClassElement>, pub container: String, - pub allow_instrumentation: bool, } #[derive(Serialize)] @@ -127,6 +124,23 @@ fn create_class_element( flag_ids: HashMap<String, u16>, rw_count: &mut i32, ) -> ClassElement { + let no_assigned_offset = !should_include_flag(pf); + + let flag_offset = match flag_ids.get(pf.name()) { + Some(offset) => offset, + None => { + // System/vendor/product RO+disabled flags have no offset in storage files. + // Assign placeholder value. + if no_assigned_offset { + &0 + } + // All other flags _must_ have an offset. + else { + panic!("{}", format!("missing flag offset for {}", pf.name())); + } + } + }; + ClassElement { readwrite_idx: if pf.permission() == ProtoFlagPermission::READ_WRITE { let index = *rw_count; @@ -144,7 +158,7 @@ fn create_class_element( }, flag_name: pf.name().to_string(), flag_macro: pf.name().to_uppercase(), - flag_offset: *flag_ids.get(pf.name()).expect("values checked at flag parse time"), + flag_offset: *flag_offset, device_config_namespace: pf.namespace().to_string(), device_config_flag: codegen::create_device_config_ident(package, pf.name()) .expect("values checked at flag parse time"), @@ -220,11 +234,11 @@ inline bool disabled_rw_in_other_namespace() { return provider_->disabled_rw_in_other_namespace(); } -inline bool enabled_fixed_ro() { +constexpr inline bool enabled_fixed_ro() { return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; } -inline bool enabled_fixed_ro_exported() { +constexpr inline bool enabled_fixed_ro_exported() { return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO_EXPORTED; } @@ -283,39 +297,23 @@ public: virtual ~flag_provider_interface() = default; virtual bool disabled_ro() = 0; - - virtual void disabled_ro(bool val) = 0; - virtual bool disabled_rw() = 0; - - virtual void disabled_rw(bool val) = 0; - virtual bool disabled_rw_exported() = 0; - - virtual void disabled_rw_exported(bool val) = 0; - virtual bool disabled_rw_in_other_namespace() = 0; - - virtual void disabled_rw_in_other_namespace(bool val) = 0; - virtual bool enabled_fixed_ro() = 0; - - virtual void enabled_fixed_ro(bool val) = 0; - virtual bool enabled_fixed_ro_exported() = 0; - - virtual void enabled_fixed_ro_exported(bool val) = 0; - virtual bool enabled_ro() = 0; - - virtual void enabled_ro(bool val) = 0; - virtual bool enabled_ro_exported() = 0; - - virtual void enabled_ro_exported(bool val) = 0; - virtual bool enabled_rw() = 0; + virtual void disabled_ro(bool val) = 0; + virtual void disabled_rw(bool val) = 0; + virtual void disabled_rw_exported(bool val) = 0; + virtual void disabled_rw_in_other_namespace(bool val) = 0; + virtual void enabled_fixed_ro(bool val) = 0; + virtual void enabled_fixed_ro_exported(bool val) = 0; + virtual void enabled_ro(bool val) = 0; + virtual void enabled_ro_exported(bool val) = 0; virtual void enabled_rw(bool val) = 0; virtual void reset_flags() {} @@ -450,56 +448,6 @@ void com_android_aconfig_test_reset_flags(); "#; - const EXPORTED_EXPORTED_HEADER_EXPECTED: &str = r#" -#pragma once - -#ifdef __cplusplus - -#include <memory> - -namespace com::android::aconfig::test { - -class flag_provider_interface { -public: - virtual ~flag_provider_interface() = default; - - virtual bool disabled_rw_exported() = 0; - - virtual bool enabled_fixed_ro_exported() = 0; - - virtual bool enabled_ro_exported() = 0; -}; - -extern std::unique_ptr<flag_provider_interface> provider_; - -inline bool disabled_rw_exported() { - return provider_->disabled_rw_exported(); -} - -inline bool enabled_fixed_ro_exported() { - return provider_->enabled_fixed_ro_exported(); -} - -inline bool enabled_ro_exported() { - return provider_->enabled_ro_exported(); -} - -} - -extern "C" { -#endif // __cplusplus - -bool com_android_aconfig_test_disabled_rw_exported(); - -bool com_android_aconfig_test_enabled_fixed_ro_exported(); - -bool com_android_aconfig_test_enabled_ro_exported(); - -#ifdef __cplusplus -} // extern "C" -#endif -"#; - const EXPORTED_FORCE_READ_ONLY_HEADER_EXPECTED: &str = r#" #pragma once @@ -548,7 +496,7 @@ inline bool disabled_rw_in_other_namespace() { return false; } -inline bool enabled_fixed_ro() { +constexpr inline bool enabled_fixed_ro() { return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; } @@ -584,7 +532,13 @@ bool com_android_aconfig_test_enabled_rw(); const PROD_SOURCE_FILE_EXPECTED: &str = r#" #include "com_android_aconfig_test.h" -#include <server_configurable_flags/get_flags.h> + +#include <unistd.h> +#include "aconfig_storage/aconfig_storage_read_api.hpp" +#include <android/log.h> +#define LOG_TAG "aconfig_cpp_codegen" +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + #include <vector> namespace com::android::aconfig::test { @@ -592,36 +546,116 @@ namespace com::android::aconfig::test { class flag_provider : public flag_provider_interface { public: + flag_provider() + : cache_(4, -1) + , boolean_start_index_() + , flag_value_file_(nullptr) + , package_exists_in_storage_(true) { + + auto package_map_file = aconfig_storage::get_mapped_file( + "system", + aconfig_storage::StorageFileType::package_map); + if (!package_map_file.ok()) { + ALOGE("error: failed to get package map file: %s", package_map_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + auto context = aconfig_storage::get_package_read_context( + **package_map_file, "com.android.aconfig.test"); + if (!context.ok()) { + ALOGE("error: failed to get package read context: %s", context.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + if (!(context->package_exists)) { + package_exists_in_storage_ = false; + return; + } + + // cache package boolean flag start index + boolean_start_index_ = context->boolean_start_index; + + // unmap package map file and free memory + delete *package_map_file; + + auto flag_value_file = aconfig_storage::get_mapped_file( + "system", + aconfig_storage::StorageFileType::flag_val); + if (!flag_value_file.ok()) { + ALOGE("error: failed to get flag value file: %s", flag_value_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + // cache flag value file + flag_value_file_ = std::unique_ptr<aconfig_storage::MappedStorageFile>( + *flag_value_file); + + } + + virtual bool disabled_ro() override { return false; } virtual bool disabled_rw() override { if (cache_[0] == -1) { - cache_[0] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 0); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } + + cache_[0] = *value; } return cache_[0]; } virtual bool disabled_rw_exported() override { if (cache_[1] == -1) { - cache_[1] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 1); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } + + cache_[1] = *value; } return cache_[1]; } virtual bool disabled_rw_in_other_namespace() override { if (cache_[2] == -1) { - cache_[2] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.other_namespace", - "com.android.aconfig.test.disabled_rw_in_other_namespace", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 2); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } + + cache_[2] = *value; } return cache_[2]; } @@ -644,16 +678,32 @@ namespace com::android::aconfig::test { virtual bool enabled_rw() override { if (cache_[3] == -1) { - cache_[3] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_rw", - "true") == "true"; + if (!package_exists_in_storage_) { + return true; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 7); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return true; + } + + cache_[3] = *value; } return cache_[3]; } private: std::vector<int8_t> cache_ = std::vector<int8_t>(4, -1); + + uint32_t boolean_start_index_; + + std::unique_ptr<aconfig_storage::MappedStorageFile> flag_value_file_; + + bool package_exists_in_storage_; }; std::unique_ptr<flag_provider_interface> provider_ = @@ -700,7 +750,13 @@ bool com_android_aconfig_test_enabled_rw() { const TEST_SOURCE_FILE_EXPECTED: &str = r#" #include "com_android_aconfig_test.h" -#include <server_configurable_flags/get_flags.h> + +#include <unistd.h> +#include "aconfig_storage/aconfig_storage_read_api.hpp" +#include <android/log.h> +#define LOG_TAG "aconfig_cpp_codegen" +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + #include <unordered_map> #include <string> @@ -710,10 +766,63 @@ namespace com::android::aconfig::test { private: std::unordered_map<std::string, bool> overrides_; + uint32_t boolean_start_index_; + + std::unique_ptr<aconfig_storage::MappedStorageFile> flag_value_file_; + + bool package_exists_in_storage_; + public: flag_provider() : overrides_() - {} + , boolean_start_index_() + , flag_value_file_(nullptr) + , package_exists_in_storage_(true) { + + auto package_map_file = aconfig_storage::get_mapped_file( + "system", + aconfig_storage::StorageFileType::package_map); + + if (!package_map_file.ok()) { + ALOGE("error: failed to get package map file: %s", package_map_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + auto context = aconfig_storage::get_package_read_context( + **package_map_file, "com.android.aconfig.test"); + + if (!context.ok()) { + ALOGE("error: failed to get package read context: %s", context.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + if (!(context->package_exists)) { + package_exists_in_storage_ = false; + return; + } + + // cache package boolean flag start index + boolean_start_index_ = context->boolean_start_index; + + // unmap package map file and free memory + delete *package_map_file; + + auto flag_value_file = aconfig_storage::get_mapped_file( + "system", + aconfig_storage::StorageFileType::flag_val); + if (!flag_value_file.ok()) { + ALOGE("error: failed to get flag value file: %s", flag_value_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + // cache flag value file + flag_value_file_ = std::unique_ptr<aconfig_storage::MappedStorageFile>( + *flag_value_file); + + } virtual bool disabled_ro() override { auto it = overrides_.find("disabled_ro"); @@ -733,10 +842,20 @@ namespace com::android::aconfig::test { if (it != overrides_.end()) { return it->second; } else { - return server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 0); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } else { + return *value; + } } } @@ -749,10 +868,20 @@ namespace com::android::aconfig::test { if (it != overrides_.end()) { return it->second; } else { - return server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 1); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } else { + return *value; + } } } @@ -765,10 +894,20 @@ namespace com::android::aconfig::test { if (it != overrides_.end()) { return it->second; } else { - return server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.other_namespace", - "com.android.aconfig.test.disabled_rw_in_other_namespace", - "false") == "true"; + if (!package_exists_in_storage_) { + return false; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 2); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return false; + } else { + return *value; + } } } @@ -833,10 +972,20 @@ namespace com::android::aconfig::test { if (it != overrides_.end()) { return it->second; } else { - return server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_rw", - "true") == "true"; + if (!package_exists_in_storage_) { + return true; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + 7); + + if (!value.ok()) { + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return true; + } else { + return *value; + } } } @@ -941,68 +1090,6 @@ void com_android_aconfig_test_reset_flags() { "#; - const EXPORTED_SOURCE_FILE_EXPECTED: &str = r#" -#include "com_android_aconfig_test.h" -#include <server_configurable_flags/get_flags.h> -#include <vector> - -namespace com::android::aconfig::test { - - class flag_provider : public flag_provider_interface { - public: - virtual bool disabled_rw_exported() override { - if (cache_[0] == -1) { - cache_[0] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"; - } - return cache_[0]; - } - - virtual bool enabled_fixed_ro_exported() override { - if (cache_[1] == -1) { - cache_[1] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_fixed_ro_exported", - "false") == "true"; - } - return cache_[1]; - } - - virtual bool enabled_ro_exported() override { - if (cache_[2] == -1) { - cache_[2] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_ro_exported", - "false") == "true"; - } - return cache_[2]; - } - - private: - std::vector<int8_t> cache_ = std::vector<int8_t>(3, -1); - }; - - std::unique_ptr<flag_provider_interface> provider_ = - std::make_unique<flag_provider>(); -} - -bool com_android_aconfig_test_disabled_rw_exported() { - return com::android::aconfig::test::disabled_rw_exported(); -} - -bool com_android_aconfig_test_enabled_fixed_ro_exported() { - return com::android::aconfig::test::enabled_fixed_ro_exported(); -} - -bool com_android_aconfig_test_enabled_ro_exported() { - return com::android::aconfig::test::enabled_ro_exported(); -} - - -"#; - const FORCE_READ_ONLY_SOURCE_FILE_EXPECTED: &str = r#" #include "com_android_aconfig_test.h" @@ -1102,7 +1189,7 @@ public: extern std::unique_ptr<flag_provider_interface> provider_; -inline bool disabled_fixed_ro() { +constexpr inline bool disabled_fixed_ro() { return COM_ANDROID_ACONFIG_TEST_DISABLED_FIXED_RO; } @@ -1110,7 +1197,7 @@ inline bool disabled_ro() { return false; } -inline bool enabled_fixed_ro() { +constexpr inline bool enabled_fixed_ro() { return COM_ANDROID_ACONFIG_TEST_ENABLED_FIXED_RO; } @@ -1187,7 +1274,6 @@ bool com_android_aconfig_test_enabled_ro() { mode: CodegenMode, expected_header: &str, expected_src: &str, - allow_instrumentation: bool, ) { let modified_parsed_flags = crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); @@ -1198,7 +1284,6 @@ bool com_android_aconfig_test_enabled_ro() { modified_parsed_flags.into_iter(), mode, flag_ids, - allow_instrumentation, ) .unwrap(); let mut generated_files_map = HashMap::new(); @@ -1238,7 +1323,6 @@ bool com_android_aconfig_test_enabled_ro() { CodegenMode::Production, EXPORTED_PROD_HEADER_EXPECTED, PROD_SOURCE_FILE_EXPECTED, - false, ); } @@ -1250,19 +1334,6 @@ bool com_android_aconfig_test_enabled_ro() { CodegenMode::Test, EXPORTED_TEST_HEADER_EXPECTED, TEST_SOURCE_FILE_EXPECTED, - false, - ); - } - - #[test] - fn test_generate_cpp_code_for_exported() { - let parsed_flags = crate::test::parse_test_flags(); - test_generate_cpp_code( - parsed_flags, - CodegenMode::Exported, - EXPORTED_EXPORTED_HEADER_EXPECTED, - EXPORTED_SOURCE_FILE_EXPECTED, - false, ); } @@ -1274,7 +1345,6 @@ bool com_android_aconfig_test_enabled_ro() { CodegenMode::ForceReadOnly, EXPORTED_FORCE_READ_ONLY_HEADER_EXPECTED, FORCE_READ_ONLY_SOURCE_FILE_EXPECTED, - false, ); } @@ -1286,7 +1356,6 @@ bool com_android_aconfig_test_enabled_ro() { CodegenMode::Production, READ_ONLY_EXPORTED_PROD_HEADER_EXPECTED, READ_ONLY_PROD_SOURCE_FILE_EXPECTED, - false, ); } } diff --git a/tools/aconfig/aconfig/src/codegen/java.rs b/tools/aconfig/aconfig/src/codegen/java.rs index dbc4ab500b..4b670a060e 100644 --- a/tools/aconfig/aconfig/src/codegen/java.rs +++ b/tools/aconfig/aconfig/src/codegen/java.rs @@ -22,30 +22,44 @@ use tinytemplate::TinyTemplate; use crate::codegen; use crate::codegen::CodegenMode; -use crate::commands::OutputFile; +use crate::commands::{should_include_flag, OutputFile}; use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag}; +use convert_finalized_flags::{FinalizedFlag, FinalizedFlagMap}; use std::collections::HashMap; +// Arguments to configure codegen for generate_java_code. +pub struct JavaCodegenConfig { + pub codegen_mode: CodegenMode, + pub flag_ids: HashMap<String, u16>, + pub allow_instrumentation: bool, + pub package_fingerprint: u64, + pub new_exported: bool, + pub single_exported_file: bool, + pub finalized_flags: FinalizedFlagMap, +} + pub fn generate_java_code<I>( package: &str, parsed_flags_iter: I, - codegen_mode: CodegenMode, - flag_ids: HashMap<String, u16>, - allow_instrumentation: bool, + config: JavaCodegenConfig, ) -> Result<Vec<OutputFile>> where I: Iterator<Item = ProtoParsedFlag>, { - let flag_elements: Vec<FlagElement> = - parsed_flags_iter.map(|pf| create_flag_element(package, &pf, flag_ids.clone())).collect(); + let flag_elements: Vec<FlagElement> = parsed_flags_iter + .map(|pf| { + create_flag_element(package, &pf, config.flag_ids.clone(), &config.finalized_flags) + }) + .collect(); let namespace_flags = gen_flags_by_namespace(&flag_elements); let properties_set: BTreeSet<String> = flag_elements.iter().map(|fe| format_property_name(&fe.device_config_namespace)).collect(); - let is_test_mode = codegen_mode == CodegenMode::Test; - let library_exported = codegen_mode == CodegenMode::Exported; + let is_test_mode = config.codegen_mode == CodegenMode::Test; + let library_exported = config.codegen_mode == CodegenMode::Exported; let runtime_lookup_required = flag_elements.iter().any(|elem| elem.is_read_write) || library_exported; let container = (flag_elements.first().expect("zero template flags").container).to_string(); + let is_platform_container = matches!(container.as_str(), "system" | "product" | "vendor"); let context = Context { flag_elements, namespace_flags, @@ -54,15 +68,22 @@ where properties_set, package_name: package.to_string(), library_exported, - allow_instrumentation, + allow_instrumentation: config.allow_instrumentation, container, + is_platform_container, + package_fingerprint: format!("0x{:X}L", config.package_fingerprint), + new_exported: config.new_exported, + single_exported_file: config.single_exported_file, }; let mut template = TinyTemplate::new(); + if library_exported && config.single_exported_file { + template.add_template( + "ExportedFlags.java", + include_str!("../../templates/ExportedFlags.java.template"), + )?; + } template.add_template("Flags.java", include_str!("../../templates/Flags.java.template"))?; - template.add_template( - "FeatureFlagsImpl.java", - include_str!("../../templates/FeatureFlagsImpl.java.template"), - )?; + add_feature_flags_impl_template(&context, &mut template)?; template.add_template( "FeatureFlags.java", include_str!("../../templates/FeatureFlags.java.template"), @@ -77,18 +98,25 @@ where )?; let path: PathBuf = package.split('.').collect(); - [ + let mut files = vec![ "Flags.java", "FeatureFlags.java", "FeatureFlagsImpl.java", "CustomFeatureFlags.java", "FakeFeatureFlagsImpl.java", - ] - .iter() - .map(|file| { - Ok(OutputFile { contents: template.render(file, &context)?.into(), path: path.join(file) }) - }) - .collect::<Result<Vec<OutputFile>>>() + ]; + if library_exported && config.single_exported_file { + files.push("ExportedFlags.java"); + } + files + .iter() + .map(|file| { + Ok(OutputFile { + contents: template.render(file, &context)?.into(), + path: path.join(file), + }) + }) + .collect::<Result<Vec<OutputFile>>>() } fn gen_flags_by_namespace(flags: &[FlagElement]) -> Vec<NamespaceFlags> { @@ -123,6 +151,10 @@ struct Context { pub library_exported: bool, pub allow_instrumentation: bool, pub container: String, + pub is_platform_container: bool, + pub package_fingerprint: String, + pub new_exported: bool, + pub single_exported_file: bool, } #[derive(Serialize, Debug)] @@ -137,30 +169,67 @@ struct FlagElement { pub default_value: bool, pub device_config_namespace: String, pub device_config_flag: String, + pub flag_name: String, pub flag_name_constant_suffix: String, pub flag_offset: u16, pub is_read_write: bool, pub method_name: String, pub properties: String, + pub finalized_sdk_present: bool, + pub finalized_sdk_value: i32, } fn create_flag_element( package: &str, pf: &ProtoParsedFlag, flag_offsets: HashMap<String, u16>, + finalized_flags: &FinalizedFlagMap, ) -> FlagElement { let device_config_flag = codegen::create_device_config_ident(package, pf.name()) .expect("values checked at flag parse time"); + + let no_assigned_offset = !should_include_flag(pf); + + let flag_offset = match flag_offsets.get(pf.name()) { + Some(offset) => offset, + None => { + // System/vendor/product RO+disabled flags have no offset in storage files. + // Assign placeholder value. + if no_assigned_offset { + &0 + } + // All other flags _must_ have an offset. + else { + panic!("{}", format!("missing flag offset for {}", pf.name())); + } + } + }; + + // An empty map is provided if check_api_level is disabled. + let mut finalized_sdk_present: bool = false; + let mut finalized_sdk_value: i32 = 0; + if !finalized_flags.is_empty() { + let finalized_sdk = finalized_flags.get_finalized_level(&FinalizedFlag { + flag_name: pf.name().to_string(), + package_name: package.to_string(), + }); + finalized_sdk_present = finalized_sdk.is_some(); + finalized_sdk_value = finalized_sdk.map(|f| f.0).unwrap_or_default(); + } + FlagElement { container: pf.container().to_string(), default_value: pf.state() == ProtoFlagState::ENABLED, device_config_namespace: pf.namespace().to_string(), device_config_flag, + flag_name: pf.name().to_string(), flag_name_constant_suffix: pf.name().to_ascii_uppercase(), - flag_offset: *flag_offsets.get(pf.name()).expect("didnt find package offset :("), + flag_offset: *flag_offset, is_read_write: pf.permission() == ProtoFlagPermission::READ_WRITE, method_name: format_java_method_name(pf.name()), properties: format_property_name(pf.namespace()), + finalized_sdk_present, + finalized_sdk_value, } } @@ -190,8 +259,62 @@ fn format_property_name(property_name: &str) -> String { format!("mProperties{}{}", &name[0..1].to_ascii_uppercase(), &name[1..]) } +fn add_feature_flags_impl_template( + context: &Context, + template: &mut TinyTemplate, +) -> Result<(), tinytemplate::error::Error> { + if context.is_test_mode { + // Test mode has its own template, so use regardless of any other settings. + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.test_mode.java.template"), + )?; + return Ok(()); + } + + match (context.library_exported, context.new_exported, context.allow_instrumentation) { + // Exported library with new_exported enabled, use new storage exported template. + (true, true, _) => { + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.exported.java.template"), + )?; + } + + // Exported library with new_exported NOT enabled, use legacy (device + // config) template, because regardless of allow_instrumentation, we use + // device config for exported libs if new_exported isn't enabled. + // Remove once new_exported is fully rolled out. + (true, false, _) => { + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.deviceConfig.java.template"), + )?; + } + + // New storage internal mode. + (false, _, true) => { + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.new_storage.java.template"), + )?; + } + + // Device config internal mode. Use legacy (device config) template. + (false, _, false) => { + template.add_template( + "FeatureFlagsImpl.java", + include_str!("../../templates/FeatureFlagsImpl.deviceConfig.java.template"), + )?; + } + }; + Ok(()) +} + #[cfg(test)] mod tests { + use convert_finalized_flags::ApiLevel; + use super::*; use crate::commands::assign_flag_ids; use std::collections::HashMap; @@ -494,12 +617,19 @@ mod tests { crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); let flag_ids = assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: false, + single_exported_file: false, + finalized_flags: FinalizedFlagMap::new(), + }; let generated_files = generate_java_code( crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), - mode, - flag_ids, - false, + config, ) .unwrap(); let expect_flags_content = EXPECTED_FLAG_COMMON_CONTENT.to_string() @@ -507,25 +637,38 @@ mod tests { private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); }"#; - let expected_featureflagsmpl_content_0 = r#" + let expected_featureflagsmpl_content = r#" package com.android.aconfig.test; // TODO(b/303773055): Remove the annotation after access issue is resolved. import android.compat.annotation.UnsupportedAppUsage; - import android.provider.DeviceConfig; - import android.provider.DeviceConfig.Properties; - "#; - - let expected_featureflagsmpl_content_1 = r#" + import android.os.Build; + import android.os.flagging.PlatformAconfigPackageInternal; + import android.util.Log; /** @hide */ public final class FeatureFlagsImpl implements FeatureFlags { - private static volatile boolean aconfig_test_is_cached = false; - private static volatile boolean other_namespace_is_cached = false; + private static final String TAG = "FeatureFlagsImpl"; + private static volatile boolean isCached = false; private static boolean disabledRw = false; private static boolean disabledRwExported = false; private static boolean disabledRwInOtherNamespace = false; private static boolean enabledRw = true; - "#; - let expected_featureflagsmpl_content_2 = r#" + private void init() { + try { + PlatformAconfigPackageInternal reader = PlatformAconfigPackageInternal.load("com.android.aconfig.test", 0x5081CE7221C77064L); + disabledRw = reader.getBooleanFlagValue(0); + disabledRwExported = reader.getBooleanFlagValue(1); + enabledRw = reader.getBooleanFlagValue(7); + disabledRwInOtherNamespace = reader.getBooleanFlagValue(2); + } catch (Exception e) { + Log.e(TAG, e.toString()); + } catch (LinkageError e) { + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.e(TAG, e.toString()); + } + isCached = true; + } + @Override @com.android.aconfig.annotations.AconfigFlagAccessor @UnsupportedAppUsage @@ -536,8 +679,8 @@ mod tests { @com.android.aconfig.annotations.AconfigFlagAccessor @UnsupportedAppUsage public boolean disabledRw() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return disabledRw; } @@ -545,8 +688,8 @@ mod tests { @com.android.aconfig.annotations.AconfigFlagAccessor @UnsupportedAppUsage public boolean disabledRwExported() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return disabledRwExported; } @@ -554,8 +697,8 @@ mod tests { @com.android.aconfig.annotations.AconfigFlagAccessor @UnsupportedAppUsage public boolean disabledRwInOtherNamespace() { - if (!other_namespace_is_cached) { - load_overrides_other_namespace(); + if (!isCached) { + init(); } return disabledRwInOtherNamespace; } @@ -587,26 +730,125 @@ mod tests { @com.android.aconfig.annotations.AconfigFlagAccessor @UnsupportedAppUsage public boolean enabledRw() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return enabledRw; } } "#; - let expect_featureflagsimpl_content_old = expected_featureflagsmpl_content_0.to_owned() - + expected_featureflagsmpl_content_1 - + r#" + let mut file_set = HashMap::from([ + ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expected_featureflagsmpl_content), + ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT), + ( + "com/android/aconfig/test/CustomFeatureFlags.java", + EXPECTED_CUSTOMFEATUREFLAGS_CONTENT, + ), + ( + "com/android/aconfig/test/FakeFeatureFlagsImpl.java", + EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT, + ), + ]); + + for file in generated_files { + let file_path = file.path.to_str().unwrap(); + assert!(file_set.contains_key(file_path), "Cannot find {}", file_path); + assert_eq!( + None, + crate::test::first_significant_code_diff( + file_set.get(file_path).unwrap(), + &String::from_utf8(file.contents).unwrap() + ), + "File {} content is not correct", + file_path + ); + file_set.remove(file_path); + } + + assert!(file_set.is_empty()); + } + + #[test] + fn test_generate_java_code_exported() { + let parsed_flags = crate::test::parse_test_flags(); + let mode = CodegenMode::Exported; + let modified_parsed_flags = + crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); + let flag_ids = + assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: false, + single_exported_file: false, + finalized_flags: FinalizedFlagMap::new(), + }; + let generated_files = generate_java_code( + crate::test::TEST_PACKAGE, + modified_parsed_flags.into_iter(), + config, + ) + .unwrap(); + + let expect_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public final class Flags { + /** @hide */ + public static final String FLAG_DISABLED_RW_EXPORTED = "com.android.aconfig.test.disabled_rw_exported"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO_EXPORTED = "com.android.aconfig.test.enabled_fixed_ro_exported"; + /** @hide */ + public static final String FLAG_ENABLED_RO_EXPORTED = "com.android.aconfig.test.enabled_ro_exported"; + public static boolean disabledRwExported() { + return FEATURE_FLAGS.disabledRwExported(); + } + public static boolean enabledFixedRoExported() { + return FEATURE_FLAGS.enabledFixedRoExported(); + } + public static boolean enabledRoExported() { + return FEATURE_FLAGS.enabledRoExported(); + } + private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); + } + "#; + + let expect_feature_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public interface FeatureFlags { + boolean disabledRwExported(); + boolean enabledFixedRoExported(); + boolean enabledRoExported(); + } + "#; + + let expect_feature_flags_impl_content = r#" + package com.android.aconfig.test; + import android.os.Binder; + import android.provider.DeviceConfig; + import android.provider.DeviceConfig.Properties; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + private static volatile boolean aconfig_test_is_cached = false; + private static boolean disabledRwExported = false; + private static boolean enabledFixedRoExported = false; + private static boolean enabledRoExported = false; + private void load_overrides_aconfig_test() { + final long ident = Binder.clearCallingIdentity(); try { Properties properties = DeviceConfig.getProperties("aconfig_test"); - disabledRw = - properties.getBoolean(Flags.FLAG_DISABLED_RW, false); disabledRwExported = properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false); - enabledRw = - properties.getBoolean(Flags.FLAG_ENABLED_RW, true); + enabledFixedRoExported = + properties.getBoolean(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, false); + enabledRoExported = + properties.getBoolean(Flags.FLAG_ENABLED_RO_EXPORTED, false); } catch (NullPointerException e) { throw new RuntimeException( "Cannot read value from namespace aconfig_test " @@ -616,39 +858,98 @@ mod tests { + "flag declaration.", e ); + } catch (SecurityException e) { + // for isolated process case, skip loading flag value from the storage, use the default + } finally { + Binder.restoreCallingIdentity(ident); } aconfig_test_is_cached = true; } - - private void load_overrides_other_namespace() { - try { - Properties properties = DeviceConfig.getProperties("other_namespace"); - disabledRwInOtherNamespace = - properties.getBoolean(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, false); - } catch (NullPointerException e) { - throw new RuntimeException( - "Cannot read value from namespace other_namespace " - + "from DeviceConfig. It could be that the code using flag " - + "executed before SettingsProvider initialization. Please use " - + "fixed read-only flag by adding is_fixed_read_only: true in " - + "flag declaration.", - e - ); + @Override + public boolean disabledRwExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return disabledRwExported; + } + @Override + public boolean enabledFixedRoExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); } - other_namespace_is_cached = true; - }"# - + expected_featureflagsmpl_content_2; + return enabledFixedRoExported; + } + @Override + public boolean enabledRoExported() { + if (!aconfig_test_is_cached) { + load_overrides_aconfig_test(); + } + return enabledRoExported; + } + }"#; + + let expect_custom_feature_flags_content = r#" + package com.android.aconfig.test; + + import java.util.Arrays; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + import java.util.function.BiPredicate; + import java.util.function.Predicate; + + /** @hide */ + public class CustomFeatureFlags implements FeatureFlags { + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) { + mGetValueImpl = getValueImpl; + } + + @Override + public boolean disabledRwExported() { + return getValue(Flags.FLAG_DISABLED_RW_EXPORTED, + FeatureFlags::disabledRwExported); + } + @Override + public boolean enabledFixedRoExported() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + FeatureFlags::enabledFixedRoExported); + } + @Override + public boolean enabledRoExported() { + return getValue(Flags.FLAG_ENABLED_RO_EXPORTED, + FeatureFlags::enabledRoExported); + } + + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + return mGetValueImpl.test(flagName, getter); + } + + public List<String> getFlagNames() { + return Arrays.asList( + Flags.FLAG_DISABLED_RW_EXPORTED, + Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + Flags.FLAG_ENABLED_RO_EXPORTED + ); + } + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + "" + ) + ); + } + "#; let mut file_set = HashMap::from([ - ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()), - ( - "com/android/aconfig/test/FeatureFlagsImpl.java", - &expect_featureflagsimpl_content_old, - ), - ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT), + ("com/android/aconfig/test/Flags.java", expect_flags_content), + ("com/android/aconfig/test/FeatureFlags.java", expect_feature_flags_content), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_feature_flags_impl_content), ( "com/android/aconfig/test/CustomFeatureFlags.java", - EXPECTED_CUSTOMFEATUREFLAGS_CONTENT, + expect_custom_feature_flags_content, ), ( "com/android/aconfig/test/FakeFeatureFlagsImpl.java", @@ -672,164 +973,178 @@ mod tests { } assert!(file_set.is_empty()); + } + #[test] + fn test_generate_java_code_new_exported() { let parsed_flags = crate::test::parse_test_flags(); - let mode = CodegenMode::Production; + let mode = CodegenMode::Exported; let modified_parsed_flags = crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); let flag_ids = assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: true, + single_exported_file: false, + finalized_flags: FinalizedFlagMap::new(), + }; let generated_files = generate_java_code( crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), - mode, - flag_ids, - true, + config, ) .unwrap(); - let expect_featureflagsimpl_content_new = expected_featureflagsmpl_content_0.to_owned() - + r#" - import android.aconfig.storage.StorageInternalReader; - import android.util.Log; - "# - + expected_featureflagsmpl_content_1 - + r#" - StorageInternalReader reader; - boolean readFromNewStorage; - - boolean useNewStorageValueAndDiscardOld = false; + let expect_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public final class Flags { + /** @hide */ + public static final String FLAG_DISABLED_RW_EXPORTED = "com.android.aconfig.test.disabled_rw_exported"; + /** @hide */ + public static final String FLAG_ENABLED_FIXED_RO_EXPORTED = "com.android.aconfig.test.enabled_fixed_ro_exported"; + /** @hide */ + public static final String FLAG_ENABLED_RO_EXPORTED = "com.android.aconfig.test.enabled_ro_exported"; + public static boolean disabledRwExported() { + return FEATURE_FLAGS.disabledRwExported(); + } + public static boolean enabledFixedRoExported() { + return FEATURE_FLAGS.enabledFixedRoExported(); + } + public static boolean enabledRoExported() { + return FEATURE_FLAGS.enabledRoExported(); + } + private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl(); + } + "#; - private final static String TAG = "AconfigJavaCodegen"; - private final static String SUCCESS_LOG = "success: %s value matches"; - private final static String MISMATCH_LOG = "error: %s value mismatch, new storage value is %s, old storage value is %s"; - private final static String ERROR_LOG = "error: failed to read flag value"; + let expect_feature_flags_content = r#" + package com.android.aconfig.test; + /** @hide */ + public interface FeatureFlags { + boolean disabledRwExported(); + boolean enabledFixedRoExported(); + boolean enabledRoExported(); + } + "#; - private void init() { - if (reader != null) return; - if (DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.storage_test_mission_1", false)) { - readFromNewStorage = true; + let expect_feature_flags_impl_content = r#" + package com.android.aconfig.test; + import android.os.Build; + import android.os.flagging.AconfigPackage; + import android.util.Log; + /** @hide */ + public final class FeatureFlagsImpl implements FeatureFlags { + private static final String TAG = "FeatureFlagsImplExport"; + private static volatile boolean isCached = false; + private static boolean disabledRwExported = false; + private static boolean enabledFixedRoExported = false; + private static boolean enabledRoExported = false; + private void init() { try { - reader = new StorageInternalReader("system", "com.android.aconfig.test"); + AconfigPackage reader = AconfigPackage.load("com.android.aconfig.test"); + disabledRwExported = reader.getBooleanFlagValue("disabled_rw_exported", false); + enabledFixedRoExported = reader.getBooleanFlagValue("enabled_fixed_ro_exported", false); + enabledRoExported = reader.getBooleanFlagValue("enabled_ro_exported", false); } catch (Exception e) { - reader = null; + // pass + Log.e(TAG, e.toString()); + } catch (LinkageError e) { + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.w(TAG, e.toString()); + } + isCached = true; + } + @Override + public boolean disabledRwExported() { + if (!isCached) { + init(); } + return disabledRwExported; } + @Override + public boolean enabledFixedRoExported() { + if (!isCached) { + init(); + } + return enabledFixedRoExported; + } + @Override + public boolean enabledRoExported() { + if (!isCached) { + init(); + } + return enabledRoExported; + } + }"#; - useNewStorageValueAndDiscardOld = - DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.use_new_storage_value", false); - } + let expect_custom_feature_flags_content = r#" + package com.android.aconfig.test; - private void load_overrides_aconfig_test() { - try { - Properties properties = DeviceConfig.getProperties("aconfig_test"); - disabledRw = - properties.getBoolean(Flags.FLAG_DISABLED_RW, false); - disabledRwExported = - properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false); - enabledRw = - properties.getBoolean(Flags.FLAG_ENABLED_RW, true); - } catch (NullPointerException e) { - throw new RuntimeException( - "Cannot read value from namespace aconfig_test " - + "from DeviceConfig. It could be that the code using flag " - + "executed before SettingsProvider initialization. Please use " - + "fixed read-only flag by adding is_fixed_read_only: true in " - + "flag declaration.", - e - ); - } - aconfig_test_is_cached = true; - init(); - if (readFromNewStorage && reader != null) { - boolean val; - try { - val = reader.getBooleanFlagValue(1); - if (val == disabledRw) { - Log.i(TAG, String.format(SUCCESS_LOG, "disabledRw")); - } else { - Log.i(TAG, String.format(MISMATCH_LOG, "disabledRw", val, disabledRw)); - } - - if (useNewStorageValueAndDiscardOld) { - disabledRw = val; - } - - val = reader.getBooleanFlagValue(2); - if (val == disabledRwExported) { - Log.i(TAG, String.format(SUCCESS_LOG, "disabledRwExported")); - } else { - Log.i(TAG, String.format(MISMATCH_LOG, "disabledRwExported", val, disabledRwExported)); - } - - if (useNewStorageValueAndDiscardOld) { - disabledRwExported = val; - } - - val = reader.getBooleanFlagValue(8); - if (val == enabledRw) { - Log.i(TAG, String.format(SUCCESS_LOG, "enabledRw")); - } else { - Log.i(TAG, String.format(MISMATCH_LOG, "enabledRw", val, enabledRw)); - } - - if (useNewStorageValueAndDiscardOld) { - enabledRw = val; - } + import java.util.Arrays; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + import java.util.function.BiPredicate; + import java.util.function.Predicate; - } catch (Exception e) { - Log.e(TAG, ERROR_LOG, e); - } + /** @hide */ + public class CustomFeatureFlags implements FeatureFlags { + + private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; + + public CustomFeatureFlags(BiPredicate<String, Predicate<FeatureFlags>> getValueImpl) { + mGetValueImpl = getValueImpl; } - } - private void load_overrides_other_namespace() { - try { - Properties properties = DeviceConfig.getProperties("other_namespace"); - disabledRwInOtherNamespace = - properties.getBoolean(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, false); - } catch (NullPointerException e) { - throw new RuntimeException( - "Cannot read value from namespace other_namespace " - + "from DeviceConfig. It could be that the code using flag " - + "executed before SettingsProvider initialization. Please use " - + "fixed read-only flag by adding is_fixed_read_only: true in " - + "flag declaration.", - e - ); + @Override + public boolean disabledRwExported() { + return getValue(Flags.FLAG_DISABLED_RW_EXPORTED, + FeatureFlags::disabledRwExported); + } + @Override + public boolean enabledFixedRoExported() { + return getValue(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + FeatureFlags::enabledFixedRoExported); + } + @Override + public boolean enabledRoExported() { + return getValue(Flags.FLAG_ENABLED_RO_EXPORTED, + FeatureFlags::enabledRoExported); } - other_namespace_is_cached = true; - init(); - if (readFromNewStorage && reader != null) { - boolean val; - try { - val = reader.getBooleanFlagValue(3); - if (val == disabledRwInOtherNamespace) { - Log.i(TAG, String.format(SUCCESS_LOG, "disabledRwInOtherNamespace")); - } else { - Log.i(TAG, String.format(MISMATCH_LOG, "disabledRwInOtherNamespace", val, disabledRwInOtherNamespace)); - } - if (useNewStorageValueAndDiscardOld) { - disabledRwInOtherNamespace = val; - } + protected boolean getValue(String flagName, Predicate<FeatureFlags> getter) { + return mGetValueImpl.test(flagName, getter); + } - } catch (Exception e) { - Log.e(TAG, ERROR_LOG, e); - } + public List<String> getFlagNames() { + return Arrays.asList( + Flags.FLAG_DISABLED_RW_EXPORTED, + Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, + Flags.FLAG_ENABLED_RO_EXPORTED + ); } - }"# + expected_featureflagsmpl_content_2; + + private Set<String> mReadOnlyFlagsSet = new HashSet<>( + Arrays.asList( + "" + ) + ); + } + "#; let mut file_set = HashMap::from([ - ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()), - ( - "com/android/aconfig/test/FeatureFlagsImpl.java", - &expect_featureflagsimpl_content_new, - ), - ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT), + ("com/android/aconfig/test/Flags.java", expect_flags_content), + ("com/android/aconfig/test/FeatureFlags.java", expect_feature_flags_content), + ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_feature_flags_impl_content), ( "com/android/aconfig/test/CustomFeatureFlags.java", - EXPECTED_CUSTOMFEATUREFLAGS_CONTENT, + expect_custom_feature_flags_content, ), ( "com/android/aconfig/test/FakeFeatureFlagsImpl.java", @@ -856,19 +1171,34 @@ mod tests { } #[test] - fn test_generate_java_code_exported() { + fn test_generate_java_code_new_exported_with_sdk_check() { let parsed_flags = crate::test::parse_test_flags(); let mode = CodegenMode::Exported; let modified_parsed_flags = crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); let flag_ids = assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let mut finalized_flags = FinalizedFlagMap::new(); + finalized_flags.insert_if_new( + ApiLevel(36), + FinalizedFlag { + flag_name: "disabled_rw_exported".to_string(), + package_name: "com.android.aconfig.test".to_string(), + }, + ); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: true, + single_exported_file: false, + finalized_flags, + }; let generated_files = generate_java_code( crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), - mode, - flag_ids, - true, + config, ) .unwrap(); @@ -907,55 +1237,50 @@ mod tests { let expect_feature_flags_impl_content = r#" package com.android.aconfig.test; - import android.provider.DeviceConfig; - import android.provider.DeviceConfig.Properties; + import android.os.Build; + import android.os.flagging.AconfigPackage; + import android.util.Log; /** @hide */ public final class FeatureFlagsImpl implements FeatureFlags { - private static volatile boolean aconfig_test_is_cached = false; + private static final String TAG = "FeatureFlagsImplExport"; + private static volatile boolean isCached = false; private static boolean disabledRwExported = false; private static boolean enabledFixedRoExported = false; private static boolean enabledRoExported = false; - - - private void load_overrides_aconfig_test() { + private void init() { try { - Properties properties = DeviceConfig.getProperties("aconfig_test"); - disabledRwExported = - properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false); - enabledFixedRoExported = - properties.getBoolean(Flags.FLAG_ENABLED_FIXED_RO_EXPORTED, false); - enabledRoExported = - properties.getBoolean(Flags.FLAG_ENABLED_RO_EXPORTED, false); - } catch (NullPointerException e) { - throw new RuntimeException( - "Cannot read value from namespace aconfig_test " - + "from DeviceConfig. It could be that the code using flag " - + "executed before SettingsProvider initialization. Please use " - + "fixed read-only flag by adding is_fixed_read_only: true in " - + "flag declaration.", - e - ); + AconfigPackage reader = AconfigPackage.load("com.android.aconfig.test"); + disabledRwExported = Build.VERSION.SDK_INT >= 36 ? true : reader.getBooleanFlagValue("disabled_rw_exported", false); + enabledFixedRoExported = reader.getBooleanFlagValue("enabled_fixed_ro_exported", false); + enabledRoExported = reader.getBooleanFlagValue("enabled_ro_exported", false); + } catch (Exception e) { + // pass + Log.e(TAG, e.toString()); + } catch (LinkageError e) { + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.w(TAG, e.toString()); } - aconfig_test_is_cached = true; + isCached = true; } @Override public boolean disabledRwExported() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return disabledRwExported; } @Override public boolean enabledFixedRoExported() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return enabledFixedRoExported; } @Override public boolean enabledRoExported() { - if (!aconfig_test_is_cached) { - load_overrides_aconfig_test(); + if (!isCached) { + init(); } return enabledRoExported; } @@ -1056,12 +1381,19 @@ mod tests { crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); let flag_ids = assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: false, + single_exported_file: false, + finalized_flags: FinalizedFlagMap::new(), + }; let generated_files = generate_java_code( crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), - mode, - flag_ids, - true, + config, ) .unwrap(); @@ -1177,12 +1509,19 @@ mod tests { crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); let flag_ids = assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap(); + let config = JavaCodegenConfig { + codegen_mode: mode, + flag_ids, + allow_instrumentation: true, + package_fingerprint: 5801144784618221668, + new_exported: false, + single_exported_file: false, + finalized_flags: FinalizedFlagMap::new(), + }; let generated_files = generate_java_code( crate::test::TEST_PACKAGE, modified_parsed_flags.into_iter(), - mode, - flag_ids, - true, + config, ) .unwrap(); let expect_featureflags_content = r#" diff --git a/tools/aconfig/aconfig/src/codegen/mod.rs b/tools/aconfig/aconfig/src/codegen/mod.rs index 1ea3b37849..9ed66dbd03 100644 --- a/tools/aconfig/aconfig/src/codegen/mod.rs +++ b/tools/aconfig/aconfig/src/codegen/mod.rs @@ -50,67 +50,6 @@ impl std::fmt::Display for CodegenMode { #[cfg(test)] mod tests { use super::*; - use aconfig_protos::is_valid_container_ident; - - #[test] - fn test_is_valid_name_ident() { - assert!(is_valid_name_ident("foo")); - assert!(is_valid_name_ident("foo_bar_123")); - assert!(is_valid_name_ident("foo_")); - - assert!(!is_valid_name_ident("")); - assert!(!is_valid_name_ident("123_foo")); - assert!(!is_valid_name_ident("foo-bar")); - assert!(!is_valid_name_ident("foo-b\u{00e5}r")); - assert!(!is_valid_name_ident("foo__bar")); - assert!(!is_valid_name_ident("_foo")); - } - - #[test] - fn test_is_valid_package_ident() { - assert!(is_valid_package_ident("foo.bar")); - assert!(is_valid_package_ident("foo.bar_baz")); - assert!(is_valid_package_ident("foo.bar.a123")); - - assert!(!is_valid_package_ident("foo_bar_123")); - assert!(!is_valid_package_ident("foo")); - assert!(!is_valid_package_ident("foo._bar")); - assert!(!is_valid_package_ident("")); - assert!(!is_valid_package_ident("123_foo")); - assert!(!is_valid_package_ident("foo-bar")); - assert!(!is_valid_package_ident("foo-b\u{00e5}r")); - assert!(!is_valid_package_ident("foo.bar.123")); - assert!(!is_valid_package_ident(".foo.bar")); - assert!(!is_valid_package_ident("foo.bar.")); - assert!(!is_valid_package_ident(".")); - assert!(!is_valid_package_ident("..")); - assert!(!is_valid_package_ident("foo..bar")); - assert!(!is_valid_package_ident("foo.__bar")); - } - - #[test] - fn test_is_valid_container_ident() { - assert!(is_valid_container_ident("foo.bar")); - assert!(is_valid_container_ident("foo.bar_baz")); - assert!(is_valid_container_ident("foo.bar.a123")); - assert!(is_valid_container_ident("foo")); - assert!(is_valid_container_ident("foo_bar_123")); - - assert!(!is_valid_container_ident("")); - assert!(!is_valid_container_ident("foo._bar")); - assert!(!is_valid_container_ident("_foo")); - assert!(!is_valid_container_ident("123_foo")); - assert!(!is_valid_container_ident("foo-bar")); - assert!(!is_valid_container_ident("foo-b\u{00e5}r")); - assert!(!is_valid_container_ident("foo.bar.123")); - assert!(!is_valid_container_ident(".foo.bar")); - assert!(!is_valid_container_ident("foo.bar.")); - assert!(!is_valid_container_ident(".")); - assert!(!is_valid_container_ident("..")); - assert!(!is_valid_container_ident("foo..bar")); - assert!(!is_valid_container_ident("foo.__bar")); - } - #[test] fn test_create_device_config_ident() { assert_eq!( diff --git a/tools/aconfig/aconfig/src/codegen/rust.rs b/tools/aconfig/aconfig/src/codegen/rust.rs index 6f3f7bfa8f..2ee5f36822 100644 --- a/tools/aconfig/aconfig/src/codegen/rust.rs +++ b/tools/aconfig/aconfig/src/codegen/rust.rs @@ -24,14 +24,13 @@ use std::collections::HashMap; use crate::codegen; use crate::codegen::CodegenMode; -use crate::commands::OutputFile; +use crate::commands::{should_include_flag, OutputFile}; pub fn generate_rust_code<I>( package: &str, flag_ids: HashMap<String, u16>, parsed_flags_iter: I, codegen_mode: CodegenMode, - allow_instrumentation: bool, ) -> Result<OutputFile> where I: Iterator<Item = ProtoParsedFlag>, @@ -46,7 +45,6 @@ where template_flags, modules: package.split('.').map(|s| s.to_string()).collect::<Vec<_>>(), has_readwrite, - allow_instrumentation, container, }; let mut template = TinyTemplate::new(); @@ -70,7 +68,6 @@ struct TemplateContext { pub template_flags: Vec<TemplateParsedFlag>, pub modules: Vec<String>, pub has_readwrite: bool, - pub allow_instrumentation: bool, pub container: String, } @@ -88,6 +85,21 @@ struct TemplateParsedFlag { impl TemplateParsedFlag { #[allow(clippy::nonminimal_bool)] fn new(package: &str, flag_offsets: HashMap<String, u16>, pf: &ProtoParsedFlag) -> Self { + let flag_offset = match flag_offsets.get(pf.name()) { + Some(offset) => offset, + None => { + // System/vendor/product RO+disabled flags have no offset in storage files. + // Assign placeholder value. + if !should_include_flag(pf) { + &0 + } + // All other flags _must_ have an offset. + else { + panic!("{}", format!("missing flag offset for {}", pf.name())); + } + } + }; + Self { readwrite: pf.permission() == ProtoFlagPermission::READ_WRITE, default_value: match pf.state() { @@ -96,7 +108,7 @@ impl TemplateParsedFlag { }, name: pf.name().to_string(), container: pf.container().to_string(), - flag_offset: *flag_offsets.get(pf.name()).expect("didnt find package offset :("), + flag_offset: *flag_offset, device_config_namespace: pf.namespace().to_string(), device_config_flag: codegen::create_device_config_ident(package, pf.name()) .expect("values checked at flag parse time"), @@ -116,154 +128,6 @@ use std::io::Write; use std::sync::LazyLock; use log::{log, LevelFilter, Level}; -static STORAGE_MIGRATION_MARKER_FILE: &str = - "/metadata/aconfig_test_missions/mission_1"; -static MIGRATION_LOG_TAG: &str = "AconfigTestMission1"; - -/// flag provider -pub struct FlagProvider; - - /// flag value cache for disabled_rw - static CACHED_disabled_rw: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw", - "false") == "true"); - - /// flag value cache for disabled_rw_exported - static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"); - - /// flag value cache for disabled_rw_in_other_namespace - static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.other_namespace", - "com.android.aconfig.test.disabled_rw_in_other_namespace", - "false") == "true"); - - /// flag value cache for enabled_rw - static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_rw", - "true") == "true"); - -impl FlagProvider { - /// query flag disabled_ro - pub fn disabled_ro(&self) -> bool { - false - } - - /// query flag disabled_rw - pub fn disabled_rw(&self) -> bool { - *CACHED_disabled_rw - } - - /// query flag disabled_rw_exported - pub fn disabled_rw_exported(&self) -> bool { - *CACHED_disabled_rw_exported - } - - /// query flag disabled_rw_in_other_namespace - pub fn disabled_rw_in_other_namespace(&self) -> bool { - *CACHED_disabled_rw_in_other_namespace - } - - /// query flag enabled_fixed_ro - pub fn enabled_fixed_ro(&self) -> bool { - true - } - - /// query flag enabled_fixed_ro_exported - pub fn enabled_fixed_ro_exported(&self) -> bool { - true - } - - /// query flag enabled_ro - pub fn enabled_ro(&self) -> bool { - true - } - - /// query flag enabled_ro_exported - pub fn enabled_ro_exported(&self) -> bool { - true - } - - /// query flag enabled_rw - pub fn enabled_rw(&self) -> bool { - *CACHED_enabled_rw - } -} - -/// flag provider -pub static PROVIDER: FlagProvider = FlagProvider; - -/// query flag disabled_ro -#[inline(always)] -pub fn disabled_ro() -> bool { - false -} - -/// query flag disabled_rw -#[inline(always)] -pub fn disabled_rw() -> bool { - PROVIDER.disabled_rw() -} - -/// query flag disabled_rw_exported -#[inline(always)] -pub fn disabled_rw_exported() -> bool { - PROVIDER.disabled_rw_exported() -} - -/// query flag disabled_rw_in_other_namespace -#[inline(always)] -pub fn disabled_rw_in_other_namespace() -> bool { - PROVIDER.disabled_rw_in_other_namespace() -} - -/// query flag enabled_fixed_ro -#[inline(always)] -pub fn enabled_fixed_ro() -> bool { - true -} - -/// query flag enabled_fixed_ro_exported -#[inline(always)] -pub fn enabled_fixed_ro_exported() -> bool { - true -} - -/// query flag enabled_ro -#[inline(always)] -pub fn enabled_ro() -> bool { - true -} - -/// query flag enabled_ro_exported -#[inline(always)] -pub fn enabled_ro_exported() -> bool { - true -} - -/// query flag enabled_rw -#[inline(always)] -pub fn enabled_rw() -> bool { - PROVIDER.enabled_rw() -} -"#; - - const PROD_INSTRUMENTED_EXPECTED: &str = r#" -//! codegenerated rust flag lib -use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context}; -use std::path::Path; -use std::io::Write; -use std::sync::LazyLock; -use log::{log, LevelFilter, Level}; - -static STORAGE_MIGRATION_MARKER_FILE: &str = - "/metadata/aconfig_test_missions/mission_1"; -static MIGRATION_LOG_TAG: &str = "AconfigTestMission1"; - /// flag provider pub struct FlagProvider; @@ -279,89 +143,53 @@ static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new /// flag value cache for disabled_rw static CACHED_disabled_rw: LazyLock<bool> = LazyLock::new(|| { - let result = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw", - "false") == "true"; - - let use_new_storage_value = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; - - if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - // This will be called multiple times. Subsequent calls after the first are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info)); - - let aconfig_storage_result = FLAG_VAL_MAP - .as_ref() - .map_err(|err| format!("failed to get flag val map: {err}")) - .and_then(|flag_val_map| { - PACKAGE_OFFSET - .as_ref() - .map_err(|err| format!("failed to get package read offset: {err}")) - .and_then(|package_offset| { - match package_offset { - Some(offset) => { - get_boolean_flag_value(&flag_val_map, offset + 1) - .map_err(|err| format!("failed to get flag: {err}")) - }, - None => Err("no context found for package 'com.android.aconfig.test'".to_string()) - } - }) - }); - - match aconfig_storage_result { - Ok(storage_result) if storage_result == result => { - if use_new_storage_value { - return storage_result; - } else { - return result; - } - }, - Ok(storage_result) => { - log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw'. Legacy storage was {result}, new storage was {storage_result}"); - if use_new_storage_value { - return storage_result; - } else { - return result; - } - }, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: {err}"); - if use_new_storage_value { - panic!("failed to read flag value: {err}"); - } - } + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); + + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: {err}")) + .and_then(|flag_val_map| { + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: {err}")) + .and_then(|package_offset| { + match package_offset { + Some(offset) => { + get_boolean_flag_value(&flag_val_map, offset + 0) + .map_err(|err| format!("failed to get flag: {err}")) + }, + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } + } + }) + }); + + match flag_value_result { + Ok(flag_value) => { + return flag_value; + }, + Err(err) => { + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; } } - - result }); /// flag value cache for disabled_rw_exported static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| { - let result = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"; - - let use_new_storage_value = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; - - if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { // This will be called multiple times. Subsequent calls after the first are noops. logger::init( logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) + .with_tag_on_device("aconfig_rust_codegen") .with_max_level(LevelFilter::Info)); - let aconfig_storage_result = FLAG_VAL_MAP + let flag_value_result = FLAG_VAL_MAP .as_ref() .map_err(|err| format!("failed to get flag val map: {err}")) .and_then(|flag_val_map| { @@ -371,62 +199,37 @@ static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| { .and_then(|package_offset| { match package_offset { Some(offset) => { - get_boolean_flag_value(&flag_val_map, offset + 2) + get_boolean_flag_value(&flag_val_map, offset + 1) .map_err(|err| format!("failed to get flag: {err}")) }, - None => Err("no context found for package 'com.android.aconfig.test'".to_string()) + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } } }) }); - match aconfig_storage_result { - Ok(storage_result) if storage_result == result => { - if use_new_storage_value { - return storage_result; - } else { - return result; - } - }, - Ok(storage_result) => { - log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_exported'. Legacy storage was {result}, new storage was {storage_result}"); - if use_new_storage_value { - return storage_result; - } else { - return result; - } + match flag_value_result { + Ok(flag_value) => { + return flag_value; }, Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: {err}"); - if use_new_storage_value { - panic!("failed to read flag value: {err}"); - } + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; } } - } - - result }); /// flag value cache for disabled_rw_in_other_namespace static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| { - let result = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.other_namespace", - "com.android.aconfig.test.disabled_rw_in_other_namespace", - "false") == "true"; - - let use_new_storage_value = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; - - if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { // This will be called multiple times. Subsequent calls after the first are noops. logger::init( logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) + .with_tag_on_device("aconfig_rust_codegen") .with_max_level(LevelFilter::Info)); - let aconfig_storage_result = FLAG_VAL_MAP + let flag_value_result = FLAG_VAL_MAP .as_ref() .map_err(|err| format!("failed to get flag val map: {err}")) .and_then(|flag_val_map| { @@ -436,63 +239,38 @@ static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| .and_then(|package_offset| { match package_offset { Some(offset) => { - get_boolean_flag_value(&flag_val_map, offset + 3) + get_boolean_flag_value(&flag_val_map, offset + 2) .map_err(|err| format!("failed to get flag: {err}")) }, - None => Err("no context found for package 'com.android.aconfig.test'".to_string()) + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } } }) }); - match aconfig_storage_result { - Ok(storage_result) if storage_result == result => { - if use_new_storage_value { - return storage_result; - } else { - return result; - } - }, - Ok(storage_result) => { - log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_in_other_namespace'. Legacy storage was {result}, new storage was {storage_result}"); - if use_new_storage_value { - return storage_result; - } else { - return result; - } + match flag_value_result { + Ok(flag_value) => { + return flag_value; }, Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: {err}"); - if use_new_storage_value { - panic!("failed to read flag value: {err}"); - } + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; } } - } - - result }); /// flag value cache for enabled_rw static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| { - let result = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_rw", - "true") == "true"; - - let use_new_storage_value = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; - - if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { // This will be called multiple times. Subsequent calls after the first are noops. logger::init( logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) + .with_tag_on_device("aconfig_rust_codegen") .with_max_level(LevelFilter::Info)); - let aconfig_storage_result = FLAG_VAL_MAP + let flag_value_result = FLAG_VAL_MAP .as_ref() .map_err(|err| format!("failed to get flag val map: {err}")) .and_then(|flag_val_map| { @@ -502,40 +280,26 @@ static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| { .and_then(|package_offset| { match package_offset { Some(offset) => { - get_boolean_flag_value(&flag_val_map, offset + 8) + get_boolean_flag_value(&flag_val_map, offset + 7) .map_err(|err| format!("failed to get flag: {err}")) }, - None => Err("no context found for package 'com.android.aconfig.test'".to_string()) + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } } }) }); - match aconfig_storage_result { - Ok(storage_result) if storage_result == result => { - if use_new_storage_value { - return storage_result; - } else { - return result; - } - }, - Ok(storage_result) => { - log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'enabled_rw'. Legacy storage was {result}, new storage was {storage_result}"); - if use_new_storage_value { - return storage_result; - } else { - return result; - } + match flag_value_result { + Ok(flag_value) => { + return flag_value; }, Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: {err}"); - if use_new_storage_value { - panic!("failed to read flag value: {err}"); - } + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return true; } } - } - - result }); impl FlagProvider { @@ -596,65 +360,7 @@ pub static PROVIDER: FlagProvider = FlagProvider; /// query flag disabled_ro #[inline(always)] pub fn disabled_ro() -> bool { - - - let result = false; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - return result; - } - - // This will be called multiple times. Subsequent calls after the first - // are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); - - unsafe { - let package_map = match get_mapped_storage_file("system", StorageFileType::PackageMap) { - Ok(file) => file, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'disabled_ro': {err}"); - return result; - } - }; - - let package_read_context = match get_package_read_context(&package_map, "com.android.aconfig.test") { - Ok(Some(context)) => context, - Ok(None) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'disabled_ro': did not get context"); - return result; - }, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'disabled_ro': {err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("system", StorageFileType::FlagVal) { - Ok(val_map) => val_map, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'disabled_ro': {err}"); - return result; - } - }; - let value = match get_boolean_flag_value(&flag_val_map, 0 + package_read_context.boolean_start_index) { - Ok(val) => val, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'disabled_ro': {err}"); - return result; - } - }; - - if result != value { - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for 'disabled_ro'. Legacy storage was {result}, new storage was {value}"); - } else { - let default_value = false; - } - } - - result - + false } /// query flag disabled_rw @@ -678,276 +384,218 @@ pub fn disabled_rw_in_other_namespace() -> bool { /// query flag enabled_fixed_ro #[inline(always)] pub fn enabled_fixed_ro() -> bool { - - - let result = true; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - return result; - } - - // This will be called multiple times. Subsequent calls after the first - // are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); - - unsafe { - let package_map = match get_mapped_storage_file("system", StorageFileType::PackageMap) { - Ok(file) => file, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro': {err}"); - return result; - } - }; - - let package_read_context = match get_package_read_context(&package_map, "com.android.aconfig.test") { - Ok(Some(context)) => context, - Ok(None) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro': did not get context"); - return result; - }, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro': {err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("system", StorageFileType::FlagVal) { - Ok(val_map) => val_map, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro': {err}"); - return result; - } - }; - let value = match get_boolean_flag_value(&flag_val_map, 4 + package_read_context.boolean_start_index) { - Ok(val) => val, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro': {err}"); - return result; - } - }; - - if result != value { - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for 'enabled_fixed_ro'. Legacy storage was {result}, new storage was {value}"); - } else { - let default_value = true; - } - } - - result - + true } /// query flag enabled_fixed_ro_exported #[inline(always)] pub fn enabled_fixed_ro_exported() -> bool { + true +} +/// query flag enabled_ro +#[inline(always)] +pub fn enabled_ro() -> bool { + true +} - let result = true; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - return result; - } - - // This will be called multiple times. Subsequent calls after the first - // are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); - - unsafe { - let package_map = match get_mapped_storage_file("system", StorageFileType::PackageMap) { - Ok(file) => file, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro_exported': {err}"); - return result; - } - }; - - let package_read_context = match get_package_read_context(&package_map, "com.android.aconfig.test") { - Ok(Some(context)) => context, - Ok(None) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro_exported': did not get context"); - return result; - }, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro_exported': {err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("system", StorageFileType::FlagVal) { - Ok(val_map) => val_map, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro_exported': {err}"); - return result; - } - }; - let value = match get_boolean_flag_value(&flag_val_map, 5 + package_read_context.boolean_start_index) { - Ok(val) => val, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_fixed_ro_exported': {err}"); - return result; - } - }; +/// query flag enabled_ro_exported +#[inline(always)] +pub fn enabled_ro_exported() -> bool { + true +} - if result != value { - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for 'enabled_fixed_ro_exported'. Legacy storage was {result}, new storage was {value}"); - } else { - let default_value = true; - } - } +/// query flag enabled_rw +#[inline(always)] +pub fn enabled_rw() -> bool { + PROVIDER.enabled_rw() +} +"#; - result + const TEST_EXPECTED: &str = r#" +//! codegenerated rust flag lib +use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context}; +use std::collections::BTreeMap; +use std::path::Path; +use std::io::Write; +use std::sync::{LazyLock, Mutex}; +use log::{log, LevelFilter, Level}; +/// flag provider +pub struct FlagProvider { + overrides: BTreeMap<&'static str, bool>, } -/// query flag enabled_ro -#[inline(always)] -pub fn enabled_ro() -> bool { - +static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = LazyLock::new(|| unsafe { + get_mapped_storage_file("system", StorageFileType::PackageMap) + .and_then(|package_map| get_package_read_context(&package_map, "com.android.aconfig.test")) + .map(|context| context.map(|c| c.boolean_start_index)) +}); - let result = true; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - return result; - } +static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new(|| unsafe { + get_mapped_storage_file("system", StorageFileType::FlagVal) +}); - // This will be called multiple times. Subsequent calls after the first - // are noops. +/// flag value cache for disabled_rw +static CACHED_disabled_rw: LazyLock<bool> = LazyLock::new(|| { + // This will be called multiple times. Subsequent calls after the first are noops. logger::init( logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); + + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: {err}")) + .and_then(|flag_val_map| { + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: {err}")) + .and_then(|package_offset| { + match package_offset { + Some(offset) => { + get_boolean_flag_value(&flag_val_map, offset + 0) + .map_err(|err| format!("failed to get flag: {err}")) + }, + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } + } + }) + }); + + match flag_value_result { + Ok(flag_value) => { + return flag_value; + }, + Err(err) => { + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; + } + } +}); - unsafe { - let package_map = match get_mapped_storage_file("system", StorageFileType::PackageMap) { - Ok(file) => file, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro': {err}"); - return result; - } - }; +/// flag value cache for disabled_rw_exported +static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| { + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); + + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: {err}")) + .and_then(|flag_val_map| { + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: {err}")) + .and_then(|package_offset| { + match package_offset { + Some(offset) => { + get_boolean_flag_value(&flag_val_map, offset + 1) + .map_err(|err| format!("failed to get flag: {err}")) + }, + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } + } + }) + }); - let package_read_context = match get_package_read_context(&package_map, "com.android.aconfig.test") { - Ok(Some(context)) => context, - Ok(None) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro': did not get context"); - return result; + match flag_value_result { + Ok(flag_value) => { + return flag_value; }, Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro': {err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("system", StorageFileType::FlagVal) { - Ok(val_map) => val_map, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro': {err}"); - return result; + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; } - }; - let value = match get_boolean_flag_value(&flag_val_map, 6 + package_read_context.boolean_start_index) { - Ok(val) => val, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro': {err}"); - return result; - } - }; - - if result != value { - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for 'enabled_ro'. Legacy storage was {result}, new storage was {value}"); - } else { - let default_value = true; } - } - - result - -} - -/// query flag enabled_ro_exported -#[inline(always)] -pub fn enabled_ro_exported() -> bool { - - - let result = true; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() { - return result; - } +}); - // This will be called multiple times. Subsequent calls after the first - // are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); +/// flag value cache for disabled_rw_in_other_namespace +static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| { + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); - unsafe { - let package_map = match get_mapped_storage_file("system", StorageFileType::PackageMap) { - Ok(file) => file, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro_exported': {err}"); - return result; - } - }; + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: {err}")) + .and_then(|flag_val_map| { + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: {err}")) + .and_then(|package_offset| { + match package_offset { + Some(offset) => { + get_boolean_flag_value(&flag_val_map, offset + 2) + .map_err(|err| format!("failed to get flag: {err}")) + }, + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } + } + }) + }); - let package_read_context = match get_package_read_context(&package_map, "com.android.aconfig.test") { - Ok(Some(context)) => context, - Ok(None) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro_exported': did not get context"); - return result; + match flag_value_result { + Ok(flag_value) => { + return flag_value; }, Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro_exported': {err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("system", StorageFileType::FlagVal) { - Ok(val_map) => val_map, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro_exported': {err}"); - return result; - } - }; - let value = match get_boolean_flag_value(&flag_val_map, 7 + package_read_context.boolean_start_index) { - Ok(val) => val, - Err(err) => { - log!(Level::Error, "AconfigTestMission1: error: failed to read flag 'enabled_ro_exported': {err}"); - return result; + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return false; } - }; - - if result != value { - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for 'enabled_ro_exported'. Legacy storage was {result}, new storage was {value}"); - } else { - let default_value = true; } - } - - result - -} +}); -/// query flag enabled_rw -#[inline(always)] -pub fn enabled_rw() -> bool { - PROVIDER.enabled_rw() -} -"#; - const TEST_EXPECTED: &str = r#" -//! codegenerated rust flag lib +/// flag value cache for enabled_rw +static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| { + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); -use std::collections::BTreeMap; -use std::sync::Mutex; + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: {err}")) + .and_then(|flag_val_map| { + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: {err}")) + .and_then(|package_offset| { + match package_offset { + Some(offset) => { + get_boolean_flag_value(&flag_val_map, offset + 7) + .map_err(|err| format!("failed to get flag: {err}")) + }, + None => { + log!(Level::Error, "no context found for package com.android.aconfig.test"); + Err(format!("failed to flag package com.android.aconfig.test")) + } + } + }) + }); -/// flag provider -pub struct FlagProvider { - overrides: BTreeMap<&'static str, bool>, -} + match flag_value_result { + Ok(flag_value) => { + return flag_value; + }, + Err(err) => { + log!(Level::Error, "aconfig_rust_codegen: error: {err}"); + return true; + } + } +}); impl FlagProvider { /// query flag disabled_ro @@ -965,10 +613,7 @@ impl FlagProvider { /// query flag disabled_rw pub fn disabled_rw(&self) -> bool { self.overrides.get("disabled_rw").copied().unwrap_or( - flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw", - "false") == "true" + *CACHED_disabled_rw ) } @@ -980,10 +625,7 @@ impl FlagProvider { /// query flag disabled_rw_exported pub fn disabled_rw_exported(&self) -> bool { self.overrides.get("disabled_rw_exported").copied().unwrap_or( - flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true" + *CACHED_disabled_rw_exported ) } @@ -995,10 +637,7 @@ impl FlagProvider { /// query flag disabled_rw_in_other_namespace pub fn disabled_rw_in_other_namespace(&self) -> bool { self.overrides.get("disabled_rw_in_other_namespace").copied().unwrap_or( - flags_rust::GetServerConfigurableFlag( - "aconfig_flags.other_namespace", - "com.android.aconfig.test.disabled_rw_in_other_namespace", - "false") == "true" + *CACHED_disabled_rw_in_other_namespace ) } @@ -1058,10 +697,7 @@ impl FlagProvider { /// query flag enabled_rw pub fn enabled_rw(&self) -> bool { self.overrides.get("enabled_rw").copied().unwrap_or( - flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_rw", - "true") == "true" + *CACHED_enabled_rw ) } @@ -1195,78 +831,6 @@ pub fn reset_flags() { } "#; - const EXPORTED_EXPECTED: &str = r#" -//! codegenerated rust flag lib -use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context}; -use std::path::Path; -use std::io::Write; -use std::sync::LazyLock; -use log::{log, LevelFilter, Level}; - -static STORAGE_MIGRATION_MARKER_FILE: &str = - "/metadata/aconfig_test_missions/mission_1"; -static MIGRATION_LOG_TAG: &str = "AconfigTestMission1"; - -/// flag provider -pub struct FlagProvider; - - /// flag value cache for disabled_rw_exported - static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.disabled_rw_exported", - "false") == "true"); - - /// flag value cache for enabled_fixed_ro_exported - static CACHED_enabled_fixed_ro_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_fixed_ro_exported", - "false") == "true"); - - /// flag value cache for enabled_ro_exported - static CACHED_enabled_ro_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.aconfig_test", - "com.android.aconfig.test.enabled_ro_exported", - "false") == "true"); - -impl FlagProvider { - /// query flag disabled_rw_exported - pub fn disabled_rw_exported(&self) -> bool { - *CACHED_disabled_rw_exported - } - - /// query flag enabled_fixed_ro_exported - pub fn enabled_fixed_ro_exported(&self) -> bool { - *CACHED_enabled_fixed_ro_exported - } - - /// query flag enabled_ro_exported - pub fn enabled_ro_exported(&self) -> bool { - *CACHED_enabled_ro_exported - } -} - -/// flag provider -pub static PROVIDER: FlagProvider = FlagProvider; - -/// query flag disabled_rw_exported -#[inline(always)] -pub fn disabled_rw_exported() -> bool { - PROVIDER.disabled_rw_exported() -} - -/// query flag enabled_fixed_ro_exported -#[inline(always)] -pub fn enabled_fixed_ro_exported() -> bool { - PROVIDER.enabled_fixed_ro_exported() -} - -/// query flag enabled_ro_exported -#[inline(always)] -pub fn enabled_ro_exported() -> bool { - PROVIDER.enabled_ro_exported() -} -"#; - const FORCE_READ_ONLY_EXPECTED: &str = r#" //! codegenerated rust flag lib use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context}; @@ -1275,10 +839,6 @@ use std::io::Write; use std::sync::LazyLock; use log::{log, LevelFilter, Level}; -static STORAGE_MIGRATION_MARKER_FILE: &str = - "/metadata/aconfig_test_missions/mission_1"; -static MIGRATION_LOG_TAG: &str = "AconfigTestMission1"; - /// flag provider pub struct FlagProvider; @@ -1355,7 +915,7 @@ pub fn enabled_rw() -> bool { "#; use crate::commands::assign_flag_ids; - fn test_generate_rust_code(mode: CodegenMode, allow_instrumentation: bool, expected: &str) { + fn test_generate_rust_code(mode: CodegenMode, expected: &str) { let parsed_flags = crate::test::parse_test_flags(); let modified_parsed_flags = crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap(); @@ -1366,7 +926,6 @@ pub fn enabled_rw() -> bool { flag_ids, modified_parsed_flags.into_iter(), mode, - allow_instrumentation, ) .unwrap(); assert_eq!("src/lib.rs", format!("{}", generated.path.display())); @@ -1381,26 +940,16 @@ pub fn enabled_rw() -> bool { #[test] fn test_generate_rust_code_for_prod() { - test_generate_rust_code(CodegenMode::Production, false, PROD_EXPECTED); - } - - #[test] - fn test_generate_rust_code_for_prod_instrumented() { - test_generate_rust_code(CodegenMode::Production, true, PROD_INSTRUMENTED_EXPECTED); + test_generate_rust_code(CodegenMode::Production, PROD_EXPECTED); } #[test] fn test_generate_rust_code_for_test() { - test_generate_rust_code(CodegenMode::Test, false, TEST_EXPECTED); - } - - #[test] - fn test_generate_rust_code_for_exported() { - test_generate_rust_code(CodegenMode::Exported, false, EXPORTED_EXPECTED); + test_generate_rust_code(CodegenMode::Test, TEST_EXPECTED); } #[test] fn test_generate_rust_code_for_force_read_only() { - test_generate_rust_code(CodegenMode::ForceReadOnly, false, FORCE_READ_ONLY_EXPECTED); + test_generate_rust_code(CodegenMode::ForceReadOnly, FORCE_READ_ONLY_EXPECTED); } } diff --git a/tools/aconfig/aconfig/src/commands.rs b/tools/aconfig/aconfig/src/commands.rs index 1a14f6415a..0c80d3b938 100644 --- a/tools/aconfig/aconfig/src/commands.rs +++ b/tools/aconfig/aconfig/src/commands.rs @@ -15,15 +15,16 @@ */ use anyhow::{bail, ensure, Context, Result}; +use convert_finalized_flags::FinalizedFlagMap; use itertools::Itertools; use protobuf::Message; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::hash::Hasher; use std::io::Read; use std::path::PathBuf; use crate::codegen::cpp::generate_cpp_code; -use crate::codegen::java::generate_java_code; +use crate::codegen::java::{generate_java_code, JavaCodegenConfig}; use crate::codegen::rust::generate_rust_code; use crate::codegen::CodegenMode; use crate::dump::{DumpFormat, DumpPredicate}; @@ -69,6 +70,7 @@ pub fn parse_flags( declarations: Vec<Input>, values: Vec<Input>, default_permission: ProtoFlagPermission, + allow_read_write: bool, ) -> Result<Vec<u8>> { let mut parsed_flags = ProtoParsedFlags::new(); @@ -79,8 +81,18 @@ pub fn parse_flags( .read_to_string(&mut contents) .with_context(|| format!("failed to read {}", input.source))?; - let flag_declarations = aconfig_protos::flag_declarations::try_from_text_proto(&contents) - .with_context(|| input.error_context())?; + let mut flag_declarations = + aconfig_protos::flag_declarations::try_from_text_proto(&contents) + .with_context(|| input.error_context())?; + + // system_ext flags should be treated as system flags as we are combining /system_ext + // and /system as one container + // TODO: remove this logic when we start enforcing that system_ext cannot be set as + // container in aconfig declaration files. + if flag_declarations.container() == "system_ext" { + flag_declarations.set_container(String::from("system")); + } + ensure!( package == flag_declarations.package(), "failed to parse {}: expected package {}, got {}", @@ -185,6 +197,16 @@ pub fn parse_flags( } } + if !allow_read_write { + if let Some(pf) = parsed_flags + .parsed_flag + .iter() + .find(|pf| pf.permission() == ProtoFlagPermission::READ_WRITE) + { + bail!("flag {} has permission READ_WRITE, but allow_read_write is false", pf.name()); + } + } + // Create a sorted parsed_flags aconfig_protos::parsed_flags::sort_parsed_flags(&mut parsed_flags); aconfig_protos::parsed_flags::verify_fields(&parsed_flags)?; @@ -197,28 +219,33 @@ pub fn create_java_lib( mut input: Input, codegen_mode: CodegenMode, allow_instrumentation: bool, + new_exported: bool, + single_exported_file: bool, + finalized_flags: FinalizedFlagMap, ) -> Result<Vec<OutputFile>> { let parsed_flags = input.try_parse_flags()?; - let modified_parsed_flags = modify_parsed_flags_based_on_mode(parsed_flags, codegen_mode)?; + let modified_parsed_flags = + modify_parsed_flags_based_on_mode(parsed_flags.clone(), codegen_mode)?; let Some(package) = find_unique_package(&modified_parsed_flags) else { bail!("no parsed flags, or the parsed flags use different packages"); }; let package = package.to_string(); + let mut flag_names = extract_flag_names(parsed_flags)?; + let package_fingerprint = compute_flags_fingerprint(&mut flag_names); let flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; - generate_java_code( - &package, - modified_parsed_flags.into_iter(), + let config = JavaCodegenConfig { codegen_mode, flag_ids, allow_instrumentation, - ) + package_fingerprint, + new_exported, + single_exported_file, + finalized_flags, + }; + generate_java_code(&package, modified_parsed_flags.into_iter(), config) } -pub fn create_cpp_lib( - mut input: Input, - codegen_mode: CodegenMode, - allow_instrumentation: bool, -) -> Result<Vec<OutputFile>> { +pub fn create_cpp_lib(mut input: Input, codegen_mode: CodegenMode) -> Result<Vec<OutputFile>> { // TODO(327420679): Enable export mode for native flag library ensure!( codegen_mode != CodegenMode::Exported, @@ -231,20 +258,10 @@ pub fn create_cpp_lib( }; let package = package.to_string(); let flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; - generate_cpp_code( - &package, - modified_parsed_flags.into_iter(), - codegen_mode, - flag_ids, - allow_instrumentation, - ) + generate_cpp_code(&package, modified_parsed_flags.into_iter(), codegen_mode, flag_ids) } -pub fn create_rust_lib( - mut input: Input, - codegen_mode: CodegenMode, - allow_instrumentation: bool, -) -> Result<OutputFile> { +pub fn create_rust_lib(mut input: Input, codegen_mode: CodegenMode) -> Result<OutputFile> { // // TODO(327420679): Enable export mode for native flag library ensure!( codegen_mode != CodegenMode::Exported, @@ -257,23 +274,18 @@ pub fn create_rust_lib( }; let package = package.to_string(); let flag_ids = assign_flag_ids(&package, modified_parsed_flags.iter())?; - generate_rust_code( - &package, - flag_ids, - modified_parsed_flags.into_iter(), - codegen_mode, - allow_instrumentation, - ) + generate_rust_code(&package, flag_ids, modified_parsed_flags.into_iter(), codegen_mode) } pub fn create_storage( caches: Vec<Input>, container: &str, file: &StorageFileType, + version: u32, ) -> Result<Vec<u8>> { let parsed_flags_vec: Vec<ProtoParsedFlags> = caches.into_iter().map(|mut input| input.try_parse_flags()).collect::<Result<Vec<_>>>()?; - generate_storage_file(container, parsed_flags_vec.iter(), file) + generate_storage_file(container, parsed_flags_vec.iter(), file, version) } pub fn create_device_config_defaults(mut input: Input) -> Result<Vec<u8>> { @@ -397,38 +409,65 @@ where { assert!(parsed_flags_iter.clone().tuple_windows().all(|(a, b)| a.name() <= b.name())); let mut flag_ids = HashMap::new(); - for (id_to_assign, pf) in (0_u32..).zip(parsed_flags_iter) { + let mut flag_idx = 0; + for pf in parsed_flags_iter { if package != pf.package() { return Err(anyhow::anyhow!("encountered a flag not in current package")); } // put a cap on how many flags a package can contain to 65535 - if id_to_assign > u16::MAX as u32 { + if flag_idx > u16::MAX as u32 { return Err(anyhow::anyhow!("the number of flags in a package cannot exceed 65535")); } - flag_ids.insert(pf.name().to_string(), id_to_assign as u16); + if should_include_flag(pf) { + flag_ids.insert(pf.name().to_string(), flag_idx as u16); + flag_idx += 1; + } } Ok(flag_ids) } -#[allow(dead_code)] // TODO: b/316357686 - Use fingerprint in codegen to - // protect hardcoded offset reads. -pub fn compute_flag_offsets_fingerprint(flags_map: &HashMap<String, u16>) -> Result<u64> { - let mut hasher = SipHasher13::new(); - - // Need to sort to ensure the data is added to the hasher in the same order - // each run. - let sorted_map: BTreeMap<&String, &u16> = flags_map.iter().collect(); +// Creates a fingerprint of the flag names (which requires sorting the vector). +// Fingerprint is used by both codegen and storage files. +pub fn compute_flags_fingerprint(flag_names: &mut Vec<String>) -> u64 { + flag_names.sort(); - for (flag, offset) in sorted_map { - // See https://docs.rs/siphasher/latest/siphasher/#note for use of write - // over write_i16. Similarly, use to_be_bytes rather than to_ne_bytes to - // ensure consistency. + let mut hasher = SipHasher13::new(); + for flag in flag_names { hasher.write(flag.as_bytes()); - hasher.write(&offset.to_be_bytes()); } - Ok(hasher.finish()) + hasher.finish() +} + +// Converts ProtoParsedFlags into a vector of strings containing all of the flag +// names. Helper fn for creating fingerprint for codegen files. Flags must all +// belong to the same package. +fn extract_flag_names(flags: ProtoParsedFlags) -> Result<Vec<String>> { + let separated_flags: Vec<ProtoParsedFlag> = flags.parsed_flag.into_iter().collect::<Vec<_>>(); + + // All flags must belong to the same package as the fingerprint is per-package. + let Some(_package) = find_unique_package(&separated_flags) else { + bail!("No parsed flags, or the parsed flags use different packages."); + }; + + Ok(separated_flags + .into_iter() + .filter(should_include_flag) + .map(|flag| flag.name.unwrap()) + .collect::<Vec<_>>()) +} + +// Exclude system/vendor/product flags that are RO+disabled. +pub fn should_include_flag(pf: &ProtoParsedFlag) -> bool { + let should_filter_container = pf.container == Some("vendor".to_string()) + || pf.container == Some("system".to_string()) + || pf.container == Some("product".to_string()); + + let disabled_ro = pf.state == Some(ProtoFlagState::DISABLED.into()) + && pf.permission == Some(ProtoFlagPermission::READ_ONLY.into()); + + !should_filter_container || !disabled_ro } #[cfg(test)] @@ -439,13 +478,49 @@ mod tests { #[test] fn test_offset_fingerprint() { let parsed_flags = crate::test::parse_test_flags(); - let package = find_unique_package(&parsed_flags.parsed_flag).unwrap().to_string(); - let flag_ids = assign_flag_ids(&package, parsed_flags.parsed_flag.iter()).unwrap(); - let expected_fingerprint = 10709892481002252132u64; + let expected_fingerprint: u64 = 11551379960324242360; - let hash_result = compute_flag_offsets_fingerprint(&flag_ids); + let mut extracted_flags = extract_flag_names(parsed_flags).unwrap(); + let hash_result = compute_flags_fingerprint(&mut extracted_flags); - assert_eq!(hash_result.unwrap(), expected_fingerprint); + assert_eq!(hash_result, expected_fingerprint); + } + + #[test] + fn test_offset_fingerprint_matches_from_package() { + let parsed_flags: ProtoParsedFlags = crate::test::parse_test_flags(); + + // All test flags are in the same package, so fingerprint from all of them. + let mut extracted_flags = extract_flag_names(parsed_flags.clone()).unwrap(); + let result_from_parsed_flags = compute_flags_fingerprint(&mut extracted_flags); + + let mut flag_names_vec = parsed_flags + .parsed_flag + .clone() + .into_iter() + .filter(should_include_flag) + .map(|flag| flag.name.unwrap()) + .map(String::from) + .collect::<Vec<_>>(); + let result_from_names = compute_flags_fingerprint(&mut flag_names_vec); + + // Assert the same hash is generated for each case. + assert_eq!(result_from_parsed_flags, result_from_names); + } + + #[test] + fn test_offset_fingerprint_different_packages_does_not_match() { + // Parse flags from two packages. + let parsed_flags: ProtoParsedFlags = crate::test::parse_test_flags(); + let second_parsed_flags = crate::test::parse_second_package_flags(); + + let mut extracted_flags = extract_flag_names(parsed_flags).unwrap(); + let result_from_parsed_flags = compute_flags_fingerprint(&mut extracted_flags); + let mut second_extracted_flags = extract_flag_names(second_parsed_flags).unwrap(); + let second_result = compute_flags_fingerprint(&mut second_extracted_flags); + + // Different flags should have a different fingerprint. + assert_ne!(result_from_parsed_flags, second_result); } #[test] @@ -519,6 +594,7 @@ mod tests { declaration, value, ProtoFlagPermission::READ_ONLY, + true, ) .unwrap(); let parsed_flags = @@ -552,6 +628,7 @@ mod tests { declaration, value, ProtoFlagPermission::READ_WRITE, + true, ) .unwrap_err(); assert_eq!( @@ -583,6 +660,7 @@ mod tests { declaration, value, ProtoFlagPermission::READ_WRITE, + true, ) .unwrap_err(); assert_eq!( @@ -590,6 +668,121 @@ mod tests { "failed to parse memory: expected container argument.container, got declaration.container" ); } + #[test] + fn test_parse_flags_no_allow_read_write_default_error() { + let first_flag = r#" + package: "com.first" + container: "com.first.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let error = crate::commands::parse_flags( + "com.first", + Some("com.first.container"), + declaration, + vec![], + ProtoFlagPermission::READ_WRITE, + false, + ) + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + "flag first has permission READ_WRITE, but allow_read_write is false" + ); + } + + #[test] + fn test_parse_flags_no_allow_read_write_value_error() { + let first_flag = r#" + package: "com.first" + container: "com.first.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let first_flag_value = r#" + flag_value { + package: "com.first" + name: "first" + state: DISABLED + permission: READ_WRITE + } + "#; + let value = vec![Input { + source: "memory".to_string(), + reader: Box::new(first_flag_value.as_bytes()), + }]; + let error = crate::commands::parse_flags( + "com.first", + Some("com.first.container"), + declaration, + value, + ProtoFlagPermission::READ_ONLY, + false, + ) + .unwrap_err(); + assert_eq!( + format!("{:?}", error), + "flag first has permission READ_WRITE, but allow_read_write is false" + ); + } + + #[test] + fn test_parse_flags_no_allow_read_write_success() { + let first_flag = r#" + package: "com.first" + container: "com.first.container" + flag { + name: "first" + namespace: "first_ns" + description: "This is the description of the first flag." + bug: "123" + } + "#; + let declaration = + vec![Input { source: "memory".to_string(), reader: Box::new(first_flag.as_bytes()) }]; + + let first_flag_value = r#" + flag_value { + package: "com.first" + name: "first" + state: DISABLED + permission: READ_ONLY + } + "#; + let value = vec![Input { + source: "memory".to_string(), + reader: Box::new(first_flag_value.as_bytes()), + }]; + let flags_bytes = crate::commands::parse_flags( + "com.first", + Some("com.first.container"), + declaration, + value, + ProtoFlagPermission::READ_ONLY, + false, + ) + .unwrap(); + let parsed_flags = + aconfig_protos::parsed_flags::try_from_binary_proto(&flags_bytes).unwrap(); + assert_eq!(1, parsed_flags.parsed_flag.len()); + let parsed_flag = parsed_flags.parsed_flag.first().unwrap(); + assert_eq!(ProtoFlagState::DISABLED, parsed_flag.state()); + assert_eq!(ProtoFlagPermission::READ_ONLY, parsed_flag.permission()); + } #[test] fn test_parse_flags_override_fixed_read_only() { @@ -625,6 +818,7 @@ mod tests { declaration, value, ProtoFlagPermission::READ_WRITE, + true, ) .unwrap_err(); assert_eq!( @@ -659,6 +853,7 @@ mod tests { declaration, value, ProtoFlagPermission::READ_ONLY, + true, ) .unwrap(); let parsed_flags = @@ -699,6 +894,30 @@ mod tests { } #[test] + fn test_dump_multiple_filters() { + let input = parse_test_flags_as_input(); + let bytes = dump_parsed_flags( + vec![input], + DumpFormat::Custom("{fully_qualified_name}".to_string()), + &["container:system+state:ENABLED", "container:system+permission:READ_WRITE"], + false, + ) + .unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + let expected_flag_list = &[ + "com.android.aconfig.test.disabled_rw", + "com.android.aconfig.test.disabled_rw_exported", + "com.android.aconfig.test.disabled_rw_in_other_namespace", + "com.android.aconfig.test.enabled_fixed_ro", + "com.android.aconfig.test.enabled_fixed_ro_exported", + "com.android.aconfig.test.enabled_ro", + "com.android.aconfig.test.enabled_ro_exported", + "com.android.aconfig.test.enabled_rw", + ]; + assert_eq!(expected_flag_list.map(|s| format!("{}\n", s)).join(""), text); + } + + #[test] fn test_dump_textproto_format_dedup() { let input = parse_test_flags_as_input(); let input2 = parse_test_flags_as_input(); @@ -760,15 +979,14 @@ mod tests { let package = find_unique_package(&parsed_flags.parsed_flag).unwrap().to_string(); let flag_ids = assign_flag_ids(&package, parsed_flags.parsed_flag.iter()).unwrap(); let expected_flag_ids = HashMap::from([ - (String::from("disabled_ro"), 0_u16), - (String::from("disabled_rw"), 1_u16), - (String::from("disabled_rw_exported"), 2_u16), - (String::from("disabled_rw_in_other_namespace"), 3_u16), - (String::from("enabled_fixed_ro"), 4_u16), - (String::from("enabled_fixed_ro_exported"), 5_u16), - (String::from("enabled_ro"), 6_u16), - (String::from("enabled_ro_exported"), 7_u16), - (String::from("enabled_rw"), 8_u16), + (String::from("disabled_rw"), 0_u16), + (String::from("disabled_rw_exported"), 1_u16), + (String::from("disabled_rw_in_other_namespace"), 2_u16), + (String::from("enabled_fixed_ro"), 3_u16), + (String::from("enabled_fixed_ro_exported"), 4_u16), + (String::from("enabled_ro"), 5_u16), + (String::from("enabled_ro_exported"), 6_u16), + (String::from("enabled_rw"), 7_u16), ]); assert_eq!(flag_ids, expected_flag_ids); } diff --git a/tools/aconfig/aconfig/src/main.rs b/tools/aconfig/aconfig/src/main.rs index 1fb64f9c56..6b294239e9 100644 --- a/tools/aconfig/aconfig/src/main.rs +++ b/tools/aconfig/aconfig/src/main.rs @@ -16,6 +16,8 @@ //! `aconfig` is a build time tool to manage build time configurations, such as feature flags. +use aconfig_storage_file::DEFAULT_FILE_VERSION; +use aconfig_storage_file::MAX_SUPPORTED_FILE_VERSION; use anyhow::{anyhow, bail, Context, Result}; use clap::{builder::ArgAction, builder::EnumValueParser, Arg, ArgMatches, Command}; use core::any::Any; @@ -31,6 +33,7 @@ mod storage; use aconfig_storage_file::StorageFileType; use codegen::CodegenMode; +use convert_finalized_flags::FinalizedFlagMap; use dump::DumpFormat; #[cfg(test)] @@ -38,9 +41,86 @@ mod test; use commands::{Input, OutputFile}; +const HELP_DUMP_CACHE: &str = r#" +An aconfig cache file, created via `aconfig create-cache`. +"#; + +const HELP_DUMP_FORMAT: &str = r#" +Change the output format for each flag. + +The argument to --format is a format string. Each flag will be a copy of this string, with certain +placeholders replaced by attributes of the flag. The placeholders are + + {package} + {name} + {namespace} + {description} + {bug} + {state} + {state:bool} + {permission} + {trace} + {trace:paths} + {is_fixed_read_only} + {is_exported} + {container} + {metadata} + {fully_qualified_name} + +Note: the format strings "textproto" and "protobuf" are handled in a special way: they output all +flag attributes in text or binary protobuf format. + +Examples: + + # See which files were read to determine the value of a flag; the files were read in the order + # listed. + --format='{fully_qualified_name} {trace}' + + # Trace the files read for a specific flag. Useful during debugging. + --filter=fully_qualified_name:com.foo.flag_name --format='{trace}' + + # Print a somewhat human readable description of each flag. + --format='The flag {name} in package {package} is {state} and has permission {permission}.' +"#; + const HELP_DUMP_FILTER: &str = r#" -Limit which flags to output. If multiple --filter arguments are provided, the output will be -limited to flags that match any of the filters. +Limit which flags to output. If --filter is omitted, all flags will be printed. If multiple +--filter options are provided, the output will be limited to flags that match any of the filters. + +The argument to --filter is a search query. Multiple queries can be AND-ed together by +concatenating them with a plus sign. + +Valid queries are: + + package:<string> + name:<string> + namespace:<string> + bug:<string> + state:ENABLED|DISABLED + permission:READ_ONLY|READ_WRITE + is_fixed_read_only:true|false + is_exported:true|false + container:<string> + fully_qualified_name:<string> + +Note: there is currently no support for filtering based on these flag attributes: description, +trace, metadata. + +Examples: + + # Print a single flag: + --filter=fully_qualified_name:com.foo.flag_name + + # Print all known information about a single flag: + --filter=fully_qualified_name:com.foo.flag_name --format=textproto + + # Print all flags in the com.foo package, and all enabled flags in the com.bar package: + --filter=package:com.foo --filter=package.com.bar+state:ENABLED +"#; + +const HELP_DUMP_DEDUP: &str = r#" +Allow the same flag to be present in multiple cache files; if duplicates are found, collapse into +a single instance. "#; fn cli() -> Command { @@ -49,8 +129,7 @@ fn cli() -> Command { .subcommand( Command::new("create-cache") .arg(Arg::new("package").long("package").required(true)) - // TODO(b/312769710): Make this argument required. - .arg(Arg::new("container").long("container")) + .arg(Arg::new("container").long("container").required(true)) .arg(Arg::new("declarations").long("declarations").action(ArgAction::Append)) .arg(Arg::new("values").long("values").action(ArgAction::Append)) .arg( @@ -61,6 +140,12 @@ fn cli() -> Command { &commands::DEFAULT_FLAG_PERMISSION, )), ) + .arg( + Arg::new("allow-read-write") + .long("allow-read-write") + .value_parser(clap::value_parser!(bool)) + .default_value("true"), + ) .arg(Arg::new("cache").long("cache").required(true)), ) .subcommand( @@ -74,10 +159,34 @@ fn cli() -> Command { .default_value("production"), ) .arg( + Arg::new("single-exported-file") + .long("single-exported-file") + .value_parser(clap::value_parser!(bool)) + .default_value("false"), + ) + // TODO: b/395899938 - clean up flags for switching to new storage + .arg( Arg::new("allow-instrumentation") .long("allow-instrumentation") .value_parser(clap::value_parser!(bool)) .default_value("false"), + ) + // TODO: b/395899938 - clean up flags for switching to new storage + .arg( + Arg::new("new-exported") + .long("new-exported") + .value_parser(clap::value_parser!(bool)) + .default_value("false"), + ) + // Allows build flag toggling of checking API level in exported + // flag lib for finalized API flags. + // TODO: b/378936061 - Remove once build flag for API level + // check is fully enabled. + .arg( + Arg::new("check-api-level") + .long("check-api-level") + .value_parser(clap::value_parser!(bool)) + .default_value("false"), ), ) .subcommand( @@ -127,22 +236,34 @@ fn cli() -> Command { .subcommand( Command::new("dump-cache") .alias("dump") - .arg(Arg::new("cache").long("cache").action(ArgAction::Append)) + .arg( + Arg::new("cache") + .long("cache") + .action(ArgAction::Append) + .long_help(HELP_DUMP_CACHE.trim()), + ) .arg( Arg::new("format") .long("format") .value_parser(|s: &str| DumpFormat::try_from(s)) .default_value( "{fully_qualified_name} [{container}]: {permission} + {state}", - ), + ) + .long_help(HELP_DUMP_FORMAT.trim()), ) .arg( Arg::new("filter") .long("filter") .action(ArgAction::Append) - .help(HELP_DUMP_FILTER.trim()), + .long_help(HELP_DUMP_FILTER.trim()), + ) + .arg( + Arg::new("dedup") + .long("dedup") + .num_args(0) + .action(ArgAction::SetTrue) + .long_help(HELP_DUMP_DEDUP.trim()), ) - .arg(Arg::new("dedup").long("dedup").num_args(0).action(ArgAction::SetTrue)) .arg(Arg::new("out").long("out").default_value("-")), ) .subcommand( @@ -159,7 +280,13 @@ fn cli() -> Command { .value_parser(|s: &str| StorageFileType::try_from(s)), ) .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true)) - .arg(Arg::new("out").long("out").required(true)), + .arg(Arg::new("out").long("out").required(true)) + .arg( + Arg::new("version") + .long("version") + .required(false) + .value_parser(|s: &str| s.parse::<u32>()), + ), ) } @@ -222,6 +349,12 @@ fn write_output_to_file_or_stdout(path: &str, data: &[u8]) -> Result<()> { Ok(()) } +fn load_finalized_flags() -> Result<FinalizedFlagMap> { + let json_str = include_str!(concat!(env!("OUT_DIR"), "/finalized_flags_record.json")); + let map = serde_json::from_str(json_str)?; + Ok(map) +} + fn main() -> Result<()> { let matches = cli().get_matches(); match matches.subcommand() { @@ -235,12 +368,15 @@ fn main() -> Result<()> { sub_matches, "default-permission", )?; + let allow_read_write = get_optional_arg::<bool>(sub_matches, "allow-read-write") + .expect("failed to parse allow-read-write"); let output = commands::parse_flags( package, container, declarations, values, *default_permission, + *allow_read_write, ) .context("failed to create cache")?; let path = get_required_arg::<String>(sub_matches, "cache")?; @@ -251,8 +387,23 @@ fn main() -> Result<()> { let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; let allow_instrumentation = get_required_arg::<bool>(sub_matches, "allow-instrumentation")?; - let generated_files = commands::create_java_lib(cache, *mode, *allow_instrumentation) - .context("failed to create java lib")?; + let new_exported = get_required_arg::<bool>(sub_matches, "new-exported")?; + let single_exported_file = + get_required_arg::<bool>(sub_matches, "single-exported-file")?; + + let check_api_level = get_required_arg::<bool>(sub_matches, "check-api-level")?; + let finalized_flags: FinalizedFlagMap = + if *check_api_level { load_finalized_flags()? } else { FinalizedFlagMap::new() }; + + let generated_files = commands::create_java_lib( + cache, + *mode, + *allow_instrumentation, + *new_exported, + *single_exported_file, + finalized_flags, + ) + .context("failed to create java lib")?; let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); generated_files .iter() @@ -261,10 +412,8 @@ fn main() -> Result<()> { Some(("create-cpp-lib", sub_matches)) => { let cache = open_single_file(sub_matches, "cache")?; let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; - let allow_instrumentation = - get_required_arg::<bool>(sub_matches, "allow-instrumentation")?; - let generated_files = commands::create_cpp_lib(cache, *mode, *allow_instrumentation) - .context("failed to create cpp lib")?; + let generated_files = + commands::create_cpp_lib(cache, *mode).context("failed to create cpp lib")?; let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); generated_files .iter() @@ -273,10 +422,8 @@ fn main() -> Result<()> { Some(("create-rust-lib", sub_matches)) => { let cache = open_single_file(sub_matches, "cache")?; let mode = get_required_arg::<CodegenMode>(sub_matches, "mode")?; - let allow_instrumentation = - get_required_arg::<bool>(sub_matches, "allow-instrumentation")?; - let generated_file = commands::create_rust_lib(cache, *mode, *allow_instrumentation) - .context("failed to create rust lib")?; + let generated_file = + commands::create_rust_lib(cache, *mode).context("failed to create rust lib")?; let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?); write_output_file_realtive_to_dir(&dir, &generated_file)?; } @@ -309,12 +456,18 @@ fn main() -> Result<()> { write_output_to_file_or_stdout(path, &output)?; } Some(("create-storage", sub_matches)) => { + let version = + get_optional_arg::<u32>(sub_matches, "version").unwrap_or(&DEFAULT_FILE_VERSION); + if *version > MAX_SUPPORTED_FILE_VERSION { + bail!("Invalid version selected ({})", version); + } let file = get_required_arg::<StorageFileType>(sub_matches, "file") .context("Invalid storage file selection")?; let cache = open_zero_or_more_files(sub_matches, "cache")?; let container = get_required_arg::<String>(sub_matches, "container")?; let path = get_required_arg::<String>(sub_matches, "out")?; - let output = commands::create_storage(cache, container, file) + + let output = commands::create_storage(cache, container, file, *version) .context("failed to create storage files")?; write_output_to_file_or_stdout(path, &output)?; } diff --git a/tools/aconfig/aconfig/src/storage/flag_info.rs b/tools/aconfig/aconfig/src/storage/flag_info.rs new file mode 100644 index 0000000000..0943daa86c --- /dev/null +++ b/tools/aconfig/aconfig/src/storage/flag_info.rs @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::commands::assign_flag_ids; +use crate::storage::FlagPackage; +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState}; +use aconfig_storage_file::{FlagInfoHeader, FlagInfoList, FlagInfoNode, StorageFileType}; +use anyhow::{anyhow, Result}; + +fn new_header(container: &str, num_flags: u32, version: u32) -> FlagInfoHeader { + FlagInfoHeader { + version, + container: String::from(container), + file_type: StorageFileType::FlagInfo as u8, + file_size: 0, + num_flags, + boolean_flag_offset: 0, + } +} + +pub fn create_flag_info( + container: &str, + packages: &[FlagPackage], + version: u32, +) -> Result<FlagInfoList> { + // Exclude system/vendor/product flags that are RO+disabled. + let mut filtered_packages = packages.to_vec(); + if container == "system" || container == "vendor" || container == "product" { + for package in filtered_packages.iter_mut() { + package.boolean_flags.retain(|b| { + !(b.state == Some(ProtoFlagState::DISABLED.into()) + && b.permission == Some(ProtoFlagPermission::READ_ONLY.into())) + }); + } + } + + let num_flags = filtered_packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); + + let mut is_flag_rw = vec![false; num_flags as usize]; + for pkg in filtered_packages { + let start_index = pkg.boolean_start_index as usize; + let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?; + for pf in pkg.boolean_flags { + let fid = flag_ids + .get(pf.name()) + .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; + is_flag_rw[start_index + (*fid as usize)] = + pf.permission() == ProtoFlagPermission::READ_WRITE; + } + } + + let mut list = FlagInfoList { + header: new_header(container, num_flags, version), + nodes: is_flag_rw.iter().map(|&rw| FlagInfoNode::create(rw)).collect(), + }; + + // initialize all header fields + list.header.boolean_flag_offset = list.header.into_bytes().len() as u32; + let bytes_per_node = FlagInfoNode::create(false).into_bytes().len() as u32; + list.header.file_size = list.header.boolean_flag_offset + num_flags * bytes_per_node; + + Ok(list) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; + use aconfig_storage_file::DEFAULT_FILE_VERSION; + + pub fn create_test_flag_info_list_from_source() -> Result<FlagInfoList> { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter(), DEFAULT_FILE_VERSION); + create_flag_info("mockup", &packages, DEFAULT_FILE_VERSION) + } + + #[test] + // this test point locks down the flag info creation and each field + fn test_list_contents() { + let flag_info_list = create_test_flag_info_list_from_source(); + assert!(flag_info_list.is_ok()); + let expected_flag_info_list = + aconfig_storage_file::test_utils::create_test_flag_info_list(DEFAULT_FILE_VERSION); + assert_eq!(flag_info_list.unwrap(), expected_flag_info_list); + } +} diff --git a/tools/aconfig/aconfig/src/storage/flag_table.rs b/tools/aconfig/aconfig/src/storage/flag_table.rs index a9712119bf..a3b4e8fe1e 100644 --- a/tools/aconfig/aconfig/src/storage/flag_table.rs +++ b/tools/aconfig/aconfig/src/storage/flag_table.rs @@ -14,18 +14,17 @@ * limitations under the License. */ -use crate::commands::assign_flag_ids; +use crate::commands::{assign_flag_ids, should_include_flag}; use crate::storage::FlagPackage; use aconfig_protos::ProtoFlagPermission; use aconfig_storage_file::{ get_table_size, FlagTable, FlagTableHeader, FlagTableNode, StorageFileType, StoredFlagType, - FILE_VERSION, }; use anyhow::{anyhow, Result}; -fn new_header(container: &str, num_flags: u32) -> FlagTableHeader { +fn new_header(container: &str, num_flags: u32, version: u32) -> FlagTableHeader { FlagTableHeader { - version: FILE_VERSION, + version, container: String::from(container), file_type: StorageFileType::FlagMap as u8, file_size: 0, @@ -63,9 +62,13 @@ impl FlagTableNodeWrapper { } fn create_nodes(package: &FlagPackage, num_buckets: u32) -> Result<Vec<Self>> { + // Exclude system/vendor/product flags that are RO+disabled. + let mut filtered_package = package.clone(); + filtered_package.boolean_flags.retain(|pf| should_include_flag(pf)); + let flag_ids = - assign_flag_ids(package.package_name, package.boolean_flags.iter().copied())?; - package + assign_flag_ids(package.package_name, filtered_package.boolean_flags.iter().copied())?; + filtered_package .boolean_flags .iter() .map(|&pf| { @@ -86,12 +89,16 @@ impl FlagTableNodeWrapper { } } -pub fn create_flag_table(container: &str, packages: &[FlagPackage]) -> Result<FlagTable> { +pub fn create_flag_table( + container: &str, + packages: &[FlagPackage], + version: u32, +) -> Result<FlagTable> { // create table let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); let num_buckets = get_table_size(num_flags)?; - let mut header = new_header(container, num_flags); + let mut header = new_header(container, num_flags, version); let mut buckets = vec![None; num_buckets as usize]; let mut node_wrappers = packages .iter() @@ -138,13 +145,15 @@ pub fn create_flag_table(container: &str, packages: &[FlagPackage]) -> Result<Fl #[cfg(test)] mod tests { + use aconfig_storage_file::DEFAULT_FILE_VERSION; + use super::*; use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; fn create_test_flag_table_from_source() -> Result<FlagTable> { let caches = parse_all_test_flags(); - let packages = group_flags_by_package(caches.iter()); - create_flag_table("mockup", &packages) + let packages = group_flags_by_package(caches.iter(), DEFAULT_FILE_VERSION); + create_flag_table("mockup", &packages, DEFAULT_FILE_VERSION) } #[test] @@ -152,7 +161,8 @@ mod tests { fn test_table_contents() { let flag_table = create_test_flag_table_from_source(); assert!(flag_table.is_ok()); - let expected_flag_table = aconfig_storage_file::test_utils::create_test_flag_table(); + let expected_flag_table = + aconfig_storage_file::test_utils::create_test_flag_table(DEFAULT_FILE_VERSION); assert_eq!(flag_table.unwrap(), expected_flag_table); } } diff --git a/tools/aconfig/aconfig/src/storage/flag_value.rs b/tools/aconfig/aconfig/src/storage/flag_value.rs index c15ba54112..3cfa447098 100644 --- a/tools/aconfig/aconfig/src/storage/flag_value.rs +++ b/tools/aconfig/aconfig/src/storage/flag_value.rs @@ -16,13 +16,13 @@ use crate::commands::assign_flag_ids; use crate::storage::FlagPackage; -use aconfig_protos::ProtoFlagState; -use aconfig_storage_file::{FlagValueHeader, FlagValueList, StorageFileType, FILE_VERSION}; +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState}; +use aconfig_storage_file::{FlagValueHeader, FlagValueList, StorageFileType}; use anyhow::{anyhow, Result}; -fn new_header(container: &str, num_flags: u32) -> FlagValueHeader { +fn new_header(container: &str, num_flags: u32, version: u32) -> FlagValueHeader { FlagValueHeader { - version: FILE_VERSION, + version, container: String::from(container), file_type: StorageFileType::FlagVal as u8, file_size: 0, @@ -31,16 +31,27 @@ fn new_header(container: &str, num_flags: u32) -> FlagValueHeader { } } -pub fn create_flag_value(container: &str, packages: &[FlagPackage]) -> Result<FlagValueList> { - // create list - let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); - +pub fn create_flag_value( + container: &str, + packages: &[FlagPackage], + version: u32, +) -> Result<FlagValueList> { + // Exclude system/vendor/product flags that are RO+disabled. + let mut filtered_packages = packages.to_vec(); + if container == "system" || container == "vendor" || container == "product" { + for package in filtered_packages.iter_mut() { + package.boolean_flags.retain(|b| { + !(b.state == Some(ProtoFlagState::DISABLED.into()) + && b.permission == Some(ProtoFlagPermission::READ_ONLY.into())) + }); + } + } + let num_flags = filtered_packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); let mut list = FlagValueList { - header: new_header(container, num_flags), + header: new_header(container, num_flags, version), booleans: vec![false; num_flags as usize], }; - - for pkg in packages.iter() { + for pkg in filtered_packages { let start_index = pkg.boolean_start_index as usize; let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?; for pf in pkg.boolean_flags.iter() { @@ -61,13 +72,15 @@ pub fn create_flag_value(container: &str, packages: &[FlagPackage]) -> Result<Fl #[cfg(test)] mod tests { + use aconfig_storage_file::DEFAULT_FILE_VERSION; + use super::*; use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; pub fn create_test_flag_value_list_from_source() -> Result<FlagValueList> { let caches = parse_all_test_flags(); - let packages = group_flags_by_package(caches.iter()); - create_flag_value("mockup", &packages) + let packages = group_flags_by_package(caches.iter(), DEFAULT_FILE_VERSION); + create_flag_value("mockup", &packages, DEFAULT_FILE_VERSION) } #[test] @@ -76,7 +89,7 @@ mod tests { let flag_value_list = create_test_flag_value_list_from_source(); assert!(flag_value_list.is_ok()); let expected_flag_value_list = - aconfig_storage_file::test_utils::create_test_flag_value_list(); + aconfig_storage_file::test_utils::create_test_flag_value_list(DEFAULT_FILE_VERSION); assert_eq!(flag_value_list.unwrap(), expected_flag_value_list); } } diff --git a/tools/aconfig/aconfig/src/storage/mod.rs b/tools/aconfig/aconfig/src/storage/mod.rs index 73339f24b3..61e65d1dfc 100644 --- a/tools/aconfig/aconfig/src/storage/mod.rs +++ b/tools/aconfig/aconfig/src/storage/mod.rs @@ -14,23 +14,27 @@ * limitations under the License. */ +pub mod flag_info; pub mod flag_table; pub mod flag_value; pub mod package_table; -use anyhow::{anyhow, Result}; +use anyhow::Result; use std::collections::{HashMap, HashSet}; +use crate::commands::compute_flags_fingerprint; use crate::storage::{ - flag_table::create_flag_table, flag_value::create_flag_value, + flag_info::create_flag_info, flag_table::create_flag_table, flag_value::create_flag_value, package_table::create_package_table, }; -use aconfig_protos::{ProtoParsedFlag, ProtoParsedFlags}; +use aconfig_protos::{ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag, ProtoParsedFlags}; use aconfig_storage_file::StorageFileType; +#[derive(Clone)] pub struct FlagPackage<'a> { pub package_name: &'a str, pub package_id: u32, + pub fingerprint: u64, pub flag_names: HashSet<&'a str>, pub boolean_flags: Vec<&'a ProtoParsedFlag>, // The index of the first boolean flag in this aconfig package among all boolean @@ -43,6 +47,7 @@ impl<'a> FlagPackage<'a> { FlagPackage { package_name, package_id, + fingerprint: 0, flag_names: HashSet::new(), boolean_flags: vec![], boolean_start_index: 0, @@ -56,7 +61,7 @@ impl<'a> FlagPackage<'a> { } } -pub fn group_flags_by_package<'a, I>(parsed_flags_vec_iter: I) -> Vec<FlagPackage<'a>> +pub fn group_flags_by_package<'a, I>(parsed_flags_vec_iter: I, version: u32) -> Vec<FlagPackage<'a>> where I: Iterator<Item = &'a ProtoParsedFlags>, { @@ -69,15 +74,33 @@ where if index == packages.len() { packages.push(FlagPackage::new(parsed_flag.package(), index as u32)); } + + // Exclude system/vendor/product flags that are RO+disabled. + if (parsed_flag.container == Some("system".to_string()) + || parsed_flag.container == Some("vendor".to_string()) + || parsed_flag.container == Some("product".to_string())) + && parsed_flag.permission == Some(ProtoFlagPermission::READ_ONLY.into()) + && parsed_flag.state == Some(ProtoFlagState::DISABLED.into()) + { + continue; + } + packages[index].insert(parsed_flag); } } - // cacluate boolean flag start index for each package + // Calculate boolean flag start index for each package let mut boolean_start_index = 0; for p in packages.iter_mut() { p.boolean_start_index = boolean_start_index; boolean_start_index += p.boolean_flags.len() as u32; + + if version >= 2 { + let mut flag_names_vec = + p.flag_names.clone().into_iter().map(String::from).collect::<Vec<_>>(); + let fingerprint = compute_flags_fingerprint(&mut flag_names_vec); + p.fingerprint = fingerprint; + } } packages @@ -87,31 +110,37 @@ pub fn generate_storage_file<'a, I>( container: &str, parsed_flags_vec_iter: I, file: &StorageFileType, + version: u32, ) -> Result<Vec<u8>> where I: Iterator<Item = &'a ProtoParsedFlags>, { - let packages = group_flags_by_package(parsed_flags_vec_iter); + let packages = group_flags_by_package(parsed_flags_vec_iter, version); match file { StorageFileType::PackageMap => { - let package_table = create_package_table(container, &packages)?; + let package_table = create_package_table(container, &packages, version)?; Ok(package_table.into_bytes()) } StorageFileType::FlagMap => { - let flag_table = create_flag_table(container, &packages)?; + let flag_table = create_flag_table(container, &packages, version)?; Ok(flag_table.into_bytes()) } StorageFileType::FlagVal => { - let flag_value = create_flag_value(container, &packages)?; + let flag_value = create_flag_value(container, &packages, version)?; Ok(flag_value.into_bytes()) } - _ => Err(anyhow!("aconfig does not support the creation of this storage file type")), + StorageFileType::FlagInfo => { + let flag_info = create_flag_info(container, &packages, version)?; + Ok(flag_info.into_bytes()) + } } } #[cfg(test)] mod tests { + use aconfig_storage_file::DEFAULT_FILE_VERSION; + use super::*; use crate::Input; @@ -154,6 +183,7 @@ mod tests { reader: Box::new(value_content), }], crate::commands::DEFAULT_FLAG_PERMISSION, + true, ) .unwrap(); aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() @@ -164,7 +194,50 @@ mod tests { #[test] fn test_flag_package() { let caches = parse_all_test_flags(); - let packages = group_flags_by_package(caches.iter()); + let packages = group_flags_by_package(caches.iter(), DEFAULT_FILE_VERSION); + + for pkg in packages.iter() { + let pkg_name = pkg.package_name; + assert_eq!(pkg.flag_names.len(), pkg.boolean_flags.len()); + for pf in pkg.boolean_flags.iter() { + assert!(pkg.flag_names.contains(pf.name())); + assert_eq!(pf.package(), pkg_name); + } + } + + assert_eq!(packages.len(), 3); + + assert_eq!(packages[0].package_name, "com.android.aconfig.storage.test_1"); + assert_eq!(packages[0].package_id, 0); + assert_eq!(packages[0].flag_names.len(), 3); + assert!(packages[0].flag_names.contains("enabled_rw")); + assert!(packages[0].flag_names.contains("disabled_rw")); + assert!(packages[0].flag_names.contains("enabled_ro")); + assert_eq!(packages[0].boolean_start_index, 0); + assert_eq!(packages[0].fingerprint, 0); + + assert_eq!(packages[1].package_name, "com.android.aconfig.storage.test_2"); + assert_eq!(packages[1].package_id, 1); + assert_eq!(packages[1].flag_names.len(), 3); + assert!(packages[1].flag_names.contains("enabled_ro")); + assert!(packages[1].flag_names.contains("disabled_rw")); + assert!(packages[1].flag_names.contains("enabled_fixed_ro")); + assert_eq!(packages[1].boolean_start_index, 3); + assert_eq!(packages[0].fingerprint, 0); + + assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4"); + assert_eq!(packages[2].package_id, 2); + assert_eq!(packages[2].flag_names.len(), 2); + assert!(packages[2].flag_names.contains("enabled_rw")); + assert!(packages[2].flag_names.contains("enabled_fixed_ro")); + assert_eq!(packages[2].boolean_start_index, 6); + assert_eq!(packages[2].fingerprint, 0); + } + + #[test] + fn test_flag_package_with_fingerprint() { + let caches = parse_all_test_flags(); + let packages = group_flags_by_package(caches.iter(), 2); for pkg in packages.iter() { let pkg_name = pkg.package_name; @@ -184,6 +257,7 @@ mod tests { assert!(packages[0].flag_names.contains("disabled_rw")); assert!(packages[0].flag_names.contains("enabled_ro")); assert_eq!(packages[0].boolean_start_index, 0); + assert_eq!(packages[0].fingerprint, 15248948510590158086u64); assert_eq!(packages[1].package_name, "com.android.aconfig.storage.test_2"); assert_eq!(packages[1].package_id, 1); @@ -192,6 +266,7 @@ mod tests { assert!(packages[1].flag_names.contains("disabled_rw")); assert!(packages[1].flag_names.contains("enabled_fixed_ro")); assert_eq!(packages[1].boolean_start_index, 3); + assert_eq!(packages[1].fingerprint, 4431940502274857964u64); assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4"); assert_eq!(packages[2].package_id, 2); @@ -199,5 +274,6 @@ mod tests { assert!(packages[2].flag_names.contains("enabled_rw")); assert!(packages[2].flag_names.contains("enabled_fixed_ro")); assert_eq!(packages[2].boolean_start_index, 6); + assert_eq!(packages[2].fingerprint, 16233229917711622375u64); } } diff --git a/tools/aconfig/aconfig/src/storage/package_table.rs b/tools/aconfig/aconfig/src/storage/package_table.rs index c53602f9cb..53daa7ff2a 100644 --- a/tools/aconfig/aconfig/src/storage/package_table.rs +++ b/tools/aconfig/aconfig/src/storage/package_table.rs @@ -18,14 +18,13 @@ use anyhow::Result; use aconfig_storage_file::{ get_table_size, PackageTable, PackageTableHeader, PackageTableNode, StorageFileType, - FILE_VERSION, }; use crate::storage::FlagPackage; -fn new_header(container: &str, num_packages: u32) -> PackageTableHeader { +fn new_header(container: &str, num_packages: u32, version: u32) -> PackageTableHeader { PackageTableHeader { - version: FILE_VERSION, + version, container: String::from(container), file_type: StorageFileType::PackageMap as u8, file_size: 0, @@ -48,6 +47,7 @@ impl PackageTableNodeWrapper { let node = PackageTableNode { package_name: String::from(package.package_name), package_id: package.package_id, + fingerprint: package.fingerprint, boolean_start_index: package.boolean_start_index, next_offset: None, }; @@ -56,20 +56,26 @@ impl PackageTableNodeWrapper { } } -pub fn create_package_table(container: &str, packages: &[FlagPackage]) -> Result<PackageTable> { +pub fn create_package_table( + container: &str, + packages: &[FlagPackage], + version: u32, +) -> Result<PackageTable> { // create table let num_packages = packages.len() as u32; let num_buckets = get_table_size(num_packages)?; - let mut header = new_header(container, num_packages); + let mut header = new_header(container, num_packages, version); let mut buckets = vec![None; num_buckets as usize]; - let mut node_wrappers: Vec<_> = - packages.iter().map(|pkg| PackageTableNodeWrapper::new(pkg, num_buckets)).collect(); + let mut node_wrappers: Vec<_> = packages + .iter() + .map(|pkg: &FlagPackage<'_>| PackageTableNodeWrapper::new(pkg, num_buckets)) + .collect(); // initialize all header fields header.bucket_offset = header.into_bytes().len() as u32; header.node_offset = header.bucket_offset + num_buckets * 4; header.file_size = header.node_offset - + node_wrappers.iter().map(|x| x.node.into_bytes().len()).sum::<usize>() as u32; + + node_wrappers.iter().map(|x| x.node.into_bytes(version).len()).sum::<usize>() as u32; // sort node_wrappers by bucket index for efficiency node_wrappers.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); @@ -87,7 +93,7 @@ pub fn create_package_table(container: &str, packages: &[FlagPackage]) -> Result if buckets[node_bucket_idx as usize].is_none() { buckets[node_bucket_idx as usize] = Some(offset); } - offset += node_wrappers[i].node.into_bytes().len() as u32; + offset += node_wrappers[i].node.into_bytes(version).len() as u32; if let Some(index) = next_node_bucket_idx { if index == node_bucket_idx { @@ -106,21 +112,59 @@ pub fn create_package_table(container: &str, packages: &[FlagPackage]) -> Result #[cfg(test)] mod tests { + use aconfig_storage_file::{DEFAULT_FILE_VERSION, MAX_SUPPORTED_FILE_VERSION}; + use super::*; use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; - pub fn create_test_package_table_from_source() -> Result<PackageTable> { + pub fn create_test_package_table_from_source(version: u32) -> Result<PackageTable> { let caches = parse_all_test_flags(); - let packages = group_flags_by_package(caches.iter()); - create_package_table("mockup", &packages) + let packages = group_flags_by_package(caches.iter(), version); + create_package_table("mockup", &packages, version) } #[test] // this test point locks down the table creation and each field - fn test_table_contents() { - let package_table = create_test_package_table_from_source(); - assert!(package_table.is_ok()); - let expected_package_table = aconfig_storage_file::test_utils::create_test_package_table(); - assert_eq!(package_table.unwrap(), expected_package_table); + fn test_table_contents_default_version() { + let package_table_result = create_test_package_table_from_source(DEFAULT_FILE_VERSION); + assert!(package_table_result.is_ok()); + let package_table = package_table_result.unwrap(); + + let expected_package_table = + aconfig_storage_file::test_utils::create_test_package_table(DEFAULT_FILE_VERSION); + + assert_eq!(package_table.header, expected_package_table.header); + assert_eq!(package_table.buckets, expected_package_table.buckets); + for (node, expected_node) in + package_table.nodes.iter().zip(expected_package_table.nodes.iter()) + { + assert_eq!(node.package_name, expected_node.package_name); + assert_eq!(node.package_id, expected_node.package_id); + assert_eq!(node.boolean_start_index, expected_node.boolean_start_index); + assert_eq!(node.next_offset, expected_node.next_offset); + } + } + + #[test] + // this test point locks down the table creation and each field + fn test_table_contents_max_version() { + let package_table_result = + create_test_package_table_from_source(MAX_SUPPORTED_FILE_VERSION); + assert!(package_table_result.is_ok()); + let package_table = package_table_result.unwrap(); + + let expected_package_table = + aconfig_storage_file::test_utils::create_test_package_table(MAX_SUPPORTED_FILE_VERSION); + + assert_eq!(package_table.header, expected_package_table.header); + assert_eq!(package_table.buckets, expected_package_table.buckets); + for (node, expected_node) in + package_table.nodes.iter().zip(expected_package_table.nodes.iter()) + { + assert_eq!(node.package_name, expected_node.package_name); + assert_eq!(node.package_id, expected_node.package_id); + assert_eq!(node.boolean_start_index, expected_node.boolean_start_index); + assert_eq!(node.next_offset, expected_node.next_offset); + } } } diff --git a/tools/aconfig/aconfig/src/test.rs b/tools/aconfig/aconfig/src/test.rs index 7409cda6e8..10da252ceb 100644 --- a/tools/aconfig/aconfig/src/test.rs +++ b/tools/aconfig/aconfig/src/test.rs @@ -266,6 +266,7 @@ parsed_flag { reader: Box::new(include_bytes!("../tests/read_only_test.values").as_slice()), }], crate::commands::DEFAULT_FLAG_PERMISSION, + true, ) .unwrap(); aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() @@ -290,6 +291,26 @@ parsed_flag { }, ], crate::commands::DEFAULT_FLAG_PERMISSION, + true, + ) + .unwrap(); + aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() + } + + pub fn parse_second_package_flags() -> ProtoParsedFlags { + let bytes = crate::commands::parse_flags( + "com.android.aconfig.second_test", + Some("system"), + vec![Input { + source: "tests/test_second_package.aconfig".to_string(), + reader: Box::new(include_bytes!("../tests/test_second_package.aconfig").as_slice()), + }], + vec![Input { + source: "tests/third.values".to_string(), + reader: Box::new(include_bytes!("../tests/third.values").as_slice()), + }], + crate::commands::DEFAULT_FLAG_PERMISSION, + true, ) .unwrap(); aconfig_protos::parsed_flags::try_from_binary_proto(&bytes).unwrap() diff --git a/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template b/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template index b82b9cb827..ef183678ef 100644 --- a/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template +++ b/tools/aconfig/aconfig/templates/CustomFeatureFlags.java.template @@ -11,7 +11,13 @@ import java.util.Set; import java.util.function.BiPredicate; import java.util.function.Predicate; +{{ -if single_exported_file }} +{{ -if library_exported }} +@Deprecated {#- PREFER ExportedFlags #} +{{ -endif }} +{{ -else }} /** @hide */ +{{ -endif }} public class CustomFeatureFlags implements FeatureFlags \{ private BiPredicate<String, Predicate<FeatureFlags>> mGetValueImpl; diff --git a/tools/aconfig/aconfig/templates/ExportedFlags.java.template b/tools/aconfig/aconfig/templates/ExportedFlags.java.template new file mode 100644 index 0000000000..4e36942154 --- /dev/null +++ b/tools/aconfig/aconfig/templates/ExportedFlags.java.template @@ -0,0 +1,49 @@ +package {package_name}; {#- CODEGEN FOR EXPORTED MODE FOR NEW STORAGE SINGLE EXPORTED FILE#} + +import android.os.Build; +import android.os.flagging.AconfigPackage; +import android.util.Log; +public final class ExportedFlags \{ +{{ -for item in flag_elements}} + public static final String FLAG_{item.flag_name_constant_suffix} = "{item.device_config_flag}"; +{{- endfor }} + private static final String TAG = "ExportedFlags"; + private static volatile boolean isCached = false; +{{ for flag in flag_elements }} + private static boolean {flag.method_name} = false; +{{ -endfor }} {#- end flag_elements #} + private ExportedFlags() \{} + + private void init() \{ + try \{ + AconfigPackage reader = AconfigPackage.load("{package_name}"); + {{ -for namespace_with_flags in namespace_flags }} + {{ -for flag in namespace_with_flags.flags }} + {{ -if flag.finalized_sdk_present }} + {flag.method_name} = Build.VERSION.SDK_INT >= {flag.finalized_sdk_value} ? true : reader.getBooleanFlagValue("{flag.flag_name}", {flag.default_value}); + {{ - else }} {#- else finalized_sdk_present #} + {flag.method_name} = reader.getBooleanFlagValue("{flag.flag_name}", {flag.default_value}); + {{ -endif}} {#- end finalized_sdk_present#} + {{ -endfor }} {#- end namespace_with_flags.flags #} + {{ -endfor }} {#- end namespace_flags #} + } catch (Exception e) \{ + // pass + Log.e(TAG, e.toString()); + } catch (LinkageError e) \{ + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.w(TAG, e.toString()); + } + isCached = true; + } + +{{ -for flag in flag_elements }} + public static boolean {flag.method_name}() \{ + if (!featureFlags.isCached) \{ + featureFlags.init(); + } + return featureFlags.{flag.method_name}; + } +{{ -endfor }} + private static ExportedFlags featureFlags = new ExportedFlags(); +} diff --git a/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template index 290d2c4b24..ed277ae27d 100644 --- a/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template +++ b/tools/aconfig/aconfig/templates/FakeFeatureFlagsImpl.java.template @@ -4,7 +4,13 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Predicate; +{{ -if single_exported_file }} +{{ -if library_exported }} +@Deprecated {#- PREFER ExportedFlags #} +{{ -endif }} +{{ -else }} /** @hide */ +{{ -endif }} public class FakeFeatureFlagsImpl extends CustomFeatureFlags \{ private final Map<String, Boolean> mFlagMap = new HashMap<>(); private final FeatureFlags mDefaults; diff --git a/tools/aconfig/aconfig/templates/FeatureFlags.java.template b/tools/aconfig/aconfig/templates/FeatureFlags.java.template index 38c8f13aaf..c8b9b7f263 100644 --- a/tools/aconfig/aconfig/templates/FeatureFlags.java.template +++ b/tools/aconfig/aconfig/templates/FeatureFlags.java.template @@ -3,7 +3,16 @@ package {package_name}; // TODO(b/303773055): Remove the annotation after access issue is resolved. import android.compat.annotation.UnsupportedAppUsage; {{ -endif }} +{{ -if single_exported_file }} +{{ -if library_exported }} +/** + * @deprecated Use \{@link ExportedFlags} instead. + */ +@Deprecated {#- PREFER ExportedFlags #} +{{ -endif }} +{{ -else }} /** @hide */ +{{ -endif }} public interface FeatureFlags \{ {{ for item in flag_elements }} {{ -if not item.is_read_write }} @@ -19,4 +28,4 @@ public interface FeatureFlags \{ {{ -endif }} boolean {item.method_name}(); {{ -endfor }} -}
\ No newline at end of file +} diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.deviceConfig.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.deviceConfig.java.template new file mode 100644 index 0000000000..44d5cc019b --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.deviceConfig.java.template @@ -0,0 +1,68 @@ +package {package_name}; +{{ if not library_exported- }} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -endif }} {#- end of not library_exported#} +{{ -if runtime_lookup_required }} +import android.os.Binder; +import android.provider.DeviceConfig; +import android.provider.DeviceConfig.Properties; +{{ -endif }} {#- end of runtime_lookup_required#} +/** @hide */ +public final class FeatureFlagsImpl implements FeatureFlags \{ +{{ -if runtime_lookup_required }} +{{ -for namespace_with_flags in namespace_flags }} + private static volatile boolean {namespace_with_flags.namespace}_is_cached = false; +{{ -endfor- }} +{{ for flag in flag_elements }} +{{- if flag.is_read_write }} + private static boolean {flag.method_name} = {flag.default_value}; +{{ -endif }} {#- end of is_read_write#} +{{ -endfor }} +{{ for namespace_with_flags in namespace_flags }} + private void load_overrides_{namespace_with_flags.namespace}() \{ + final long ident = Binder.clearCallingIdentity(); + try \{ + Properties properties = DeviceConfig.getProperties("{namespace_with_flags.namespace}"); +{{ -for flag in namespace_with_flags.flags }} +{{ -if flag.is_read_write }} + {flag.method_name} = + properties.getBoolean(Flags.FLAG_{flag.flag_name_constant_suffix}, {flag.default_value}); +{{ -endif }} {#- end of is_read_write#} +{{ -endfor }} + } catch (NullPointerException e) \{ + throw new RuntimeException( + "Cannot read value from namespace {namespace_with_flags.namespace} " + + "from DeviceConfig. It could be that the code using flag " + + "executed before SettingsProvider initialization. Please use " + + "fixed read-only flag by adding is_fixed_read_only: true in " + + "flag declaration.", + e + ); + } catch (SecurityException e) \{ + // for isolated process case, skip loading flag value from the storage, use the default + } finally \{ + Binder.restoreCallingIdentity(ident); + } + {namespace_with_flags.namespace}_is_cached = true; +} +{{ endfor- }} +{{ -endif }}{#- end of runtime_lookup_required #} +{{ -for flag in flag_elements }} + @Override +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage +{{ -endif }}{#- end of not library_exported #} + public boolean {flag.method_name}() \{ +{{ -if flag.is_read_write }} + if (!{flag.device_config_namespace}_is_cached) \{ + load_overrides_{flag.device_config_namespace}(); + } + return {flag.method_name}; +{{ -else }} {#- else is_read_write #} + return {flag.default_value}; +{{ -endif }}{#- end of is_read_write #} + } +{{ endfor }} +} diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.exported.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.exported.java.template new file mode 100644 index 0000000000..b843ec2441 --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.exported.java.template @@ -0,0 +1,53 @@ +package {package_name}; {#- CODEGEN FOR EXPORTED MODE FOR NEW STORAGE #} + +import android.os.Build; +import android.os.flagging.AconfigPackage; +import android.util.Log; +{{ -if single_exported_file }} +{{ -if library_exported }} +/** + * @deprecated Use \{@link ExportedFlags} instead. + */ +@Deprecated {#- PREFER ExportedFlags #} +{{ -endif }} +{{ -else }} +/** @hide */ +{{ -endif }} +public final class FeatureFlagsImpl implements FeatureFlags \{ + private static final String TAG = "FeatureFlagsImplExport"; + private static volatile boolean isCached = false; +{{ for flag in flag_elements }} + private static boolean {flag.method_name} = false; +{{ -endfor }} {#- end flag_elements #} + private void init() \{ + try \{ + AconfigPackage reader = AconfigPackage.load("{package_name}"); + {{ -for namespace_with_flags in namespace_flags }} + {{ -for flag in namespace_with_flags.flags }} + {{ -if flag.finalized_sdk_present }} + {flag.method_name} = Build.VERSION.SDK_INT >= {flag.finalized_sdk_value} ? true : reader.getBooleanFlagValue("{flag.flag_name}", {flag.default_value}); + {{ - else }} {#- else finalized_sdk_present #} + {flag.method_name} = reader.getBooleanFlagValue("{flag.flag_name}", {flag.default_value}); + {{ -endif}} {#- end finalized_sdk_present#} + {{ -endfor }} {#- end namespace_with_flags.flags #} + {{ -endfor }} {#- end namespace_flags #} + } catch (Exception e) \{ + // pass + Log.e(TAG, e.toString()); + } catch (LinkageError e) \{ + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.w(TAG, e.toString()); + } + isCached = true; + } +{{ -for flag in flag_elements }} + @Override + public boolean {flag.method_name}() \{ + if (!isCached) \{ + init(); + } + return {flag.method_name}; + } +{{ endfor }} {#- end flag_elements #} +} diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template deleted file mode 100644 index 9970b1f300..0000000000 --- a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template +++ /dev/null @@ -1,147 +0,0 @@ -package {package_name}; -{{ -if not is_test_mode }} -{{ if not library_exported- }} -// TODO(b/303773055): Remove the annotation after access issue is resolved. -import android.compat.annotation.UnsupportedAppUsage; -{{ -endif }} - -{{ -if runtime_lookup_required }} -import android.provider.DeviceConfig; -import android.provider.DeviceConfig.Properties; - - -{{ -if not library_exported }} -{{ -if allow_instrumentation }} -import android.aconfig.storage.StorageInternalReader; -import android.util.Log; -{{ -endif }} -{{ -endif }} - -{{ -endif }} -/** @hide */ -public final class FeatureFlagsImpl implements FeatureFlags \{ -{{ -if runtime_lookup_required }} -{{ -for namespace_with_flags in namespace_flags }} - private static volatile boolean {namespace_with_flags.namespace}_is_cached = false; -{{ -endfor- }} - -{{ for flag in flag_elements }} -{{- if flag.is_read_write }} - private static boolean {flag.method_name} = {flag.default_value}; -{{ -endif }} -{{ -endfor }} -{{ -if not library_exported }} -{{ -if allow_instrumentation }} - StorageInternalReader reader; - boolean readFromNewStorage; - - boolean useNewStorageValueAndDiscardOld = false; - - private final static String TAG = "AconfigJavaCodegen"; - private final static String SUCCESS_LOG = "success: %s value matches"; - private final static String MISMATCH_LOG = "error: %s value mismatch, new storage value is %s, old storage value is %s"; - private final static String ERROR_LOG = "error: failed to read flag value"; - - private void init() \{ - if (reader != null) return; - if (DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.storage_test_mission_1", false)) \{ - readFromNewStorage = true; - try \{ - reader = new StorageInternalReader("{container}", "{package_name}"); - } catch (Exception e) \{ - reader = null; - } - } - - useNewStorageValueAndDiscardOld = - DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.use_new_storage_value", false); - } - -{{ -endif }} -{{ -endif }} -{{ for namespace_with_flags in namespace_flags }} - private void load_overrides_{namespace_with_flags.namespace}() \{ - try \{ - Properties properties = DeviceConfig.getProperties("{namespace_with_flags.namespace}"); -{{ -for flag in namespace_with_flags.flags }} -{{ -if flag.is_read_write }} - {flag.method_name} = - properties.getBoolean(Flags.FLAG_{flag.flag_name_constant_suffix}, {flag.default_value}); -{{ -endif }} -{{ -endfor }} - } catch (NullPointerException e) \{ - throw new RuntimeException( - "Cannot read value from namespace {namespace_with_flags.namespace} " - + "from DeviceConfig. It could be that the code using flag " - + "executed before SettingsProvider initialization. Please use " - + "fixed read-only flag by adding is_fixed_read_only: true in " - + "flag declaration.", - e - ); - } - {namespace_with_flags.namespace}_is_cached = true; -{{ -if not library_exported }} -{{ -if allow_instrumentation }} - init(); - if (readFromNewStorage && reader != null) \{ - boolean val; - try \{ -{{ -for flag in namespace_with_flags.flags }} -{{ -if flag.is_read_write }} - - val = reader.getBooleanFlagValue({flag.flag_offset}); - if (val == {flag.method_name}) \{ - Log.i(TAG, String.format(SUCCESS_LOG, "{flag.method_name}")); - } else \{ - Log.i(TAG, String.format(MISMATCH_LOG, "{flag.method_name}", val, {flag.method_name})); - } - - if (useNewStorageValueAndDiscardOld) \{ - {flag.method_name} = val; - } - -{{ -endif }} -{{ -endfor }} - } catch (Exception e) \{ - Log.e(TAG, ERROR_LOG, e); - } - } -{{ -endif }} -{{ -endif }} - } -{{ endfor- }} -{{ -endif }}{#- end of runtime_lookup_required #} -{{ -for flag in flag_elements }} - @Override -{{ -if not library_exported }} - @com.android.aconfig.annotations.AconfigFlagAccessor - @UnsupportedAppUsage -{{ -endif }} - public boolean {flag.method_name}() \{ -{{ -if flag.is_read_write }} - if (!{flag.device_config_namespace}_is_cached) \{ - load_overrides_{flag.device_config_namespace}(); - } - return {flag.method_name}; -{{ -else }} - return {flag.default_value}; -{{ -endif }} - } -{{ endfor }} -} -{{ else }} -{#- Generate only stub if in test mode #} -/** @hide */ -public final class FeatureFlagsImpl implements FeatureFlags \{ -{{ for flag in flag_elements }} - @Override -{{ -if not library_exported }} - @com.android.aconfig.annotations.AconfigFlagAccessor -{{ -endif }} - public boolean {flag.method_name}() \{ - throw new UnsupportedOperationException( - "Method is not implemented."); - } -{{ endfor- }} -} -{{ endif }} diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.new_storage.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.new_storage.java.template new file mode 100644 index 0000000000..9492a8335a --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.new_storage.java.template @@ -0,0 +1,63 @@ +package {package_name}; {#- CODEGEN FOR INTERNAL MODE FOR NEW STORAGE #} +// TODO(b/303773055): Remove the annotation after access issue is resolved. +import android.compat.annotation.UnsupportedAppUsage; +{{ -if runtime_lookup_required }} +import android.os.Build; +{{ if is_platform_container }} +import android.os.flagging.PlatformAconfigPackageInternal; +{{ -else }} {#- else is_platform_container #} +import android.os.flagging.AconfigPackageInternal; +{{ -endif }} {#- end of is_platform_container#} +import android.util.Log; +{{ -endif }} {#- end of runtime_lookup_required#} +/** @hide */ +public final class FeatureFlagsImpl implements FeatureFlags \{ +{{ -if runtime_lookup_required }} + private static final String TAG = "FeatureFlagsImpl"; + private static volatile boolean isCached = false; +{{ for flag in flag_elements }} +{{ -if flag.is_read_write }} + private static boolean {flag.method_name} = {flag.default_value}; +{{ -endif }} {#- end of is_read_write#} +{{ -endfor }} {#- else flag_elements #} + + private void init() \{ + try \{ +{{ if is_platform_container }} + PlatformAconfigPackageInternal reader = PlatformAconfigPackageInternal.load("{package_name}", {package_fingerprint}); +{{ -else }} {#- else is_platform_container #} + AconfigPackageInternal reader = AconfigPackageInternal.load("{package_name}", {package_fingerprint}); +{{ -endif }} {#- end of is_platform_container#} + {{ -for namespace_with_flags in namespace_flags }} + {{ -for flag in namespace_with_flags.flags }} + {{ -if flag.is_read_write }} + {flag.method_name} = reader.getBooleanFlagValue({flag.flag_offset}); + {{ -endif }} {#- is_read_write#} + {{ -endfor }} {#- else namespace_with_flags.flags #} + {{ -endfor }} {#- else namespace_flags #} + } catch (Exception e) \{ + Log.e(TAG, e.toString()); + } catch (LinkageError e) \{ + // for mainline module running on older devices. + // This should be replaces to version check, after the version bump. + Log.e(TAG, e.toString()); + } + isCached = true; + } +{{ -endif }}{#- end of runtime_lookup_required #} +{{ -for flag in flag_elements }} + @Override + @com.android.aconfig.annotations.AconfigFlagAccessor + @UnsupportedAppUsage + public boolean {flag.method_name}() \{ +{{ -if flag.is_read_write }} + if (!isCached) \{ + init(); + } + return {flag.method_name}; +{{ -else }}{#- else is_read_write #} + return {flag.default_value}; +{{ -endif }} {#- end of is_read_write#} + } +{{ endfor }} {#- else flag_elements #} +} diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.test_mode.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.test_mode.java.template new file mode 100644 index 0000000000..8eda26310e --- /dev/null +++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.test_mode.java.template @@ -0,0 +1,14 @@ +package {package_name}; {#- CODEGEN FOR TEST MODE #} +/** @hide */ +public final class FeatureFlagsImpl implements FeatureFlags \{ +{{ for flag in flag_elements }} + @Override +{{ -if not library_exported }} + @com.android.aconfig.annotations.AconfigFlagAccessor +{{ -endif }} + public boolean {flag.method_name}() \{ + throw new UnsupportedOperationException( + "Method is not implemented."); + } +{{ endfor- }} +} diff --git a/tools/aconfig/aconfig/templates/Flags.java.template b/tools/aconfig/aconfig/templates/Flags.java.template index e2f70b95fa..8a92d331b1 100644 --- a/tools/aconfig/aconfig/templates/Flags.java.template +++ b/tools/aconfig/aconfig/templates/Flags.java.template @@ -3,7 +3,16 @@ package {package_name}; // TODO(b/303773055): Remove the annotation after access issue is resolved. import android.compat.annotation.UnsupportedAppUsage; {{ -endif }} +{{ -if single_exported_file }} +{{ -if library_exported }} +/** + * @deprecated Use \{@link ExportedFlags} instead. + */ +@Deprecated {#- PREFER ExportedFlags #} +{{ -endif }} +{{ -else }} /** @hide */ +{{ -endif }} public final class Flags \{ {{ -for item in flag_elements}} /** @hide */ diff --git a/tools/aconfig/aconfig/templates/cpp_exported_header.template b/tools/aconfig/aconfig/templates/cpp_exported_header.template index 0f7853e405..f6f576a29e 100644 --- a/tools/aconfig/aconfig/templates/cpp_exported_header.template +++ b/tools/aconfig/aconfig/templates/cpp_exported_header.template @@ -27,12 +27,13 @@ public: {{ -for item in class_elements}} virtual bool {item.flag_name}() = 0; + {{ -endfor }} + {{ -if is_test_mode }} + {{ -for item in class_elements}} virtual void {item.flag_name}(bool val) = 0; - {{ -endif }} {{ -endfor }} - {{ -if is_test_mode }} virtual void reset_flags() \{} {{ -endif }} }; @@ -40,6 +41,7 @@ public: extern std::unique_ptr<flag_provider_interface> provider_; {{ for item in class_elements}} +{{ if not is_test_mode }}{{ if item.is_fixed_read_only }}constexpr {{ endif }}{{ endif -}} inline bool {item.flag_name}() \{ {{ -if is_test_mode }} return provider_->{item.flag_name}(); diff --git a/tools/aconfig/aconfig/templates/cpp_source_file.template b/tools/aconfig/aconfig/templates/cpp_source_file.template index b6012e7a0d..36ab774f54 100644 --- a/tools/aconfig/aconfig/templates/cpp_source_file.template +++ b/tools/aconfig/aconfig/templates/cpp_source_file.template @@ -1,18 +1,13 @@ #include "{header}.h" -{{ if allow_instrumentation }} {{ if readwrite- }} -#include <sys/stat.h> +#include <unistd.h> #include "aconfig_storage/aconfig_storage_read_api.hpp" #include <android/log.h> #define LOG_TAG "aconfig_cpp_codegen" -#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) {{ -endif }} -{{ endif }} -{{ if readwrite- }} -#include <server_configurable_flags/get_flags.h> -{{ endif }} {{ if is_test_mode }} #include <unordered_map> #include <string> @@ -29,32 +24,103 @@ namespace {cpp_namespace} \{ private: std::unordered_map<std::string, bool> overrides_; + {{ if readwrite- }} + uint32_t boolean_start_index_; + + std::unique_ptr<aconfig_storage::MappedStorageFile> flag_value_file_; + + bool package_exists_in_storage_; + {{ -endif }} + public: + {{ if readwrite- }} + flag_provider() + : overrides_() + , boolean_start_index_() + , flag_value_file_(nullptr) + , package_exists_in_storage_(true) \{ + + auto package_map_file = aconfig_storage::get_mapped_file( + "{container}", + aconfig_storage::StorageFileType::package_map); + + if (!package_map_file.ok()) \{ + ALOGE("error: failed to get package map file: %s", package_map_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + auto context = aconfig_storage::get_package_read_context( + **package_map_file, "{package}"); + + if (!context.ok()) \{ + ALOGE("error: failed to get package read context: %s", context.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + if (!(context->package_exists)) \{ + package_exists_in_storage_ = false; + return; + } + + // cache package boolean flag start index + boolean_start_index_ = context->boolean_start_index; + + // unmap package map file and free memory + delete *package_map_file; + + auto flag_value_file = aconfig_storage::get_mapped_file( + "{container}", + aconfig_storage::StorageFileType::flag_val); + if (!flag_value_file.ok()) \{ + ALOGE("error: failed to get flag value file: %s", flag_value_file.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + // cache flag value file + flag_value_file_ = std::unique_ptr<aconfig_storage::MappedStorageFile>( + *flag_value_file); + + } + {{ -else }} flag_provider() : overrides_() \{} + {{ -endif }} -{{ for item in class_elements }} + {{ for item in class_elements }} virtual bool {item.flag_name}() override \{ auto it = overrides_.find("{item.flag_name}"); - if (it != overrides_.end()) \{ - return it->second; + if (it != overrides_.end()) \{ + return it->second; } else \{ - {{ if item.readwrite- }} - return server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.{item.device_config_namespace}", - "{item.device_config_flag}", - "{item.default_value}") == "true"; - {{ -else }} - return {item.default_value}; - {{ -endif }} + {{ if item.readwrite- }} + if (!package_exists_in_storage_) \{ + return {item.default_value}; + } + + auto value = aconfig_storage::get_boolean_flag_value( + *flag_value_file_, + boolean_start_index_ + {item.flag_offset}); + + if (!value.ok()) \{ + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return {item.default_value}; + } else \{ + return *value; + } + {{ -else }} + return {item.default_value}; + {{ -endif }} } } virtual void {item.flag_name}(bool val) override \{ overrides_["{item.flag_name}"] = val; } -{{ endfor }} + {{ endfor }} virtual void reset_flags() override \{ overrides_.clear(); @@ -66,38 +132,32 @@ namespace {cpp_namespace} \{ class flag_provider : public flag_provider_interface \{ public: - {{ if allow_instrumentation- }} {{ if readwrite- }} flag_provider() - {{ if readwrite- }} : cache_({readwrite_count}, -1) , boolean_start_index_() - {{ -else- }} - : boolean_start_index_() - {{ -endif }} , flag_value_file_(nullptr) - , read_from_new_storage_(false) - , use_new_storage_value(false) \{ - - struct stat buffer; - if (stat("/metadata/aconfig_test_missions/mission_1", &buffer) == 0) \{ - read_from_new_storage_ = true; - } else \{ - return; - } + , package_exists_in_storage_(true) \{ auto package_map_file = aconfig_storage::get_mapped_file( "{container}", aconfig_storage::StorageFileType::package_map); if (!package_map_file.ok()) \{ - ALOGI("error: failed to get package map file: %s", package_map_file.error().c_str()); + ALOGE("error: failed to get package map file: %s", package_map_file.error().c_str()); + package_exists_in_storage_ = false; return; } auto context = aconfig_storage::get_package_read_context( **package_map_file, "{package}"); if (!context.ok()) \{ - ALOGI("error: failed to get package read context: %s", context.error().c_str()); + ALOGE("error: failed to get package read context: %s", context.error().c_str()); + package_exists_in_storage_ = false; + return; + } + + if (!(context->package_exists)) \{ + package_exists_in_storage_ = false; return; } @@ -111,7 +171,8 @@ namespace {cpp_namespace} \{ "{container}", aconfig_storage::StorageFileType::flag_val); if (!flag_value_file.ok()) \{ - ALOGI("error: failed to get flag value file: %s", flag_value_file.error().c_str()); + ALOGE("error: failed to get flag value file: %s", flag_value_file.error().c_str()); + package_exists_in_storage_ = false; return; } @@ -119,30 +180,15 @@ namespace {cpp_namespace} \{ flag_value_file_ = std::unique_ptr<aconfig_storage::MappedStorageFile>( *flag_value_file); - use_new_storage_value = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; } {{ -endif }} - {{ -endif }} {{ -for item in class_elements }} virtual bool {item.flag_name}() override \{ {{ -if item.readwrite }} if (cache_[{item.readwrite_idx}] == -1) \{ - cache_[{item.readwrite_idx}] = server_configurable_flags::GetServerConfigurableFlag( - "aconfig_flags.{item.device_config_namespace}", - "{item.device_config_flag}", - "{item.default_value}") == "true"; - } - - - {{ if allow_instrumentation- }} - if (read_from_new_storage_) \{ - if (!flag_value_file_) \{ - ALOGI("error: failed to get flag {item.flag_name}: flag value file is null"); - return cache_[{item.readwrite_idx}]; + if (!package_exists_in_storage_) \{ + return {item.default_value}; } auto value = aconfig_storage::get_boolean_flag_value( @@ -150,25 +196,12 @@ namespace {cpp_namespace} \{ boolean_start_index_ + {item.flag_offset}); if (!value.ok()) \{ - ALOGI("error: failed to read flag value: %s", value.error().c_str()); - return cache_[{item.readwrite_idx}]; + ALOGE("error: failed to read flag value: %s", value.error().c_str()); + return {item.default_value}; } - bool expected_value = cache_[{item.readwrite_idx}]; - if (*value != expected_value) \{ - ALOGI("error: {item.flag_name} value mismatch, new storage value is %s, old storage value is %s", - *value ? "true" : "false", expected_value ? "true" : "false"); - } - - if (use_new_storage_value) \{ - return *value; - } else \{ - return expected_value; - } + cache_[{item.readwrite_idx}] = *value; } - {{ -endif }} - - return cache_[{item.readwrite_idx}]; {{ -else }} {{ -if item.is_fixed_read_only }} @@ -183,14 +216,12 @@ namespace {cpp_namespace} \{ {{ if readwrite- }} private: std::vector<int8_t> cache_ = std::vector<int8_t>({readwrite_count}, -1); - {{ if allow_instrumentation- }} + uint32_t boolean_start_index_; std::unique_ptr<aconfig_storage::MappedStorageFile> flag_value_file_; - bool read_from_new_storage_; - bool use_new_storage_value; - {{ -endif }} + bool package_exists_in_storage_; {{ -endif }} }; diff --git a/tools/aconfig/aconfig/templates/rust.template b/tools/aconfig/aconfig/templates/rust.template index ea1c60081c..56323e25ca 100644 --- a/tools/aconfig/aconfig/templates/rust.template +++ b/tools/aconfig/aconfig/templates/rust.template @@ -5,15 +5,10 @@ use std::io::Write; use std::sync::LazyLock; use log::\{log, LevelFilter, Level}; -static STORAGE_MIGRATION_MARKER_FILE: &str = - "/metadata/aconfig_test_missions/mission_1"; -static MIGRATION_LOG_TAG: &str = "AconfigTestMission1"; - /// flag provider pub struct FlagProvider; {{ if has_readwrite- }} -{{ if allow_instrumentation }} static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = LazyLock::new(|| unsafe \{ get_mapped_storage_file("{container}", StorageFileType::PackageMap) .and_then(|package_map| get_package_read_context(&package_map, "{package}")) @@ -23,81 +18,50 @@ static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = Lazy static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new(|| unsafe \{ get_mapped_storage_file("{container}", StorageFileType::FlagVal) }); -{{ -endif }} {{ -for flag in template_flags }} {{ -if flag.readwrite }} /// flag value cache for {flag.name} -{{ if allow_instrumentation }} static CACHED_{flag.name}: LazyLock<bool> = LazyLock::new(|| \{ - let result = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.{flag.device_config_namespace}", - "{flag.device_config_flag}", - "{flag.default_value}") == "true"; - - let use_new_storage_value = flags_rust::GetServerConfigurableFlag( - "aconfig_flags.core_experiments_team_internal", - "com.android.providers.settings.use_new_storage_value", - "false") == "true"; - if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() \{ - // This will be called multiple times. Subsequent calls after the first are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info)); - - let aconfig_storage_result = FLAG_VAL_MAP - .as_ref() - .map_err(|err| format!("failed to get flag val map: \{err}")) - .and_then(|flag_val_map| \{ - PACKAGE_OFFSET - .as_ref() - .map_err(|err| format!("failed to get package read offset: \{err}")) - .and_then(|package_offset| \{ - match package_offset \{ - Some(offset) => \{ - get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset}) - .map_err(|err| format!("failed to get flag: \{err}")) - }, - None => Err("no context found for package '{package}'".to_string()) + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); + + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: \{err}")) + .and_then(|flag_val_map| \{ + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: \{err}")) + .and_then(|package_offset| \{ + match package_offset \{ + Some(offset) => \{ + get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset}) + .map_err(|err| format!("failed to get flag: \{err}")) + }, + None => \{ + log!(Level::Error, "no context found for package {package}"); + Err(format!("failed to flag package {package}")) } - }) - }); - - match aconfig_storage_result \{ - Ok(storage_result) if storage_result == result => \{ - if use_new_storage_value \{ - return storage_result; - } else \{ - return result; - } - }, - Ok(storage_result) => \{ - log!(Level::Error, "AconfigTestMission1: error: mismatch for flag '{flag.name}'. Legacy storage was \{result}, new storage was \{storage_result}"); - if use_new_storage_value \{ - return storage_result; - } else \{ - return result; - } - }, - Err(err) => \{ - log!(Level::Error, "AconfigTestMission1: error: \{err}"); - if use_new_storage_value \{ - panic!("failed to read flag value: \{err}"); - } - } + } + }) + }); + + match flag_value_result \{ + Ok(flag_value) => \{ + return flag_value; + }, + Err(err) => \{ + log!(Level::Error, "aconfig_rust_codegen: error: \{err}"); + return {flag.default_value}; } } - result - }); -{{ else }} -static CACHED_{flag.name}: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag( - "aconfig_flags.{flag.device_config_namespace}", - "{flag.device_config_flag}", - "{flag.default_value}") == "true"); -{{ endif }} +}); {{ -endif }} {{ -endfor }} {{ -endif }} @@ -123,72 +87,11 @@ pub static PROVIDER: FlagProvider = FlagProvider; {{ for flag in template_flags }} /// query flag {flag.name} #[inline(always)] -{{ -if flag.readwrite }} pub fn {flag.name}() -> bool \{ +{{ -if flag.readwrite }} PROVIDER.{flag.name}() {{ -else }} -pub fn {flag.name}() -> bool \{ - {{ if not allow_instrumentation }} {flag.default_value} - {{ else }} - - let result = {flag.default_value}; - if !Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() \{ - return result; - } - - // This will be called multiple times. Subsequent calls after the first - // are noops. - logger::init( - logger::Config::default() - .with_tag_on_device(MIGRATION_LOG_TAG) - .with_max_level(LevelFilter::Info), - ); - - unsafe \{ - let package_map = match get_mapped_storage_file("{flag.container}", StorageFileType::PackageMap) \{ - Ok(file) => file, - Err(err) => \{ - log!(Level::Error, "AconfigTestMission1: error: failed to read flag '{flag.name}': \{err}"); - return result; - } - }; - - let package_read_context = match get_package_read_context(&package_map, "{package}") \{ - Ok(Some(context)) => context, - Ok(None) => \{ - log!(Level::Error, "AconfigTestMission1: error: failed to read flag '{flag.name}': did not get context"); - return result; - }, - Err(err) => \{ - log!(Level::Error, "AconfigTestMission1: error: failed to read flag '{flag.name}': \{err}"); - return result; - } - }; - let flag_val_map = match get_mapped_storage_file("{flag.container}", StorageFileType::FlagVal) \{ - Ok(val_map) => val_map, - Err(err) => \{ - log!(Level::Error, "AconfigTestMission1: error: failed to read flag '{flag.name}': \{err}"); - return result; - } - }; - let value = match get_boolean_flag_value(&flag_val_map, {flag.flag_offset} + package_read_context.boolean_start_index) \{ - Ok(val) => val, - Err(err) => \{ - log!(Level::Error, "AconfigTestMission1: error: failed to read flag '{flag.name}': \{err}"); - return result; - } - }; - - if result != value \{ - log!(Level::Error, "AconfigTestMission1: error: flag mismatch for '{flag.name}'. Legacy storage was \{result}, new storage was \{value}"); - } else \{ - let default_value = {flag.default_value}; - } - } - - result - {{ endif }} {{ -endif }} } {{ endfor }} diff --git a/tools/aconfig/aconfig/templates/rust_test.template b/tools/aconfig/aconfig/templates/rust_test.template index d01f40aab7..139a5ec62a 100644 --- a/tools/aconfig/aconfig/templates/rust_test.template +++ b/tools/aconfig/aconfig/templates/rust_test.template @@ -1,23 +1,81 @@ //! codegenerated rust flag lib - +use aconfig_storage_read_api::\{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context}; use std::collections::BTreeMap; -use std::sync::Mutex; +use std::path::Path; +use std::io::Write; +use std::sync::\{LazyLock, Mutex}; +use log::\{log, LevelFilter, Level}; /// flag provider pub struct FlagProvider \{ overrides: BTreeMap<&'static str, bool>, } +{{ if has_readwrite- }} +static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = LazyLock::new(|| unsafe \{ + get_mapped_storage_file("{container}", StorageFileType::PackageMap) + .and_then(|package_map| get_package_read_context(&package_map, "{package}")) + .map(|context| context.map(|c| c.boolean_start_index)) +}); + +static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new(|| unsafe \{ + get_mapped_storage_file("{container}", StorageFileType::FlagVal) +}); + +{{ -for flag in template_flags }} +{{ -if flag.readwrite }} +/// flag value cache for {flag.name} +static CACHED_{flag.name}: LazyLock<bool> = LazyLock::new(|| \{ + + // This will be called multiple times. Subsequent calls after the first are noops. + logger::init( + logger::Config::default() + .with_tag_on_device("aconfig_rust_codegen") + .with_max_level(LevelFilter::Info)); + + let flag_value_result = FLAG_VAL_MAP + .as_ref() + .map_err(|err| format!("failed to get flag val map: \{err}")) + .and_then(|flag_val_map| \{ + PACKAGE_OFFSET + .as_ref() + .map_err(|err| format!("failed to get package read offset: \{err}")) + .and_then(|package_offset| \{ + match package_offset \{ + Some(offset) => \{ + get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset}) + .map_err(|err| format!("failed to get flag: \{err}")) + }, + None => \{ + log!(Level::Error, "no context found for package {package}"); + Err(format!("failed to flag package {package}")) + } + } + }) + }); + + match flag_value_result \{ + Ok(flag_value) => \{ + return flag_value; + }, + Err(err) => \{ + log!(Level::Error, "aconfig_rust_codegen: error: \{err}"); + return {flag.default_value}; + } + } + +}); +{{ -endif }} +{{ -endfor }} +{{ -endif }} + impl FlagProvider \{ {{ for flag in template_flags }} /// query flag {flag.name} pub fn {flag.name}(&self) -> bool \{ self.overrides.get("{flag.name}").copied().unwrap_or( {{ if flag.readwrite -}} - flags_rust::GetServerConfigurableFlag( - "aconfig_flags.{flag.device_config_namespace}", - "{flag.device_config_flag}", - "{flag.default_value}") == "true" + *CACHED_{flag.name} {{ -else- }} {flag.default_value} {{ -endif }} diff --git a/tools/aconfig/aconfig/tests/test_second_package.aconfig b/tools/aconfig/aconfig/tests/test_second_package.aconfig new file mode 100644 index 0000000000..188bc96cfb --- /dev/null +++ b/tools/aconfig/aconfig/tests/test_second_package.aconfig @@ -0,0 +1,10 @@ +package: "com.android.aconfig.second_test" +container: "system" + +flag { + name: "testing_flag" + namespace: "another_namespace" + description: "This is a flag for testing." + bug: "123" +} + diff --git a/tools/aconfig/aconfig/tests/third.values b/tools/aconfig/aconfig/tests/third.values new file mode 100644 index 0000000000..675832a4bc --- /dev/null +++ b/tools/aconfig/aconfig/tests/third.values @@ -0,0 +1,6 @@ +flag_value { + package: "com.android.aconfig.second_test" + name: "testing_flag" + state: DISABLED + permission: READ_WRITE +} diff --git a/tools/aconfig/aconfig_device_paths/Android.bp b/tools/aconfig/aconfig_device_paths/Android.bp index 95cecf4ada..3531450e49 100644 --- a/tools/aconfig/aconfig_device_paths/Android.bp +++ b/tools/aconfig/aconfig_device_paths/Android.bp @@ -26,7 +26,6 @@ rust_defaults { "libaconfig_protos", "libanyhow", "libprotobuf", - "libregex", ], } @@ -35,6 +34,11 @@ rust_library { crate_name: "aconfig_device_paths", host_supported: true, defaults: ["libaconfig_device_paths.defaults"], + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } genrule { @@ -51,4 +55,47 @@ java_library { static_libs: [ "libaconfig_java_proto_nano", ], + sdk_version: "core_platform", + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", +} + +genrule { + name: "libaconfig_java_host_device_paths_src", + srcs: ["src/HostDeviceProtosTemplate.java"], + out: ["HostDeviceProtos.java"], + tool_files: [ + "partition_aconfig_flags_paths.txt", + "mainline_aconfig_flags_paths.txt", + ], + cmd: "sed -e '/TEMPLATE/{r$(location partition_aconfig_flags_paths.txt)' -e 'd}' $(in) > $(out).tmp && " + + "sed -e '/MAINLINE_T/{r$(location mainline_aconfig_flags_paths.txt)' -e 'd}' $(out).tmp > $(out)", +} + +java_library_host { + name: "aconfig_host_device_paths_java", + srcs: [":libaconfig_java_host_device_paths_src"], +} + +genrule { + name: "java_device_paths_test_util_src", + srcs: ["src/DeviceProtosTestUtilTemplate.java"], + out: ["DeviceProtosTestUtil.java"], + tool_files: ["partition_aconfig_flags_paths.txt"], + cmd: "sed -e '/TEMPLATE/{r$(location partition_aconfig_flags_paths.txt)' -e 'd}' $(in) > $(out)", +} + +java_library { + name: "aconfig_device_paths_java_util", + srcs: [":java_device_paths_test_util_src"], + static_libs: [ + "libaconfig_java_proto_nano", + ], + sdk_version: "core_platform", + apex_available: [ + "//apex_available:platform", + ], } diff --git a/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt b/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt new file mode 100644 index 0000000000..aad2b23896 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt @@ -0,0 +1,20 @@ +"/apex/com.android.adservices/etc/aconfig_flags.pb", +"/apex/com.android.appsearch/etc/aconfig_flags.pb", +"/apex/com.android.art/etc/aconfig_flags.pb", +"/apex/com.android.bt/etc/aconfig_flags.pb", +"/apex/com.android.cellbroadcast/etc/aconfig_flags.pb", +"/apex/com.android.configinfrastructure/etc/aconfig_flags.pb", +"/apex/com.android.conscrypt/etc/aconfig_flags.pb", +"/apex/com.android.devicelock/etc/aconfig_flags.pb", +"/apex/com.android.healthfitness/etc/aconfig_flags.pb", +"/apex/com.android.ipsec/etc/aconfig_flags.pb", +"/apex/com.android.media/etc/aconfig_flags.pb", +"/apex/com.android.mediaprovider/etc/aconfig_flags.pb", +"/apex/com.android.ondevicepersonalization/etc/aconfig_flags.pb", +"/apex/com.android.os.statsd/etc/aconfig_flags.pb", +"/apex/com.android.permission/etc/aconfig_flags.pb", +"/apex/com.android.profiling/etc/aconfig_flags.pb", +"/apex/com.android.tethering/etc/aconfig_flags.pb", +"/apex/com.android.uwb/etc/aconfig_flags.pb", +"/apex/com.android.virt/etc/aconfig_flags.pb", +"/apex/com.android.wifi/etc/aconfig_flags.pb", diff --git a/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt b/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt index 140cd21ac8..e997e3ddfa 100644 --- a/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt +++ b/tools/aconfig/aconfig_device_paths/partition_aconfig_flags_paths.txt @@ -1,4 +1,3 @@ "/system/etc/aconfig_flags.pb", -"/system_ext/etc/aconfig_flags.pb", "/product/etc/aconfig_flags.pb", "/vendor/etc/aconfig_flags.pb", diff --git a/tools/aconfig/aconfig_device_paths/src/DeviceProtosTemplate.java b/tools/aconfig/aconfig_device_paths/src/DeviceProtosTemplate.java index 58c58deb47..4d4119947f 100644 --- a/tools/aconfig/aconfig_device_paths/src/DeviceProtosTemplate.java +++ b/tools/aconfig/aconfig_device_paths/src/DeviceProtosTemplate.java @@ -29,7 +29,7 @@ import java.util.List; * @hide */ public class DeviceProtos { - static final String[] PATHS = { + public static final String[] PATHS = { TEMPLATE }; @@ -50,10 +50,11 @@ public class DeviceProtos { ArrayList<parsed_flag> result = new ArrayList(); for (String path : parsedFlagsProtoPaths()) { - FileInputStream inputStream = new FileInputStream(path); - parsed_flags parsedFlags = parsed_flags.parseFrom(inputStream.readAllBytes()); - for (parsed_flag flag : parsedFlags.parsedFlag) { - result.add(flag); + try (FileInputStream inputStream = new FileInputStream(path)) { + parsed_flags parsedFlags = parsed_flags.parseFrom(inputStream.readAllBytes()); + for (parsed_flag flag : parsedFlags.parsedFlag) { + result.add(flag); + } } } @@ -64,7 +65,7 @@ public class DeviceProtos { * Returns the list of all on-device aconfig protos paths. * @hide */ - private static List<String> parsedFlagsProtoPaths() { + public static List<String> parsedFlagsProtoPaths() { ArrayList<String> paths = new ArrayList(Arrays.asList(PATHS)); File apexDirectory = new File(APEX_DIR); diff --git a/tools/aconfig/aconfig_device_paths/src/DeviceProtosTestUtilTemplate.java b/tools/aconfig/aconfig_device_paths/src/DeviceProtosTestUtilTemplate.java new file mode 100644 index 0000000000..45d67663ef --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/src/DeviceProtosTestUtilTemplate.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.aconfig; + +import android.aconfig.nano.Aconfig.parsed_flag; +import android.aconfig.nano.Aconfig.parsed_flags; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** @hide */ +public class DeviceProtosTestUtil { + public static final String[] PATHS = { + TEMPLATE + }; + + private static final String APEX_DIR = "/apex/"; + private static final String APEX_ACONFIG_PATH_SUFFIX = "/etc/aconfig_flags.pb"; + private static final String SYSTEM_APEX_DIR = "/system/apex"; + + /** + * Returns a list of all on-device aconfig protos. + * + * <p>May throw an exception if the protos can't be read at the call site. For example, some of + * the protos are in the apex/ partition, which is mounted somewhat late in the boot process. + * + * @throws IOException if we can't read one of the protos yet + * @return a list of all on-device aconfig protos + */ + public static List<parsed_flag> loadAndParseFlagProtos() throws IOException { + ArrayList<parsed_flag> result = new ArrayList(); + + for (String path : parsedFlagsProtoPaths()) { + try (FileInputStream inputStream = new FileInputStream(path)) { + parsed_flags parsedFlags = parsed_flags.parseFrom(inputStream.readAllBytes()); + for (parsed_flag flag : parsedFlags.parsedFlag) { + result.add(flag); + } + } + } + + return result; + } + + /** + * Returns the list of all on-device aconfig protos paths. + * + * @hide + */ + public static List<String> parsedFlagsProtoPaths() { + ArrayList<String> paths = new ArrayList(Arrays.asList(PATHS)); + + File apexDirectory = new File(SYSTEM_APEX_DIR); + if (!apexDirectory.isDirectory()) { + return paths; + } + + File[] subdirs = apexDirectory.listFiles(); + if (subdirs == null) { + return paths; + } + + for (File prefix : subdirs) { + String apexName = prefix.getName().replace("com.google", "com"); + apexName = apexName.substring(0, apexName.lastIndexOf('.')); + + File protoPath = new File(APEX_DIR + apexName + APEX_ACONFIG_PATH_SUFFIX); + if (!protoPath.exists()) { + continue; + } + + paths.add(protoPath.getAbsolutePath()); + } + return paths; + } +} diff --git a/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java b/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java new file mode 100644 index 0000000000..e7d0a76a8a --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.aconfig; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A host lib that can read all aconfig proto file paths on a given device. + * This lib is only available on device with root access (userdebug/eng). + */ +public class HostDeviceProtos { + /** + * An interface that executes ADB command and return the result. + */ + public static interface AdbCommandExecutor { + /** Executes the ADB command. */ + String executeAdbCommand(String command); + } + + static final String[] PATHS = { + TEMPLATE + }; + + static final String[] MAINLINE_PATHS = { + MAINLINE_T + }; + + private static final String APEX_DIR = "/apex"; + private static final String RECURSIVELY_LIST_APEX_DIR_COMMAND = + "shell su 0 find /apex | grep aconfig_flags"; + private static final String APEX_ACONFIG_PATH_SUFFIX = "/etc/aconfig_flags.pb"; + + + /** + * Returns the list of all on-device aconfig proto paths from host side. + */ + public static List<String> parsedFlagsProtoPaths(AdbCommandExecutor adbCommandExecutor) { + ArrayList<String> paths = new ArrayList(Arrays.asList(PATHS)); + + String adbCommandOutput = adbCommandExecutor.executeAdbCommand( + RECURSIVELY_LIST_APEX_DIR_COMMAND); + + if (adbCommandOutput == null || adbCommandOutput.isEmpty()) { + paths.addAll(Arrays.asList(MAINLINE_PATHS)); + return paths; + } + + Set<String> allFiles = new HashSet<>(Arrays.asList(adbCommandOutput.split("\n"))); + + Set<String> subdirs = allFiles.stream().map(file -> { + String[] filePaths = file.split("/"); + // The first element is "", the second element is "apex". + return filePaths.length > 2 ? filePaths[2] : ""; + }).collect(Collectors.toSet()); + + for (String prefix : subdirs) { + // For each mainline modules, there are two directories, one <modulepackage>/, + // and one <modulepackage>@<versioncode>/. Just read the former. + if (prefix.contains("@")) { + continue; + } + + String protoPath = APEX_DIR + "/" + prefix + APEX_ACONFIG_PATH_SUFFIX; + if (allFiles.contains(protoPath)) { + paths.add(protoPath); + } + } + return paths; + } +} diff --git a/tools/aconfig/aconfig_device_paths/src/lib.rs b/tools/aconfig/aconfig_device_paths/src/lib.rs index 9ab9cea267..8871b4f8ac 100644 --- a/tools/aconfig/aconfig_device_paths/src/lib.rs +++ b/tools/aconfig/aconfig_device_paths/src/lib.rs @@ -62,13 +62,12 @@ mod tests { #[test] fn test_read_partition_paths() { - assert_eq!(read_partition_paths().len(), 4); + assert_eq!(read_partition_paths().len(), 3); assert_eq!( read_partition_paths(), vec![ PathBuf::from("/system/etc/aconfig_flags.pb"), - PathBuf::from("/system_ext/etc/aconfig_flags.pb"), PathBuf::from("/product/etc/aconfig_flags.pb"), PathBuf::from("/vendor/etc/aconfig_flags.pb") ] diff --git a/tools/aconfig/aconfig_device_paths/test/Android.bp b/tools/aconfig/aconfig_device_paths/test/Android.bp new file mode 100644 index 0000000000..37f561ff81 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/test/Android.bp @@ -0,0 +1,35 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_team: "trendy_team_android_core_experiments", + default_applicable_licenses: ["Android-Apache-2.0"], +} + +android_test { + name: "aconfig_device_paths_java_test", + srcs: [ + "src/**/*.java", + ], + static_libs: [ + "androidx.test.runner", + "junit", + "aconfig_device_paths_java_util", + ], + test_suites: [ + "general-tests", + ], + platform_apis: true, + certificate: "platform", +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/AndroidManifest.xml b/tools/aconfig/aconfig_device_paths/test/AndroidManifest.xml index 78bfb37dc9..5e01879157 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/java/AndroidManifest.xml +++ b/tools/aconfig/aconfig_device_paths/test/AndroidManifest.xml @@ -15,12 +15,13 @@ ~ limitations under the License. --> -<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="android.aconfig_storage.test"> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="android.aconfig.storage.test"> <application> <uses-library android:name="android.test.runner" /> </application> <instrumentation android:name="androidx.test.runner.AndroidJUnitRunner" - android:targetPackage="android.aconfig_storage.test" /> + android:targetPackage="android.aconfig.storage.test" /> </manifest> diff --git a/tools/aconfig/aconfig_device_paths/test/src/DeviceProtosTestUtilTest.java b/tools/aconfig/aconfig_device_paths/test/src/DeviceProtosTestUtilTest.java new file mode 100644 index 0000000000..8dd0fd0065 --- /dev/null +++ b/tools/aconfig/aconfig_device_paths/test/src/DeviceProtosTestUtilTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.aconfig.test; + +import static org.junit.Assert.assertTrue; + +import android.aconfig.DeviceProtosTestUtil; +import android.aconfig.nano.Aconfig.parsed_flag; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; +import java.util.Set; + +@RunWith(JUnit4.class) +public class DeviceProtosTestUtilTest { + + private static final Set<String> PLATFORM_CONTAINERS = Set.of("system", "vendor", "product"); + + @Test + public void testDeviceProtos_loadAndParseFlagProtos() throws Exception { + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); + int platformFlags = 0; + int mainlineFlags = 0; + for (parsed_flag pf : flags) { + if (PLATFORM_CONTAINERS.contains(pf.container)) { + platformFlags++; + } else { + mainlineFlags++; + } + } + + assertTrue(platformFlags > 3); + assertTrue(mainlineFlags > 3); + } +} diff --git a/tools/aconfig/aconfig_flags/Android.bp b/tools/aconfig/aconfig_flags/Android.bp index e327ced26c..1b4e148ce3 100644 --- a/tools/aconfig/aconfig_flags/Android.bp +++ b/tools/aconfig/aconfig_flags/Android.bp @@ -24,6 +24,11 @@ rust_library { "libaconfig_flags_rust", ], host_supported: true, + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } aconfig_declarations { @@ -38,9 +43,19 @@ rust_aconfig_library { crate_name: "aconfig_flags_rust", aconfig_declarations: "aconfig_flags", host_supported: true, + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } cc_aconfig_library { name: "libaconfig_flags_cc", aconfig_declarations: "aconfig_flags", } + +java_aconfig_library { + name: "aconfig_flags_java", + aconfig_declarations: "aconfig_flags", +} diff --git a/tools/aconfig/aconfig_flags/flags.aconfig b/tools/aconfig/aconfig_flags/flags.aconfig index db8b1b7904..2488b5c8ab 100644 --- a/tools/aconfig/aconfig_flags/flags.aconfig +++ b/tools/aconfig/aconfig_flags/flags.aconfig @@ -7,3 +7,37 @@ flag { bug: "312235596" description: "When enabled, aconfig flags are read from the new aconfig storage only." } + +flag { + name: "enable_aconfigd_from_mainline" + namespace: "core_experiments_team_internal" + bug: "369808805" + description: "When enabled, launch aconfigd from config infra module." +} + +flag { + name: "tools_read_from_new_storage" + namespace: "core_experiments_team_internal" + bug: "370499640" + description: "When enabled, tools read directly from the new aconfig storage." +} + +flag { + name: "tools_read_from_new_storage_bugfix" + namespace: "core_experiments_team_internal" + bug: "370499640" + description: "When enabled, tools read directly from the new aconfig storage." + metadata { + purpose: PURPOSE_BUGFIX + } +} + +flag { + name: "invoke_updatable_aflags" + namespace: "core_experiments_team_internal" + bug: "385383899" + description: "When enabled, the system aflags binary invokes the updatable aflags." + metadata { + purpose: PURPOSE_BUGFIX + } +} diff --git a/tools/aconfig/aconfig_flags/src/lib.rs b/tools/aconfig/aconfig_flags/src/lib.rs index a607efb7d4..dc507aef6f 100644 --- a/tools/aconfig/aconfig_flags/src/lib.rs +++ b/tools/aconfig/aconfig_flags/src/lib.rs @@ -34,6 +34,16 @@ pub mod auto_generated { pub fn enable_only_new_storage() -> bool { aconfig_flags_rust::enable_only_new_storage() } + + /// Returns the value for the enable_aconfigd_from_mainline flag. + pub fn enable_aconfigd_from_mainline() -> bool { + aconfig_flags_rust::enable_only_new_storage() + } + + /// Returns the value for the invoke_updatable_aflags flag. + pub fn invoke_updatable_aflags() -> bool { + aconfig_flags_rust::invoke_updatable_aflags() + } } /// Module used when building with cargo @@ -44,4 +54,16 @@ pub mod auto_generated { // Used only to enable typechecking and testing with cargo true } + + /// Returns a placeholder value for the enable_aconfigd_from_mainline flag. + pub fn enable_aconfigd_from_mainline() -> bool { + // Used only to enable typechecking and testing with cargo + true + } + + /// Returns the value for the invoke_updatable_aflags flag. + pub fn invoke_updatable_aflags() -> bool { + // Used only to enable typechecking and testing with cargo + true + } } diff --git a/tools/aconfig/aconfig_protos/Android.bp b/tools/aconfig/aconfig_protos/Android.bp index d24199443c..080688ebbc 100644 --- a/tools/aconfig/aconfig_protos/Android.bp +++ b/tools/aconfig/aconfig_protos/Android.bp @@ -58,6 +58,11 @@ rust_protobuf { crate_name: "aconfig_rust_proto", source_stem: "aconfig_rust_proto", host_supported: true, + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } rust_defaults { @@ -81,6 +86,11 @@ rust_library { crate_name: "aconfig_protos", host_supported: true, defaults: ["aconfig_protos.defaults"], + apex_available: [ + "//apex_available:platform", + "com.android.configinfrastructure", + ], + min_sdk_version: "34", } rust_test_host { @@ -88,3 +98,13 @@ rust_test_host { test_suites: ["general-tests"], defaults: ["aconfig_protos.defaults"], } + +// Internal protos + +python_library_host { + name: "aconfig_internal_proto_python", + srcs: ["protos/aconfig_internal.proto"], + proto: { + canonical_path_from_root: false, + }, +} diff --git a/tools/aconfig/aconfig_protos/protos/aconfig_internal.proto b/tools/aconfig/aconfig_protos/protos/aconfig_internal.proto new file mode 100644 index 0000000000..7930f568fc --- /dev/null +++ b/tools/aconfig/aconfig_protos/protos/aconfig_internal.proto @@ -0,0 +1,42 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +// This is the schema definition for protos intended for internal aconfig +// use ONLY. There are no guarantees regarding backwards compatibility. +// Do not put protos here intended for storage or communication. + +syntax = "proto2"; + +package android.aconfig_internal; + + +// This protobuf defines messages used to store data about flags used to guard +// APIs which are finalized for a given SDK. +message finalized_flag { + // Name of the flag (required). Does not include package name. + // Must match flag name in the aconfig declaration header. + optional string name = 1; + + // Package the flag belongs to (required). Must match package in the aconfig declaration header. + optional string package = 2; + + // SDK level in which the flag was finalized. + optional int32 min_sdk = 3; + + // TODO - b/378936061: Add support for minor SDK version & SDK extension. +}; + +message finalized_flags { + repeated finalized_flag finalized_flag = 1; +} diff --git a/tools/aconfig/aconfig_protos/src/lib.rs b/tools/aconfig/aconfig_protos/src/lib.rs index 81bbd7e130..64b82d6796 100644 --- a/tools/aconfig/aconfig_protos/src/lib.rs +++ b/tools/aconfig/aconfig_protos/src/lib.rs @@ -1073,4 +1073,63 @@ parsed_flag { // two identical flags with dedup enabled assert_eq!(first, parsed_flags::merge(vec![first.clone(), first.clone()], true).unwrap()); } + + #[test] + fn test_is_valid_name_ident() { + assert!(is_valid_name_ident("foo")); + assert!(is_valid_name_ident("foo_bar_123")); + assert!(is_valid_name_ident("foo_")); + + assert!(!is_valid_name_ident("")); + assert!(!is_valid_name_ident("123_foo")); + assert!(!is_valid_name_ident("foo-bar")); + assert!(!is_valid_name_ident("foo-b\u{00e5}r")); + assert!(!is_valid_name_ident("foo__bar")); + assert!(!is_valid_name_ident("_foo")); + } + + #[test] + fn test_is_valid_package_ident() { + assert!(is_valid_package_ident("foo.bar")); + assert!(is_valid_package_ident("foo.bar_baz")); + assert!(is_valid_package_ident("foo.bar.a123")); + + assert!(!is_valid_package_ident("foo_bar_123")); + assert!(!is_valid_package_ident("foo")); + assert!(!is_valid_package_ident("foo._bar")); + assert!(!is_valid_package_ident("")); + assert!(!is_valid_package_ident("123_foo")); + assert!(!is_valid_package_ident("foo-bar")); + assert!(!is_valid_package_ident("foo-b\u{00e5}r")); + assert!(!is_valid_package_ident("foo.bar.123")); + assert!(!is_valid_package_ident(".foo.bar")); + assert!(!is_valid_package_ident("foo.bar.")); + assert!(!is_valid_package_ident(".")); + assert!(!is_valid_package_ident("..")); + assert!(!is_valid_package_ident("foo..bar")); + assert!(!is_valid_package_ident("foo.__bar")); + } + + #[test] + fn test_is_valid_container_ident() { + assert!(is_valid_container_ident("foo.bar")); + assert!(is_valid_container_ident("foo.bar_baz")); + assert!(is_valid_container_ident("foo.bar.a123")); + assert!(is_valid_container_ident("foo")); + assert!(is_valid_container_ident("foo_bar_123")); + + assert!(!is_valid_container_ident("")); + assert!(!is_valid_container_ident("foo._bar")); + assert!(!is_valid_container_ident("_foo")); + assert!(!is_valid_container_ident("123_foo")); + assert!(!is_valid_container_ident("foo-bar")); + assert!(!is_valid_container_ident("foo-b\u{00e5}r")); + assert!(!is_valid_container_ident("foo.bar.123")); + assert!(!is_valid_container_ident(".foo.bar")); + assert!(!is_valid_container_ident("foo.bar.")); + assert!(!is_valid_container_ident(".")); + assert!(!is_valid_container_ident("..")); + assert!(!is_valid_container_ident("foo..bar")); + assert!(!is_valid_container_ident("foo.__bar")); + } } diff --git a/tools/aconfig/aconfig_storage_file/Android.bp b/tools/aconfig/aconfig_storage_file/Android.bp index 40b4464167..e875c7be6a 100644 --- a/tools/aconfig/aconfig_storage_file/Android.bp +++ b/tools/aconfig/aconfig_storage_file/Android.bp @@ -14,6 +14,7 @@ rust_defaults { "libclap", "libcxx", "libaconfig_storage_protos", + "libserde", ], } @@ -36,7 +37,10 @@ rust_binary_host { name: "aconfig-storage", defaults: ["aconfig_storage_file.defaults"], srcs: ["src/main.rs"], - rustlibs: ["libaconfig_storage_file"], + rustlibs: [ + "libaconfig_storage_file", + "libserde_json", + ], } rust_test_host { diff --git a/tools/aconfig/aconfig_storage_file/Cargo.toml b/tools/aconfig/aconfig_storage_file/Cargo.toml index 192dfad40a..a40557803f 100644 --- a/tools/aconfig/aconfig_storage_file/Cargo.toml +++ b/tools/aconfig/aconfig_storage_file/Cargo.toml @@ -14,6 +14,8 @@ tempfile = "3.9.0" thiserror = "1.0.56" clap = { version = "4.1.8", features = ["derive"] } cxx = "1.0" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" [[bin]] name = "aconfig-storage" diff --git a/tools/aconfig/aconfig_storage_file/src/flag_info.rs b/tools/aconfig/aconfig_storage_file/src/flag_info.rs index beac38d156..cf16834be2 100644 --- a/tools/aconfig/aconfig_storage_file/src/flag_info.rs +++ b/tools/aconfig/aconfig_storage_file/src/flag_info.rs @@ -20,10 +20,11 @@ use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; use crate::{AconfigStorageError, StorageFileType}; use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use std::fmt; /// Flag info header struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagInfoHeader { pub version: u32, pub container: String, @@ -89,7 +90,7 @@ impl FlagInfoHeader { } /// bit field for flag info -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum FlagInfoBit { HasServerOverride = 1 << 0, IsReadWrite = 1 << 1, @@ -97,7 +98,7 @@ pub enum FlagInfoBit { } /// Flag info node struct -#[derive(PartialEq, Clone)] +#[derive(PartialEq, Clone, Serialize, Deserialize)] pub struct FlagInfoNode { pub attributes: u8, } @@ -138,7 +139,7 @@ impl FlagInfoNode { } /// Flag info list struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagInfoList { pub header: FlagInfoHeader, pub nodes: Vec<FlagInfoNode>, @@ -193,12 +194,15 @@ impl FlagInfoList { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::create_test_flag_info_list; + use crate::{ + test_utils::create_test_flag_info_list, DEFAULT_FILE_VERSION, MAX_SUPPORTED_FILE_VERSION, + }; - #[test] // this test point locks down the value list serialization - fn test_serialization() { - let flag_info_list = create_test_flag_info_list(); + // TODO: b/376108268 - Use parameterized tests. + #[test] + fn test_serialization_default() { + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); let header: &FlagInfoHeader = &flag_info_list.header; let reinterpreted_header = FlagInfoHeader::from_bytes(&header.into_bytes()); @@ -219,20 +223,42 @@ mod tests { } #[test] + fn test_serialization_max() { + let flag_info_list = create_test_flag_info_list(MAX_SUPPORTED_FILE_VERSION); + + let header: &FlagInfoHeader = &flag_info_list.header; + let reinterpreted_header = FlagInfoHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<FlagInfoNode> = &flag_info_list.nodes; + for node in nodes.iter() { + let reinterpreted_node = FlagInfoNode::from_bytes(&node.into_bytes()).unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let flag_info_bytes = flag_info_list.into_bytes(); + let reinterpreted_info_list = FlagInfoList::from_bytes(&flag_info_bytes); + assert!(reinterpreted_info_list.is_ok()); + assert_eq!(&flag_info_list, &reinterpreted_info_list.unwrap()); + assert_eq!(flag_info_bytes.len() as u32, header.file_size); + } + // this test point locks down that version number should be at the top of serialized // bytes + #[test] fn test_version_number() { - let flag_info_list = create_test_flag_info_list(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); let bytes = &flag_info_list.into_bytes(); let mut head = 0; - let version = read_u32_from_bytes(bytes, &mut head).unwrap(); - assert_eq!(version, 1); + let version_from_file = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version_from_file, DEFAULT_FILE_VERSION); } - #[test] // this test point locks down file type check + #[test] fn test_file_type_check() { - let mut flag_info_list = create_test_flag_info_list(); + let mut flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); flag_info_list.header.file_type = 123u8; let error = FlagInfoList::from_bytes(&flag_info_list.into_bytes()).unwrap_err(); assert_eq!( diff --git a/tools/aconfig/aconfig_storage_file/src/flag_table.rs b/tools/aconfig/aconfig_storage_file/src/flag_table.rs index 660edac0c7..6fbee023ce 100644 --- a/tools/aconfig/aconfig_storage_file/src/flag_table.rs +++ b/tools/aconfig/aconfig_storage_file/src/flag_table.rs @@ -23,10 +23,11 @@ use crate::{ }; use crate::{AconfigStorageError, StorageFileType, StoredFlagType}; use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use std::fmt; /// Flag table header struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagTableHeader { pub version: u32, pub container: String, @@ -95,7 +96,7 @@ impl FlagTableHeader { } /// Flag table node struct -#[derive(PartialEq, Clone)] +#[derive(PartialEq, Clone, Serialize, Deserialize)] pub struct FlagTableNode { pub package_id: u32, pub flag_name: String, @@ -154,7 +155,7 @@ impl FlagTableNode { } } -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagTable { pub header: FlagTableHeader, pub buckets: Vec<Option<u32>>, @@ -219,12 +220,15 @@ impl FlagTable { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::create_test_flag_table; + use crate::{ + test_utils::create_test_flag_table, DEFAULT_FILE_VERSION, MAX_SUPPORTED_FILE_VERSION, + }; - #[test] // this test point locks down the table serialization - fn test_serialization() { - let flag_table = create_test_flag_table(); + // TODO: b/376108268 - Use parameterized tests. + #[test] + fn test_serialization_default() { + let flag_table = create_test_flag_table(DEFAULT_FILE_VERSION); let header: &FlagTableHeader = &flag_table.header; let reinterpreted_header = FlagTableHeader::from_bytes(&header.into_bytes()); @@ -245,20 +249,42 @@ mod tests { } #[test] + fn test_serialization_max() { + let flag_table = create_test_flag_table(MAX_SUPPORTED_FILE_VERSION); + + let header: &FlagTableHeader = &flag_table.header; + let reinterpreted_header = FlagTableHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<FlagTableNode> = &flag_table.nodes; + for node in nodes.iter() { + let reinterpreted_node = FlagTableNode::from_bytes(&node.into_bytes()).unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let flag_table_bytes = flag_table.into_bytes(); + let reinterpreted_table = FlagTable::from_bytes(&flag_table_bytes); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&flag_table, &reinterpreted_table.unwrap()); + assert_eq!(flag_table_bytes.len() as u32, header.file_size); + } + // this test point locks down that version number should be at the top of serialized // bytes + #[test] fn test_version_number() { - let flag_table = create_test_flag_table(); + let flag_table = create_test_flag_table(DEFAULT_FILE_VERSION); let bytes = &flag_table.into_bytes(); let mut head = 0; - let version = read_u32_from_bytes(bytes, &mut head).unwrap(); - assert_eq!(version, 1); + let version_from_file = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version_from_file, DEFAULT_FILE_VERSION); } - #[test] // this test point locks down file type check + #[test] fn test_file_type_check() { - let mut flag_table = create_test_flag_table(); + let mut flag_table = create_test_flag_table(DEFAULT_FILE_VERSION); flag_table.header.file_type = 123u8; let error = FlagTable::from_bytes(&flag_table.into_bytes()).unwrap_err(); assert_eq!( diff --git a/tools/aconfig/aconfig_storage_file/src/flag_value.rs b/tools/aconfig/aconfig_storage_file/src/flag_value.rs index 506924b339..9a14bec7de 100644 --- a/tools/aconfig/aconfig_storage_file/src/flag_value.rs +++ b/tools/aconfig/aconfig_storage_file/src/flag_value.rs @@ -20,10 +20,11 @@ use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; use crate::{AconfigStorageError, StorageFileType}; use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use std::fmt; /// Flag value header struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagValueHeader { pub version: u32, pub container: String, @@ -89,7 +90,7 @@ impl FlagValueHeader { } /// Flag value list struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct FlagValueList { pub header: FlagValueHeader, pub booleans: Vec<bool>, @@ -131,12 +132,32 @@ impl FlagValueList { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::create_test_flag_value_list; + use crate::{ + test_utils::create_test_flag_value_list, DEFAULT_FILE_VERSION, MAX_SUPPORTED_FILE_VERSION, + }; #[test] // this test point locks down the value list serialization - fn test_serialization() { - let flag_value_list = create_test_flag_value_list(); + // TODO: b/376108268 - Use parameterized tests. + fn test_serialization_default() { + let flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); + + let header: &FlagValueHeader = &flag_value_list.header; + let reinterpreted_header = FlagValueHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let flag_value_bytes = flag_value_list.into_bytes(); + let reinterpreted_value_list = FlagValueList::from_bytes(&flag_value_bytes); + assert!(reinterpreted_value_list.is_ok()); + assert_eq!(&flag_value_list, &reinterpreted_value_list.unwrap()); + assert_eq!(flag_value_bytes.len() as u32, header.file_size); + } + + #[test] + // this test point locks down the value list serialization + fn test_serialization_max() { + let flag_value_list = create_test_flag_value_list(MAX_SUPPORTED_FILE_VERSION); let header: &FlagValueHeader = &flag_value_list.header; let reinterpreted_header = FlagValueHeader::from_bytes(&header.into_bytes()); @@ -154,17 +175,17 @@ mod tests { // this test point locks down that version number should be at the top of serialized // bytes fn test_version_number() { - let flag_value_list = create_test_flag_value_list(); + let flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); let bytes = &flag_value_list.into_bytes(); let mut head = 0; - let version = read_u32_from_bytes(bytes, &mut head).unwrap(); - assert_eq!(version, 1); + let version_from_file = read_u32_from_bytes(bytes, &mut head).unwrap(); + assert_eq!(version_from_file, DEFAULT_FILE_VERSION); } #[test] // this test point locks down file type check fn test_file_type_check() { - let mut flag_value_list = create_test_flag_value_list(); + let mut flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); flag_value_list.header.file_type = 123u8; let error = FlagValueList::from_bytes(&flag_value_list.into_bytes()).unwrap_err(); assert_eq!( diff --git a/tools/aconfig/aconfig_storage_file/src/lib.rs b/tools/aconfig/aconfig_storage_file/src/lib.rs index b6367ffa35..e99132092d 100644 --- a/tools/aconfig/aconfig_storage_file/src/lib.rs +++ b/tools/aconfig/aconfig_storage_file/src/lib.rs @@ -41,6 +41,7 @@ pub mod sip_hasher13; pub mod test_utils; use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::fs::File; use std::hash::Hasher; @@ -56,8 +57,13 @@ use crate::AconfigStorageError::{ BytesParseFail, HashTableSizeLimit, InvalidFlagValueType, InvalidStoredFlagType, }; -/// Storage file version -pub const FILE_VERSION: u32 = 1; +/// The max storage file version from which we can safely read/write. May be +/// experimental. +pub const MAX_SUPPORTED_FILE_VERSION: u32 = 2; + +/// The newest fully-released version. Unless otherwise specified, this is the +/// version we will write. +pub const DEFAULT_FILE_VERSION: u32 = 1; /// Good hash table prime number pub(crate) const HASH_PRIMES: [u32; 29] = [ @@ -107,7 +113,7 @@ impl TryFrom<u8> for StorageFileType { /// Flag type enum as stored by storage file /// ONLY APPEND, NEVER REMOVE FOR BACKWARD COMPATIBILITY. THE MAX IS U16. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum StoredFlagType { ReadWriteBoolean = 0, ReadOnlyBoolean = 1, @@ -243,6 +249,11 @@ pub(crate) fn read_u16_from_bytes( Ok(val) } +/// Read and parse the first 4 bytes of buf as u32. +pub fn read_u32_from_start_of_bytes(buf: &[u8]) -> Result<u32, AconfigStorageError> { + read_u32_from_bytes(buf, &mut 0) +} + /// Read and parse bytes as u32 pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32, AconfigStorageError> { let val = @@ -253,6 +264,16 @@ pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32, AconfigS Ok(val) } +// Read and parse bytes as u64 +pub fn read_u64_from_bytes(buf: &[u8], head: &mut usize) -> Result<u64, AconfigStorageError> { + let val = + u64::from_le_bytes(buf[*head..*head + 8].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u64 from bytes: {}", errmsg)) + })?); + *head += 8; + Ok(val) +} + /// Read and parse bytes as string pub(crate) fn read_str_from_bytes( buf: &[u8], @@ -515,10 +536,15 @@ mod tests { // this test point locks down the flag list api fn test_list_flag() { let package_table = - write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); - let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); - let flag_value_list = - write_bytes_to_temp_file(&create_test_flag_value_list().into_bytes()).unwrap(); + write_bytes_to_temp_file(&create_test_package_table(DEFAULT_FILE_VERSION).into_bytes()) + .unwrap(); + let flag_table = + write_bytes_to_temp_file(&create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes()) + .unwrap(); + let flag_value_list = write_bytes_to_temp_file( + &create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(), + ) + .unwrap(); let package_table_path = package_table.path().display().to_string(); let flag_table_path = flag_table.path().display().to_string(); @@ -583,12 +609,19 @@ mod tests { // this test point locks down the flag list with info api fn test_list_flag_with_info() { let package_table = - write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); - let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); - let flag_value_list = - write_bytes_to_temp_file(&create_test_flag_value_list().into_bytes()).unwrap(); - let flag_info_list = - write_bytes_to_temp_file(&create_test_flag_info_list().into_bytes()).unwrap(); + write_bytes_to_temp_file(&create_test_package_table(DEFAULT_FILE_VERSION).into_bytes()) + .unwrap(); + let flag_table = + write_bytes_to_temp_file(&create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes()) + .unwrap(); + let flag_value_list = write_bytes_to_temp_file( + &create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(), + ) + .unwrap(); + let flag_info_list = write_bytes_to_temp_file( + &create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(), + ) + .unwrap(); let package_table_path = package_table.path().display().to_string(); let flag_table_path = flag_table.path().display().to_string(); diff --git a/tools/aconfig/aconfig_storage_file/src/main.rs b/tools/aconfig/aconfig_storage_file/src/main.rs index 8b9e38da02..a9cfd19066 100644 --- a/tools/aconfig/aconfig_storage_file/src/main.rs +++ b/tools/aconfig/aconfig_storage_file/src/main.rs @@ -20,9 +20,29 @@ use aconfig_storage_file::{ list_flags, list_flags_with_info, read_file_to_bytes, AconfigStorageError, FlagInfoList, FlagTable, FlagValueList, PackageTable, StorageFileType, }; - use clap::{builder::ArgAction, Arg, Command}; +use serde::Serialize; +use serde_json; +use std::fmt; +use std::fs; +use std::fs::File; +use std::io::Write; +/** + * Usage Examples + * + * Print file: + * $ aconfig-storage print --file=path/to/flag.map --type=flag_map + * + * List flags: + * $ aconfig-storage list --flag-map=path/to/flag.map \ + * --flag-val=path/to/flag.val --package-map=path/to/package.map + * + * Write binary file for testing: + * $ aconfig-storage print --file=path/to/flag.map --type=flag_map --format=json > flag_map.json + * $ vim flag_map.json // Manually make updates + * $ aconfig-storage write-bytes --input-file=flag_map.json --output-file=path/to/flag.map --type=flag_map + */ fn cli() -> Command { Command::new("aconfig-storage") .subcommand_required(true) @@ -34,7 +54,8 @@ fn cli() -> Command { .long("type") .required(true) .value_parser(|s: &str| StorageFileType::try_from(s)), - ), + ) + .arg(Arg::new("format").long("format").required(false).action(ArgAction::Set)), ) .subcommand( Command::new("list") @@ -50,41 +71,75 @@ fn cli() -> Command { Arg::new("flag-info").long("flag-info").required(false).action(ArgAction::Set), ), ) + .subcommand( + Command::new("write-bytes") + // Where to write the output bytes. Suggest to use the StorageFileType names (e.g. flag.map). + .arg( + Arg::new("output-file") + .long("output-file") + .required(true) + .action(ArgAction::Set), + ) + // Input file should be json. + .arg( + Arg::new("input-file").long("input-file").required(true).action(ArgAction::Set), + ) + .arg( + Arg::new("type") + .long("type") + .required(true) + .value_parser(|s: &str| StorageFileType::try_from(s)), + ), + ) } fn print_storage_file( file_path: &str, file_type: &StorageFileType, + as_json: bool, ) -> Result<(), AconfigStorageError> { let bytes = read_file_to_bytes(file_path)?; match file_type { StorageFileType::PackageMap => { let package_table = PackageTable::from_bytes(&bytes)?; - println!("{:?}", package_table); + println!("{}", to_print_format(package_table, as_json)); } StorageFileType::FlagMap => { let flag_table = FlagTable::from_bytes(&bytes)?; - println!("{:?}", flag_table); + println!("{}", to_print_format(flag_table, as_json)); } StorageFileType::FlagVal => { let flag_value = FlagValueList::from_bytes(&bytes)?; - println!("{:?}", flag_value); + println!("{}", to_print_format(flag_value, as_json)); } StorageFileType::FlagInfo => { let flag_info = FlagInfoList::from_bytes(&bytes)?; - println!("{:?}", flag_info); + println!("{}", to_print_format(flag_info, as_json)); } } Ok(()) } +fn to_print_format<T>(file_contents: T, as_json: bool) -> String +where + T: Serialize + fmt::Debug, +{ + if as_json { + serde_json::to_string(&file_contents).unwrap() + } else { + format!("{:?}", file_contents) + } +} + fn main() -> Result<(), AconfigStorageError> { let matches = cli().get_matches(); match matches.subcommand() { Some(("print", sub_matches)) => { let file_path = sub_matches.get_one::<String>("file").unwrap(); let file_type = sub_matches.get_one::<StorageFileType>("type").unwrap(); - print_storage_file(file_path, file_type)? + let format = sub_matches.get_one::<String>("format"); + let as_json: bool = format == Some(&"json".to_string()); + print_storage_file(file_path, file_type, as_json)? } Some(("list", sub_matches)) => { let package_map = sub_matches.get_one::<String>("package-map").unwrap(); @@ -96,10 +151,10 @@ fn main() -> Result<(), AconfigStorageError> { let flags = list_flags_with_info(package_map, flag_map, flag_val, info_file)?; for flag in flags.iter() { println!( - "{} {} {} {:?} IsReadWrite: {}, HasServerOverride: {}, HasLocalOverride: {}", - flag.package_name, flag.flag_name, flag.flag_value, flag.value_type, - flag.is_readwrite, flag.has_server_override, flag.has_local_override, - ); + "{} {} {} {:?} IsReadWrite: {}, HasServerOverride: {}, HasLocalOverride: {}", + flag.package_name, flag.flag_name, flag.flag_value, flag.value_type, + flag.is_readwrite, flag.has_server_override, flag.has_local_override, + ); } } None => { @@ -113,6 +168,40 @@ fn main() -> Result<(), AconfigStorageError> { } } } + // Converts JSON of the file into raw bytes (as is used on-device). + // Intended to generate/easily update these files for testing. + Some(("write-bytes", sub_matches)) => { + let input_file_path = sub_matches.get_one::<String>("input-file").unwrap(); + let input_json = fs::read_to_string(input_file_path).unwrap(); + + let file_type = sub_matches.get_one::<StorageFileType>("type").unwrap(); + let output_bytes: Vec<u8>; + match file_type { + StorageFileType::FlagVal => { + let list: FlagValueList = serde_json::from_str(&input_json).unwrap(); + output_bytes = list.into_bytes(); + } + StorageFileType::FlagInfo => { + let list: FlagInfoList = serde_json::from_str(&input_json).unwrap(); + output_bytes = list.into_bytes(); + } + StorageFileType::FlagMap => { + let table: FlagTable = serde_json::from_str(&input_json).unwrap(); + output_bytes = table.into_bytes(); + } + StorageFileType::PackageMap => { + let table: PackageTable = serde_json::from_str(&input_json).unwrap(); + output_bytes = table.into_bytes(); + } + } + + let output_file_path = sub_matches.get_one::<String>("output-file").unwrap(); + let file = File::create(output_file_path); + if file.is_err() { + panic!("can't make file"); + } + let _ = file.unwrap().write_all(&output_bytes); + } _ => unreachable!(), } Ok(()) diff --git a/tools/aconfig/aconfig_storage_file/src/package_table.rs b/tools/aconfig/aconfig_storage_file/src/package_table.rs index 007f86ed1a..21357c7e4a 100644 --- a/tools/aconfig/aconfig_storage_file/src/package_table.rs +++ b/tools/aconfig/aconfig_storage_file/src/package_table.rs @@ -17,13 +17,17 @@ //! package table module defines the package table file format and methods for serialization //! and deserialization -use crate::{get_bucket_index, read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; +use crate::{ + get_bucket_index, read_str_from_bytes, read_u32_from_bytes, read_u64_from_bytes, + read_u8_from_bytes, +}; use crate::{AconfigStorageError, StorageFileType}; use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use std::fmt; /// Package table header struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct PackageTableHeader { pub version: u32, pub container: String, @@ -92,10 +96,11 @@ impl PackageTableHeader { } /// Package table node struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct PackageTableNode { pub package_name: String, pub package_id: u32, + pub fingerprint: u64, // The index of the first boolean flag in this aconfig package among all boolean // flags in this container. pub boolean_start_index: u32, @@ -107,8 +112,12 @@ impl fmt::Debug for PackageTableNode { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { writeln!( f, - "Package: {}, Id: {}, Boolean flag start index: {}, Next: {:?}", - self.package_name, self.package_id, self.boolean_start_index, self.next_offset + "Package: {}, Id: {}, Fingerprint: {}, Boolean flag start index: {}, Next: {:?}", + self.package_name, + self.package_id, + self.fingerprint, + self.boolean_start_index, + self.next_offset )?; Ok(()) } @@ -116,7 +125,16 @@ impl fmt::Debug for PackageTableNode { impl PackageTableNode { /// Serialize to bytes - pub fn into_bytes(&self) -> Vec<u8> { + pub fn into_bytes(&self, version: u32) -> Vec<u8> { + match version { + 1 => Self::into_bytes_v1(self), + 2 => Self::into_bytes_v2(self), + // TODO(b/316357686): into_bytes should return a Result. + _ => Self::into_bytes_v2(&self), + } + } + + fn into_bytes_v1(&self) -> Vec<u8> { let mut result = Vec::new(); let name_bytes = self.package_name.as_bytes(); result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); @@ -127,18 +145,60 @@ impl PackageTableNode { result } - /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + fn into_bytes_v2(&self) -> Vec<u8> { + let mut result = Vec::new(); + let name_bytes = self.package_name.as_bytes(); + result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(name_bytes); + result.extend_from_slice(&self.package_id.to_le_bytes()); + result.extend_from_slice(&self.fingerprint.to_le_bytes()); + result.extend_from_slice(&self.boolean_start_index.to_le_bytes()); + result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); + result + } + + /// Deserialize from bytes based on file version. + pub fn from_bytes(bytes: &[u8], version: u32) -> Result<Self, AconfigStorageError> { + match version { + 1 => Self::from_bytes_v1(bytes), + 2 => Self::from_bytes_v2(bytes), + _ => { + return Err(AconfigStorageError::BytesParseFail(anyhow!( + "Binary file is an unsupported version: {}", + version + ))) + } + } + } + + fn from_bytes_v1(bytes: &[u8]) -> Result<Self, AconfigStorageError> { let mut head = 0; - let node = Self { - package_name: read_str_from_bytes(bytes, &mut head)?, - package_id: read_u32_from_bytes(bytes, &mut head)?, - boolean_start_index: read_u32_from_bytes(bytes, &mut head)?, - next_offset: match read_u32_from_bytes(bytes, &mut head)? { - 0 => None, - val => Some(val), - }, + let package_name = read_str_from_bytes(bytes, &mut head)?; + let package_id = read_u32_from_bytes(bytes, &mut head)?; + // v1 does not have fingerprint, so just set to 0. + let fingerprint: u64 = 0; + let boolean_start_index = read_u32_from_bytes(bytes, &mut head)?; + let next_offset = match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), + }; + + let node = Self { package_name, package_id, fingerprint, boolean_start_index, next_offset }; + Ok(node) + } + + fn from_bytes_v2(bytes: &[u8]) -> Result<Self, AconfigStorageError> { + let mut head = 0; + let package_name = read_str_from_bytes(bytes, &mut head)?; + let package_id = read_u32_from_bytes(bytes, &mut head)?; + let fingerprint = read_u64_from_bytes(bytes, &mut head)?; + let boolean_start_index = read_u32_from_bytes(bytes, &mut head)?; + let next_offset = match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), }; + + let node = Self { package_name, package_id, fingerprint, boolean_start_index, next_offset }; Ok(node) } @@ -151,7 +211,7 @@ impl PackageTableNode { } /// Package table struct -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] pub struct PackageTable { pub header: PackageTableHeader, pub buckets: Vec<Option<u32>>, @@ -179,7 +239,11 @@ impl PackageTable { [ self.header.into_bytes(), self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::<Vec<_>>().concat(), - self.nodes.iter().map(|v| v.into_bytes()).collect::<Vec<_>>().concat(), + self.nodes + .iter() + .map(|v| v.into_bytes(self.header.version)) + .collect::<Vec<_>>() + .concat(), ] .concat() } @@ -198,8 +262,8 @@ impl PackageTable { .collect(); let nodes = (0..num_packages) .map(|_| { - let node = PackageTableNode::from_bytes(&bytes[head..])?; - head += node.into_bytes().len(); + let node = PackageTableNode::from_bytes(&bytes[head..], header.version)?; + head += node.into_bytes(header.version).len(); Ok(node) }) .collect::<Result<Vec<_>, AconfigStorageError>>() @@ -219,11 +283,13 @@ impl PackageTable { mod tests { use super::*; use crate::test_utils::create_test_package_table; + use crate::{read_u32_from_start_of_bytes, DEFAULT_FILE_VERSION, MAX_SUPPORTED_FILE_VERSION}; #[test] // this test point locks down the table serialization - fn test_serialization() { - let package_table = create_test_package_table(); + // TODO: b/376108268 - Use parameterized tests. + fn test_serialization_default() { + let package_table = create_test_package_table(DEFAULT_FILE_VERSION); let header: &PackageTableHeader = &package_table.header; let reinterpreted_header = PackageTableHeader::from_bytes(&header.into_bytes()); assert!(reinterpreted_header.is_ok()); @@ -231,7 +297,32 @@ mod tests { let nodes: &Vec<PackageTableNode> = &package_table.nodes; for node in nodes.iter() { - let reinterpreted_node = PackageTableNode::from_bytes(&node.into_bytes()).unwrap(); + let reinterpreted_node = + PackageTableNode::from_bytes(&node.into_bytes(header.version), header.version) + .unwrap(); + assert_eq!(node, &reinterpreted_node); + } + + let package_table_bytes = package_table.into_bytes(); + let reinterpreted_table = PackageTable::from_bytes(&package_table_bytes); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&package_table, &reinterpreted_table.unwrap()); + assert_eq!(package_table_bytes.len() as u32, header.file_size); + } + + #[test] + fn test_serialization_max() { + let package_table = create_test_package_table(MAX_SUPPORTED_FILE_VERSION); + let header: &PackageTableHeader = &package_table.header; + let reinterpreted_header = PackageTableHeader::from_bytes(&header.into_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec<PackageTableNode> = &package_table.nodes; + for node in nodes.iter() { + let reinterpreted_node = + PackageTableNode::from_bytes(&node.into_bytes(header.version), header.version) + .unwrap(); assert_eq!(node, &reinterpreted_node); } @@ -246,17 +337,36 @@ mod tests { // this test point locks down that version number should be at the top of serialized // bytes fn test_version_number() { - let package_table = create_test_package_table(); + let package_table = create_test_package_table(DEFAULT_FILE_VERSION); let bytes = &package_table.into_bytes(); - let mut head = 0; - let version = read_u32_from_bytes(bytes, &mut head).unwrap(); - assert_eq!(version, 1); + let unpacked_version = read_u32_from_start_of_bytes(bytes).unwrap(); + assert_eq!(unpacked_version, DEFAULT_FILE_VERSION); + } + + #[test] + fn test_round_trip_default() { + let table: PackageTable = create_test_package_table(DEFAULT_FILE_VERSION); + let table_bytes = table.into_bytes(); + + let reinterpreted_table = PackageTable::from_bytes(&table_bytes).unwrap(); + + assert_eq!(table, reinterpreted_table); + } + + #[test] + fn test_round_trip_max() { + let table: PackageTable = create_test_package_table(MAX_SUPPORTED_FILE_VERSION); + let table_bytes = table.into_bytes(); + + let reinterpreted_table = PackageTable::from_bytes(&table_bytes).unwrap(); + + assert_eq!(table, reinterpreted_table); } #[test] // this test point locks down file type check fn test_file_type_check() { - let mut package_table = create_test_package_table(); + let mut package_table = create_test_package_table(DEFAULT_FILE_VERSION); package_table.header.file_type = 123u8; let error = PackageTable::from_bytes(&package_table.into_bytes()).unwrap_err(); assert_eq!( diff --git a/tools/aconfig/aconfig_storage_file/src/test_utils.rs b/tools/aconfig/aconfig_storage_file/src/test_utils.rs index 106666c47f..7c603df40e 100644 --- a/tools/aconfig/aconfig_storage_file/src/test_utils.rs +++ b/tools/aconfig/aconfig_storage_file/src/test_utils.rs @@ -24,32 +24,59 @@ use anyhow::anyhow; use std::io::Write; use tempfile::NamedTempFile; -pub fn create_test_package_table() -> PackageTable { +pub fn create_test_package_table(version: u32) -> PackageTable { let header = PackageTableHeader { - version: 1, + version: version, container: String::from("mockup"), file_type: StorageFileType::PackageMap as u8, - file_size: 209, + file_size: match version { + 1 => 209, + 2 => 233, + _ => panic!("Unsupported version."), + }, num_packages: 3, bucket_offset: 31, node_offset: 59, }; - let buckets: Vec<Option<u32>> = vec![Some(59), None, None, Some(109), None, None, None]; + let buckets: Vec<Option<u32>> = match version { + 1 => vec![Some(59), None, None, Some(109), None, None, None], + 2 => vec![Some(59), None, None, Some(117), None, None, None], + _ => panic!("Unsupported version."), + }; let first_node = PackageTableNode { package_name: String::from("com.android.aconfig.storage.test_2"), package_id: 1, + fingerprint: match version { + 1 => 0, + 2 => 4431940502274857964u64, + _ => panic!("Unsupported version."), + }, boolean_start_index: 3, next_offset: None, }; let second_node = PackageTableNode { package_name: String::from("com.android.aconfig.storage.test_1"), package_id: 0, + fingerprint: match version { + 1 => 0, + 2 => 15248948510590158086u64, + _ => panic!("Unsupported version."), + }, boolean_start_index: 0, - next_offset: Some(159), + next_offset: match version { + 1 => Some(159), + 2 => Some(175), + _ => panic!("Unsupported version."), + }, }; let third_node = PackageTableNode { package_name: String::from("com.android.aconfig.storage.test_4"), package_id: 2, + fingerprint: match version { + 1 => 0, + 2 => 16233229917711622375u64, + _ => panic!("Unsupported version."), + }, boolean_start_index: 6, next_offset: None, }; @@ -76,9 +103,9 @@ impl FlagTableNode { } } -pub fn create_test_flag_table() -> FlagTable { +pub fn create_test_flag_table(version: u32) -> FlagTable { let header = FlagTableHeader { - version: 1, + version: version, container: String::from("mockup"), file_type: StorageFileType::FlagMap as u8, file_size: 321, @@ -118,9 +145,9 @@ pub fn create_test_flag_table() -> FlagTable { FlagTable { header, buckets, nodes } } -pub fn create_test_flag_value_list() -> FlagValueList { +pub fn create_test_flag_value_list(version: u32) -> FlagValueList { let header = FlagValueHeader { - version: 1, + version: version, container: String::from("mockup"), file_type: StorageFileType::FlagVal as u8, file_size: 35, @@ -131,9 +158,9 @@ pub fn create_test_flag_value_list() -> FlagValueList { FlagValueList { header, booleans } } -pub fn create_test_flag_info_list() -> FlagInfoList { +pub fn create_test_flag_info_list(version: u32) -> FlagInfoList { let header = FlagInfoHeader { - version: 1, + version: version, container: String::from("mockup"), file_type: StorageFileType::FlagInfo as u8, file_size: 35, diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/AconfigStorageException.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/AconfigStorageException.java index 86a75f2f65..324c55d57d 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/AconfigStorageException.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/AconfigStorageException.java @@ -16,12 +16,122 @@ package android.aconfig.storage; +/** + * Exception thrown when an error occurs while accessing Aconfig Storage. + * + * <p>This exception indicates a general problem with Aconfig Storage, such as an inability to read + * or write data. + */ public class AconfigStorageException extends RuntimeException { + + /** Generic error code indicating an unspecified Aconfig Storage error. */ + public static final int ERROR_GENERIC = 0; + + /** Error code indicating that the Aconfig Storage system is not found on the device. */ + public static final int ERROR_STORAGE_SYSTEM_NOT_FOUND = 1; + + /** Error code indicating that the requested configuration package is not found. */ + public static final int ERROR_PACKAGE_NOT_FOUND = 2; + + /** Error code indicating that the specified container is not found. */ + public static final int ERROR_CONTAINER_NOT_FOUND = 3; + + /** Error code indicating that there was an error reading the Aconfig Storage file. */ + public static final int ERROR_CANNOT_READ_STORAGE_FILE = 4; + + public static final int ERROR_FILE_FINGERPRINT_MISMATCH = 5; + + private final int mErrorCode; + + /** + * Constructs a new {@code AconfigStorageException} with a generic error code and the specified + * detail message. + * + * @param msg The detail message for this exception. + */ public AconfigStorageException(String msg) { super(msg); + mErrorCode = ERROR_GENERIC; } + /** + * Constructs a new {@code AconfigStorageException} with a generic error code, the specified + * detail message, and cause. + * + * @param msg The detail message for this exception. + * @param cause The cause of this exception. + */ public AconfigStorageException(String msg, Throwable cause) { super(msg, cause); + mErrorCode = ERROR_GENERIC; + } + + /** + * Constructs a new {@code AconfigStorageException} with the specified error code and detail + * message. + * + * @param errorCode The error code for this exception. + * @param msg The detail message for this exception. + */ + public AconfigStorageException(int errorCode, String msg) { + super(msg); + mErrorCode = errorCode; + } + + /** + * Constructs a new {@code AconfigStorageException} with the specified error code, detail + * message, and cause. + * + * @param errorCode The error code for this exception. + * @param msg The detail message for this exception. + * @param cause The cause of this exception. + */ + public AconfigStorageException(int errorCode, String msg, Throwable cause) { + super(msg, cause); + mErrorCode = errorCode; + } + + /** + * Returns the error code associated with this exception. + * + * @return The error code. + */ + public int getErrorCode() { + return mErrorCode; + } + + /** + * Returns the error message for this exception, including the error code and the original + * message. + * + * @return The error message. + */ + @Override + public String getMessage() { + return errorString() + ": " + super.getMessage(); + } + + /** + * Returns a string representation of the error code. + * + * @return The error code string. + */ + private String errorString() { + switch (mErrorCode) { + case ERROR_GENERIC: + return "ERROR_GENERIC"; + case ERROR_STORAGE_SYSTEM_NOT_FOUND: + return "ERROR_STORAGE_SYSTEM_NOT_FOUND"; + case ERROR_PACKAGE_NOT_FOUND: + return "ERROR_PACKAGE_NOT_FOUND"; + case ERROR_CONTAINER_NOT_FOUND: + return "ERROR_CONTAINER_NOT_FOUND"; + case ERROR_CANNOT_READ_STORAGE_FILE: + return "ERROR_CANNOT_READ_STORAGE_FILE"; + case ERROR_FILE_FINGERPRINT_MISMATCH: + return "ERROR_FILE_FINGERPRINT_MISMATCH"; + default: + return "<Unknown error code " + mErrorCode + ">"; + } } } diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/ByteBufferReader.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/ByteBufferReader.java index 4bea0836f0..14fc468f11 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/ByteBufferReader.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/ByteBufferReader.java @@ -19,10 +19,12 @@ package android.aconfig.storage; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.Objects; public class ByteBufferReader { private ByteBuffer mByteBuffer; + private int mPosition; public ByteBufferReader(ByteBuffer byteBuffer) { this.mByteBuffer = byteBuffer; @@ -30,21 +32,29 @@ public class ByteBufferReader { } public int readByte() { - return Byte.toUnsignedInt(mByteBuffer.get()); + return Byte.toUnsignedInt(mByteBuffer.get(nextGetIndex(1))); } public int readShort() { - return Short.toUnsignedInt(mByteBuffer.getShort()); + return Short.toUnsignedInt(mByteBuffer.getShort(nextGetIndex(2))); } public int readInt() { - return this.mByteBuffer.getInt(); + return this.mByteBuffer.getInt(nextGetIndex(4)); + } + + public long readLong() { + return this.mByteBuffer.getLong(nextGetIndex(8)); } public String readString() { int length = readInt(); + if (length > 1024) { + throw new AconfigStorageException( + "String length exceeds maximum allowed size (1024 bytes): " + length); + } byte[] bytes = new byte[length]; - mByteBuffer.get(bytes, 0, length); + getArray(nextGetIndex(length), bytes, 0, length); return new String(bytes, StandardCharsets.UTF_8); } @@ -53,6 +63,26 @@ public class ByteBufferReader { } public void position(int newPosition) { - mByteBuffer.position(newPosition); + mPosition = newPosition; + } + + public int position() { + return mPosition; + } + + private int nextGetIndex(int nb) { + int p = mPosition; + mPosition += nb; + return p; + } + + private void getArray(int index, byte[] dst, int offset, int length) { + Objects.checkFromIndexSize(index, length, mByteBuffer.limit()); + Objects.checkFromIndexSize(offset, length, dst.length); + + int end = offset + length; + for (int i = offset, j = index; i < end; i++, j++) { + dst[i] = mByteBuffer.get(j); + } } } diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FileType.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FileType.java index b0b1b9b186..c35487358d 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FileType.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FileType.java @@ -42,4 +42,20 @@ public enum FileType { return null; } } + + @Override + public String toString() { + switch (type) { + case 0: + return "PACKAGE_MAP"; + case 1: + return "FLAG_MAP"; + case 2: + return "FLAG_VAL"; + case 3: + return "FLAG_INFO"; + default: + return "unrecognized type"; + } + } } diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java index 9838a7c780..ee60b18dcb 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java @@ -24,12 +24,12 @@ import java.util.Objects; public class FlagTable { private Header mHeader; - private ByteBufferReader mReader; + private ByteBuffer mBuffer; public static FlagTable fromBytes(ByteBuffer bytes) { FlagTable flagTable = new FlagTable(); - flagTable.mReader = new ByteBufferReader(bytes); - flagTable.mHeader = Header.fromBytes(flagTable.mReader); + flagTable.mBuffer = bytes; + flagTable.mHeader = Header.fromBytes(new ByteBufferReader(bytes)); return flagTable; } @@ -37,20 +37,27 @@ public class FlagTable { public Node get(int packageId, String flagName) { int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4; int bucketIndex = TableUtils.getBucketIndex(makeKey(packageId, flagName), numBuckets); - - mReader.position(mHeader.mBucketOffset + bucketIndex * 4); - int nodeIndex = mReader.readInt(); + int newPosition = mHeader.mBucketOffset + bucketIndex * 4; + if (newPosition >= mHeader.mNodeOffset) { + return null; + } + ByteBufferReader reader = new ByteBufferReader(mBuffer) ; + reader.position(newPosition); + int nodeIndex = reader.readInt(); + if (nodeIndex < mHeader.mNodeOffset || nodeIndex >= mHeader.mFileSize) { + return null; + } while (nodeIndex != -1) { - mReader.position(nodeIndex); - Node node = Node.fromBytes(mReader); + reader.position(nodeIndex); + Node node = Node.fromBytes(reader); if (Objects.equals(flagName, node.mFlagName) && packageId == node.mPackageId) { return node; } nodeIndex = node.mNextOffset; } - throw new AconfigStorageException("get cannot find flag: " + flagName); + return null; } public Header getHeader() { diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java index 773b882f4a..215616e781 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java @@ -19,39 +19,65 @@ package android.aconfig.storage; import static java.nio.charset.StandardCharsets.UTF_8; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; public class PackageTable { + private static final int FINGERPRINT_BYTES = 8; + // int: mPackageId + int: mBooleanStartIndex + int: mNextOffset + private static final int NODE_SKIP_BYTES = 12; + private Header mHeader; - private ByteBufferReader mReader; + private ByteBuffer mBuffer; public static PackageTable fromBytes(ByteBuffer bytes) { PackageTable packageTable = new PackageTable(); - packageTable.mReader = new ByteBufferReader(bytes); - packageTable.mHeader = Header.fromBytes(packageTable.mReader); + packageTable.mBuffer = bytes; + packageTable.mHeader = Header.fromBytes(new ByteBufferReader(bytes)); return packageTable; } public Node get(String packageName) { - int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4; int bucketIndex = TableUtils.getBucketIndex(packageName.getBytes(UTF_8), numBuckets); + int newPosition = mHeader.mBucketOffset + bucketIndex * 4; + if (newPosition >= mHeader.mNodeOffset) { + return null; + } + ByteBufferReader reader = new ByteBufferReader(mBuffer); + reader.position(newPosition); + int nodeIndex = reader.readInt(); - mReader.position(mHeader.mBucketOffset + bucketIndex * 4); - int nodeIndex = mReader.readInt(); + if (nodeIndex < mHeader.mNodeOffset || nodeIndex >= mHeader.mFileSize) { + return null; + } while (nodeIndex != -1) { - mReader.position(nodeIndex); - Node node = Node.fromBytes(mReader); + reader.position(nodeIndex); + Node node = Node.fromBytes(reader, mHeader.mVersion); if (Objects.equals(packageName, node.mPackageName)) { return node; } nodeIndex = node.mNextOffset; } - throw new AconfigStorageException("get cannot find package: " + packageName); + return null; + } + + public List<String> getPackageList() { + List<String> list = new ArrayList<>(mHeader.mNumPackages); + ByteBufferReader reader = new ByteBufferReader(mBuffer); + reader.position(mHeader.mNodeOffset); + int fingerprintBytes = mHeader.mVersion == 1 ? 0 : FINGERPRINT_BYTES; + int skipBytes = fingerprintBytes + NODE_SKIP_BYTES; + for (int i = 0; i < mHeader.mNumPackages; i++) { + list.add(reader.readString()); + reader.position(reader.position() + skipBytes); + } + return list; } public Header getHeader() { @@ -68,7 +94,7 @@ public class PackageTable { private int mBucketOffset; private int mNodeOffset; - public static Header fromBytes(ByteBufferReader reader) { + private static Header fromBytes(ByteBufferReader reader) { Header header = new Header(); header.mVersion = reader.readInt(); header.mContainer = reader.readString(); @@ -118,10 +144,24 @@ public class PackageTable { private String mPackageName; private int mPackageId; + private long mPackageFingerprint; private int mBooleanStartIndex; private int mNextOffset; + private boolean mHasPackageFingerprint; + + private static Node fromBytes(ByteBufferReader reader, int version) { + switch (version) { + case 1: + return fromBytesV1(reader); + case 2: + return fromBytesV2(reader); + default: + // Do we want to throw here? + return new Node(); + } + } - public static Node fromBytes(ByteBufferReader reader) { + private static Node fromBytesV1(ByteBufferReader reader) { Node node = new Node(); node.mPackageName = reader.readString(); node.mPackageId = reader.readInt(); @@ -131,6 +171,18 @@ public class PackageTable { return node; } + private static Node fromBytesV2(ByteBufferReader reader) { + Node node = new Node(); + node.mPackageName = reader.readString(); + node.mPackageId = reader.readInt(); + node.mPackageFingerprint = reader.readLong(); + node.mBooleanStartIndex = reader.readInt(); + node.mNextOffset = reader.readInt(); + node.mNextOffset = node.mNextOffset == 0 ? -1 : node.mNextOffset; + node.mHasPackageFingerprint = true; + return node; + } + @Override public int hashCode() { return Objects.hash(mPackageName, mPackageId, mBooleanStartIndex, mNextOffset); @@ -161,6 +213,10 @@ public class PackageTable { return mPackageId; } + public long getPackageFingerprint() { + return mPackageFingerprint; + } + public int getBooleanStartIndex() { return mBooleanStartIndex; } @@ -168,5 +224,9 @@ public class PackageTable { public int getNextOffset() { return mNextOffset; } + + public boolean hasPackageFingerprint() { + return mHasPackageFingerprint; + } } } diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/StorageFileProvider.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/StorageFileProvider.java new file mode 100644 index 0000000000..f75ac36f7d --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/StorageFileProvider.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.aconfig.storage; + +import java.io.Closeable; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** @hide */ +public class StorageFileProvider { + + private static final String DEFAULT_MAP_PATH = "/metadata/aconfig/maps/"; + private static final String DEFAULT_BOOT_PATH = "/metadata/aconfig/boot/"; + private static final String PMAP_FILE_EXT = ".package.map"; + private static final String FMAP_FILE_EXT = ".flag.map"; + private static final String VAL_FILE_EXT = ".val"; + private static final StorageFileProvider DEFAULT_INSTANCE = + new StorageFileProvider(DEFAULT_MAP_PATH, DEFAULT_BOOT_PATH); + + private final String mMapPath; + private final String mBootPath; + + /** @hide */ + public static StorageFileProvider getDefaultProvider() { + return DEFAULT_INSTANCE; + } + + /** @hide */ + public StorageFileProvider(String mapPath, String bootPath) { + mMapPath = mapPath; + mBootPath = bootPath; + } + + /** @hide */ + public List<String> listContainers(String[] excludes) { + List<String> result = new ArrayList<>(); + Set<String> set = new HashSet<>(Arrays.asList(excludes)); + + try { + DirectoryStream<Path> stream = + Files.newDirectoryStream(Paths.get(mMapPath), "*" + PMAP_FILE_EXT); + for (Path entry : stream) { + String fileName = entry.getFileName().toString(); + String container = + fileName.substring(0, fileName.length() - PMAP_FILE_EXT.length()); + if (!set.contains(container)) { + result.add(container); + } + } + } catch (NoSuchFileException e) { + return result; + } catch (Exception e) { + throw new AconfigStorageException( + String.format("Fail to list map files in path %s", mMapPath), e); + } + + return result; + } + + /** @hide */ + public PackageTable getPackageTable(String container) { + return PackageTable.fromBytes( + mapStorageFile( + Paths.get(mMapPath, container + PMAP_FILE_EXT), FileType.PACKAGE_MAP)); + } + + /** @hide */ + public FlagTable getFlagTable(String container) { + return FlagTable.fromBytes( + mapStorageFile(Paths.get(mMapPath, container + FMAP_FILE_EXT), FileType.FLAG_MAP)); + } + + /** @hide */ + public FlagValueList getFlagValueList(String container) { + return FlagValueList.fromBytes( + mapStorageFile(Paths.get(mBootPath, container + VAL_FILE_EXT), FileType.FLAG_VAL)); + } + + // Map a storage file given file path + private static MappedByteBuffer mapStorageFile(Path file, FileType type) { + FileChannel channel = null; + try { + channel = FileChannel.open(file, StandardOpenOption.READ); + return channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); + } catch (Exception e) { + throw new AconfigStorageException( + AconfigStorageException.ERROR_CANNOT_READ_STORAGE_FILE, + String.format("Fail to mmap storage %s file %s", type.toString(), file), + e); + } finally { + quietlyDispose(channel); + } + } + + private static void quietlyDispose(Closeable closable) { + try { + if (closable != null) { + closable.close(); + } + } catch (Exception e) { + // no need to care, at least as of now + } + } +} diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java index 81168f538e..d4269dac3f 100644 --- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java +++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java @@ -63,4 +63,16 @@ public class TableUtils { long hashVal = SipHasher13.hash(val); return (int) Long.remainderUnsigned(hashVal, numBuckets); } + + public static class StorageFilesBundle { + public final PackageTable packageTable; + public final FlagTable flagTable; + public final FlagValueList flagValueList; + + public StorageFilesBundle (PackageTable pTable, FlagTable fTable, FlagValueList fValueList) { + this.packageTable = pTable; + this.flagTable = fTable; + this.flagValueList = fValueList; + } + } } diff --git a/tools/aconfig/aconfig_storage_file/tests/Android.bp b/tools/aconfig/aconfig_storage_file/tests/Android.bp index 12e4acad1b..bd46d5f0ab 100644 --- a/tools/aconfig/aconfig_storage_file/tests/Android.bp +++ b/tools/aconfig/aconfig_storage_file/tests/Android.bp @@ -10,10 +10,14 @@ cc_test { "libbase", ], data: [ - "package.map", - "flag.map", - "flag.val", - "flag.info", + "data/v1/package_v1.map", + "data/v1/flag_v1.map", + "data/v1/flag_v1.val", + "data/v1/flag_v1.info", + "data/v2/package_v2.map", + "data/v2/flag_v2.map", + "data/v2/flag_v2.val", + "data/v2/flag_v2.info", ], test_suites: [ "device-tests", @@ -35,10 +39,14 @@ android_test { test_config: "AndroidStorageJaveTest.xml", sdk_version: "test_current", data: [ - "package.map", - "flag.map", - "flag.val", - "flag.info", + "data/v1/package_v1.map", + "data/v1/flag_v1.map", + "data/v1/flag_v1.val", + "data/v1/flag_v1.info", + "data/v2/package_v2.map", + "data/v2/flag_v2.map", + "data/v2/flag_v2.val", + "data/v2/flag_v2.info", ], test_suites: [ "general-tests", diff --git a/tools/aconfig/aconfig_storage_file/tests/AndroidStorageJaveTest.xml b/tools/aconfig/aconfig_storage_file/tests/AndroidStorageJaveTest.xml index 2d52d44c57..bfc238e320 100644 --- a/tools/aconfig/aconfig_storage_file/tests/AndroidStorageJaveTest.xml +++ b/tools/aconfig/aconfig_storage_file/tests/AndroidStorageJaveTest.xml @@ -21,13 +21,18 @@ </target_preparer> <target_preparer class="com.android.compatibility.common.tradefed.targetprep.FilePusher"> <option name="cleanup" value="true" /> - <option name="push" value="package.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/package.map" /> - <option name="push" value="flag.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/flag.map" /> - <option name="push" value="flag.val->/data/local/tmp/aconfig_storage_file_test_java/testdata/flag.val" /> - <option name="push" value="flag.info->/data/local/tmp/aconfig_storage_file_test_java/testdata/flag.info" /> + <option name="push" value="package_v1.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v1.package.map" /> + <option name="push" value="flag_v1.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v1.flag.map" /> + <option name="push" value="flag_v1.val->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v1.val" /> + <option name="push" value="flag_v1.info->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v1.info" /> + <option name="push" value="package_v2.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v2.package.map" /> + <option name="push" value="flag_v2.map->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v2.flag.map" /> + <option name="push" value="flag_v2.val->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v2.val" /> + <option name="push" value="flag_v2.info->/data/local/tmp/aconfig_storage_file_test_java/testdata/mock.v2.info" /> + <option name="post-push" value="chmod +r /data/local/tmp/aconfig_storage_file_test_java/testdata/" /> </target_preparer> <test class="com.android.tradefed.testtype.AndroidJUnitTest" > <option name="package" value="android.aconfig.storage.test" /> <option name="runtime-hint" value="1m" /> </test> -</configuration>
\ No newline at end of file +</configuration> diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.info b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.info Binary files differindex 6223edf369..6223edf369 100644 --- a/tools/aconfig/aconfig_storage_file/tests/flag.info +++ b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.info diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.map b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.map Binary files differindex e868f53d7e..e868f53d7e 100644 --- a/tools/aconfig/aconfig_storage_file/tests/flag.map +++ b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.map diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.val b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.val Binary files differindex ed203d4d13..ed203d4d13 100644 --- a/tools/aconfig/aconfig_storage_file/tests/flag.val +++ b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag_v1.val diff --git a/tools/aconfig/aconfig_storage_file/tests/package.map b/tools/aconfig/aconfig_storage_file/tests/data/v1/package_v1.map Binary files differindex 6c46a0339c..6c46a0339c 100644 --- a/tools/aconfig/aconfig_storage_file/tests/package.map +++ b/tools/aconfig/aconfig_storage_file/tests/data/v1/package_v1.map diff --git a/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.info b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.info Binary files differnew file mode 100644 index 0000000000..9db7fde7ae --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.info diff --git a/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.map b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.map Binary files differnew file mode 100644 index 0000000000..cf4685ceb4 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.map diff --git a/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.val b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.val Binary files differnew file mode 100644 index 0000000000..37d4750206 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/data/v2/flag_v2.val diff --git a/tools/aconfig/aconfig_storage_file/tests/data/v2/package_v2.map b/tools/aconfig/aconfig_storage_file/tests/data/v2/package_v2.map Binary files differnew file mode 100644 index 0000000000..0a9f95ec85 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/data/v2/package_v2.map diff --git a/tools/aconfig/aconfig_storage_file/tests/jarjar.txt b/tools/aconfig/aconfig_storage_file/tests/jarjar.txt index a6c17fa476..24952ecfdf 100644 --- a/tools/aconfig/aconfig_storage_file/tests/jarjar.txt +++ b/tools/aconfig/aconfig_storage_file/tests/jarjar.txt @@ -7,6 +7,8 @@ rule android.aconfig.storage.SipHasher13 android.aconfig.storage.test.SipHasher1 rule android.aconfig.storage.FileType android.aconfig.storage.test.FileType rule android.aconfig.storage.FlagValueList android.aconfig.storage.test.FlagValueList rule android.aconfig.storage.TableUtils android.aconfig.storage.test.TableUtils +rule android.aconfig.storage.AconfigPackageImpl android.aconfig.storage.test.AconfigPackageImpl +rule android.aconfig.storage.StorageFileProvider android.aconfig.storage.test.StorageFileProvider rule android.aconfig.storage.FlagTable$* android.aconfig.storage.test.FlagTable$@1 diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/FlagTableTest.java b/tools/aconfig/aconfig_storage_file/tests/srcs/FlagTableTest.java index fd40d4c4ef..dc465b658d 100644 --- a/tools/aconfig/aconfig_storage_file/tests/srcs/FlagTableTest.java +++ b/tools/aconfig/aconfig_storage_file/tests/srcs/FlagTableTest.java @@ -31,7 +31,7 @@ public class FlagTableTest { @Test public void testFlagTable_rightHeader() throws Exception { - FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer()); + FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer(1)); FlagTable.Header header = flagTable.getHeader(); assertEquals(1, header.getVersion()); assertEquals("mockup", header.getContainer()); @@ -44,7 +44,7 @@ public class FlagTableTest { @Test public void testFlagTable_rightNode() throws Exception { - FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer()); + FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer(1)); FlagTable.Node node1 = flagTable.get(0, "enabled_ro"); FlagTable.Node node2 = flagTable.get(0, "enabled_rw"); diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/FlagValueListTest.java b/tools/aconfig/aconfig_storage_file/tests/srcs/FlagValueListTest.java index 1b0de630c7..306df7da5f 100644 --- a/tools/aconfig/aconfig_storage_file/tests/srcs/FlagValueListTest.java +++ b/tools/aconfig/aconfig_storage_file/tests/srcs/FlagValueListTest.java @@ -34,7 +34,7 @@ public class FlagValueListTest { @Test public void testFlagValueList_rightHeader() throws Exception { FlagValueList flagValueList = - FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer()); + FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer(1)); FlagValueList.Header header = flagValueList.getHeader(); assertEquals(1, header.getVersion()); assertEquals("mockup", header.getContainer()); @@ -47,7 +47,7 @@ public class FlagValueListTest { @Test public void testFlagValueList_rightNode() throws Exception { FlagValueList flagValueList = - FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer()); + FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer(1)); boolean[] expected = new boolean[] {false, true, true, false, true, true, true, true}; assertEquals(expected.length, flagValueList.size()); @@ -60,11 +60,11 @@ public class FlagValueListTest { @Test public void testFlagValueList_getValue() throws Exception { PackageTable packageTable = - PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer()); - FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer()); + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(1)); + FlagTable flagTable = FlagTable.fromBytes(TestDataUtils.getTestFlagMapByteBuffer(1)); FlagValueList flagValueList = - FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer()); + FlagValueList.fromBytes(TestDataUtils.getTestFlagValByteBuffer(1)); PackageTable.Node pNode = packageTable.get("com.android.aconfig.storage.test_1"); FlagTable.Node fNode = flagTable.get(pNode.getPackageId(), "enabled_rw"); diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/PackageTableTest.java b/tools/aconfig/aconfig_storage_file/tests/srcs/PackageTableTest.java index e7e19d8d51..4b68e5bb92 100644 --- a/tools/aconfig/aconfig_storage_file/tests/srcs/PackageTableTest.java +++ b/tools/aconfig/aconfig_storage_file/tests/srcs/PackageTableTest.java @@ -17,6 +17,8 @@ package android.aconfig.storage.test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import android.aconfig.storage.FileType; import android.aconfig.storage.PackageTable; @@ -25,13 +27,18 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CyclicBarrier; + @RunWith(JUnit4.class) public class PackageTableTest { @Test public void testPackageTable_rightHeader() throws Exception { PackageTable packageTable = - PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer()); + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(1)); PackageTable.Header header = packageTable.getHeader(); assertEquals(1, header.getVersion()); assertEquals("mockup", header.getContainer()); @@ -43,9 +50,23 @@ public class PackageTableTest { } @Test + public void testPackageTable_rightHeader_v2() throws Exception { + PackageTable packageTable = + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(2)); + PackageTable.Header header = packageTable.getHeader(); + assertEquals(2, header.getVersion()); + assertEquals("mockup", header.getContainer()); + assertEquals(FileType.PACKAGE_MAP, header.getFileType()); + assertEquals(233, header.getFileSize()); + assertEquals(3, header.getNumPackages()); + assertEquals(31, header.getBucketOffset()); + assertEquals(59, header.getNodeOffset()); + } + + @Test public void testPackageTable_rightNode() throws Exception { PackageTable packageTable = - PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer()); + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(1)); PackageTable.Node node1 = packageTable.get("com.android.aconfig.storage.test_1"); PackageTable.Node node2 = packageTable.get("com.android.aconfig.storage.test_2"); @@ -66,5 +87,103 @@ public class PackageTableTest { assertEquals(159, node1.getNextOffset()); assertEquals(-1, node2.getNextOffset()); assertEquals(-1, node4.getNextOffset()); + + assertFalse(node1.hasPackageFingerprint()); + assertFalse(node2.hasPackageFingerprint()); + assertFalse(node4.hasPackageFingerprint()); + } + + @Test + public void testPackageTable_rightNode_v2() throws Exception { + PackageTable packageTable = + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(2)); + + PackageTable.Node node1 = packageTable.get("com.android.aconfig.storage.test_1"); + PackageTable.Node node2 = packageTable.get("com.android.aconfig.storage.test_2"); + PackageTable.Node node4 = packageTable.get("com.android.aconfig.storage.test_4"); + + assertEquals("com.android.aconfig.storage.test_1", node1.getPackageName()); + assertEquals("com.android.aconfig.storage.test_2", node2.getPackageName()); + assertEquals("com.android.aconfig.storage.test_4", node4.getPackageName()); + + assertEquals(0, node1.getPackageId()); + assertEquals(1, node2.getPackageId()); + assertEquals(2, node4.getPackageId()); + + assertEquals(0, node1.getBooleanStartIndex()); + assertEquals(3, node2.getBooleanStartIndex()); + assertEquals(6, node4.getBooleanStartIndex()); + + assertEquals(175, node1.getNextOffset()); + assertEquals(-1, node2.getNextOffset()); + assertEquals(-1, node4.getNextOffset()); + + assertTrue(node1.hasPackageFingerprint()); + assertTrue(node2.hasPackageFingerprint()); + assertTrue(node4.hasPackageFingerprint()); + + assertEquals(-3197795563119393530L, node1.getPackageFingerprint()); + assertEquals(4431940502274857964L, node2.getPackageFingerprint()); + assertEquals(-2213514155997929241L, node4.getPackageFingerprint()); + } + + @Test + public void testPackageTable_getPackageList() throws Exception { + PackageTable packageTable = + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(2)); + Set<String> packages = new HashSet<>(packageTable.getPackageList()); + assertEquals(3, packages.size()); + assertTrue(packages.contains("com.android.aconfig.storage.test_1")); + assertTrue(packages.contains("com.android.aconfig.storage.test_2")); + assertTrue(packages.contains("com.android.aconfig.storage.test_4")); + + packageTable = PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(1)); + packages = new HashSet<>(packageTable.getPackageList()); + assertEquals(3, packages.size()); + assertTrue(packages.contains("com.android.aconfig.storage.test_1")); + assertTrue(packages.contains("com.android.aconfig.storage.test_2")); + assertTrue(packages.contains("com.android.aconfig.storage.test_4")); + } + + @Test + public void testPackageTable_multithreadsRead() throws Exception { + PackageTable packageTable = + PackageTable.fromBytes(TestDataUtils.getTestPackageMapByteBuffer(2)); + int numberOfThreads = 3; + Thread[] threads = new Thread[numberOfThreads]; + final CyclicBarrier gate = new CyclicBarrier(numberOfThreads + 1); + String[] expects = { + "com.android.aconfig.storage.test_1", + "com.android.aconfig.storage.test_2", + "com.android.aconfig.storage.test_4" + }; + + for (int i = 0; i < numberOfThreads; i++) { + final String packageName = expects[i]; + threads[i] = + new Thread() { + @Override + public void run() { + try { + gate.await(); + } catch (Exception e) { + } + for (int j = 0; j < 10; j++) { + if (!Objects.equals( + packageName, + packageTable.get(packageName).getPackageName())) { + throw new RuntimeException(); + } + } + } + }; + threads[i].start(); + } + + gate.await(); + + for (int i = 0; i < numberOfThreads; i++) { + threads[i].join(); + } } } diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/StorageFileProviderTest.java b/tools/aconfig/aconfig_storage_file/tests/srcs/StorageFileProviderTest.java new file mode 100644 index 0000000000..c2720f9544 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/tests/srcs/StorageFileProviderTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.aconfig.storage.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import android.aconfig.storage.FlagTable; +import android.aconfig.storage.FlagValueList; +import android.aconfig.storage.PackageTable; +import android.aconfig.storage.StorageFileProvider; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.nio.file.Paths; +import java.util.List; + +@RunWith(JUnit4.class) +public class StorageFileProviderTest { + + @Test + public void testlistContainers() throws Exception { + StorageFileProvider p = + new StorageFileProvider(TestDataUtils.TESTDATA_PATH, TestDataUtils.TESTDATA_PATH); + String[] excludes = {}; + List<String> containers = p.listContainers(excludes); + assertEquals(2, containers.size()); + + excludes = new String[] {"mock.v1"}; + containers = p.listContainers(excludes); + assertEquals(1, containers.size()); + + p = new StorageFileProvider("fake/path/", "fake/path/"); + containers = p.listContainers(excludes); + assertTrue(containers.isEmpty()); + } + + @Test + public void testLoadFiles() throws Exception { + StorageFileProvider p = + new StorageFileProvider(TestDataUtils.TESTDATA_PATH, TestDataUtils.TESTDATA_PATH); + PackageTable pt = p.getPackageTable("mock.v1"); + assertNotNull(pt); + FlagTable f = p.getFlagTable("mock.v1"); + assertNotNull(f); + FlagValueList v = p.getFlagValueList("mock.v1"); + assertNotNull(v); + } +} diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/TestDataUtils.java b/tools/aconfig/aconfig_storage_file/tests/srcs/TestDataUtils.java index f35952d392..388971e28b 100644 --- a/tools/aconfig/aconfig_storage_file/tests/srcs/TestDataUtils.java +++ b/tools/aconfig/aconfig_storage_file/tests/srcs/TestDataUtils.java @@ -21,28 +21,27 @@ import java.io.InputStream; import java.nio.ByteBuffer; public final class TestDataUtils { - private static final String TEST_PACKAGE_MAP_PATH = "package.map"; - private static final String TEST_FLAG_MAP_PATH = "flag.map"; - private static final String TEST_FLAG_VAL_PATH = "flag.val"; - private static final String TEST_FLAG_INFO_PATH = "flag.info"; + private static final String TEST_PACKAGE_MAP_PATH = "mock.v%d.package.map"; + private static final String TEST_FLAG_MAP_PATH = "mock.v%d.flag.map"; + private static final String TEST_FLAG_VAL_PATH = "mock.v%d.val"; + private static final String TEST_FLAG_INFO_PATH = "mock.v%d.info"; - private static final String TESTDATA_PATH = - "/data/local/tmp/aconfig_storage_file_test_java/testdata/"; + public static final String TESTDATA_PATH = "/data/local/tmp/aconfig_storage_file_test_java/testdata/"; - public static ByteBuffer getTestPackageMapByteBuffer() throws Exception { - return readFile(TESTDATA_PATH + TEST_PACKAGE_MAP_PATH); + public static ByteBuffer getTestPackageMapByteBuffer(int version) throws Exception { + return readFile(TESTDATA_PATH + String.format(TEST_PACKAGE_MAP_PATH, version)); } - public static ByteBuffer getTestFlagMapByteBuffer() throws Exception { - return readFile(TESTDATA_PATH + TEST_FLAG_MAP_PATH); + public static ByteBuffer getTestFlagMapByteBuffer(int version) throws Exception { + return readFile(TESTDATA_PATH + String.format(TEST_FLAG_MAP_PATH, version)); } - public static ByteBuffer getTestFlagValByteBuffer() throws Exception { - return readFile(TESTDATA_PATH + TEST_FLAG_VAL_PATH); + public static ByteBuffer getTestFlagValByteBuffer(int version) throws Exception { + return readFile(TESTDATA_PATH + String.format(TEST_FLAG_VAL_PATH, version)); } - public static ByteBuffer getTestFlagInfoByteBuffer() throws Exception { - return readFile(TESTDATA_PATH + TEST_FLAG_INFO_PATH); + public static ByteBuffer getTestFlagInfoByteBuffer(int version) throws Exception { + return readFile(TESTDATA_PATH + String.format(TEST_FLAG_INFO_PATH, version)); } private static ByteBuffer readFile(String fileName) throws Exception { diff --git a/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp index ebd1dd89bd..5c008afbf1 100644 --- a/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp +++ b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp @@ -24,10 +24,8 @@ using namespace android::base; using namespace aconfig_storage; -void verify_value(const FlagValueSummary& flag, - const std::string& package_name, - const std::string& flag_name, - const std::string& flag_val, +void verify_value(const FlagValueSummary& flag, const std::string& package_name, + const std::string& flag_name, const std::string& flag_val, const std::string& value_type) { ASSERT_EQ(flag.package_name, package_name); ASSERT_EQ(flag.flag_name, flag_name); @@ -39,10 +37,8 @@ void verify_value_info(const FlagValueAndInfoSummary& flag, const std::string& package_name, const std::string& flag_name, const std::string& flag_val, - const std::string& value_type, - bool is_readwrite, - bool has_server_override, - bool has_local_override) { + const std::string& value_type, bool is_readwrite, + bool has_server_override, bool has_local_override) { ASSERT_EQ(flag.package_name, package_name); ASSERT_EQ(flag.flag_name, flag_name); ASSERT_EQ(flag.flag_value, flag_val); @@ -52,61 +48,137 @@ void verify_value_info(const FlagValueAndInfoSummary& flag, ASSERT_EQ(flag.has_local_override, has_local_override); } +Result<std::vector<FlagValueSummary>> get_flag_list_result( + const std::string version) { + auto const test_base_dir = GetExecutableDirectory(); + auto const test_dir = test_base_dir + "/data/v" + version; + auto const package_map = test_dir + "/package_v" + version + ".map"; + auto const flag_map = test_dir + "/flag_v" + version + ".map"; + auto const flag_val = test_dir + "/flag_v" + version + ".val"; + return aconfig_storage::list_flags(package_map, flag_map, flag_val); +} + +Result<std::vector<FlagValueAndInfoSummary>> get_flag_list_result_with_info( + const std::string version) { + auto const test_base_dir = GetExecutableDirectory(); + auto const test_dir = test_base_dir + "/data/v" + version; + auto const package_map = test_dir + "/package_v" + version + ".map"; + auto const flag_map = test_dir + "/flag_v" + version + ".map"; + auto const flag_val = test_dir + "/flag_v" + version + ".val"; + auto const flag_info = test_dir + "/flag_v" + version + ".info"; + return aconfig_storage::list_flags_with_info(package_map, flag_map, flag_val, + flag_info); +} + TEST(AconfigStorageFileTest, test_list_flag) { - auto const test_dir = GetExecutableDirectory(); - auto const package_map = test_dir + "/package.map"; - auto const flag_map = test_dir + "/flag.map"; - auto const flag_val = test_dir + "/flag.val"; - auto flag_list_result = aconfig_storage::list_flags( - package_map, flag_map, flag_val); + auto flag_list_result = get_flag_list_result("1"); + ASSERT_TRUE(flag_list_result.ok()); + + auto const& flag_list = *flag_list_result; + ASSERT_EQ(flag_list.size(), 8); + verify_value(flag_list[0], "com.android.aconfig.storage.test_1", + "disabled_rw", "false", "ReadWriteBoolean"); + verify_value(flag_list[1], "com.android.aconfig.storage.test_1", "enabled_ro", + "true", "ReadOnlyBoolean"); + verify_value(flag_list[2], "com.android.aconfig.storage.test_1", "enabled_rw", + "true", "ReadWriteBoolean"); + verify_value(flag_list[3], "com.android.aconfig.storage.test_2", + "disabled_rw", "false", "ReadWriteBoolean"); + verify_value(flag_list[4], "com.android.aconfig.storage.test_2", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean"); + verify_value(flag_list[5], "com.android.aconfig.storage.test_2", "enabled_ro", + "true", "ReadOnlyBoolean"); + verify_value(flag_list[6], "com.android.aconfig.storage.test_4", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean"); + verify_value(flag_list[7], "com.android.aconfig.storage.test_4", "enabled_rw", + "true", "ReadWriteBoolean"); +} + +// TODO: b/376256472 - Use parameterized tests. +TEST(AconfigStorageFileTest, test_list_flag_v2) { + auto flag_list_result = get_flag_list_result("2"); ASSERT_TRUE(flag_list_result.ok()); auto const& flag_list = *flag_list_result; ASSERT_EQ(flag_list.size(), 8); - verify_value(flag_list[0], "com.android.aconfig.storage.test_1", "disabled_rw", - "false", "ReadWriteBoolean"); + verify_value(flag_list[0], "com.android.aconfig.storage.test_1", + "disabled_rw", "false", "ReadWriteBoolean"); verify_value(flag_list[1], "com.android.aconfig.storage.test_1", "enabled_ro", "true", "ReadOnlyBoolean"); verify_value(flag_list[2], "com.android.aconfig.storage.test_1", "enabled_rw", "true", "ReadWriteBoolean"); - verify_value(flag_list[3], "com.android.aconfig.storage.test_2", "disabled_rw", - "false", "ReadWriteBoolean"); - verify_value(flag_list[4], "com.android.aconfig.storage.test_2", "enabled_fixed_ro", - "true", "FixedReadOnlyBoolean"); + verify_value(flag_list[3], "com.android.aconfig.storage.test_2", + "disabled_rw", "false", "ReadWriteBoolean"); + verify_value(flag_list[4], "com.android.aconfig.storage.test_2", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean"); verify_value(flag_list[5], "com.android.aconfig.storage.test_2", "enabled_ro", "true", "ReadOnlyBoolean"); - verify_value(flag_list[6], "com.android.aconfig.storage.test_4", "enabled_fixed_ro", - "true", "FixedReadOnlyBoolean"); + verify_value(flag_list[6], "com.android.aconfig.storage.test_4", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean"); verify_value(flag_list[7], "com.android.aconfig.storage.test_4", "enabled_rw", "true", "ReadWriteBoolean"); } TEST(AconfigStorageFileTest, test_list_flag_with_info) { - auto const test_dir = GetExecutableDirectory(); - auto const package_map = test_dir + "/package.map"; - auto const flag_map = test_dir + "/flag.map"; - auto const flag_val = test_dir + "/flag.val"; - auto const flag_info = test_dir + "/flag.info"; - auto flag_list_result = aconfig_storage::list_flags_with_info( - package_map, flag_map, flag_val, flag_info); + auto flag_list_result = get_flag_list_result_with_info("1"); + ASSERT_TRUE(flag_list_result.ok()); + + auto const& flag_list = *flag_list_result; + ASSERT_EQ(flag_list.size(), 8); + verify_value_info(flag_list[0], "com.android.aconfig.storage.test_1", + "disabled_rw", "false", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[1], "com.android.aconfig.storage.test_1", + "enabled_ro", "true", "ReadOnlyBoolean", false, false, + false); + verify_value_info(flag_list[2], "com.android.aconfig.storage.test_1", + "enabled_rw", "true", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[3], "com.android.aconfig.storage.test_2", + "disabled_rw", "false", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[4], "com.android.aconfig.storage.test_2", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean", false, + false, false); + verify_value_info(flag_list[5], "com.android.aconfig.storage.test_2", + "enabled_ro", "true", "ReadOnlyBoolean", false, false, + false); + verify_value_info(flag_list[6], "com.android.aconfig.storage.test_4", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean", false, + false, false); + verify_value_info(flag_list[7], "com.android.aconfig.storage.test_4", + "enabled_rw", "true", "ReadWriteBoolean", true, false, + false); +} + +TEST(AconfigStorageFileTest, test_list_flag_with_info_v2) { + auto flag_list_result = get_flag_list_result_with_info("2"); ASSERT_TRUE(flag_list_result.ok()); auto const& flag_list = *flag_list_result; ASSERT_EQ(flag_list.size(), 8); - verify_value_info(flag_list[0], "com.android.aconfig.storage.test_1", "disabled_rw", - "false", "ReadWriteBoolean", true, false, false); - verify_value_info(flag_list[1], "com.android.aconfig.storage.test_1", "enabled_ro", - "true", "ReadOnlyBoolean", false, false, false); - verify_value_info(flag_list[2], "com.android.aconfig.storage.test_1", "enabled_rw", - "true", "ReadWriteBoolean", true, false, false); - verify_value_info(flag_list[3], "com.android.aconfig.storage.test_2", "disabled_rw", - "false", "ReadWriteBoolean", true, false, false); - verify_value_info(flag_list[4], "com.android.aconfig.storage.test_2", "enabled_fixed_ro", - "true", "FixedReadOnlyBoolean", false, false, false); - verify_value_info(flag_list[5], "com.android.aconfig.storage.test_2", "enabled_ro", - "true", "ReadOnlyBoolean", false, false, false); - verify_value_info(flag_list[6], "com.android.aconfig.storage.test_4", "enabled_fixed_ro", - "true", "FixedReadOnlyBoolean", false, false, false); - verify_value_info(flag_list[7], "com.android.aconfig.storage.test_4", "enabled_rw", - "true", "ReadWriteBoolean", true, false, false); + verify_value_info(flag_list[0], "com.android.aconfig.storage.test_1", + "disabled_rw", "false", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[1], "com.android.aconfig.storage.test_1", + "enabled_ro", "true", "ReadOnlyBoolean", false, false, + false); + verify_value_info(flag_list[2], "com.android.aconfig.storage.test_1", + "enabled_rw", "true", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[3], "com.android.aconfig.storage.test_2", + "disabled_rw", "false", "ReadWriteBoolean", true, false, + false); + verify_value_info(flag_list[4], "com.android.aconfig.storage.test_2", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean", false, + false, false); + verify_value_info(flag_list[5], "com.android.aconfig.storage.test_2", + "enabled_ro", "true", "ReadOnlyBoolean", false, false, + false); + verify_value_info(flag_list[6], "com.android.aconfig.storage.test_4", + "enabled_fixed_ro", "true", "FixedReadOnlyBoolean", false, + false, false); + verify_value_info(flag_list[7], "com.android.aconfig.storage.test_4", + "enabled_rw", "true", "ReadWriteBoolean", true, false, + false); } diff --git a/tools/aconfig/aconfig_storage_read_api/Android.bp b/tools/aconfig/aconfig_storage_read_api/Android.bp index f96b2230d1..16341b9273 100644 --- a/tools/aconfig/aconfig_storage_read_api/Android.bp +++ b/tools/aconfig/aconfig_storage_read_api/Android.bp @@ -36,10 +36,10 @@ rust_test_host { "librand", ], data: [ - "tests/package.map", - "tests/flag.map", - "tests/flag.val", - "tests/flag.info", + "tests/data/v1/package_v1.map", + "tests/data/v1/flag_v1.map", + "tests/data/v1/flag_v1.val", + "tests/data/v1/flag_v1.info", ], } @@ -107,31 +107,12 @@ cc_library { afdo: true, } -soong_config_module_type { - name: "aconfig_lib_cc_shared_link_defaults", - module_type: "cc_defaults", - config_namespace: "Aconfig", - bool_variables: [ - "read_from_new_storage", - ], - properties: [ - "shared_libs", - ], -} - -soong_config_bool_variable { - name: "read_from_new_storage", -} - -aconfig_lib_cc_shared_link_defaults { +cc_defaults { name: "aconfig_lib_cc_shared_link.defaults", - soong_config_variables: { - read_from_new_storage: { - shared_libs: [ - "libaconfig_storage_read_api_cc", - ], - }, - }, + shared_libs: select(release_flag("RELEASE_READ_FROM_NEW_STORAGE"), { + true: ["libaconfig_storage_read_api_cc"], + default: [], + }), } cc_defaults { @@ -173,37 +154,17 @@ java_library { java_library { name: "aconfig_storage_reader_java", srcs: [ - "srcs/android/aconfig/storage/StorageInternalReader.java", + "srcs/android/os/flagging/*.java", ], libs: [ "unsupportedappusage", - "strict_mode_stub", ], static_libs: [ "aconfig_storage_file_java", ], - sdk_version: "core_current", - host_supported: true, - min_sdk_version: "29", - apex_available: [ - "//apex_available:platform", - "//apex_available:anyapex", - ], -} - -java_library { - name: "aconfig_storage_reader_java_none", - srcs: [ - "srcs/android/aconfig/storage/StorageInternalReader.java", - ], - libs: [ - "unsupportedappusage-sdk-none", - "fake_device_config", - ], - static_libs: [ - "aconfig_storage_file_java_none", + sdk_version: "current", + visibility: [ + "//frameworks/base", + "//build/make/tools/aconfig/aconfig_storage_read_api/tests", ], - sdk_version: "none", - system_modules: "core-all-system-modules", - host_supported: true, } diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs index 6d03377683..68b6193079 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_info_query.rs @@ -16,8 +16,10 @@ //! flag value query module defines the flag value file read from mapped bytes -use crate::{AconfigStorageError, FILE_VERSION}; -use aconfig_storage_file::{flag_info::FlagInfoHeader, read_u8_from_bytes, FlagValueType}; +use crate::AconfigStorageError; +use aconfig_storage_file::{ + flag_info::FlagInfoHeader, read_u8_from_bytes, FlagValueType, MAX_SUPPORTED_FILE_VERSION, +}; use anyhow::anyhow; /// Get flag attribute bitfield @@ -27,11 +29,11 @@ pub fn find_flag_attribute( flag_index: u32, ) -> Result<u8, AconfigStorageError> { let interpreted_header = FlagInfoHeader::from_bytes(buf)?; - if interpreted_header.version > crate::FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -53,12 +55,14 @@ pub fn find_flag_attribute( #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::{test_utils::create_test_flag_info_list, FlagInfoBit}; + use aconfig_storage_file::{ + test_utils::create_test_flag_info_list, FlagInfoBit, DEFAULT_FILE_VERSION, + }; #[test] // this test point locks down query if flag has server override fn test_is_flag_sticky() { - let flag_info_list = create_test_flag_info_list().into_bytes(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(); for offset in 0..8 { let attribute = find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, offset).unwrap(); @@ -69,7 +73,7 @@ mod tests { #[test] // this test point locks down query if flag is readwrite fn test_is_flag_readwrite() { - let flag_info_list = create_test_flag_info_list().into_bytes(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(); let baseline: Vec<bool> = vec![true, false, true, true, false, false, false, true]; for offset in 0..8 { let attribute = @@ -84,7 +88,7 @@ mod tests { #[test] // this test point locks down query if flag has local override fn test_flag_has_override() { - let flag_info_list = create_test_flag_info_list().into_bytes(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(); for offset in 0..8 { let attribute = find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, offset).unwrap(); @@ -95,7 +99,7 @@ mod tests { #[test] // this test point locks down query beyond the end of boolean section fn test_boolean_out_of_range() { - let flag_info_list = create_test_flag_info_list().into_bytes(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(); let error = find_flag_attribute(&flag_info_list[..], FlagValueType::Boolean, 8).unwrap_err(); assert_eq!( @@ -107,16 +111,16 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut info_list = create_test_flag_info_list(); - info_list.header.version = crate::FILE_VERSION + 1; + let mut info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); + info_list.header.version = MAX_SUPPORTED_FILE_VERSION + 1; let flag_info = info_list.into_bytes(); let error = find_flag_attribute(&flag_info[..], FlagValueType::Boolean, 4).unwrap_err(); assert_eq!( format!("{:?}", error), format!( "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", - crate::FILE_VERSION + 1, - crate::FILE_VERSION + MAX_SUPPORTED_FILE_VERSION + 1, + MAX_SUPPORTED_FILE_VERSION ) ); } diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs index a1a4793bc2..3e87acc43b 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_table_query.rs @@ -16,9 +16,10 @@ //! flag table query module defines the flag table file read from mapped bytes -use crate::{AconfigStorageError, FILE_VERSION}; +use crate::AconfigStorageError; use aconfig_storage_file::{ flag_table::FlagTableHeader, flag_table::FlagTableNode, read_u32_from_bytes, StoredFlagType, + MAX_SUPPORTED_FILE_VERSION, }; use anyhow::anyhow; @@ -36,11 +37,11 @@ pub fn find_flag_read_context( flag: &str, ) -> Result<Option<FlagReadContext>, AconfigStorageError> { let interpreted_header = FlagTableHeader::from_bytes(buf)?; - if interpreted_header.version > crate::FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -73,12 +74,12 @@ pub fn find_flag_read_context( #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::test_utils::create_test_flag_table; + use aconfig_storage_file::{test_utils::create_test_flag_table, DEFAULT_FILE_VERSION}; #[test] // this test point locks down table query fn test_flag_query() { - let flag_table = create_test_flag_table().into_bytes(); + let flag_table = create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes(); let baseline = vec![ (0, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 1u16), (0, "enabled_rw", StoredFlagType::ReadWriteBoolean, 2u16), @@ -100,7 +101,7 @@ mod tests { #[test] // this test point locks down table query of a non exist flag fn test_not_existed_flag_query() { - let flag_table = create_test_flag_table().into_bytes(); + let flag_table = create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes(); let flag_context = find_flag_read_context(&flag_table[..], 1, "disabled_fixed_ro").unwrap(); assert_eq!(flag_context, None); let flag_context = find_flag_read_context(&flag_table[..], 2, "disabled_rw").unwrap(); @@ -110,16 +111,16 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut table = create_test_flag_table(); - table.header.version = crate::FILE_VERSION + 1; + let mut table = create_test_flag_table(DEFAULT_FILE_VERSION); + table.header.version = MAX_SUPPORTED_FILE_VERSION + 1; let flag_table = table.into_bytes(); let error = find_flag_read_context(&flag_table[..], 0, "enabled_ro").unwrap_err(); assert_eq!( format!("{:?}", error), format!( "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", - crate::FILE_VERSION + 1, - crate::FILE_VERSION + MAX_SUPPORTED_FILE_VERSION + 1, + MAX_SUPPORTED_FILE_VERSION ) ); } diff --git a/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs b/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs index 9d32a16ac8..35f56929a9 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/flag_value_query.rs @@ -16,18 +16,20 @@ //! flag value query module defines the flag value file read from mapped bytes -use crate::{AconfigStorageError, FILE_VERSION}; -use aconfig_storage_file::{flag_value::FlagValueHeader, read_u8_from_bytes}; +use crate::AconfigStorageError; +use aconfig_storage_file::{ + flag_value::FlagValueHeader, read_u8_from_bytes, MAX_SUPPORTED_FILE_VERSION, +}; use anyhow::anyhow; /// Query flag value pub fn find_boolean_flag_value(buf: &[u8], flag_index: u32) -> Result<bool, AconfigStorageError> { let interpreted_header = FlagValueHeader::from_bytes(buf)?; - if interpreted_header.version > crate::FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -46,12 +48,12 @@ pub fn find_boolean_flag_value(buf: &[u8], flag_index: u32) -> Result<bool, Acon #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::test_utils::create_test_flag_value_list; + use aconfig_storage_file::{test_utils::create_test_flag_value_list, DEFAULT_FILE_VERSION}; #[test] // this test point locks down flag value query fn test_flag_value_query() { - let flag_value_list = create_test_flag_value_list().into_bytes(); + let flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(); let baseline: Vec<bool> = vec![false, true, true, false, true, true, true, true]; for (offset, expected_value) in baseline.into_iter().enumerate() { let flag_value = find_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap(); @@ -62,7 +64,7 @@ mod tests { #[test] // this test point locks down query beyond the end of boolean section fn test_boolean_out_of_range() { - let flag_value_list = create_test_flag_value_list().into_bytes(); + let flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(); let error = find_boolean_flag_value(&flag_value_list[..], 8).unwrap_err(); assert_eq!( format!("{:?}", error), @@ -73,16 +75,16 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut value_list = create_test_flag_value_list(); - value_list.header.version = crate::FILE_VERSION + 1; + let mut value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); + value_list.header.version = MAX_SUPPORTED_FILE_VERSION + 1; let flag_value = value_list.into_bytes(); let error = find_boolean_flag_value(&flag_value[..], 4).unwrap_err(); assert_eq!( format!("{:?}", error), format!( "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", - crate::FILE_VERSION + 1, - crate::FILE_VERSION + MAX_SUPPORTED_FILE_VERSION + 1, + MAX_SUPPORTED_FILE_VERSION ) ); } diff --git a/tools/aconfig/aconfig_storage_read_api/src/lib.rs b/tools/aconfig/aconfig_storage_read_api/src/lib.rs index d76cf3fe4e..d3cc9d427d 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/lib.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/lib.rs @@ -44,9 +44,10 @@ pub mod package_table_query; pub use aconfig_storage_file::{AconfigStorageError, FlagValueType, StorageFileType}; pub use flag_table_query::FlagReadContext; +pub use mapped_file::map_file; pub use package_table_query::PackageReadContext; -use aconfig_storage_file::{read_u32_from_bytes, FILE_VERSION}; +use aconfig_storage_file::read_u32_from_bytes; use flag_info_query::find_flag_attribute; use flag_table_query::find_flag_read_context; use flag_value_query::find_boolean_flag_value; @@ -114,13 +115,13 @@ pub fn get_flag_read_context( /// Get the boolean flag value. /// -/// \input file: mapped flag file +/// \input file: a byte slice, can be either &Mmap or &MapMut /// \input index: boolean flag offset /// /// \return /// If the provide offset is valid, it returns the boolean flag value, otherwise it /// returns the error message. -pub fn get_boolean_flag_value(file: &Mmap, index: u32) -> Result<bool, AconfigStorageError> { +pub fn get_boolean_flag_value(file: &[u8], index: u32) -> Result<bool, AconfigStorageError> { find_boolean_flag_value(file, index) } @@ -148,7 +149,7 @@ pub fn get_storage_file_version(file_path: &str) -> Result<u32, AconfigStorageEr /// Get the flag attribute. /// -/// \input file: mapped flag info file +/// \input file: a byte slice, can be either &Mmap or &MapMut /// \input flag_type: flag value type /// \input flag_index: flag index /// @@ -156,7 +157,7 @@ pub fn get_storage_file_version(file_path: &str) -> Result<u32, AconfigStorageEr /// If the provide offset is valid, it returns the flag attribute bitfiled, otherwise it /// returns the error message. pub fn get_flag_attribute( - file: &Mmap, + file: &[u8], flag_type: FlagValueType, flag_index: u32, ) -> Result<u8, AconfigStorageError> { @@ -412,10 +413,10 @@ mod tests { let flag_map = storage_dir.clone() + "/maps/mockup.flag.map"; let flag_val = storage_dir.clone() + "/boot/mockup.val"; let flag_info = storage_dir.clone() + "/boot/mockup.info"; - fs::copy("./tests/package.map", &package_map).unwrap(); - fs::copy("./tests/flag.map", &flag_map).unwrap(); - fs::copy("./tests/flag.val", &flag_val).unwrap(); - fs::copy("./tests/flag.info", &flag_info).unwrap(); + fs::copy("./tests/data/v1/package_v1.map", &package_map).unwrap(); + fs::copy("./tests/data/v1/flag_v1.map", &flag_map).unwrap(); + fs::copy("./tests/data/v1/flag_v1.val", &flag_val).unwrap(); + fs::copy("./tests/data/v1/flag_v1.info", &flag_info).unwrap(); return storage_dir; } @@ -432,21 +433,24 @@ mod tests { get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_1") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + let expected_package_context = + PackageReadContext { package_id: 0, boolean_start_index: 0, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_2") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + let expected_package_context = + PackageReadContext { package_id: 1, boolean_start_index: 3, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_4") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + let expected_package_context = + PackageReadContext { package_id: 2, boolean_start_index: 6, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); } @@ -507,9 +511,9 @@ mod tests { #[test] // this test point locks down flag storage file version number query api fn test_storage_version_query() { - assert_eq!(get_storage_file_version("./tests/package.map").unwrap(), 1); - assert_eq!(get_storage_file_version("./tests/flag.map").unwrap(), 1); - assert_eq!(get_storage_file_version("./tests/flag.val").unwrap(), 1); - assert_eq!(get_storage_file_version("./tests/flag.info").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/data/v1/package_v1.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/data/v1/flag_v1.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/data/v1/flag_v1.val").unwrap(), 1); + assert_eq!(get_storage_file_version("./tests/data/v1/flag_v1.info").unwrap(), 1); } } diff --git a/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs index 5a1664535f..f4e269e68b 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs @@ -28,7 +28,7 @@ use crate::StorageFileType; /// The memory mapped file may have undefined behavior if there are writes to this /// file after being mapped. Ensure no writes can happen to this file while this /// mapping stays alive. -unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> { +pub unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> { let file = File::open(file_path) .map_err(|errmsg| FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)))?; unsafe { @@ -97,10 +97,10 @@ mod tests { let flag_map = storage_dir.clone() + "/maps/mockup.flag.map"; let flag_val = storage_dir.clone() + "/boot/mockup.val"; let flag_info = storage_dir.clone() + "/boot/mockup.info"; - fs::copy("./tests/package.map", &package_map).unwrap(); - fs::copy("./tests/flag.map", &flag_map).unwrap(); - fs::copy("./tests/flag.val", &flag_val).unwrap(); - fs::copy("./tests/flag.info", &flag_info).unwrap(); + fs::copy("./tests/data/v1/package_v1.map", &package_map).unwrap(); + fs::copy("./tests/data/v1/flag_v1.map", &flag_map).unwrap(); + fs::copy("./tests/data/v1/flag_v1.val", &flag_val).unwrap(); + fs::copy("./tests/data/v1/flag_v1.info", &flag_info).unwrap(); return storage_dir; } @@ -108,9 +108,9 @@ mod tests { #[test] fn test_mapped_file_contents() { let storage_dir = create_test_storage_files(); - map_and_verify(&storage_dir, StorageFileType::PackageMap, "./tests/package.map"); - map_and_verify(&storage_dir, StorageFileType::FlagMap, "./tests/flag.map"); - map_and_verify(&storage_dir, StorageFileType::FlagVal, "./tests/flag.val"); - map_and_verify(&storage_dir, StorageFileType::FlagInfo, "./tests/flag.info"); + map_and_verify(&storage_dir, StorageFileType::PackageMap, "./tests/data/v1/package_v1.map"); + map_and_verify(&storage_dir, StorageFileType::FlagMap, "./tests/data/v1/flag_v1.map"); + map_and_verify(&storage_dir, StorageFileType::FlagVal, "./tests/data/v1/flag_v1.val"); + map_and_verify(&storage_dir, StorageFileType::FlagInfo, "./tests/data/v1/flag_v1.info"); } } diff --git a/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs b/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs index 2cb854b1b1..b20668f9c2 100644 --- a/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs +++ b/tools/aconfig/aconfig_storage_read_api/src/package_table_query.rs @@ -16,9 +16,10 @@ //! package table query module defines the package table file read from mapped bytes -use crate::{AconfigStorageError, FILE_VERSION}; +use crate::AconfigStorageError; use aconfig_storage_file::{ package_table::PackageTableHeader, package_table::PackageTableNode, read_u32_from_bytes, + MAX_SUPPORTED_FILE_VERSION, }; use anyhow::anyhow; @@ -27,6 +28,7 @@ use anyhow::anyhow; pub struct PackageReadContext { pub package_id: u32, pub boolean_start_index: u32, + pub fingerprint: u64, } /// Query package read context: package id and start index @@ -35,11 +37,11 @@ pub fn find_package_read_context( package: &str, ) -> Result<Option<PackageReadContext>, AconfigStorageError> { let interpreted_header = PackageTableHeader::from_bytes(buf)?; - if interpreted_header.version > FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -55,11 +57,13 @@ pub fn find_package_read_context( } loop { - let interpreted_node = PackageTableNode::from_bytes(&buf[package_node_offset..])?; + let interpreted_node = + PackageTableNode::from_bytes(&buf[package_node_offset..], interpreted_header.version)?; if interpreted_node.package_name == package { return Ok(Some(PackageReadContext { package_id: interpreted_node.package_id, boolean_start_index: interpreted_node.boolean_start_index, + fingerprint: interpreted_node.fingerprint, })); } match interpreted_node.next_offset { @@ -72,29 +76,68 @@ pub fn find_package_read_context( #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::test_utils::create_test_package_table; + use aconfig_storage_file::{test_utils::create_test_package_table, DEFAULT_FILE_VERSION}; #[test] // this test point locks down table query fn test_package_query() { - let package_table = create_test_package_table().into_bytes(); + let package_table = create_test_package_table(DEFAULT_FILE_VERSION).into_bytes(); let package_context = find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_1") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + let expected_package_context = + PackageReadContext { package_id: 0, boolean_start_index: 0, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_2") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + let expected_package_context = + PackageReadContext { package_id: 1, boolean_start_index: 3, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_4") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + let expected_package_context = + PackageReadContext { package_id: 2, boolean_start_index: 6, fingerprint: 0 }; + assert_eq!(package_context, expected_package_context); + } + + #[test] + // this test point locks down table query + fn test_package_query_v2() { + let package_table = create_test_package_table(2).into_bytes(); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_1") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 0, + boolean_start_index: 0, + fingerprint: 15248948510590158086u64, + }; + assert_eq!(package_context, expected_package_context); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_2") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 1, + boolean_start_index: 3, + fingerprint: 4431940502274857964u64, + }; + assert_eq!(package_context, expected_package_context); + let package_context = + find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_4") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 2, + boolean_start_index: 6, + fingerprint: 16233229917711622375u64, + }; assert_eq!(package_context, expected_package_context); } @@ -102,7 +145,7 @@ mod tests { // this test point locks down table query of a non exist package fn test_not_existed_package_query() { // this will land at an empty bucket - let package_table = create_test_package_table().into_bytes(); + let package_table = create_test_package_table(DEFAULT_FILE_VERSION).into_bytes(); let package_context = find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_3") .unwrap(); @@ -117,8 +160,8 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut table = create_test_package_table(); - table.header.version = crate::FILE_VERSION + 1; + let mut table = create_test_package_table(DEFAULT_FILE_VERSION); + table.header.version = MAX_SUPPORTED_FILE_VERSION + 1; let package_table = table.into_bytes(); let error = find_package_read_context(&package_table[..], "com.android.aconfig.storage.test_1") @@ -127,8 +170,8 @@ mod tests { format!("{:?}", error), format!( "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", - crate::FILE_VERSION + 1, - crate::FILE_VERSION + MAX_SUPPORTED_FILE_VERSION + 1, + MAX_SUPPORTED_FILE_VERSION ) ); } diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java b/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java deleted file mode 100644 index 29ebee5ab4..0000000000 --- a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.aconfig.storage; - -import android.compat.annotation.UnsupportedAppUsage; -import android.os.StrictMode; - -import java.io.Closeable; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; - -/** @hide */ -public class StorageInternalReader { - - private static final String MAP_PATH = "/metadata/aconfig/maps/"; - private static final String BOOT_PATH = "/metadata/aconfig/boot/"; - - private PackageTable mPackageTable; - private FlagValueList mFlagValueList; - - private int mPackageBooleanStartOffset; - - @UnsupportedAppUsage - public StorageInternalReader(String container, String packageName) { - this(packageName, MAP_PATH + container + ".package.map", BOOT_PATH + container + ".val"); - } - - @UnsupportedAppUsage - public StorageInternalReader(String packageName, String packageMapFile, String flagValueFile) { - StrictMode.ThreadPolicy oldPolicy = StrictMode.allowThreadDiskReads(); - mPackageTable = PackageTable.fromBytes(mapStorageFile(packageMapFile)); - mFlagValueList = FlagValueList.fromBytes(mapStorageFile(flagValueFile)); - StrictMode.setThreadPolicy(oldPolicy); - mPackageBooleanStartOffset = getPackageBooleanStartOffset(packageName); - } - - @UnsupportedAppUsage - public boolean getBooleanFlagValue(int index) { - index += mPackageBooleanStartOffset; - if (index >= mFlagValueList.size()) { - throw new AconfigStorageException("Fail to get boolean flag value"); - } - return mFlagValueList.getBoolean(index); - } - - private int getPackageBooleanStartOffset(String packageName) { - PackageTable.Node pNode = mPackageTable.get(packageName); - if (pNode == null) { - PackageTable.Header header = mPackageTable.getHeader(); - throw new AconfigStorageException( - String.format( - "Fail to get package %s from container %s", - packageName, header.getContainer())); - } - return pNode.getBooleanStartIndex(); - } - - // Map a storage file given file path - private static MappedByteBuffer mapStorageFile(String file) { - FileChannel channel = null; - try { - channel = FileChannel.open(Paths.get(file), StandardOpenOption.READ); - return channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); - } catch (Exception e) { - throw new AconfigStorageException( - String.format("Fail to mmap storage file %s", file), e); - } finally { - quietlyDispose(channel); - } - } - - private static void quietlyDispose(Closeable closable) { - try { - if (closable != null) { - closable.close(); - } - } catch (Exception e) { - // no need to care, at least as of now - } - } -} diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackage.java b/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackage.java new file mode 100644 index 0000000000..ddad249fa0 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackage.java @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os.flagging; + +import static android.aconfig.storage.TableUtils.StorageFilesBundle; + +import android.aconfig.storage.AconfigStorageException; +import android.aconfig.storage.FlagTable; +import android.aconfig.storage.FlagValueList; +import android.aconfig.storage.PackageTable; +import android.compat.annotation.UnsupportedAppUsage; +import android.util.Log; + +import java.io.Closeable; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * An {@code aconfig} package containing the enabled state of its flags. + * + * <p><strong>Note: this is intended only to be used by generated code. To determine if a given flag + * is enabled in app code, the generated android flags should be used.</strong> + * + * <p>This class is used to read the flag from platform Aconfig Package.Each instance of this class + * will cache information related to one package. To read flags from a different package, a new + * instance of this class should be {@link #load loaded}. + * + * @hide + */ +public class PlatformAconfigPackage { + private static final String TAG = "PlatformAconfigPackage"; + private static final String MAP_PATH = "/metadata/aconfig/maps/"; + private static final String BOOT_PATH = "/metadata/aconfig/boot/"; + + private FlagTable mFlagTable; + private FlagValueList mFlagValueList; + + private int mPackageBooleanStartOffset = -1; + private int mPackageId = -1; + + private PlatformAconfigPackage() {} + + /** @hide */ + static final Map<String, StorageFilesBundle> sStorageFilesCache = new HashMap<>(); + + /** @hide */ + @UnsupportedAppUsage + public static final Set<String> PLATFORM_PACKAGE_MAP_FILES = + Set.of("system.package.map", "vendor.package.map", "product.package.map"); + + static { + for (String pf : PLATFORM_PACKAGE_MAP_FILES) { + try { + PackageTable pTable = PackageTable.fromBytes(mapStorageFile(MAP_PATH + pf)); + String container = pTable.getHeader().getContainer(); + FlagTable fTable = + FlagTable.fromBytes(mapStorageFile(MAP_PATH + container + ".flag.map")); + FlagValueList fValueList = + FlagValueList.fromBytes(mapStorageFile(BOOT_PATH + container + ".val")); + StorageFilesBundle files = new StorageFilesBundle(pTable, fTable, fValueList); + for (String packageName : pTable.getPackageList()) { + sStorageFilesCache.put(packageName, files); + } + } catch (Exception e) { + // pass + Log.w(TAG, e.toString()); + } + } + } + + /** + * Loads a platform Aconfig Package from Aconfig Storage. + * + * <p>This method attempts to load the specified platform Aconfig package. + * + * @param packageName The name of the Aconfig package to load. + * @return An instance of {@link PlatformAconfigPackage}, which may be empty if the package is + * not found in the container. Null if the package is not found in platform partitions. + * @throws AconfigStorageReadException if there is an error reading from Aconfig Storage, such + * as if the storage system is not found, or there is an error reading the storage file. The + * specific error code can be got using {@link AconfigStorageReadException#getErrorCode()}. + * @hide + */ + @UnsupportedAppUsage + public static PlatformAconfigPackage load(String packageName) { + try { + PlatformAconfigPackage aconfigPackage = new PlatformAconfigPackage(); + StorageFilesBundle files = sStorageFilesCache.get(packageName); + if (files == null) { + return null; + } + PackageTable.Node pNode = files.packageTable.get(packageName); + aconfigPackage.mFlagTable = files.flagTable; + aconfigPackage.mFlagValueList = files.flagValueList; + aconfigPackage.mPackageBooleanStartOffset = pNode.getBooleanStartIndex(); + aconfigPackage.mPackageId = pNode.getPackageId(); + return aconfigPackage; + } catch (AconfigStorageException e) { + throw new AconfigStorageReadException( + e.getErrorCode(), "Fail to create AconfigPackage", e); + } catch (Exception e) { + throw new AconfigStorageReadException( + AconfigStorageReadException.ERROR_GENERIC, + "Fail to create PlatformAconfigPackage", + e); + } + } + + /** + * Retrieves the value of a boolean flag. + * + * <p>This method retrieves the value of the specified flag. If the flag exists within the + * loaded Aconfig Package, its value is returned. Otherwise, the provided `defaultValue` is + * returned. + * + * @param flagName The name of the flag (excluding any package name prefix). + * @param defaultValue The value to return if the flag is not found. + * @return The boolean value of the flag, or `defaultValue` if the flag is not found. + * @hide + */ + @UnsupportedAppUsage + public boolean getBooleanFlagValue(String flagName, boolean defaultValue) { + FlagTable.Node fNode = mFlagTable.get(mPackageId, flagName); + if (fNode == null) { + return defaultValue; + } + return mFlagValueList.getBoolean(fNode.getFlagIndex() + mPackageBooleanStartOffset); + } + + // Map a storage file given file path + private static MappedByteBuffer mapStorageFile(String file) { + FileChannel channel = null; + try { + channel = FileChannel.open(Paths.get(file), StandardOpenOption.READ); + return channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); + } catch (Exception e) { + throw new AconfigStorageReadException( + AconfigStorageReadException.ERROR_CANNOT_READ_STORAGE_FILE, + "Fail to mmap storage", + e); + } finally { + quietlyDispose(channel); + } + } + + private static void quietlyDispose(Closeable closable) { + try { + if (closable != null) { + closable.close(); + } + } catch (Exception e) { + // no need to care, at least as of now + } + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackageInternal.java b/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackageInternal.java new file mode 100644 index 0000000000..da18fb9fe0 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/srcs/android/os/flagging/PlatformAconfigPackageInternal.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os.flagging; + +import static android.aconfig.storage.TableUtils.StorageFilesBundle; + +import android.aconfig.storage.AconfigStorageException; +import android.aconfig.storage.FlagValueList; +import android.aconfig.storage.PackageTable; +import android.compat.annotation.UnsupportedAppUsage; + +/** + * An {@code aconfig} package containing the enabled state of its flags. + * + * <p><strong>Note: this is intended only to be used by generated code. To determine if a given flag + * is enabled in app code, the generated android flags should be used.</strong> + * + * <p>This class is not part of the public API and should be used by Acnofig Flag internally </b> It + * is intended for internal use only and will be changed or removed without notice. + * + * <p>This class is used to read the flag from Aconfig Package.Each instance of this class will + * cache information related to one package. To read flags from a different package, a new instance + * of this class should be {@link #load loaded}. + * + * @hide + */ +public class PlatformAconfigPackageInternal { + + private final FlagValueList mFlagValueList; + private final int mPackageBooleanStartOffset; + + private PlatformAconfigPackageInternal( + FlagValueList flagValueList, int packageBooleanStartOffset) { + this.mFlagValueList = flagValueList; + this.mPackageBooleanStartOffset = packageBooleanStartOffset; + } + + /** + * Loads an Aconfig package from the specified container and verifies its fingerprint. + * + * <p>This method is intended for internal use only and may be changed or removed without + * notice. + * + * @param packageName The name of the Aconfig package. + * @param packageFingerprint The expected fingerprint of the package. + * @return An instance of {@link PlatformAconfigPackageInternal} representing the loaded + * package. + * @hide + */ + @UnsupportedAppUsage + public static PlatformAconfigPackageInternal load(String packageName, long packageFingerprint) { + StorageFilesBundle files = PlatformAconfigPackage.sStorageFilesCache.get(packageName); + if (files == null) { + throw new AconfigStorageException( + AconfigStorageException.ERROR_PACKAGE_NOT_FOUND, + "package " + packageName + " cannot be found on the device"); + } + PackageTable.Node pNode = files.packageTable.get(packageName); + FlagValueList vList = files.flagValueList; + + if (pNode.hasPackageFingerprint() && packageFingerprint != pNode.getPackageFingerprint()) { + throw new AconfigStorageException( + AconfigStorageException.ERROR_FILE_FINGERPRINT_MISMATCH, + "package " + packageName + "fingerprint doesn't match the one on device"); + } + + return new PlatformAconfigPackageInternal(vList, pNode.getBooleanStartIndex()); + } + + /** + * Retrieves the value of a boolean flag using its index. + * + * <p>This method is intended for internal use only and may be changed or removed without + * notice. + * + * <p>This method retrieves the value of a flag within the loaded Aconfig package using its + * index. The index is generated at build time and may vary between builds. + * + * @param index The index of the flag within the package. + * @return The boolean value of the flag. + * @hide + */ + @UnsupportedAppUsage + public boolean getBooleanFlagValue(int index) { + return mFlagValueList.getBoolean(index + mPackageBooleanStartOffset); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/AndroidTest.xml b/tools/aconfig/aconfig_storage_read_api/tests/AconfigStorageReadFunctionalTest.xml index 99c9e2566e..ee50060208 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/java/AndroidTest.xml +++ b/tools/aconfig/aconfig_storage_read_api/tests/AconfigStorageReadFunctionalTest.xml @@ -26,7 +26,7 @@ </target_preparer> <target_preparer class="com.android.tradefed.targetprep.TestAppInstallSetup"> - <option name="test-file-name" value="aconfig_storage_read_api.test.java.apk" /> + <option name="test-file-name" value="aconfig_storage_read_functional.apk" /> </target_preparer> <target_preparer class="com.android.tradefed.targetprep.DisableSELinuxTargetPreparer" /> @@ -35,17 +35,17 @@ <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer"> <option name="cleanup" value="true" /> <option name="abort-on-push-failure" value="true" /> - <option name="push-file" key="package.map" + <option name="push-file" key="package_v1.map" value="/data/local/tmp/aconfig_java_api_test/maps/mockup.package.map" /> - <option name="push-file" key="flag.map" + <option name="push-file" key="flag_v1.map" value="/data/local/tmp/aconfig_java_api_test/maps/mockup.flag.map" /> - <option name="push-file" key="flag.val" + <option name="push-file" key="flag_v1.val" value="/data/local/tmp/aconfig_java_api_test/boot/mockup.val" /> </target_preparer> <test class="com.android.tradefed.testtype.AndroidJUnitTest" > <option name="runner" value="androidx.test.runner.AndroidJUnitRunner" /> - <option name="package" value="android.aconfig_storage.test" /> + <option name="package" value="android.aconfig.storage.test" /> <option name="runtime-hint" value="1m" /> </test> -</configuration> +</configuration>
\ No newline at end of file diff --git a/tools/aconfig/aconfig_storage_read_api/tests/Android.bp b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp index ed0c728215..c071f7cd88 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/Android.bp +++ b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp @@ -1,9 +1,14 @@ filegroup { name: "read_api_test_storage_files", - srcs: ["package.map", - "flag.map", - "flag.val", - "flag.info" + srcs: [ + "data/v1/package_v1.map", + "data/v1/flag_v1.map", + "data/v1/flag_v1.val", + "data/v1/flag_v1.info", + "data/v2/package_v2.map", + "data/v2/flag_v2.map", + "data/v2/flag_v2.val", + "data/v2/flag_v2.info", ], } @@ -43,3 +48,30 @@ cc_test { "general-tests", ], } + +android_test { + name: "aconfig_storage_read_functional", + srcs: [ + "functional/srcs/**/*.java", + ], + static_libs: [ + "aconfig_device_paths_java_util", + "aconfig_storage_file_java", + "androidx.test.rules", + "libaconfig_storage_read_api_java", + "junit", + ], + jni_libs: [ + "libaconfig_storage_read_api_rust_jni", + ], + data: [ + ":read_api_test_storage_files", + ], + platform_apis: true, + certificate: "platform", + test_suites: [ + "general-tests", + ], + test_config: "AconfigStorageReadFunctionalTest.xml", + team: "trendy_team_android_core_experiments", +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/AndroidManifest.xml b/tools/aconfig/aconfig_storage_read_api/tests/AndroidManifest.xml new file mode 100644 index 0000000000..5e01879157 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/AndroidManifest.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + ~ Copyright (C) 2024 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="android.aconfig.storage.test"> + <application> + <uses-library android:name="android.test.runner" /> + </application> + + <instrumentation android:name="androidx.test.runner.AndroidJUnitRunner" + android:targetPackage="android.aconfig.storage.test" /> + +</manifest> diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.info b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.info Binary files differindex 6223edf369..6223edf369 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/flag.info +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.info diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.map Binary files differindex e868f53d7e..e868f53d7e 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/flag.map +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.val b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.val Binary files differindex ed203d4d13..ed203d4d13 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/flag.val +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag_v1.val diff --git a/tools/aconfig/aconfig_storage_read_api/tests/package.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/package_v1.map Binary files differindex 6c46a0339c..6c46a0339c 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/package.map +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/package_v1.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.info b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.info Binary files differnew file mode 100644 index 0000000000..9db7fde7ae --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.info diff --git a/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.map Binary files differnew file mode 100644 index 0000000000..cf4685ceb4 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.val b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.val Binary files differnew file mode 100644 index 0000000000..37d4750206 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/flag_v2.val diff --git a/tools/aconfig/aconfig_storage_read_api/tests/data/v2/package_v2.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/package_v2.map Binary files differnew file mode 100644 index 0000000000..0a9f95ec85 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/data/v2/package_v2.map diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/AconfigStorageReadAPITest.java index 191741ef51..0587e9d4c5 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java +++ b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/AconfigStorageReadAPITest.java @@ -19,14 +19,13 @@ package android.aconfig.storage.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import android.aconfig.DeviceProtos; +import android.aconfig.DeviceProtosTestUtil; import android.aconfig.nano.Aconfig.parsed_flag; import android.aconfig.storage.AconfigStorageReadAPI; import android.aconfig.storage.FlagReadContext; import android.aconfig.storage.FlagReadContext.StoredFlagType; import android.aconfig.storage.PackageReadContext; import android.aconfig.storage.SipHasher13; -import android.aconfig.storage.StorageInternalReader; import org.junit.Test; import org.junit.runner.RunWith; @@ -211,7 +210,7 @@ public class AconfigStorageReadAPITest { @Test public void testRustJavaEqualHash() throws IOException { - List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos(); + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); for (parsed_flag flag : flags) { String packageName = flag.package_; String flagName = flag.name; @@ -225,46 +224,4 @@ public class AconfigStorageReadAPITest { assertEquals(rHash, jHash); } } - - @Test - public void testRustJavaEqualFlag() throws IOException { - List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos(); - - String mapPath = "/metadata/aconfig/maps/"; - String flagsPath = "/metadata/aconfig/boot/"; - - for (parsed_flag flag : flags) { - - String container = flag.container; - String packageName = flag.package_; - String flagName = flag.name; - String fullFlagName = packageName + "/" + flagName; - - MappedByteBuffer packageMap = - AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".package.map"); - MappedByteBuffer flagMap = - AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".flag.map"); - MappedByteBuffer flagValList = - AconfigStorageReadAPI.mapStorageFile(flagsPath + container + ".val"); - - PackageReadContext packageContext = - AconfigStorageReadAPI.getPackageReadContext(packageMap, packageName); - - FlagReadContext flagContext = - AconfigStorageReadAPI.getFlagReadContext( - flagMap, packageContext.mPackageId, flagName); - - boolean rVal = - AconfigStorageReadAPI.getBooleanFlagValue( - flagValList, - packageContext.mBooleanStartIndex + flagContext.mFlagIndex); - - StorageInternalReader reader = new StorageInternalReader(container, packageName); - boolean jVal = reader.getBooleanFlagValue(flagContext.mFlagIndex); - - long rHash = AconfigStorageReadAPI.hash(packageName); - long jHash = SipHasher13.hash(packageName.getBytes()); - assertEquals(rVal, jVal); - } - } } diff --git a/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageInternalTest.java b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageInternalTest.java new file mode 100644 index 0000000000..9896baff87 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageInternalTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.aconfig.storage.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import android.aconfig.DeviceProtosTestUtil; +import android.aconfig.nano.Aconfig; +import android.aconfig.nano.Aconfig.parsed_flag; +import android.aconfig.storage.FlagTable; +import android.aconfig.storage.FlagValueList; +import android.aconfig.storage.PackageTable; +import android.aconfig.storage.StorageFileProvider; +import android.internal.aconfig.storage.AconfigStorageException; +import android.os.flagging.PlatformAconfigPackageInternal; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@RunWith(JUnit4.class) +public class PlatformAconfigPackageInternalTest { + + private static final Set<String> PLATFORM_CONTAINERS = Set.of("system", "vendor", "product"); + + @Test + public void testPlatformAconfigPackageInternal_load() throws IOException { + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); + Map<String, PlatformAconfigPackageInternal> readerMap = new HashMap<>(); + StorageFileProvider fp = StorageFileProvider.getDefaultProvider(); + + for (parsed_flag flag : flags) { + if (flag.permission == Aconfig.READ_ONLY && flag.state == Aconfig.DISABLED) { + continue; + } + String container = flag.container; + String packageName = flag.package_; + String flagName = flag.name; + if (!PLATFORM_CONTAINERS.contains(container)) continue; + + PackageTable pTable = fp.getPackageTable(container); + PackageTable.Node pNode = pTable.get(packageName); + FlagTable fTable = fp.getFlagTable(container); + FlagTable.Node fNode = fTable.get(pNode.getPackageId(), flagName); + FlagValueList fList = fp.getFlagValueList(container); + + int index = pNode.getBooleanStartIndex() + fNode.getFlagIndex(); + boolean rVal = fList.getBoolean(index); + + long fingerprint = pNode.getPackageFingerprint(); + + PlatformAconfigPackageInternal reader = readerMap.get(packageName); + if (reader == null) { + reader = PlatformAconfigPackageInternal.load(packageName, fingerprint); + readerMap.put(packageName, reader); + } + boolean jVal = reader.getBooleanFlagValue(fNode.getFlagIndex()); + + assertEquals(rVal, jVal); + } + } + + @Test + public void testPlatformAconfigPackage_load_withError() throws IOException { + // package not found + AconfigStorageException e = + assertThrows( + AconfigStorageException.class, + () -> PlatformAconfigPackageInternal.load("fake_package", 0)); + assertEquals(AconfigStorageException.ERROR_PACKAGE_NOT_FOUND, e.getErrorCode()); + + // fingerprint doesn't match + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); + StorageFileProvider fp = StorageFileProvider.getDefaultProvider(); + + parsed_flag flag = flags.get(0); + + String container = flag.container; + String packageName = flag.package_; + boolean value = flag.state == Aconfig.ENABLED; + + PackageTable pTable = fp.getPackageTable(container); + PackageTable.Node pNode = pTable.get(packageName); + + if (pNode.hasPackageFingerprint()) { + long fingerprint = pNode.getPackageFingerprint(); + e = + assertThrows( + AconfigStorageException.class, + () -> + PlatformAconfigPackageInternal.load( + packageName, fingerprint + 1)); + assertEquals(AconfigStorageException.ERROR_FILE_FINGERPRINT_MISMATCH, e.getErrorCode()); + } + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageTest.java b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageTest.java new file mode 100644 index 0000000000..1c6c238542 --- /dev/null +++ b/tools/aconfig/aconfig_storage_read_api/tests/functional/srcs/PlatformAconfigPackageTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.aconfig.storage.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import android.aconfig.DeviceProtosTestUtil; +import android.aconfig.nano.Aconfig; +import android.aconfig.nano.Aconfig.parsed_flag; +import android.aconfig.storage.FlagTable; +import android.aconfig.storage.FlagValueList; +import android.aconfig.storage.PackageTable; +import android.aconfig.storage.StorageFileProvider; +import android.os.flagging.PlatformAconfigPackage; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@RunWith(JUnit4.class) +public class PlatformAconfigPackageTest { + + private static final Set<String> PLATFORM_CONTAINERS = Set.of("system", "vendor", "product"); + + @Test + public void testPlatformAconfigPackage_StorageFilesCache() throws IOException { + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); + for (parsed_flag flag : flags) { + if (flag.permission == Aconfig.READ_ONLY && flag.state == Aconfig.DISABLED) { + continue; + } + String container = flag.container; + String packageName = flag.package_; + if (!PLATFORM_CONTAINERS.contains(container)) continue; + assertNotNull(PlatformAconfigPackage.load(packageName)); + } + } + + @Test + public void testPlatformAconfigPackage_load() throws IOException { + List<parsed_flag> flags = DeviceProtosTestUtil.loadAndParseFlagProtos(); + Map<String, PlatformAconfigPackage> readerMap = new HashMap<>(); + StorageFileProvider fp = StorageFileProvider.getDefaultProvider(); + + for (parsed_flag flag : flags) { + if (flag.permission == Aconfig.READ_ONLY && flag.state == Aconfig.DISABLED) { + continue; + } + String container = flag.container; + String packageName = flag.package_; + String flagName = flag.name; + if (!PLATFORM_CONTAINERS.contains(container)) continue; + + PackageTable pTable = fp.getPackageTable(container); + PackageTable.Node pNode = pTable.get(packageName); + FlagTable fTable = fp.getFlagTable(container); + FlagTable.Node fNode = fTable.get(pNode.getPackageId(), flagName); + FlagValueList fList = fp.getFlagValueList(container); + + int index = pNode.getBooleanStartIndex() + fNode.getFlagIndex(); + boolean rVal = fList.getBoolean(index); + + long fingerprint = pNode.getPackageFingerprint(); + + PlatformAconfigPackage reader = readerMap.get(packageName); + if (reader == null) { + reader = PlatformAconfigPackage.load(packageName); + readerMap.put(packageName, reader); + } + boolean jVal = reader.getBooleanFlagValue(flagName, !rVal); + + assertEquals(rVal, jVal); + } + } + + @Test + public void testPlatformAconfigPackage_load_withError() throws IOException { + // package not found + assertNull(PlatformAconfigPackage.load("fake_container")); + } +} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp b/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp deleted file mode 100644 index 3d4e9ad218..0000000000 --- a/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp +++ /dev/null @@ -1,24 +0,0 @@ -android_test { - name: "aconfig_storage_read_api.test.java", - srcs: ["./**/*.java"], - static_libs: [ - "aconfig_device_paths_java", - "aconfig_storage_file_java", - "aconfig_storage_reader_java", - "androidx.test.rules", - "libaconfig_storage_read_api_java", - "junit", - ], - jni_libs: [ - "libaconfig_storage_read_api_rust_jni", - ], - data: [ - ":read_api_test_storage_files", - ], - platform_apis: true, - certificate: "platform", - test_suites: [ - "general-tests", - ], - team: "trendy_team_android_core_experiments", -} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/StorageInternalReaderTest.java b/tools/aconfig/aconfig_storage_read_api/tests/java/StorageInternalReaderTest.java deleted file mode 100644 index 3a1bba0fad..0000000000 --- a/tools/aconfig/aconfig_storage_read_api/tests/java/StorageInternalReaderTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2024 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.aconfig.storage.test; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import android.aconfig.storage.StorageInternalReader; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class StorageInternalReaderTest { - - private String mStorageDir = "/data/local/tmp/aconfig_java_api_test"; - - @Test - public void testStorageInternalReader_getFlag() { - - String packageMapFile = mStorageDir + "/maps/mockup.package.map"; - String flagValueFile = mStorageDir + "/boot/mockup.val"; - - StorageInternalReader reader = - new StorageInternalReader( - "com.android.aconfig.storage.test_1", packageMapFile, flagValueFile); - assertFalse(reader.getBooleanFlagValue(0)); - assertTrue(reader.getBooleanFlagValue(1)); - } -} diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp index 6d29045efe..5289faa6de 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp +++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp @@ -45,7 +45,8 @@ class AconfigStorageTest : public ::testing::Test { } void SetUp() override { - auto const test_dir = android::base::GetExecutableDirectory(); + auto const test_base_dir = android::base::GetExecutableDirectory(); + auto const test_dir = test_base_dir + "/data/v1"; storage_dir = std::string(root_dir.path); auto maps_dir = storage_dir + "/maps"; auto boot_dir = storage_dir + "/boot"; @@ -55,10 +56,10 @@ class AconfigStorageTest : public ::testing::Test { flag_map = std::string(maps_dir) + "/mockup.flag.map"; flag_val = std::string(boot_dir) + "/mockup.val"; flag_info = std::string(boot_dir) + "/mockup.info"; - copy_file(test_dir + "/package.map", package_map); - copy_file(test_dir + "/flag.map", flag_map); - copy_file(test_dir + "/flag.val", flag_val); - copy_file(test_dir + "/flag.info", flag_info); + copy_file(test_dir + "/package_v1.map", package_map); + copy_file(test_dir + "/flag_v1.map", flag_map); + copy_file(test_dir + "/flag_v1.val", flag_val); + copy_file(test_dir + "/flag_v1.info", flag_info); } void TearDown() override { diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs index afc44d4d70..2a8edf3302 100644 --- a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs +++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs @@ -9,7 +9,7 @@ mod aconfig_storage_rust_test { use rand::Rng; use std::fs; - fn create_test_storage_files() -> String { + fn create_test_storage_files(version: u32) -> String { let mut rng = rand::thread_rng(); let number: u32 = rng.gen(); let storage_dir = String::from("/tmp/") + &number.to_string(); @@ -26,17 +26,17 @@ mod aconfig_storage_rust_test { let flag_map = storage_dir.clone() + "/maps/mockup.flag.map"; let flag_val = storage_dir.clone() + "/boot/mockup.val"; let flag_info = storage_dir.clone() + "/boot/mockup.info"; - fs::copy("./package.map", package_map).unwrap(); - fs::copy("./flag.map", flag_map).unwrap(); - fs::copy("./flag.val", flag_val).unwrap(); - fs::copy("./flag.info", flag_info).unwrap(); + fs::copy(format!("./data/v{0}/package_v{0}.map", version), package_map).unwrap(); + fs::copy(format!("./data/v{0}/flag_v{0}.map", version), flag_map).unwrap(); + fs::copy(format!("./data/v{}/flag_v{0}.val", version), flag_val).unwrap(); + fs::copy(format!("./data/v{}/flag_v{0}.info", version), flag_info).unwrap(); storage_dir } #[test] - fn test_unavailable_stoarge() { - let storage_dir = create_test_storage_files(); + fn test_unavailable_storage() { + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let err = unsafe { @@ -53,7 +53,7 @@ mod aconfig_storage_rust_test { #[test] fn test_package_context_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let package_mapped_file = unsafe { @@ -64,27 +64,73 @@ mod aconfig_storage_rust_test { get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_1") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 0, boolean_start_index: 0 }; + let expected_package_context = + PackageReadContext { package_id: 0, boolean_start_index: 0, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_2") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 1, boolean_start_index: 3 }; + let expected_package_context = + PackageReadContext { package_id: 1, boolean_start_index: 3, fingerprint: 0 }; assert_eq!(package_context, expected_package_context); let package_context = get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_4") .unwrap() .unwrap(); - let expected_package_context = PackageReadContext { package_id: 2, boolean_start_index: 6 }; + let expected_package_context = + PackageReadContext { package_id: 2, boolean_start_index: 6, fingerprint: 0 }; + assert_eq!(package_context, expected_package_context); + } + + #[test] + fn test_package_context_query_with_fingerprint() { + let storage_dir = create_test_storage_files(2); + // SAFETY: + // The safety here is ensured as the test process will not write to temp storage file + let package_mapped_file = unsafe { + get_mapped_file(&storage_dir, "mockup", StorageFileType::PackageMap).unwrap() + }; + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_1") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 0, + boolean_start_index: 0, + fingerprint: 15248948510590158086u64, + }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_2") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 1, + boolean_start_index: 3, + fingerprint: 4431940502274857964u64, + }; + assert_eq!(package_context, expected_package_context); + + let package_context = + get_package_read_context(&package_mapped_file, "com.android.aconfig.storage.test_4") + .unwrap() + .unwrap(); + let expected_package_context = PackageReadContext { + package_id: 2, + boolean_start_index: 6, + fingerprint: 16233229917711622375u64, + }; assert_eq!(package_context, expected_package_context); } #[test] fn test_none_exist_package_context_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let package_mapped_file = unsafe { @@ -99,7 +145,7 @@ mod aconfig_storage_rust_test { #[test] fn test_flag_context_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_mapped_file = @@ -125,7 +171,7 @@ mod aconfig_storage_rust_test { #[test] fn test_none_exist_flag_context_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_mapped_file = @@ -141,7 +187,7 @@ mod aconfig_storage_rust_test { #[test] fn test_boolean_flag_value_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_value_file = @@ -155,7 +201,7 @@ mod aconfig_storage_rust_test { #[test] fn test_invalid_boolean_flag_value_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_value_file = @@ -169,7 +215,7 @@ mod aconfig_storage_rust_test { #[test] fn test_flag_info_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_info_file = @@ -186,7 +232,7 @@ mod aconfig_storage_rust_test { #[test] fn test_invalid_boolean_flag_info_query() { - let storage_dir = create_test_storage_files(); + let storage_dir = create_test_storage_files(1); // SAFETY: // The safety here is ensured as the test process will not write to temp storage file let flag_info_file = @@ -199,10 +245,18 @@ mod aconfig_storage_rust_test { } #[test] - fn test_storage_version_query() { - assert_eq!(get_storage_file_version("./package.map").unwrap(), 1); - assert_eq!(get_storage_file_version("./flag.map").unwrap(), 1); - assert_eq!(get_storage_file_version("./flag.val").unwrap(), 1); - assert_eq!(get_storage_file_version("./flag.info").unwrap(), 1); + fn test_storage_version_query_v1() { + assert_eq!(get_storage_file_version("./data/v1/package_v1.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./data/v1/flag_v1.map").unwrap(), 1); + assert_eq!(get_storage_file_version("./data/v1/flag_v1.val").unwrap(), 1); + assert_eq!(get_storage_file_version("./data/v1/flag_v1.info").unwrap(), 1); + } + + #[test] + fn test_storage_version_query_v2() { + assert_eq!(get_storage_file_version("./data/v2/package_v2.map").unwrap(), 2); + assert_eq!(get_storage_file_version("./data/v2/flag_v2.map").unwrap(), 2); + assert_eq!(get_storage_file_version("./data/v2/flag_v2.val").unwrap(), 2); + assert_eq!(get_storage_file_version("./data/v2/flag_v2.info").unwrap(), 2); } } diff --git a/tools/aconfig/aconfig_storage_write_api/Android.bp b/tools/aconfig/aconfig_storage_write_api/Android.bp index 0f1962c3ac..4c882b4b9a 100644 --- a/tools/aconfig/aconfig_storage_write_api/Android.bp +++ b/tools/aconfig/aconfig_storage_write_api/Android.bp @@ -16,6 +16,11 @@ rust_defaults { "libaconfig_storage_file", "libaconfig_storage_read_api", ], + min_sdk_version: "34", + apex_available: [ + "//apex_available:anyapex", + "//apex_available:platform", + ], } rust_library { diff --git a/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp b/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp index 7b435746da..03a8fa284a 100644 --- a/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp +++ b/tools/aconfig/aconfig_storage_write_api/aconfig_storage_write_api.cpp @@ -100,18 +100,4 @@ android::base::Result<void> set_flag_has_local_override( return {}; } -android::base::Result<void> create_flag_info( - std::string const& package_map, - std::string const& flag_map, - std::string const& flag_info_out) { - auto creation_cxx = create_flag_info_cxx( - rust::Str(package_map.c_str()), - rust::Str(flag_map.c_str()), - rust::Str(flag_info_out.c_str())); - if (creation_cxx.success) { - return {}; - } else { - return android::base::Error() << creation_cxx.error_message; - } -} } // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp b/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp index 0bba7ffcfc..50a51889b1 100644 --- a/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp +++ b/tools/aconfig/aconfig_storage_write_api/include/aconfig_storage/aconfig_storage_write_api.hpp @@ -36,13 +36,4 @@ android::base::Result<void> set_flag_has_local_override( uint32_t offset, bool value); -/// Create flag info file based on package and flag map -/// \input package_map: package map file -/// \input flag_map: flag map file -/// \input flag_info_out: flag info file to be created -android::base::Result<void> create_flag_info( - std::string const& package_map, - std::string const& flag_map, - std::string const& flag_info_out); - } // namespace aconfig_storage diff --git a/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs b/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs index 7e6071340c..5721105d86 100644 --- a/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs +++ b/tools/aconfig/aconfig_storage_write_api/src/flag_info_update.rs @@ -18,7 +18,7 @@ use aconfig_storage_file::{ read_u8_from_bytes, AconfigStorageError, FlagInfoBit, FlagInfoHeader, FlagValueType, - FILE_VERSION, + MAX_SUPPORTED_FILE_VERSION, }; use anyhow::anyhow; @@ -28,11 +28,11 @@ fn get_flag_info_offset( flag_index: u32, ) -> Result<usize, AconfigStorageError> { let interpreted_header = FlagInfoHeader::from_bytes(buf)?; - if interpreted_header.version > FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot write to storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -94,13 +94,13 @@ pub fn update_flag_has_local_override( #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::test_utils::create_test_flag_info_list; + use aconfig_storage_file::{test_utils::create_test_flag_info_list, DEFAULT_FILE_VERSION}; use aconfig_storage_read_api::flag_info_query::find_flag_attribute; #[test] // this test point locks down has server override update fn test_update_flag_has_server_override() { - let flag_info_list = create_test_flag_info_list(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); let mut buf = flag_info_list.into_bytes(); for i in 0..flag_info_list.header.num_flags { update_flag_has_server_override(&mut buf, FlagValueType::Boolean, i, true).unwrap(); @@ -115,7 +115,7 @@ mod tests { #[test] // this test point locks down has local override update fn test_update_flag_has_local_override() { - let flag_info_list = create_test_flag_info_list(); + let flag_info_list = create_test_flag_info_list(DEFAULT_FILE_VERSION); let mut buf = flag_info_list.into_bytes(); for i in 0..flag_info_list.header.num_flags { update_flag_has_local_override(&mut buf, FlagValueType::Boolean, i, true).unwrap(); diff --git a/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs b/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs index dd15c996a6..9772db9ee8 100644 --- a/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs +++ b/tools/aconfig/aconfig_storage_write_api/src/flag_value_update.rs @@ -16,7 +16,7 @@ //! flag value update module defines the flag value file write to mapped bytes -use aconfig_storage_file::{AconfigStorageError, FlagValueHeader, FILE_VERSION}; +use aconfig_storage_file::{AconfigStorageError, FlagValueHeader, MAX_SUPPORTED_FILE_VERSION}; use anyhow::anyhow; /// Set flag value @@ -26,11 +26,11 @@ pub fn update_boolean_flag_value( flag_value: bool, ) -> Result<usize, AconfigStorageError> { let interpreted_header = FlagValueHeader::from_bytes(buf)?; - if interpreted_header.version > FILE_VERSION { + if interpreted_header.version > MAX_SUPPORTED_FILE_VERSION { return Err(AconfigStorageError::HigherStorageFileVersion(anyhow!( "Cannot write to storage file with a higher version of {} with lib version {}", interpreted_header.version, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION ))); } @@ -49,12 +49,12 @@ pub fn update_boolean_flag_value( #[cfg(test)] mod tests { use super::*; - use aconfig_storage_file::test_utils::create_test_flag_value_list; + use aconfig_storage_file::{test_utils::create_test_flag_value_list, DEFAULT_FILE_VERSION}; #[test] // this test point locks down flag value update fn test_boolean_flag_value_update() { - let flag_value_list = create_test_flag_value_list(); + let flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); let value_offset = flag_value_list.header.boolean_value_offset; let mut content = flag_value_list.into_bytes(); let true_byte = u8::from(true).to_le_bytes()[0]; @@ -72,7 +72,7 @@ mod tests { #[test] // this test point locks down update beyond the end of boolean section fn test_boolean_out_of_range() { - let mut flag_value_list = create_test_flag_value_list().into_bytes(); + let mut flag_value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(); let error = update_boolean_flag_value(&mut flag_value_list[..], 8, true).unwrap_err(); assert_eq!( format!("{:?}", error), @@ -83,16 +83,16 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut value_list = create_test_flag_value_list(); - value_list.header.version = FILE_VERSION + 1; + let mut value_list = create_test_flag_value_list(DEFAULT_FILE_VERSION); + value_list.header.version = MAX_SUPPORTED_FILE_VERSION + 1; let mut flag_value = value_list.into_bytes(); let error = update_boolean_flag_value(&mut flag_value[..], 4, true).unwrap_err(); assert_eq!( format!("{:?}", error), format!( "HigherStorageFileVersion(Cannot write to storage file with a higher version of {} with lib version {})", - FILE_VERSION + 1, - FILE_VERSION + MAX_SUPPORTED_FILE_VERSION + 1, + MAX_SUPPORTED_FILE_VERSION ) ); } diff --git a/tools/aconfig/aconfig_storage_write_api/src/lib.rs b/tools/aconfig/aconfig_storage_write_api/src/lib.rs index 0396a63d4e..09bb41f54f 100644 --- a/tools/aconfig/aconfig_storage_write_api/src/lib.rs +++ b/tools/aconfig/aconfig_storage_write_api/src/lib.rs @@ -24,15 +24,10 @@ pub mod mapped_file; #[cfg(test)] mod test_utils; -use aconfig_storage_file::{ - AconfigStorageError, FlagInfoHeader, FlagInfoList, FlagInfoNode, FlagTable, FlagValueType, - PackageTable, StorageFileType, StoredFlagType, FILE_VERSION, -}; +use aconfig_storage_file::{AconfigStorageError, FlagValueType}; use anyhow::anyhow; use memmap2::MmapMut; -use std::fs::File; -use std::io::{Read, Write}; /// Get read write mapped storage files. /// @@ -104,86 +99,6 @@ pub fn set_flag_has_local_override( }) } -/// Read in storage file as bytes -fn read_file_to_bytes(file_path: &str) -> Result<Vec<u8>, AconfigStorageError> { - let mut file = File::open(file_path).map_err(|errmsg| { - AconfigStorageError::FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)) - })?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer).map_err(|errmsg| { - AconfigStorageError::FileReadFail(anyhow!( - "Failed to read bytes from file {}: {}", - file_path, - errmsg - )) - })?; - Ok(buffer) -} - -/// Create flag info file given package map file and flag map file -/// \input package_map: package map file -/// \input flag_map: flag map file -/// \output flag_info_out: created flag info file -pub fn create_flag_info( - package_map: &str, - flag_map: &str, - flag_info_out: &str, -) -> Result<(), AconfigStorageError> { - let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?; - let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?; - - if package_table.header.container != flag_table.header.container { - return Err(AconfigStorageError::FileCreationFail(anyhow!( - "container for package map {} and flag map {} does not match", - package_table.header.container, - flag_table.header.container, - ))); - } - - let mut package_start_index = vec![0; package_table.header.num_packages as usize]; - for node in package_table.nodes.iter() { - package_start_index[node.package_id as usize] = node.boolean_start_index; - } - - let mut is_flag_rw = vec![false; flag_table.header.num_flags as usize]; - for node in flag_table.nodes.iter() { - let flag_index = package_start_index[node.package_id as usize] + node.flag_index as u32; - is_flag_rw[flag_index as usize] = node.flag_type == StoredFlagType::ReadWriteBoolean; - } - - let mut list = FlagInfoList { - header: FlagInfoHeader { - version: FILE_VERSION, - container: flag_table.header.container, - file_type: StorageFileType::FlagInfo as u8, - file_size: 0, - num_flags: flag_table.header.num_flags, - boolean_flag_offset: 0, - }, - nodes: is_flag_rw.iter().map(|&rw| FlagInfoNode::create(rw)).collect(), - }; - - list.header.boolean_flag_offset = list.header.into_bytes().len() as u32; - list.header.file_size = list.into_bytes().len() as u32; - - let mut file = File::create(flag_info_out).map_err(|errmsg| { - AconfigStorageError::FileCreationFail(anyhow!( - "fail to create file {}: {}", - flag_info_out, - errmsg - )) - })?; - file.write_all(&list.into_bytes()).map_err(|errmsg| { - AconfigStorageError::FileCreationFail(anyhow!( - "fail to write to file {}: {}", - flag_info_out, - errmsg - )) - })?; - - Ok(()) -} - // *************************************** // // CC INTERLOP // *************************************** // @@ -212,12 +127,6 @@ mod ffi { pub error_message: String, } - // Flag info file creation return for cc interlop - pub struct FlagInfoCreationCXX { - pub success: bool, - pub error_message: String, - } - // Rust export to c++ extern "Rust" { pub fn update_boolean_flag_value_cxx( @@ -239,12 +148,6 @@ mod ffi { offset: u32, value: bool, ) -> FlagHasLocalOverrideUpdateCXX; - - pub fn create_flag_info_cxx( - package_map: &str, - flag_map: &str, - flag_info_out: &str, - ) -> FlagInfoCreationCXX; } } @@ -329,34 +232,15 @@ pub(crate) fn update_flag_has_local_override_cxx( } } -/// Create flag info file cc interlop -pub(crate) fn create_flag_info_cxx( - package_map: &str, - flag_map: &str, - flag_info_out: &str, -) -> ffi::FlagInfoCreationCXX { - match create_flag_info(package_map, flag_map, flag_info_out) { - Ok(()) => ffi::FlagInfoCreationCXX { success: true, error_message: String::from("") }, - Err(errmsg) => { - ffi::FlagInfoCreationCXX { success: false, error_message: format!("{:?}", errmsg) } - } - } -} - #[cfg(test)] mod tests { use super::*; use crate::test_utils::copy_to_temp_file; - use aconfig_storage_file::test_utils::{ - create_test_flag_info_list, create_test_flag_table, create_test_package_table, - write_bytes_to_temp_file, - }; use aconfig_storage_file::FlagInfoBit; use aconfig_storage_read_api::flag_info_query::find_flag_attribute; use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value; use std::fs::File; use std::io::Read; - use tempfile::NamedTempFile; fn get_boolean_flag_value_at_offset(file: &str, offset: u32) -> bool { let mut f = File::open(&file).unwrap(); @@ -439,31 +323,4 @@ mod tests { } } } - - fn create_empty_temp_file() -> Result<NamedTempFile, AconfigStorageError> { - let file = NamedTempFile::new().map_err(|_| { - AconfigStorageError::FileCreationFail(anyhow!("Failed to create temp file")) - })?; - Ok(file) - } - - #[test] - // this test point locks down the flag info creation - fn test_create_flag_info() { - let package_table = - write_bytes_to_temp_file(&create_test_package_table().into_bytes()).unwrap(); - let flag_table = write_bytes_to_temp_file(&create_test_flag_table().into_bytes()).unwrap(); - let flag_info = create_empty_temp_file().unwrap(); - - let package_table_path = package_table.path().display().to_string(); - let flag_table_path = flag_table.path().display().to_string(); - let flag_info_path = flag_info.path().display().to_string(); - - assert!(create_flag_info(&package_table_path, &flag_table_path, &flag_info_path).is_ok()); - - let flag_info = - FlagInfoList::from_bytes(&read_file_to_bytes(&flag_info_path).unwrap()).unwrap(); - let expected_flag_info = create_test_flag_info_list(); - assert_eq!(flag_info, expected_flag_info); - } } diff --git a/tools/aconfig/aflags/Android.bp b/tools/aconfig/aflags/Android.bp index 2040cc635b..341975daa4 100644 --- a/tools/aconfig/aflags/Android.bp +++ b/tools/aconfig/aflags/Android.bp @@ -12,7 +12,7 @@ rust_defaults { "libaconfig_device_paths", "libaconfig_flags", "libaconfig_protos", - "libaconfigd_protos", + "libaconfigd_protos_rust", "libaconfig_storage_read_api", "libaconfig_storage_file", "libanyhow", @@ -20,6 +20,10 @@ rust_defaults { "libnix", "libprotobuf", "libregex", + // TODO: b/371021174 remove this fake dependency once we find a proper strategy to + // deal with test aconfig libs are not present in storage because they are never used + // by the actual build + "libaconfig_test_rust_library", ], } @@ -27,6 +31,9 @@ rust_binary { name: "aflags", host_supported: true, defaults: ["aflags.defaults"], + apex_available: [ + "//apex_available:platform", + ], } rust_test_host { diff --git a/tools/aconfig/aflags/Cargo.toml b/tools/aconfig/aflags/Cargo.toml index 7efce6dc96..d31e232975 100644 --- a/tools/aconfig/aflags/Cargo.toml +++ b/tools/aconfig/aflags/Cargo.toml @@ -9,10 +9,10 @@ paste = "1.0.11" protobuf = "3.2.0" regex = "1.10.3" aconfig_protos = { path = "../aconfig_protos" } -aconfigd_protos = { version = "0.1.0", path = "../../../../../system/server_configurable_flags/aconfigd"} +aconfigd_protos = { version = "0.1.0", path = "../../../../../packages/modules/ConfigInfrastructure/aconfigd/proto"} nix = { version = "0.28.0", features = ["user"] } aconfig_storage_file = { version = "0.1.0", path = "../aconfig_storage_file" } aconfig_storage_read_api = { version = "0.1.0", path = "../aconfig_storage_read_api" } clap = {version = "4.5.2" } aconfig_device_paths = { version = "0.1.0", path = "../aconfig_device_paths" } -aconfig_flags = { version = "0.1.0", path = "../aconfig_flags" }
\ No newline at end of file +aconfig_flags = { version = "0.1.0", path = "../aconfig_flags" } diff --git a/tools/aconfig/aflags/src/aconfig_storage_source.rs b/tools/aconfig/aflags/src/aconfig_storage_source.rs index b2fd3c92ac..766807acef 100644 --- a/tools/aconfig/aflags/src/aconfig_storage_source.rs +++ b/tools/aconfig/aflags/src/aconfig_storage_source.rs @@ -1,3 +1,4 @@ +use crate::load_protos; use crate::{Flag, FlagSource}; use crate::{FlagPermission, FlagValue, ValuePickedFrom}; use aconfigd_protos::{ @@ -9,13 +10,35 @@ use anyhow::anyhow; use anyhow::Result; use protobuf::Message; use protobuf::SpecialFields; +use std::collections::HashMap; use std::io::{Read, Write}; use std::net::Shutdown; use std::os::unix::net::UnixStream; pub struct AconfigStorageSource {} -fn convert(msg: ProtoFlagQueryReturnMessage) -> Result<Flag> { +static ACONFIGD_SYSTEM_SOCKET_NAME: &str = "/dev/socket/aconfigd_system"; +static ACONFIGD_MAINLINE_SOCKET_NAME: &str = "/dev/socket/aconfigd_mainline"; + +enum AconfigdSocket { + System, + Mainline, +} + +impl AconfigdSocket { + pub fn name(&self) -> &str { + match self { + AconfigdSocket::System => ACONFIGD_SYSTEM_SOCKET_NAME, + AconfigdSocket::Mainline => ACONFIGD_MAINLINE_SOCKET_NAME, + } + } +} + +fn load_flag_to_container() -> Result<HashMap<String, String>> { + Ok(load_protos::load()?.into_iter().map(|p| (p.qualified_name(), p.container)).collect()) +} + +fn convert(msg: ProtoFlagQueryReturnMessage, containers: &HashMap<String, String>) -> Result<Flag> { let (value, value_picked_from) = match ( &msg.boot_flag_value, msg.default_flag_value, @@ -55,21 +78,27 @@ fn convert(msg: ProtoFlagQueryReturnMessage) -> Result<Flag> { None => return Err(anyhow!("missing permission")), }; + let name = msg.flag_name.ok_or(anyhow!("missing flag name"))?; + let package = msg.package_name.ok_or(anyhow!("missing package name"))?; + let qualified_name = format!("{package}.{name}"); Ok(Flag { - name: msg.flag_name.ok_or(anyhow!("missing flag name"))?, - package: msg.package_name.ok_or(anyhow!("missing package name"))?, + name, + package, value, permission, value_picked_from, staged_value, - container: "-".to_string(), - + container: containers + .get(&qualified_name) + .cloned() + .unwrap_or_else(|| "<no container>".to_string()) + .to_string(), // TODO: remove once DeviceConfig is not in the CLI. namespace: "-".to_string(), }) } -fn read_from_socket() -> Result<Vec<ProtoFlagQueryReturnMessage>> { +fn read_from_socket(socket: AconfigdSocket) -> Result<Vec<ProtoFlagQueryReturnMessage>> { let messages = ProtoStorageRequestMessages { msgs: vec![ProtoStorageRequestMessage { msg: Some(ProtoStorageRequestMessageMsg::ListStorageMessage(ProtoListStorageMessage { @@ -81,7 +110,7 @@ fn read_from_socket() -> Result<Vec<ProtoFlagQueryReturnMessage>> { special_fields: SpecialFields::new(), }; - let mut socket = UnixStream::connect("/dev/socket/aconfigd")?; + let mut socket = UnixStream::connect(socket.name())?; let message_buffer = messages.write_to_bytes()?; let mut message_length_buffer: [u8; 4] = [0; 4]; @@ -114,11 +143,21 @@ fn read_from_socket() -> Result<Vec<ProtoFlagQueryReturnMessage>> { impl FlagSource for AconfigStorageSource { fn list_flags() -> Result<Vec<Flag>> { - read_from_socket() - .map(|query_messages| { - query_messages.iter().map(|message| convert(message.clone())).collect::<Vec<_>>() - })? + let containers = load_flag_to_container()?; + let system_messages = read_from_socket(AconfigdSocket::System); + let mainline_messages = read_from_socket(AconfigdSocket::Mainline); + + let mut all_messages = vec![]; + if let Ok(system_messages) = system_messages { + all_messages.extend_from_slice(&system_messages); + } + if let Ok(mainline_messages) = mainline_messages { + all_messages.extend_from_slice(&mainline_messages); + } + + all_messages .into_iter() + .map(|query_message| convert(query_message.clone(), &containers)) .collect() } diff --git a/tools/aconfig/aflags/src/load_protos.rs b/tools/aconfig/aflags/src/load_protos.rs index 90d8599145..c5ac8ff9dc 100644 --- a/tools/aconfig/aflags/src/load_protos.rs +++ b/tools/aconfig/aflags/src/load_protos.rs @@ -51,7 +51,10 @@ pub(crate) fn load() -> Result<Vec<Flag>> { let paths = aconfig_device_paths::parsed_flags_proto_paths()?; for path in paths { - let bytes = fs::read(path.clone())?; + let Ok(bytes) = fs::read(&path) else { + eprintln!("warning: failed to read {:?}", path); + continue; + }; let parsed_flags: ProtoParsedFlags = protobuf::Message::parse_from_bytes(&bytes)?; for flag in parsed_flags.parsed_flag { // TODO(b/334954748): enforce one-container-per-flag invariant. @@ -60,3 +63,10 @@ pub(crate) fn load() -> Result<Vec<Flag>> { } Ok(result) } + +pub(crate) fn list_containers() -> Result<Vec<String>> { + Ok(aconfig_device_paths::parsed_flags_proto_paths()? + .into_iter() + .map(|p| infer_container(&p)) + .collect()) +} diff --git a/tools/aconfig/aflags/src/main.rs b/tools/aconfig/aflags/src/main.rs index a726cc0369..568ad999e0 100644 --- a/tools/aconfig/aflags/src/main.rs +++ b/tools/aconfig/aflags/src/main.rs @@ -16,6 +16,9 @@ //! `aflags` is a device binary to read and write aconfig flags. +use std::env; +use std::process::{Command as OsCommand, Stdio}; + use anyhow::{anyhow, ensure, Result}; use clap::Parser; @@ -116,9 +119,10 @@ impl Flag { } fn display_staged_value(&self) -> String { - match self.staged_value { - Some(v) => format!("(->{})", v), - None => "-".to_string(), + match (&self.permission, self.staged_value) { + (FlagPermission::ReadOnly, _) => "-".to_string(), + (FlagPermission::ReadWrite, None) => "-".to_string(), + (FlagPermission::ReadWrite, Some(v)) => format!("(->{})", v), } } } @@ -252,6 +256,14 @@ fn list(source_type: FlagSourceType, container: Option<String>) -> Result<String FlagSourceType::DeviceConfig => DeviceConfigSource::list_flags()?, FlagSourceType::AconfigStorage => AconfigStorageSource::list_flags()?, }; + + if let Some(ref c) = container { + ensure!( + load_protos::list_containers()?.contains(c), + format!("container '{}' not found", &c) + ); + } + let flags = (Filter { container }).apply(&flags_unfiltered); let padding_info = PaddingInfo { longest_flag_col: flags.iter().map(|f| f.qualified_name().len()).max().unwrap_or(0), @@ -289,7 +301,37 @@ fn display_which_backing() -> String { } } +fn invoke_updatable_aflags() { + let updatable_command = "/apex/com.android.configinfrastructure/bin/aflags_updatable"; + + let args: Vec<String> = env::args().collect(); + let command_args = if args.len() >= 2 { &args[1..] } else { &["--help".to_string()] }; + + let mut child = OsCommand::new(updatable_command); + for arg in command_args { + child.arg(arg); + } + + let output = child + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("failed to execute child") + .wait_with_output() + .expect("failed to execute command"); + + let output_str = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !output_str.is_empty() { + println!("{}", output_str); + } +} + fn main() -> Result<()> { + if aconfig_flags::auto_generated::invoke_updatable_aflags() { + invoke_updatable_aflags(); + return Ok(()); + } + ensure!(nix::unistd::Uid::current().is_root(), "must be root"); let cli = Cli::parse(); @@ -297,7 +339,7 @@ fn main() -> Result<()> { Command::List { container } => { if aconfig_flags::auto_generated::enable_only_new_storage() { list(FlagSourceType::AconfigStorage, container) - .map_err(|err| anyhow!("storage may not be enabled: {err}")) + .map_err(|err| anyhow!("could not list flags: {err}")) .map(Some) } else { list(FlagSourceType::DeviceConfig, container).map(Some) diff --git a/tools/aconfig/convert_finalized_flags/Android.bp b/tools/aconfig/convert_finalized_flags/Android.bp new file mode 100644 index 0000000000..5b3956062f --- /dev/null +++ b/tools/aconfig/convert_finalized_flags/Android.bp @@ -0,0 +1,56 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "convert_finalized_flags.defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + rustlibs: [ + "libanyhow", + "libclap", + "libitertools", + "libprotobuf", + "libserde", + "libserde_json", + "libtempfile", + "libtinytemplate", + ], +} + +rust_library_host { + name: "libconvert_finalized_flags", + crate_name: "convert_finalized_flags", + defaults: ["convert_finalized_flags.defaults"], + srcs: [ + "src/lib.rs", + ], +} + +rust_binary_host { + name: "convert_finalized_flags", + defaults: ["convert_finalized_flags.defaults"], + srcs: ["src/main.rs"], + rustlibs: [ + "libconvert_finalized_flags", + "libserde_json", + ], +} + +rust_test_host { + name: "convert_finalized_flags.test", + defaults: ["convert_finalized_flags.defaults"], + test_suites: ["general-tests"], + srcs: ["src/lib.rs"], +} + +genrule { + name: "finalized_flags_record.json", + srcs: [ + "//prebuilts/sdk:finalized-api-flags", + ], + out: ["finalized_flags_record.json"], + tools: ["convert_finalized_flags"], + cmd: "args=\"\" && for f in $(locations //prebuilts/sdk:finalized-api-flags); do args=\"$$args --flag_file_path $$f\"; done && $(location convert_finalized_flags) $$args > $(out)", +} diff --git a/tools/aconfig/convert_finalized_flags/Cargo.toml b/tools/aconfig/convert_finalized_flags/Cargo.toml new file mode 100644 index 0000000000..e34e030841 --- /dev/null +++ b/tools/aconfig/convert_finalized_flags/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "convert_finalized_flags" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" +clap = { version = "4.1.8", features = ["derive"] } +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +tempfile = "3.13.0" diff --git a/tools/aconfig/convert_finalized_flags/src/lib.rs b/tools/aconfig/convert_finalized_flags/src/lib.rs new file mode 100644 index 0000000000..10faa39893 --- /dev/null +++ b/tools/aconfig/convert_finalized_flags/src/lib.rs @@ -0,0 +1,395 @@ +/* +* Copyright (C) 2025 The Android Open Source Project +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +//! Functions to extract finalized flag information from +//! /prebuilts/sdk/#/finalized-flags.txt. +//! These functions are very specific to that file setup as well as the format +//! of the files (just a list of the fully-qualified flag names). +//! There are also some helper functions for local building using cargo. These +//! functions are only invoked via cargo for quick local testing and will not +//! be used during actual soong building. They are marked as such. +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::io::{self, BufRead}; + +/// Just the fully qualified flag name (package_name.flag_name). +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct FinalizedFlag { + /// Name of the flag. + pub flag_name: String, + /// Name of the package. + pub package_name: String, +} + +/// API level in which the flag was finalized. +#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct ApiLevel(pub i32); + +/// Contains all flags finalized for a given API level. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct FinalizedFlagMap(HashMap<ApiLevel, HashSet<FinalizedFlag>>); + +impl FinalizedFlagMap { + /// Creates a new, empty instance. + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Convenience method for is_empty on the underlying map. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Returns the API level in which the flag was finalized . + pub fn get_finalized_level(&self, flag: &FinalizedFlag) -> Option<ApiLevel> { + for (api_level, flags_for_level) in &self.0 { + if flags_for_level.contains(flag) { + return Some(*api_level); + } + } + None + } + + /// Insert the flag into the map for the given level if the flag is not + /// present in the map already - for *any* level (not just the one given). + pub fn insert_if_new(&mut self, level: ApiLevel, flag: FinalizedFlag) { + if self.contains(&flag) { + return; + } + self.0.entry(level).or_default().insert(flag); + } + + fn contains(&self, flag: &FinalizedFlag) -> bool { + self.0.values().any(|flags_set| flags_set.contains(flag)) + } +} + +/// Converts a string to an int. Will parse to int even if the string is "X.0". +/// Returns error for "X.1". +fn str_to_api_level(numeric_string: &str) -> Result<ApiLevel> { + let float_value = numeric_string.parse::<f64>()?; + + if float_value.fract() == 0.0 { + Ok(ApiLevel(float_value as i32)) + } else { + Err(anyhow!("Numeric string is float, can't parse to int.")) + } +} + +/// For each file, extracts the qualified flag names into a FinalizedFlag, then +/// enters them in a map at the API level corresponding to their directory. +/// Ex: /prebuilts/sdk/35/finalized-flags.txt -> {36, [flag1, flag2]}. +pub fn read_files_to_map_using_path(flag_files: Vec<String>) -> Result<FinalizedFlagMap> { + let mut data_map = FinalizedFlagMap::new(); + + for flag_file in flag_files { + // Split /path/sdk/<int.int>/finalized-flags.txt -> ['/path/sdk', 'int.int', 'finalized-flags.txt']. + let flag_file_split: Vec<String> = + flag_file.clone().rsplitn(3, '/').map(|s| s.to_string()).collect(); + + if &flag_file_split[0] != "finalized-flags.txt" { + return Err(anyhow!("Provided incorrect file, must be finalized-flags.txt")); + } + + let api_level_string = &flag_file_split[1]; + + // For now, skip any directory with full API level, e.g. "36.1". The + // finalized flag files each contain all flags finalized *up to* that + // level (including prior levels), so skipping intermediate levels means + // the flags will be included at the next full number. + // TODO: b/378936061 - Support full SDK version. + // In the future, we should error if provided a non-numeric directory. + let Ok(api_level) = str_to_api_level(api_level_string) else { + continue; + }; + + let file = fs::File::open(flag_file)?; + let reader = io::BufReader::new(file); + + for qualified_flag_name in reader.lines() { + // Split the qualified flag name into package and flag name: + // com.my.package.name.my_flag_name -> ['com.my.package.name', 'my_flag_name']. + let mut flag: Vec<String> = + qualified_flag_name?.rsplitn(2, '.').map(|s| s.to_string()).collect(); + + if flag.len() != 2 { + continue; + } + + let package_name = flag.pop().ok_or(anyhow!("Missing flag package."))?; + let flag_name = flag.pop().ok_or(anyhow!("Missing flag name."))?; + + // Only add the flag if it wasn't added in a prior file. + data_map.insert_if_new(api_level, FinalizedFlag { flag_name, package_name }); + } + } + + Ok(data_map) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + const FLAG_FILE_NAME: &str = "finalized-flags.txt"; + + // Creates some flags for testing. + fn create_test_flags() -> Vec<FinalizedFlag> { + vec![ + FinalizedFlag { flag_name: "name1".to_string(), package_name: "package1".to_string() }, + FinalizedFlag { flag_name: "name2".to_string(), package_name: "package2".to_string() }, + FinalizedFlag { flag_name: "name3".to_string(), package_name: "package3".to_string() }, + ] + } + + // Writes the fully qualified flag names in the given file. + fn add_flags_to_file(flag_file: &mut File, flags: &[FinalizedFlag]) { + for flag in flags { + let _unused = writeln!(flag_file, "{}.{}", flag.package_name, flag.flag_name); + } + } + + #[test] + fn test_read_flags_one_file() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt. + let temp_dir = tempdir().unwrap(); + let mut file_path = temp_dir.path().to_path_buf(); + file_path.push("35"); + fs::create_dir_all(&file_path).unwrap(); + file_path.push(FLAG_FILE_NAME); + let mut file = File::create(&file_path).unwrap(); + + // Write all flags to the file. + add_flags_to_file(&mut file, &[flags[0].clone(), flags[1].clone()]); + let flag_file_path = file_path.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![flag_file_path]).unwrap(); + + assert_eq!(map.0.len(), 1); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[0])); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[1])); + } + + #[test] + fn test_read_flags_two_files() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt and for 36. + let temp_dir = tempdir().unwrap(); + let mut file_path1 = temp_dir.path().to_path_buf(); + file_path1.push("35"); + fs::create_dir_all(&file_path1).unwrap(); + file_path1.push(FLAG_FILE_NAME); + let mut file1 = File::create(&file_path1).unwrap(); + + let mut file_path2 = temp_dir.path().to_path_buf(); + file_path2.push("36"); + fs::create_dir_all(&file_path2).unwrap(); + file_path2.push(FLAG_FILE_NAME); + let mut file2 = File::create(&file_path2).unwrap(); + + // Write all flags to the files. + add_flags_to_file(&mut file1, &[flags[0].clone()]); + add_flags_to_file(&mut file2, &[flags[0].clone(), flags[1].clone(), flags[2].clone()]); + let flag_file_path1 = file_path1.to_string_lossy().to_string(); + let flag_file_path2 = file_path2.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![flag_file_path1, flag_file_path2]).unwrap(); + + // Assert there are two API levels, 35 and 36. + assert_eq!(map.0.len(), 2); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[0])); + + // 36 should not have the first flag in the set, as it was finalized in + // an earlier API level. + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[1])); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[2])); + } + + #[test] + fn test_read_flags_full_numbers() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt and for 36. + let temp_dir = tempdir().unwrap(); + let mut file_path1 = temp_dir.path().to_path_buf(); + file_path1.push("35.0"); + fs::create_dir_all(&file_path1).unwrap(); + file_path1.push(FLAG_FILE_NAME); + let mut file1 = File::create(&file_path1).unwrap(); + + let mut file_path2 = temp_dir.path().to_path_buf(); + file_path2.push("36.0"); + fs::create_dir_all(&file_path2).unwrap(); + file_path2.push(FLAG_FILE_NAME); + let mut file2 = File::create(&file_path2).unwrap(); + + // Write all flags to the files. + add_flags_to_file(&mut file1, &[flags[0].clone()]); + add_flags_to_file(&mut file2, &[flags[0].clone(), flags[1].clone(), flags[2].clone()]); + let flag_file_path1 = file_path1.to_string_lossy().to_string(); + let flag_file_path2 = file_path2.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![flag_file_path1, flag_file_path2]).unwrap(); + + assert_eq!(map.0.len(), 2); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[0])); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[1])); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[2])); + } + + #[test] + fn test_read_flags_fractions_round_up() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt and for 36. + let temp_dir = tempdir().unwrap(); + let mut file_path1 = temp_dir.path().to_path_buf(); + file_path1.push("35.1"); + fs::create_dir_all(&file_path1).unwrap(); + file_path1.push(FLAG_FILE_NAME); + let mut file1 = File::create(&file_path1).unwrap(); + + let mut file_path2 = temp_dir.path().to_path_buf(); + file_path2.push("36.0"); + fs::create_dir_all(&file_path2).unwrap(); + file_path2.push(FLAG_FILE_NAME); + let mut file2 = File::create(&file_path2).unwrap(); + + // Write all flags to the files. + add_flags_to_file(&mut file1, &[flags[0].clone()]); + add_flags_to_file(&mut file2, &[flags[0].clone(), flags[1].clone(), flags[2].clone()]); + let flag_file_path1 = file_path1.to_string_lossy().to_string(); + let flag_file_path2 = file_path2.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![flag_file_path1, flag_file_path2]).unwrap(); + + // No flags were added in 35. All 35.1 flags were rolled up to 36. + assert_eq!(map.0.len(), 1); + assert!(!map.0.contains_key(&ApiLevel(35))); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[0])); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[1])); + assert!(map.0.get(&ApiLevel(36)).unwrap().contains(&flags[2])); + } + + #[test] + fn test_read_flags_non_numeric() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt. + let temp_dir = tempdir().unwrap(); + let mut file_path = temp_dir.path().to_path_buf(); + file_path.push("35"); + fs::create_dir_all(&file_path).unwrap(); + file_path.push(FLAG_FILE_NAME); + let mut flag_file = File::create(&file_path).unwrap(); + + let mut invalid_path = temp_dir.path().to_path_buf(); + invalid_path.push("sdk-annotations"); + fs::create_dir_all(&invalid_path).unwrap(); + invalid_path.push(FLAG_FILE_NAME); + File::create(&invalid_path).unwrap(); + + // Write all flags to the file. + add_flags_to_file(&mut flag_file, &[flags[0].clone(), flags[1].clone()]); + let flag_file_path = file_path.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![ + flag_file_path, + invalid_path.to_string_lossy().to_string(), + ]) + .unwrap(); + + // No set should be created for sdk-annotations. + assert_eq!(map.0.len(), 1); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[0])); + assert!(map.0.get(&ApiLevel(35)).unwrap().contains(&flags[1])); + } + + #[test] + fn test_read_flags_wrong_file_err() { + let flags = create_test_flags(); + + // Create the file <temp_dir>/35/finalized-flags.txt. + let temp_dir = tempdir().unwrap(); + let mut file_path = temp_dir.path().to_path_buf(); + file_path.push("35"); + fs::create_dir_all(&file_path).unwrap(); + file_path.push(FLAG_FILE_NAME); + let mut flag_file = File::create(&file_path).unwrap(); + + let mut pre_flag_path = temp_dir.path().to_path_buf(); + pre_flag_path.push("18"); + fs::create_dir_all(&pre_flag_path).unwrap(); + pre_flag_path.push("some_random_file.txt"); + File::create(&pre_flag_path).unwrap(); + + // Write all flags to the file. + add_flags_to_file(&mut flag_file, &[flags[0].clone(), flags[1].clone()]); + let flag_file_path = file_path.to_string_lossy().to_string(); + + // Convert to map. + let map = read_files_to_map_using_path(vec![ + flag_file_path, + pre_flag_path.to_string_lossy().to_string(), + ]); + + assert!(map.is_err()); + } + + #[test] + fn test_flags_map_insert_if_new() { + let flags = create_test_flags(); + let mut map = FinalizedFlagMap::new(); + let l35 = ApiLevel(35); + let l36 = ApiLevel(36); + + map.insert_if_new(l35, flags[0].clone()); + map.insert_if_new(l35, flags[1].clone()); + map.insert_if_new(l35, flags[2].clone()); + map.insert_if_new(l36, flags[0].clone()); + + assert!(map.0.get(&l35).unwrap().contains(&flags[0])); + assert!(map.0.get(&l35).unwrap().contains(&flags[1])); + assert!(map.0.get(&l35).unwrap().contains(&flags[2])); + assert!(!map.0.contains_key(&l36)); + } + + #[test] + fn test_flags_map_get_level() { + let flags = create_test_flags(); + let mut map = FinalizedFlagMap::new(); + let l35 = ApiLevel(35); + let l36 = ApiLevel(36); + + map.insert_if_new(l35, flags[0].clone()); + map.insert_if_new(l36, flags[1].clone()); + + assert_eq!(map.get_finalized_level(&flags[0]).unwrap(), l35); + assert_eq!(map.get_finalized_level(&flags[1]).unwrap(), l36); + } +} diff --git a/tools/aconfig/convert_finalized_flags/src/main.rs b/tools/aconfig/convert_finalized_flags/src/main.rs new file mode 100644 index 0000000000..38300f6776 --- /dev/null +++ b/tools/aconfig/convert_finalized_flags/src/main.rs @@ -0,0 +1,57 @@ +/* +* Copyright (C) 2025 The Android Open Source Project +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +//! convert_finalized_flags is a build time tool used to convert the finalized +//! flags text files under prebuilts/sdk into structured data (FinalizedFlag +//! struct). +//! This binary is intended to run as part of a genrule to create a json file +//! which is provided to the aconfig binary that creates the codegen. +//! Usage: +//! cargo run -- --flag-files-path path/to/prebuilts/sdk/finalized-flags.txt file2.txt etc +use anyhow::Result; +use clap::Parser; + +use convert_finalized_flags::read_files_to_map_using_path; + +const ABOUT_TEXT: &str = "Tool for processing finalized-flags.txt files. + +These files contain the list of qualified flag names that have been finalized, +each on a newline. The directory of the flag file is the finalized API level. + +The output is a json map of API level to set of FinalizedFlag objects. The only +supported use case for this tool is via a genrule at build time for aconfig +codegen. + +Args: +* `flag-files-path`: Space-separated list of absolute paths for the finalized +flags files. +"; + +#[derive(Parser, Debug)] +#[clap(long_about=ABOUT_TEXT, bin_name="convert-finalized-flags")] +struct Cli { + /// Flags files. + #[arg(long = "flag_file_path")] + flag_file_path: Vec<String>, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let finalized_flags_map = read_files_to_map_using_path(cli.flag_file_path)?; + + let json_str = serde_json::to_string(&finalized_flags_map)?; + println!("{}", json_str); + Ok(()) +} diff --git a/tools/aconfig/fake_device_config/Android.bp b/tools/aconfig/fake_device_config/Android.bp index 7704742601..bf98058895 100644 --- a/tools/aconfig/fake_device_config/Android.bp +++ b/tools/aconfig/fake_device_config/Android.bp @@ -15,9 +15,7 @@ java_library { name: "fake_device_config", srcs: [ - "src/android/util/Log.java", - "src/android/provider/DeviceConfig.java", - "src/android/os/StrictMode.java", + "src/**/*.java", ], sdk_version: "none", system_modules: "core-all-system-modules", @@ -26,11 +24,22 @@ java_library { } java_library { - name: "strict_mode_stub", + name: "aconfig_storage_stub", srcs: [ - "src/android/os/StrictMode.java", + "src/android/os/flagging/**/*.java", ], sdk_version: "core_current", host_supported: true, is_stubs_module: true, } + +java_library { + name: "aconfig_storage_stub_none", + srcs: [ + "src/android/os/flagging/**/*.java", + ], + sdk_version: "none", + system_modules: "core-all-system-modules", + host_supported: true, + is_stubs_module: true, +} diff --git a/tools/aconfig/fake_device_config/src/android/os/Build.java b/tools/aconfig/fake_device_config/src/android/os/Build.java new file mode 100644 index 0000000000..8ec72fb2dc --- /dev/null +++ b/tools/aconfig/fake_device_config/src/android/os/Build.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os; + +public class Build { + public static class VERSION { + public static final int SDK_INT = 0; + } +} diff --git a/tools/aconfig/fake_device_config/src/android/os/StrictMode.java b/tools/aconfig/fake_device_config/src/android/os/flagging/AconfigPackage.java index 641625206c..3cac5168d1 100644 --- a/tools/aconfig/fake_device_config/src/android/os/StrictMode.java +++ b/tools/aconfig/fake_device_config/src/android/os/flagging/AconfigPackage.java @@ -14,16 +14,17 @@ * limitations under the License. */ -package android.os; +package android.os.flagging; -public class StrictMode { - public static ThreadPolicy allowThreadDiskReads() { +/* + * This class allows generated aconfig code to compile independently of the framework. + */ +public class AconfigPackage { + public static AconfigPackage load(String packageName) { throw new UnsupportedOperationException("Stub!"); } - public static void setThreadPolicy(final ThreadPolicy policy) { + public boolean getBooleanFlagValue(String flagName, boolean defaultValue) { throw new UnsupportedOperationException("Stub!"); } - - public static final class ThreadPolicy {} } diff --git a/tools/aconfig/fake_device_config/src/android/provider/DeviceConfig.java b/tools/aconfig/fake_device_config/src/android/os/flagging/AconfigPackageInternal.java index dbb07ac983..46058b664f 100644 --- a/tools/aconfig/fake_device_config/src/android/provider/DeviceConfig.java +++ b/tools/aconfig/fake_device_config/src/android/os/flagging/AconfigPackageInternal.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 The Android Open Source Project + * Copyright (C) 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,26 +14,18 @@ * limitations under the License. */ -package android.provider; +package android.os.flagging; /* * This class allows generated aconfig code to compile independently of the framework. */ -public class DeviceConfig { - private DeviceConfig() { - } +public class AconfigPackageInternal { - public static boolean getBoolean(String ns, String name, boolean def) { - return false; - } + public static AconfigPackageInternal load(String packageName, long packageFingerprint) { + throw new UnsupportedOperationException("Stub!"); + } - public static Properties getProperties(String namespace, String... names) { - return new Properties(); - } - - public static class Properties { - public boolean getBoolean(String name, boolean def) { - return false; - } - } + public boolean getBooleanFlagValue(int index) { + throw new UnsupportedOperationException("Stub!"); + } } diff --git a/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackage.java b/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackage.java new file mode 100644 index 0000000000..ec79f7daa1 --- /dev/null +++ b/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackage.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os.flagging; + +import java.util.Set; + +/* + * This class allows generated aconfig code to compile independently of the framework. + */ +public class PlatformAconfigPackage { + + public static final Set<String> PLATFORM_PACKAGE_MAP_FILES = + Set.of("system.package.map", "vendor.package.map", "product.package.map"); + + public static PlatformAconfigPackage load(String packageName) { + throw new UnsupportedOperationException("Stub!"); + } + + public boolean getBooleanFlagValue(String flagName, boolean defaultValue) { + throw new UnsupportedOperationException("Stub!"); + } +} diff --git a/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackageInternal.java b/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackageInternal.java new file mode 100644 index 0000000000..378c963ba4 --- /dev/null +++ b/tools/aconfig/fake_device_config/src/android/os/flagging/PlatformAconfigPackageInternal.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.os.flagging; + +/* + * This class allows generated aconfig code to compile independently of the framework. + */ +public class PlatformAconfigPackageInternal { + + public static PlatformAconfigPackageInternal load(String packageName, long packageFingerprint) { + throw new UnsupportedOperationException("Stub!"); + } + + public boolean getBooleanFlagValue(int index) { + throw new UnsupportedOperationException("Stub!"); + } +} diff --git a/tools/aconfig/fake_device_config/src/android/util/Log.java b/tools/aconfig/fake_device_config/src/android/util/Log.java index 3e7fd0f386..e40790a432 100644 --- a/tools/aconfig/fake_device_config/src/android/util/Log.java +++ b/tools/aconfig/fake_device_config/src/android/util/Log.java @@ -2,14 +2,18 @@ package android.util; public final class Log { public static int i(String tag, String msg) { - return 0; + throw new UnsupportedOperationException("Stub!"); + } + + public static int w(String tag, String msg) { + throw new UnsupportedOperationException("Stub!"); } public static int e(String tag, String msg) { - return 0; + throw new UnsupportedOperationException("Stub!"); } public static int e(String tag, String msg, Throwable tr) { - return 0; + throw new UnsupportedOperationException("Stub!"); } } diff --git a/tools/aconfig/printflags/Android.bp b/tools/aconfig/printflags/Android.bp index d50a77d072..6f7bca3529 100644 --- a/tools/aconfig/printflags/Android.bp +++ b/tools/aconfig/printflags/Android.bp @@ -19,6 +19,9 @@ rust_defaults { rust_binary { name: "printflags", defaults: ["printflags.defaults"], + apex_available: [ + "//apex_available:platform", + ], } rust_test_host { diff --git a/tools/auto_gen_test_config.py b/tools/auto_gen_test_config.py index 8ee599a1ec..d54c4121e4 100755 --- a/tools/auto_gen_test_config.py +++ b/tools/auto_gen_test_config.py @@ -34,6 +34,7 @@ PLACEHOLDER_MODULE = '{MODULE}' PLACEHOLDER_PACKAGE = '{PACKAGE}' PLACEHOLDER_RUNNER = '{RUNNER}' PLACEHOLDER_TEST_TYPE = '{TEST_TYPE}' +PLACEHOLDER_EXTRA_TEST_RUNNER_CONFIGS = '{EXTRA_TEST_RUNNER_CONFIGS}' def main(argv): @@ -59,6 +60,7 @@ def main(argv): "instrumentation_test_config_template", help="Path to the instrumentation test config template.") parser.add_argument("--extra-configs", default="") + parser.add_argument("--extra-test-runner-configs", default="") args = parser.parse_args(argv) target_config = args.target_config @@ -66,6 +68,7 @@ def main(argv): empty_config = args.empty_config instrumentation_test_config_template = args.instrumentation_test_config_template extra_configs = '\n'.join(args.extra_configs.split('\\n')) + extra_test_runner_configs = '\n'.join(args.extra_test_runner_configs.split('\\n')) module = os.path.splitext(os.path.basename(target_config))[0] @@ -131,6 +134,7 @@ def main(argv): config = config.replace(PLACEHOLDER_PACKAGE, package) config = config.replace(PLACEHOLDER_TEST_TYPE, test_type) config = config.replace(PLACEHOLDER_EXTRA_CONFIGS, extra_configs) + config = config.replace(PLACEHOLDER_EXTRA_TEST_RUNNER_CONFIGS, extra_test_runner_configs) config = config.replace(PLACEHOLDER_RUNNER, runner) with open(target_config, 'w') as config_file: config_file.write(config) diff --git a/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt b/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt index e07ac1dfd4..5acb54a082 100644 --- a/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt +++ b/tools/check-flagged-apis/src/com/android/checkflaggedapis/CheckFlaggedApisTest.kt @@ -34,7 +34,7 @@ private val API_SIGNATURE = ctor @FlaggedApi("android.flag.foo") public Clazz(); field @FlaggedApi("android.flag.foo") public static final int FOO = 1; // 0x1 method @FlaggedApi("android.flag.foo") public int getErrorCode(); - method @FlaggedApi("android.flag.foo") public boolean setData(int, int[][], @NonNull android.util.Utility<T, U>); + method @FlaggedApi("android.flag.foo") public <T,U> boolean setData(int, int[][], @NonNull android.util.Utility<T, U>); method @FlaggedApi("android.flag.foo") public boolean setVariableData(int, android.util.Atom...); method @FlaggedApi("android.flag.foo") public boolean innerClassArg(android.Clazz.Builder); } diff --git a/tools/check_elf_file.py b/tools/check_elf_file.py index 1fd7950bfe..064004179e 100755 --- a/tools/check_elf_file.py +++ b/tools/check_elf_file.py @@ -42,8 +42,9 @@ _EM_ARM = 40 _EM_X86_64 = 62 _EM_AARCH64 = 183 -_KNOWN_MACHINES = {_EM_386, _EM_ARM, _EM_X86_64, _EM_AARCH64} - +_32_BIT_MACHINES = {_EM_386, _EM_ARM} +_64_BIT_MACHINES = {_EM_X86_64, _EM_AARCH64} +_KNOWN_MACHINES = _32_BIT_MACHINES | _64_BIT_MACHINES # ELF header struct _ELF_HEADER_STRUCT = ( @@ -483,6 +484,11 @@ class Checker(object): sys.exit(2) def check_max_page_size(self, max_page_size): + if self._file_under_test.header.e_machine in _32_BIT_MACHINES: + # Skip test on 32-bit machines. 16 KB pages is an arm64 feature + # and no 32-bit systems in Android use it. + return + for alignment in self._file_under_test.alignments: if alignment % max_page_size != 0: self._error(f'Load segment has alignment {alignment} but ' diff --git a/tools/compliance/go.work b/tools/compliance/go.work index a24d2ea541..506e619436 100644 --- a/tools/compliance/go.work +++ b/tools/compliance/go.work @@ -1,4 +1,4 @@ -go 1.22 +go 1.23 use ( . diff --git a/tools/edit_monitor/Android.bp b/tools/edit_monitor/Android.bp new file mode 100644 index 0000000000..b8ac5bff53 --- /dev/null +++ b/tools/edit_monitor/Android.bp @@ -0,0 +1,118 @@ +// Copyright 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Set of error prone rules to ensure code quality +// PackageLocation check requires the androidCompatible=false otherwise it does not do anything. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], + default_team: "trendy_team_adte", +} + +python_library_host { + name: "edit_event_proto", + srcs: [ + "proto/edit_event.proto", + ], + proto: { + canonical_path_from_root: false, + }, +} + +python_library_host { + name: "edit_monitor_lib", + pkg_path: "edit_monitor", + srcs: [ + "daemon_manager.py", + "edit_monitor.py", + "utils.py", + ], + libs: [ + "asuite_cc_client", + "edit_event_proto", + "watchdog", + ], +} + +python_test_host { + name: "daemon_manager_test", + main: "daemon_manager_test.py", + pkg_path: "edit_monitor", + srcs: [ + "daemon_manager_test.py", + ], + libs: [ + "edit_monitor_lib", + ], + test_options: { + unit_test: true, + }, +} + +python_test_host { + name: "edit_monitor_test", + main: "edit_monitor_test.py", + pkg_path: "edit_monitor", + srcs: [ + "edit_monitor_test.py", + ], + libs: [ + "edit_monitor_lib", + ], + test_options: { + unit_test: true, + }, +} + +python_test_host { + name: "edit_monitor_utils_test", + main: "utils_test.py", + pkg_path: "edit_monitor", + srcs: [ + "utils_test.py", + ], + libs: [ + "edit_monitor_lib", + ], + test_options: { + unit_test: true, + }, +} + +python_test_host { + name: "edit_monitor_integration_test", + main: "edit_monitor_integration_test.py", + pkg_path: "testdata", + srcs: [ + "edit_monitor_integration_test.py", + ], + test_options: { + unit_test: true, + }, + data: [ + ":edit_monitor", + ], +} + +python_binary_host { + name: "edit_monitor", + pkg_path: "edit_monitor", + srcs: [ + "main.py", + ], + libs: [ + "edit_monitor_lib", + ], + main: "main.py", +} diff --git a/tools/edit_monitor/OWNERS b/tools/edit_monitor/OWNERS new file mode 100644 index 0000000000..8f0f3646dd --- /dev/null +++ b/tools/edit_monitor/OWNERS @@ -0,0 +1 @@ +include platform/tools/asuite:/OWNERS_ADTE_TEAM
\ No newline at end of file diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py new file mode 100644 index 0000000000..7d666fed55 --- /dev/null +++ b/tools/edit_monitor/daemon_manager.py @@ -0,0 +1,463 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import errno +import fcntl +import getpass +import hashlib +import logging +import multiprocessing +import os +import pathlib +import platform +import signal +import subprocess +import sys +import tempfile +import time + +from atest.metrics import clearcut_client +from atest.proto import clientanalytics_pb2 +from edit_monitor import utils +from proto import edit_event_pb2 + +DEFAULT_PROCESS_TERMINATION_TIMEOUT_SECONDS = 5 +DEFAULT_MONITOR_INTERVAL_SECONDS = 5 +DEFAULT_MEMORY_USAGE_THRESHOLD = 0.02 # 2% of total memory +DEFAULT_CPU_USAGE_THRESHOLD = 200 +DEFAULT_REBOOT_TIMEOUT_SECONDS = 60 * 60 * 24 +BLOCK_SIGN_FILE = "edit_monitor_block_sign" +# Enum of the Clearcut log source defined under +# /google3/wireless/android/play/playlog/proto/log_source_enum.proto +LOG_SOURCE = 2524 + + +def default_daemon_target(): + """Place holder for the default daemon target.""" + print("default daemon target") + + +class DaemonManager: + """Class to manage and monitor the daemon run as a subprocess.""" + + def __init__( + self, + binary_path: str, + daemon_target: callable = default_daemon_target, + daemon_args: tuple = (), + cclient: clearcut_client.Clearcut | None = None, + ): + self.binary_path = binary_path + self.daemon_target = daemon_target + self.daemon_args = daemon_args + self.cclient = cclient or clearcut_client.Clearcut(LOG_SOURCE) + + self.user_name = getpass.getuser() + self.host_name = platform.node() + self.source_root = os.environ.get("ANDROID_BUILD_TOP", "") + self.pid = os.getpid() + self.daemon_process = None + + self.max_memory_usage = 0 + self.max_cpu_usage = 0 + self.total_memory_size = os.sysconf("SC_PAGE_SIZE") * os.sysconf( + "SC_PHYS_PAGES" + ) + + pid_file_dir = pathlib.Path(tempfile.gettempdir()).joinpath("edit_monitor") + pid_file_dir.mkdir(parents=True, exist_ok=True) + self.pid_file_path = self._get_pid_file_path(pid_file_dir) + self.block_sign = pathlib.Path(tempfile.gettempdir()).joinpath( + BLOCK_SIGN_FILE + ) + + def start(self): + """Writes the pidfile and starts the daemon proces.""" + if not utils.is_feature_enabled( + "edit_monitor", + self.user_name, + "ENABLE_ANDROID_EDIT_MONITOR", + 100, + ): + logging.warning("Edit monitor is disabled, exiting...") + return + + if self.block_sign.exists(): + logging.warning("Block sign found, exiting...") + return + + if self.binary_path.startswith("/google/cog/"): + logging.warning("Edit monitor for cog is not supported, exiting...") + return + + setup_lock_file = pathlib.Path(tempfile.gettempdir()).joinpath( + self.pid_file_path.name + ".setup" + ) + logging.info("setup lock file: %s", setup_lock_file) + with open(setup_lock_file, "w") as f: + try: + # Acquire an exclusive lock + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + self._stop_any_existing_instance() + self._write_pid_to_pidfile() + self._start_daemon_process() + except Exception as e: + if ( + isinstance(e, IOError) and e.errno == errno.EAGAIN + ): # Failed to acquire the file lock. + logging.warning("Another edit monitor is starting, exitinng...") + return + else: + logging.exception("Failed to start daemon manager with error %s", e) + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR + ) + raise e + finally: + # Release the lock + fcntl.flock(f, fcntl.LOCK_UN) + + def monitor_daemon( + self, + interval: int = DEFAULT_MONITOR_INTERVAL_SECONDS, + memory_threshold: float = DEFAULT_MEMORY_USAGE_THRESHOLD, + cpu_threshold: float = DEFAULT_CPU_USAGE_THRESHOLD, + reboot_timeout: int = DEFAULT_REBOOT_TIMEOUT_SECONDS, + ): + """Monits the daemon process status. + + Periodically check the CPU/Memory usage of the daemon process as long as the + process is still running and kill the process if the resource usage is above + given thresholds. + """ + if not self.daemon_process: + return + + logging.info("start monitoring daemon process %d.", self.daemon_process.pid) + reboot_time = time.time() + reboot_timeout + while self.daemon_process.is_alive(): + if time.time() > reboot_time: + self.reboot() + try: + memory_usage = self._get_process_memory_percent(self.daemon_process.pid) + self.max_memory_usage = max(self.max_memory_usage, memory_usage) + + cpu_usage = self._get_process_cpu_percent(self.daemon_process.pid) + self.max_cpu_usage = max(self.max_cpu_usage, cpu_usage) + + time.sleep(interval) + except Exception as e: + # Logging the error and continue. + logging.warning("Failed to monitor daemon process with error: %s", e) + + if self.max_memory_usage >= memory_threshold: + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE + ) + logging.error( + "Daemon process is consuming too much memory, rebooting..." + ) + self.reboot() + + if self.max_cpu_usage >= cpu_threshold: + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE + ) + logging.error("Daemon process is consuming too much cpu, killing...") + self._terminate_process(self.daemon_process.pid) + + logging.info( + "Daemon process %d terminated. Max memory usage: %f, Max cpu" + " usage: %f.", + self.daemon_process.pid, + self.max_memory_usage, + self.max_cpu_usage, + ) + + def stop(self): + """Stops the daemon process and removes the pidfile.""" + + logging.info("in daemon manager cleanup.") + try: + if self.daemon_process: + # The daemon process might already in termination process, + # wait some time before kill it explicitly. + self._wait_for_process_terminate(self.daemon_process.pid, 1) + if self.daemon_process.is_alive(): + self._terminate_process(self.daemon_process.pid) + self._remove_pidfile(self.pid) + logging.info("Successfully stopped daemon manager.") + except Exception as e: + logging.exception("Failed to stop daemon manager with error %s", e) + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR + ) + sys.exit(1) + finally: + self.cclient.flush_events() + + def reboot(self): + """Reboots the current process. + + Stops the current daemon manager and reboots the entire process based on + the binary file. Exits directly If the binary file no longer exists. + """ + logging.info("Rebooting process based on binary %s.", self.binary_path) + + # Stop the current daemon manager first. + self.stop() + + # If the binary no longer exists, exit directly. + if not os.path.exists(self.binary_path): + logging.info("binary %s no longer exists, exiting.", self.binary_path) + sys.exit(0) + + try: + os.execv(self.binary_path, sys.argv) + except OSError as e: + logging.exception("Failed to reboot process with error: %s.", e) + self._send_error_event_to_clearcut( + edit_event_pb2.EditEvent.FAILED_TO_REBOOT_EDIT_MONITOR + ) + sys.exit(1) # Indicate an error occurred + + def cleanup(self): + """Wipes out all edit monitor instances in the system. + + Stops all the existing edit monitor instances and place a block sign + to prevent any edit monitor process to start. This method is only used + in emergency case when there's something goes wrong with the edit monitor + that requires immediate cleanup to prevent damanger to the system. + """ + logging.debug("Start cleaning up all existing instances.") + self._send_error_event_to_clearcut(edit_event_pb2.EditEvent.FORCE_CLEANUP) + + try: + # First places a block sign to prevent any edit monitor process to start. + self.block_sign.touch() + except (FileNotFoundError, PermissionError, OSError): + logging.exception("Failed to place the block sign") + + # Finds and kills all the existing instances of edit monitor. + existing_instances_pids = self._find_all_instances_pids() + for pid in existing_instances_pids: + logging.info( + "Found existing edit monitor instance with pid %d, killing...", pid + ) + try: + self._terminate_process(pid) + except Exception: + logging.exception("Failed to terminate process %d", pid) + + def _stop_any_existing_instance(self): + if not self.pid_file_path.exists(): + logging.debug("No existing instances.") + return + + ex_pid = self._read_pid_from_pidfile() + + if ex_pid: + logging.info("Found another instance with pid %d.", ex_pid) + self._terminate_process(ex_pid) + self._remove_pidfile(ex_pid) + + def _read_pid_from_pidfile(self) -> int | None: + try: + with open(self.pid_file_path, "r") as f: + return int(f.read().strip()) + except FileNotFoundError as e: + logging.warning("pidfile %s does not exist.", self.pid_file_path) + return None + + def _write_pid_to_pidfile(self): + """Creates a pidfile and writes the current pid to the file. + + Raise FileExistsError if the pidfile already exists. + """ + try: + # Use the 'x' mode to open the file for exclusive creation + with open(self.pid_file_path, "x") as f: + f.write(f"{self.pid}") + except FileExistsError as e: + # This could be caused due to race condition that a user is trying + # to start two edit monitors at the same time. Or because there is + # already an existing edit monitor running and we can not kill it + # for some reason. + logging.exception("pidfile %s already exists.", self.pid_file_path) + raise e + + def _start_daemon_process(self): + """Starts a subprocess to run the daemon.""" + p = multiprocessing.Process( + target=self.daemon_target, args=self.daemon_args + ) + p.daemon = True + p.start() + + logging.info("Start subprocess with PID %d", p.pid) + self.daemon_process = p + + def _terminate_process( + self, pid: int, timeout: int = DEFAULT_PROCESS_TERMINATION_TIMEOUT_SECONDS + ): + """Terminates a process with given pid. + + It first sends a SIGTERM to the process to allow it for proper + termination with a timeout. If the process is not terminated within + the timeout, kills it forcefully. + """ + try: + os.kill(pid, signal.SIGTERM) + if not self._wait_for_process_terminate(pid, timeout): + logging.warning( + "Process %d not terminated within timeout, try force kill", pid + ) + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + logging.info("Process with PID %d not found (already terminated)", pid) + + def _wait_for_process_terminate(self, pid: int, timeout: int) -> bool: + start_time = time.time() + + while time.time() < start_time + timeout: + if not self._is_process_alive(pid): + return True + time.sleep(1) + + logging.error("Process %d not terminated within %d seconds.", pid, timeout) + return False + + def _is_process_alive(self, pid: int) -> bool: + try: + output = subprocess.check_output( + ["ps", "-p", str(pid), "-o", "state="], text=True + ).strip() + state = output.split()[0] + return state != "Z" # Check if the state is not 'Z' (zombie) + except subprocess.CalledProcessError: + # Process not found (already dead). + return False + except (FileNotFoundError, OSError, ValueError) as e: + logging.warning( + "Unable to check the status for process %d with error: %s.", pid, e + ) + return True + + def _remove_pidfile(self, expected_pid: int): + recorded_pid = self._read_pid_from_pidfile() + + if recorded_pid is None: + logging.info("pid file %s already removed.", self.pid_file_path) + return + + if recorded_pid != expected_pid: + logging.warning( + "pid file contains pid from a different process, expected pid: %d," + " actual pid: %d.", + expected_pid, + recorded_pid, + ) + return + + logging.debug("removing pidfile written by process %s", expected_pid) + try: + os.remove(self.pid_file_path) + except FileNotFoundError: + logging.info("pid file %s already removed.", self.pid_file_path) + + def _get_pid_file_path(self, pid_file_dir: pathlib.Path) -> pathlib.Path: + """Generates the path to store the pidfile. + + The file path should have the format of "/tmp/edit_monitor/xxxx.lock" + where xxxx is a hashed value based on the binary path that starts the + process. + """ + hash_object = hashlib.sha256() + hash_object.update(self.binary_path.encode("utf-8")) + pid_file_path = pid_file_dir.joinpath(hash_object.hexdigest() + ".lock") + logging.info("pid_file_path: %s", pid_file_path) + + return pid_file_path + + def _get_process_memory_percent(self, pid: int) -> float: + with open(f"/proc/{pid}/stat", "r") as f: + stat_data = f.readline().split() + # RSS is the 24th field in /proc/[pid]/stat + rss_pages = int(stat_data[23]) + process_memory = rss_pages * 4 * 1024 # Convert to bytes + + return ( + process_memory / self.total_memory_size + if self.total_memory_size + else 0.0 + ) + + def _get_process_cpu_percent(self, pid: int, interval: int = 1) -> float: + total_start_time = self._get_total_cpu_time(pid) + with open("/proc/uptime", "r") as f: + uptime_start = float(f.readline().split()[0]) + + time.sleep(interval) + + total_end_time = self._get_total_cpu_time(pid) + with open("/proc/uptime", "r") as f: + uptime_end = float(f.readline().split()[0]) + + return ( + (total_end_time - total_start_time) / (uptime_end - uptime_start) * 100 + ) + + def _get_total_cpu_time(self, pid: int) -> float: + with open(f"/proc/{str(pid)}/stat", "r") as f: + stats = f.readline().split() + # utime is the 14th field in /proc/[pid]/stat measured in clock ticks. + utime = int(stats[13]) + # stime is the 15th field in /proc/[pid]/stat measured in clock ticks. + stime = int(stats[14]) + return (utime + stime) / os.sysconf(os.sysconf_names["SC_CLK_TCK"]) + + def _find_all_instances_pids(self) -> list[int]: + pids = [] + + try: + output = subprocess.check_output(["ps", "-ef", "--no-headers"], text=True) + for line in output.splitlines(): + parts = line.split() + process_path = parts[7] + if pathlib.Path(process_path).name == "edit_monitor": + pid = int(parts[1]) + if pid != self.pid: # exclude the current process + pids.append(pid) + except Exception: + logging.exception( + "Failed to get pids of existing edit monitors from ps command." + ) + + return pids + + def _send_error_event_to_clearcut(self, error_type): + edit_monitor_error_event_proto = edit_event_pb2.EditEvent( + user_name=self.user_name, + host_name=self.host_name, + source_root=self.source_root, + ) + edit_monitor_error_event_proto.edit_monitor_error_event.CopyFrom( + edit_event_pb2.EditEvent.EditMonitorErrorEvent(error_type=error_type) + ) + log_event = clientanalytics_pb2.LogEvent( + event_time_ms=int(time.time() * 1000), + source_extension=edit_monitor_error_event_proto.SerializeToString(), + ) + self.cclient.log(log_event) diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py new file mode 100644 index 0000000000..a7c175dbca --- /dev/null +++ b/tools/edit_monitor/daemon_manager_test.py @@ -0,0 +1,524 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittests for DaemonManager.""" + +import fcntl +import logging +import multiprocessing +import os +import pathlib +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock +from edit_monitor import daemon_manager +from proto import edit_event_pb2 + + +TEST_BINARY_FILE = '/path/to/test_binary' +TEST_PID_FILE_PATH = ( + '587239c2d1050afdf54512e2d799f3b929f86b43575eb3c7b4bab105dd9bd25e.lock' +) + + +def simple_daemon(output_file): + with open(output_file, 'w') as f: + f.write('running daemon target') + + +def long_running_daemon(): + while True: + time.sleep(1) + + +def memory_consume_daemon_target(size_mb): + try: + size_bytes = size_mb * 1024 * 1024 + dummy_data = bytearray(size_bytes) + time.sleep(10) + except MemoryError: + print(f'Process failed to allocate {size_mb} MB of memory.') + + +def cpu_consume_daemon_target(target_usage_percent): + while True: + start_time = time.time() + while time.time() - start_time < target_usage_percent / 100: + pass # Busy loop to consume CPU + + # Sleep to reduce CPU usage + time.sleep(1 - target_usage_percent / 100) + + +class DaemonManagerTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Configure to print logging to stdout. + logging.basicConfig(filename=None, level=logging.DEBUG) + console = logging.StreamHandler(sys.stdout) + logging.getLogger('').addHandler(console) + + def setUp(self): + super().setUp() + self.original_tempdir = tempfile.tempdir + self.working_dir = tempfile.TemporaryDirectory() + # Sets the tempdir under the working dir so any temp files created during + # tests will be cleaned. + tempfile.tempdir = self.working_dir.name + self.patch = mock.patch.dict( + os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'true'} + ) + self.patch.start() + + def tearDown(self): + # Cleans up any child processes left by the tests. + self._cleanup_child_processes() + self.working_dir.cleanup() + # Restores tempdir. + tempfile.tempdir = self.original_tempdir + self.patch.stop() + super().tearDown() + + def test_start_success_with_no_existing_instance(self): + self.assert_run_simple_daemon_success() + + def test_start_success_with_existing_instance_running(self): + # Create a running daemon subprocess + p = self._create_fake_deamon_process() + + self.assert_run_simple_daemon_success() + self.assert_no_subprocess_running() + + def test_start_success_with_existing_instance_already_dead(self): + # Create a pidfile with pid that does not exist. + pid_file_path_dir = pathlib.Path(self.working_dir.name).joinpath( + 'edit_monitor' + ) + pid_file_path_dir.mkdir(parents=True, exist_ok=True) + with open(pid_file_path_dir.joinpath(TEST_PID_FILE_PATH), 'w') as f: + f.write('123456') + + self.assert_run_simple_daemon_success() + + def test_start_success_with_existing_instance_from_different_binary(self): + # First start an instance based on "some_binary_path" + existing_dm = daemon_manager.DaemonManager( + 'some_binary_path', + daemon_target=long_running_daemon, + ) + existing_dm.start() + + self.assert_run_simple_daemon_success() + existing_dm.stop() + + def test_start_return_directly_if_block_sign_exists(self): + # Creates the block sign. + pathlib.Path(self.working_dir.name).joinpath( + daemon_manager.BLOCK_SIGN_FILE + ).touch() + + dm = daemon_manager.DaemonManager(TEST_BINARY_FILE) + dm.start() + + # Verify no daemon process is started. + self.assertIsNone(dm.daemon_process) + + @mock.patch.dict( + os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'false'}, clear=True + ) + def test_start_return_directly_if_disabled(self): + dm = daemon_manager.DaemonManager(TEST_BINARY_FILE) + dm.start() + + # Verify no daemon process is started. + self.assertIsNone(dm.daemon_process) + + def test_start_return_directly_if_in_cog_env(self): + dm = daemon_manager.DaemonManager( + '/google/cog/cloud/user/workspace/edit_monitor' + ) + dm.start() + + # Verify no daemon process is started. + self.assertIsNone(dm.daemon_process) + + def test_start_failed_other_instance_is_starting(self): + f = open( + pathlib.Path(self.working_dir.name).joinpath( + TEST_PID_FILE_PATH + '.setup' + ), + 'w', + ) + # Acquire an exclusive lock + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + + dm = daemon_manager.DaemonManager(TEST_BINARY_FILE) + dm.start() + + # Release the lock + fcntl.flock(f, fcntl.LOCK_UN) + f.close() + # Verify no daemon process is started. + self.assertIsNone(dm.daemon_process) + + @mock.patch('os.kill') + def test_start_failed_to_kill_existing_instance(self, mock_kill): + mock_kill.side_effect = OSError('Unknown OSError') + pid_file_path_dir = pathlib.Path(self.working_dir.name).joinpath( + 'edit_monitor' + ) + pid_file_path_dir.mkdir(parents=True, exist_ok=True) + with open(pid_file_path_dir.joinpath(TEST_PID_FILE_PATH), 'w') as f: + f.write('123456') + + fake_cclient = FakeClearcutClient() + with self.assertRaises(OSError): + dm = daemon_manager.DaemonManager(TEST_BINARY_FILE, cclient=fake_cclient) + dm.start() + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR + ) + + def test_start_failed_to_write_pidfile(self): + pid_file_path_dir = pathlib.Path(self.working_dir.name).joinpath( + 'edit_monitor' + ) + pid_file_path_dir.mkdir(parents=True, exist_ok=True) + + # Makes the directory read-only so write pidfile will fail. + os.chmod(pid_file_path_dir, 0o555) + + fake_cclient = FakeClearcutClient() + with self.assertRaises(PermissionError): + dm = daemon_manager.DaemonManager(TEST_BINARY_FILE, cclient=fake_cclient) + dm.start() + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR + ) + + def test_start_failed_to_start_daemon_process(self): + fake_cclient = FakeClearcutClient() + with self.assertRaises(TypeError): + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target='wrong_target', + daemon_args=(1), + cclient=fake_cclient, + ) + dm.start() + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR + ) + + @mock.patch('os.execv') + def test_monitor_reboot_with_high_memory_usage(self, mock_execv): + fake_cclient = FakeClearcutClient() + binary_file = tempfile.NamedTemporaryFile( + dir=self.working_dir.name, delete=False + ) + + dm = daemon_manager.DaemonManager( + binary_file.name, + daemon_target=memory_consume_daemon_target, + daemon_args=(2,), + cclient=fake_cclient, + ) + # set the fake total_memory_size + dm.total_memory_size = 100 * 1024 * 1024 + dm.start() + dm.monitor_daemon(interval=1) + + self.assertTrue(dm.max_memory_usage >= 0.02) + self.assert_no_subprocess_running() + self._assert_error_event_logged( + fake_cclient, + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE, + ) + mock_execv.assert_called_once() + + def test_monitor_daemon_subprocess_killed_high_cpu_usage(self): + fake_cclient = FakeClearcutClient() + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target=cpu_consume_daemon_target, + daemon_args=(20,), + cclient=fake_cclient, + ) + dm.start() + dm.monitor_daemon(interval=1, cpu_threshold=20) + + self.assertTrue(dm.max_cpu_usage >= 20) + self.assert_no_subprocess_running() + self._assert_error_event_logged( + fake_cclient, + edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE, + ) + + @mock.patch('subprocess.check_output') + def test_monitor_daemon_failed_does_not_matter(self, mock_output): + mock_output.side_effect = OSError('Unknown OSError') + self.assert_run_simple_daemon_success() + + @mock.patch('os.execv') + def test_monitor_daemon_reboot_triggered(self, mock_execv): + binary_file = tempfile.NamedTemporaryFile( + dir=self.working_dir.name, delete=False + ) + + dm = daemon_manager.DaemonManager( + binary_file.name, + daemon_target=long_running_daemon, + ) + dm.start() + dm.monitor_daemon(reboot_timeout=0.5) + mock_execv.assert_called_once() + + def test_stop_success(self): + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, daemon_target=long_running_daemon + ) + dm.start() + dm.stop() + + self.assert_no_subprocess_running() + self.assertFalse(dm.pid_file_path.exists()) + + @mock.patch('os.kill') + def test_stop_failed_to_kill_daemon_process(self, mock_kill): + mock_kill.side_effect = OSError('Unknown OSError') + fake_cclient = FakeClearcutClient() + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target=long_running_daemon, + cclient=fake_cclient, + ) + + with self.assertRaises(SystemExit): + dm.start() + dm.stop() + self.assertTrue(dm.daemon_process.is_alive()) + self.assertTrue(dm.pid_file_path.exists()) + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR + ) + + @mock.patch('os.remove') + def test_stop_failed_to_remove_pidfile(self, mock_remove): + mock_remove.side_effect = OSError('Unknown OSError') + + fake_cclient = FakeClearcutClient() + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target=long_running_daemon, + cclient=fake_cclient, + ) + + with self.assertRaises(SystemExit): + dm.start() + dm.stop() + self.assert_no_subprocess_running() + self.assertTrue(dm.pid_file_path.exists()) + + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR + ) + + @mock.patch('os.execv') + def test_reboot_success(self, mock_execv): + binary_file = tempfile.NamedTemporaryFile( + dir=self.working_dir.name, delete=False + ) + + dm = daemon_manager.DaemonManager( + binary_file.name, daemon_target=long_running_daemon + ) + dm.start() + dm.reboot() + + # Verifies the old process is stopped + self.assert_no_subprocess_running() + self.assertFalse(dm.pid_file_path.exists()) + + mock_execv.assert_called_once() + + @mock.patch('os.execv') + def test_reboot_binary_no_longer_exists(self, mock_execv): + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, daemon_target=long_running_daemon + ) + dm.start() + + with self.assertRaises(SystemExit): + dm.reboot() + mock_execv.assert_not_called() + self.assertEqual(cm.exception.code, 0) + + @mock.patch('os.execv') + def test_reboot_failed(self, mock_execv): + mock_execv.side_effect = OSError('Unknown OSError') + fake_cclient = FakeClearcutClient() + binary_file = tempfile.NamedTemporaryFile( + dir=self.working_dir.name, delete=False + ) + + dm = daemon_manager.DaemonManager( + binary_file.name, + daemon_target=long_running_daemon, + cclient=fake_cclient, + ) + dm.start() + + with self.assertRaises(SystemExit): + dm.reboot() + self.assertEqual(cm.exception.code, 1) + self._assert_error_event_logged( + fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_REBOOT_EDIT_MONITOR + ) + + @mock.patch('subprocess.check_output') + def test_cleanup_success(self, mock_check_output): + p = self._create_fake_deamon_process() + fake_cclient = FakeClearcutClient() + mock_check_output.return_value = f'user {p.pid} 1 1 1 1 1 edit_monitor arg' + + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target=long_running_daemon, + cclient=fake_cclient, + ) + dm.cleanup() + + self.assertFalse(p.is_alive()) + self.assertTrue( + pathlib.Path(self.working_dir.name) + .joinpath(daemon_manager.BLOCK_SIGN_FILE) + .exists() + ) + + def assert_run_simple_daemon_success(self): + damone_output_file = tempfile.NamedTemporaryFile( + dir=self.working_dir.name, delete=False + ) + dm = daemon_manager.DaemonManager( + TEST_BINARY_FILE, + daemon_target=simple_daemon, + daemon_args=(damone_output_file.name,), + ) + dm.start() + dm.monitor_daemon(interval=1) + + # Verifies the expected pid file is created. + expected_pid_file_path = pathlib.Path(self.working_dir.name).joinpath( + 'edit_monitor', TEST_PID_FILE_PATH + ) + self.assertTrue(expected_pid_file_path.exists()) + + # Verify the daemon process is executed successfully. + with open(damone_output_file.name, 'r') as f: + contents = f.read() + self.assertEqual(contents, 'running daemon target') + + def assert_no_subprocess_running(self): + child_pids = self._get_child_processes(os.getpid()) + for child_pid in child_pids: + self.assertFalse( + self._is_process_alive(child_pid), f'process {child_pid} still alive' + ) + + def _get_child_processes(self, parent_pid: int) -> list[int]: + try: + output = subprocess.check_output( + ['ps', '-o', 'pid,ppid', '--no-headers'], text=True + ) + + child_processes = [] + for line in output.splitlines(): + pid, ppid = line.split() + if int(ppid) == parent_pid: + child_processes.append(int(pid)) + return child_processes + except subprocess.CalledProcessError as e: + self.fail(f'failed to get child process, error: {e}') + + def _is_process_alive(self, pid: int) -> bool: + try: + output = subprocess.check_output( + ['ps', '-p', str(pid), '-o', 'state='], text=True + ).strip() + state = output.split()[0] + return state != 'Z' # Check if the state is not 'Z' (zombie) + except subprocess.CalledProcessError: + return False + + def _cleanup_child_processes(self): + child_pids = self._get_child_processes(os.getpid()) + for child_pid in child_pids: + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + # process already terminated + pass + + def _create_fake_deamon_process( + self, name: str = TEST_PID_FILE_PATH + ) -> multiprocessing.Process: + # Create a long running subprocess + p = multiprocessing.Process(target=long_running_daemon) + p.start() + + # Create the pidfile with the subprocess pid + pid_file_path_dir = pathlib.Path(self.working_dir.name).joinpath( + 'edit_monitor' + ) + pid_file_path_dir.mkdir(parents=True, exist_ok=True) + with open(pid_file_path_dir.joinpath(name), 'w') as f: + f.write(str(p.pid)) + return p + + def _assert_error_event_logged(self, fake_cclient, error_type): + error_events = fake_cclient.get_sent_events() + self.assertEqual(len(error_events), 1) + self.assertEqual( + edit_event_pb2.EditEvent.FromString( + error_events[0].source_extension + ).edit_monitor_error_event.error_type, + error_type, + ) + + +class FakeClearcutClient: + + def __init__(self): + self.pending_log_events = [] + self.sent_log_event = [] + + def log(self, log_event): + self.pending_log_events.append(log_event) + + def flush_events(self): + self.sent_log_event.extend(self.pending_log_events) + self.pending_log_events.clear() + + def get_sent_events(self): + return self.sent_log_event + self.pending_log_events + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/edit_monitor/edit_monitor.py b/tools/edit_monitor/edit_monitor.py new file mode 100644 index 0000000000..ab528e870f --- /dev/null +++ b/tools/edit_monitor/edit_monitor.py @@ -0,0 +1,220 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import getpass +import logging +import multiprocessing.connection +import os +import pathlib +import platform +import threading +import time + +from atest.metrics import clearcut_client +from atest.proto import clientanalytics_pb2 +from proto import edit_event_pb2 +from watchdog.events import FileSystemEvent +from watchdog.events import PatternMatchingEventHandler +from watchdog.observers import Observer + +# Enum of the Clearcut log source defined under +# /google3/wireless/android/play/playlog/proto/log_source_enum.proto +LOG_SOURCE = 2524 +DEFAULT_FLUSH_INTERVAL_SECONDS = 5 +DEFAULT_SINGLE_EVENTS_SIZE_THRESHOLD = 100 + + +class ClearcutEventHandler(PatternMatchingEventHandler): + + def __init__( + self, + path: str, + flush_interval_sec: int, + single_events_size_threshold: int, + is_dry_run: bool = False, + cclient: clearcut_client.Clearcut | None = None, + ): + + super().__init__(patterns=["*"], ignore_directories=True) + self.root_monitoring_path = path + self.flush_interval_sec = flush_interval_sec + self.single_events_size_threshold = single_events_size_threshold + self.is_dry_run = is_dry_run + self.cclient = cclient or clearcut_client.Clearcut(LOG_SOURCE) + + self.user_name = getpass.getuser() + self.host_name = platform.node() + self.source_root = os.environ.get("ANDROID_BUILD_TOP", "") + + self.pending_events = [] + self._scheduled_log_thread = None + self._pending_events_lock = threading.Lock() + + def on_moved(self, event: FileSystemEvent): + self._log_edit_event(event, edit_event_pb2.EditEvent.MOVE) + + def on_created(self, event: FileSystemEvent): + self._log_edit_event(event, edit_event_pb2.EditEvent.CREATE) + + def on_deleted(self, event: FileSystemEvent): + self._log_edit_event(event, edit_event_pb2.EditEvent.DELETE) + + def on_modified(self, event: FileSystemEvent): + self._log_edit_event(event, edit_event_pb2.EditEvent.MODIFY) + + def flushall(self): + logging.info("flushing all pending events.") + if self._scheduled_log_thread: + logging.info("canceling log thread") + self._scheduled_log_thread.cancel() + self._scheduled_log_thread = None + + self._log_clearcut_events() + self.cclient.flush_events() + + def _log_edit_event( + self, event: FileSystemEvent, edit_type: edit_event_pb2.EditEvent.EditType + ): + try: + event_time = time.time() + + if self._is_hidden_file(pathlib.Path(event.src_path)): + logging.debug("ignore hidden file: %s.", event.src_path) + return + + if not self._is_under_git_project(pathlib.Path(event.src_path)): + logging.debug( + "ignore file %s which does not belong to a git project", + event.src_path, + ) + return + + logging.info("%s: %s", event.event_type, event.src_path) + + event_proto = edit_event_pb2.EditEvent( + user_name=self.user_name, + host_name=self.host_name, + source_root=self.source_root, + ) + event_proto.single_edit_event.CopyFrom( + edit_event_pb2.EditEvent.SingleEditEvent( + file_path=event.src_path, edit_type=edit_type + ) + ) + with self._pending_events_lock: + self.pending_events.append((event_proto, event_time)) + if not self._scheduled_log_thread: + logging.debug( + "Scheduling thread to run in %d seconds", self.flush_interval_sec + ) + self._scheduled_log_thread = threading.Timer( + self.flush_interval_sec, self._log_clearcut_events + ) + self._scheduled_log_thread.start() + + except Exception: + logging.exception("Failed to log edit event.") + + def _is_hidden_file(self, file_path: pathlib.Path) -> bool: + return any( + part.startswith(".") + for part in file_path.relative_to(self.root_monitoring_path).parts + ) + + def _is_under_git_project(self, file_path: pathlib.Path) -> bool: + root_path = pathlib.Path(self.root_monitoring_path).resolve() + return any( + root_path.joinpath(dir).joinpath('.git').exists() + for dir in file_path.relative_to(root_path).parents + ) + + def _log_clearcut_events(self): + with self._pending_events_lock: + self._scheduled_log_thread = None + edit_events = self.pending_events + self.pending_events = [] + + pending_events_size = len(edit_events) + if pending_events_size > self.single_events_size_threshold: + logging.info( + "got %d events in %d seconds, sending aggregated events instead", + pending_events_size, + self.flush_interval_sec, + ) + aggregated_event_time = edit_events[0][1] + aggregated_event_proto = edit_event_pb2.EditEvent( + user_name=self.user_name, + host_name=self.host_name, + source_root=self.source_root, + ) + aggregated_event_proto.aggregated_edit_event.CopyFrom( + edit_event_pb2.EditEvent.AggregatedEditEvent( + num_edits=pending_events_size + ) + ) + edit_events = [(aggregated_event_proto, aggregated_event_time)] + + if self.is_dry_run: + logging.info("Sent %d edit events in dry run.", len(edit_events)) + return + + for event_proto, event_time in edit_events: + log_event = clientanalytics_pb2.LogEvent( + event_time_ms=int(event_time * 1000), + source_extension=event_proto.SerializeToString(), + ) + self.cclient.log(log_event) + + logging.info("sent %d edit events", len(edit_events)) + + +def start( + path: str, + is_dry_run: bool = False, + flush_interval_sec: int = DEFAULT_FLUSH_INTERVAL_SECONDS, + single_events_size_threshold: int = DEFAULT_SINGLE_EVENTS_SIZE_THRESHOLD, + cclient: clearcut_client.Clearcut | None = None, + pipe_sender: multiprocessing.connection.Connection | None = None, +): + """Method to start the edit monitor. + + This is the entry point to start the edit monitor as a subprocess of + the daemon manager. + + params: + path: The root path to monitor + cclient: The clearcut client to send the edit logs. + conn: the sender of the pipe to communicate with the deamon manager. + """ + event_handler = ClearcutEventHandler( + path, flush_interval_sec, single_events_size_threshold, is_dry_run, cclient) + observer = Observer() + + logging.info("Starting observer on path %s.", path) + observer.schedule(event_handler, path, recursive=True) + observer.start() + logging.info("Observer started.") + if pipe_sender: + pipe_sender.send("Observer started.") + + try: + while True: + time.sleep(1) + finally: + event_handler.flushall() + observer.stop() + observer.join() + if pipe_sender: + pipe_sender.close() diff --git a/tools/edit_monitor/edit_monitor_integration_test.py b/tools/edit_monitor/edit_monitor_integration_test.py new file mode 100644 index 0000000000..f39b93667d --- /dev/null +++ b/tools/edit_monitor/edit_monitor_integration_test.py @@ -0,0 +1,169 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration tests for Edit Monitor.""" + +import glob +from importlib import resources +import logging +import os +import pathlib +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + + +class EditMonitorIntegrationTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Configure to print logging to stdout. + logging.basicConfig(filename=None, level=logging.DEBUG) + console = logging.StreamHandler(sys.stdout) + logging.getLogger("").addHandler(console) + + def setUp(self): + super().setUp() + self.working_dir = tempfile.TemporaryDirectory() + self.root_monitoring_path = pathlib.Path(self.working_dir.name).joinpath( + "files" + ) + self.root_monitoring_path.mkdir() + self.edit_monitor_binary_path = self._import_executable("edit_monitor") + self.patch = mock.patch.dict( + os.environ, {"ENABLE_ANDROID_EDIT_MONITOR": "true"} + ) + self.patch.start() + + def tearDown(self): + self.patch.stop() + self.working_dir.cleanup() + super().tearDown() + + def test_log_single_edit_event_success(self): + p = self._start_edit_monitor_process() + + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath(".git").touch() + + # Create and modify a file. + test_file = self.root_monitoring_path.joinpath("test.txt") + with open(test_file, "w") as f: + f.write("something") + + # Move the file. + test_file_moved = self.root_monitoring_path.joinpath("new_test.txt") + test_file.rename(test_file_moved) + + # Delete the file. + test_file_moved.unlink() + + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.communicate() + + self.assertEqual(self._get_logged_events_num(), 4) + + def test_start_multiple_edit_monitor_only_one_started(self): + p1 = self._start_edit_monitor_process(wait_for_observer_start=False) + p2 = self._start_edit_monitor_process(wait_for_observer_start=False) + p3 = self._start_edit_monitor_process(wait_for_observer_start=False) + + live_processes = self._get_live_processes([p1, p2, p3]) + + # Cleanup all live processes. + for p in live_processes: + os.kill(p.pid, signal.SIGINT) + p.communicate() + + self.assertEqual(len(live_processes), 1) + + def _start_edit_monitor_process(self, wait_for_observer_start=True): + command = f""" + export TMPDIR="{self.working_dir.name}" + {self.edit_monitor_binary_path} --path={self.root_monitoring_path} --dry_run""" + p = subprocess.Popen( + command, + shell=True, + text=True, + start_new_session=True, + executable="/bin/bash", + ) + if wait_for_observer_start: + self._wait_for_observer_start(time_out=5) + + return p + + def _wait_for_observer_start(self, time_out): + start_time = time.time() + + while time.time() < start_time + time_out: + log_files = glob.glob(self.working_dir.name + "/edit_monitor_*/*.log") + if log_files: + with open(log_files[0], "r") as f: + for line in f: + logging.debug("initial log: %s", line) + if line.rstrip("\n").endswith("Observer started."): + return + else: + time.sleep(1) + + self.fail(f"Observer not started in {time_out} seconds.") + + def _get_logged_events_num(self): + log_files = glob.glob(self.working_dir.name + "/edit_monitor_*/*.log") + self.assertEqual(len(log_files), 1) + + with open(log_files[0], "r") as f: + for line in f: + logging.debug("complete log: %s", line) + if line.rstrip("\n").endswith("in dry run."): + return int(line.split(":")[-1].split(" ")[2]) + + return 0 + + def _get_live_processes(self, processes): + live_processes = [] + for p in processes: + try: + p.wait(timeout=5) + except subprocess.TimeoutExpired as e: + live_processes.append(p) + logging.info("process: %d still alive.", p.pid) + else: + logging.info("process: %d stopped.", p.pid) + return live_processes + + def _import_executable(self, executable_name: str) -> pathlib.Path: + binary_dir = pathlib.Path(self.working_dir.name).joinpath("binary") + binary_dir.mkdir() + executable_path = binary_dir.joinpath(executable_name) + with resources.as_file( + resources.files("testdata").joinpath(executable_name) + ) as binary: + shutil.copy(binary, executable_path) + executable_path.chmod(0o755) + return executable_path + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/edit_monitor/edit_monitor_test.py b/tools/edit_monitor/edit_monitor_test.py new file mode 100644 index 0000000000..deb73e724b --- /dev/null +++ b/tools/edit_monitor/edit_monitor_test.py @@ -0,0 +1,301 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittests for Edit Monitor.""" + +import logging +import multiprocessing +import os +import pathlib +import signal +import sys +import tempfile +import time +import unittest + +from atest.proto import clientanalytics_pb2 +from edit_monitor import edit_monitor +from proto import edit_event_pb2 + + +class EditMonitorTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Configure to print logging to stdout. + logging.basicConfig(filename=None, level=logging.DEBUG) + console = logging.StreamHandler(sys.stdout) + logging.getLogger('').addHandler(console) + + def setUp(self): + super().setUp() + self.working_dir = tempfile.TemporaryDirectory() + self.root_monitoring_path = pathlib.Path(self.working_dir.name).joinpath( + 'files' + ) + self.root_monitoring_path.mkdir() + self.log_event_dir = pathlib.Path(self.working_dir.name).joinpath('logs') + self.log_event_dir.mkdir() + + def tearDown(self): + self.working_dir.cleanup() + super().tearDown() + + def test_log_single_edit_event_success(self): + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath('.git').touch() + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output') + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create and modify a file. + test_file = self.root_monitoring_path.joinpath('test.txt') + with open(test_file, 'w') as f: + f.write('something') + # Move the file. + test_file_moved = self.root_monitoring_path.joinpath('new_test.txt') + test_file.rename(test_file_moved) + # Delete the file. + test_file_moved.unlink() + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 4) + expected_create_event = edit_event_pb2.EditEvent.SingleEditEvent( + file_path=str( + self.root_monitoring_path.joinpath('test.txt').resolve() + ), + edit_type=edit_event_pb2.EditEvent.CREATE, + ) + expected_modify_event = edit_event_pb2.EditEvent.SingleEditEvent( + file_path=str( + self.root_monitoring_path.joinpath('test.txt').resolve() + ), + edit_type=edit_event_pb2.EditEvent.MODIFY, + ) + expected_move_event = edit_event_pb2.EditEvent.SingleEditEvent( + file_path=str( + self.root_monitoring_path.joinpath('test.txt').resolve() + ), + edit_type=edit_event_pb2.EditEvent.MOVE, + ) + expected_delete_event = edit_event_pb2.EditEvent.SingleEditEvent( + file_path=str( + self.root_monitoring_path.joinpath('new_test.txt').resolve() + ), + edit_type=edit_event_pb2.EditEvent.DELETE, + ) + self.assertEqual( + expected_create_event, + edit_event_pb2.EditEvent.FromString( + logged_events[0].source_extension + ).single_edit_event, + ) + self.assertEqual( + expected_modify_event, + edit_event_pb2.EditEvent.FromString( + logged_events[1].source_extension + ).single_edit_event, + ) + self.assertEqual( + expected_move_event, + edit_event_pb2.EditEvent.FromString( + logged_events[2].source_extension + ).single_edit_event, + ) + self.assertEqual( + expected_delete_event, + edit_event_pb2.EditEvent.FromString( + logged_events[3].source_extension + ).single_edit_event, + ) + + + def test_log_aggregated_edit_event_success(self): + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath('.git').touch() + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output') + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create 6 test files + for i in range(6): + test_file = self.root_monitoring_path.joinpath('test_' + str(i)) + test_file.touch() + + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 1) + + expected_aggregated_edit_event = ( + edit_event_pb2.EditEvent.AggregatedEditEvent( + num_edits=6, + ) + ) + + self.assertEqual( + expected_aggregated_edit_event, + edit_event_pb2.EditEvent.FromString( + logged_events[0].source_extension + ).aggregated_edit_event, + ) + + def test_do_not_log_edit_event_for_directory_change(self): + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath('.git').touch() + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output') + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create a sub directory + self.root_monitoring_path.joinpath('test_dir').mkdir() + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 0) + + def test_do_not_log_edit_event_for_hidden_file(self): + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath('.git').touch() + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output') + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create a hidden file. + self.root_monitoring_path.joinpath('.test.txt').touch() + # Create a hidden dir. + hidden_dir = self.root_monitoring_path.joinpath('.test') + hidden_dir.mkdir() + hidden_dir.joinpath('test.txt').touch() + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 0) + + def test_do_not_log_edit_event_for_non_git_project_file(self): + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output') + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create a file. + self.root_monitoring_path.joinpath('test.txt').touch() + # Create a file under a sub dir. + sub_dir = self.root_monitoring_path.joinpath('.test') + sub_dir.mkdir() + sub_dir.joinpath('test.txt').touch() + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 0) + + def test_log_edit_event_fail(self): + # Create the .git file under the monitoring dir. + self.root_monitoring_path.joinpath('.git').touch() + fake_cclient = FakeClearcutClient( + log_output_file=self.log_event_dir.joinpath('logs.output'), + raise_log_exception=True, + ) + p = self._start_test_edit_monitor_process(fake_cclient) + + # Create a file. + self.root_monitoring_path.joinpath('test.txt').touch() + # Give some time for the edit monitor to receive the edit event. + time.sleep(1) + # Stop the edit monitor and flush all events. + os.kill(p.pid, signal.SIGINT) + p.join() + + logged_events = self._get_logged_events() + self.assertEqual(len(logged_events), 0) + + def _start_test_edit_monitor_process( + self, cclient + ) -> multiprocessing.Process: + receiver, sender = multiprocessing.Pipe() + # Start edit monitor in a subprocess. + p = multiprocessing.Process( + target=edit_monitor.start, + args=(str(self.root_monitoring_path.resolve()), False, 0.5, 5, cclient, sender), + ) + p.daemon = True + p.start() + + # Wait until observer started. + received_data = receiver.recv() + self.assertEqual(received_data, 'Observer started.') + + receiver.close() + return p + + def _get_logged_events(self): + with open(self.log_event_dir.joinpath('logs.output'), 'rb') as f: + data = f.read() + + return [ + clientanalytics_pb2.LogEvent.FromString(record) + for record in data.split(b'\x00') + if record + ] + + +class FakeClearcutClient: + + def __init__(self, log_output_file, raise_log_exception=False): + self.pending_log_events = [] + self.raise_log_exception = raise_log_exception + self.log_output_file = log_output_file + + def log(self, log_event): + if self.raise_log_exception: + raise Exception('unknown exception') + self.pending_log_events.append(log_event) + + def flush_events(self): + delimiter = b'\x00' # Use a null byte as the delimiter + with open(self.log_output_file, 'wb') as f: + for log_event in self.pending_log_events: + f.write(log_event.SerializeToString() + delimiter) + + self.pending_log_events.clear() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/edit_monitor/main.py b/tools/edit_monitor/main.py new file mode 100644 index 0000000000..3c2d183aed --- /dev/null +++ b/tools/edit_monitor/main.py @@ -0,0 +1,119 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import signal +import sys +import tempfile + +from edit_monitor import daemon_manager +from edit_monitor import edit_monitor + + +def create_arg_parser(): + """Creates an instance of the default arg parser.""" + + parser = argparse.ArgumentParser( + description=( + 'Monitors edits in Android source code and uploads the edit logs.' + ), + add_help=True, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + '--path', + type=str, + required=True, + help='Root path to monitor the edit events.', + ) + + parser.add_argument( + '--dry_run', + action='store_true', + help='Dry run the edit monitor. This starts the edit monitor process without actually send the edit logs to clearcut.', + ) + + parser.add_argument( + '--force_cleanup', + action='store_true', + help=( + 'Instead of start a new edit monitor, force stop all existing edit' + ' monitors in the system. This option is only used in emergent cases' + ' when we want to prevent user damage by the edit monitor.' + ), + ) + + parser.add_argument( + '--verbose', + action='store_true', + help=( + 'Log verbose info in the log file for debugging purpose.' + ), + ) + + return parser + + +def configure_logging(verbose=False): + root_logging_dir = tempfile.mkdtemp(prefix='edit_monitor_') + _, log_path = tempfile.mkstemp(dir=root_logging_dir, suffix='.log') + + + log_fmt = '%(asctime)s.%(msecs)03d %(filename)s:%(lineno)s:%(levelname)s: %(message)s' + date_fmt = '%Y-%m-%d %H:%M:%S' + log_level = logging.DEBUG if verbose else logging.INFO + + logging.basicConfig( + filename=log_path, level=log_level, format=log_fmt, datefmt=date_fmt + ) + # Filter out logs from inotify_buff to prevent log pollution. + logging.getLogger('watchdog.observers.inotify_buffer').addFilter( + lambda record: record.filename != 'inotify_buffer.py') + print(f'logging to file {log_path}') + + +def term_signal_handler(_signal_number, _frame): + logging.info('Process %d received SIGTERM, Terminating...', os.getpid()) + sys.exit(0) + + +def main(argv: list[str]): + args = create_arg_parser().parse_args(argv[1:]) + configure_logging(args.verbose) + if args.dry_run: + logging.info('This is a dry run.') + dm = daemon_manager.DaemonManager( + binary_path=argv[0], + daemon_target=edit_monitor.start, + daemon_args=(args.path, args.dry_run), + ) + + try: + if args.force_cleanup: + dm.cleanup() + else: + dm.start() + dm.monitor_daemon() + except Exception: + logging.exception('Unexpected exception raised when run daemon.') + finally: + dm.stop() + + +if __name__ == '__main__': + signal.signal(signal.SIGTERM, term_signal_handler) + main(sys.argv) diff --git a/tools/edit_monitor/proto/edit_event.proto b/tools/edit_monitor/proto/edit_event.proto new file mode 100644 index 0000000000..9acc2e754b --- /dev/null +++ b/tools/edit_monitor/proto/edit_event.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package tools.asuite.edit_monitor; + +message EditEvent { + enum EditType { + UNSUPPORTED_TYPE = 0; + CREATE = 1; + MODIFY = 2; + DELETE = 3; + MOVE = 4; + } + + enum ErrorType { + UNKNOWN_ERROR = 0; + FAILED_TO_START_EDIT_MONITOR = 1; + FAILED_TO_STOP_EDIT_MONITOR = 2; + FAILED_TO_REBOOT_EDIT_MONITOR = 3; + KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE = 4; + FORCE_CLEANUP = 5; + KILLED_DUE_TO_EXCEEDED_CPU_USAGE = 6; + } + + // Event that logs a single edit + message SingleEditEvent { + // Full path of the file that edited. + string file_path = 1; + // Type of the edit. + EditType edit_type = 2; + } + + // Event that logs aggregated info for a set of edits. + message AggregatedEditEvent { + int32 num_edits = 1; + } + + // Event that logs errors happened in the edit monitor. + message EditMonitorErrorEvent { + ErrorType error_type = 1; + } + + // ------------------------ + // FIELDS FOR EditEvent + // ------------------------ + // Internal user name. + string user_name = 1; + // The root of Android source. + string source_root = 2; + // Name of the host workstation. + string host_name = 3; + + oneof event { + SingleEditEvent single_edit_event = 4; + AggregatedEditEvent aggregated_edit_event = 5; + EditMonitorErrorEvent edit_monitor_error_event = 6; + } +} diff --git a/tools/edit_monitor/utils.py b/tools/edit_monitor/utils.py new file mode 100644 index 0000000000..b88949d300 --- /dev/null +++ b/tools/edit_monitor/utils.py @@ -0,0 +1,53 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import logging +import os + + +def is_feature_enabled( + feature_name: str, + user_name: str, + enable_flag: str = None, + rollout_percent: int = 100, +) -> bool: + """Determine whether the given feature is enabled. + + Whether a given feature is enabled or not depends on two flags: 1) the + enable_flag that explicitly enable/disable the feature and 2) the rollout_flag + that controls the rollout percentage. + + Args: + feature_name: name of the feature. + user_name: system user name. + enable_flag: name of the env var that enables/disables the feature + explicitly. + rollout_flg: name of the env var that controls the rollout percentage, the + value stored in the env var should be an int between 0 and 100 string + """ + if enable_flag: + if os.environ.get(enable_flag, "") == "false": + logging.info("feature: %s is disabled", feature_name) + return False + + if os.environ.get(enable_flag, "") == "true": + logging.info("feature: %s is enabled", feature_name) + return True + + hash_object = hashlib.sha256() + hash_object.update((user_name + feature_name).encode("utf-8")) + hash_number = int(hash_object.hexdigest(), 16) % 100 + + return hash_number < rollout_percent diff --git a/tools/edit_monitor/utils_test.py b/tools/edit_monitor/utils_test.py new file mode 100644 index 0000000000..1c30aa1acc --- /dev/null +++ b/tools/edit_monitor/utils_test.py @@ -0,0 +1,71 @@ +# Copyright 2024, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unittests for edit monitor utils.""" +import os +import unittest +from unittest import mock + +from edit_monitor import utils + +TEST_USER = 'test_user' +TEST_FEATURE = 'test_feature' +ENABLE_TEST_FEATURE_FLAG = 'ENABLE_TEST_FEATURE' +ROLLOUT_TEST_FEATURE_FLAG = 'ROLLOUT_TEST_FEATURE' + + +class EnableFeatureTest(unittest.TestCase): + + def test_feature_enabled_without_flag(self): + self.assertTrue(utils.is_feature_enabled(TEST_FEATURE, TEST_USER)) + + @mock.patch.dict(os.environ, {ENABLE_TEST_FEATURE_FLAG: 'false'}, clear=True) + def test_feature_disabled_with_flag(self): + self.assertFalse( + utils.is_feature_enabled( + TEST_FEATURE, TEST_USER, ENABLE_TEST_FEATURE_FLAG + ) + ) + + @mock.patch.dict(os.environ, {ENABLE_TEST_FEATURE_FLAG: 'true'}, clear=True) + def test_feature_enabled_with_flag(self): + self.assertTrue( + utils.is_feature_enabled( + TEST_FEATURE, TEST_USER, ENABLE_TEST_FEATURE_FLAG + ) + ) + + def test_feature_enabled_with_rollout_percentage(self): + self.assertTrue( + utils.is_feature_enabled( + TEST_FEATURE, + TEST_USER, + ENABLE_TEST_FEATURE_FLAG, + 90, + ) + ) + + def test_feature_disabled_with_rollout_percentage(self): + self.assertFalse( + utils.is_feature_enabled( + TEST_FEATURE, + TEST_USER, + ENABLE_TEST_FEATURE_FLAG, + 10, + ) + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/event_log_tags.py b/tools/event_log_tags.py index a6ae9f193e..e859b6b3b1 100644 --- a/tools/event_log_tags.py +++ b/tools/event_log_tags.py @@ -14,21 +14,21 @@ """A module for reading and parsing event-log-tags files.""" +import dataclasses import re import sys +from typing import Optional -class Tag(object): - __slots__ = ["tagnum", "tagname", "description", "filename", "linenum"] - - def __init__(self, tagnum, tagname, description, filename, linenum): - self.tagnum = tagnum - self.tagname = tagname - self.description = description - self.filename = filename - self.linenum = linenum +@dataclasses.dataclass +class Tag: + tagnum: int + tagname: str + description: Optional[str] + filename: str + linenum: int -class TagFile(object): +class TagFile: """Read an input event-log-tags file.""" def AddError(self, msg, linenum=None): if linenum is None: @@ -76,14 +76,11 @@ class TagFile(object): self.options[parts[1]] = parts[2:] continue - if parts[0] == "?": - tag = None - else: - try: - tag = int(parts[0]) - except ValueError: - self.AddError("\"%s\" isn't an integer tag or '?'" % (parts[0],)) - continue + try: + tag = int(parts[0]) + except ValueError: + self.AddError("\"%s\" isn't an integer tag" % (parts[0],)) + continue tagname = parts[1] if len(parts) == 3: @@ -128,8 +125,8 @@ def WriteOutput(output_file, data): out = sys.stdout output_file = "<stdout>" else: - out = open(output_file, "wb") - out.write(str.encode(data)) + out = open(output_file, "w") + out.write(data) out.close() except (IOError, OSError) as e: print("failed to write %s: %s" % (output_file, e), file=sys.stderr) diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp index ab766d6d93..3826e50ff3 100644 --- a/tools/filelistdiff/Android.bp +++ b/tools/filelistdiff/Android.bp @@ -24,4 +24,9 @@ python_binary_host { prebuilt_etc_host { name: "system_image_diff_allowlist", src: "allowlist", -}
\ No newline at end of file +} + +prebuilt_etc_host { + name: "system_image_diff_allowlist_next", + src: "allowlist_next", +} diff --git a/tools/filelistdiff/OWNERS b/tools/filelistdiff/OWNERS new file mode 100644 index 0000000000..690fb178fc --- /dev/null +++ b/tools/filelistdiff/OWNERS @@ -0,0 +1 @@ +per-file allowlist = justinyun@google.com, jeongik@google.com, kiyoungkim@google.com, inseob@google.com diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist index c4a464dd16..d8979d6983 100644 --- a/tools/filelistdiff/allowlist +++ b/tools/filelistdiff/allowlist @@ -1,49 +1,3 @@ -# Known diffs only in the KATI system image -etc/NOTICE.xml.gz -framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex -framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta -framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex -framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta -lib/aaudio-aidl-cpp.so -lib/android.hardware.biometrics.fingerprint@2.1.so -lib/android.hardware.radio.config@1.0.so -lib/android.hardware.radio.deprecated@1.0.so -lib/android.hardware.radio@1.0.so -lib/android.hardware.radio@1.1.so -lib/android.hardware.radio@1.2.so -lib/android.hardware.radio@1.3.so -lib/android.hardware.radio@1.4.so -lib/android.hardware.secure_element@1.0.so -lib/com.android.media.aaudio-aconfig-cc.so -lib/heapprofd_client.so -lib/heapprofd_client_api.so -lib/libaaudio.so -lib/libaaudio_internal.so -lib/libalarm_jni.so -lib/libamidi.so -lib/libcups.so -lib/libjni_deviceAsWebcam.so -lib/libprintspooler_jni.so -lib/libvendorsupport.so -lib/libwfds.so -lib/libyuv.so - -# b/351258461 -adb_keys +# Known diffs that are installed in either system image with the configuration +# b/353429422 init.environ.rc - -# Known diffs only in the Soong system image -lib/libhidcommand_jni.so -lib/libuinputcommand_jni.so - -# Known diffs in internal source -bin/uprobestats -etc/aconfig/flag.map -etc/aconfig/flag.val -etc/aconfig/package.map -etc/bpf/uprobestats/BitmapAllocation.o -etc/bpf/uprobestats/GenericInstrumentation.o -etc/init/UprobeStats.rc -lib/libuprobestats_client.so -lib64/libuprobestats_client.so -priv-app/DeviceDiagnostics/DeviceDiagnostics.apk
\ No newline at end of file diff --git a/tools/filelistdiff/allowlist_next b/tools/filelistdiff/allowlist_next new file mode 100644 index 0000000000..9cc7f34aec --- /dev/null +++ b/tools/filelistdiff/allowlist_next @@ -0,0 +1,3 @@ +# Allowlist only for the next release configuration. +# TODO(b/369678122): The list will be cleared when the trunk configurations are +# available to the next. diff --git a/tools/filelistdiff/file_list_diff.py b/tools/filelistdiff/file_list_diff.py index cdc5b2ee41..a6408e87cc 100644 --- a/tools/filelistdiff/file_list_diff.py +++ b/tools/filelistdiff/file_list_diff.py @@ -19,38 +19,54 @@ COLOR_WARNING = '\033[93m' COLOR_ERROR = '\033[91m' COLOR_NORMAL = '\033[0m' -def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name): +def find_unique_items(kati_installed_files, soong_installed_files, system_module_name, allowlists): with open(kati_installed_files, 'r') as kati_list_file, \ - open(soong_installed_files, 'r') as soong_list_file, \ - open(allowlist, 'r') as allowlist_file: + open(soong_installed_files, 'r') as soong_list_file: kati_files = set(kati_list_file.read().split()) soong_files = set(soong_list_file.read().split()) - allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n')))) + + allowed_files = set() + for allowlist in allowlists: + with open(allowlist, 'r') as allowlist_file: + allowed_files.update(set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n'))))) def is_unknown_diff(filepath): - return not filepath in allowed_files + return filepath not in allowed_files + + def is_unnecessary_allowlist(filepath): + return filepath not in kati_files.symmetric_difference(soong_files) unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files)) unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files)) + unnecessary_allowlists = set(filter(is_unnecessary_allowlist, allowed_files)) if unique_in_kati: - print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}') - print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}') + print('') + print(f'{COLOR_ERROR}Missing required modules in {system_module_name} module.{COLOR_NORMAL}') + print(f'To resolve this issue, please add the modules to the Android.bp file for the {system_module_name} to install the following KATI only installed files.') + print(f'You can find the correct Android.bp file using the command "gomod {system_module_name}".') + print(f'{COLOR_WARNING}KATI only installed file(s):{COLOR_NORMAL}') for item in sorted(unique_in_kati): - print(item) + print(' '+item) if unique_in_soong: - if unique_in_kati: - print('') - - print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}') - print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}') + print('') + print(f'{COLOR_ERROR}Missing packages in base_system.mk.{COLOR_NORMAL}') + print('Please add packages into build/make/target/product/base_system.mk or build/make/tools/filelistdiff/allowlist to install or skip the following Soong only installed files.') + print(f'{COLOR_WARNING}Soong only installed file(s):{COLOR_NORMAL}') for item in sorted(unique_in_soong): - print(item) + print(' '+item) + + if unnecessary_allowlists: + print('') + print(f'{COLOR_ERROR}Unnecessary files in allowlist.{COLOR_NORMAL}') + print('Please remove these entries from build/make/tools/filelistdiff/allowlist') + for item in sorted(unnecessary_allowlists): + print(' '+item) + - if unique_in_kati or unique_in_soong: + if unique_in_kati or unique_in_soong or unnecessary_allowlists: print('') - print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}') sys.exit(1) @@ -59,8 +75,8 @@ if __name__ == '__main__': parser.add_argument('kati_installed_file_list') parser.add_argument('soong_installed_file_list') - parser.add_argument('allowlist') parser.add_argument('system_module_name') + parser.add_argument('--allowlists', nargs='*', default=[]) args = parser.parse_args() - find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name)
\ No newline at end of file + find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.system_module_name, args.allowlists)
\ No newline at end of file diff --git a/tools/finalization/build-step-0-and-m.sh b/tools/finalization/build-step-0-and-m.sh new file mode 100755 index 0000000000..484380045e --- /dev/null +++ b/tools/finalization/build-step-0-and-m.sh @@ -0,0 +1,20 @@ + +#!/bin/bash +# Copyright 2024 Google Inc. All rights reserved. +set -ex +function help() { + echo "Finalize VINTF and build a target for test." + echo "usage: $(basename "$0") target [goals...]" +} +function finalize_main_step0_and_m() { + if [ $# == 0 ] ; then + help + exit 1 + fi; + local top="$(dirname "$0")"/../../../.. + source $top/build/make/tools/finalization/build-step-0.sh + local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=$1 TARGET_RELEASE=fina_0 TARGET_BUILD_VARIANT=userdebug" + # This command tests the release state for AIDL. + AIDL_FROZEN_REL=true $m ${@:2} +} +finalize_main_step0_and_m $@ diff --git a/tools/finalization/build-step-0.sh b/tools/finalization/build-step-0.sh index f81b720b2d..8826b35c0f 100755 --- a/tools/finalization/build-step-0.sh +++ b/tools/finalization/build-step-0.sh @@ -7,11 +7,26 @@ function finalize_main_step0() { local top="$(dirname "$0")"/../../../.. source $top/build/make/tools/finalization/environment.sh + local need_vintf_finalize=false if [ "$FINAL_STATE" = "unfinalized" ] ; then - # VINTF finalization + need_vintf_finalize=true + else + # build-step-0.sh tests the vintf finalization step (step-0) when the + # FINAL_BOARD_API_LEVEL is the same as the RELEASE_BOARD_API_LEVEL; and + # RELEASE_BOARD_API_LEVEL_FROZEN is not true from the fina_0 configuration. + # The FINAL_BOARD_API_LEVEL must be the next vendor API level to be finalized. + local board_api_level_vars=$(TARGET_RELEASE=fina_0 $top/build/soong/soong_ui.bash --dumpvars-mode -vars "RELEASE_BOARD_API_LEVEL_FROZEN RELEASE_BOARD_API_LEVEL") + local target_board_api_level_vars="RELEASE_BOARD_API_LEVEL_FROZEN='' +RELEASE_BOARD_API_LEVEL='$FINAL_BOARD_API_LEVEL'" + if [ "$board_api_level_vars" = "$target_board_api_level_vars" ] ; then + echo The target is a finalization candidate. + need_vintf_finalize=true + fi; + fi; + + if [ "$need_vintf_finalize" = true ] ; then # VINTF finalization source $top/build/make/tools/finalization/finalize-vintf-resources.sh fi; } finalize_main_step0 - diff --git a/tools/finalization/command-line-options.sh b/tools/finalization/command-line-options.sh index d9397c2699..3a1e0491f3 100644 --- a/tools/finalization/command-line-options.sh +++ b/tools/finalization/command-line-options.sh @@ -3,6 +3,7 @@ eval set -- "$ARGV" while true; do case "$1" in --dry-run) repo_upload_dry_run_arg="--dry-run"; repo_branch="finalization-dry-run"; shift ;; + --) shift; break;; *) break esac done diff --git a/tools/finalization/environment.sh b/tools/finalization/environment.sh index f68a68b650..0d3a9e130a 100755 --- a/tools/finalization/environment.sh +++ b/tools/finalization/environment.sh @@ -29,4 +29,8 @@ export BUILD_FROM_SOURCE_STUB=true # FINAL versions for VINTF # TODO(b/323985297): The version must match with that from the release configuration. # Instead of hardcoding the version here, read it from a release configuration. -export FINAL_BOARD_API_LEVEL='202404' +export FINAL_BOARD_API_LEVEL='202504' +export FINAL_CORRESPONDING_VERSION_LETTER='B' +export FINAL_CORRESPONDING_PLATFORM_VERSION='16' +export FINAL_NEXT_BOARD_API_LEVEL='202604' +export FINAL_NEXT_CORRESPONDING_VERSION_LETTER='C' diff --git a/tools/finalization/finalize-vintf-resources.sh b/tools/finalization/finalize-vintf-resources.sh index d532b25346..6f1a6f646e 100755 --- a/tools/finalization/finalize-vintf-resources.sh +++ b/tools/finalization/finalize-vintf-resources.sh @@ -33,18 +33,18 @@ function finalize_vintf_resources() { function create_new_compat_matrix_and_kernel_configs() { # The compatibility matrix versions are bumped during vFRC # These will change every time we have a new vFRC - local CURRENT_COMPATIBILITY_MATRIX_LEVEL='202404' - local NEXT_COMPATIBILITY_MATRIX_LEVEL='202504' + local CURRENT_COMPATIBILITY_MATRIX_LEVEL="$FINAL_BOARD_API_LEVEL" + local NEXT_COMPATIBILITY_MATRIX_LEVEL="$FINAL_NEXT_BOARD_API_LEVEL" # The kernel configs need the letter of the Android release - local CURRENT_RELEASE_LETTER='v' - local NEXT_RELEASE_LETTER='w' + local CURRENT_RELEASE_LETTER="$FINAL_CORRESPONDING_VERSION_LETTER" + local NEXT_RELEASE_LETTER="$FINAL_NEXT_CORRESPONDING_VERSION_LETTER" # build the targets required before touching the Android.bp/Android.mk files local build_cmd="$top/build/soong/soong_ui.bash --make-mode" $build_cmd bpmodify - "$top/prebuilts/build-tools/path/linux-x86/python3" "$top/hardware/interfaces/compatibility_matrices/bump.py" "$CURRENT_COMPATIBILITY_MATRIX_LEVEL" "$NEXT_COMPATIBILITY_MATRIX_LEVEL" "$CURRENT_RELEASE_LETTER" "$NEXT_RELEASE_LETTER" + "$top/prebuilts/build-tools/path/linux-x86/python3" "$top/hardware/interfaces/compatibility_matrices/bump.py" "$CURRENT_COMPATIBILITY_MATRIX_LEVEL" "$NEXT_COMPATIBILITY_MATRIX_LEVEL" "$CURRENT_RELEASE_LETTER" "$NEXT_RELEASE_LETTER" "$FINAL_CORRESPONDING_PLATFORM_VERSION" # Freeze the current framework manifest file. This relies on the # aosp_cf_x86_64-trunk_staging build target to get the right manifest diff --git a/tools/fs_config/Android.bp b/tools/fs_config/Android.bp index 6aa528963d..a5b6fd0a4c 100644 --- a/tools/fs_config/Android.bp +++ b/tools/fs_config/Android.bp @@ -277,6 +277,7 @@ genrule_defaults { out: ["out"], } +// system genrule { name: "fs_config_dirs_system_gen", defaults: ["fs_config_defaults"], @@ -307,6 +308,7 @@ prebuilt_etc { src: ":fs_config_files_system_gen", } +// system_ext genrule { name: "fs_config_dirs_system_ext_gen", defaults: ["fs_config_defaults"], @@ -337,6 +339,7 @@ prebuilt_etc { system_ext_specific: true, } +// product genrule { name: "fs_config_dirs_product_gen", defaults: ["fs_config_defaults"], @@ -367,6 +370,7 @@ prebuilt_etc { product_specific: true, } +// vendor genrule { name: "fs_config_dirs_vendor_gen", defaults: ["fs_config_defaults"], @@ -397,6 +401,7 @@ prebuilt_etc { vendor: true, } +// odm genrule { name: "fs_config_dirs_odm_gen", defaults: ["fs_config_defaults"], @@ -427,4 +432,214 @@ prebuilt_etc { device_specific: true, } -// TODO(jiyong): add fs_config for oem, system_dlkm, vendor_dlkm, odm_dlkm partitions +// system_dlkm +genrule { + name: "fs_config_dirs_system_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition system_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_system_dlkm", + filename: "fs_config_dirs", + src: ":fs_config_dirs_system_dlkm_gen", + system_dlkm_specific: true, +} + +genrule { + name: "fs_config_files_system_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition system_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_system_dlkm", + filename: "fs_config_files", + src: ":fs_config_files_system_dlkm_gen", + system_dlkm_specific: true, +} + +// vendor_dlkm +genrule { + name: "fs_config_dirs_vendor_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition vendor_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_vendor_dlkm", + filename: "fs_config_dirs", + src: ":fs_config_dirs_vendor_dlkm_gen", + vendor_dlkm_specific: true, +} + +genrule { + name: "fs_config_files_vendor_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition vendor_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_vendor_dlkm", + filename: "fs_config_files", + src: ":fs_config_files_vendor_dlkm_gen", + vendor_dlkm_specific: true, +} + +// odm_dlkm +genrule { + name: "fs_config_dirs_odm_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition odm_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_odm_dlkm", + filename: "fs_config_dirs", + src: ":fs_config_dirs_odm_dlkm_gen", + odm_dlkm_specific: true, +} + +genrule { + name: "fs_config_files_odm_dlkm_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition odm_dlkm " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_odm_dlkm", + filename: "fs_config_files", + src: ":fs_config_files_odm_dlkm_gen", + odm_dlkm_specific: true, +} + +// oem +genrule { + name: "fs_config_dirs_oem_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_dirs + + "--partition oem " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_dirs_oem", + filename: "fs_config_dirs", + src: ":fs_config_dirs_oem_gen", + oem_specific: true, +} + +genrule { + name: "fs_config_files_oem_gen", + defaults: ["fs_config_defaults"], + cmd: fs_config_cmd_files + + "--partition oem " + + "$(locations :target_fs_config_gen)", +} + +prebuilt_etc { + name: "fs_config_files_oem", + filename: "fs_config_files", + src: ":fs_config_files_oem_gen", + oem_specific: true, +} + +// Generate the <p>/etc/fs_config_dirs binary files for each partition. +// Add fs_config_dirs to PRODUCT_PACKAGES in the device make file to enable. +phony { + name: "fs_config_dirs", + required: [ + "fs_config_dirs_system", + "fs_config_dirs_system_ext", + "fs_config_dirs_product", + "fs_config_dirs_nonsystem", + ], +} + +// Generate the <p>/etc/fs_config_files binary files for each partition. +// Add fs_config_files to PRODUCT_PACKAGES in the device make file to enable. +phony { + name: "fs_config_files", + required: [ + "fs_config_files_system", + "fs_config_files_system_ext", + "fs_config_files_product", + "fs_config_files_nonsystem", + ], +} + +// Generate the <p>/etc/fs_config_dirs binary files for all enabled partitions +// excluding /system, /system_ext and /product. Add fs_config_dirs_nonsystem to +// PRODUCT_PACKAGES in the device make file to enable. +phony { + name: "fs_config_dirs_nonsystem", + required: [] + + select(soong_config_variable("fs_config", "vendor"), { + true: ["fs_config_dirs_vendor"], + default: [], + }) + + select(soong_config_variable("fs_config", "oem"), { + true: ["fs_config_dirs_oem"], + default: [], + }) + + select(soong_config_variable("fs_config", "odm"), { + true: ["fs_config_dirs_odm"], + default: [], + }) + + select(soong_config_variable("fs_config", "vendor_dlkm"), { + true: ["fs_config_dirs_vendor_dlkm"], + default: [], + }) + + select(soong_config_variable("fs_config", "odm_dlkm"), { + true: ["fs_config_dirs_odm_dlkm"], + default: [], + }) + + select(soong_config_variable("fs_config", "system_dlkm"), { + true: ["fs_config_dirs_system_dlkm"], + default: [], + }), +} + +// Generate the <p>/etc/fs_config_files binary files for all enabled partitions +// excluding /system, /system_ext and /product. Add fs_config_files_nonsystem to +// PRODUCT_PACKAGES in the device make file to enable. +phony { + name: "fs_config_files_nonsystem", + required: [] + + select(soong_config_variable("fs_config", "vendor"), { + true: ["fs_config_files_vendor"], + default: [], + }) + + select(soong_config_variable("fs_config", "oem"), { + true: ["fs_config_files_oem"], + default: [], + }) + + select(soong_config_variable("fs_config", "odm"), { + true: ["fs_config_files_odm"], + default: [], + }) + + select(soong_config_variable("fs_config", "vendor_dlkm"), { + true: ["fs_config_files_vendor_dlkm"], + default: [], + }) + + select(soong_config_variable("fs_config", "odm_dlkm"), { + true: ["fs_config_files_odm_dlkm"], + default: [], + }) + + select(soong_config_variable("fs_config", "system_dlkm"), { + true: ["fs_config_files_system_dlkm"], + default: [], + }), +} diff --git a/tools/fs_config/Android.mk b/tools/fs_config/Android.mk deleted file mode 100644 index e4c362630f..0000000000 --- a/tools/fs_config/Android.mk +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -LOCAL_PATH := $(call my-dir) - -# One can override the default android_filesystem_config.h file by using TARGET_FS_CONFIG_GEN. -# Set TARGET_FS_CONFIG_GEN to contain a list of intermediate format files -# for generating the android_filesystem_config.h file. -# -# More information can be found in the README - -ifneq ($(wildcard $(TARGET_DEVICE_DIR)/android_filesystem_config.h),) -$(error Using $(TARGET_DEVICE_DIR)/android_filesystem_config.h is deprecated, please use TARGET_FS_CONFIG_GEN instead) -endif - -android_filesystem_config := system/core/libcutils/include/private/android_filesystem_config.h -capability_header := bionic/libc/kernel/uapi/linux/capability.h - -# List of supported vendor, oem, odm, vendor_dlkm, odm_dlkm, and system_dlkm Partitions -fs_config_generate_extra_partition_list := $(strip \ - $(if $(BOARD_USES_VENDORIMAGE)$(BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE),vendor) \ - $(if $(BOARD_USES_OEMIMAGE)$(BOARD_OEMIMAGE_FILE_SYSTEM_TYPE),oem) \ - $(if $(BOARD_USES_ODMIMAGE)$(BOARD_ODMIMAGE_FILE_SYSTEM_TYPE),odm) \ - $(if $(BOARD_USES_VENDOR_DLKMIMAGE)$(BOARD_VENDOR_DLKMIMAGE_FILE_SYSTEM_TYPE),vendor_dlkm) \ - $(if $(BOARD_USES_ODM_DLKMIMAGE)$(BOARD_ODM_DLKMIMAGE_FILE_SYSTEM_TYPE),odm_dlkm) \ - $(if $(BOARD_USES_SYSTEM_DLKMIMAGE)$(BOARD_SYSTEM_DLKMIMAGE_FILE_SYSTEM_TYPE),system_dlkm) \ -) - -################################## -# Generate the <p>/etc/fs_config_dirs binary files for each partition. -# Add fs_config_dirs to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := \ - fs_config_dirs_system \ - fs_config_dirs_system_ext \ - fs_config_dirs_product \ - fs_config_dirs_nonsystem -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the <p>/etc/fs_config_files binary files for each partition. -# Add fs_config_files to PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := \ - fs_config_files_system \ - fs_config_files_system_ext \ - fs_config_files_product \ - fs_config_files_nonsystem -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the <p>/etc/fs_config_dirs binary files for all enabled partitions -# excluding /system, /system_ext and /product. Add fs_config_dirs_nonsystem to -# PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_nonsystem -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),fs_config_dirs_$(t)) -include $(BUILD_PHONY_PACKAGE) - -################################## -# Generate the <p>/etc/fs_config_files binary files for all enabled partitions -# excluding /system, /system_ext and /product. Add fs_config_files_nonsystem to -# PRODUCT_PACKAGES in the device make file to enable. -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_nonsystem -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_REQUIRED_MODULES := $(foreach t,$(fs_config_generate_extra_partition_list),fs_config_files_$(t)) -include $(BUILD_PHONY_PACKAGE) - -ifneq ($(filter oem,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the oem/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_oem -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_OEM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition oem \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the oem/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_oem -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_OEM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition oem \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - -ifneq ($(filter vendor_dlkm,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the vendor_dlkm/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_vendor_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition vendor_dlkm \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the vendor_dlkm/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES in -# the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_vendor_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_VENDOR_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition vendor_dlkm \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - -ifneq ($(filter odm_dlkm,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the odm_dlkm/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_odm_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition odm_dlkm \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the odm_dlkm/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_odm_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_ODM_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition odm_dlkm \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - -ifneq ($(filter system_dlkm,$(fs_config_generate_extra_partition_list)),) -################################## -# Generate the system_dlkm/etc/fs_config_dirs binary file for the target -# Add fs_config_dirs or fs_config_dirs_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_dirs_system_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_dirs -LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system_dlkm \ - --dirs \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -################################## -# Generate the system_dlkm/etc/fs_config_files binary file for the target -# Add fs_config_files or fs_config_files_nonsystem to PRODUCT_PACKAGES -# in the device make file to enable -include $(CLEAR_VARS) - -LOCAL_MODULE := fs_config_files_system_dlkm -LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0 -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE -LOCAL_MODULE_CLASS := ETC -LOCAL_INSTALLED_MODULE_STEM := fs_config_files -LOCAL_MODULE_PATH := $(TARGET_OUT_SYSTEM_DLKM)/etc -include $(BUILD_SYSTEM)/base_rules.mk -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_FS_HDR := $(android_filesystem_config) -$(LOCAL_BUILT_MODULE): PRIVATE_ANDROID_CAP_HDR := $(capability_header) -$(LOCAL_BUILT_MODULE): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN) -$(LOCAL_BUILT_MODULE): $(LOCAL_PATH)/fs_config_generator.py $(TARGET_FS_CONFIG_GEN) $(android_filesystem_config) $(capability_header) - @mkdir -p $(dir $@) - $< fsconfig \ - --aid-header $(PRIVATE_ANDROID_FS_HDR) \ - --capability-header $(PRIVATE_ANDROID_CAP_HDR) \ - --partition system_dlkm \ - --files \ - --out_file $@ \ - $(or $(PRIVATE_TARGET_FS_CONFIG_GEN),/dev/null) - -endif - -android_filesystem_config := -capability_header := -fs_config_generate_extra_partition_list := diff --git a/tools/ide_query/cc_analyzer/README.md b/tools/ide_query/cc_analyzer/README.md new file mode 100644 index 0000000000..7b822d205f --- /dev/null +++ b/tools/ide_query/cc_analyzer/README.md @@ -0,0 +1,3 @@ +See instructions in +[Android Clang/LLVM-based Tools Readme Doc](https://android.googlesource.com/platform/prebuilts/clang-tools/+/main/README.md) +for cutting a new release. diff --git a/tools/ide_query/cc_analyzer/include_scanner.cc b/tools/ide_query/cc_analyzer/include_scanner.cc index 8916a3edd6..1d3f26e737 100644 --- a/tools/ide_query/cc_analyzer/include_scanner.cc +++ b/tools/ide_query/cc_analyzer/include_scanner.cc @@ -94,6 +94,11 @@ class IncludeScanningAction final : public clang::PreprocessOnlyAction { std::unordered_map<std::string, std::string> &abs_paths) : abs_paths_(abs_paths) {} bool BeginSourceFileAction(clang::CompilerInstance &ci) override { + // Be more resilient against all warnings/errors, as we want + // include-scanning to work even on incomplete sources. + ci.getDiagnostics().setEnableAllWarnings(false); + ci.getDiagnostics().setSeverityForAll(clang::diag::Flavor::WarningOrError, + clang::diag::Severity::Ignored); std::string cwd; auto cwd_or_err = ci.getVirtualFileSystem().getCurrentWorkingDirectory(); if (!cwd_or_err || cwd_or_err.get().empty()) return false; @@ -154,6 +159,8 @@ llvm::Expected<std::vector<std::pair<std::string, std::string>>> ScanIncludes( main_file.get()->getBuffer().str()); std::vector<std::string> argv = cmd.CommandLine; + // Disable all warnings to be more robust in analysis. + argv.insert(llvm::find(argv, "--"), {"-Wno-error", "-w"}); fs = OverlayBuiltinHeaders(argv, std::move(fs)); llvm::IntrusiveRefCntPtr<clang::FileManager> files( diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go index 23c7abd2a0..6caa29c1f3 100644 --- a/tools/ide_query/ide_query.go +++ b/tools/ide_query/ide_query.go @@ -116,8 +116,8 @@ func main() { var targets []string javaTargetsByFile := findJavaModules(javaFiles, javaModules) - for _, t := range javaTargetsByFile { - targets = append(targets, t) + for _, target := range javaTargetsByFile { + targets = append(targets, javaModules[target].Jars...) } ccTargets, err := getCCTargets(ctx, env, ccFiles) @@ -293,11 +293,23 @@ func getCCInputs(ctx context.Context, env Env, filePaths []string) ([]*pb.Analys // If a file is covered by multiple modules, the first module is returned. func findJavaModules(paths []string, modules map[string]*javaModule) map[string]string { ret := make(map[string]string) - for name, module := range modules { + // A file may be part of multiple modules. To make the result deterministic, + // check the modules in sorted order. + keys := make([]string, 0, len(modules)) + for name := range modules { + keys = append(keys, name) + } + slices.Sort(keys) + for _, name := range keys { if strings.HasSuffix(name, ".impl") { continue } + module := modules[name] + if len(module.Jars) == 0 { + continue + } + for i, p := range paths { if slices.Contains(module.Srcs, p) { ret[p] = name @@ -309,6 +321,7 @@ func findJavaModules(paths []string, modules map[string]*javaModule) map[string] break } } + return ret } @@ -341,6 +354,8 @@ func getJavaInputs(env Env, modulesByPath map[string]string, modules map[string] Id: moduleName, Language: pb.Language_LANGUAGE_JAVA, SourceFilePaths: m.Srcs, + GeneratedFiles: genFiles(env, m), + DependencyIds: m.Deps, } unitsById[u.Id] = u @@ -355,14 +370,11 @@ func getJavaInputs(env Env, modulesByPath map[string]string, modules map[string] continue } - var paths []string - paths = append(paths, mod.Srcs...) - paths = append(paths, mod.SrcJars...) - paths = append(paths, mod.Jars...) unitsById[name] = &pb.BuildableUnit{ Id: name, SourceFilePaths: mod.Srcs, - GeneratedFiles: genFiles(env, paths), + GeneratedFiles: genFiles(env, mod), + DependencyIds: mod.Deps, } for _, d := range mod.Deps { @@ -379,8 +391,13 @@ func getJavaInputs(env Env, modulesByPath map[string]string, modules map[string] } // genFiles returns the generated files (paths that start with outDir/) for the -// given paths. Generated files that do not exist are ignored. -func genFiles(env Env, paths []string) []*pb.GeneratedFile { +// given module. Generated files that do not exist are ignored. +func genFiles(env Env, mod *javaModule) []*pb.GeneratedFile { + var paths []string + paths = append(paths, mod.Srcs...) + paths = append(paths, mod.SrcJars...) + paths = append(paths, mod.Jars...) + prefix := env.OutDir + "/" var ret []*pb.GeneratedFile for _, p := range paths { diff --git a/tools/ide_query/ide_query.sh b/tools/ide_query/ide_query.sh index 6f9b0c4b8b..8dfffc1cfa 100755 --- a/tools/ide_query/ide_query.sh +++ b/tools/ide_query/ide_query.sh @@ -19,7 +19,7 @@ source $(pwd)/../../shell_utils.sh require_top # Ensure cogsetup (out/ will be symlink outside the repo) -. ${TOP}/build/make/cogsetup.sh +setup_cog_env_if_needed case $(uname -s) in Linux) diff --git a/tools/ide_query/prober_scripts/cpp/general.cc b/tools/ide_query/prober_scripts/cpp/general.cc index 0f0639be5e..ac882829c0 100644 --- a/tools/ide_query/prober_scripts/cpp/general.cc +++ b/tools/ide_query/prober_scripts/cpp/general.cc @@ -56,7 +56,7 @@ void TestCompletion() { void TestNavigation() { std::vector<int> ints; - // | | ints + // ^ ^ ints // ^ // step diff --git a/tools/ide_query/prober_scripts/ide_query.out b/tools/ide_query/prober_scripts/ide_query.out index cd7ce6d258..be48da1424 100644 --- a/tools/ide_query/prober_scripts/ide_query.out +++ b/tools/ide_query/prober_scripts/ide_query.out @@ -1,7 +1,9 @@ -out–a -8build/make/tools/ide_query/prober_scripts/cpp/general.cc8prebuilts/clang/host/linux-x86/clang-r522817/bin/clang++-mthumb-Os-fomit-frame-pointer-mllvm-enable-shrink-wrap=false-O2-Wall-Wextra -Winit-self-Wpointer-arith-Wunguarded-availability-Werror=date-time-Werror=int-conversion-Werror=pragma-pack&-Werror=pragma-pack-suspicious-include-Werror=sizeof-array-div-Werror=string-plus-int'-Werror=unreachable-code-loop-increment"-Wno-error=deprecated-declarations-Wno-c99-designator-Wno-gnu-folding-constant"-Wno-inconsistent-missing-override -Wno-error=reorder-init-list-Wno-reorder-init-list-Wno-sign-compare -Wno-unused -DANDROID-DNDEBUG-UDEBUG(-D__compiler_offsetof=__builtin_offsetof*-D__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__ -faddrsig-fdebug-default-version=5-fcolor-diagnostics-ffp-contract=off-fno-exceptions-fno-strict-aliasing-fmessage-length=0#-fno-relaxed-template-template-args-gsimple-template-names-gz=zstd-no-canonical-prefixes-Wno-error=format"-fdebug-prefix-map=/proc/self/cwd= -ftrivial-auto-var-init=zero-g-ffunction-sections-fdata-sections-fno-short-enums-funwind-tables-fstack-protector-strong-Wa,--noexecstack-D_FORTIFY_SOURCE=2-Wstrict-aliasing=2-Werror=return-type-Werror=non-virtual-dtor-Werror=address-Werror=sequence-point-Werror=format-security -nostdlibinc-fdebug-info-for-profiling -msoft-float-march=armv7-a-mfloat-abi=softfp --mfpu=neon/-Ibuild/make/tools/ide_query/prober_scripts/cpp³-Iout/soong/.intermediates/build/make/tools/ide_query/prober_scripts/cpp/ide_query_proberscript_cc/android_arm_armv7-a-neon/gen/proto/build/make/tools/ide_query/prober_scripts/cpp…-Iout/soong/.intermediates/build/make/tools/ide_query/prober_scripts/cpp/ide_query_proberscript_cc/android_arm_armv7-a-neon/gen/proto-D__LIBC_API__=10000-D__LIBM_API__=10000-D__LIBDL_API__=10000-Iexternal/protobuf/srcY-Iprebuilts/clang/host/linux-x86/clang-r522817/android_libc++/platform/arm/include/c++/v1=-Iprebuilts/clang/host/linux-x86/clang-r522817/include/c++/v1 -Ibionic/libc/async_safe/include -Isystem/logging/liblog/include'-Ibionic/libc/system_properties/include<-Isystem/core/property_service/libpropertyinfoparser/include-isystembionic/libc/include-isystem bionic/libc/kernel/uapi/asm-arm-isystembionic/libc/kernel/uapi-isystem bionic/libc/kernel/android/scsi-isystem bionic/libc/kernel/android/uapi-target armv7a-linux-androideabi10000-DANDROID_STRICT-fPIE-Werror-Wno-unused-parameter-DGOOGLE_PROTOBUF_NO_RTTI-Wimplicit-fallthrough*-D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS-Wno-gnu-include-next-fvisibility-inlines-hidden-mllvm-enable-shrink-wrap=false -std=gnu++20 -fno-rtti-Isystem/core/include -Isystem/logging/liblog/include -Isystem/media/audio/include -Ihardware/libhardware/include%-Ihardware/libhardware_legacy/include-Ihardware/ril/include-Iframeworks/native/include"-Iframeworks/native/opengl/include-Iframeworks/av/include-Werror=bool-operation -Werror=format-insufficient-args%-Werror=implicit-int-float-conversion-Werror=int-in-bool-context-Werror=int-to-pointer-cast-Werror=pointer-to-int-cast-Werror=xor-used-as-pow -Wno-void-pointer-to-enum-cast -Wno-void-pointer-to-int-cast-Wno-pointer-to-int-cast-Werror=fortify-source-Wno-unused-variable -Wno-missing-field-initializers-Wno-packed-non-pod -Werror=address-of-temporary+-Werror=incompatible-function-pointer-types-Werror=null-dereference-Werror=return-type"-Wno-tautological-constant-compare$-Wno-tautological-type-limit-compare"-Wno-implicit-int-float-conversion!-Wno-tautological-overlap-compare-Wno-deprecated-copy-Wno-range-loop-construct"-Wno-zero-as-null-pointer-constant)-Wno-deprecated-anon-enum-enum-conversion$-Wno-deprecated-enum-enum-conversion-Wno-pessimizing-move -Wno-non-c-typedef-for-linkage-Wno-align-mismatch"-Wno-error=unused-but-set-variable#-Wno-error=unused-but-set-parameter -Wno-error=deprecated-builtins-Wno-error=deprecated2-Wno-error=single-bit-bitfield-constant-conversion$-Wno-error=enum-constexpr-conversion-Wno-error=invalid-offsetof&-Wno-deprecated-dynamic-exception-spec8build/make/tools/ide_query/prober_scripts/cpp/general.cc"Õ? +out2x +8build/make/tools/ide_query/prober_scripts/cpp/general.cc8build/make/tools/ide_query/prober_scripts/cpp/general.cc:—" +8build/make/tools/ide_query/prober_scripts/cpp/general.cc8build/make/tools/ide_query/prober_scripts/cpp/general.cc"8prebuilts/clang/host/linux-x86/clang-r530567/bin/clang++" -nostdlibinc"-mthumb"-Os"-fomit-frame-pointer"-mllvm"-enable-shrink-wrap=false"-O2"-Wall"-Wextra" -Winit-self"-Wpointer-arith"-Wunguarded-availability"-Werror=date-time"-Werror=int-conversion"-Werror=pragma-pack"&-Werror=pragma-pack-suspicious-include"-Werror=sizeof-array-div"-Werror=string-plus-int"'-Werror=unreachable-code-loop-increment""-Wno-error=deprecated-declarations"-Wno-c23-extensions"-Wno-c99-designator"-Wno-gnu-folding-constant""-Wno-inconsistent-missing-override" -Wno-error=reorder-init-list"-Wno-reorder-init-list"-Wno-sign-compare" -Wno-unused" -DANDROID"-DNDEBUG"-UDEBUG"(-D__compiler_offsetof=__builtin_offsetof"*-D__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__" -faddrsig"-fdebug-default-version=5"-fcolor-diagnostics"-ffp-contract=off"-fno-exceptions"-fno-strict-aliasing"-fmessage-length=0"-gsimple-template-names"-gz=zstd"-no-canonical-prefixes""-fdebug-prefix-map=/proc/self/cwd=" -ftrivial-auto-var-init=zero"-g"-ffunction-sections"-fdata-sections"-fno-short-enums"-funwind-tables"-fstack-protector-strong"-Wa,--noexecstack"-D_FORTIFY_SOURCE=2"-Wstrict-aliasing=2"-Werror=return-type"-Werror=non-virtual-dtor"-Werror=address"-Werror=sequence-point"-Werror=format-security" -msoft-float"-march=armv7-a"-mfloat-abi=softfp" +-mfpu=neon"/-Ibuild/make/tools/ide_query/prober_scripts/cpp"³-Iout/soong/.intermediates/build/make/tools/ide_query/prober_scripts/cpp/ide_query_proberscript_cc/android_arm_armv7-a-neon/gen/proto/build/make/tools/ide_query/prober_scripts/cpp"…-Iout/soong/.intermediates/build/make/tools/ide_query/prober_scripts/cpp/ide_query_proberscript_cc/android_arm_armv7-a-neon/gen/proto"-D__LIBC_API__=10000"-D__LIBM_API__=10000"-D__LIBDL_API__=10000"-Iexternal/protobuf/src"Y-Iprebuilts/clang/host/linux-x86/clang-r530567/android_libc++/platform/arm/include/c++/v1"=-Iprebuilts/clang/host/linux-x86/clang-r530567/include/c++/v1" -Ibionic/libc/async_safe/include" -Isystem/logging/liblog/include"'-Ibionic/libc/system_properties/include"<-Isystem/core/property_service/libpropertyinfoparser/include"-isystem"bionic/libc/include"-isystem" bionic/libc/kernel/uapi/asm-arm"-isystem"bionic/libc/kernel/uapi"-isystem" bionic/libc/kernel/android/scsi"-isystem" bionic/libc/kernel/android/uapi"-target" armv7a-linux-androideabi10000"-DANDROID_STRICT"-fPIE"-Werror"-Wno-unused-parameter"-DGOOGLE_PROTOBUF_NO_RTTI"-Wimplicit-fallthrough"*-D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS"-Wno-gnu-include-next"-fvisibility-inlines-hidden"-mllvm"-enable-shrink-wrap=false" -std=gnu++20" -fno-rtti"-Isystem/core/include" -Isystem/logging/liblog/include" -Isystem/media/audio/include" -Ihardware/libhardware/include"%-Ihardware/libhardware_legacy/include"-Ihardware/ril/include"-Iframeworks/native/include""-Iframeworks/native/opengl/include"-Iframeworks/av/include"-Werror=bool-operation" -Werror=format-insufficient-args"%-Werror=implicit-int-float-conversion"-Werror=int-in-bool-context"-Werror=int-to-pointer-cast"-Werror=pointer-to-int-cast"-Werror=xor-used-as-pow" -Wno-void-pointer-to-enum-cast" -Wno-void-pointer-to-int-cast"-Wno-pointer-to-int-cast"-Werror=fortify-source"-Wno-unused-variable" -Wno-missing-field-initializers"-Wno-packed-non-pod" -Werror=address-of-temporary"+-Werror=incompatible-function-pointer-types"-Werror=null-dereference"-Werror=return-type""-Wno-tautological-constant-compare"$-Wno-tautological-type-limit-compare""-Wno-implicit-int-float-conversion"!-Wno-tautological-overlap-compare"-Wno-deprecated-copy"-Wno-range-loop-construct""-Wno-zero-as-null-pointer-constant")-Wno-deprecated-anon-enum-enum-conversion"$-Wno-deprecated-enum-enum-conversion"-Wno-error=pessimizing-move" -Wno-non-c-typedef-for-linkage"-Wno-align-mismatch""-Wno-error=unused-but-set-variable"#-Wno-error=unused-but-set-parameter" -Wno-error=deprecated-builtins"-Wno-error=deprecated"&-Wno-deprecated-dynamic-exception-spec"$-Wno-error=enum-constexpr-conversion"-Wno-error=invalid-offsetof")-Wno-error=thread-safety-reference-return"-Wno-vla-cxx-extension"8build/make/tools/ide_query/prober_scripts/cpp/general.cc2Egenfiles_for_build/make/tools/ide_query/prober_scripts/cpp/general.cc:Ÿ@ +Egenfiles_for_build/make/tools/ide_query/prober_scripts/cpp/general.cc*Õ? ¶soong/.intermediates/build/make/tools/ide_query/prober_scripts/cpp/ide_query_proberscript_cc/android_arm_armv7-a-neon/gen/proto/build/make/tools/ide_query/prober_scripts/cpp/foo.pb.h™>// Generated by the protocol buffer compiler. DO NOT EDIT! // source: build/make/tools/ide_query/prober_scripts/cpp/foo.proto diff --git a/tools/ide_query/prober_scripts/jvm/Android.bp b/tools/ide_query/prober_scripts/jvm/Android.bp new file mode 100644 index 0000000000..84d00b52fd --- /dev/null +++ b/tools/ide_query/prober_scripts/jvm/Android.bp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_library { + name: "ide_query_proberscript_jvm", + srcs: [ + "Foo.java", + "Bar.java", + "other/Other.java", + ], +} diff --git a/tools/ide_query/prober_scripts/jvm/Bar.java b/tools/ide_query/prober_scripts/jvm/Bar.java new file mode 100644 index 0000000000..8d51576901 --- /dev/null +++ b/tools/ide_query/prober_scripts/jvm/Bar.java @@ -0,0 +1,32 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jvm; + +/** Bar class. The class for testing code assist within the same build module. */ +class Bar<K extends Number, V extends Number> { + Bar() { + foo(new Foo()); + } + + void foo(Foo f) {} + + void foo(Object o) {} + + void bar(Foo f) {} + + void baz(Object o) {} +}
\ No newline at end of file diff --git a/tools/ide_query/prober_scripts/jvm/Foo.java b/tools/ide_query/prober_scripts/jvm/Foo.java index a043f72e32..2c8ceb62db 100644 --- a/tools/ide_query/prober_scripts/jvm/Foo.java +++ b/tools/ide_query/prober_scripts/jvm/Foo.java @@ -16,22 +16,109 @@ package jvm; -import java.util.ArrayList; -import java.util.HashSet; +import jvm.other.Other; /** Foo class. */ public final class Foo { +// ^ ^ foo_def + + void testParameterInfo() { + // Test signature help for type parameters. + + Bar<Integer, Double> b = new Bar<>(); + // ^ ctor + // ^ decl_1 + // ^ decl_2 + System.out.println(b); + + // step at ctor + // workspace.waitForReady() + // paraminfo.trigger() + // assert paraminfo.items.filter( + // label="K extends Number, V extends Number", + // selection="K extends Number", + // ) + + // step at decl_1 + // workspace.waitForReady() + // paraminfo.trigger() + // assert paraminfo.items.filter( + // label="K extends Number, V extends Number", + // selection="K extends Number", + // ) + + // step at decl_2 + // workspace.waitForReady() + // paraminfo.trigger() + // assert paraminfo.items.filter( + // label="K extends Number, V extends Number", + // selection="V extends Number", + // ) + + // Test signature help for constructor parameters. + + Other other = new Other(123, "foo"); + // ^ param_1 + // ^ param_2 + System.out.println(other); + + // step at param_1 + // workspace.waitForReady() + // paraminfo.trigger() + // assert paraminfo.items.filter( + // label="\\(int first, String second\\)", + // selection="int first", + // ) + + // step at param_2 + // workspace.waitForReady() + // paraminfo.trigger() + // assert paraminfo.items.empty() + } void testCompletion() { - ArrayList<Integer> list = new ArrayList<>(); - System.out.println(list); + Bar<Integer, Double> b = new Bar<>(); + System.out.println(b); // ^ // step - // ; Test completion on the standard types. - // type("list.") + // ; Test completion on types from the same package. + // workspace.waitForReady() + // type("b.") // completion.trigger() - // assert completion.items.filter(label="add.*") + // assert completion.items.filter(label="foo.*") + // delline() + + Other other = new Other(1, "foo"); + System.out.println(other); + + // ^ + + // step + // ; Test completion on types from a different package. + // workspace.waitForReady() + // type("other.") + // completion.trigger() + // apply(completion.items.filter(label="other.*").first()) + // type(".") + // completion.trigger() + // apply(completion.items.filter(label="other.*").first()) + // delline() + } + + void testDiagnostics() { + + // ^ + + // step + // ; Test diagnostics about wrong type argument bounds. + // workspace.waitForReady() + // type("Bar<String, Double> b;") + // assert diagnostics.items.filter( + // message="type argument .* is not within bounds .*", + // code="compiler.err.not.within.bounds", + // ) + // delline() } } diff --git a/tools/ide_query/prober_scripts/jvm/ide_query.out b/tools/ide_query/prober_scripts/jvm/ide_query.out new file mode 100644 index 0000000000..af9fb86e83 --- /dev/null +++ b/tools/ide_query/prober_scripts/jvm/ide_query.out @@ -0,0 +1,4 @@ + +out2X +6build/make/tools/ide_query/prober_scripts/jvm/Foo.javaide_query_proberscript_jvm:Î +ide_query_proberscript_jvm6build/make/tools/ide_query/prober_scripts/jvm/Foo.java6build/make/tools/ide_query/prober_scripts/jvm/Bar.java>build/make/tools/ide_query/prober_scripts/jvm/other/Other.java
\ No newline at end of file diff --git a/tools/ide_query/prober_scripts/jvm/other/Other.java b/tools/ide_query/prober_scripts/jvm/other/Other.java new file mode 100644 index 0000000000..822662a66e --- /dev/null +++ b/tools/ide_query/prober_scripts/jvm/other/Other.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jvm.other; + +/** Other class */ +public class Other { + public Other(int first, String second) {} + + public Other other() { + return new Other(0, ""); + } +} diff --git a/tools/ide_query/prober_scripts/regen.sh b/tools/ide_query/prober_scripts/regen.sh index 2edfe53ec3..04a02640d7 100755 --- a/tools/ide_query/prober_scripts/regen.sh +++ b/tools/ide_query/prober_scripts/regen.sh @@ -21,13 +21,8 @@ # ide_query.sh. The prober doesn't check-out the full source code, so it # can't run ide_query.sh itself. -cd $(dirname $BASH_SOURCE) -source $(pwd)/../../../shell_utils.sh -require_top - files_to_build=( build/make/tools/ide_query/prober_scripts/cpp/general.cc ) -cd ${TOP} build/make/tools/ide_query/ide_query.sh --lunch_target=aosp_arm-trunk_staging-eng ${files_to_build[@]} > build/make/tools/ide_query/prober_scripts/ide_query.out diff --git a/tools/java-event-log-tags.py b/tools/java-event-log-tags.py index bbd65fa4bf..e3dc07e4ab 100755 --- a/tools/java-event-log-tags.py +++ b/tools/java-event-log-tags.py @@ -15,16 +15,12 @@ # limitations under the License. """ -Usage: java-event-log-tags.py [-o output_file] <input_file> <merged_tags_file> - Generate a java class containing constants for each of the event log tags in the given input file. - --h to display this usage message and exit. """ from io import StringIO -import getopt +import argparse import os import os.path import re @@ -32,57 +28,14 @@ import sys import event_log_tags -output_file = None - -try: - opts, args = getopt.getopt(sys.argv[1:], "ho:") -except getopt.GetoptError as err: - print(str(err)) - print(__doc__) - sys.exit(2) - -for o, a in opts: - if o == "-h": - print(__doc__) - sys.exit(2) - elif o == "-o": - output_file = a - else: - print("unhandled option %s" % (o,), file=sys.stderr) - sys.exit(1) - -if len(args) != 1 and len(args) != 2: - print("need one or two input files, not %d" % (len(args),)) - print(__doc__) - sys.exit(1) +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('-o', dest='output_file') +parser.add_argument('file') +args = parser.parse_args() -fn = args[0] +fn = args.file tagfile = event_log_tags.TagFile(fn) -if len(args) > 1: - # Load the merged tag file (which should have numbers assigned for all - # tags. Use the numbers from the merged file to fill in any missing - # numbers from the input file. - merged_fn = args[1] - merged_tagfile = event_log_tags.TagFile(merged_fn) - merged_by_name = dict([(t.tagname, t) for t in merged_tagfile.tags]) - for t in tagfile.tags: - if t.tagnum is None: - if t.tagname in merged_by_name: - t.tagnum = merged_by_name[t.tagname].tagnum - else: - # We're building something that's not being included in the - # product, so its tags don't appear in the merged file. Assign - # them all an arbitrary number so we can emit the java and - # compile the (unused) package. - t.tagnum = 999999 -else: - # Not using the merged tag file, so all tags must have manually assigned - # numbers - for t in tagfile.tags: - if t.tagnum is None: - tagfilef.AddError("tag \"%s\" has no number" % (tagname,), tag.linenum) - if "java_package" not in tagfile.options: tagfile.AddError("java_package option not specified", linenum=0) @@ -141,11 +94,11 @@ javaTypes = ["ERROR", "int", "long", "String", "Object[]", "float"] for t in tagfile.tags: methodName = javaName("write_" + t.tagname) if t.description: - args = [arg.strip("() ").split("|") for arg in t.description.split(",")] + fn_args = [arg.strip("() ").split("|") for arg in t.description.split(",")] else: - args = [] - argTypesNames = ", ".join([javaTypes[int(arg[1])] + " " + javaName(arg[0]) for arg in args]) - argNames = "".join([", " + javaName(arg[0]) for arg in args]) + fn_args = [] + argTypesNames = ", ".join([javaTypes[int(arg[1])] + " " + javaName(arg[0]) for arg in fn_args]) + argNames = "".join([", " + javaName(arg[0]) for arg in fn_args]) buffer.write("\n public static void %s(%s) {" % (methodName, argTypesNames)) buffer.write("\n android.util.EventLog.writeEvent(%s%s);" % (t.tagname.upper(), argNames)) buffer.write("\n }\n") @@ -153,8 +106,8 @@ for t in tagfile.tags: buffer.write("}\n"); -output_dir = os.path.dirname(output_file) +output_dir = os.path.dirname(args.output_file) if not os.path.exists(output_dir): os.makedirs(output_dir) -event_log_tags.WriteOutput(output_file, buffer) +event_log_tags.WriteOutput(args.output_file, buffer) diff --git a/tools/merge-event-log-tags.py b/tools/merge-event-log-tags.py index 292604c469..5730c11c43 100755 --- a/tools/merge-event-log-tags.py +++ b/tools/merge-event-log-tags.py @@ -15,22 +15,13 @@ # limitations under the License. """ -Usage: merge-event-log-tags.py [-o output_file] [input_files...] - Merge together zero or more event-logs-tags files to produce a single output file, stripped of comments. Checks that no tag numbers conflict and fails if they do. - --h to display this usage message and exit. """ from io import StringIO -import getopt -try: - import hashlib -except ImportError: - import md5 as hashlib -import struct +import argparse import sys import event_log_tags @@ -38,32 +29,10 @@ import event_log_tags errors = [] warnings = [] -output_file = None -pre_merged_file = None - -# Tags with a tag number of ? are assigned a tag in the range -# [ASSIGN_START, ASSIGN_LIMIT). -ASSIGN_START = 900000 -ASSIGN_LIMIT = 1000000 - -try: - opts, args = getopt.getopt(sys.argv[1:], "ho:m:") -except getopt.GetoptError as err: - print(str(err)) - print(__doc__) - sys.exit(2) - -for o, a in opts: - if o == "-h": - print(__doc__) - sys.exit(2) - elif o == "-o": - output_file = a - elif o == "-m": - pre_merged_file = a - else: - print("unhandled option %s" % (o,), file=sys.stderr) - sys.exit(1) +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('-o', dest='output_file') +parser.add_argument('files', nargs='*') +args = parser.parse_args() # Restrictions on tags: # @@ -77,12 +46,7 @@ for o, a in opts: by_tagname = {} by_tagnum = {} -pre_merged_tags = {} -if pre_merged_file: - for t in event_log_tags.TagFile(pre_merged_file).tags: - pre_merged_tags[t.tagname] = t - -for fn in args: +for fn in args.files: tagfile = event_log_tags.TagFile(fn) for t in tagfile.tags: @@ -93,12 +57,6 @@ for fn in args: if t.tagname in by_tagname: orig = by_tagname[t.tagname] - # Allow an explicit tag number to define an implicit tag number - if orig.tagnum is None: - orig.tagnum = t.tagnum - elif t.tagnum is None: - t.tagnum = orig.tagnum - if (t.tagnum == orig.tagnum and t.description == orig.description): # if the name and description are identical, issue a warning @@ -114,7 +72,7 @@ for fn in args: linenum=t.linenum) continue - if t.tagnum is not None and t.tagnum in by_tagnum: + if t.tagnum in by_tagnum: orig = by_tagnum[t.tagnum] if t.tagname != orig.tagname: @@ -125,8 +83,7 @@ for fn in args: continue by_tagname[t.tagname] = t - if t.tagnum is not None: - by_tagnum[t.tagnum] = t + by_tagnum[t.tagnum] = t errors.extend(tagfile.errors) warnings.extend(tagfile.warnings) @@ -140,38 +97,6 @@ if warnings: for fn, ln, msg in warnings: print("%s:%d: warning: %s" % (fn, ln, msg), file=sys.stderr) -# Python's hash function (a) isn't great and (b) varies between -# versions of python. Using md5 is overkill here but is the same from -# platform to platform and speed shouldn't matter in practice. -def hashname(str): - d = hashlib.md5(str).digest()[:4] - return struct.unpack("!I", d)[0] - -# Assign a tag number to all the entries that say they want one -# assigned. We do this based on a hash of the tag name so that the -# numbers should stay relatively stable as tags are added. - -# If we were provided pre-merged tags (w/ the -m option), then don't -# ever try to allocate one, just fail if we don't have a number - -for name, t in sorted(by_tagname.items()): - if t.tagnum is None: - if pre_merged_tags: - try: - t.tagnum = pre_merged_tags[t.tagname] - except KeyError: - print("Error: Tag number not defined for tag `%s'. Have you done a full build?" % t.tagname, - file=sys.stderr) - sys.exit(1) - else: - while True: - x = (hashname(name) % (ASSIGN_LIMIT - ASSIGN_START - 1)) + ASSIGN_START - if x not in by_tagnum: - t.tagnum = x - by_tagnum[x] = t - break - name = "_" + name - # by_tagnum should be complete now; we've assigned numbers to all tags. buffer = StringIO() @@ -181,4 +106,4 @@ for n, t in sorted(by_tagnum.items()): else: buffer.write("%d %s\n" % (t.tagnum, t.tagname)) -event_log_tags.WriteOutput(output_file, buffer) +event_log_tags.WriteOutput(args.output_file, buffer) diff --git a/tools/metadata/Android.bp b/tools/metadata/Android.bp deleted file mode 100644 index 77d106d705..0000000000 --- a/tools/metadata/Android.bp +++ /dev/null @@ -1,16 +0,0 @@ -package { - default_applicable_licenses: ["Android-Apache-2.0"], -} - -blueprint_go_binary { - name: "metadata", - deps: [ - "soong-testing-test_spec_proto", - "soong-testing-code_metadata_proto", - "soong-testing-code_metadata_internal_proto", - "golang-protobuf-proto", - ], - srcs: [ - "generator.go", - ] -}
\ No newline at end of file diff --git a/tools/metadata/OWNERS b/tools/metadata/OWNERS deleted file mode 100644 index 03bcdf1c40..0000000000 --- a/tools/metadata/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -dariofreni@google.com -joeo@google.com -ronish@google.com -caditya@google.com diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go deleted file mode 100644 index b7668be44f..0000000000 --- a/tools/metadata/generator.go +++ /dev/null @@ -1,328 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io" - "log" - "os" - "sort" - "strings" - "sync" - - "android/soong/testing/code_metadata_internal_proto" - "android/soong/testing/code_metadata_proto" - "android/soong/testing/test_spec_proto" - "google.golang.org/protobuf/proto" -) - -type keyToLocksMap struct { - locks sync.Map -} - -func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex { - mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{}) - return mutex.(*sync.Mutex) -} - -// Define a struct to hold the combination of team ID and multi-ownership flag for validation -type sourceFileAttributes struct { - TeamID string - MultiOwnership bool - Path string -} - -func getSortedKeys(syncMap *sync.Map) []string { - var allKeys []string - syncMap.Range( - func(key, _ interface{}) bool { - allKeys = append(allKeys, key.(string)) - return true - }, - ) - - sort.Strings(allKeys) - return allKeys -} - -// writeProtoToFile marshals a protobuf message and writes it to a file -func writeProtoToFile(outputFile string, message proto.Message) { - data, err := proto.Marshal(message) - if err != nil { - log.Fatal(err) - } - file, err := os.Create(outputFile) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - _, err = file.Write(data) - if err != nil { - log.Fatal(err) - } -} - -func readFileToString(filePath string) string { - file, err := os.Open(filePath) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - data, err := io.ReadAll(file) - if err != nil { - log.Fatal(err) - } - return string(data) -} - -func writeEmptyOutputProto(outputFile string, metadataRule string) { - file, err := os.Create(outputFile) - if err != nil { - log.Fatal(err) - } - var message proto.Message - if metadataRule == "test_spec" { - message = &test_spec_proto.TestSpec{} - } else if metadataRule == "code_metadata" { - message = &code_metadata_proto.CodeMetadata{} - } - data, err := proto.Marshal(message) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - _, err = file.Write([]byte(data)) - if err != nil { - log.Fatal(err) - } -} - -func processTestSpecProtobuf( - filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap, - errCh chan error, wg *sync.WaitGroup, -) { - defer wg.Done() - - fileContent := strings.TrimRight(readFileToString(filePath), "\n") - testData := test_spec_proto.TestSpec{} - err := proto.Unmarshal([]byte(fileContent), &testData) - if err != nil { - errCh <- err - return - } - - ownershipMetadata := testData.GetOwnershipMetadataList() - for _, metadata := range ownershipMetadata { - key := metadata.GetTargetName() - lock := keyLocks.GetLockForKey(key) - lock.Lock() - - value, loaded := ownershipMetadataMap.LoadOrStore( - key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata}, - ) - if loaded { - existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) - isDuplicate := false - for _, existing := range existingMetadata { - if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() { - errCh <- fmt.Errorf( - "Conflicting trendy team IDs found for %s at:\n%s with teamId"+ - ": %s,\n%s with teamId: %s", - key, - metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(), - existing.GetTrendyTeamId(), - ) - - lock.Unlock() - return - } - if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() { - isDuplicate = true - break - } - } - if !isDuplicate { - existingMetadata = append(existingMetadata, metadata) - ownershipMetadataMap.Store(key, existingMetadata) - } - } - - lock.Unlock() - } -} - -// processCodeMetadataProtobuf processes CodeMetadata protobuf files -func processCodeMetadataProtobuf( - filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap, - errCh chan error, wg *sync.WaitGroup, -) { - defer wg.Done() - - fileContent := strings.TrimRight(readFileToString(filePath), "\n") - internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{} - err := proto.Unmarshal([]byte(fileContent), &internalCodeData) - if err != nil { - errCh <- err - return - } - - // Process each TargetOwnership entry - for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() { - key := internalMetadata.GetTargetName() - lock := keyLocks.GetLockForKey(key) - lock.Lock() - - for _, srcFile := range internalMetadata.GetSourceFiles() { - srcFileKey := srcFile - srcFileLock := keyLocks.GetLockForKey(srcFileKey) - srcFileLock.Lock() - attributes := sourceFileAttributes{ - TeamID: internalMetadata.GetTrendyTeamId(), - MultiOwnership: internalMetadata.GetMultiOwnership(), - Path: internalMetadata.GetPath(), - } - - existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey) - if exists { - existing := existingAttributes.(sourceFileAttributes) - if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) { - errCh <- fmt.Errorf( - "Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+ - " If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+ - "Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.", - srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path, - ) - srcFileLock.Unlock() - lock.Unlock() - return - } - } else { - // Store the metadata if no conflict - sourceFileMetadataMap.Store(srcFileKey, attributes) - } - srcFileLock.Unlock() - } - - value, loaded := ownershipMetadataMap.LoadOrStore( - key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata}, - ) - if loaded { - existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership) - isDuplicate := false - for _, existing := range existingMetadata { - if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() { - isDuplicate = true - break - } - } - if !isDuplicate { - existingMetadata = append(existingMetadata, internalMetadata) - ownershipMetadataMap.Store(key, existingMetadata) - } - } - - lock.Unlock() - } -} - -func main() { - inputFile := flag.String("inputFile", "", "Input file path") - outputFile := flag.String("outputFile", "", "Output file path") - rule := flag.String( - "rule", "", "Metadata rule (Hint: test_spec or code_metadata)", - ) - flag.Parse() - - if *inputFile == "" || *outputFile == "" || *rule == "" { - fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>") - os.Exit(1) - } - - inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n") - filePaths := strings.Split(inputFileData, " ") - if len(filePaths) == 1 && filePaths[0] == "" { - writeEmptyOutputProto(*outputFile, *rule) - return - } - ownershipMetadataMap := &sync.Map{} - keyLocks := &keyToLocksMap{} - errCh := make(chan error, len(filePaths)) - var wg sync.WaitGroup - - switch *rule { - case "test_spec": - for _, filePath := range filePaths { - wg.Add(1) - go processTestSpecProtobuf( - filePath, ownershipMetadataMap, keyLocks, errCh, &wg, - ) - } - - wg.Wait() - close(errCh) - - for err := range errCh { - log.Fatal(err) - } - - allKeys := getSortedKeys(ownershipMetadataMap) - var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata - - for _, key := range allKeys { - value, _ := ownershipMetadataMap.Load(key) - metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata) - allMetadata = append(allMetadata, metadataList...) - } - - testSpec := &test_spec_proto.TestSpec{ - OwnershipMetadataList: allMetadata, - } - writeProtoToFile(*outputFile, testSpec) - break - case "code_metadata": - sourceFileMetadataMap := &sync.Map{} - for _, filePath := range filePaths { - wg.Add(1) - go processCodeMetadataProtobuf( - filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg, - ) - } - - wg.Wait() - close(errCh) - - for err := range errCh { - log.Fatal(err) - } - - sortedKeys := getSortedKeys(ownershipMetadataMap) - allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0) - for _, key := range sortedKeys { - value, _ := ownershipMetadataMap.Load(key) - metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership) - for _, m := range metadata { - targetName := m.GetTargetName() - path := m.GetPath() - trendyTeamId := m.GetTrendyTeamId() - - allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{ - TargetName: &targetName, - Path: &path, - TrendyTeamId: &trendyTeamId, - SourceFiles: m.GetSourceFiles(), - }) - } - } - - finalMetadata := &code_metadata_proto.CodeMetadata{ - TargetOwnershipList: allMetadata, - } - writeProtoToFile(*outputFile, finalMetadata) - break - default: - log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule) - } -} diff --git a/tools/metadata/go.mod b/tools/metadata/go.mod deleted file mode 100644 index e9d04b16f6..0000000000 --- a/tools/metadata/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module android/soong/tools/metadata - -require google.golang.org/protobuf v0.0.0 - -replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf - -go 1.18
\ No newline at end of file diff --git a/tools/metadata/go.work b/tools/metadata/go.work deleted file mode 100644 index f2cdf8ec98..0000000000 --- a/tools/metadata/go.work +++ /dev/null @@ -1,11 +0,0 @@ -go 1.18 - -use ( - . - ../../../../external/golang-protobuf - ../../../soong/testing/test_spec_proto - ../../../soong/testing/code_metadata_proto - ../../../soong/testing/code_metadata_proto_internal -) - -replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf diff --git a/tools/metadata/testdata/emptyInputFile.txt b/tools/metadata/testdata/emptyInputFile.txt deleted file mode 100644 index 8b13789179..0000000000 --- a/tools/metadata/testdata/emptyInputFile.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/metadata/testdata/expectedCodeMetadataOutput.txt b/tools/metadata/testdata/expectedCodeMetadataOutput.txt deleted file mode 100644 index 755cf40a30..0000000000 --- a/tools/metadata/testdata/expectedCodeMetadataOutput.txt +++ /dev/null @@ -1,7 +0,0 @@ - - -bar -Android.bp12346"b.java - -foo -Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/expectedOutputFile.txt b/tools/metadata/testdata/expectedOutputFile.txt deleted file mode 100644 index b0d382f279..0000000000 --- a/tools/metadata/testdata/expectedOutputFile.txt +++ /dev/null @@ -1,22 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Android.bp12346 -. -java-test-module-name-six -Aqwerty.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-two -Android.bp12345 -. -java-test-module-name-two -Asdfghj.bp12345 -. -java-test-module-name-two -Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/file1.txt b/tools/metadata/testdata/file1.txt deleted file mode 100644 index 81beed00ab..0000000000 --- a/tools/metadata/testdata/file1.txt +++ /dev/null @@ -1,13 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-two -Android.bp12345 -. -java-test-module-name-two -Asdfghj.bp12345 -. -java-test-module-name-two -Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file2.txt b/tools/metadata/testdata/file2.txt deleted file mode 100644 index 32a753fef5..0000000000 --- a/tools/metadata/testdata/file2.txt +++ /dev/null @@ -1,25 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Android.bp12346 -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Aqwerty.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/file3.txt b/tools/metadata/testdata/file3.txt deleted file mode 100644 index 81beed00ab..0000000000 --- a/tools/metadata/testdata/file3.txt +++ /dev/null @@ -1,13 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-two -Android.bp12345 -. -java-test-module-name-two -Asdfghj.bp12345 -. -java-test-module-name-two -Azxcvbn.bp12345 diff --git a/tools/metadata/testdata/file4.txt b/tools/metadata/testdata/file4.txt deleted file mode 100644 index 6a7590021d..0000000000 --- a/tools/metadata/testdata/file4.txt +++ /dev/null @@ -1,25 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Android.bp12346 -. -java-test-module-name-one -Android.bp12346 -. -java-test-module-name-six -Aqwerty.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 diff --git a/tools/metadata/testdata/file5.txt b/tools/metadata/testdata/file5.txt deleted file mode 100644 index d8de06457d..0000000000 --- a/tools/metadata/testdata/file5.txt +++ /dev/null @@ -1,4 +0,0 @@ - - -foo -Android.bp12345"a.java diff --git a/tools/metadata/testdata/file6.txt b/tools/metadata/testdata/file6.txt deleted file mode 100644 index 9c7cdcd505..0000000000 --- a/tools/metadata/testdata/file6.txt +++ /dev/null @@ -1,4 +0,0 @@ - - -bar -Android.bp12346"b.java diff --git a/tools/metadata/testdata/file7.txt b/tools/metadata/testdata/file7.txt deleted file mode 100644 index d8de06457d..0000000000 --- a/tools/metadata/testdata/file7.txt +++ /dev/null @@ -1,4 +0,0 @@ - - -foo -Android.bp12345"a.java diff --git a/tools/metadata/testdata/file8.txt b/tools/metadata/testdata/file8.txt deleted file mode 100644 index a931690022..0000000000 --- a/tools/metadata/testdata/file8.txt +++ /dev/null @@ -1,4 +0,0 @@ - - -foo -Android.gp12346"a.java diff --git a/tools/metadata/testdata/generatedCodeMetadataOutput.txt b/tools/metadata/testdata/generatedCodeMetadataOutput.txt deleted file mode 100644 index 755cf40a30..0000000000 --- a/tools/metadata/testdata/generatedCodeMetadataOutput.txt +++ /dev/null @@ -1,7 +0,0 @@ - - -bar -Android.bp12346"b.java - -foo -Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt b/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt deleted file mode 100644 index 755cf40a30..0000000000 --- a/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt +++ /dev/null @@ -1,7 +0,0 @@ - - -bar -Android.bp12346"b.java - -foo -Android.bp12345"a.java
\ No newline at end of file diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt deleted file mode 100644 index e69de29bb2..0000000000 --- a/tools/metadata/testdata/generatedEmptyOutputFile.txt +++ /dev/null diff --git a/tools/metadata/testdata/generatedOutputFile.txt b/tools/metadata/testdata/generatedOutputFile.txt deleted file mode 100644 index b0d382f279..0000000000 --- a/tools/metadata/testdata/generatedOutputFile.txt +++ /dev/null @@ -1,22 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Android.bp12346 -. -java-test-module-name-six -Aqwerty.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-two -Android.bp12345 -. -java-test-module-name-two -Asdfghj.bp12345 -. -java-test-module-name-two -Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/metadata/testdata/inputCodeMetadata.txt b/tools/metadata/testdata/inputCodeMetadata.txt deleted file mode 100644 index 7a81b7d523..0000000000 --- a/tools/metadata/testdata/inputCodeMetadata.txt +++ /dev/null @@ -1 +0,0 @@ -file5.txt file6.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputCodeMetadataNegative.txt b/tools/metadata/testdata/inputCodeMetadataNegative.txt deleted file mode 100644 index 26668e44a9..0000000000 --- a/tools/metadata/testdata/inputCodeMetadataNegative.txt +++ /dev/null @@ -1 +0,0 @@ -file7.txt file8.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFiles.txt b/tools/metadata/testdata/inputFiles.txt deleted file mode 100644 index e44bc94d32..0000000000 --- a/tools/metadata/testdata/inputFiles.txt +++ /dev/null @@ -1 +0,0 @@ -file1.txt file2.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/inputFilesNegativeCase.txt b/tools/metadata/testdata/inputFilesNegativeCase.txt deleted file mode 100644 index a37aa3fd5d..0000000000 --- a/tools/metadata/testdata/inputFilesNegativeCase.txt +++ /dev/null @@ -1 +0,0 @@ -file3.txt file4.txt
\ No newline at end of file diff --git a/tools/metadata/testdata/metadata_test.go b/tools/metadata/testdata/metadata_test.go deleted file mode 100644 index 314add352f..0000000000 --- a/tools/metadata/testdata/metadata_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - "os/exec" - "strings" - "testing" -) - -func TestMetadata(t *testing.T) { - cmd := exec.Command( - "metadata", "-rule", "test_spec", "-inputFile", "./inputFiles.txt", "-outputFile", - "./generatedOutputFile.txt", - ) - stderr, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) - } - - // Read the contents of the expected output file - expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt") - if err != nil { - t.Fatalf("Error reading expected output file: %s", err) - } - - // Read the contents of the generated output file - generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt") - if err != nil { - t.Fatalf("Error reading generated output file: %s", err) - } - - fmt.Println() - - // Compare the contents - if string(expectedOutput) != string(generatedOutput) { - t.Errorf("Generated file contents do not match the expected output") - } -} - -func TestMetadataNegativeCase(t *testing.T) { - cmd := exec.Command( - "metadata", "-rule", "test_spec", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile", - "./generatedOutputFileNegativeCase.txt", - ) - stderr, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf( - "Expected an error, but the metadata command executed successfully. Output: %s", - stderr, - ) - } - - expectedError := "Conflicting trendy team IDs found for java-test-module" + - "-name-one at:\nAndroid.bp with teamId: 12346," + - "\nAndroid.bp with teamId: 12345" - if !strings.Contains( - strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError), - ) { - t.Errorf( - "Unexpected error message. Expected to contain: %s, Got: %s", - expectedError, stderr, - ) - } -} - -func TestEmptyInputFile(t *testing.T) { - cmd := exec.Command( - "metadata", "-rule", "test_spec", "-inputFile", "./emptyInputFile.txt", "-outputFile", - "./generatedEmptyOutputFile.txt", - ) - stderr, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) - } - - // Read the contents of the generated output file - generatedOutput, err := ioutil.ReadFile("./generatedEmptyOutputFile.txt") - if err != nil { - t.Fatalf("Error reading generated output file: %s", err) - } - - fmt.Println() - - // Compare the contents - if string(generatedOutput) != "\n" { - t.Errorf("Generated file contents do not match the expected output") - } -} - -func TestCodeMetadata(t *testing.T) { - cmd := exec.Command( - "metadata", "-rule", "code_metadata", "-inputFile", "./inputCodeMetadata.txt", "-outputFile", - "./generatedCodeMetadataOutputFile.txt", - ) - stderr, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err) - } - - // Read the contents of the expected output file - expectedOutput, err := ioutil.ReadFile("./expectedCodeMetadataOutput.txt") - if err != nil { - t.Fatalf("Error reading expected output file: %s", err) - } - - // Read the contents of the generated output file - generatedOutput, err := ioutil.ReadFile("./generatedCodeMetadataOutputFile.txt") - if err != nil { - t.Fatalf("Error reading generated output file: %s", err) - } - - fmt.Println() - - // Compare the contents - if string(expectedOutput) != string(generatedOutput) { - t.Errorf("Generated file contents do not match the expected output") - } -} diff --git a/tools/metadata/testdata/outputFile.txt b/tools/metadata/testdata/outputFile.txt deleted file mode 100644 index b0d382f279..0000000000 --- a/tools/metadata/testdata/outputFile.txt +++ /dev/null @@ -1,22 +0,0 @@ - -. -java-test-module-name-one -Android.bp12345 -. -java-test-module-name-six -Android.bp12346 -. -java-test-module-name-six -Aqwerty.bp12346 -. -java-test-module-name-six -Apoiuyt.bp12346 -. -java-test-module-name-two -Android.bp12345 -. -java-test-module-name-two -Asdfghj.bp12345 -. -java-test-module-name-two -Azxcvbn.bp12345
\ No newline at end of file diff --git a/tools/missing_soong_module_info.py b/tools/missing_soong_module_info.py new file mode 100755 index 0000000000..6fa7f2bccb --- /dev/null +++ b/tools/missing_soong_module_info.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import sys + +def main(): + try: + product_out = os.environ["ANDROID_PRODUCT_OUT"] + except KeyError: + sys.stderr.write("Can't get ANDROID_PRODUCT_OUT. Run lunch first.\n") + sys.exit(1) + + filename = os.path.join(product_out, "module-info.json") + try: + with open(filename) as f: + modules = json.load(f) + except FileNotFoundError: + sys.stderr.write(f"File not found: {filename}\n") + sys.exit(1) + except json.JSONDecodeError: + sys.stderr.write(f"Invalid json: {filename}\n") + return None + + classes = {} + + for name, info in modules.items(): + make = info.get("make") + make_gen = info.get("make_generated_module_info") + if not make and make_gen: + classes.setdefault(frozenset(info.get("class")), []).append(name) + + for cl, names in classes.items(): + print(" ".join(cl)) + for name in names: + print(" ", name) + +if __name__ == "__main__": + main() diff --git a/tools/perf/benchmarks b/tools/perf/benchmarks index 6998ecd5c2..228c1d0c94 100755 --- a/tools/perf/benchmarks +++ b/tools/perf/benchmarks @@ -337,6 +337,12 @@ class Runner(): def Run(self): """Run all of the user-selected benchmarks.""" + + # With `--list`, just list the benchmarks available. + if self._options.List(): + print(" ".join(self._options.BenchmarkIds())) + return + # Clean out the log dir or create it if necessary prepare_log_dir(self._options.LogDir()) @@ -546,6 +552,8 @@ benchmarks: parser.add_argument("--benchmark", nargs="*", default=[b.id for b in self._benchmarks], metavar="BENCHMARKS", help="Benchmarks to run. Default suite will be run if omitted.") + parser.add_argument("--list", action="store_true", + help="list the available benchmarks. No benchmark is run.") parser.add_argument("--dist-one", action="store_true", help="Copy logs and metrics to the given dist dir. Requires that only" + " one benchmark be supplied. Postroll steps will be skipped.") @@ -615,6 +623,12 @@ benchmarks: def DryRun(self): return self._args.dry_run + def List(self): + return self._args.list + + def BenchmarkIds(self) : + return [benchmark.id for benchmark in self._benchmarks] + def _lunches(self): def parse_lunch(lunch): parts = lunch.split("-") @@ -786,6 +800,32 @@ benchmarks: preroll=1, postroll=2, ), + Benchmark(id="add_systemui_field_with_tests", + title="Add SystemUI field with tests", + change=AddJavaField("frameworks/base/packages/SystemUI/src/com/android/systemui/wmshell/WMShell.java", + "public"), + modules=["SystemUiRavenTests"], + preroll=1, + postroll=2, + ), + Benchmark(id="systemui_flicker_add_log_call", + title="Add a Log call to flicker", + change=Modify("platform_testing/libraries/flicker/src/android/tools/flicker/FlickerServiceResultsCollector.kt", + lambda: f'Log.v(LOG_TAG, "BENCHMARK = {random.randint(0, 1000000)}");\n', + before="Log.v(LOG_TAG,"), + modules=["WMShellFlickerTestsPip"], + preroll=1, + postroll=2, + ), + Benchmark(id="systemui_core_add_log_call", + title="Add a Log call SystemUIApplication", + change=Modify("frameworks/base/packages/SystemUI/src/com/android/systemui/SystemUIApplication.java", + lambda: f'Log.v(TAG, "BENCHMARK = {random.randint(0, 1000000)}");\n', + before="Log.wtf(TAG,"), + modules=["SystemUI-core"], + preroll=1, + postroll=2, + ), ] def _error(self, message): diff --git a/tools/record-finalized-flags/.gitignore b/tools/record-finalized-flags/.gitignore new file mode 100644 index 0000000000..1e7caa9ea8 --- /dev/null +++ b/tools/record-finalized-flags/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +target/ diff --git a/tools/record-finalized-flags/Android.bp b/tools/record-finalized-flags/Android.bp new file mode 100644 index 0000000000..55a3a389e0 --- /dev/null +++ b/tools/record-finalized-flags/Android.bp @@ -0,0 +1,28 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_defaults { + name: "record-finalized-flags-defaults", + edition: "2021", + clippy_lints: "android", + lints: "android", + srcs: ["src/main.rs"], + rustlibs: [ + "libaconfig_protos", + "libanyhow", + "libclap", + "libregex", + ], +} + +rust_binary_host { + name: "record-finalized-flags", + defaults: ["record-finalized-flags-defaults"], +} + +rust_test_host { + name: "record-finalized-flags-test", + defaults: ["record-finalized-flags-defaults"], + test_suites: ["general-tests"], +} diff --git a/tools/record-finalized-flags/Cargo.toml b/tools/record-finalized-flags/Cargo.toml new file mode 100644 index 0000000000..0fc795363f --- /dev/null +++ b/tools/record-finalized-flags/Cargo.toml @@ -0,0 +1,15 @@ +# Cargo.toml file to allow rapid development of record-finalized-flags using +# cargo. Soong is the official Android build system, and the only system +# guaranteed to support record-finalized-flags. If there is ever any issue with +# the cargo setup, support for cargo will be dropped and this file removed. + +[package] +name = "record-finalized-flags" +version = "0.1.0" +edition = "2021" + +[dependencies] +aconfig_protos = { path = "../aconfig/aconfig_protos" } +anyhow = { path = "../../../../external/rust/android-crates-io/crates/anyhow" } +clap = { path = "../../../../external/rust/android-crates-io/crates/clap", features = ["derive"] } +regex = { path = "../../../../external/rust/android-crates-io/crates/regex" } diff --git a/tools/record-finalized-flags/OWNERS b/tools/record-finalized-flags/OWNERS new file mode 100644 index 0000000000..2864a2c23c --- /dev/null +++ b/tools/record-finalized-flags/OWNERS @@ -0,0 +1 @@ +include platform/frameworks/base:/SDK_OWNERS diff --git a/tools/record-finalized-flags/src/api_signature_files.rs b/tools/record-finalized-flags/src/api_signature_files.rs new file mode 100644 index 0000000000..af8f4d1957 --- /dev/null +++ b/tools/record-finalized-flags/src/api_signature_files.rs @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use regex::Regex; +use std::{collections::HashSet, io::Read}; + +use crate::FlagId; + +/// Grep for all flags used with @FlaggedApi annotations in an API signature file (*current.txt +/// file). +pub(crate) fn extract_flagged_api_flags<R: Read>(mut reader: R) -> Result<HashSet<FlagId>> { + let mut haystack = String::new(); + reader.read_to_string(&mut haystack)?; + let regex = Regex::new(r#"(?ms)@FlaggedApi\("(.*?)"\)"#).unwrap(); + let iter = regex.captures_iter(&haystack).map(|cap| cap[1].to_owned()); + Ok(HashSet::from_iter(iter)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test() { + let api_signature_file = include_bytes!("../tests/api-signature-file.txt"); + let flags = extract_flagged_api_flags(&api_signature_file[..]).unwrap(); + assert_eq!( + flags, + HashSet::from_iter(vec![ + "record_finalized_flags.test.foo".to_string(), + "this.flag.is.not.used".to_string(), + ]) + ); + } +} diff --git a/tools/record-finalized-flags/src/finalized_flags.rs b/tools/record-finalized-flags/src/finalized_flags.rs new file mode 100644 index 0000000000..1ae4c4d789 --- /dev/null +++ b/tools/record-finalized-flags/src/finalized_flags.rs @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use anyhow::Result; +use std::{collections::HashSet, io::Read}; + +use crate::FlagId; + +/// Read a list of flag names. The input is expected to be plain text, with each line containing +/// the name of a single flag. +pub(crate) fn read_finalized_flags<R: Read>(mut reader: R) -> Result<HashSet<FlagId>> { + let mut contents = String::new(); + reader.read_to_string(&mut contents)?; + let iter = contents.lines().map(|s| s.to_owned()); + Ok(HashSet::from_iter(iter)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test() { + let input = include_bytes!("../tests/finalized-flags.txt"); + let flags = read_finalized_flags(&input[..]).unwrap(); + assert_eq!( + flags, + HashSet::from_iter(vec![ + "record_finalized_flags.test.bar".to_string(), + "record_finalized_flags.test.baz".to_string(), + ]) + ); + } +} diff --git a/tools/record-finalized-flags/src/flag_values.rs b/tools/record-finalized-flags/src/flag_values.rs new file mode 100644 index 0000000000..cc16d12f3c --- /dev/null +++ b/tools/record-finalized-flags/src/flag_values.rs @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use aconfig_protos::{ParsedFlagExt, ProtoFlagPermission, ProtoFlagState}; +use anyhow::{anyhow, Result}; +use std::{collections::HashSet, io::Read}; + +use crate::FlagId; + +/// Parse a ProtoParsedFlags binary protobuf blob and return the fully qualified names of flags +/// that are slated for API finalization (i.e. are both ENABLED and READ_ONLY). +pub(crate) fn get_relevant_flags_from_binary_proto<R: Read>( + mut reader: R, +) -> Result<HashSet<FlagId>> { + let mut buffer = Vec::new(); + reader.read_to_end(&mut buffer)?; + let parsed_flags = aconfig_protos::parsed_flags::try_from_binary_proto(&buffer) + .map_err(|_| anyhow!("failed to parse binary proto"))?; + let iter = parsed_flags + .parsed_flag + .into_iter() + .filter(|flag| { + flag.state() == ProtoFlagState::ENABLED + && flag.permission() == ProtoFlagPermission::READ_ONLY + }) + .map(|flag| flag.fully_qualified_name()); + Ok(HashSet::from_iter(iter)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_disabled_or_read_write_flags_are_ignored() { + let bytes = include_bytes!("../tests/flags.protobuf"); + let flags = get_relevant_flags_from_binary_proto(&bytes[..]).unwrap(); + assert_eq!(flags, HashSet::from_iter(vec!["record_finalized_flags.test.foo".to_string()])); + } +} diff --git a/tools/record-finalized-flags/src/main.rs b/tools/record-finalized-flags/src/main.rs new file mode 100644 index 0000000000..efdbc9be8e --- /dev/null +++ b/tools/record-finalized-flags/src/main.rs @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! `record-finalized-flags` is a tool to create a snapshot (intended to be stored in +//! prebuilts/sdk) of the flags used with @FlaggedApi APIs +use anyhow::Result; +use clap::Parser; +use std::{collections::HashSet, fs::File, path::PathBuf}; + +mod api_signature_files; +mod finalized_flags; +mod flag_values; + +pub(crate) type FlagId = String; + +const ABOUT: &str = "Create a new prebuilts/sdk/<version>/finalized-flags.txt file + +The prebuilts/sdk/<version>/finalized-flags.txt files list all aconfig flags that have been used +with @FlaggedApi annotations on APIs that have been finalized. These files are used to prevent +flags from being re-used for new, unfinalized, APIs, and by the aconfig code generation. + +This tool works as follows: + + - Read API signature files from source tree (*current.txt files) [--api-signature-file] + - Read the current aconfig flag values from source tree [--parsed-flags-file] + - Read the previous finalized-flags.txt files from prebuilts/sdk [--finalized-flags-file] + - Extract the flags slated for API finalization by scanning through the API signature files for + flags that are ENABLED and READ_ONLY + - Merge the found flags with the recorded flags from previous API finalizations + - Print the set of flags to stdout +"; + +#[derive(Parser, Debug)] +#[clap(about=ABOUT)] +struct Cli { + #[arg(long)] + parsed_flags_file: PathBuf, + + #[arg(long)] + api_signature_file: Vec<PathBuf>, + + #[arg(long)] + finalized_flags_file: PathBuf, +} + +/// Filter out the ENABLED and READ_ONLY flags used with @FlaggedApi annotations in the source +/// tree, and add those flags to the set of previously finalized flags. +fn calculate_new_finalized_flags( + flags_used_with_flaggedapi_annotation: &HashSet<FlagId>, + all_flags_to_be_finalized: &HashSet<FlagId>, + already_finalized_flags: &HashSet<FlagId>, +) -> HashSet<FlagId> { + let new_flags: HashSet<_> = flags_used_with_flaggedapi_annotation + .intersection(all_flags_to_be_finalized) + .map(|s| s.to_owned()) + .collect(); + already_finalized_flags.union(&new_flags).map(|s| s.to_owned()).collect() +} + +fn main() -> Result<()> { + let args = Cli::parse(); + + let mut flags_used_with_flaggedapi_annotation = HashSet::new(); + for path in args.api_signature_file { + let file = File::open(path)?; + for flag in api_signature_files::extract_flagged_api_flags(file)?.drain() { + flags_used_with_flaggedapi_annotation.insert(flag); + } + } + + let file = File::open(args.parsed_flags_file)?; + let all_flags_to_be_finalized = flag_values::get_relevant_flags_from_binary_proto(file)?; + + let file = File::open(args.finalized_flags_file)?; + let already_finalized_flags = finalized_flags::read_finalized_flags(file)?; + + let mut new_finalized_flags = Vec::from_iter(calculate_new_finalized_flags( + &flags_used_with_flaggedapi_annotation, + &all_flags_to_be_finalized, + &already_finalized_flags, + )); + new_finalized_flags.sort(); + + println!("{}", new_finalized_flags.join("\n")); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test() { + let input = include_bytes!("../tests/api-signature-file.txt"); + let flags_used_with_flaggedapi_annotation = + api_signature_files::extract_flagged_api_flags(&input[..]).unwrap(); + + let input = include_bytes!("../tests/flags.protobuf"); + let all_flags_to_be_finalized = + flag_values::get_relevant_flags_from_binary_proto(&input[..]).unwrap(); + + let input = include_bytes!("../tests/finalized-flags.txt"); + let already_finalized_flags = finalized_flags::read_finalized_flags(&input[..]).unwrap(); + + let new_finalized_flags = calculate_new_finalized_flags( + &flags_used_with_flaggedapi_annotation, + &all_flags_to_be_finalized, + &already_finalized_flags, + ); + + assert_eq!( + new_finalized_flags, + HashSet::from_iter(vec![ + "record_finalized_flags.test.foo".to_string(), + "record_finalized_flags.test.bar".to_string(), + "record_finalized_flags.test.baz".to_string(), + ]) + ); + } +} diff --git a/tools/record-finalized-flags/tests/api-signature-file.txt b/tools/record-finalized-flags/tests/api-signature-file.txt new file mode 100644 index 0000000000..2ad559f0ad --- /dev/null +++ b/tools/record-finalized-flags/tests/api-signature-file.txt @@ -0,0 +1,15 @@ +// Signature format: 2.0 +package android { + + public final class C { + ctor public C(); + } + + public static final class C.inner { + ctor public C.inner(); + field @FlaggedApi("record_finalized_flags.test.foo") public static final String FOO = "foo"; + field @FlaggedApi("this.flag.is.not.used") public static final String BAR = "bar"; + } + +} + diff --git a/tools/record-finalized-flags/tests/finalized-flags.txt b/tools/record-finalized-flags/tests/finalized-flags.txt new file mode 100644 index 0000000000..7fbcb3dc65 --- /dev/null +++ b/tools/record-finalized-flags/tests/finalized-flags.txt @@ -0,0 +1,2 @@ +record_finalized_flags.test.bar +record_finalized_flags.test.baz diff --git a/tools/record-finalized-flags/tests/flags.declarations b/tools/record-finalized-flags/tests/flags.declarations new file mode 100644 index 0000000000..b45ef62523 --- /dev/null +++ b/tools/record-finalized-flags/tests/flags.declarations @@ -0,0 +1,16 @@ +package: "record_finalized_flags.test" +container: "system" + +flag { + name: "foo" + namespace: "test" + description: "FIXME" + bug: "" +} + +flag { + name: "not_enabled" + namespace: "test" + description: "FIXME" + bug: "" +} diff --git a/tools/record-finalized-flags/tests/flags.protobuf b/tools/record-finalized-flags/tests/flags.protobuf Binary files differnew file mode 100644 index 0000000000..7c6e63eca8 --- /dev/null +++ b/tools/record-finalized-flags/tests/flags.protobuf diff --git a/tools/record-finalized-flags/tests/flags.values b/tools/record-finalized-flags/tests/flags.values new file mode 100644 index 0000000000..ff6225d822 --- /dev/null +++ b/tools/record-finalized-flags/tests/flags.values @@ -0,0 +1,13 @@ +flag_value { + package: "record_finalized_flags.test" + name: "foo" + state: ENABLED + permission: READ_ONLY +} + +flag_value { + package: "record_finalized_flags.test" + name: "not_enabled" + state: DISABLED + permission: READ_ONLY +} diff --git a/tools/record-finalized-flags/tests/generate-flags-protobuf.sh b/tools/record-finalized-flags/tests/generate-flags-protobuf.sh new file mode 100755 index 0000000000..701189cd5c --- /dev/null +++ b/tools/record-finalized-flags/tests/generate-flags-protobuf.sh @@ -0,0 +1,7 @@ +#!/bin/bash +aconfig create-cache \ + --package record_finalized_flags.test \ + --container system \ + --declarations flags.declarations \ + --values flags.values \ + --cache flags.protobuf diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp index 8c710449f9..e371b2354c 100644 --- a/tools/releasetools/Android.bp +++ b/tools/releasetools/Android.bp @@ -637,6 +637,8 @@ python_defaults { ], data: [ "testdata/**/*", + ], + device_common_data: [ ":com.android.apex.compressed.v1", ":com.android.apex.vendor.foo.with_vintf", ], diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index c25ff2718c..30a6accf32 100644 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -1100,7 +1100,7 @@ def AddImagesToTargetFiles(filename): vbmeta_partitions = common.AVB_PARTITIONS[:] + tuple(avb_custom_partitions) vbmeta_system = OPTIONS.info_dict.get("avb_vbmeta_system", "").strip() - if vbmeta_system: + if vbmeta_system and set(vbmeta_system.split()).intersection(partitions): banner("vbmeta_system") partitions["vbmeta_system"] = AddVBMeta( output_zip, partitions, "vbmeta_system", vbmeta_system.split()) @@ -1110,7 +1110,7 @@ def AddImagesToTargetFiles(filename): vbmeta_partitions.append("vbmeta_system") vbmeta_vendor = OPTIONS.info_dict.get("avb_vbmeta_vendor", "").strip() - if vbmeta_vendor: + if vbmeta_vendor and set(vbmeta_vendor.split()).intersection(partitions): banner("vbmeta_vendor") partitions["vbmeta_vendor"] = AddVBMeta( output_zip, partitions, "vbmeta_vendor", vbmeta_vendor.split()) @@ -1137,7 +1137,7 @@ def AddImagesToTargetFiles(filename): if item not in included_partitions] vbmeta_partitions.append(partition_name) - if OPTIONS.info_dict.get("avb_building_vbmeta_image") == "true": + if OPTIONS.info_dict.get("avb_building_vbmeta_image") == "true" and set(vbmeta_partitions).intersection(partitions): banner("vbmeta") AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions) diff --git a/tools/releasetools/apex_utils.py b/tools/releasetools/apex_utils.py index 54df955e9f..08f2b83388 100644 --- a/tools/releasetools/apex_utils.py +++ b/tools/releasetools/apex_utils.py @@ -79,15 +79,10 @@ class ApexApkSigner(object): Returns: The repacked apex file containing the signed apk files. """ - if not os.path.exists(self.debugfs_path): - raise ApexSigningError( - "Couldn't find location of debugfs_static: " + - "Path {} does not exist. ".format(self.debugfs_path) + - "Make sure bin/debugfs_static can be found in -p <path>") - list_cmd = ['deapexer', '--debugfs_path', self.debugfs_path, - 'list', self.apex_path] - entries_names = common.RunAndCheckOutput(list_cmd).split() - apk_entries = [name for name in entries_names if name.endswith('.apk')] + payload_dir = self.ExtractApexPayload(self.apex_path) + apk_entries = [] + for base_dir, _, files in os.walk(payload_dir): + apk_entries.extend(os.path.join(base_dir, file) for file in files if file.endswith('.apk')) # No need to sign and repack, return the original apex path. if not apk_entries and self.sign_tool is None: @@ -105,16 +100,16 @@ class ApexApkSigner(object): logger.warning('Apk path does not contain the intended directory name:' ' %s', entry) - payload_dir, has_signed_content = self.ExtractApexPayloadAndSignContents( - apk_entries, apk_keys, payload_key, signing_args) + has_signed_content = self.SignContentsInPayload( + payload_dir, apk_entries, apk_keys, payload_key, signing_args) if not has_signed_content: logger.info('No contents has been signed in %s', self.apex_path) return self.apex_path return self.RepackApexPayload(payload_dir, payload_key, signing_args) - def ExtractApexPayloadAndSignContents(self, apk_entries, apk_keys, payload_key, signing_args): - """Extracts the payload image and signs the containing apk files.""" + def ExtractApexPayload(self, apex_path): + """Extracts the contents of an APEX and returns the directory of the contents""" if not os.path.exists(self.debugfs_path): raise ApexSigningError( "Couldn't find location of debugfs_static: " + @@ -129,9 +124,12 @@ class ApexApkSigner(object): extract_cmd = ['deapexer', '--debugfs_path', self.debugfs_path, '--fsckerofs_path', self.fsckerofs_path, 'extract', - self.apex_path, payload_dir] + apex_path, payload_dir] common.RunAndCheckOutput(extract_cmd) + return payload_dir + def SignContentsInPayload(self, payload_dir, apk_entries, apk_keys, payload_key, signing_args): + """Signs the contents in payload.""" has_signed_content = False for entry in apk_entries: apk_path = os.path.join(payload_dir, entry) @@ -163,7 +161,7 @@ class ApexApkSigner(object): common.RunAndCheckOutput(cmd) has_signed_content = True - return payload_dir, has_signed_content + return has_signed_content def RepackApexPayload(self, payload_dir, payload_key, signing_args=None): """Rebuilds the apex file with the updated payload directory.""" diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index 464ad9b4cc..b6c96c4bf3 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -677,24 +677,31 @@ def TryParseFingerprint(glob_dict: dict): glob_dict["fingerprint"] = fingerprint return - -def ImagePropFromGlobalDict(glob_dict, mount_point): - """Build an image property dictionary from the global dictionary. +def TryParseFingerprintAndTimestamp(glob_dict): + """Helper function that parses fingerprint and timestamp from the global dictionary. Args: glob_dict: the global dictionary from the build system. - mount_point: such as "system", "data" etc. """ - d = {} TryParseFingerprint(glob_dict) # Set fixed timestamp for building the OTA package. if "use_fixed_timestamp" in glob_dict: - d["timestamp"] = FIXED_FILE_TIMESTAMP + glob_dict["timestamp"] = FIXED_FILE_TIMESTAMP if "build.prop" in glob_dict: timestamp = glob_dict["build.prop"].GetProp("ro.build.date.utc") if timestamp: - d["timestamp"] = timestamp + glob_dict["timestamp"] = timestamp + +def ImagePropFromGlobalDict(glob_dict, mount_point): + """Build an image property dictionary from the global dictionary. + + Args: + glob_dict: the global dictionary from the build system. + mount_point: such as "system", "data" etc. + """ + d = {} + TryParseFingerprintAndTimestamp(glob_dict) def copy_prop(src_p, dest_p): """Copy a property from the global dictionary. @@ -730,6 +737,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point): "avb_avbtool", "use_dynamic_partition_size", "fingerprint", + "timestamp", ) for p in common_props: copy_prop(p, p) @@ -992,6 +1000,7 @@ def main(argv): # The caller knows the mount point and provides a dictionary needed by # BuildImage(). image_properties = glob_dict + TryParseFingerprintAndTimestamp(image_properties) else: image_filename = os.path.basename(args.out_file) mount_point = "" diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index f04dfb703d..b6cbb15222 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -23,7 +23,7 @@ import fnmatch import getopt import getpass import gzip -import imp +import importlib.util import json import logging import logging.config @@ -3132,16 +3132,19 @@ class DeviceSpecificParams(object): return try: if os.path.isdir(path): - info = imp.find_module("releasetools", [path]) - else: - d, f = os.path.split(path) - b, x = os.path.splitext(f) - if x == ".py": - f = b - info = imp.find_module(f, [d]) + path = os.path.join(path, "releasetools") + if os.path.isdir(path): + path = os.path.join(path, "__init__.py") + if not os.path.exists(path) and os.path.exists(path + ".py"): + path = path + ".py" + spec = importlib.util.spec_from_file_location("device_specific", path) + if not spec: + raise FileNotFoundError(path) logger.info("loaded device-specific extensions from %s", path) - self.module = imp.load_module("device_specific", *info) - except ImportError: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.module = module + except (ImportError, FileNotFoundError): logger.info("unable to load device-specific module; assuming none") def _DoCall(self, function_name, *args, **kwargs): diff --git a/tools/releasetools/fsverity_metadata_generator.py b/tools/releasetools/fsverity_metadata_generator.py index fa7cd3934a..e531cca7db 100644 --- a/tools/releasetools/fsverity_metadata_generator.py +++ b/tools/releasetools/fsverity_metadata_generator.py @@ -104,16 +104,13 @@ class FSVerityMetadataGenerator: out = subprocess.check_output(cmd, universal_newlines=True).strip() return bytes(bytearray.fromhex(out)) - def generate(self, input_file, output_file=None): + def generate(self, input_file, output_file): if self._signature != 'none': if not self._key: raise RuntimeError("key must be specified.") if not self._cert: raise RuntimeError("cert must be specified.") - if not output_file: - output_file = input_file + '.fsv_meta' - with TempDirectory() as temp_dir: self._do_generate(input_file, output_file, temp_dir) @@ -229,6 +226,21 @@ if __name__ == '__main__': required=True) args = p.parse_args(sys.argv[1:]) + output_file = args.output + if not output_file: + output_file = input_file + '.fsv_meta' + + # remove the output file first, as switching between a file and a symlink can be complicated + try: + os.remove(output_file) + except FileNotFoundError: + pass + + if os.path.islink(args.input): + target = os.readlink(args.input) + '.fsv_meta' + os.symlink(target, output_file) + sys.exit(0) + generator = FSVerityMetadataGenerator(args.fsverity_path) generator.set_signature(args.signature) if args.signature == 'none': @@ -241,4 +253,4 @@ if __name__ == '__main__': generator.set_cert(args.cert) generator.set_key_format(args.key_format) generator.set_hash_alg(args.hash_alg) - generator.generate(args.input, args.output) + generator.generate(args.input, output_file) diff --git a/tools/releasetools/merge_ota.py b/tools/releasetools/merge_ota.py index fb5957a857..e8732a2c52 100644 --- a/tools/releasetools/merge_ota.py +++ b/tools/releasetools/merge_ota.py @@ -226,9 +226,21 @@ def main(argv): logger.setLevel(logging.INFO) logger.info(args) + if args.java_path: + common.OPTIONS.java_path = args.java_path + if args.search_path: common.OPTIONS.search_path = args.search_path + if args.signapk_path: + common.OPTIONS.signapk_path = args.signapk_path + + if args.extra_signapk_args: + common.OPTIONS.extra_signapk_args = args.extra_signapk_args + + if args.signapk_shared_library_path: + common.OPTIONS.signapk_shared_library_path = args.signapk_shared_library_path + metadata_ota = args.packages[-1] if args.metadata_ota is not None: metadata_ota = args.metadata_ota diff --git a/tools/releasetools/ota_from_raw_img.py b/tools/releasetools/ota_from_raw_img.py index 03b44f15d6..3b9374ab13 100644 --- a/tools/releasetools/ota_from_raw_img.py +++ b/tools/releasetools/ota_from_raw_img.py @@ -105,9 +105,6 @@ def main(argv): if args.package_key: logger.info("Signing payload...") - # TODO: remove OPTIONS when no longer used as fallback in payload_signer - common.OPTIONS.payload_signer_args = None - common.OPTIONS.payload_signer_maximum_signature_size = None signer = PayloadSigner(args.package_key, args.private_key_suffix, key_passwords[args.package_key], payload_signer=args.payload_signer, diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index 985cd56cb0..76d168cb8e 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -264,6 +264,10 @@ A/B OTA specific options --compression_factor Specify the maximum block size to be compressed at once during OTA. supported options: 4k, 8k, 16k, 32k, 64k, 128k, 256k + + --full_ota_partitions + Specify list of partitions should be updated in full OTA fashion, even if + an incremental OTA is about to be generated """ from __future__ import print_function @@ -283,7 +287,7 @@ import common import ota_utils import payload_signer from ota_utils import (VABC_COMPRESSION_PARAM_SUPPORT, FinalizeMetadata, GetPackageMetadata, - PayloadGenerator, SECURITY_PATCH_LEVEL_PROP_NAME, ExtractTargetFiles, CopyTargetFilesDir) + PayloadGenerator, SECURITY_PATCH_LEVEL_PROP_NAME, ExtractTargetFiles, CopyTargetFilesDir, TARGET_FILES_IMAGES_SUBDIR) from common import DoesInputFileContain, IsSparseImage import target_files_diff from non_ab_ota import GenerateNonAbOtaPackage @@ -337,6 +341,7 @@ OPTIONS.security_patch_level = None OPTIONS.max_threads = None OPTIONS.vabc_cow_version = None OPTIONS.compression_factor = None +OPTIONS.full_ota_partitions = None POSTINSTALL_CONFIG = 'META/postinstall_config.txt' @@ -892,6 +897,14 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): if source_file is not None: source_file = ExtractTargetFiles(source_file) + if OPTIONS.full_ota_partitions: + for partition in OPTIONS.full_ota_partitions: + for subdir in TARGET_FILES_IMAGES_SUBDIR: + image_path = os.path.join(source_file, subdir, partition + ".img") + if os.path.exists(image_path): + logger.info( + "Ignoring source image %s for partition %s because it is configured to use full OTA", image_path, partition) + os.remove(image_path) assert "ab_partitions" in OPTIONS.source_info_dict, \ "META/ab_partitions.txt is required for ab_update." assert "ab_partitions" in OPTIONS.target_info_dict, \ @@ -1026,6 +1039,9 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): # Prepare custom images. if OPTIONS.custom_images: + if source_file is not None: + source_file = GetTargetFilesZipForCustomImagesUpdates( + source_file, OPTIONS.custom_images) target_file = GetTargetFilesZipForCustomImagesUpdates( target_file, OPTIONS.custom_images) @@ -1108,17 +1124,18 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): additional_args += ["--enable_lz4diff=" + str(OPTIONS.enable_lz4diff).lower()] + env_override = {} if source_file and OPTIONS.enable_lz4diff: - input_tmp = common.UnzipTemp(source_file, ["META/liblz4.so"]) - liblz4_path = os.path.join(input_tmp, "META", "liblz4.so") + liblz4_path = os.path.join(source_file, "META", "liblz4.so") assert os.path.exists( liblz4_path), "liblz4.so not found in META/ dir of target file {}".format(liblz4_path) logger.info("Enabling lz4diff %s", liblz4_path) - additional_args += ["--liblz4_path", liblz4_path] erofs_compression_param = OPTIONS.target_info_dict.get( "erofs_default_compressor") assert erofs_compression_param is not None, "'erofs_default_compressor' not found in META/misc_info.txt of target build. This is required to enable lz4diff." additional_args += ["--erofs_compression_param", erofs_compression_param] + env_override["LD_PRELOAD"] = liblz4_path + \ + ":" + os.environ.get("LD_PRELOAD", "") if OPTIONS.disable_vabc: additional_args += ["--disable_vabc=true"] @@ -1128,10 +1145,15 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): additional_args += ["--compressor_types", OPTIONS.compressor_types] additional_args += ["--max_timestamp", max_timestamp] + env = dict(os.environ) + if env_override: + logger.info("Using environment variables %s", env_override) + env.update(env_override) payload.Generate( target_file, source_file, - additional_args + partition_timestamps_flags + additional_args + partition_timestamps_flags, + env=env ) # Sign the payload. @@ -1193,7 +1215,7 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None): def main(argv): - def option_handler(o, a): + def option_handler(o, a: str): if o in ("-i", "--incremental_from"): OPTIONS.incremental_source = a elif o == "--full_radio": @@ -1320,6 +1342,9 @@ def main(argv): else: raise ValueError("Cannot parse value %r for option %r - only " "integers are allowed." % (a, o)) + elif o == "--full_ota_partitions": + OPTIONS.full_ota_partitions = set( + a.strip().strip("\"").strip("'").split(",")) else: return False return True @@ -1370,6 +1395,7 @@ def main(argv): "max_threads=", "vabc_cow_version=", "compression_factor=", + "full_ota_partitions=", ], extra_option_handler=[option_handler, payload_signer.signer_options]) common.InitLogging() diff --git a/tools/releasetools/ota_signing_utils.py b/tools/releasetools/ota_signing_utils.py index 60c8c94f91..9d04c3bbb5 100644 --- a/tools/releasetools/ota_signing_utils.py +++ b/tools/releasetools/ota_signing_utils.py @@ -23,10 +23,18 @@ def ParseSignerArgs(args): def AddSigningArgumentParse(parser: argparse.ArgumentParser): + parser.add_argument('--java_path', type=str, + help='Path to JVM if other than default') parser.add_argument('--package_key', type=str, help='Paths to private key for signing payload') parser.add_argument('--search_path', '--path', type=str, help='Search path for framework/signapk.jar') + parser.add_argument('--signapk_path', type=str, + help='Path to signapk.jar, relative to search_path') + parser.add_argument('--extra_signapk_args', type=ParseSignerArgs, + help='Extra arguments for signapk.jar') + parser.add_argument('--signapk_shared_library_path', type=str, + help='Path to lib64 libraries used by signapk.jar') parser.add_argument('--payload_signer', type=str, help='Path to custom payload signer') parser.add_argument('--payload_signer_args', type=ParseSignerArgs, diff --git a/tools/releasetools/ota_utils.py b/tools/releasetools/ota_utils.py index 81b53dce36..852d62bb0f 100644 --- a/tools/releasetools/ota_utils.py +++ b/tools/releasetools/ota_utils.py @@ -845,16 +845,16 @@ class PayloadGenerator(object): self.is_partial_update = is_partial_update self.spl_downgrade = spl_downgrade - def _Run(self, cmd): # pylint: disable=no-self-use + def _Run(self, cmd, **kwargs): # pylint: disable=no-self-use # Don't pipe (buffer) the output if verbose is set. Let # brillo_update_payload write to stdout/stderr directly, so its progress can # be monitored. if OPTIONS.verbose: - common.RunAndCheckOutput(cmd, stdout=None, stderr=None) + common.RunAndCheckOutput(cmd, stdout=None, stderr=None, **kwargs) else: - common.RunAndCheckOutput(cmd) + common.RunAndCheckOutput(cmd, **kwargs) - def Generate(self, target_file, source_file=None, additional_args=None): + def Generate(self, target_file, source_file=None, additional_args=None, **kwargs): """Generates a payload from the given target-files zip(s). Args: @@ -863,6 +863,7 @@ class PayloadGenerator(object): generating a full OTA. additional_args: A list of additional args that should be passed to delta_generator binary; or None. + kwargs: Any additional args to pass to subprocess.Popen """ if additional_args is None: additional_args = [] @@ -918,7 +919,7 @@ class PayloadGenerator(object): if self.is_partial_update: cmd.extend(["--is_partial_update=true"]) cmd.extend(additional_args) - self._Run(cmd) + self._Run(cmd, **kwargs) self.payload_file = payload_file self.payload_properties = None diff --git a/tools/releasetools/sign_target_files_apks.py b/tools/releasetools/sign_target_files_apks.py index 0254f37be5..2fa3fb5e89 100755 --- a/tools/releasetools/sign_target_files_apks.py +++ b/tools/releasetools/sign_target_files_apks.py @@ -184,6 +184,7 @@ import re import shutil import stat import sys +import shlex import tempfile import zipfile from xml.etree import ElementTree @@ -377,6 +378,37 @@ def GetApexKeys(keys_info, key_map): return keys_info +def GetMicrodroidVbmetaKey(virt_apex_path, avbtool_path): + """Extracts the AVB public key from microdroid_vbmeta.img within a virt apex. + + Args: + virt_apex_path: The path to the com.android.virt.apex file. + avbtool_path: The path to the avbtool executable. + + Returns: + The AVB public key (bytes). + """ + # Creates an ApexApkSigner to extract microdroid_vbmeta.img. + # No need to set key_passwords/codename_to_api_level_map since + # we won't do signing here. + apex_signer = apex_utils.ApexApkSigner( + virt_apex_path, + None, # key_passwords + None) # codename_to_api_level_map + payload_dir = apex_signer.ExtractApexPayload(virt_apex_path) + microdroid_vbmeta_image = os.path.join( + payload_dir, 'etc', 'fs', 'microdroid_vbmeta.img') + + # Extracts the avb public key from microdroid_vbmeta.img. + with tempfile.NamedTemporaryFile() as microdroid_pubkey: + common.RunAndCheckOutput([ + avbtool_path, 'info_image', + '--image', microdroid_vbmeta_image, + '--output_pubkey', microdroid_pubkey.name]) + with open(microdroid_pubkey.name, 'rb') as f: + return f.read() + + def GetApkFileInfo(filename, compressed_extension, skipped_prefixes): """Returns the APK info based on the given filename. @@ -589,7 +621,7 @@ def GetOtaSigningArgs(): if OPTIONS.payload_signer: args.extend(["--payload_signer=" + OPTIONS.payload_signer]) if OPTIONS.payload_signer_args: - args.extend(["--payload_signer_args=" + OPTIONS.payload_signer_args]) + args.extend(["--payload_signer_args=" + shlex.join(OPTIONS.payload_signer_args)]) if OPTIONS.search_path: args.extend(["--search_path", OPTIONS.search_path]) if OPTIONS.payload_signer_maximum_signature_size: @@ -861,21 +893,34 @@ def ProcessTargetFiles(input_tf_zip: zipfile.ZipFile, output_tf_zip: zipfile.Zip # Updates pvmfw embedded public key with the virt APEX payload key. elif filename == "PREBUILT_IMAGES/pvmfw.img": - # Find the name of the virt APEX in the target files. + # Find the path of the virt APEX in the target files. namelist = input_tf_zip.namelist() - apex_gen = (GetApexFilename(f) for f in namelist if IsApexFile(f)) - virt_apex_re = re.compile("^com\.([^\.]+\.)?android\.virt\.apex$") - virt_apex = next((a for a in apex_gen if virt_apex_re.match(a)), None) - if not virt_apex: + apex_gen = (f for f in namelist if IsApexFile(f)) + virt_apex_re = re.compile("^.*com\.([^\.]+\.)?android\.virt\.apex$") + virt_apex_path = next( + (a for a in apex_gen if virt_apex_re.match(a)), None) + if not virt_apex_path: print("Removing %s from ramdisk: virt APEX not found" % filename) else: - print("Replacing %s embedded key with %s key" % (filename, virt_apex)) + print("Replacing %s embedded key with %s key" % (filename, + virt_apex_path)) # Get the current and new embedded keys. + virt_apex = GetApexFilename(virt_apex_path) payload_key, container_key, sign_tool = apex_keys[virt_apex] - new_pubkey_path = common.ExtractAvbPublicKey( - misc_info['avb_avbtool'], payload_key) - with open(new_pubkey_path, 'rb') as f: - new_pubkey = f.read() + + # b/384813199: handles the pre-signed com.android.virt.apex in GSI. + if payload_key == 'PRESIGNED': + with tempfile.NamedTemporaryFile() as virt_apex_temp_file: + virt_apex_temp_file.write(input_tf_zip.read(virt_apex_path)) + virt_apex_temp_file.flush() + new_pubkey = GetMicrodroidVbmetaKey(virt_apex_temp_file.name, + misc_info['avb_avbtool']) + else: + new_pubkey_path = common.ExtractAvbPublicKey( + misc_info['avb_avbtool'], payload_key) + with open(new_pubkey_path, 'rb') as f: + new_pubkey = f.read() + pubkey_info = copy.copy( input_tf_zip.getinfo("PREBUILT_IMAGES/pvmfw_embedded.avbpubkey")) old_pubkey = input_tf_zip.read(pubkey_info.filename) diff --git a/tools/releasetools/test_common.py b/tools/releasetools/test_common.py index 89933a00fc..62f425ae6e 100644 --- a/tools/releasetools/test_common.py +++ b/tools/releasetools/test_common.py @@ -2157,3 +2157,11 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase): 'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys', 'ro.product.odm.device': 'coral', }, copied_props.build_props) + + +class DeviceSpecificParamsTest(test_utils.ReleaseToolsTestCase): + + def test_missingSource(self): + common.OPTIONS.device_specific = '/does_not_exist' + ds = DeviceSpecificParams() + self.assertIsNone(ds.module) diff --git a/tools/sbom/Android.bp b/tools/sbom/Android.bp index 6901b06720..7e2840fae1 100644 --- a/tools/sbom/Android.bp +++ b/tools/sbom/Android.bp @@ -33,6 +33,13 @@ python_binary_host { ], } +python_library_host { + name: "compliance_metadata", + srcs: [ + "compliance_metadata.py", + ], +} + python_binary_host { name: "gen_sbom", srcs: [ @@ -44,6 +51,7 @@ python_binary_host { }, }, libs: [ + "compliance_metadata", "metadata_file_proto_py", "libprotobuf-python", "sbom_lib", @@ -109,3 +117,19 @@ python_binary_host { "sbom_lib", ], } + +python_binary_host { + name: "gen_notice_xml", + srcs: [ + "gen_notice_xml.py", + ], + version: { + py3: { + embedded_launcher: true, + }, + }, + libs: [ + "compliance_metadata", + "metadata_file_proto_py", + ], +} diff --git a/tools/sbom/compliance_metadata.py b/tools/sbom/compliance_metadata.py new file mode 100644 index 0000000000..502c110154 --- /dev/null +++ b/tools/sbom/compliance_metadata.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlite3 + +class MetadataDb: + def __init__(self, db): + self.conn = sqlite3.connect(':memory:') + self.conn.row_factory = sqlite3.Row + with sqlite3.connect(db) as c: + c.backup(self.conn) + self.reorg() + + def reorg(self): + # package_license table + self.conn.execute("create table package_license as " + "select name as package, pkg_default_applicable_licenses as license " + "from modules " + "where module_type = 'package' ") + cursor = self.conn.execute("select package,license from package_license where license like '% %'") + multi_licenses_packages = cursor.fetchall() + cursor.close() + rows = [] + for p in multi_licenses_packages: + licenses = p['license'].strip().split(' ') + for lic in licenses: + rows.append((p['package'], lic)) + self.conn.executemany('insert into package_license values (?, ?)', rows) + self.conn.commit() + + self.conn.execute("delete from package_license where license like '% %'") + self.conn.commit() + + # module_license table + self.conn.execute("create table module_license as " + "select distinct name as module, package, licenses as license " + "from modules " + "where licenses != '' ") + cursor = self.conn.execute("select module,package,license from module_license where license like '% %'") + multi_licenses_modules = cursor.fetchall() + cursor.close() + rows = [] + for m in multi_licenses_modules: + licenses = m['license'].strip().split(' ') + for lic in licenses: + rows.append((m['module'], m['package'],lic)) + self.conn.executemany('insert into module_license values (?, ?, ?)', rows) + self.conn.commit() + + self.conn.execute("delete from module_license where license like '% %'") + self.conn.commit() + + # module_installed_file table + self.conn.execute("create table module_installed_file as " + "select id as module_id, name as module_name, package, installed_files as installed_file " + "from modules " + "where installed_files != '' ") + cursor = self.conn.execute("select module_id, module_name, package, installed_file " + "from module_installed_file where installed_file like '% %'") + multi_installed_file_modules = cursor.fetchall() + cursor.close() + rows = [] + for m in multi_installed_file_modules: + installed_files = m['installed_file'].strip().split(' ') + for f in installed_files: + rows.append((m['module_id'], m['module_name'], m['package'], f)) + self.conn.executemany('insert into module_installed_file values (?, ?, ?, ?)', rows) + self.conn.commit() + + self.conn.execute("delete from module_installed_file where installed_file like '% %'") + self.conn.commit() + + # module_built_file table + self.conn.execute("create table module_built_file as " + "select id as module_id, name as module_name, package, built_files as built_file " + "from modules " + "where built_files != '' ") + cursor = self.conn.execute("select module_id, module_name, package, built_file " + "from module_built_file where built_file like '% %'") + multi_built_file_modules = cursor.fetchall() + cursor.close() + rows = [] + for m in multi_built_file_modules: + built_files = m['built_file'].strip().split(' ') + for f in built_files: + rows.append((m['module_id'], m['module_name'], m['package'], f)) + self.conn.executemany('insert into module_built_file values (?, ?, ?, ?)', rows) + self.conn.commit() + + self.conn.execute("delete from module_built_file where built_file like '% %'") + self.conn.commit() + + + # Indexes + self.conn.execute('create index idx_modules_id on modules (id)') + self.conn.execute('create index idx_modules_name on modules (name)') + self.conn.execute('create index idx_package_licnese_package on package_license (package)') + self.conn.execute('create index idx_package_licnese_license on package_license (license)') + self.conn.execute('create index idx_module_licnese_module on module_license (module)') + self.conn.execute('create index idx_module_licnese_license on module_license (license)') + self.conn.execute('create index idx_module_installed_file_module_id on module_installed_file (module_id)') + self.conn.execute('create index idx_module_installed_file_installed_file on module_installed_file (installed_file)') + self.conn.execute('create index idx_module_built_file_module_id on module_built_file (module_id)') + self.conn.execute('create index idx_module_built_file_built_file on module_built_file (built_file)') + self.conn.commit() + + def dump_debug_db(self, debug_db): + with sqlite3.connect(debug_db) as c: + self.conn.backup(c) + + def get_installed_files(self): + # Get all records from table make_metadata, which contains all installed files and corresponding make modules' metadata + cursor = self.conn.execute('select installed_file, module_path, is_prebuilt_make_module, product_copy_files, kernel_module_copy_files, is_platform_generated, license_text from make_metadata') + rows = cursor.fetchall() + cursor.close() + installed_files_metadata = [] + for row in rows: + metadata = dict(zip(row.keys(), row)) + installed_files_metadata.append(metadata) + return installed_files_metadata + + def get_installed_file_in_dir(self, dir): + dir = dir.removesuffix('/') + cursor = self.conn.execute( + 'select installed_file, module_path, is_prebuilt_make_module, product_copy_files, ' + ' kernel_module_copy_files, is_platform_generated, license_text ' + 'from make_metadata ' + 'where installed_file like ?', (dir + '/%',)) + rows = cursor.fetchall() + cursor.close() + installed_files_metadata = [] + for row in rows: + metadata = dict(zip(row.keys(), row)) + installed_files_metadata.append(metadata) + return installed_files_metadata + + def get_soong_modules(self): + # Get all records from table modules, which contains metadata of all soong modules + cursor = self.conn.execute('select name, package, package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files from modules') + rows = cursor.fetchall() + cursor.close() + soong_modules = [] + for row in rows: + soong_module = dict(zip(row.keys(), row)) + soong_modules.append(soong_module) + return soong_modules + + def get_package_licenses(self, package): + cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' + 'from package_license pl join modules m on pl.license = m.name ' + 'where pl.package = ?', + ('//' + package,)) + rows = cursor.fetchall() + licenses = {} + for r in rows: + licenses[r['name']] = r['license_text'] + return licenses + + def get_module_licenses(self, module_name, package): + licenses = {} + # If property "licenses" is defined on module + cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' + 'from module_license ml join modules m on ml.license = m.name ' + 'where ml.module = ? and ml.package = ?', + (module_name, package)) + rows = cursor.fetchall() + for r in rows: + licenses[r['name']] = r['license_text'] + if len(licenses) > 0: + return licenses + + # Use default package license + cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' + 'from package_license pl join modules m on pl.license = m.name ' + 'where pl.package = ?', + ('//' + package,)) + rows = cursor.fetchall() + for r in rows: + licenses[r['name']] = r['license_text'] + return licenses + + def get_soong_module_of_installed_file(self, installed_file): + cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files ' + 'from modules m join module_installed_file mif on m.id = mif.module_id ' + 'where mif.installed_file = ?', + (installed_file,)) + rows = cursor.fetchall() + cursor.close() + if rows: + soong_module = dict(zip(rows[0].keys(), rows[0])) + return soong_module + + return None + + def get_soong_module_of_built_file(self, built_file): + cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files ' + 'from modules m join module_built_file mbf on m.id = mbf.module_id ' + 'where mbf.built_file = ?', + (built_file,)) + rows = cursor.fetchall() + cursor.close() + if rows: + soong_module = dict(zip(rows[0].keys(), rows[0])) + return soong_module + + return None
\ No newline at end of file diff --git a/tools/sbom/gen_notice_xml.py b/tools/sbom/gen_notice_xml.py new file mode 100644 index 0000000000..8478b1fdd4 --- /dev/null +++ b/tools/sbom/gen_notice_xml.py @@ -0,0 +1,224 @@ +# !/usr/bin/env python3 +# +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Generate NOTICE.xml.gz of a partition. +Usage example: + gen_notice_xml.py --output_file out/soong/.intermediate/.../NOTICE.xml.gz \ + --metadata out/soong/compliance-metadata/aosp_cf_x86_64_phone/compliance-metadata.db \ + --partition system \ + --product_out out/target/vsoc_x86_64 \ + --soong_out out/soong +""" + +import argparse +import compliance_metadata +import google.protobuf.text_format as text_format +import gzip +import hashlib +import metadata_file_pb2 +import os +import queue +import xml.sax.saxutils + + +FILE_HEADER = '''\ +<?xml version="1.0" encoding="utf-8"?> +<licenses> +''' +FILE_FOOTER = '''\ +</licenses> +''' + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') + parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') + parser.add_argument('--output_file', required=True, help='The path of the generated NOTICE.xml.gz file.') + parser.add_argument('--partition', required=True, help='The name of partition for which the NOTICE.xml.gz is generated.') + parser.add_argument('--metadata', required=True, help='The path of compliance metadata DB file.') + parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.') + parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong') + + return parser.parse_args() + + +def log(*info): + if args.verbose: + for i in info: + print(i) + + +def new_file_name_tag(file_metadata, package_name, content_id): + file_path = file_metadata['installed_file'].removeprefix(args.product_out) + lib = 'Android' + if package_name: + lib = package_name + return f'<file-name contentId="{content_id}" lib="{lib}">{file_path}</file-name>\n' + + +def new_file_content_tag(content_id, license_text): + escaped_license_text = xml.sax.saxutils.escape(license_text, {'\t': '	', '\n': '
', '\r': '
'}) + return f'<file-content contentId="{content_id}"><![CDATA[{escaped_license_text}]]></file-content>\n\n' + +def get_metadata_file_path(file_metadata): + """Search for METADATA file of a package and return its path.""" + metadata_path = '' + if file_metadata['module_path']: + metadata_path = file_metadata['module_path'] + elif file_metadata['kernel_module_copy_files']: + metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) + + while metadata_path and not os.path.exists(metadata_path + '/METADATA'): + metadata_path = os.path.dirname(metadata_path) + + return metadata_path + +def md5_file_content(filepath): + h = hashlib.md5() + with open(filepath, 'rb') as f: + h.update(f.read()) + return h.hexdigest() + +def get_transitive_static_dep_modules(installed_file_metadata, db): + # Find all transitive static dep files of the installed files + q = queue.Queue() + if installed_file_metadata['static_dep_files']: + for f in installed_file_metadata['static_dep_files'].split(' '): + q.put(f) + if installed_file_metadata['whole_static_dep_files']: + for f in installed_file_metadata['whole_static_dep_files'].split(' '): + q.put(f) + + static_dep_files = {} + while not q.empty(): + dep_file = q.get() + if dep_file in static_dep_files: + # It has been processed + continue + + soong_module = db.get_soong_module_of_built_file(dep_file) + if not soong_module: + continue + + static_dep_files[dep_file] = soong_module + + if soong_module['static_dep_files']: + for f in soong_module['static_dep_files'].split(' '): + if f not in static_dep_files: + q.put(f) + if soong_module['whole_static_dep_files']: + for f in soong_module['whole_static_dep_files'].split(' '): + if f not in static_dep_files: + q.put(f) + + return static_dep_files.values() + +def main(): + global args + args = get_args() + log('Args:', vars(args)) + + global db + db = compliance_metadata.MetadataDb(args.metadata) + if args.debug: + db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db') + + # NOTICE.xml + notice_xml_file_path = os.path.dirname(args.output_file) + '/NOTICE.xml' + with open(notice_xml_file_path, 'w', encoding="utf-8") as notice_xml_file: + notice_xml_file.write(FILE_HEADER) + + all_license_files = {} + for metadata in db.get_installed_file_in_dir(args.product_out + '/' + args.partition): + soong_module = db.get_soong_module_of_installed_file(metadata['installed_file']) + if soong_module: + metadata.update(soong_module) + else: + # For make modules soong_module_type should be empty + metadata['soong_module_type'] = '' + metadata['static_dep_files'] = '' + metadata['whole_static_dep_files'] = '' + + installed_file_metadata_list = [metadata] + if args.partition in ('vendor', 'product', 'system_ext'): + # For transitive static dependencies of an installed file, make it as if an installed file are + # also created from static dependency modules whose licenses are also collected + static_dep_modules = get_transitive_static_dep_modules(metadata, db) + for dep in static_dep_modules: + dep['installed_file'] = metadata['installed_file'] + installed_file_metadata_list.append(dep) + + for installed_file_metadata in installed_file_metadata_list: + package_name = 'Android' + licenses = {} + if installed_file_metadata['module_path']: + metadata_file_path = get_metadata_file_path(installed_file_metadata) + if metadata_file_path: + proto = metadata_file_pb2.Metadata() + with open(metadata_file_path + '/METADATA', 'rt') as f: + text_format.Parse(f.read(), proto) + if proto.name: + package_name = proto.name + if proto.third_party and proto.third_party.version: + if proto.third_party.version.startswith('v'): + package_name = package_name + '_' + proto.third_party.version + else: + package_name = package_name + '_v_' + proto.third_party.version + else: + package_name = metadata_file_path + if metadata_file_path.startswith('external/'): + package_name = metadata_file_path.removeprefix('external/') + + # Every license file is in a <file-content> element + licenses = db.get_module_licenses(installed_file_metadata.get('name', ''), installed_file_metadata['module_path']) + + # Installed file is from PRODUCT_COPY_FILES + elif metadata['product_copy_files']: + licenses['unused_name'] = metadata['license_text'] + + # Installed file is generated by the platform in builds + elif metadata['is_platform_generated']: + licenses['unused_name'] = metadata['license_text'] + + if licenses: + # Each value is a space separated filepath list + for license_files in licenses.values(): + if not license_files: + continue + for filepath in license_files.split(' '): + if filepath not in all_license_files: + all_license_files[filepath] = md5_file_content(filepath) + md5 = all_license_files[filepath] + notice_xml_file.write(new_file_name_tag(installed_file_metadata, package_name, md5)) + + # Licenses + processed_md5 = [] + for filepath, md5 in all_license_files.items(): + if md5 not in processed_md5: + processed_md5.append(md5) + with open(filepath, 'rt', errors='backslashreplace') as f: + notice_xml_file.write(new_file_content_tag(md5, f.read())) + + notice_xml_file.write(FILE_FOOTER) + + # NOTICE.xml.gz + with open(notice_xml_file_path, 'rb') as notice_xml_file, gzip.open(args.output_file, 'wb') as gz_file: + gz_file.writelines(notice_xml_file) + +if __name__ == '__main__': + main() diff --git a/tools/sbom/gen_sbom.py b/tools/sbom/gen_sbom.py index a203258b96..77bccbb73a 100644 --- a/tools/sbom/gen_sbom.py +++ b/tools/sbom/gen_sbom.py @@ -26,6 +26,7 @@ Usage example: """ import argparse +import compliance_metadata import datetime import google.protobuf.text_format as text_format import hashlib @@ -35,7 +36,6 @@ import queue import metadata_file_pb2 import sbom_data import sbom_writers -import sqlite3 # Package type PKG_SOURCE = 'SOURCE' @@ -92,6 +92,7 @@ THIRD_PARTY_IDENTIFIER_TYPES = [ 'SVN', 'Hg', 'Darcs', + 'Piper', 'VCS', 'Archive', 'PrebuiltByAlphabet', @@ -568,202 +569,16 @@ def get_all_transitive_static_dep_files_of_installed_files(installed_files_metad return sorted(all_static_dep_files.keys()) -class MetadataDb: - def __init__(self, db): - self.conn = sqlite3.connect(':memory') - self.conn.row_factory = sqlite3.Row - with sqlite3.connect(db) as c: - c.backup(self.conn) - self.reorg() - - def reorg(self): - # package_license table - self.conn.execute("create table package_license as " - "select name as package, pkg_default_applicable_licenses as license " - "from modules " - "where module_type = 'package' ") - cursor = self.conn.execute("select package,license from package_license where license like '% %'") - multi_licenses_packages = cursor.fetchall() - cursor.close() - rows = [] - for p in multi_licenses_packages: - licenses = p['license'].strip().split(' ') - for lic in licenses: - rows.append((p['package'], lic)) - self.conn.executemany('insert into package_license values (?, ?)', rows) - self.conn.commit() - - self.conn.execute("delete from package_license where license like '% %'") - self.conn.commit() - - # module_license table - self.conn.execute("create table module_license as " - "select distinct name as module, package, licenses as license " - "from modules " - "where licenses != '' ") - cursor = self.conn.execute("select module,package,license from module_license where license like '% %'") - multi_licenses_modules = cursor.fetchall() - cursor.close() - rows = [] - for m in multi_licenses_modules: - licenses = m['license'].strip().split(' ') - for lic in licenses: - rows.append((m['module'], m['package'],lic)) - self.conn.executemany('insert into module_license values (?, ?, ?)', rows) - self.conn.commit() - - self.conn.execute("delete from module_license where license like '% %'") - self.conn.commit() - - # module_installed_file table - self.conn.execute("create table module_installed_file as " - "select id as module_id, name as module_name, package, installed_files as installed_file " - "from modules " - "where installed_files != '' ") - cursor = self.conn.execute("select module_id, module_name, package, installed_file " - "from module_installed_file where installed_file like '% %'") - multi_installed_file_modules = cursor.fetchall() - cursor.close() - rows = [] - for m in multi_installed_file_modules: - installed_files = m['installed_file'].strip().split(' ') - for f in installed_files: - rows.append((m['module_id'], m['module_name'], m['package'], f)) - self.conn.executemany('insert into module_installed_file values (?, ?, ?, ?)', rows) - self.conn.commit() - - self.conn.execute("delete from module_installed_file where installed_file like '% %'") - self.conn.commit() - - # module_built_file table - self.conn.execute("create table module_built_file as " - "select id as module_id, name as module_name, package, built_files as built_file " - "from modules " - "where built_files != '' ") - cursor = self.conn.execute("select module_id, module_name, package, built_file " - "from module_built_file where built_file like '% %'") - multi_built_file_modules = cursor.fetchall() - cursor.close() - rows = [] - for m in multi_built_file_modules: - built_files = m['installed_file'].strip().split(' ') - for f in built_files: - rows.append((m['module_id'], m['module_name'], m['package'], f)) - self.conn.executemany('insert into module_built_file values (?, ?, ?, ?)', rows) - self.conn.commit() - - self.conn.execute("delete from module_built_file where built_file like '% %'") - self.conn.commit() - - - # Indexes - self.conn.execute('create index idx_modules_id on modules (id)') - self.conn.execute('create index idx_modules_name on modules (name)') - self.conn.execute('create index idx_package_licnese_package on package_license (package)') - self.conn.execute('create index idx_package_licnese_license on package_license (license)') - self.conn.execute('create index idx_module_licnese_module on module_license (module)') - self.conn.execute('create index idx_module_licnese_license on module_license (license)') - self.conn.execute('create index idx_module_installed_file_module_id on module_installed_file (module_id)') - self.conn.execute('create index idx_module_installed_file_installed_file on module_installed_file (installed_file)') - self.conn.execute('create index idx_module_built_file_module_id on module_built_file (module_id)') - self.conn.execute('create index idx_module_built_file_built_file on module_built_file (built_file)') - self.conn.commit() - - if args.debug: - with sqlite3.connect(os.path.dirname(args.metadata) + '/compliance-metadata-debug.db') as c: - self.conn.backup(c) - - - def get_installed_files(self): - # Get all records from table make_metadata, which contains all installed files and corresponding make modules' metadata - cursor = self.conn.execute('select installed_file, module_path, is_prebuilt_make_module, product_copy_files, kernel_module_copy_files, is_platform_generated, license_text from make_metadata') - rows = cursor.fetchall() - cursor.close() - installed_files_metadata = [] - for row in rows: - metadata = dict(zip(row.keys(), row)) - installed_files_metadata.append(metadata) - return installed_files_metadata - - def get_soong_modules(self): - # Get all records from table modules, which contains metadata of all soong modules - cursor = self.conn.execute('select name, package, package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files from modules') - rows = cursor.fetchall() - cursor.close() - soong_modules = [] - for row in rows: - soong_module = dict(zip(row.keys(), row)) - soong_modules.append(soong_module) - return soong_modules - - def get_package_licenses(self, package): - cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' - 'from package_license pl join modules m on pl.license = m.name ' - 'where pl.package = ?', - ('//' + package,)) - rows = cursor.fetchall() - licenses = {} - for r in rows: - licenses[r['name']] = r['license_text'] - return licenses - - def get_module_licenses(self, module_name, package): - licenses = {} - # If property "licenses" is defined on module - cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' - 'from module_license ml join modules m on ml.license = m.name ' - 'where ml.module = ? and ml.package = ?', - (module_name, package)) - rows = cursor.fetchall() - for r in rows: - licenses[r['name']] = r['license_text'] - if len(licenses) > 0: - return licenses - - # Use default package license - cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text ' - 'from package_license pl join modules m on pl.license = m.name ' - 'where pl.package = ?', - ('//' + package,)) - rows = cursor.fetchall() - for r in rows: - licenses[r['name']] = r['license_text'] - return licenses - - def get_soong_module_of_installed_file(self, installed_file): - cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files ' - 'from modules m join module_installed_file mif on m.id = mif.module_id ' - 'where mif.installed_file = ?', - (installed_file,)) - rows = cursor.fetchall() - cursor.close() - if rows: - soong_module = dict(zip(rows[0].keys(), rows[0])) - return soong_module - - return None - - def get_soong_module_of_built_file(self, built_file): - cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files ' - 'from modules m join module_built_file mbf on m.id = mbf.module_id ' - 'where mbf.built_file = ?', - (built_file,)) - rows = cursor.fetchall() - cursor.close() - if rows: - soong_module = dict(zip(rows[0].keys(), rows[0])) - return soong_module - - return None - - def main(): global args args = get_args() log('Args:', vars(args)) global db - db = MetadataDb(args.metadata) + db = compliance_metadata.MetadataDb(args.metadata) + if args.debug: + db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db') + global metadata_file_protos metadata_file_protos = {} global licenses_text @@ -894,8 +709,17 @@ def main(): 'installed_file': dep_file, 'is_prebuilt_make_module': False } - file_metadata.update(db.get_soong_module_of_built_file(dep_file)) - add_package_of_file(file_id, file_metadata, doc, report) + soong_module = db.get_soong_module_of_built_file(dep_file) + if not soong_module: + continue + file_metadata.update(soong_module) + if is_source_package(file_metadata) or is_prebuilt_package(file_metadata): + add_package_of_file(file_id, file_metadata, doc, report) + else: + # Other static lib files are generated from the platform + doc.add_relationship(sbom_data.Relationship(id1=file_id, + relationship=sbom_data.RelationshipType.GENERATED_FROM, + id2=sbom_data.SPDXID_PLATFORM)) # Add relationships for static deps of static libraries add_static_deps_of_file(file_id, file_metadata, doc) diff --git a/tools/signapk/src/com/android/signapk/SignApk.java b/tools/signapk/src/com/android/signapk/SignApk.java index 6b2341bc80..654e19675d 100644 --- a/tools/signapk/src/com/android/signapk/SignApk.java +++ b/tools/signapk/src/com/android/signapk/SignApk.java @@ -302,7 +302,6 @@ class SignApk { final KeyStore keyStore, final String keyName) throws CertificateException, KeyStoreException, NoSuchAlgorithmException, UnrecoverableKeyException, UnrecoverableEntryException { - final Key key = keyStore.getKey(keyName, readPassword(keyName)); final PrivateKeyEntry privateKeyEntry = (PrivateKeyEntry) keyStore.getEntry(keyName, null); if (privateKeyEntry == null) { throw new Error( diff --git a/tools/tool_event_logger/OWNERS b/tools/tool_event_logger/OWNERS index b692c9edf3..e93d20f126 100644 --- a/tools/tool_event_logger/OWNERS +++ b/tools/tool_event_logger/OWNERS @@ -1,4 +1,3 @@ include platform/tools/asuite:/OWNERS zhuoyao@google.com -hzalek@google.com
\ No newline at end of file |