Merge pull request #582 from seedvault-app/feature/e2e-test-logs
Fix and improve E2E tests
diff --git a/.cirrus.yml b/.cirrus.yml
index 8566502..613c4b0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,7 +1,7 @@
task:
name: Build with AOSP
only_if: $CIRRUS_PR_LABELS =~ ".*aosp-build.*"
- timeout_in: 120m
+ timeout_in: 70m
container:
image: ubuntu:23.04
cpu: 8
@@ -10,4 +10,4 @@
- ./.github/scripts/build_aosp.sh aosp_arm64 userdebug android-14.0.0_r1
always:
seedvault_artifacts:
- path: $CIRRUS_WORKING_DIR/Seedvault.apk
+ path: Seedvault.apk
diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh
new file mode 100755
index 0000000..48a8657
--- /dev/null
+++ b/.github/scripts/run_tests.sh
@@ -0,0 +1,35 @@
+adb root
+sleep 5
+adb remount
+
+echo "Installing Seedvault app..."
+adb shell mkdir -p /system/priv-app/Seedvault
+adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk
+
+echo "Installing Seedvault permissions..."
+adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml
+adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml
+
+echo "Setting Seedvault transport..."
+sleep 10
+adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport
+
+large_test_exit_code=0
+./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$?
+
+adb pull /sdcard/seedvault_test_results
+
+if [ "$large_test_exit_code" -ne 0 ]; then
+ echo 'Large tests failed.'
+ exit 1
+fi
+
+medium_test_exit_code=0
+./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$?
+
+if [ "$medium_test_exit_code" -ne 0 ]; then
+ echo 'Medium tests failed.'
+ exit 1
+fi
+
+exit 0
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 82ca5c4..52dd750 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1,6 +1,10 @@
name: Build
on: [push, pull_request]
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
jobs:
build:
name: Build
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 925a65b..a624c77 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,14 +13,13 @@
jobs:
instrumentation_tests:
runs-on: macos-11
+ if: github.repository == 'seedvault-app/seedvault'
+ timeout-minutes: 80
strategy:
fail-fast: false
matrix:
- android_target: [33, 34]
- emulator_type: [default, google_apis]
- exclude:
- - android_target: 34
- emulator_type: default
+ android_target: [ 33, 34 ]
+ emulator_type: [ default ]
steps:
- name: Checkout Code
uses: actions/checkout@v3
@@ -31,88 +30,33 @@
java-version: '17'
cache: 'gradle'
- - name: AVD cache
- uses: actions/cache@v3
- id: avd-cache
- with:
- path: |
- ~/.android/avd/*
- ~/.android/adb*
- key: aosp-${{ matrix.emulator_type }}-${{ matrix.android_target }}-${{ runner.os }}
-
- name: Build Release APK
run: ./gradlew :app:assembleRelease
- - name: Create AVD snapshot
- if: steps.avd-cache.outputs.cache-hit != 'true'
- uses: reactivecircus/android-emulator-runner@v2
- with:
- api-level: ${{ matrix.android_target }}
- target: ${{ matrix.emulator_type }}
- arch: x86_64
- force-avd-creation: false
- emulator-options: -writable-system -no-snapshot-load -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
- disable-animations: true
- script: |
- ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64"
- echo "Generated AVD snapshot for caching."
-
- name: Assemble tests
run: ./gradlew :app:assembleAndroidTest
- name: Run tests
- uses: reactivecircus/android-emulator-runner@v2
+ uses: Wandalen/wretry.action@v1.3.0
with:
- api-level: ${{ matrix.android_target }}
- target: ${{ matrix.emulator_type }}
- arch: x86_64
- force-avd-creation: false
- emulator-options: -writable-system -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
- profile: pixel_6a
- heap-size: '512M'
- ram-size: '4096M'
- disk-size: '14G'
- sdcard-path-or-size: '4096M'
- cores: 3
- disable-animations: false
- script: |
- adb root
- sleep 5
- adb remount
+ attempt_limit: 3
+ action: reactivecircus/android-emulator-runner@v2
+ with: |
+ api-level: ${{ matrix.android_target }}
+ target: ${{ matrix.emulator_type }}
+ arch: x86_64
+ force-avd-creation: true
+ emulator-options: -cores 2 -writable-system -no-snapshot-load -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+ disk-size: '14G'
+ sdcard-path-or-size: '4096M'
+ disable-animations: true
+ script: |
+ ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64"
+ ./.github/scripts/run_tests.sh
- echo "Installing Seedvault app..."
- adb shell mkdir -p /system/priv-app/Seedvault
- adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk
-
- echo "Installing Seedvault permissions..."
- adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml
- adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml
-
- echo "Setting Seedvault transport..."
- sleep 10
- adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport
-
- wget --quiet https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz
- adb shell mkdir -p /sdcard/seedvault_baseline
- adb push backup.tar.gz /sdcard/seedvault_baseline
- adb wait-for-device
- adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline
- adb shell rm /sdcard/seedvault_baseline/backup.tar.gz
-
- large_test_exit_code=0
- ./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$?
-
- medium_test_exit_code=0
- ./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$?
-
- adb pull /sdcard/seedvault_test_videos
-
- if [ $large_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi
- if [ $medium_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi
-
- - name: Upload screenshots and videos
+ - name: Upload test results
if: always()
uses: actions/upload-artifact@v3
with:
- name: seedvault_test_videos
- path: seedvault_test_videos/**/*.mp4
+ name: ${{ matrix.emulator_type }}-${{ matrix.android_target }}-results
+ path: seedvault_test_results/**/*
diff --git a/app/build.gradle b/app/build.gradle
index 56508f3..fcb46e4 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -162,7 +162,7 @@
androidTestImplementation 'androidx.test:runner:1.4.0'
androidTestImplementation 'androidx.test:rules:1.4.0'
androidTestImplementation 'androidx.test.ext:junit:1.1.3'
- androidTestImplementation "io.mockk:mockk-android:$mockk_version"
+ androidTestImplementation "io.mockk:mockk-android:1.13.8"
androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0'
}
@@ -200,7 +200,7 @@
doFirst {
commandLine "${project.projectDir}/development/scripts/provision_emulator.sh",
"seedvault",
- "system-images;android-34;google_apis;x86_64"
+ "system-images;android-34;default;x86_64"
environment "ANDROID_HOME", android.sdkDirectory.absolutePath
environment "JAVA_HOME", System.properties['java.home']
diff --git a/app/development/scripts/provision_emulator.sh b/app/development/scripts/provision_emulator.sh
index cfdcc88..fef04b8 100755
--- a/app/development/scripts/provision_emulator.sh
+++ b/app/development/scripts/provision_emulator.sh
@@ -20,7 +20,7 @@
ROOT_PROJECT_DIR=$SCRIPT_DIR/../../..
echo "Downloading system image..."
-$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE"
+yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE"
# create AVD if it doesn't exist
if $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager list avd | grep -q "$EMULATOR_NAME"; then
diff --git a/app/development/scripts/start_emulator.sh b/app/development/scripts/start_emulator.sh
index 4c07118..b211348 100755
--- a/app/development/scripts/start_emulator.sh
+++ b/app/development/scripts/start_emulator.sh
@@ -14,9 +14,5 @@
EMULATOR_NAME=$1
-SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
-DEVELOPMENT_DIR=$SCRIPT_DIR/..
-ROOT_PROJECT_DIR=$SCRIPT_DIR/../../..
-
echo "Starting emulator..."
nohup $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -gpu swiftshader_indirect -writable-system >/dev/null 2>&1 &
diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt
index b386214..86f14a2 100644
--- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt
+++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt
@@ -44,7 +44,7 @@
companion object {
private const val TEST_STORAGE_FOLDER = "seedvault_test"
- private const val TEST_VIDEO_FOLDER = "seedvault_test_videos"
+ private const val TEST_VIDEO_FOLDER = "seedvault_test_results"
}
val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath
@@ -106,19 +106,23 @@
uiAutomation.executeShellCommand(command).close()
}
+ fun testResultFilename(testName: String): String {
+ val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss")
+ val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
+ return "${timeStamp}_${testName.replace(" ", "_")}"
+ }
+
@OptIn(DelicateCoroutinesApi::class)
@WorkerThread
- suspend fun startScreenRecord(
+ suspend fun startRecordingTest(
keepRecordingScreen: AtomicBoolean,
testName: String,
) {
- val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss")
- val timeStamp = simpleDateFormat.format(Calendar.getInstance().time)
- val fileName = "${timeStamp}_${testName.replace(" ", "_")}"
-
val folder = testVideoPath
runCommand("mkdir -p $folder")
+ val fileName = testResultFilename(testName)
+
// screen record automatically stops after 3 minutes
// we need to block on a loop and split it into multiple files
GlobalScope.launch(Dispatchers.IO) {
@@ -131,10 +135,16 @@
}
@WorkerThread
- fun stopScreenRecord(keepRecordingScreen: AtomicBoolean) {
+ fun stopRecordingTest(
+ keepRecordingScreen: AtomicBoolean,
+ testName: String,
+ ) {
keepRecordingScreen.set(false)
-
runCommand("pkill -2 screenrecord")
+
+ // write logcat to file
+ val fileName = testResultFilename(testName)
+ runCommand("logcat -d -f $testVideoPath/$fileName.log")
}
fun uninstallPackages(packages: Collection<PackageInfo>) {
diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt
index 423c461..2d2be5f 100644
--- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt
+++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt
@@ -38,13 +38,13 @@
resetApplicationState()
clearTestBackups()
- startScreenRecord(keepRecordingScreen, name.methodName)
+ startRecordingTest(keepRecordingScreen, name.methodName)
restoreBaselineBackup()
}
@After
open fun tearDown() {
- stopScreenRecord(keepRecordingScreen)
+ stopRecordingTest(keepRecordingScreen, name.methodName)
}
/**
diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt
index fdcdd67..bc13414 100644
--- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt
+++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt
@@ -11,6 +11,10 @@
abstract class UiDeviceScreen<T> {
+ companion object {
+ private const val SELECTOR_TIMEOUT = 180000L
+ }
+
operator fun invoke(function: T.() -> Unit) {
function.invoke(this as T)
}
@@ -18,8 +22,11 @@
fun UiObject.scrollTo(
scrollSelector: UiSelector = UiSelector().className(ScrollView::class.java),
): UiObject {
- UiScrollable(scrollSelector).scrollIntoView(this)
- waitForExists(15000)
+ val uiScrollable = UiScrollable(scrollSelector)
+ uiScrollable.waitForExists(SELECTOR_TIMEOUT)
+ uiScrollable.scrollIntoView(this)
+ waitForExists(SELECTOR_TIMEOUT)
+
sleep(2000)
return this
}
@@ -32,6 +39,6 @@
private fun device() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
.also {
- Configurator.getInstance().waitForSelectorTimeout = 60000
+ Configurator.getInstance().waitForSelectorTimeout = SELECTOR_TIMEOUT
}
}