Add simple benchmark & a test framework (#2022)

See #2019, #2020 and #2021 for failed attempts at using a benchmark framework. So, for now I'll just go with timing the results manually. 

For trying this out:
- Backup your current account first, maybe there are some bugs in switching accounts.
- You can run benchmarks on either an emulated device or a real device. For better benchmark results, you should run the benchmark on a real device and make sure that the core is compiled in release mode.
- Disable animations on your device, otherwise the test may fail. (Developer options -> Window animation scale, Transition animation scale & Animatior duration scale -> set all three of them to 0x)
- In Android Studio: File -> Sync project with gradle files
- In Android Studio: Run -> Edit configurations -> `+` -> Android Instrumented test -> Either select a specific class or select "All in Module" -> OK -> Select your configuration in the toolbar -> Click on the green "run" button in the toolbar to run the tests

When the benchmark is done, you will get a result like `MEASURED RESULTS (Benchmark) - Going thorough all 10 chats: 11635,11207,11363,11352,11279,11183,11137,11145,11032,11057`, Paste `11635,11207,11363,11352,11279,11183,11137,11145,11032,11057` into a cell in a LibreOffice spreadsheet, do `Data -> Text to columns`, choose `,` as a separator, hit `OK`, and create a diagram.
This commit is contained in:
Hocuri 2021-08-11 14:01:15 +02:00 committed by GitHub
parent e8cf64d5ea
commit f7b74e4963
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 198 additions and 10 deletions

View file

@ -155,6 +155,32 @@ environment.
`export PATH=$PATH:$ANDROID_NDK`.
# Run UI Tests and Benchmarks
- You don't necessarily need a dedicated testing device.
Backup your current account first, maybe there are some bugs in switching accounts.
- You can run benchmarks on either an emulated device or a real device.
You need at least Android 9. For better benchmark results,
you should run the benchmark on a real device and make sure that the core is compiled in release mode.
- Disable animations on your device, otherwise the test may fail:
at "Developer options"
set all of "Window animation scale", "Transition animation scale" and "Animatior duration scale" to 0x
- In Android Studio: "File" / "Sync project with gradle files"
- In Android Studio: "Run" / "Edit configurations" / "+" / "Android Instrumented test":
Either select a specific class or select "All in Module" / "OK" /
Select your configuration in the toolbar / Click on the green "run" button in the toolbar to run the tests
When the benchmark is done, you will get a result like
`MEASURED RESULTS (Benchmark) - Going thorough all 10 chats: 11635,11207,11363,11352,11279,11183,11137,11145,11032,11057`.
You can paste `11635,11207,11363,11352,11279,11183,11137,11145,11032,11057`
into a cell in a LibreOffice spreadsheet, do "Data" / "Text to columns",
choose `,` as a separator, hit "OK", and create a diagram.
# Credits
The user interface classes are based on the Signal messenger.

View file

@ -0,0 +1,144 @@
package com.b44t.messenger;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
import androidx.test.espresso.contrib.RecyclerViewActions;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.LargeTest;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.thoughtcrime.securesms.ConversationListActivity;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.connect.AccountManager;
import org.thoughtcrime.securesms.connect.DcHelper;
import org.thoughtcrime.securesms.util.Prefs;
import static androidx.test.espresso.Espresso.onView;
import static androidx.test.espresso.Espresso.pressBack;
import static androidx.test.espresso.action.ViewActions.click;
import static androidx.test.espresso.action.ViewActions.replaceText;
import static androidx.test.espresso.action.ViewActions.typeText;
import static androidx.test.espresso.matcher.ViewMatchers.withContentDescription;
import static androidx.test.espresso.matcher.ViewMatchers.withHint;
import static androidx.test.espresso.matcher.ViewMatchers.withId;
import static androidx.test.espresso.matcher.ViewMatchers.withText;
import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
@RunWith(AndroidJUnit4.class)
@LargeTest
public class EnterChatsBenchmark {
// ==============================================================================================
// Set this to true if you already have at least 10 chats on your existing DeltaChat installation
// and want to traverse through them instead of 10 newly created chats
private final static boolean USE_EXISTING_CHATS = false;
// ==============================================================================================
private final static int GO_THROUGH_ALL_CHATS_N_TIMES = 8;
// ==============================================================================================
// PLEASE BACKUP YOUR ACCOUNT BEFORE RUNNING THIS!
// ==============================================================================================
private final static String TAG = EnterChatsBenchmark.class.getSimpleName();
@Rule
public ActivityScenarioRule<ConversationListActivity> activityRule = new ActivityScenarioRule<>(getConversationsListIntent());
private Intent getConversationsListIntent() {
Intent intent =
Intent.makeMainActivity(
new ComponentName(getInstrumentation().getTargetContext(), ConversationListActivity.class));
if (!USE_EXISTING_CHATS) {
Context context = getInstrumentation().getTargetContext();
AccountManager.getInstance().beginAccountCreation(context);
DcContext c = DcHelper.getContext(context);
c.setConfig("configured_addr", "alice@example.org");
c.setConfig("configured_mail_pw", "abcd");
c.setConfig("configured", "1");
}
return intent;
}
@Test
public void createAndEnterNChats() {
Prefs.setEnterSendsEnabled(getInstrumentation().getTargetContext(), true);
if (!USE_EXISTING_CHATS) {
createChatAndGoBack("Group #1", "Hello!", "Some links: https://testrun.org", "And a command: /help");
createChatAndGoBack("Group #2", "example.org, alice@example.org", "aaaaaaa", "bbbbbb");
createChatAndGoBack("Group #3", repeat("Some string ", 600), repeat("Another string", 200), "Hi!!!");
createChatAndGoBack("Group #4", "xyzabc", "Hi!!!!", "Let's meet!");
createChatAndGoBack("Group #5", repeat("aaaa", 40), "bbbbbbbbbbbbbbbbbb", "ccccccccccccccc");
createChatAndGoBack("Group #6", "aaaaaaaaaaa", repeat("Hi! ", 1000), "bbbbbbbbbb");
createChatAndGoBack("Group #7", repeat("abcdefg ", 500), repeat("xxxxx", 100), "yrrrrrrrrrrrrr");
createChatAndGoBack("Group #8", "and a number: 037362/384756", "ccccc", "Nice!");
createChatAndGoBack("Group #9", "ddddddddddddddddd", "zuuuuuuuuuuuuuuuu", "ccccc");
createChatAndGoBack("Group #10", repeat("xxxxxxyyyyy", 100), repeat("String!!", 10), "abcd");
}
String[] times = new String[GO_THROUGH_ALL_CHATS_N_TIMES];
for (int i = 0; i<GO_THROUGH_ALL_CHATS_N_TIMES; i++) {
times[i] = "" + timeGoToAllChats();
}
Log.i(TAG, "MEASURED RESULTS (Benchmark) - Going thorough all 10 chats: " + String.join(",", times));
}
private long timeGoToAllChats() {
long start = System.currentTimeMillis();
for (int i=0; i<10; i++) {
onView(withId(R.id.list)).perform(RecyclerViewActions.actionOnItemAtPosition(i, click()));
pressBack();
}
long diff = System.currentTimeMillis() - start;
Log.i(TAG, "Measured (Benchmark): Going through all chats took " + diff + "ms");
return diff;
}
private String repeat(String string, int n) {
StringBuilder s = new StringBuilder();
for (int i = 0; i < n; i++) {
s.append(string);
}
return s.toString();
}
private void createChatAndGoBack(String groupName, String text1, String text2, String text3) {
onView(withId(R.id.fab)).perform(click());
onView(withText(R.string.menu_new_group)).perform(click());
onView(withHint(R.string.group_name)).perform(replaceText(groupName));
onView(withContentDescription(R.string.group_create_button)).perform(click());
sendText(text1);
sendText(text2);
sendText(text3);
sendText(text1);
sendText(text2);
sendText(text3);
pressBack();
pressBack();
}
private void sendText(String text1) {
onView(withHint(R.string.chat_input_placeholder)).perform(replaceText(text1));
onView(withHint(R.string.chat_input_placeholder)).perform(typeText("\n"));
}
@After
public void removeMockAccount() {
if (!USE_EXISTING_CHATS) {
Context context = getInstrumentation().getTargetContext();
DcAccounts accounts = DcHelper.getAccounts(context);
DcContext selectedAccount = accounts.getSelectedAccount();
accounts.removeAccount(selectedAccount.getAccountId());
}
}
}

View file

@ -75,6 +75,13 @@ dependencies {
testImplementation 'org.powermock:powermock-module-junit4-rule:1.6.1'
testImplementation 'org.powermock:powermock-classloading-xstream:1.6.1'
androidTestImplementation 'androidx.test:runner:1.2.0'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
androidTestImplementation 'androidx.test.espresso:espresso-contrib:3.3.0'
androidTestImplementation 'androidx.test:rules:1.2.0'
androidTestImplementation 'androidx.test.ext:junit:1.1.1'
androidTestImplementation 'com.android.support:support-annotations:24.0.0'
androidTestImplementation ('org.assertj:assertj-core:1.7.1') {
exclude group: 'org.hamcrest', module: 'hamcrest-core'
}
@ -111,6 +118,8 @@ android {
ndk {
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
}
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}
compileOptions {
@ -197,8 +206,8 @@ android {
assets.srcDirs = ['assets']
jniLibs.srcDirs = ['libs']
}
test {
java.srcDirs = ['test']
androidTest {
java.srcDirs = ['androidTest']
}
}

View file

@ -2,11 +2,12 @@ package org.thoughtcrime.securesms;
import android.content.Intent;
import android.os.Bundle;
import android.util.Log;
import androidx.annotation.IdRes;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.fragment.app.Fragment;
import android.util.Log;
import org.thoughtcrime.securesms.connect.DcHelper;
import org.thoughtcrime.securesms.service.GenericForegroundService;

View file

@ -3,10 +3,7 @@ package org.thoughtcrime.securesms.connect;
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.SharedPreferences;
import android.preference.PreferenceManager;
import android.util.Log;
import android.widget.Toast;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AlertDialog;
@ -19,7 +16,6 @@ import org.thoughtcrime.securesms.ConversationListActivity;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.WelcomeActivity;
import org.thoughtcrime.securesms.notifications.NotificationCenter;
import org.thoughtcrime.securesms.util.Prefs;
import org.thoughtcrime.securesms.util.task.ProgressDialogAsyncTask;
import java.io.File;
@ -87,7 +83,7 @@ public class AccountManager {
// add accounts
private void beginAccountCreation(Context context) {
public void beginAccountCreation(Context context) {
DcHelper.getAccounts(context).addAccount();
resetDcContext(context);
}

View file

@ -8,11 +8,11 @@ import android.net.Uri;
import android.preference.PreferenceManager;
import android.provider.ContactsContract;
import android.provider.Settings;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.core.app.NotificationCompat;
import com.b44t.messenger.DcChat;
import com.b44t.messenger.DcContext;
import com.mapbox.mapboxsdk.geometry.LatLng;
@ -121,6 +121,10 @@ public class Prefs {
return getBooleanPreference(context, IN_THREAD_NOTIFICATION_PREF, true);
}
public static void setEnterSendsEnabled(Context context, boolean value) {
setBooleanPreference(context, ENTER_SENDS_PREF, value);
}
public static boolean isEnterSendsEnabled(Context context) {
return getBooleanPreference(context, ENTER_SENDS_PREF, false);
}

View file

@ -137,6 +137,9 @@ public class ViewUtil {
final SettableFuture future = new SettableFuture();
if (view.getVisibility() == visibility) {
future.set(true);
} else if (AccessibilityUtil.areAnimationsDisabled(view.getContext())) {
view.setVisibility(visibility);
future.set(true);
} else {
view.clearAnimation();
animation.reset();
@ -162,6 +165,11 @@ public class ViewUtil {
public static void animateIn(final @NonNull View view, final @NonNull Animation animation) {
if (view.getVisibility() == View.VISIBLE) return;
if (AccessibilityUtil.areAnimationsDisabled(view.getContext())) {
view.setVisibility(View.VISIBLE);
return;
}
view.clearAnimation();
animation.reset();
animation.setStartTime(0);