This commit is contained in:
Felitendo
2025-05-20 15:17:20 +02:00
parent 034f1210fb
commit c24d95627e
399 changed files with 35059 additions and 2 deletions

View File

@@ -0,0 +1,92 @@
package com.looker.droidify.index
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.SmallTest
import androidx.test.platform.app.InstrumentationRegistry
import com.looker.droidify.model.Repository
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import java.io.File
import kotlin.math.sqrt
import kotlin.system.measureTimeMillis
@RunWith(AndroidJUnit4::class)
@SmallTest
class RepositoryUpdaterTest {
private lateinit var context: Context
private lateinit var repository: Repository
@Before
fun setup() {
context = InstrumentationRegistry.getInstrumentation().context
repository = Repository(
id = 15,
address = "https://apt.izzysoft.de/fdroid/repo",
mirrors = emptyList(),
name = "IzzyOnDroid F-Droid Repo",
description = "",
version = 20002,
enabled = true,
fingerprint = "3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A",
lastModified = "",
entityTag = "",
updated = 1735315749835,
timestamp = 1725352450000,
authentication = "",
)
}
@Test
fun processFile() {
testRepetition(1) {
val createFile = File.createTempFile("index", "entry")
val mergerFile = File.createTempFile("index", "merger")
val jarStream = context.resources.assets.open("index-v1.jar")
jarStream.copyTo(createFile.outputStream())
process(createFile, mergerFile)
}
}
private fun process(file: File, merger: File) = measureTimeMillis {
RepositoryUpdater.processFile(
context = context,
repository = repository,
indexType = RepositoryUpdater.IndexType.INDEX_V1,
unstable = false,
file = file,
mergerFile = merger,
lastModified = "",
entityTag = "",
callback = { stage, current, total ->
},
)
}
private inline fun testRepetition(repetition: Int, block: () -> Long) {
val times = (1..repetition).map {
System.gc()
System.runFinalization()
block().toDouble()
}
val meanAndDeviation = times.culledMeanAndDeviation()
println(times)
println("${meanAndDeviation.first} ± ${meanAndDeviation.second}")
}
}
private fun List<Double>.culledMeanAndDeviation(): Pair<Double, Double> = when {
isEmpty() -> Double.NaN to Double.NaN
size == 1 || size == 2 -> this.meanAndDeviation()
else -> sorted().subList(1, size - 1).meanAndDeviation()
}
private fun List<Double>.meanAndDeviation(): Pair<Double, Double> {
val mean = average()
return mean to sqrt(fold(0.0) { acc, value -> acc + (value - mean).squared() } / size)
}
private fun Double.squared() = this * this

View File

@@ -0,0 +1,68 @@
package com.looker.droidify.sync
import com.looker.droidify.network.Downloader
import com.looker.droidify.network.NetworkResponse
import com.looker.droidify.network.ProgressListener
import com.looker.droidify.network.header.HeadersBuilder
import com.looker.droidify.network.validation.FileValidator
import com.looker.droidify.sync.common.assets
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.withContext
import java.io.File
import java.io.InputStream
import java.net.Proxy
val FakeDownloader = object : Downloader {
override fun setProxy(proxy: Proxy) {
TODO("Not yet implemented")
}
override suspend fun headCall(
url: String,
headers: HeadersBuilder.() -> Unit
): NetworkResponse {
TODO("Not yet implemented")
}
override suspend fun downloadToFile(
url: String,
target: File,
validator: FileValidator?,
headers: HeadersBuilder.() -> Unit,
block: ProgressListener?
): NetworkResponse {
return if (url.endsWith("fail")) NetworkResponse.Error.Unknown(Exception("You asked for it"))
else {
val index = when {
url.endsWith("fdroid-index-v1.jar") -> assets("fdroid_index_v1.jar")
url.endsWith("fdroid-index-v1.json") -> assets("fdroid_index_v1.json")
url.endsWith("fdroid-index-v2.json") -> assets("fdroid_index_v2.json")
url.endsWith("index-v1.jar") -> assets("izzy_index_v1.jar")
url.endsWith("index-v2.json") -> assets("izzy_index_v2.json")
url.endsWith("index-v2-updated.json") -> assets("izzy_index_v2_updated.json")
url.endsWith("entry.jar") -> assets("izzy_entry.jar")
url.endsWith("/diff/1725731263000.json") -> assets("izzy_diff.json")
// Just in case we try these in future
url.endsWith("index-v1.json") -> assets("izzy_index_v1.json")
url.endsWith("entry.json") -> assets("izzy_entry.json")
else -> error("Unknown URL: $url")
}
index.writeTo(target)
NetworkResponse.Success(200, null, null)
}
}
suspend infix fun InputStream.writeTo(file: File) = withContext(Dispatchers.IO) {
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
var bytesRead = read(buffer)
file.outputStream().use { output ->
while (bytesRead != -1) {
ensureActive()
output.write(buffer, 0, bytesRead)
bytesRead = read(buffer)
}
output.flush()
}
}
}

View File

@@ -0,0 +1,130 @@
package com.looker.droidify.sync
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import com.looker.droidify.domain.model.Repo
import com.looker.droidify.sync.common.IndexJarValidator
import com.looker.droidify.sync.common.Izzy
import com.looker.droidify.sync.common.JsonParser
import com.looker.droidify.sync.common.assets
import com.looker.droidify.sync.common.downloadIndex
import com.looker.droidify.sync.common.benchmark
import com.looker.droidify.sync.v2.EntryParser
import com.looker.droidify.sync.v2.EntrySyncable
import com.looker.droidify.sync.v2.model.Entry
import com.looker.droidify.sync.v2.model.IndexV2
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.test.StandardTestDispatcher
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.json.decodeFromStream
import org.junit.Before
import org.junit.runner.RunWith
import kotlin.system.measureTimeMillis
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
@RunWith(AndroidJUnit4::class)
class EntrySyncableTest {
private lateinit var dispatcher: CoroutineDispatcher
private lateinit var context: Context
private lateinit var syncable: Syncable<Entry>
private lateinit var parser: Parser<Entry>
private lateinit var validator: IndexValidator
private lateinit var repo: Repo
private lateinit var newIndex: IndexV2
/**
* In this particular test 1 package is removed and 36 packages are updated
*/
@OptIn(ExperimentalSerializationApi::class)
@Before
fun before() {
context = InstrumentationRegistry.getInstrumentation().context
dispatcher = StandardTestDispatcher()
validator = IndexJarValidator(dispatcher)
parser = EntryParser(dispatcher, JsonParser, validator)
syncable = EntrySyncable(context, FakeDownloader, dispatcher)
newIndex = JsonParser.decodeFromStream<IndexV2>(assets("izzy_index_v2_updated.json"))
repo = Izzy
}
@Test
fun benchmark_sync_full() = runTest(dispatcher) {
val output = benchmark(10) {
measureTimeMillis { syncable.sync(repo) }
}
println(output)
}
@Test
fun benchmark_entry_parser() = runTest(dispatcher) {
val output = benchmark(10) {
measureTimeMillis {
parser.parse(
file = FakeDownloader.downloadIndex(
context = context,
repo = repo,
fileName = "izzy",
url = "entry.jar"
),
repo = repo
)
}
}
println(output)
}
@Test
fun check_if_patch_applies() = runTest(dispatcher) {
// Downloads old index file as the index file does not exist
val (fingerprint1, index1) = syncable.sync(repo)
assert(index1 != null)
// Downloads the diff as the index file exists and is older than entry version
val (fingerprint2, index2) = syncable.sync(
repo.copy(
versionInfo = repo.versionInfo.copy(
timestamp = index1!!.repo.timestamp
)
)
)
assert(index2 != null)
// Does not download anything
val (fingerprint3, index3) = syncable.sync(
repo.copy(
versionInfo = repo.versionInfo.copy(
timestamp = index2!!.repo.timestamp
)
)
)
assert(index3 == null)
// Check if all the packages are same
assertContentEquals(newIndex.packages.keys.sorted(), index2.packages.keys.sorted())
// Check if all the version hashes are same
assertContentEquals(
newIndex.packages.values.flatMap { it.versions.keys }.sorted(),
index2.packages.values.flatMap { it.versions.keys }.sorted(),
)
// Check if repo antifeatures are same
assertContentEquals(
newIndex.repo.antiFeatures.keys.sorted(),
index2.repo.antiFeatures.keys.sorted()
)
// Check if repo categories are same
assertContentEquals(
newIndex.repo.categories.keys.sorted(),
index2.repo.categories.keys.sorted()
)
assertEquals(fingerprint1, fingerprint2)
assertEquals(fingerprint2, fingerprint3)
}
}

View File

@@ -0,0 +1,13 @@
package com.looker.droidify.sync
import com.looker.droidify.domain.model.Fingerprint
import java.util.jar.JarEntry
val FakeIndexValidator = object : IndexValidator {
override suspend fun validate(
jarEntry: JarEntry,
expectedFingerprint: Fingerprint?
): Fingerprint {
return expectedFingerprint ?: Fingerprint("0".repeat(64))
}
}

View File

@@ -0,0 +1,308 @@
package com.looker.droidify.sync
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import com.looker.droidify.domain.model.Repo
import com.looker.droidify.sync.common.IndexJarValidator
import com.looker.droidify.sync.common.Izzy
import com.looker.droidify.sync.common.JsonParser
import com.looker.droidify.sync.common.downloadIndex
import com.looker.droidify.sync.common.benchmark
import com.looker.droidify.sync.common.toV2
import com.looker.droidify.sync.v1.V1Parser
import com.looker.droidify.sync.v1.V1Syncable
import com.looker.droidify.sync.v1.model.IndexV1
import com.looker.droidify.sync.v2.V2Parser
import com.looker.droidify.sync.v2.model.FileV2
import com.looker.droidify.sync.v2.model.IndexV2
import com.looker.droidify.sync.v2.model.MetadataV2
import com.looker.droidify.sync.v2.model.VersionV2
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.test.StandardTestDispatcher
import kotlinx.coroutines.test.runTest
import org.junit.Before
import org.junit.runner.RunWith
import kotlin.system.measureTimeMillis
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
@RunWith(AndroidJUnit4::class)
class V1SyncableTest {
private lateinit var dispatcher: CoroutineDispatcher
private lateinit var context: Context
private lateinit var syncable: Syncable<IndexV1>
private lateinit var parser: Parser<IndexV1>
private lateinit var v2Parser: Parser<IndexV2>
private lateinit var validator: IndexValidator
private lateinit var repo: Repo
@Before
fun before() {
context = InstrumentationRegistry.getInstrumentation().context
dispatcher = StandardTestDispatcher()
validator = IndexJarValidator(dispatcher)
parser = V1Parser(dispatcher, JsonParser, validator)
v2Parser = V2Parser(dispatcher, JsonParser)
syncable = V1Syncable(context, FakeDownloader, dispatcher)
repo = Izzy
}
@Test
fun benchmark_sync_v1() = runTest(dispatcher) {
val output = benchmark(10) {
measureTimeMillis { syncable.sync(repo) }
}
println(output)
}
@Test
fun benchmark_v1_parser() = runTest(dispatcher) {
val file = FakeDownloader.downloadIndex(context, repo, "izzy", "index-v1.jar")
val output = benchmark(10) {
measureTimeMillis {
parser.parse(
file = file,
repo = repo
)
}
}
println(output)
}
@Test
fun benchmark_v1_vs_v2_parser() = runTest(dispatcher) {
val v1File = FakeDownloader.downloadIndex(context, repo, "izzy-v1", "index-v1.jar")
val v2File = FakeDownloader.downloadIndex(context, repo, "izzy-v2", "index-v2.json")
val output1 = benchmark(10) {
measureTimeMillis {
parser.parse(
file = v1File,
repo = repo
)
}
}
val output2 = benchmark(10) {
measureTimeMillis {
parser.parse(
file = v2File,
repo = repo,
)
}
}
println(output1)
println(output2)
}
@Test
fun v1tov2() = runTest(dispatcher) {
testIndexConversion("index-v1.jar", "index-v2-updated.json")
}
// @Test
fun v1tov2FDroidRepo() = runTest(dispatcher) {
testIndexConversion("fdroid-index-v1.jar", "fdroid-index-v2.json")
}
private suspend fun testIndexConversion(
v1: String,
v2: String,
targeted: String? = null,
) {
val fileV1 = FakeDownloader.downloadIndex(context, repo, "data-v1", v1)
val fileV2 = FakeDownloader.downloadIndex(context, repo, "data-v2", v2)
val (fingerV1, foundIndexV1) = parser.parse(fileV1, repo)
val (fingerV2, expectedIndex) = v2Parser.parse(fileV2, repo)
val foundIndex = foundIndexV1.toV2()
assertEquals(fingerV2, fingerV1)
assertNotNull(foundIndex)
assertNotNull(expectedIndex)
assertEquals(expectedIndex.repo.timestamp, foundIndex.repo.timestamp)
assertEquals(expectedIndex.packages.size, foundIndex.packages.size)
assertContentEquals(
expectedIndex.packages.keys.sorted(),
foundIndex.packages.keys.sorted(),
)
if (targeted == null) {
expectedIndex.packages.keys.forEach { key ->
val expectedPackage = expectedIndex.packages[key]
val foundPackage = foundIndex.packages[key]
println("**".repeat(25))
println("Testing: ${expectedPackage?.metadata?.name?.get("en-US")} <$key>")
assertNotNull(expectedPackage)
assertNotNull(foundPackage)
assertMetadata(expectedPackage.metadata, foundPackage.metadata)
assertVersion(expectedPackage.versions, foundPackage.versions)
}
} else {
val expectedPackage = expectedIndex.packages[targeted]
val foundPackage = foundIndex.packages[targeted]
println("**".repeat(25))
println("Testing: ${expectedPackage?.metadata?.name?.get("en-US")} <$targeted>")
assertNotNull(expectedPackage)
assertNotNull(foundPackage)
assertMetadata(expectedPackage.metadata, foundPackage.metadata)
assertVersion(expectedPackage.versions, foundPackage.versions)
}
}
}
/*
* Cannot assert following:
* - `name` => because fdroidserver behaves weirdly
* */
private fun assertMetadata(expectedMetaData: MetadataV2, foundMetadata: MetadataV2) {
assertEquals(expectedMetaData.preferredSigner, foundMetadata.preferredSigner)
// assertLocalizedString(expectedMetaData.name, foundMetadata.name)
assertLocalizedString(expectedMetaData.summary, foundMetadata.summary)
assertLocalizedString(expectedMetaData.description, foundMetadata.description)
assertContentEquals(expectedMetaData.categories, foundMetadata.categories)
// Update
assertEquals(expectedMetaData.changelog, foundMetadata.changelog)
assertEquals(expectedMetaData.added, foundMetadata.added)
assertEquals(expectedMetaData.lastUpdated, foundMetadata.lastUpdated)
// Author
assertEquals(expectedMetaData.authorEmail, foundMetadata.authorEmail)
assertEquals(expectedMetaData.authorName, foundMetadata.authorName)
assertEquals(expectedMetaData.authorPhone, foundMetadata.authorPhone)
assertEquals(expectedMetaData.authorWebSite, foundMetadata.authorWebSite)
// Donate
assertEquals(expectedMetaData.bitcoin, foundMetadata.bitcoin)
assertEquals(expectedMetaData.liberapay, foundMetadata.liberapay)
assertEquals(expectedMetaData.flattrID, foundMetadata.flattrID)
assertEquals(expectedMetaData.openCollective, foundMetadata.openCollective)
assertEquals(expectedMetaData.litecoin, foundMetadata.litecoin)
assertContentEquals(expectedMetaData.donate, foundMetadata.donate)
// Source
assertEquals(expectedMetaData.translation, foundMetadata.translation)
assertEquals(expectedMetaData.issueTracker, foundMetadata.issueTracker)
assertEquals(expectedMetaData.license, foundMetadata.license)
assertEquals(expectedMetaData.sourceCode, foundMetadata.sourceCode)
// Graphics
assertLocalizedString(expectedMetaData.video, foundMetadata.video)
assertLocalized(expectedMetaData.icon, foundMetadata.icon) { expected, found ->
assertEquals(expected.name, found.name)
}
assertLocalized(expectedMetaData.promoGraphic, foundMetadata.promoGraphic) { expected, found ->
assertEquals(expected.name, found.name)
}
assertLocalized(expectedMetaData.tvBanner, foundMetadata.tvBanner) { expected, found ->
assertEquals(expected.name, found.name)
}
assertLocalized(
expectedMetaData.featureGraphic,
foundMetadata.featureGraphic
) { expected, found ->
assertEquals(expected.name, found.name)
}
assertLocalized(
expectedMetaData.screenshots?.phone,
foundMetadata.screenshots?.phone
) { expected, found ->
assertFiles(expected, found)
}
assertLocalized(
expectedMetaData.screenshots?.sevenInch,
foundMetadata.screenshots?.sevenInch
) { expected, found ->
assertFiles(expected, found)
}
assertLocalized(
expectedMetaData.screenshots?.tenInch,
foundMetadata.screenshots?.tenInch
) { expected, found ->
assertFiles(expected, found)
}
assertLocalized(
expectedMetaData.screenshots?.tv,
foundMetadata.screenshots?.tv
) { expected, found ->
assertFiles(expected, found)
}
assertLocalized(
expectedMetaData.screenshots?.wear,
foundMetadata.screenshots?.wear
) { expected, found ->
assertFiles(expected, found)
}
}
/*
* Cannot assert following:
* - `whatsNew` => we added same changelog to all versions
* - `antiFeatures` => anti features are now version specific
* */
private fun assertVersion(
expected: Map<String, VersionV2>,
found: Map<String, VersionV2>,
) {
assertEquals(expected.keys.size, found.keys.size)
assertContentEquals(expected.keys.sorted(), found.keys.sorted().asIterable())
expected.keys.forEach { versionHash ->
val expectedVersion = expected[versionHash]
val foundVersion = found[versionHash]
assertNotNull(expectedVersion)
assertNotNull(foundVersion)
assertEquals(expectedVersion.added, foundVersion.added)
assertEquals(expectedVersion.file.name, foundVersion.file.name)
assertEquals(expectedVersion.src?.name, foundVersion.src?.name)
val expectedMan = expectedVersion.manifest
val foundMan = foundVersion.manifest
assertEquals(expectedMan.versionCode, foundMan.versionCode)
assertEquals(expectedMan.versionName, foundMan.versionName)
assertEquals(expectedMan.maxSdkVersion, foundMan.maxSdkVersion)
assertEquals(expectedMan.usesSdk, foundMan.usesSdk)
assertContentEquals(
expectedMan.features.sortedBy { it.name },
foundMan.features.sortedBy { it.name },
)
assertContentEquals(expectedMan.usesPermission, foundMan.usesPermission)
assertContentEquals(expectedMan.usesPermissionSdk23, foundMan.usesPermissionSdk23)
assertContentEquals(expectedMan.signer?.sha256?.sorted(), foundMan.signer?.sha256?.sorted())
assertContentEquals(expectedMan.nativecode.sorted(), foundMan.nativecode.sorted())
}
}
private fun assertLocalizedString(
expected: Map<String, String>?,
found: Map<String, String>?,
message: String? = null,
) {
assertLocalized(expected, found) { one, two ->
assertEquals(one, two, message)
}
}
private fun <T> assertLocalized(
expected: Map<String, T>?,
found: Map<String, T>?,
block: (expected: T, found: T) -> Unit,
) {
if (expected == null || found == null) {
assertEquals(expected, found)
return
}
assertNotNull(expected)
assertNotNull(found)
assertEquals(expected.size, found.size)
assertContentEquals(expected.keys.sorted(), found.keys.sorted().asIterable())
expected.keys.forEach {
if (expected[it] != null && found[it] != null) block(expected[it]!!, found[it]!!)
}
}
private fun assertFiles(expected: List<FileV2>, found: List<FileV2>, message: String? = null) {
// Only check name, because we cannot add sha to old index
assertContentEquals(expected.map { it.name }, found.map { it.name }.asIterable(), message)
}

View File

@@ -0,0 +1,43 @@
package com.looker.droidify.sync.common
import kotlin.math.pow
import kotlin.math.sqrt
internal inline fun benchmark(
repetition: Int,
extraMessage: String? = null,
block: () -> Long,
): String {
if (extraMessage != null) {
println("=".repeat(50))
println(extraMessage)
println("=".repeat(50))
}
val times = DoubleArray(repetition)
repeat(repetition) { iteration ->
System.gc()
System.runFinalization()
times[iteration] = block().toDouble()
}
val meanAndDeviation = times.culledMeanAndDeviation()
return buildString {
append("=".repeat(50))
append("\n")
append(times.joinToString(" | "))
append("\n")
append("${meanAndDeviation.first} ms ± ${meanAndDeviation.second.toFloat()} ms")
append("\n")
append("=".repeat(50))
append("\n")
}
}
private fun DoubleArray.culledMeanAndDeviation(): Pair<Double, Double> {
sort()
return meanAndDeviation()
}
private fun DoubleArray.meanAndDeviation(): Pair<Double, Double> {
val mean = average()
return mean to sqrt(fold(0.0) { acc, value -> acc + (value - mean).pow(2) } / size)
}

View File

@@ -0,0 +1,20 @@
package com.looker.droidify.sync.common
import com.looker.droidify.domain.model.Authentication
import com.looker.droidify.domain.model.Fingerprint
import com.looker.droidify.domain.model.Repo
import com.looker.droidify.domain.model.VersionInfo
val Izzy = Repo(
id = 1L,
enabled = true,
address = "https://apt.izzysoft.de/fdroid/repo",
name = "IzzyOnDroid F-Droid Repo",
description = "This is a repository of apps to be used with F-Droid. Applications in this repository are official binaries built by the original application developers, taken from their resp. repositories (mostly Github, GitLab, Codeberg). Updates for the apps are usually fetched daily, and you can expect daily index updates.",
fingerprint = Fingerprint("3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A"),
authentication = Authentication("", ""),
versionInfo = VersionInfo(0L, null),
mirrors = emptyList(),
antiFeatures = emptyList(),
categories = emptyList(),
)

View File

@@ -0,0 +1,8 @@
package com.looker.droidify.sync.common
import androidx.test.platform.app.InstrumentationRegistry
import java.io.InputStream
fun assets(name: String): InputStream {
return InstrumentationRegistry.getInstrumentation().context.assets.open(name)
}