This commit is contained in:
Felitendo
2025-05-20 15:17:20 +02:00
parent 034f1210fb
commit c24d95627e
399 changed files with 35059 additions and 2 deletions

View File

@@ -0,0 +1,118 @@
package com.looker.droidify.index
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import com.fasterxml.jackson.core.JsonToken
import com.looker.droidify.utility.common.extension.Json
import com.looker.droidify.utility.common.extension.asSequence
import com.looker.droidify.utility.common.extension.collectNotNull
import com.looker.droidify.utility.common.extension.writeDictionary
import com.looker.droidify.model.Product
import com.looker.droidify.model.Release
import com.looker.droidify.utility.serialization.product
import com.looker.droidify.utility.serialization.release
import com.looker.droidify.utility.serialization.serialize
import java.io.ByteArrayOutputStream
import java.io.Closeable
import java.io.File
class IndexMerger(file: File) : Closeable {
private val db = SQLiteDatabase.openOrCreateDatabase(file, null)
init {
db.execWithResult("PRAGMA synchronous = OFF")
db.execWithResult("PRAGMA journal_mode = OFF")
db.execSQL(
"CREATE TABLE product (" +
"package_name TEXT PRIMARY KEY," +
"description TEXT NOT NULL, " +
"data BLOB NOT NULL)"
)
db.execSQL("CREATE TABLE releases (package_name TEXT PRIMARY KEY, data BLOB NOT NULL)")
db.beginTransaction()
}
fun addProducts(products: List<Product>) {
for (product in products) {
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream)
.use { it.writeDictionary(product::serialize) }
db.insert(
"product",
null,
ContentValues().apply {
put("package_name", product.packageName)
put("description", product.description)
put("data", outputStream.toByteArray())
}
)
}
}
fun addReleases(pairs: List<Pair<String, List<Release>>>) {
for (pair in pairs) {
val (packageName, releases) = pair
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream).use {
it.writeStartArray()
for (release in releases) {
it.writeDictionary(release::serialize)
}
it.writeEndArray()
}
db.insert(
"releases",
null,
ContentValues().apply {
put("package_name", packageName)
put("data", outputStream.toByteArray())
}
)
}
}
private fun closeTransaction() {
if (db.inTransaction()) {
db.setTransactionSuccessful()
db.endTransaction()
}
}
fun forEach(repositoryId: Long, windowSize: Int, callback: (List<Product>, Int) -> Unit) {
closeTransaction()
db.rawQuery(
"""SELECT product.description, product.data AS pd, releases.data AS rd FROM product
LEFT JOIN releases ON product.package_name = releases.package_name""",
null
).use { cursor ->
cursor.asSequence().map { currentCursor ->
val description = currentCursor.getString(0)
val product = Json.factory.createParser(currentCursor.getBlob(1)).use {
it.nextToken()
it.product().apply {
this.repositoryId = repositoryId
this.description = description
}
}
val releases = currentCursor.getBlob(2)?.let { bytes ->
Json.factory.createParser(bytes).use {
it.nextToken()
it.collectNotNull(
JsonToken.START_OBJECT
) { release() }
}
}.orEmpty()
product.copy(releases = releases)
}.windowed(windowSize, windowSize, true)
.forEach { products -> callback(products, cursor.count) }
}
}
override fun close() {
db.use { closeTransaction() }
}
private inline fun SQLiteDatabase.execWithResult(sql: String) {
rawQuery(sql, null).use { it.count }
}
}

View File

@@ -0,0 +1,559 @@
package com.looker.droidify.index
import android.content.res.Resources
import androidx.core.os.ConfigurationCompat.getLocales
import androidx.core.os.LocaleListCompat
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonToken
import com.looker.droidify.utility.common.extension.Json
import com.looker.droidify.utility.common.extension.collectDistinctNotEmptyStrings
import com.looker.droidify.utility.common.extension.collectNotNull
import com.looker.droidify.utility.common.extension.forEach
import com.looker.droidify.utility.common.extension.forEachKey
import com.looker.droidify.utility.common.extension.illegal
import com.looker.droidify.model.Product
import com.looker.droidify.model.Product.Donate.Bitcoin
import com.looker.droidify.model.Product.Donate.Liberapay
import com.looker.droidify.model.Product.Donate.Litecoin
import com.looker.droidify.model.Product.Donate.OpenCollective
import com.looker.droidify.model.Product.Donate.Regular
import com.looker.droidify.model.Product.Screenshot.Type.LARGE_TABLET
import com.looker.droidify.model.Product.Screenshot.Type.PHONE
import com.looker.droidify.model.Product.Screenshot.Type.SMALL_TABLET
import com.looker.droidify.model.Product.Screenshot.Type.TV
import com.looker.droidify.model.Product.Screenshot.Type.VIDEO
import com.looker.droidify.model.Product.Screenshot.Type.WEAR
import com.looker.droidify.model.Release
import com.looker.droidify.utility.common.SdkCheck
import com.looker.droidify.utility.common.nullIfEmpty
import java.io.InputStream
object IndexV1Parser {
interface Callback {
fun onRepository(
mirrors: List<String>,
name: String,
description: String,
version: Int,
timestamp: Long
)
fun onProduct(product: Product)
fun onReleases(packageName: String, releases: List<Release>)
}
private class Screenshots(
val video: List<String>,
val phone: List<String>,
val smallTablet: List<String>,
val largeTablet: List<String>,
val wear: List<String>,
val tv: List<String>,
)
private class Localized(
val name: String,
val summary: String,
val description: String,
val whatsNew: String,
val metadataIcon: String,
val screenshots: Screenshots?
)
private fun <T> Map<String, Localized>.getAndCall(
key: String,
callback: (String, Localized) -> T?
): T? {
return this[key]?.let { callback(key, it) }
}
/**
* Gets the best localization for the given [localeList]
* from collections.
*/
private fun <T> Map<String, T>?.getBestLocale(localeList: LocaleListCompat): T? {
if (isNullOrEmpty()) return null
val firstMatch = localeList.getFirstMatch(keys.toTypedArray()) ?: return null
val tag = firstMatch.toLanguageTag()
// try first matched tag first (usually has region tag, e.g. de-DE)
return get(tag) ?: run {
// split away stuff like script and try language and region only
val langCountryTag = "${firstMatch.language}-${firstMatch.country}"
getOrStartsWith(langCountryTag) ?: run {
// split away region tag and try language only
val langTag = firstMatch.language
// try language, then English and then just take the first of the list
getOrStartsWith(langTag) ?: get("en-US") ?: get("en") ?: values.first()
}
}
}
/**
* Returns the value from the map with the given key or if that key is not contained in the map,
* tries the first map key that starts with the given key.
* If nothing matches, null is returned.
*
* This is useful when looking for a language tag like `fr_CH` and falling back to `fr`
* in a map that has `fr_FR` as a key.
*/
private fun <T> Map<String, T>.getOrStartsWith(s: String): T? = get(s) ?: run {
entries.forEach { (key, value) ->
if (key.startsWith(s)) return value
}
return null
}
private fun <T> Map<String, Localized>.find(callback: (String, Localized) -> T?): T? {
return getAndCall("en-US", callback)
?: getAndCall("en_US", callback)
?: getAndCall("en", callback)
}
private fun <T> Map<String, Localized>.findLocalized(callback: (Localized) -> T?): T? {
return getBestLocale(getLocales(Resources.getSystem().configuration))?.let { callback(it) }
}
private fun Map<String, Localized>.findString(
fallback: String,
callback: (Localized) -> String
): String {
return (find { _, localized -> callback(localized).nullIfEmpty() } ?: fallback).trim()
}
private fun Map<String, Localized>.findLocalizedString(
fallback: String,
callback: (Localized) -> String
): String {
// @BLumia: it's possible a key of a certain Localized object is empty, so we still need a fallback
return (
findLocalized { localized -> callback(localized).trim().nullIfEmpty() } ?: findString(
fallback,
callback
)
).trim()
}
internal object DonateComparator : Comparator<Product.Donate> {
private val classes = listOf(
Regular::class,
Bitcoin::class,
Litecoin::class,
Liberapay::class,
OpenCollective::class
)
override fun compare(donate1: Product.Donate, donate2: Product.Donate): Int {
val index1 = classes.indexOf(donate1::class)
val index2 = classes.indexOf(donate2::class)
return when {
index1 >= 0 && index2 == -1 -> -1
index2 >= 0 && index1 == -1 -> 1
else -> index1.compareTo(index2)
}
}
}
private const val DICT_REPO = "repo"
private const val DICT_PRODUCT = "apps"
private const val DICT_RELEASE = "packages"
private const val KEY_REPO_ADDRESS = "address"
private const val KEY_REPO_MIRRORS = "mirrors"
private const val KEY_REPO_NAME = "name"
private const val KEY_REPO_DESC = "description"
private const val KEY_REPO_VER = "version"
private const val KEY_REPO_TIME = "timestamp"
fun parse(repositoryId: Long, inputStream: InputStream, callback: Callback) {
val jsonParser = Json.factory.createParser(inputStream)
if (jsonParser.nextToken() != JsonToken.START_OBJECT) {
jsonParser.illegal()
} else {
jsonParser.forEachKey { key ->
when {
key.dictionary(DICT_REPO) -> {
var address = ""
var mirrors = emptyList<String>()
var name = ""
var description = ""
var version = 0
var timestamp = 0L
forEachKey {
when {
it.string(KEY_REPO_ADDRESS) -> address = valueAsString
it.array(KEY_REPO_MIRRORS) -> mirrors =
collectDistinctNotEmptyStrings()
it.string(KEY_REPO_NAME) -> name = valueAsString
it.string(KEY_REPO_DESC) -> description = valueAsString
it.number(KEY_REPO_VER) -> version = valueAsInt
it.number(KEY_REPO_TIME) -> timestamp = valueAsLong
else -> skipChildren()
}
}
val realMirrors = (
if (address.isNotEmpty()) {
listOf(address)
} else {
emptyList()
}
) + mirrors
callback.onRepository(
mirrors = realMirrors.distinct(),
name = name,
description = description,
version = version,
timestamp = timestamp
)
}
key.array(DICT_PRODUCT) -> forEach(JsonToken.START_OBJECT) {
val product = parseProduct(repositoryId)
callback.onProduct(product)
}
key.dictionary(DICT_RELEASE) -> forEachKey {
if (it.token == JsonToken.START_ARRAY) {
val packageName = it.key
val releases = collectNotNull(JsonToken.START_OBJECT) { parseRelease() }
callback.onReleases(packageName, releases)
} else {
skipChildren()
}
}
else -> skipChildren()
}
}
}
}
private const val KEY_PRODUCT_PACKAGENAME = "packageName"
private const val KEY_PRODUCT_NAME = "name"
private const val KEY_PRODUCT_SUMMARY = "summary"
private const val KEY_PRODUCT_DESCRIPTION = "description"
private const val KEY_PRODUCT_ICON = "icon"
private const val KEY_PRODUCT_AUTHORNAME = "authorName"
private const val KEY_PRODUCT_AUTHOREMAIL = "authorEmail"
private const val KEY_PRODUCT_AUTHORWEBSITE = "authorWebSite"
private const val KEY_PRODUCT_SOURCECODE = "sourceCode"
private const val KEY_PRODUCT_CHANGELOG = "changelog"
private const val KEY_PRODUCT_WEBSITE = "webSite"
private const val KEY_PRODUCT_ISSUETRACKER = "issueTracker"
private const val KEY_PRODUCT_ADDED = "added"
private const val KEY_PRODUCT_LASTUPDATED = "lastUpdated"
private const val KEY_PRODUCT_SUGGESTEDVERSIONCODE = "suggestedVersionCode"
private const val KEY_PRODUCT_CATEGORIES = "categories"
private const val KEY_PRODUCT_ANTIFEATURES = "antiFeatures"
private const val KEY_PRODUCT_LICENSE = "license"
private const val KEY_PRODUCT_DONATE = "donate"
private const val KEY_PRODUCT_BITCOIN = "bitcoin"
private const val KEY_PRODUCT_LIBERAPAYID = "liberapay"
private const val KEY_PRODUCT_LITECOIN = "litecoin"
private const val KEY_PRODUCT_OPENCOLLECTIVE = "openCollective"
private const val KEY_PRODUCT_LOCALIZED = "localized"
private const val KEY_PRODUCT_WHATSNEW = "whatsNew"
private const val KEY_PRODUCT_PHONE_SCREENSHOTS = "phoneScreenshots"
private const val KEY_PRODUCT_SEVEN_INCH_SCREENSHOTS = "sevenInchScreenshots"
private const val KEY_PRODUCT_TEN_INCH_SCREENSHOTS = "tenInchScreenshots"
private const val KEY_PRODUCT_WEAR_SCREENSHOTS = "wearScreenshots"
private const val KEY_PRODUCT_TV_SCREENSHOTS = "tvScreenshots"
private const val KEY_PRODUCT_VIDEO = "video"
private fun JsonParser.parseProduct(repositoryId: Long): Product {
var packageName = ""
var nameFallback = ""
var summaryFallback = ""
var descriptionFallback = ""
var icon = ""
var authorName = ""
var authorEmail = ""
var authorWeb = ""
var source = ""
var changelog = ""
var web = ""
var tracker = ""
var added = 0L
var updated = 0L
var suggestedVersionCode = 0L
var categories = emptyList<String>()
var antiFeatures = emptyList<String>()
val licenses = mutableListOf<String>()
val donates = mutableListOf<Product.Donate>()
val localizedMap = mutableMapOf<String, Localized>()
forEachKey { key ->
when {
key.string(KEY_PRODUCT_PACKAGENAME) -> packageName = valueAsString
key.string(KEY_PRODUCT_NAME) -> nameFallback = valueAsString
key.string(KEY_PRODUCT_SUMMARY) -> summaryFallback = valueAsString
key.string(KEY_PRODUCT_DESCRIPTION) -> descriptionFallback = valueAsString
key.string(KEY_PRODUCT_ICON) -> icon = validateIcon(valueAsString)
key.string(KEY_PRODUCT_AUTHORNAME) -> authorName = valueAsString
key.string(KEY_PRODUCT_AUTHOREMAIL) -> authorEmail = valueAsString
key.string(KEY_PRODUCT_AUTHORWEBSITE) -> authorWeb = valueAsString
key.string(KEY_PRODUCT_SOURCECODE) -> source = valueAsString
key.string(KEY_PRODUCT_CHANGELOG) -> changelog = valueAsString
key.string(KEY_PRODUCT_WEBSITE) -> web = valueAsString
key.string(KEY_PRODUCT_ISSUETRACKER) -> tracker = valueAsString
key.number(KEY_PRODUCT_ADDED) -> added = valueAsLong
key.number(KEY_PRODUCT_LASTUPDATED) -> updated = valueAsLong
key.string(KEY_PRODUCT_SUGGESTEDVERSIONCODE) ->
suggestedVersionCode =
valueAsString.toLongOrNull() ?: 0L
key.array(KEY_PRODUCT_CATEGORIES) -> categories = collectDistinctNotEmptyStrings()
key.array(KEY_PRODUCT_ANTIFEATURES) -> antiFeatures =
collectDistinctNotEmptyStrings()
key.string(KEY_PRODUCT_LICENSE) -> licenses += valueAsString.split(',')
.filter { it.isNotEmpty() }
key.string(KEY_PRODUCT_DONATE) -> donates += Regular(valueAsString)
key.string(KEY_PRODUCT_BITCOIN) -> donates += Bitcoin(valueAsString)
key.string(KEY_PRODUCT_LIBERAPAYID) -> donates += Liberapay(valueAsString)
key.string(KEY_PRODUCT_LITECOIN) -> donates += Litecoin(valueAsString)
key.string(KEY_PRODUCT_OPENCOLLECTIVE) -> donates += OpenCollective(valueAsString)
key.dictionary(KEY_PRODUCT_LOCALIZED) -> forEachKey { localizedKey ->
if (localizedKey.token == JsonToken.START_OBJECT) {
val locale = localizedKey.key
var name = ""
var summary = ""
var description = ""
var whatsNew = ""
var metadataIcon = ""
var phone = emptyList<String>()
var smallTablet = emptyList<String>()
var largeTablet = emptyList<String>()
var wear = emptyList<String>()
var tv = emptyList<String>()
var video = emptyList<String>()
forEachKey {
when {
it.string(KEY_PRODUCT_NAME) -> name = valueAsString
it.string(KEY_PRODUCT_SUMMARY) -> summary = valueAsString
it.string(KEY_PRODUCT_DESCRIPTION) -> description = valueAsString
it.string(KEY_PRODUCT_WHATSNEW) -> whatsNew = valueAsString
it.string(KEY_PRODUCT_ICON) -> metadataIcon = valueAsString
it.string(KEY_PRODUCT_VIDEO) -> video = listOf(valueAsString)
it.array(KEY_PRODUCT_PHONE_SCREENSHOTS) ->
phone = collectDistinctNotEmptyStrings()
it.array(KEY_PRODUCT_SEVEN_INCH_SCREENSHOTS) ->
smallTablet = collectDistinctNotEmptyStrings()
it.array(KEY_PRODUCT_TEN_INCH_SCREENSHOTS) ->
largeTablet = collectDistinctNotEmptyStrings()
it.array(KEY_PRODUCT_WEAR_SCREENSHOTS) ->
wear = collectDistinctNotEmptyStrings()
it.array(KEY_PRODUCT_TV_SCREENSHOTS) ->
tv = collectDistinctNotEmptyStrings()
else -> skipChildren()
}
}
val isScreenshotEmpty =
arrayOf(video, phone, smallTablet, largeTablet, wear, tv)
.any { it.isNotEmpty() }
val screenshots =
if (isScreenshotEmpty) {
Screenshots(video, phone, smallTablet, largeTablet, wear, tv)
} else {
null
}
localizedMap[locale] = Localized(
name = name,
summary = summary,
description = description,
whatsNew = whatsNew,
metadataIcon = metadataIcon.nullIfEmpty()?.let { "$locale/$it" }
.orEmpty(),
screenshots = screenshots,
)
} else {
skipChildren()
}
}
else -> skipChildren()
}
}
val name = localizedMap.findLocalizedString(nameFallback) { it.name }
val summary = localizedMap.findLocalizedString(summaryFallback) { it.summary }
val description =
localizedMap.findLocalizedString(descriptionFallback) { it.description }.replace(
"\n",
"<br/>"
)
val whatsNew = localizedMap.findLocalizedString("") { it.whatsNew }.replace("\n", "<br/>")
val metadataIcon = localizedMap.findLocalizedString("") { it.metadataIcon }.ifEmpty {
localizedMap.firstNotNullOfOrNull { it.value.metadataIcon }.orEmpty()
}
val screenshotPairs =
localizedMap.find { key, localized -> localized.screenshots?.let { Pair(key, it) } }
val screenshots = screenshotPairs?.let { (key, screenshots) ->
screenshots.video.map { Product.Screenshot(key, VIDEO, it) } +
screenshots.phone.map { Product.Screenshot(key, PHONE, it) } +
screenshots.smallTablet.map { Product.Screenshot(key, SMALL_TABLET, it) } +
screenshots.largeTablet.map { Product.Screenshot(key, LARGE_TABLET, it) } +
screenshots.wear.map { Product.Screenshot(key, WEAR, it) } +
screenshots.tv.map { Product.Screenshot(key, TV, it) }
}.orEmpty()
return Product(
repositoryId = repositoryId,
packageName = packageName,
name = name,
summary = summary,
description = description,
whatsNew = whatsNew,
icon = icon,
metadataIcon = metadataIcon,
author = Product.Author(authorName, authorEmail, authorWeb),
source = source,
changelog = changelog,
web = web,
tracker = tracker,
added = added,
updated = updated,
suggestedVersionCode = suggestedVersionCode,
categories = categories,
antiFeatures = antiFeatures,
licenses = licenses,
donates = donates.sortedWith(DonateComparator),
screenshots = screenshots,
releases = emptyList()
)
}
private const val KEY_RELEASE_VERSIONNAME = "versionName"
private const val KEY_RELEASE_VERSIONCODE = "versionCode"
private const val KEY_RELEASE_ADDED = "added"
private const val KEY_RELEASE_SIZE = "size"
private const val KEY_RELEASE_MINSDKVERSION = "minSdkVersion"
private const val KEY_RELEASE_TARGETSDKVERSION = "targetSdkVersion"
private const val KEY_RELEASE_MAXSDKVERSION = "maxSdkVersion"
private const val KEY_RELEASE_SRCNAME = "srcname"
private const val KEY_RELEASE_APKNAME = "apkName"
private const val KEY_RELEASE_HASH = "hash"
private const val KEY_RELEASE_HASHTYPE = "hashType"
private const val KEY_RELEASE_SIG = "sig"
private const val KEY_RELEASE_OBBMAINFILE = "obbMainFile"
private const val KEY_RELEASE_OBBMAINFILESHA256 = "obbMainFileSha256"
private const val KEY_RELEASE_OBBPATCHFILE = "obbPatchFile"
private const val KEY_RELEASE_OBBPATCHFILESHA256 = "obbPatchFileSha256"
private const val KEY_RELEASE_USESPERMISSION = "uses-permission"
private const val KEY_RELEASE_USESPERMISSIONSDK23 = "uses-permission-sdk-23"
private const val KEY_RELEASE_FEATURES = "features"
private const val KEY_RELEASE_NATIVECODE = "nativecode"
private fun JsonParser.parseRelease(): Release {
var version = ""
var versionCode = 0L
var added = 0L
var size = 0L
var minSdkVersion = 0
var targetSdkVersion = 0
var maxSdkVersion = 0
var source = ""
var release = ""
var hash = ""
var hashTypeCandidate = ""
var signature = ""
var obbMain = ""
var obbMainHash = ""
var obbPatch = ""
var obbPatchHash = ""
val permissions = linkedSetOf<String>()
var features = emptyList<String>()
var platforms = emptyList<String>()
forEachKey { key ->
when {
key.string(KEY_RELEASE_VERSIONNAME) -> version = valueAsString
key.number(KEY_RELEASE_VERSIONCODE) -> versionCode = valueAsLong
key.number(KEY_RELEASE_ADDED) -> added = valueAsLong
key.number(KEY_RELEASE_SIZE) -> size = valueAsLong
key.number(KEY_RELEASE_MINSDKVERSION) -> minSdkVersion = valueAsInt
key.number(KEY_RELEASE_TARGETSDKVERSION) -> targetSdkVersion = valueAsInt
key.number(KEY_RELEASE_MAXSDKVERSION) -> maxSdkVersion = valueAsInt
key.string(KEY_RELEASE_SRCNAME) -> source = valueAsString
key.string(KEY_RELEASE_APKNAME) -> release = valueAsString
key.string(KEY_RELEASE_HASH) -> hash = valueAsString
key.string(KEY_RELEASE_HASHTYPE) -> hashTypeCandidate = valueAsString
key.string(KEY_RELEASE_SIG) -> signature = valueAsString
key.string(KEY_RELEASE_OBBMAINFILE) -> obbMain = valueAsString
key.string(KEY_RELEASE_OBBMAINFILESHA256) -> obbMainHash = valueAsString
key.string(KEY_RELEASE_OBBPATCHFILE) -> obbPatch = valueAsString
key.string(KEY_RELEASE_OBBPATCHFILESHA256) -> obbPatchHash = valueAsString
key.array(KEY_RELEASE_USESPERMISSION) -> collectPermissions(permissions, 0)
key.array(KEY_RELEASE_USESPERMISSIONSDK23) -> collectPermissions(permissions, 23)
key.array(KEY_RELEASE_FEATURES) -> features = collectDistinctNotEmptyStrings()
key.array(KEY_RELEASE_NATIVECODE) -> platforms = collectDistinctNotEmptyStrings()
else -> skipChildren()
}
}
val hashType =
if (hash.isNotEmpty() && hashTypeCandidate.isEmpty()) "sha256" else hashTypeCandidate
val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
return Release(
selected = false,
version = version,
versionCode = versionCode,
added = added,
size = size,
minSdkVersion = minSdkVersion,
targetSdkVersion = targetSdkVersion,
maxSdkVersion = maxSdkVersion,
source = source,
release = release,
hash = hash,
hashType = hashType,
signature = signature,
obbMain = obbMain,
obbMainHash = obbMainHash,
obbMainHashType = obbMainHashType,
obbPatch = obbPatch,
obbPatchHash = obbPatchHash,
obbPatchHashType = obbPatchHashType,
permissions = permissions.toList(),
features = features,
platforms = platforms,
incompatibilities = emptyList()
)
}
private fun JsonParser.collectPermissions(permissions: LinkedHashSet<String>, minSdk: Int) {
forEach(JsonToken.START_ARRAY) {
val firstToken = nextToken()
val permission = if (firstToken == JsonToken.VALUE_STRING) valueAsString else ""
if (firstToken != JsonToken.END_ARRAY) {
val secondToken = nextToken()
val maxSdk = if (secondToken == JsonToken.VALUE_NUMBER_INT) valueAsInt else 0
if (permission.isNotEmpty() &&
SdkCheck.sdk >= minSdk && (
maxSdk <= 0 ||
SdkCheck.sdk <= maxSdk
)
) {
permissions.add(permission)
}
if (secondToken != JsonToken.END_ARRAY) {
while (true) {
val token = nextToken()
if (token == JsonToken.END_ARRAY) {
break
} else if (token.isStructStart) {
skipChildren()
}
}
}
}
}
}
private fun validateIcon(icon: String): String {
return if (icon.endsWith(".xml")) "" else icon
}
}

View File

@@ -0,0 +1,461 @@
package com.looker.droidify.index
import android.content.Context
import android.net.Uri
import com.looker.droidify.database.Database
import com.looker.droidify.domain.model.fingerprint
import com.looker.droidify.model.Product
import com.looker.droidify.model.Release
import com.looker.droidify.model.Repository
import com.looker.droidify.network.Downloader
import com.looker.droidify.network.NetworkResponse
import com.looker.droidify.utility.common.SdkCheck
import com.looker.droidify.utility.common.cache.Cache
import com.looker.droidify.utility.common.extension.toFormattedString
import com.looker.droidify.utility.common.result.Result
import com.looker.droidify.utility.extension.android.Android
import com.looker.droidify.utility.getProgress
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.drop
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.map
import java.io.File
import java.security.CodeSigner
import java.security.cert.Certificate
import java.util.jar.JarEntry
import java.util.jar.JarFile
object RepositoryUpdater {
enum class Stage {
DOWNLOAD, PROCESS, MERGE, COMMIT
}
// TODO Add support for Index-V2 and also cleanup everything here
enum class IndexType(
val jarName: String,
val contentName: String
) {
INDEX_V1("index-v1.jar", "index-v1.json")
}
enum class ErrorType {
NETWORK, HTTP, VALIDATION, PARSING
}
class UpdateException : Exception {
val errorType: ErrorType
constructor(errorType: ErrorType, message: String) : super(message) {
this.errorType = errorType
}
constructor(errorType: ErrorType, message: String, cause: Exception) : super(
message,
cause
) {
this.errorType = errorType
}
}
private val updaterLock = Any()
private val cleanupLock = Any()
private lateinit var downloader: Downloader
fun init(scope: CoroutineScope, downloader: Downloader) {
this.downloader = downloader
scope.launch {
// No need of mutex because it is in same coroutine scope
var lastDisabled = emptyMap<Long, Boolean>()
Database.RepositoryAdapter
.getAllRemovedStream()
.map { deletedRepos ->
deletedRepos
.filterNot { it.key in lastDisabled.keys }
.also { lastDisabled = deletedRepos }
}
// To not perform complete cleanup on startup
.drop(1)
.filter { it.isNotEmpty() }
.collect(Database.RepositoryAdapter::cleanup)
}
}
fun await() {
synchronized(updaterLock) { }
}
suspend fun update(
context: Context,
repository: Repository,
unstable: Boolean,
callback: (Stage, Long, Long?) -> Unit
) = update(
context = context,
repository = repository,
unstable = unstable,
indexTypes = listOf(IndexType.INDEX_V1),
callback = callback
)
private suspend fun update(
context: Context,
repository: Repository,
unstable: Boolean,
indexTypes: List<IndexType>,
callback: (Stage, Long, Long?) -> Unit
): Result<Boolean> = withContext(Dispatchers.IO) {
val indexType = indexTypes[0]
when (val request = downloadIndex(context, repository, indexType, callback)) {
is Result.Error -> {
val result = request.data
?: return@withContext Result.Error(request.exception, false)
val file = request.data?.file
?: return@withContext Result.Error(request.exception, false)
file.delete()
if (result.statusCode == 404 && indexTypes.isNotEmpty()) {
update(
context = context,
repository = repository,
indexTypes = indexTypes.subList(1, indexTypes.size),
unstable = unstable,
callback = callback
)
} else {
Result.Error(
UpdateException(
ErrorType.HTTP,
"Invalid response: HTTP ${result.statusCode}"
)
)
}
}
is Result.Success -> {
if (request.data.isUnmodified) {
request.data.file.delete()
Result.Success(false)
} else {
try {
val isFileParsedSuccessfully = processFile(
context = context,
repository = repository,
indexType = indexType,
unstable = unstable,
file = request.data.file,
lastModified = request.data.lastModified,
entityTag = request.data.entityTag,
callback = callback
)
Result.Success(isFileParsedSuccessfully)
} catch (e: UpdateException) {
Result.Error(e)
}
}
}
}
}
private suspend fun downloadIndex(
context: Context,
repository: Repository,
indexType: IndexType,
callback: (Stage, Long, Long?) -> Unit
): Result<IndexFile> = withContext(Dispatchers.IO) {
val file = Cache.getTemporaryFile(context)
val result = downloader.downloadToFile(
url = Uri.parse(repository.address).buildUpon()
.appendPath(indexType.jarName).build().toString(),
target = file,
headers = {
ifModifiedSince(repository.lastModified)
etag(repository.entityTag)
authentication(repository.authentication)
}
) { read, total ->
callback(Stage.DOWNLOAD, read.value, total.value.takeIf { it != 0L })
}
when (result) {
is NetworkResponse.Success -> {
Result.Success(
IndexFile(
isUnmodified = result.statusCode == 304,
lastModified = result.lastModified?.toFormattedString() ?: "",
entityTag = result.etag ?: "",
statusCode = result.statusCode,
file = file
)
)
}
is NetworkResponse.Error -> {
file.delete()
when (result) {
is NetworkResponse.Error.Http -> {
val errorType = if (result.statusCode in 400..499) {
ErrorType.HTTP
} else {
ErrorType.NETWORK
}
Result.Error(
UpdateException(
errorType = errorType,
message = "Failed with Status: ${result.statusCode}"
)
)
}
is NetworkResponse.Error.ConnectionTimeout -> Result.Error(result.exception)
is NetworkResponse.Error.IO -> Result.Error(result.exception)
is NetworkResponse.Error.SocketTimeout -> Result.Error(result.exception)
is NetworkResponse.Error.Unknown -> Result.Error(result.exception)
// TODO: Add Validator
is NetworkResponse.Error.Validation -> Result.Error()
}
}
}
}
fun processFile(
context: Context,
repository: Repository,
indexType: IndexType,
unstable: Boolean,
file: File,
mergerFile: File = Cache.getTemporaryFile(context),
lastModified: String,
entityTag: String,
callback: (Stage, Long, Long?) -> Unit
): Boolean {
var rollback = true
return synchronized(updaterLock) {
try {
val jarFile = JarFile(file, true)
val indexEntry = jarFile.getEntry(indexType.contentName) as JarEntry
val total = indexEntry.size
Database.UpdaterAdapter.createTemporaryTable()
val features = context.packageManager.systemAvailableFeatures
.asSequence().map { it.name }.toSet() + setOf("android.hardware.touchscreen")
var changedRepository: Repository? = null
try {
val unmergedProducts = mutableListOf<Product>()
val unmergedReleases = mutableListOf<Pair<String, List<Release>>>()
IndexMerger(mergerFile).use { indexMerger ->
jarFile.getInputStream(indexEntry).getProgress {
callback(Stage.PROCESS, it, total)
}.use { entryStream ->
IndexV1Parser.parse(
repository.id,
entryStream,
object : IndexV1Parser.Callback {
override fun onRepository(
mirrors: List<String>,
name: String,
description: String,
version: Int,
timestamp: Long
) {
changedRepository = repository.update(
mirrors,
name,
description,
version,
lastModified,
entityTag,
timestamp
)
}
override fun onProduct(product: Product) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedProducts += product
if (unmergedProducts.size >= 50) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
}
override fun onReleases(
packageName: String,
releases: List<Release>
) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedReleases += Pair(packageName, releases)
if (unmergedReleases.size >= 50) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
}
}
)
if (Thread.interrupted()) {
throw InterruptedException()
}
if (unmergedProducts.isNotEmpty()) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
if (unmergedReleases.isNotEmpty()) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
var progress = 0
indexMerger.forEach(repository.id, 50) { products, totalCount ->
if (Thread.interrupted()) {
throw InterruptedException()
}
progress += products.size
callback(
Stage.MERGE,
progress.toLong(),
totalCount.toLong()
)
Database.UpdaterAdapter.putTemporary(
products
.map { transformProduct(it, features, unstable) }
)
}
}
}
} finally {
mergerFile.delete()
}
val workRepository = changedRepository ?: repository
if (workRepository.timestamp < repository.timestamp) {
throw UpdateException(
ErrorType.VALIDATION,
"New index is older than current index:" +
" ${workRepository.timestamp} < ${repository.timestamp}"
)
}
val fingerprint = indexEntry
.codeSigner
.certificate
.fingerprint()
.toString()
.uppercase()
val commitRepository = if (!workRepository.fingerprint.equals(
fingerprint,
ignoreCase = true
)
) {
if (workRepository.fingerprint.isNotEmpty()) {
throw UpdateException(
ErrorType.VALIDATION,
"Certificate fingerprints do not match"
)
}
workRepository.copy(fingerprint = fingerprint)
} else {
workRepository
}
if (Thread.interrupted()) {
throw InterruptedException()
}
callback(Stage.COMMIT, 0, null)
synchronized(cleanupLock) {
Database.UpdaterAdapter.finishTemporary(commitRepository, true)
}
rollback = false
true
} catch (e: Exception) {
throw when (e) {
is UpdateException, is InterruptedException -> e
else -> UpdateException(ErrorType.PARSING, "Error parsing index", e)
}
} finally {
file.delete()
if (rollback) {
Database.UpdaterAdapter.finishTemporary(repository, false)
}
}
}
}
@get:Throws(UpdateException::class)
private val JarEntry.codeSigner: CodeSigner
get() = codeSigners?.singleOrNull()
?: throw UpdateException(
ErrorType.VALIDATION,
"index.jar must be signed by a single code signer"
)
@get:Throws(UpdateException::class)
private val CodeSigner.certificate: Certificate
get() = signerCertPath?.certificates?.singleOrNull()
?: throw UpdateException(
ErrorType.VALIDATION,
"index.jar code signer should have only one certificate"
)
private fun transformProduct(
product: Product,
features: Set<String>,
unstable: Boolean
): Product {
val releasePairs = product.releases
.distinctBy { it.identifier }
.sortedByDescending { it.versionCode }
.map { release ->
val incompatibilities = mutableListOf<Release.Incompatibility>()
if (release.minSdkVersion > 0 && SdkCheck.sdk < release.minSdkVersion) {
incompatibilities += Release.Incompatibility.MinSdk
}
if (release.maxSdkVersion > 0 && SdkCheck.sdk > release.maxSdkVersion) {
incompatibilities += Release.Incompatibility.MaxSdk
}
if (release.platforms.isNotEmpty() &&
(release.platforms intersect Android.platforms).isEmpty()
) {
incompatibilities += Release.Incompatibility.Platform
}
incompatibilities += (release.features - features).sorted()
.map { Release.Incompatibility.Feature(it) }
Pair(release, incompatibilities.toList())
}
val predicate: (Release) -> Boolean = {
unstable ||
product.suggestedVersionCode <= 0 ||
it.versionCode <= product.suggestedVersionCode
}
val firstSelected =
releasePairs.firstOrNull { it.second.isEmpty() && predicate(it.first) }
?: releasePairs.firstOrNull { predicate(it.first) }
val releases = releasePairs
.map { (release, incompatibilities) ->
release.copy(
incompatibilities = incompatibilities,
selected = firstSelected?.let {
it.first.versionCode == release.versionCode &&
it.second == incompatibilities
} ?: false
)
}
return product.copy(releases = releases)
}
}
data class IndexFile(
val isUnmodified: Boolean,
val lastModified: String,
val entityTag: String,
val statusCode: Int,
val file: File
)