Improve handling when own app lists at the server are defect

app/src/main/java/io/timelimit/android/data/dao/CryptContainerDao.kt

@@ -38,9 +38,6 @@ interface CryptContainerDao {
     @Query("SELECT * FROM crypt_container_metadata WHERE category_id = :categoryId AND device_id IS NULL AND type = :type")
     fun getCryptoMetadataSyncByCategoryId(categoryId: String, type: Int): CryptContainerMetadata?
 
-    @Query("DELETE FROM crypt_container_metadata WHERE category_id IS NULL AND device_id = :deviceId AND type in (:types)")
-    fun removeDeviceCryptoMetadata(deviceId: String, types: List<Int>)
-
     @Query("SELECT * FROM crypt_container_metadata WHERE status = :processingStatus")
     fun getMetadataByProcessingStatus(processingStatus: CryptContainerMetadata.ProcessingStatus): List<CryptContainerMetadata>
 

app/src/main/java/io/timelimit/android/data/model/CryptContainerMetadata.kt

@@ -117,7 +117,10 @@ data class CryptContainerMetadata (
                     nextCounter = 1,
                     currentGenerationFirstTimestamp = System.currentTimeMillis(),
                     currentGenerationKey = newKey
-                )
+                ),
+                type =
+                if (currentGenerationKey == null) PrepareEncryptionResult.Type.NewContainer
+                else PrepareEncryptionResult.Type.IncrementedGeneration
             )
         } else {
             PrepareEncryptionResult(
@@ -128,15 +131,23 @@ data class CryptContainerMetadata (
                 ),
                 newMetadata = copy(
                     nextCounter = nextCounter + 1
-                )
+                ),
+                type = PrepareEncryptionResult.Type.IncrementedCounter
             )
         }
     }
 
     data class PrepareEncryptionResult (
         val params: CryptContainer.EncryptParameters,
-        val newMetadata: CryptContainerMetadata
-    )
+        val newMetadata: CryptContainerMetadata,
+        val type: Type
+    ) {
+        enum class Type {
+            NewContainer,
+            IncrementedGeneration,
+            IncrementedCounter
+        }
+    }
 }
 
 class CryptContainerMetadataProcessingStatusConverter {

app/src/main/java/io/timelimit/android/extensions/Wire.kt

@@ -0,0 +1,20 @@
+/*
+ * TimeLimit Copyright <C> 2019 - 2022 Jonas Lochmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+package io.timelimit.android.extensions
+
+import com.squareup.wire.Message
+
+fun <M : Message<M, B>, B : Message.Builder<M, B>> Message<M, B>.encodedSize() = this.adapter.encodedSize(this as M)

app/src/main/java/io/timelimit/android/logic/applist/CryptoAppListSync.kt

@@ -21,6 +21,7 @@ import io.timelimit.android.crypto.CryptContainer
 import io.timelimit.android.data.Database
 import io.timelimit.android.data.model.CryptContainerData
 import io.timelimit.android.data.model.CryptContainerMetadata
+import io.timelimit.android.extensions.encodedSize
 import io.timelimit.android.logic.ServerApiLevelInfo
 import io.timelimit.android.proto.build
 import io.timelimit.android.proto.encodeDeflated
@@ -43,154 +44,165 @@ object CryptoAppListSync {
         disableLegacySync: Boolean,
         serverApiLevelInfo: ServerApiLevelInfo
     ) {
-        fun dispatch(action: AppLogicAction) {
+        val compressedDataSizeLimit =
+            if (serverApiLevelInfo.hasLevelOrIsOffline(5)) 1024 * 512
+            else 1024 * 256
+
+        fun dispatchSync(action: AppLogicAction) {
             if (deviceState.isConnectedMode) {
                 ApplyActionUtil.addAppLogicActionToDatabaseSync(action, database)
             }
         }
 
-        val compressedDataSizeLimit =
-            if (serverApiLevelInfo.hasLevelOrIsOffline(5)) 1024 * 512
-            else 1024 * 256
+        fun <T> prepareEncryption(encrypted: InstalledAppsUtil.Encrypted<T>?, type: Int, forceNewGeneration: Boolean = false) = if (encrypted == null) {
+            val key = CryptContainer.EncryptParameters.generate()
 
-        val savedCrypt = Threads.database.executeAndWait {
-            InstalledAppsUtil.getEncryptedInstalledAppsFromDatabaseSync(database, deviceState.id)
+            val metadata = CryptContainerMetadata.buildFor(
+                deviceId = deviceState.id,
+                categoryId = null,
+                type = type,
+                params = key
+            )
+
+            CryptContainerMetadata.PrepareEncryptionResult(key, metadata, CryptContainerMetadata.PrepareEncryptionResult.Type.NewContainer)
+        } else {
+            if (encrypted.meta.type != type) throw IllegalStateException()
+
+            encrypted.meta.copy(
+                status = CryptContainerMetadata.ProcessingStatus.Finished
+            ).prepareEncryption(forceNewGeneration)
         }
 
-        if (savedCrypt == null) {
-            val baseKey = CryptContainer.EncryptParameters.generate()
-            val diffKey = CryptContainer.EncryptParameters.generate()
-
-            val baseEncrypted = CryptContainer.encrypt(installed.encodeDeflated(), baseKey)
-            val diffEncrypted = CryptContainer.encrypt(SavedAppsDifferenceProto.build(baseEncrypted, InstalledAppsDifferenceProto()).encodeDeflated(), diffKey)
-
-            if (baseEncrypted.size > compressedDataSizeLimit) throw TooLargeException(baseEncrypted.size)
-
-            Threads.database.executeAndWait {
-                database.cryptContainer().removeDeviceCryptoMetadata(
-                    deviceId = deviceState.id,
-                    types = listOf(
-                        CryptContainerMetadata.TYPE_APP_LIST_BASE,
-                        CryptContainerMetadata.TYPE_APP_LIST_DIFF
-                    )
-                )
-
-                val baseId = database.cryptContainer().insertMetadata(
-                    CryptContainerMetadata.buildFor(
-                        deviceId = deviceState.id,
-                        categoryId = null,
-                        type = CryptContainerMetadata.TYPE_APP_LIST_BASE,
-                        params = baseKey
-                    )
-                )
-
-                val diffId = database.cryptContainer().insertMetadata(
-                    CryptContainerMetadata.buildFor(
-                        deviceId = deviceState.id,
-                        categoryId = null,
-                        type = CryptContainerMetadata.TYPE_APP_LIST_DIFF,
-                        params = diffKey
-                    )
-                )
-
-                database.cryptContainer().insertData(
-                    CryptContainerData(
-                        cryptContainerId = baseId,
-                        encryptedData = baseEncrypted
-                    )
-                )
-
-                database.cryptContainer().insertData(
-                    CryptContainerData(
-                        cryptContainerId = diffId,
-                        encryptedData = diffEncrypted
-                    )
-                )
-
-                dispatch(UpdateInstalledAppsAction(
-                    base = baseEncrypted,
-                    diff = diffEncrypted,
-                    wipe = disableLegacySync
-                ))
+        fun throwIfTooLarge(data: ByteArray) {
+            data.size.let {
+                if (it > compressedDataSizeLimit) throw TooLargeException(it)
             }
+        }
 
-            syncUtil.requestImportantSync()
-        } else {
-            val diffCrypto = AppsDifferenceUtil.calculateAppsDifference(savedCrypt.base, installed)
+        val savedCrypt = InstalledAppsUtil.getEncryptedInstalledAppsFromDatabase(database, deviceState.id)
 
-            if (diffCrypto != savedCrypt.diff) {
-                val baseSize = savedCrypt.base.adapter.encodedSize(savedCrypt.base)
-                val diffSize = diffCrypto.adapter.encodedSize(diffCrypto)
-                val needsNewBySize = diffSize >= baseSize / 10
-                val baseNeedsNewGeneration = savedCrypt.baseMeta.needsNewGeneration()
-                val diffNeedsNewGeneration = savedCrypt.diffMeta.needsNewGeneration() or baseNeedsNewGeneration
+        val baseCryptConfig = prepareEncryption(
+            encrypted = savedCrypt.base,
+            type = CryptContainerMetadata.TYPE_APP_LIST_BASE
+        )
 
-                val diffCryptParams = savedCrypt.diffMeta.prepareEncryption(diffNeedsNewGeneration)
+        val diffCryptConfig = prepareEncryption(
+            encrypted = savedCrypt.diff,
+            type = CryptContainerMetadata.TYPE_APP_LIST_DIFF,
+            forceNewGeneration = baseCryptConfig.type != CryptContainerMetadata.PrepareEncryptionResult.Type.IncrementedCounter || savedCrypt.base?.decrypted == null
+        )
 
-                if (needsNewBySize or baseNeedsNewGeneration) {
-                    val baseCryptParams = savedCrypt.baseMeta.prepareEncryption(baseNeedsNewGeneration)
+        val diffCrypto: InstalledAppsDifferenceProto? = if (savedCrypt.base?.decrypted == null) null
+        else AppsDifferenceUtil.calculateAppsDifference(savedCrypt.base.decrypted.data, installed)
 
-                    val baseEncrypted = CryptContainer.encrypt(installed.encodeDeflated(), baseCryptParams.params)
+        if (
+            savedCrypt.base?.decrypted == null ||
+            savedCrypt.diff?.decrypted == null ||
+            diffCrypto != savedCrypt.diff.decrypted.data
+        ) {
+            if (
+                savedCrypt.base?.decrypted == null ||
+                savedCrypt.diff?.decrypted == null ||
+                diffCrypto == null ||
+                baseCryptConfig.type != CryptContainerMetadata.PrepareEncryptionResult.Type.IncrementedCounter ||
+                diffCrypto.encodedSize() >= savedCrypt.base.decrypted.data.encodedSize() / 10
+            ) {
+                val (baseEncrypted, diffEncrypted) = Threads.crypto.executeAndWait {
+                    val baseEncrypted = CryptContainer.encrypt(
+                        installed.encodeDeflated(),
+                        baseCryptConfig.params
+                    )
 
                     val diffEncrypted = CryptContainer.encrypt(
-                        SavedAppsDifferenceProto.build(baseEncrypted, InstalledAppsDifferenceProto()).encodeDeflated(),
-                        diffCryptParams.params
+                        SavedAppsDifferenceProto.build(
+                            baseEncrypted,
+                            InstalledAppsDifferenceProto()
+                        ).encodeDeflated(),
+                        diffCryptConfig.params
                     )
 
-                    if (baseEncrypted.size > compressedDataSizeLimit) throw TooLargeException(baseEncrypted.size)
+                    Pair(baseEncrypted, diffEncrypted)
+                }
 
-                    Threads.database.executeAndWait {
-                        database.cryptContainer().updateMetadata(listOf(
-                            baseCryptParams.newMetadata,
-                            diffCryptParams.newMetadata
-                        ))
+                throwIfTooLarge(baseEncrypted)
+                throwIfTooLarge(diffEncrypted)
 
-                        database.cryptContainer().updateData(listOf(
+                Threads.database.executeAndWait {
+                    if (savedCrypt.base == null) {
+                        val baseId = database.cryptContainer().insertMetadata(baseCryptConfig.newMetadata)
+
+                        database.cryptContainer().insertData(
                             CryptContainerData(
-                                cryptContainerId = savedCrypt.baseMeta.cryptContainerId,
+                                cryptContainerId = baseId,
                                 encryptedData = baseEncrypted
-                            ),
-                            CryptContainerData(
-                                cryptContainerId = savedCrypt.diffMeta.cryptContainerId,
-                                encryptedData = diffEncrypted
                             )
-                        ))
+                        )
+                    } else {
+                        database.cryptContainer().updateMetadata(baseCryptConfig.newMetadata)
 
-                        dispatch(UpdateInstalledAppsAction(
-                            base = baseEncrypted,
-                            diff = diffEncrypted,
-                            wipe = disableLegacySync
+                        database.cryptContainer().updateData(CryptContainerData(
+                            cryptContainerId = savedCrypt.base.meta.cryptContainerId,
+                            encryptedData = baseEncrypted
                         ))
                     }
 
-                    syncUtil.requestImportantSync()
-                } else {
-                    val diffEncrypted = CryptContainer.encrypt(
-                        SavedAppsDifferenceProto.build(savedCrypt.baseHeader, diffCrypto).encodeDeflated(),
-                        diffCryptParams.params
-                    )
+                    if (savedCrypt.diff == null) {
+                        val diffId = database.cryptContainer().insertMetadata(diffCryptConfig.newMetadata)
 
-                    if (diffEncrypted.size > compressedDataSizeLimit) throw TooLargeException(diffEncrypted.size)
-
-                    Threads.database.executeAndWait {
-                        database.cryptContainer().updateMetadata(diffCryptParams.newMetadata)
-
-                        database.cryptContainer().updateData(
+                        database.cryptContainer().insertData(
                             CryptContainerData(
-                                cryptContainerId = savedCrypt.diffMeta.cryptContainerId,
+                                cryptContainerId = diffId,
                                 encryptedData = diffEncrypted
                             )
                         )
+                    } else {
+                        database.cryptContainer().updateMetadata(diffCryptConfig.newMetadata)
 
-                        dispatch(UpdateInstalledAppsAction(
-                            base = null,
-                            diff = diffEncrypted,
-                            wipe = disableLegacySync
+                        database.cryptContainer().updateData(CryptContainerData(
+                            cryptContainerId = savedCrypt.diff.meta.cryptContainerId,
+                            encryptedData = diffEncrypted
                         ))
                     }
 
-                    syncUtil.requestImportantSync()
+                    dispatchSync(UpdateInstalledAppsAction(
+                        base = baseEncrypted,
+                        diff = diffEncrypted,
+                        wipe = disableLegacySync
+                    ))
                 }
+
+                syncUtil.requestImportantSync()
+            } else {
+                val diffEncrypted = Threads.crypto.executeAndWait {
+                    CryptContainer.encrypt(
+                        SavedAppsDifferenceProto.build(
+                            savedCrypt.base.decrypted.header,
+                            diffCrypto
+                        ).encodeDeflated(),
+                        diffCryptConfig.params
+                    )
+                }
+
+                throwIfTooLarge(diffEncrypted)
+
+                Threads.database.executeAndWait {
+                    database.cryptContainer().updateMetadata(diffCryptConfig.newMetadata)
+
+                    database.cryptContainer().updateData(
+                        CryptContainerData(
+                            cryptContainerId = savedCrypt.diff.meta.cryptContainerId,
+                            encryptedData = diffEncrypted
+                        )
+                    )
+
+                    dispatchSync(UpdateInstalledAppsAction(
+                        base = null,
+                        diff = diffEncrypted,
+                        wipe = disableLegacySync
+                    ))
+                }
+
+                syncUtil.requestImportantSync()
             }
         }
     }

app/src/main/java/io/timelimit/android/logic/applist/InstalledAppsUtil.kt

@@ -45,90 +45,114 @@ object InstalledAppsUtil {
         )
     }
 
-    fun getEncryptedInstalledAppsFromDatabaseSync(database: Database, deviceId: String): DecryptedInstalledApps? {
+    suspend fun getEncryptedInstalledAppsFromDatabase(database: Database, deviceId: String): EncryptedInstalledApps {
         if (BuildConfig.DEBUG) {
-            Log.d(LOG_TAG, "getEncryptedInstalledAppsFromDatabaseSync()")
+            Log.d(LOG_TAG, "getEncryptedInstalledAppsFromDatabase()")
         }
 
-        val baseValue = database.cryptContainer().getCryptoFullDataSyncByDeviceId(
-            deviceId = deviceId,
-            type = CryptContainerMetadata.TYPE_APP_LIST_BASE
-        )
+        val (baseValue, diffValue) = Threads.database.executeAndWait {
+            database.runInTransaction {
+                val baseValue = database.cryptContainer().getCryptoFullDataSyncByDeviceId(
+                    deviceId = deviceId,
+                    type = CryptContainerMetadata.TYPE_APP_LIST_BASE
+                )
 
-        val diffValue = database.cryptContainer().getCryptoFullDataSyncByDeviceId(
-            deviceId = deviceId,
-            type = CryptContainerMetadata.TYPE_APP_LIST_DIFF
-        )
+                val diffValue = database.cryptContainer().getCryptoFullDataSyncByDeviceId(
+                    deviceId = deviceId,
+                    type = CryptContainerMetadata.TYPE_APP_LIST_DIFF
+                )
 
-        if (
-            baseValue == null ||
-            baseValue.metadata.currentGenerationKey == null ||
-            baseValue.metadata.status != CryptContainerMetadata.ProcessingStatus.Finished ||
-            diffValue == null ||
-            diffValue.metadata.currentGenerationKey == null ||
-            diffValue.metadata.status != CryptContainerMetadata.ProcessingStatus.Finished
-        ) {
-            if (BuildConfig.DEBUG) {
-                Log.d(LOG_TAG, "incomplete data")
+                Pair(baseValue, diffValue)
+            }
+        }
+
+        return Threads.crypto.executeAndWait {
+            val baseDecrypted = try {
+                if (
+                    baseValue != null &&
+                    baseValue.metadata.currentGenerationKey != null &&
+                    baseValue.metadata.status == CryptContainerMetadata.ProcessingStatus.Finished
+                ) {
+                    val baseHeader = CryptContainer.Header.read(baseValue.encryptedData)
+
+                    val baseDecrypted = CryptContainer.decrypt(
+                        baseValue.metadata.currentGenerationKey,
+                        baseValue.encryptedData
+                    )
+
+                    val baseData = InstalledAppsProto.ADAPTER.decodeInflated(baseDecrypted)
+
+                    Decrypted(
+                        data = baseData,
+                        header = baseHeader
+                    )
+                } else null
+            } catch (ex: CryptException) {
+                if (BuildConfig.DEBUG) {
+                    Log.d(LOG_TAG, "could not decrypt previous base data", ex)
+                }
+
+                null
+            } catch (ex: IOException) {
+                if (BuildConfig.DEBUG) {
+                    Log.d(LOG_TAG, "could not decode previous base data", ex)
+                }
+
+                null
             }
 
-            return null
-        }
+            val diffDecrypted = try {
+                if (
+                    diffValue != null &&
+                    diffValue.metadata.currentGenerationKey != null &&
+                    diffValue.metadata.status == CryptContainerMetadata.ProcessingStatus.Finished
+                ) {
+                    val diffHeader = CryptContainer.Header.read(diffValue.encryptedData)
 
-        val (baseHeader, baseDecrypted, diffDecrypted) = try {
-            val baseHeader = CryptContainer.Header.read(baseValue.encryptedData)
+                    val diffDecrypted = CryptContainer.decrypt(
+                        diffValue.metadata.currentGenerationKey,
+                        diffValue.encryptedData
+                    )
 
-            val baseDecrypted = CryptContainer.decrypt(
-                baseValue.metadata.currentGenerationKey,
-                baseValue.encryptedData
+                    val diffData =
+                        SavedAppsDifferenceProto.ADAPTER.decodeInflated(diffDecrypted).apps
+                            ?: InstalledAppsDifferenceProto()
+
+                    Decrypted(
+                        data = diffData,
+                        header = diffHeader
+                    )
+                } else null
+            } catch (ex: CryptException) {
+                if (BuildConfig.DEBUG) {
+                    Log.d(LOG_TAG, "could not decrypt previous diff data", ex)
+                }
+
+                null
+            } catch (ex: IOException) {
+                if (BuildConfig.DEBUG) {
+                    Log.d(LOG_TAG, "could not decode previous diff data", ex)
+                }
+
+                null
+            }
+
+            EncryptedInstalledApps(
+                base = baseValue?.let { Encrypted(it.metadata, baseDecrypted) },
+                diff = diffValue?.let { Encrypted(it.metadata, diffDecrypted) }
             )
-
-            val diffDecrypted = CryptContainer.decrypt(
-                diffValue.metadata.currentGenerationKey,
-                diffValue.encryptedData
-            )
-
-            Triple(baseHeader, baseDecrypted, diffDecrypted)
-        } catch (ex: CryptException) {
-            if (BuildConfig.DEBUG) {
-                Log.d(LOG_TAG, "could not decrypt previous data", ex)
-            }
-
-            return null
         }
-
-        val (base, diff) = try {
-            val base = InstalledAppsProto.ADAPTER.decodeInflated(baseDecrypted)
-
-            val diff = SavedAppsDifferenceProto.ADAPTER.decodeInflated(diffDecrypted).apps
-                ?: InstalledAppsDifferenceProto()
-
-            Pair(base, diff)
-        } catch (ex: IOException) {
-            if (BuildConfig.DEBUG) {
-                Log.d(LOG_TAG, "could not decode data", ex)
-            }
-
-            return null
-        }
-
-        return DecryptedInstalledApps(
-            base = base,
-            baseMeta = baseValue.metadata,
-            baseHeader = baseHeader,
-            diff = diff,
-            diffMeta = diffValue.metadata
-        )
     }
 
-    data class DecryptedInstalledApps(
-        val base: InstalledAppsProto,
-        val baseHeader: CryptContainer.Header,
-        val baseMeta: CryptContainerMetadata,
-        val diff: InstalledAppsDifferenceProto,
-        val diffMeta: CryptContainerMetadata
+    data class EncryptedInstalledApps(
+        val base: Encrypted<InstalledAppsProto>?,
+        val diff: Encrypted<InstalledAppsDifferenceProto>?
     )
 
+    data class Encrypted<T>(val meta: CryptContainerMetadata, val decrypted: Decrypted<T>?)
+
+    data class Decrypted<T>(val data: T, val header: CryptContainer.Header)
+
     suspend fun getInstalledAppsFromOs(appLogic: AppLogic, deviceState: DeviceState): InstalledAppsProto {
         val apps = kotlin.run {
             val currentlyInstalled = Threads.backgroundOSInteraction.executeAndWait {

app/src/main/java/io/timelimit/android/logic/crypto/CryptDataHandler.kt

@@ -59,7 +59,15 @@ object CryptDataHandler {
         } else oldItem.copy(serverVersion = data.version)
 
         if (deviceId == database.config().getOwnDeviceIdSync()) {
-            database.cryptContainer().updateMetadata(currentItem)
+            val updatedItem = if (isUnmodified)
+                currentItem
+            else
+                // this tells the sync logic to not use the existing encrypted data and start a new generation
+                // there is no need to actually save the data from the server because it is not used at all
+                // the DecryptProcessor ignores data for the device itself
+                currentItem.copy(status = CryptContainerMetadata.ProcessingStatus.Unprocessed)
+
+            database.cryptContainer().updateMetadata(updatedItem)
 
             return Result(didCreateKeyRequests = false)
         }