Retry transactions

src/database/main.ts

@@ -42,6 +42,7 @@ import { createU2fKeyModel, U2fKeyModelStatic } from './u2fkey'
 import { createUsedTimeModel, UsedTimeModelStatic } from './usedtime'
 import { createUserModel, UserModelStatic } from './user'
 import { createUserLimitLoginCategoryModel, UserLimitLoginCategoryModelStatic } from './userlimitlogincategory'
+import { shouldRetryWithException } from './utils/serialized'
 
 export type Transaction = Sequelize.Transaction
 
@@ -71,10 +72,18 @@ export interface Database {
   usedTime: UsedTimeModelStatic
   user: UserModelStatic
   userLimitLoginCategory: UserLimitLoginCategoryModelStatic
-  transaction: <T> (autoCallback: (t: Transaction) => Promise<T>, options?: { transaction: Transaction }) => Promise<T>
+  transaction: <T> (
+    autoCallback: (t: Transaction) => Promise<T>,
+    options?: TransactionOptions
+  ) => Promise<T>
   dialect: string
 }
 
+interface TransactionOptions {
+  transaction?: Transaction
+  disableRetry?: boolean
+}
+
 const createDatabase = (sequelize: Sequelize.Sequelize): Database => ({
   addDeviceToken: createAddDeviceTokenModel(sequelize),
   authtoken: createAuthtokenModel(sequelize),
@@ -101,10 +110,39 @@ const createDatabase = (sequelize: Sequelize.Sequelize): Database => ({
   usedTime: createUsedTimeModel(sequelize),
   user: createUserModel(sequelize),
   userLimitLoginCategory: createUserLimitLoginCategoryModel(sequelize),
-  transaction: <T> (autoCallback: (transaction: Transaction) => Promise<T>, options?: { transaction: Transaction }) => (sequelize.transaction({
+  async transaction<T>(
+    autoCallback: (transaction: Transaction) => Promise<T>,
+    options?: TransactionOptions
+  ): Promise<T> {
+    const runAttempt = () => sequelize.transaction({
       isolationLevel: Sequelize.Transaction.ISOLATION_LEVELS.SERIALIZABLE,
       transaction: options?.transaction
-  }, autoCallback)) as Promise<T>,
+    }, autoCallback)
+
+    const delay = (time: number) => new Promise((resolve) => setTimeout(resolve, time))
+
+    try {
+      return await runAttempt()
+    } catch (ex) {
+      if (
+        options?.disableRetry ||
+        options?.transaction ||
+        !shouldRetryWithException(this, ex)
+      ) throw ex
+    }
+
+    await delay(10 * (1 + Math.random()))
+
+    try {
+      return await runAttempt()
+    } catch (ex) {
+      if (!shouldRetryWithException(this, ex)) throw ex
+    }
+
+    await delay(100 * (1 + Math.random()))
+
+    return await runAttempt()
+  },
   dialect: sequelize.getDialect()
 })
 

src/database/utils/serialized.ts

@@ -86,8 +86,8 @@ export async function assertSerializeableTransactionsAreWorking (database: Datab
             await database.config.update({ value: 'd' }, { where: { id: configItemIds.secondSelfTestData }, transaction: transactionTwo })
           })()
         ])
-      })
+      }, { disableRetry: true })
-    })
+    }, { disableRetry: true })
 
     throw new SerializationFeatureCheckException()
   } catch (ex) {