Skip to content

Commit cbb3c09

Browse files
greyson-signalgreyson-signal
committed
Create backups from copies of the database file.
Still more work here to do with regards to certain tables, like SignalStore and Recipient.
1 parent 890facc commit cbb3c09

9 files changed

Lines changed: 140 additions & 64 deletions

File tree

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt‎

Lines changed: 92 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import kotlinx.coroutines.withContext
1212
import org.greenrobot.eventbus.EventBus
1313
import org.signal.core.util.Base64
1414
import org.signal.core.util.EventTimer
15+
import org.signal.core.util.fullWalCheckpoint
1516
import org.signal.core.util.logging.Log
1617
import org.signal.core.util.money.FiatMoney
1718
import org.signal.core.util.withinTransaction
@@ -43,6 +44,8 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter
4344
import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType
4445
import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFeature
4546
import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository
47+
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
48+
import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
4649
import org.thoughtcrime.securesms.database.DistributionListTables
4750
import org.thoughtcrime.securesms.database.SignalDatabase
4851
import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
@@ -71,6 +74,7 @@ import org.whispersystems.signalservice.internal.push.SubscriptionsConfiguration
7174
import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
7275
import java.io.ByteArrayOutputStream
7376
import java.io.File
77+
import java.io.IOException
7478
import java.io.InputStream
7579
import java.io.OutputStream
7680
import java.math.BigDecimal
@@ -83,6 +87,7 @@ object BackupRepository {
8387

8488
private val TAG = Log.tag(BackupRepository::class.java)
8589
private const val VERSION = 1L
90+
private const val DB_SNAPSHOT_NAME = "signal-snapshot.db"
8691

8792
private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error ->
8893
when (error.code) {
@@ -105,64 +110,106 @@ object BackupRepository {
105110
SignalStore.backup().backupTier = null
106111
}
107112

108-
fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) {
109-
val eventTimer = EventTimer()
110-
val writer: BackupExportWriter = if (plaintext) {
111-
PlainTextBackupWriter(outputStream)
112-
} else {
113-
EncryptedBackupWriter(
114-
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
115-
aci = SignalStore.account().aci!!,
116-
outputStream = outputStream,
117-
append = append
113+
private fun createSignalDatabaseSnapshot(): SignalDatabase {
114+
// Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes
115+
if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) {
116+
Log.w(TAG, "Failed to checkpoint WAL! Not guaranteed to be using the most recent data.")
117+
}
118+
119+
// We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file
120+
return SignalDatabase.rawDatabase.withinTransaction {
121+
val context = AppDependencies.application
122+
123+
val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME)
124+
val targetFile = File(existingDbFile.parentFile, DB_SNAPSHOT_NAME)
125+
126+
try {
127+
existingDbFile.copyTo(targetFile, overwrite = true)
128+
} catch (e: IOException) {
129+
// TODO [backup] Gracefully handle this error
130+
throw IllegalStateException("Failed to copy database file!", e)
131+
}
132+
133+
SignalDatabase(
134+
context = context,
135+
databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context),
136+
attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
137+
name = DB_SNAPSHOT_NAME
118138
)
119139
}
140+
}
141+
142+
private fun deleteDatabaseSnapshot() {
143+
val targetFile = AppDependencies.application.getDatabasePath(DB_SNAPSHOT_NAME)
144+
if (!targetFile.delete()) {
145+
Log.w(TAG, "Failed to delete database snapshot!")
146+
}
147+
}
120148

121-
val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia)
149+
fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) {
150+
val eventTimer = EventTimer()
151+
val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot()
122152

123-
writer.use {
124-
writer.write(
125-
BackupInfo(
126-
version = VERSION,
127-
backupTimeMs = exportState.backupTime
153+
try {
154+
val writer: BackupExportWriter = if (plaintext) {
155+
PlainTextBackupWriter(outputStream)
156+
} else {
157+
EncryptedBackupWriter(
158+
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
159+
aci = SignalStore.account().aci!!,
160+
outputStream = outputStream,
161+
append = append
128162
)
129-
)
130-
// Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
131-
// writes from other threads are blocked. This is something to think more about.
132-
SignalDatabase.rawDatabase.withinTransaction {
133-
AccountDataProcessor.export {
134-
writer.write(it)
135-
eventTimer.emit("account")
136-
}
163+
}
137164

138-
RecipientBackupProcessor.export(exportState) {
139-
writer.write(it)
140-
eventTimer.emit("recipient")
141-
}
165+
val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia)
142166

143-
ChatBackupProcessor.export(exportState) { frame ->
144-
writer.write(frame)
145-
eventTimer.emit("thread")
146-
}
167+
writer.use {
168+
writer.write(
169+
BackupInfo(
170+
version = VERSION,
171+
backupTimeMs = exportState.backupTime
172+
)
173+
)
174+
// Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
175+
// writes from other threads are blocked. This is something to think more about.
176+
dbSnapshot.rawWritableDatabase.withinTransaction {
177+
AccountDataProcessor.export(dbSnapshot) {
178+
writer.write(it)
179+
eventTimer.emit("account")
180+
}
147181

148-
AdHocCallBackupProcessor.export { frame ->
149-
writer.write(frame)
150-
eventTimer.emit("call")
151-
}
182+
RecipientBackupProcessor.export(dbSnapshot, exportState) {
183+
writer.write(it)
184+
eventTimer.emit("recipient")
185+
}
152186

153-
StickerBackupProcessor.export { frame ->
154-
writer.write(frame)
155-
eventTimer.emit("sticker-pack")
156-
}
187+
ChatBackupProcessor.export(dbSnapshot, exportState) { frame ->
188+
writer.write(frame)
189+
eventTimer.emit("thread")
190+
}
157191

158-
ChatItemBackupProcessor.export(exportState) { frame ->
159-
writer.write(frame)
160-
eventTimer.emit("message")
192+
AdHocCallBackupProcessor.export(dbSnapshot) { frame ->
193+
writer.write(frame)
194+
eventTimer.emit("call")
195+
}
196+
197+
StickerBackupProcessor.export(dbSnapshot) { frame ->
198+
writer.write(frame)
199+
eventTimer.emit("sticker-pack")
200+
}
201+
202+
ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame ->
203+
writer.write(frame)
204+
eventTimer.emit("message")
205+
}
161206
}
162207
}
163-
}
164208

165-
Log.d(TAG, "export() ${eventTimer.stop().summary}")
209+
Log.d(TAG, "export() ${eventTimer.stop().summary}")
210+
} finally {
211+
deleteDatabaseSnapshot()
212+
}
166213
}
167214

168215
fun export(plaintext: Boolean = false): ByteArray {

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter
1414
import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository
1515
import org.thoughtcrime.securesms.components.settings.app.usernamelinks.UsernameQrCodeColorScheme
1616
import org.thoughtcrime.securesms.database.SignalDatabase
17-
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.recipients
1817
import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
1918
import org.thoughtcrime.securesms.dependencies.AppDependencies
2019
import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob
@@ -34,12 +33,14 @@ import kotlin.jvm.optionals.getOrNull
3433

3534
object AccountDataProcessor {
3635

37-
fun export(emitter: BackupFrameEmitter) {
36+
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
3837
val context = AppDependencies.application
3938

39+
// TODO [backup] Need to get it from the db snapshot
4040
val self = Recipient.self().fresh()
41-
val record = recipients.getRecordForSync(self.id)
41+
val record = db.recipientTable.getRecordForSync(self.id)
4242

43+
// TODO [backup] Need to get it from the db snapshot
4344
val subscriber: InAppPaymentSubscriberRecord? = InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION)
4445

4546
emitter.emit(
@@ -80,7 +81,7 @@ object AccountDataProcessor {
8081
}
8182

8283
fun import(accountData: AccountData, selfId: RecipientId) {
83-
recipients.restoreSelfFromBackup(accountData, selfId)
84+
SignalDatabase.recipients.restoreSelfFromBackup(accountData, selfId)
8485

8586
SignalStore.account().setRegistered(true)
8687

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ object AdHocCallBackupProcessor {
1818

1919
val TAG = Log.tag(AdHocCallBackupProcessor::class.java)
2020

21-
fun export(emitter: BackupFrameEmitter) {
22-
SignalDatabase.calls.getAdhocCallsForBackup().use { reader ->
21+
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
22+
db.callTable.getAdhocCallsForBackup().use { reader ->
2323
for (callLog in reader) {
2424
if (callLog != null) {
2525
emitter.emit(Frame(adHocCall = callLog))

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ import org.thoughtcrime.securesms.recipients.RecipientId
1919
object ChatBackupProcessor {
2020
val TAG = Log.tag(ChatBackupProcessor::class.java)
2121

22-
fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
23-
SignalDatabase.threads.getThreadsForBackup().use { reader ->
22+
fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
23+
db.threadTable.getThreadsForBackup().use { reader ->
2424
for (chat in reader) {
2525
if (exportState.recipientIds.contains(chat.recipientId)) {
2626
exportState.threadIds.add(chat.id)

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ import org.thoughtcrime.securesms.database.SignalDatabase
1818
object ChatItemBackupProcessor {
1919
val TAG = Log.tag(ChatItemBackupProcessor::class.java)
2020

21-
fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
22-
SignalDatabase.messages.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
21+
fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
22+
db.messageTable.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
2323
while (chatItems.hasNext()) {
2424
val chatItem = chatItems.next()
2525
if (chatItem != null) {

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ object RecipientBackupProcessor {
3030

3131
val TAG = Log.tag(RecipientBackupProcessor::class.java)
3232

33-
fun export(state: ExportState, emitter: BackupFrameEmitter) {
33+
fun export(db: SignalDatabase, state: ExportState, emitter: BackupFrameEmitter) {
34+
// TODO [backup] Need to get it from the db snapshot
3435
val selfId = Recipient.self().id.toLong()
3536
val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId
3637
if (releaseChannelId != null) {
@@ -44,7 +45,7 @@ object RecipientBackupProcessor {
4445
)
4546
}
4647

47-
SignalDatabase.recipients.getContactsForBackup(selfId).use { reader ->
48+
db.recipientTable.getContactsForBackup(selfId).use { reader ->
4849
for (backupRecipient in reader) {
4950
if (backupRecipient != null) {
5051
state.recipientIds.add(backupRecipient.id)
@@ -53,19 +54,19 @@ object RecipientBackupProcessor {
5354
}
5455
}
5556

56-
SignalDatabase.recipients.getGroupsForBackup().use { reader ->
57+
db.recipientTable.getGroupsForBackup().use { reader ->
5758
for (backupRecipient in reader) {
5859
state.recipientIds.add(backupRecipient.id)
5960
emitter.emit(Frame(recipient = backupRecipient))
6061
}
6162
}
6263

63-
SignalDatabase.distributionLists.getAllForBackup().forEach {
64+
db.distributionListTables.getAllForBackup().forEach {
6465
state.recipientIds.add(it.id)
6566
emitter.emit(Frame(recipient = it))
6667
}
6768

68-
SignalDatabase.callLinks.getCallLinksForBackup().forEach {
69+
db.callLinkTable.getCallLinksForBackup().forEach {
6970
state.recipientIds.add(it.id)
7071
emitter.emit(Frame(recipient = it))
7172
}

‎app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
1717
import org.thoughtcrime.securesms.jobs.StickerPackDownloadJob
1818

1919
object StickerBackupProcessor {
20-
fun export(emitter: BackupFrameEmitter) {
21-
StickerPackRecordReader(SignalDatabase.stickers.allStickerPacks).use { reader ->
20+
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
21+
StickerPackRecordReader(db.stickerTable.allStickerPacks).use { reader ->
2222
var record: StickerPackRecord? = reader.next
2323
while (record != null) {
2424
if (record.isInstalled) {

‎app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.thoughtcrime.securesms.service.KeyCachingService
2323
import org.thoughtcrime.securesms.util.TextSecurePreferences
2424
import java.io.File
2525

26-
open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret) :
26+
open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, private val name: String = DATABASE_NAME) :
2727
SQLiteOpenHelper(
2828
context,
2929
DATABASE_NAME,
@@ -219,7 +219,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
219219

220220
companion object {
221221
private val TAG = Log.tag(SignalDatabase::class.java)
222-
private const val DATABASE_NAME = "signal.db"
222+
const val DATABASE_NAME = "signal.db"
223223

224224
@JvmStatic
225225
@Volatile

‎core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,33 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean {
8989
}
9090
}
9191

92+
/**
93+
* Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out).
94+
* Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint
95+
* can't complete in a reasonable amount of time.
96+
*
97+
* See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint
98+
*/
99+
fun SupportSQLiteDatabase.fullWalCheckpoint(maxAttempts: Int = 3): Boolean {
100+
var attempts = 0
101+
102+
while (attempts < maxAttempts) {
103+
if (this.walCheckpoint()) {
104+
return true
105+
}
106+
107+
attempts++
108+
}
109+
110+
return false
111+
}
112+
113+
private fun SupportSQLiteDatabase.walCheckpoint(): Boolean {
114+
return this.query("PRAGMA wal_checkpoint(TRUNCATE)").use { cursor ->
115+
cursor.moveToFirst() && cursor.getInt(0) == 0
116+
}
117+
}
118+
92119
fun SupportSQLiteDatabase.getIndexes(): List<Index> {
93120
return this.query("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name ASC").readToList { cursor ->
94121
val indexName = cursor.requireNonNullString("name")

0 commit comments

Comments
 (0)