improve concurrency, misc fixes, tests

frank06 · frank06 · commit bf3005f9926a · 2024-04-30T11:27:22.000-03:00
diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@
 
 ## Persistent reactive models in Flutter with zero boilerplate
 
-Flutter Data is an offline-first data framework with a customizable REST client and powerful model relationships, built on Riverpod.
+Flutter Data is a [local-first](https://www.inkandswitch.com/local-first/) data framework with a customizable REST client and powerful model relationships, built on Riverpod.
 
 <small>Inspired by [Ember Data](https://github.com/emberjs/data) and [ActiveRecord](https://guides.rubyonrails.org/active_record_basics.html).</small>
 
diff --git a/lib/src/adapter/adapter.dart b/lib/src/adapter/adapter.dart
@@ -175,7 +175,7 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
     if (model._key == null) {
       throw Exception("Model must be initialized:\n\n$model");
     }
-    final key = model._key!.detypifyKey()!;
+    final key = model._key!.detypifyKey();
     final map = serializeLocal(model, withRelationships: false);
     final data = jsonEncode(map);
     db.execute(
@@ -192,6 +192,8 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
     final internalProvidersMap = _internalProvidersMap!;
     final _internalType = internalType;
 
+    final _refProvider = Provider((ref) => ref);
+
     return await Isolate.run(() async {
       late final ProviderContainer container;
       try {
@@ -203,8 +205,18 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
           ],
         );
 
-        await container
-            .read(initializeFlutterData(internalProvidersMap).future);
+        final _ref = container.read(_refProvider);
+        _internalProvidersMap = internalProvidersMap;
+        _internalAdaptersMap = internalProvidersMap
+            .map((key, value) => MapEntry(key, _ref.read(value)));
+
+        await container.read(localStorageProvider).initialize(inIsolate: true);
+
+        // initialize and register
+        for (final adapter in _internalAdaptersMap!.values) {
+          adapter.dispose();
+          await adapter.initialize(ref: _ref);
+        }
 
         final adapter = internalProvidersMap[_internalType]!;
         return fn(container.read(adapter));
@@ -220,24 +232,43 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
   List<String> _saveManyLocal(
       Adapter adapter, Iterable<DataModelMixin> models) {
     final db = adapter.db;
-    final savedKeys = <String>[];
 
+    final savedKeys = <int>[];
+    final pssMap = <PreparedStatement, List<(int, String)>>{};
+    final keyMap = <int, String>{};
+
+    // per adapter, create prepared statements for each type and serialize models
     final grouped = models.groupSetsBy((e) => e._adapter);
     for (final e in grouped.entries) {
       final adapter = e.key;
+
       final ps = db.prepare(
           'REPLACE INTO ${adapter.internalType} (key, data) VALUES (?, ?) RETURNING key;');
+      pssMap[ps] = [];
+
       for (final model in e.value) {
-        final key = model._key!.detypifyKey();
+        final [type, _key] = model._key!.split('#');
+        final intKey = int.parse(_key);
+        keyMap[intKey] = type;
         final map = adapter.serializeLocal(model, withRelationships: false);
         final data = jsonEncode(map);
+        pssMap[ps]!.add((intKey, data));
+      }
+    }
+
+    // with everything ready, execute transaction
+    db.execute('BEGIN');
+    for (final MapEntry(key: ps, value: record) in pssMap.entries) {
+      for (final (key, data) in record) {
         final result = ps.select([key, data]);
-        savedKeys
-            .add((result.first['key'] as int).typifyWith(adapter.internalType));
+        savedKeys.add(result.first['key'] as int);
       }
       ps.dispose();
     }
-    return savedKeys;
+    db.execute('COMMIT');
+
+    // read keys returned by queries and typify with their original type
+    return savedKeys.map((key) => key.typifyWith(keyMap[key]!)).toList();
   }
 
   Future<List<String>?> saveManyLocal(Iterable<DataModelMixin> models,
@@ -268,7 +299,7 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
 
   /// Deletes models with [keys] from local storage.
   void deleteLocalByKeys(Iterable<String> keys, {bool notify = true}) {
-    final intKeys = keys.map((k) => k.detypifyKey()!).toList();
+    final intKeys = keys.map((k) => k.detypifyKey()).toList();
     db.execute(
         'DELETE FROM $internalType WHERE key IN (${keys.map((_) => '?').join(', ')})',
         intKeys);
diff --git a/lib/src/core/core_notifier.dart b/lib/src/core/core_notifier.dart
@@ -38,9 +38,6 @@ class CoreNotifier extends DelayedStateNotifier<DataGraphEvent> {
   /// Finds an ID, given a [key].
   Object? getIdForKey(String key) {
     final intKey = key.detypifyKey();
-    if (intKey == null) {
-      return null;
-    }
     final result = storage.db
         .select('SELECT id, is_int FROM _keys WHERE key = ?', [intKey]);
     if (result.isEmpty) {
@@ -56,7 +53,7 @@ class CoreNotifier extends DelayedStateNotifier<DataGraphEvent> {
   @protected
   Future<void> deleteKeysWithEdges(Iterable<String> keys) async {
     final params = keys.map((_) => '?').join(', ');
-    final intKeys = keys.map((k) => k.detypifyKey()!).toList();
+    final intKeys = keys.map((k) => k.detypifyKey()).toList();
 
     storage.db.execute('BEGIN');
     storage.db.execute('DELETE FROM _keys WHERE key IN ($params);', intKeys);
diff --git a/lib/src/model/relationship/has_many.dart b/lib/src/model/relationship/has_many.dart
@@ -44,7 +44,9 @@ class HasMany<E extends DataModelMixin<E>> extends Relationship<E, Set<E>> {
   }
 
   void addAll(Iterable<E> values) {
+    db.execute('BEGIN');
     _addAll(values.map((e) => e._key!).toSet());
+    db.execute('COMMIT');
   }
 
   bool contains(E element) {
diff --git a/lib/src/model/relationship/relationship.dart b/lib/src/model/relationship/relationship.dart
@@ -53,10 +53,10 @@ sealed class Relationship<E extends DataModelMixin<E>, N> with EquatableMixin {
 
     // setting up from scratch, remove all and add keys
 
-    db.execute('BEGIN');
     final existingKeys = keys;
     final keysToAdd = _uninitializedKeys!.difference(existingKeys);
     final keysToRemove = existingKeys.difference(_uninitializedKeys!);
+    db.execute('BEGIN');
     _removeAll(keysToRemove);
     _addAll(keysToAdd);
     db.execute('COMMIT');
diff --git a/lib/src/storage/local_storage.dart b/lib/src/storage/local_storage.dart
@@ -4,17 +4,21 @@ class LocalStorage {
   LocalStorage({
     required this.baseDirFn,
     LocalStorageClearStrategy? clear,
+    this.busyTimeout = 5000,
   }) : clear = clear ?? LocalStorageClearStrategy.never;
 
   var isInitialized = false;
 
   final FutureOr<String> Function() baseDirFn;
   final LocalStorageClearStrategy clear;
+  final int busyTimeout;
 
   late final String path;
+
+  @protected
   late final Database db;
 
-  Future<LocalStorage> initialize() async {
+  Future<LocalStorage> initialize({bool inIsolate = false}) async {
     if (isInitialized) return this;
 
     final baseDirPath = await baseDirFn();
@@ -25,12 +29,19 @@ class LocalStorage {
         await destroy();
       }
 
-      db = sqlite3.open(path);
+      db = sqlite3.open(path, mutex: false);
 
-      db.execute('''
+      if (inIsolate) {
+        db.execute('''
+          PRAGMA journal_mode = WAL;
+          PRAGMA busy_timeout = $busyTimeout;
+        ''');
+      } else {
+        db.execute('''
           PRAGMA journal_mode = WAL;
+          PRAGMA busy_timeout = $busyTimeout;
           VACUUM;
-          
+      
           CREATE TABLE IF NOT EXISTS _edges (
             key_ INTEGER NOT NULL,
             name_ TEXT,
@@ -59,6 +70,7 @@ class LocalStorage {
             key TEXT
           );
         ''');
+      }
     } catch (e, stackTrace) {
       print('[flutter_data] Failed to open:\n$e\n$stackTrace');
       if (clear == LocalStorageClearStrategy.whenError) {
diff --git a/lib/src/utils/extensions.dart b/lib/src/utils/extensions.dart
@@ -65,12 +65,8 @@ extension StringUtilsX on String {
   }
 
   /// Returns key as int
-  int? detypifyKey() {
+  int detypifyKey() {
     final [_, key] = split('#');
-    if (key.isEmpty) {
-      // enters here if there is only a type, e.g. `people`
-      return null;
-    }
     return int.parse(key);
   }
 
diff --git a/test/repository/remote_adapter_serialization_test.dart b/test/repository/remote_adapter_serialization_test.dart
@@ -245,4 +245,40 @@ void main() async {
       'number_of_sales': 0,
     });
   });
+
+  test('deserialize async', () async {
+    final data = await container.people.deserializeAsync([
+      {'_id': '23', 'name': 'Ko', 'age': 24}
+    ]);
+
+    expect(data.models, [Person(id: '23', name: 'Ko', age: 24)]);
+
+    final data2 = await container.people.deserializeAsync([
+      {'_id': '26', 'name': 'Ze', 'age': 58}
+    ]);
+
+    expect(data2.models, [Person(id: '26', name: 'Ze', age: 58)]);
+  });
+
+  test('deserialize async concurrent', () async {
+    final p1 = container.people.deserializeAsync(
+        List.generate(500, (i) {
+          return {'_id': '$i', 'name': 'Tom $i', 'age': 18 + i};
+        }),
+        save: true);
+    final p2 = container.people.deserializeAsync(
+        List.generate(500, (i) {
+          final j = i + 500;
+          return {'_id': '$j', 'name': 'Jack $j', 'age': 18 + i};
+        }),
+        save: true);
+    final p3 = container.people.deserializeAsync(
+        List.generate(500, (i) {
+          final j = i + 1000;
+          return {'_id': '$j', 'name': 'Rob $j', 'age': 18 + i};
+        }),
+        save: true);
+    await Future.wait([p1, p2, p3]);
+    expect(container.people.countLocal, 1500);
+  });
 }

Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,9 @@ class HasMany<E extends DataModelMixin<E>> extends Relationship<E, Set<E>> {`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`void addAll(Iterable<E> values) {`
	`47`	`+ db.execute('BEGIN');`
`47`	`48`	`_addAll(values.map((e) => e._key!).toSet());`
	`49`	`+ db.execute('COMMIT');`
`48`	`50`	`}`
`49`	`51`
`50`	`52`	`bool contains(E element) {`
Original file line number	Diff line number	Diff line change
`@@ -65,12 +65,8 @@ extension StringUtilsX on String {`
`65`	`65`	`}`
`66`	`66`
`67`	`67`	`/// Returns key as int`
`68`		`- int? detypifyKey() {`
	`68`	`+ int detypifyKey() {`
`69`	`69`	`final [_, key] = split('#');`
`70`		`- if (key.isEmpty) {`
`71`		- // enters here if there is only a type, e.g. `people`
`72`		`- return null;`
`73`		`- }`
`74`	`70`	`return int.parse(key);`
`75`	`71`	`}`
`76`	`72`