diff --git a/src/L2.php b/src/L2.php index 4ed0042..2ac5520 100644 --- a/src/L2.php +++ b/src/L2.php @@ -9,6 +9,5 @@ abstract public function set($pool, Address $address, $value = null, $expiration abstract public function delete($pool, Address $address); abstract public function deleteTag(L1 $l1, $tag); abstract public function getAddressesForTag($tag); - abstract public function collectGarbage($item_limit = null); abstract public function countGarbage(); } diff --git a/src/LX.php b/src/LX.php index 58bf890..24cb405 100644 --- a/src/LX.php +++ b/src/LX.php @@ -35,4 +35,9 @@ public function exists(Address $address) $value = $this->get($address); return !is_null($value); } + + public function collectGarbage($item_limit = null) + { + return 0; + } } diff --git a/src/SQLiteL1.php b/src/SQLiteL1.php new file mode 100644 index 0000000..933a69f --- /dev/null +++ b/src/SQLiteL1.php @@ -0,0 +1,294 @@ +query('SELECT 1 FROM ' . $table_name . ' LIMIT 1'); + } catch (\PDOException $e) { + if (in_array($e->getCode(), ['42S02', 'HY000'])) { + return false; + } + // Rethrow anything else. + // @codeCoverageIgnoreStart + throw $e; + // @codeCoverageIgnoreEnd + } + return true; + } + + protected static function initializeSchema(\PDO $dbh) + { + if (!self::tableExists($dbh, 'entries')) { + $dbh->exec('CREATE TABLE IF NOT EXISTS entries("address" TEXT PRIMARY KEY, "value" BLOB, "expiration" INTEGER, "created" INTEGER, "event_id" INTEGER NOT NULL DEFAULT 0, "reads" INTEGER NOT NULL DEFAULT 0, "writes" INTEGER NOT NULL DEFAULT 0)'); + $dbh->exec('CREATE INDEX IF NOT EXISTS expiration ON entries ("expiration")'); + } + } + + protected static function getDatabaseHandle($pool) + { + $path = join(DIRECTORY_SEPARATOR, array(sys_get_temp_dir(), 'lcache-' . $pool)); + $dbh = new \PDO('sqlite:' . $path . '.sqlite3'); + $dbh->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); + $dbh->exec('PRAGMA synchronous = OFF'); + $dbh->exec('PRAGMA foreign_keys = ON'); + self::initializeSchema($dbh); + return $dbh; + } + + public function __construct($pool = null) + { + parent::__construct($pool); + $this->dbh = self::getDatabaseHandle($this->pool); + + $this->statusKeyHits = 'lcache_status:' . $this->pool . ':hits'; + $this->statusKeyMisses = 'lcache_status:' . $this->pool . ':misses'; + $this->statusKeyLastAppliedEventId = 'lcache_status:' . $this->pool . ':last_applied_event_id'; + } + + protected function pruneExpiredEntries() + { + $sth = $this->dbh->prepare('DELETE FROM entries WHERE expiration < :now'); + $sth->bindValue(':now', $_SERVER['REQUEST_TIME'], \PDO::PARAM_INT); + try { + $sth->execute(); + // @codeCoverageIgnoreStart + } catch (\PDOException $e) { + $text = 'LCache SQLiteL1: Pruning Failed: ' . $e->getMessage(); + trigger_error($text, E_USER_WARNING); + return false; + } + // @codeCoverageIgnoreEnd + return $sth->rowCount(); + } + + public function __destruct() + { + $this->pruneExpiredEntries(); + } + + public function collectGarbage($item_limit = null) + { + $items = $this->pruneExpiredEntries(); + return $items; + } + + public function getKeyOverhead(Address $address) + { + $sth = $this->dbh->prepare('SELECT "reads", "writes" FROM entries WHERE "address" = :address'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->execute(); + $result = $sth->fetchObject(); + + if ($result === false) { + return 0; + } + + return $result->writes - $result->reads; + } + + public function setWithExpiration($event_id, Address $address, $value, $created, $expiration = null) + { + $serialized_value = null; + if (!is_null($value)) { + $serialized_value = serialize($value); + } + + $sth = $this->dbh->prepare('INSERT OR IGNORE INTO entries ("address", "value", "expiration", "created", "event_id", "writes") VALUES (:address, :value, :expiration, :created, :event_id, 1)'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':value', $serialized_value, \PDO::PARAM_LOB); + $sth->bindValue(':expiration', $expiration, \PDO::PARAM_INT); + $sth->bindValue(':created', $created, \PDO::PARAM_INT); + $sth->bindValue(':event_id', $event_id, \PDO::PARAM_INT); + $sth->execute(); + + // A count of zero means a conflict during the insertion. Update in a way + // that avoids stomping on newer writes. + if ($sth->rowCount() === 0) { + $bump_writes = ', "writes" = "writes" + 1'; + // Don't bump write counts for negative cache entries. + if (is_null($value)) { + $bump_writes = ''; + } + + // Always allow overwrites of event ID zero so when there's been a + // read (which creates a row with a read count of one) and then we + // still miss L2 (which creates an L1 tombstone here), the update + // goes through. + $sth = $this->dbh->prepare('UPDATE entries SET "value" = :value, "expiration" = :expiration, "created" = :created, "event_id" = :event_id ' . $bump_writes . ' WHERE "address" = :address AND ("event_id" < :event_id OR "event_id" = 0)'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':value', $serialized_value, \PDO::PARAM_LOB); + $sth->bindValue(':expiration', $expiration, \PDO::PARAM_INT); + $sth->bindValue(':created', $created, \PDO::PARAM_INT); + $sth->bindValue(':event_id', $event_id, \PDO::PARAM_INT); + $sth->execute(); + } + + return true; + } + + public function exists(Address $address) + { + $sth = $this->dbh->prepare('SELECT COUNT(*) AS existing FROM entries WHERE "address" = :address AND ("expiration" >= :now OR "expiration" IS NULL) AND "value" IS NOT NULL'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':now', $_SERVER['REQUEST_TIME'], \PDO::PARAM_INT); + $sth->execute(); + $result = $sth->fetchObject(); + return $result->existing > 0; + } + + public function isNegativeCache(Address $address) + { + $sth = $this->dbh->prepare('SELECT COUNT(*) AS entry_count FROM entries WHERE "address" = :address AND ("expiration" >= :now OR "expiration" IS NULL) AND "value" IS NULL'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':now', $_SERVER['REQUEST_TIME'], \PDO::PARAM_INT); + $sth->execute(); + $result = $sth->fetchObject(); + return ($result->entry_count > 0); + } + + /** + * @codeCoverageIgnore + */ + public function debugDumpState() + { + echo PHP_EOL . PHP_EOL . 'Entries:' . PHP_EOL; + $sth = $this->dbh->prepare('SELECT * FROM "entries" ORDER BY "address"'); + $sth->execute(); + while ($event = $sth->fetchObject()) { + print_r($event); + } + echo PHP_EOL; + } + + public function getEntry(Address $address) + { + $sth = $this->dbh->prepare('SELECT "value", "expiration", "reads", "writes", "created" FROM entries WHERE "address" = :address AND ("expiration" >= :now OR "expiration" IS NULL)'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':now', $_SERVER['REQUEST_TIME'], \PDO::PARAM_INT); + $sth->execute(); + $entry = $sth->fetchObject(); + + // If there are under 10X reads versus writes, bump the read count. We + // do this to simultaneously track useful overhead data but not unnecessarily + // record reads after they massively outweigh writes for an address. + // @TODO: Make this adapt to overhead thresholds. + if ($entry === false || $entry->reads < 10 * $entry->writes || $entry->reads < 10) { + $sth = $this->dbh->prepare('UPDATE entries SET "reads" = "reads" + 1 WHERE "address" = :address'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->execute(); + if ($sth->rowCount() === 0) { + // Use a zero expiration so this row is only used for counts, not negative caching. + // Use the default event ID of zero to ensure any writes win over this stub. + $sth = $this->dbh->prepare('INSERT OR IGNORE INTO entries ("address", "expiration", "reads") VALUES (:address, 0, 1)'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->execute(); + } + } + + if ($entry === false) { + $this->recordMiss(); + return null; + } + + // Unserialize non-null values. + if (!is_null($entry->value)) { + $entry->value = unserialize($entry->value); + } + + $this->recordHit(); + return $entry; + } + + public function delete($event_id, Address $address) + { + if ($address->isEntireCache() || $address->isEntireBin()) { + $pattern = $address->serialize() . '%'; + $sth = $this->dbh->prepare('DELETE FROM entries WHERE "address" LIKE :pattern'); + $sth->bindValue('pattern', $pattern, \PDO::PARAM_STR); + $sth->execute(); + return true; + } + + $sth = $this->dbh->prepare('DELETE FROM entries WHERE "address" = :address AND event_id < :event_id'); + $sth->bindValue(':address', $address->serialize(), \PDO::PARAM_STR); + $sth->bindValue(':event_id', $event_id, \PDO::PARAM_INT); + $sth->execute(); + return true; + } + + // @TODO: Update hit/miss functions to either record nothing or fall back + // to SQLite storage if APCu is not available. + + protected function recordHit() + { + $success = false; // Make Scrutinizer happy. + apcu_inc($this->statusKeyHits, 1, $success); + if (!$success) { + // @TODO: Remove this fallback when we drop APCu 4.x support. + // @codeCoverageIgnoreStart + // Ignore coverage because (1) it's tested with other code and + // (2) APCu 5.x does not use it. + apcu_store($this->statusKeyHits, 1); + // @codeCoverageIgnoreEnd + } + } + + protected function recordMiss() + { + $success = false; // Make Scrutinizer happy. + apcu_inc($this->statusKeyMisses, 1, $success); + if (!$success) { + // @TODO: Remove this fallback when we drop APCu 4.x support. + // @codeCoverageIgnoreStart + // Ignore coverage because (1) it's tested with other code and + // (2) APCu 5.x does not use it. + apcu_store($this->statusKeyMisses, 1); + // @codeCoverageIgnoreEnd + } + } + + public function getHits() + { + $value = apcu_fetch($this->statusKeyHits); + return $value ? $value : 0; + } + + public function getMisses() + { + $value = apcu_fetch($this->statusKeyMisses); + return $value ? $value : 0; + } + + // @TODO: Update event ID tracking to either record fall back + // to SQLite storage if APCu is not available. + + public function getLastAppliedEventID() + { + $value = apcu_fetch($this->statusKeyLastAppliedEventId); + if ($value === false) { + $value = null; + } + return $value; + } + + public function setLastAppliedEventID($eid) + { + return apcu_store($this->statusKeyLastAppliedEventId, $eid); + } +} diff --git a/tests/LCacheTest.php b/tests/LCacheTest.php index a490494..a82a1f0 100644 --- a/tests/LCacheTest.php +++ b/tests/LCacheTest.php @@ -46,42 +46,68 @@ public function testNullL1() $this->assertEquals(PHP_INT_MAX, $cache->getLastAppliedEventID()); } - public function testStaticL1SetGetDelete() + protected function performSetGetDeleteTest($l1) { $event_id = 1; - $cache = new StaticL1(); $myaddr = new Address('mybin', 'mykey'); - $this->assertEquals(0, $cache->getHits()); - $this->assertEquals(0, $cache->getMisses()); + $this->assertEquals(0, $l1->getHits()); + $this->assertEquals(0, $l1->getMisses()); - // Try to get an entry from an empty cache. - $entry = $cache->get($myaddr); + // Try to get an entry from an empty L1. + $entry = $l1->get($myaddr); $this->assertNull($entry); - $this->assertEquals(0, $cache->getHits()); - $this->assertEquals(1, $cache->getMisses()); + $this->assertEquals(0, $l1->getHits()); + $this->assertEquals(1, $l1->getMisses()); // Set and get an entry. - $cache->set($event_id++, $myaddr, 'myvalue'); - $entry = $cache->get($myaddr); + $l1->set($event_id++, $myaddr, 'myvalue'); + $entry = $l1->get($myaddr); $this->assertEquals('myvalue', $entry); - $this->assertEquals(1, $cache->getHits()); - $this->assertEquals(1, $cache->getMisses()); + $this->assertEquals(1, $l1->getHits()); + $this->assertEquals(1, $l1->getMisses()); // Delete the entry and try to get it again. - $cache->delete($event_id++, $myaddr); - $entry = $cache->get($myaddr); + $l1->delete($event_id++, $myaddr); + $entry = $l1->get($myaddr); $this->assertNull($entry); - $this->assertEquals(1, $cache->getHits()); - $this->assertEquals(2, $cache->getMisses()); + $this->assertEquals(1, $l1->getHits()); + $this->assertEquals(2, $l1->getMisses()); // Clear everything and try to read. - $cache->delete($event_id++, new Address()); - $entry = $cache->get($myaddr); + $l1->delete($event_id++, new Address()); + $entry = $l1->get($myaddr); $this->assertNull($entry); - $this->assertEquals(1, $cache->getHits()); - $this->assertEquals(3, $cache->getMisses()); + $this->assertEquals(1, $l1->getHits()); + $this->assertEquals(3, $l1->getMisses()); + + // This is a no-op for most L1 implementations, but it should not + // return false, regardless. + $this->assertTrue(false !== $l1->collectGarbage()); + + // Test complex values that need serialization. + $myarray = [1, 2, 3]; + $l1->set($event_id++, $myaddr, $myarray); + $entry = $l1->get($myaddr); + $this->assertEquals($myarray, $entry); + + // Test creation tracking. + $l1->setWithExpiration($event_id++, $myaddr, 'myvalue', 42); + $entry = $l1->getEntry($myaddr); + $this->assertEquals(42, $entry->created); + } + + public function testStaticL1SetGetDelete() + { + $l1 = new StaticL1(); + $this->performSetGetDeleteTest($l1); + } + + public function testSQLiteL1SetGetDelete() + { + $l1 = new SQLiteL1(); + $this->performSetGetDeleteTest($l1); } public function testStaticL1Antirollback() @@ -177,6 +203,42 @@ public function testNewPoolSynchronization() $this->assertEquals($pool1->getLastAppliedEventID(), $pool2->getLastAppliedEventID()); } + protected function performTombstoneTest($l1) + { + $central = new Integrated($l1, new StaticL2()); + + $dne = new Address('mypool', 'mykey-dne'); + $this->assertNull($central->get($dne)); + + $tombstone = $central->getEntry($dne, true); + $this->assertNotNull($tombstone); + $this->assertNull($tombstone->value); + // The L1 should return the tombstone entry so the integrated cache + // can avoid rewriting it. + $tombstone = $l1->getEntry($dne); + $this->assertNotNull($tombstone); + $this->assertNull($tombstone->value); + + // The tombstone should also count as non-existence. + $this->assertFalse($central->exists($dne)); + + // This is a no-op for most L1 implementations, but it should not + // return false, regardless. + $this->assertTrue(false !== $l1->collectGarbage()); + } + + public function testAPCuL1Tombstone() + { + $l1 = new APCuL1('testAPCuL1Tombstone'); + $this->performTombstoneTest($l1); + } + + public function testSQLiteL1Tombstone() + { + $l1 = new SQLiteL1(); + $this->performTombstoneTest($l1); + } + protected function performSynchronizationTest($central, $first_l1, $second_l1) { // Create two integrated pools with independent L1s. @@ -242,22 +304,6 @@ protected function performSynchronizationTest($central, $first_l1, $second_l1) $pool2->synchronize(); $this->assertNull($pool2->get($mybin1_mykey)); $this->assertEquals('myvalue2', $pool2->get($mybin2_mykey)); - - // An L2 miss should place a tombstone into L1. - $dne = new Address('mypool', 'mykey-dne'); - $this->assertNull($pool1->get($mybin1_mykey)); - $tombstone = $pool1->getEntry($mybin1_mykey, true); - $this->assertNotNull($tombstone); - $this->assertNull($tombstone->value); - - // The L1 should return the tombstone entry so the integrated cache - // can avoid rewriting it. - $tombstone = $first_l1->getEntry($mybin1_mykey); - $this->assertNotNull($tombstone); - $this->assertNull($tombstone->value); - - // The tombstone should also count as non-existence. - $this->assertFalse($pool1->exists($mybin1_mykey)); } protected function performClearSynchronizationTest($central, $first_l1, $second_l1) @@ -390,6 +436,16 @@ public function testSynchronizationAPCu() } } + public function testSynchronizationSQLiteL1() + { + $central = new StaticL2(); + $this->performSynchronizationTest($central, new SQLiteL1(), new SQLiteL1()); + + $this->performClearSynchronizationTest($central, new SQLiteL1(), new StaticL1()); + $this->performClearSynchronizationTest($central, new StaticL1(), new SQLiteL1()); + $this->performClearSynchronizationTest($central, new SQLiteL1(), new SQLiteL1()); + } + public function testSynchronizationDatabase() { $this->createSchema(); @@ -467,9 +523,8 @@ public function testEmptyCleanUpDatabaseL2() $l2 = new DatabaseL2($this->dbh); } - public function testExistsAPCuL1() + protected function performExistsTest($l1) { - $l1 = new APCuL1('first'); $myaddr = new Address('mybin', 'mykey'); $l1->set(1, $myaddr, 'myvalue'); $this->assertTrue($l1->exists($myaddr)); @@ -477,14 +532,22 @@ public function testExistsAPCuL1() $this->assertFalse($l1->exists($myaddr)); } + public function testExistsAPCuL1() + { + $l1 = new APCuL1('first'); + $this->performExistsTest($l1); + } + public function testExistsStaticL1() { $l1 = new StaticL1(); - $myaddr = new Address('mybin', 'mykey'); - $l1->set(1, $myaddr, 'myvalue'); - $this->assertTrue($l1->exists($myaddr)); - $l1->delete(2, $myaddr); - $this->assertFalse($l1->exists($myaddr)); + $this->performExistsTest($l1); + } + + public function testExistsSQLiteL1() + { + $l1 = new SQLiteL1(); + $this->performExistsTest($l1); } public function testExistsIntegrated() @@ -541,12 +604,18 @@ public function testAPCuL1Antirollback() $this->performL1AntirollbackTest($l1); } + public function testSQLite1Antirollback() + { + $l1 = new SQLiteL1(); + $this->performL1AntirollbackTest($l1); + } + protected function performL1HitMissTest($l1) { $myaddr = new Address('mybin', 'mykey'); $current_hits = $l1->getHits(); $current_misses = $l1->getMisses(); - $current_event_id = $l1->getLastAppliedEventID(); + $current_event_id = 1; $l1->get($myaddr); $this->assertEquals($current_misses + 1, $l1->getMisses()); $l1->set($current_event_id++, $myaddr, 'myvalue'); @@ -554,9 +623,15 @@ protected function performL1HitMissTest($l1) $this->assertEquals($current_hits + 1, $l1->getHits()); } - public function testAPCuHitMiss() + public function testAPCuL1HitMiss() { - $l1 = new APCuL1('testAPCuHitMiss'); + $l1 = new APCuL1('testAPCuL1HitMiss'); + $this->performL1HitMissTest($l1); + } + + public function testSQLiteL1HitMiss() + { + $l1 = new SQLiteL1(); $this->performL1HitMissTest($l1); } @@ -725,6 +800,11 @@ public function testAPCuL1Counters() $this->performHitSetCounterTest(new APCuL1('counters')); } + public function testSQLiteL1Counters() + { + $this->performHitSetCounterTest(new SQLiteL1()); + } + protected function performExcessiveOverheadSkippingTest($l1) { $pool = new Integrated($l1, new StaticL2(), 2); @@ -735,6 +815,8 @@ protected function performExcessiveOverheadSkippingTest($l1) $this->assertNotNull($pool->set($myaddr, 'myvalue2')); // This should return an event_id but delete the item. + $this->assertEquals(2, $l1->getKeyOverhead($myaddr)); + $this->assertFalse($l1->isNegativeCache($myaddr)); $this->assertNotNull($pool->set($myaddr, 'myvalue3')); $this->assertFalse($pool->exists($myaddr)); @@ -769,6 +851,11 @@ public function testAPCuL1ExcessiveOverheadSkipping() $this->performExcessiveOverheadSkippingTest(new APCuL1('overhead')); } + public function testSQLiteL1ExcessiveOverheadSkipping() + { + $this->performExcessiveOverheadSkippingTest(new SQLiteL1()); + } + public function testAPCuL1Expiration() { $l1 = new APCuL1(); @@ -803,6 +890,23 @@ public function testDatabaseL2BatchDeletion() $this->assertNull($l2->get($myaddr)); } + public function testSQLiteL1SchemaErrorHandling() + { + $pool_name = uniqid('', true) . '-' . mt_rand(); + $l1_a = new SQLiteL1($pool_name); + + // Opening a second instance of the same pool should work. + $l1_b = new SQLiteL1($pool_name); + + $myaddr = new Address('mybin', 'mykey'); + + $l1_a->set(1, $myaddr, 'myvalue'); + + // Reading from the second handle should show the value written to the + // first. + $this->assertEquals('myvalue', $l1_b->get($myaddr)); + } + /** * @return PHPUnit_Extensions_Database_DataSet_IDataSet */