From b8c334f7f34ad88f68c3ba8f6641138ec3245c48 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:35:51 +0000 Subject: [PATCH 1/8] fix(cache): keep invalidation state outside evicting data caches --- docs/cache-invalidation-state.md | 21 +++++++ src/Database/Cache/Query.php | 49 ++++++++-------- src/Database/Database.php | 21 +++++++ src/Database/Trait/Documents.php | 56 +++++++++---------- tests/unit/Cache/QueryTest.php | 26 +++++++++ .../unit/Documents/DocumentCacheEpochTest.php | 51 ++++++++++++++++- 6 files changed, 173 insertions(+), 51 deletions(-) create mode 100644 docs/cache-invalidation-state.md diff --git a/docs/cache-invalidation-state.md b/docs/cache-invalidation-state.md new file mode 100644 index 000000000..1ab80c555 --- /dev/null +++ b/docs/cache-invalidation-state.md @@ -0,0 +1,21 @@ +# Invalidation state and evicting caches + +Document caching keeps a collection epoch, started/finished generations and writer registrations. These are coordination state: independently evicting one key can make the generations unequal even after every writer has finished. A replacement blocked epoch then disables document caching until the writer timeout; further eviction can restart that wait. No key TTL does not mean no eviction. + +Keep this state in a shared, non-evicting store, separately from disposable document/schema/query values: + +```php +$database = new Database($adapter, $dataCache); +$database->setInvalidationCache($stateCache); +$queryCache = new Utopia\Database\Cache\Query($dataCache, invalidationCache: $stateCache); +``` + +Configure the same state store on every reader and writer before using the handles. For Redis use `noeviction`; for Dragonfly leave `cache_mode` disabled. Size and monitor this store for the collection inventory; do not add expiry to its accounting keys. Do not wrap it in a circuit breaker that substitutes `0`, empty lists or cache misses for failed reads. A failed accounting operation must remain distinguishable from missing state. Document/schema values and their fill generations remain in the data cache. + +Without an explicit state cache the previous single-store behavior remains for compatibility. `setCache()` changes the data cache; it does not replace an explicitly configured state cache. Flushing data/query results does not clear state or erase in-flight writer barriers. + +## Existing installations + +Changing stores is a coordinated cutover, not a rolling environment-variable update. Pause all database writers (API, workers, tasks and migrations), drain in-flight transactions, stop old processes, and configure every process with the same state store. Clear the disposable data cache before starting the new processes and resuming writes, so no cached schema carries an epoch from the previous store. Retain rollback access to the old store. To roll back, repeat the drain/stop/data-cache-clear process with the previous configuration. + +Never flush an invalidation store with active writers. Availability and failover of this store are part of the cache consistency protocol. diff --git a/src/Database/Cache/Query.php b/src/Database/Cache/Query.php index dbc499b2c..ece39c866 100644 --- a/src/Database/Cache/Query.php +++ b/src/Database/Cache/Query.php @@ -40,13 +40,18 @@ class Query /** @var array */ private array $regions = []; + private readonly Cache $invalidationCache; + /** * @param int $slots Results a collection scope keeps at most; queries sharing a slot evict each other */ public function __construct( private readonly Cache $cache, private readonly int $slots = self::SLOTS, + ?Cache $invalidationCache = null, ) { + $this->invalidationCache = $invalidationCache ?? $cache; + if ($slots < 1) { throw new InvalidArgumentException('A query cache needs at least one slot'); } @@ -209,15 +214,15 @@ public function createToken(): string */ public function blockCollection(string $key, string $token): void { - if (! (new Owners($this->cache))->register($key, $token)) { + if (! (new Owners($this->invalidationCache))->register($key, $token)) { throw new Unavailable("Failed to register query cache owner for '{$key}'"); } - if ($this->cache->save($this->getEpochKey($key), self::BLOCKED_PREFIX.$token.self::SEPARATOR.\time()) === false) { + if ($this->invalidationCache->save($this->getEpochKey($key), self::BLOCKED_PREFIX.$token.self::SEPARATOR.\time()) === false) { throw new Unavailable("Failed to block query cache epoch for '{$key}'"); } - $this->cache->purge($this->getStartedKey($key)); + $this->invalidationCache->purge($this->getStartedKey($key)); $this->cache->purge($key, self::BLOCK_FIELD); } @@ -229,14 +234,14 @@ public function blockCollection(string $key, string $token): void */ public function activateCollection(string $key, string $token, int $writerTimeout): void { - $registration = (new Owners($this->cache))->find($key, $token); - $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); + $registration = (new Owners($this->invalidationCache))->find($key, $token); + $owner = $this->invalidationCache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid query cache owner for '{$key}'"); } $owned = $owner === $token; - if ($owned && ! $this->cache->purge($registration->key, $registration->field)) { - $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); + if ($owned && ! $this->invalidationCache->purge($registration->key, $registration->field)) { + $owner = $this->invalidationCache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null) { throw new Unavailable("Failed to release query cache owner for '{$key}'"); } @@ -246,9 +251,9 @@ public function activateCollection(string $key, string $token, int $writerTimeou $epochKey = $this->getEpochKey($key); $startedKey = $this->getStartedKey($key); $finishedKey = $this->getFinishedKey($key); - $started = $this->cache->getGeneration($startedKey); - $finished = $this->cache->getGeneration($finishedKey); - $current = $this->cache->load($epochKey, self::PERMANENT); + $started = $this->invalidationCache->getGeneration($startedKey); + $finished = $this->invalidationCache->getGeneration($finishedKey); + $current = $this->invalidationCache->load($epochKey, self::PERMANENT); $ours = $this->isTombstoneOf($current, $token); if ($started === $finished) { @@ -267,9 +272,9 @@ public function activateCollection(string $key, string $token, int $writerTimeou return; } - $this->cache->purge($finishedKey); - $nextStarted = $this->cache->getGeneration($startedKey); - $nextFinished = $this->cache->getGeneration($finishedKey); + $this->invalidationCache->purge($finishedKey); + $nextStarted = $this->invalidationCache->getGeneration($startedKey); + $nextFinished = $this->invalidationCache->getGeneration($finishedKey); if ($nextStarted === $nextFinished) { $this->publish($key, $nextFinished); @@ -277,7 +282,7 @@ public function activateCollection(string $key, string $token, int $writerTimeou return; } - if ($nextFinished === $finished && $this->isTombstoneOf($this->cache->load($epochKey, self::PERMANENT), $token)) { + if ($nextFinished === $finished && $this->isTombstoneOf($this->invalidationCache->load($epochKey, self::PERMANENT), $token)) { throw new Unavailable("Failed to finish query cache invalidation for '{$key}'"); } @@ -302,7 +307,7 @@ public function flush(): void */ private function getEpoch(string $key, string $collection, int $writerTimeout): ?string { - $value = $this->cache->load($this->getEpochKey($key), self::PERMANENT); + $value = $this->invalidationCache->load($this->getEpochKey($key), self::PERMANENT); if ($value === false || $value === null) { return $this->cache->getGeneration($key) === self::NEVER_STARTED @@ -329,8 +334,8 @@ private function getEpoch(string $key, string $collection, int $writerTimeout): return null; } - $started = $this->cache->getGeneration($this->getStartedKey($key)); - if ($started !== $stamp && $started !== $this->cache->getGeneration($this->getFinishedKey($key))) { + $started = $this->invalidationCache->getGeneration($this->getStartedKey($key)); + if ($started !== $stamp && $started !== $this->invalidationCache->getGeneration($this->getFinishedKey($key))) { return null; } @@ -351,8 +356,8 @@ private function getLapsedEpoch(string $key, string $collection, string $tombsto return null; } - $started = $this->cache->getGeneration($this->getStartedKey($key)); - $finished = $this->cache->getGeneration($this->getFinishedKey($key)); + $started = $this->invalidationCache->getGeneration($this->getStartedKey($key)); + $finished = $this->invalidationCache->getGeneration($this->getFinishedKey($key)); if ($stamp + ($started === $finished ? $ttl : $writerTimeout) > $now) { return null; } @@ -368,7 +373,7 @@ private function releaseAbandonedOwners(string $owners, int $writerTimeout): boo { $now = \time(); $abandoned = []; - foreach ($this->cache->list($owners) as $token) { + foreach ($this->invalidationCache->list($owners) as $token) { $created = $this->getTokenTime($token); if ($created === null || $created + $writerTimeout > $now) { return false; @@ -378,7 +383,7 @@ private function releaseAbandonedOwners(string $owners, int $writerTimeout): boo } foreach ($abandoned as $token) { - $this->cache->purge($owners, $token); + $this->invalidationCache->purge($owners, $token); } return true; @@ -411,7 +416,7 @@ private function publish(string $key, string $finished): void { $epoch = self::ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::SEPARATOR.$finished; - if ($this->cache->save($this->getEpochKey($key), $epoch) === false) { + if ($this->invalidationCache->save($this->getEpochKey($key), $epoch) === false) { throw new Unavailable("Failed to activate query cache for '{$key}'"); } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 12d85006b..6e85f4bb0 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -206,6 +206,8 @@ class Database protected Cache $cache; + private ?Cache $invalidationCache = null; + protected string $cacheName = Scope::NAME; /** @@ -954,6 +956,25 @@ public function setCache(Cache $cache): static return $this; } + /** + * Store epochs, writer registrations and counters outside an evicting data cache. + * Configure every reader and writer with the same non-evicting store before use. + * Do not wrap this store in a circuit breaker that substitutes missing values. + * + * @return $this + */ + public function setInvalidationCache(?Cache $cache): static + { + $this->invalidationCache = $cache; + + return $this; + } + + private function getInvalidationCache(): Cache + { + return $this->invalidationCache ?? $this->cache; + } + public function getCache(): Cache { return $this->cache; diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 478602787..79a03aaa7 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -3687,7 +3687,7 @@ private function loadDocumentCacheState(string $collectionKey): Epoch $now = \time(); try { - $record = $this->cache->load($collectionKey.'#epoch', self::DOCUMENT_CACHE_PERMANENT); + $record = $this->getInvalidationCache()->load($collectionKey.'#epoch', self::DOCUMENT_CACHE_PERMANENT); if (! \is_string($record) || $record === '') { return $this->restoreDocumentCacheEpoch($collectionKey, $now); } @@ -3703,15 +3703,15 @@ private function loadDocumentCacheState(string $collectionKey): Epoch } $tombstone = \substr($record, \strlen(self::DOCUMENT_CACHE_BLOCKED_PREFIX)); - $finished = $this->cache->getGeneration($collectionKey.'#finished'); + $finished = $this->getInvalidationCache()->getGeneration($collectionKey.'#finished'); return new Epoch(self::DOCUMENT_CACHE_LAPSED_PREFIX.$tombstone.self::DOCUMENT_CACHE_SEPARATOR.$finished); } - $started = $this->cache->getGeneration($collectionKey.'#started'); + $started = $this->getInvalidationCache()->getGeneration($collectionKey.'#started'); if ( ($separator !== false && $started === $stamp) - || $started === $this->cache->getGeneration($collectionKey.'#finished') + || $started === $this->getInvalidationCache()->getGeneration($collectionKey.'#finished') ) { return new Epoch($marker); } @@ -3731,15 +3731,15 @@ private function loadDocumentCacheState(string $collectionKey): Epoch */ private function restoreDocumentCacheEpoch(string $collectionKey, int $now): Epoch { - $started = $this->cache->getGeneration($collectionKey.'#started'); - if ($started !== $this->cache->getGeneration($collectionKey.'#finished')) { - $this->cache->save($collectionKey.'#epoch', self::DOCUMENT_CACHE_BLOCKED_PREFIX.$this->createDocumentCacheToken().self::DOCUMENT_CACHE_SEPARATOR.$now); + $started = $this->getInvalidationCache()->getGeneration($collectionKey.'#started'); + if ($started !== $this->getInvalidationCache()->getGeneration($collectionKey.'#finished')) { + $this->getInvalidationCache()->save($collectionKey.'#epoch', self::DOCUMENT_CACHE_BLOCKED_PREFIX.$this->createDocumentCacheToken().self::DOCUMENT_CACHE_SEPARATOR.$now); return new Epoch(blockedAt: $now); } $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)); - if ($this->cache->save($collectionKey.'#epoch', $epoch.self::DOCUMENT_CACHE_SEPARATOR.$started) === false) { + if ($this->getInvalidationCache()->save($collectionKey.'#epoch', $epoch.self::DOCUMENT_CACHE_SEPARATOR.$started) === false) { return new Epoch(blockedAt: $now); } @@ -3790,21 +3790,21 @@ private function blockDocumentCacheEpoch(string $collectionKey, string $token, s { $epochKey = $collectionKey.'#epoch'; if (! $this->registerDocumentCacheOwner($collectionKey, $token)) { - $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); + $epoch = $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); if ($epoch === false || $epoch === null) { - $this->withRetries(fn () => $this->cache->purge($epochKey)); + $this->withRetries(fn () => $this->getInvalidationCache()->purge($epochKey)); return false; } } $this->withRetries(function () use ($epochKey, $token): void { - if ($this->cache->save($epochKey, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR.\time()) === false) { + if ($this->getInvalidationCache()->save($epochKey, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR.\time()) === false) { throw new CacheUnavailable("Failed to block document cache epoch '{$epochKey}'"); } }); - $this->withRetries(fn () => $this->cache->purge($collectionKey.'#started')); + $this->withRetries(fn () => $this->getInvalidationCache()->purge($collectionKey.'#started')); $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); return true; @@ -3812,7 +3812,7 @@ private function blockDocumentCacheEpoch(string $collectionKey, string $token, s private function registerDocumentCacheOwner(string $collectionKey, string $token): bool { - $owners = new Owners($this->cache); + $owners = new Owners($this->getInvalidationCache()); for ($attempt = 0; $attempt < self::DOCUMENT_CACHE_REGISTRATION_ATTEMPTS; $attempt++) { if ($owners->register($collectionKey, $token)) { return true; @@ -3859,8 +3859,8 @@ protected function activateDocumentInvalidation(array $tokens): void */ private function activateDocumentCacheEpoch(string $collectionKey, string $token, string $definitionKey): void { - $registration = (new Owners($this->cache))->find($collectionKey, $token); - $owner = $this->cache->load($registration->key, self::TTL, $registration->field); + $registration = (new Owners($this->getInvalidationCache()))->find($collectionKey, $token); + $owner = $this->getInvalidationCache()->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid document cache owner '{$token}' for '{$collectionKey}'"); } @@ -3868,10 +3868,10 @@ private function activateDocumentCacheEpoch(string $collectionKey, string $token $startedKey = $collectionKey.'#started'; $finishedKey = $collectionKey.'#finished'; - $started = $this->cache->getGeneration($startedKey); - $finished = $this->cache->getGeneration($finishedKey); + $started = $this->getInvalidationCache()->getGeneration($startedKey); + $finished = $this->getInvalidationCache()->getGeneration($finishedKey); $epochKey = $collectionKey.'#epoch'; - $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); + $epoch = $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); $blocked = \is_string($epoch) && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX); $ours = $blocked && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR); @@ -3895,7 +3895,7 @@ private function activateDocumentCacheEpoch(string $collectionKey, string $token $nextFinished = $this->finishDocumentCacheWrite($collectionKey, $finished, $epoch); - $nextStarted = $this->cache->getGeneration($startedKey); + $nextStarted = $this->getInvalidationCache()->getGeneration($startedKey); if ($nextStarted === $nextFinished) { $this->publishDocumentCacheEpoch($collectionKey, $nextFinished, $definitionKey); @@ -3920,7 +3920,7 @@ private function publishDocumentCacheEpoch(string $collectionKey, string $starte $epochKey = $collectionKey.'#epoch'; $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::DOCUMENT_CACHE_SEPARATOR.$started; $this->withRetries(function () use ($epochKey, $epoch): void { - if ($this->cache->save($epochKey, $epoch) === false) { + if ($this->getInvalidationCache()->save($epochKey, $epoch) === false) { throw new CacheUnavailable("Failed to activate document cache epoch '{$epochKey}'"); } }); @@ -3936,11 +3936,11 @@ private function releaseDocumentCacheOwner(Registration $registration, string $c { $released = true; $this->withRetries(function () use ($registration, $collectionKey, $token, &$released): void { - if ($this->cache->purge($registration->key, $registration->field)) { + if ($this->getInvalidationCache()->purge($registration->key, $registration->field)) { return; } - $owner = $this->cache->load($registration->key, self::TTL, $registration->field); + $owner = $this->getInvalidationCache()->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null) { throw new CacheUnavailable("Failed to release document cache owner '{$token}' for '{$collectionKey}'"); } @@ -3963,18 +3963,18 @@ private function finishDocumentCacheWrite(string $collectionKey, string $finishe $attempted = false; $this->withRetries(function () use ($finishedKey, $epochKey, $finished, $epoch, &$next, &$attempted): void { if ($attempted) { - $next = $this->cache->getGeneration($finishedKey); + $next = $this->getInvalidationCache()->getGeneration($finishedKey); } if ($next === $finished) { $attempted = true; - $this->cache->purge($finishedKey); + $this->getInvalidationCache()->purge($finishedKey); } - $next = $this->cache->getGeneration($finishedKey); + $next = $this->getInvalidationCache()->getGeneration($finishedKey); if ( $next === $finished && \is_string($epoch) - && $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT) === $epoch + && $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT) === $epoch ) { throw new CacheUnavailable("Failed to finish document cache invalidation '{$epochKey}'"); } @@ -3991,7 +3991,7 @@ private function releaseAbandonedDocumentCacheOwners(string $owners): bool { $now = \time(); $abandoned = []; - foreach ($this->cache->list($owners) as $token) { + foreach ($this->getInvalidationCache()->list($owners) as $token) { $separator = \strpos($token, self::DOCUMENT_CACHE_TOKEN_SEPARATOR); $created = $separator === false ? '' : \substr($token, 0, $separator); if (! \ctype_digit($created) || (int) $created + $this->cacheWriterTimeout > $now) { @@ -4002,7 +4002,7 @@ private function releaseAbandonedDocumentCacheOwners(string $owners): bool } foreach ($abandoned as $token) { - $this->cache->purge($owners, $token); + $this->getInvalidationCache()->purge($owners, $token); } return true; diff --git a/tests/unit/Cache/QueryTest.php b/tests/unit/Cache/QueryTest.php index ac7de9b0a..8efe193ec 100644 --- a/tests/unit/Cache/QueryTest.php +++ b/tests/unit/Cache/QueryTest.php @@ -34,6 +34,32 @@ protected function setUp(): void $this->queryCache = new ResultCache($this->cache); } + public function testDataCacheFlushDoesNotEraseAnInFlightWriter(): void + { + $data = new RedisLeasableCache(); + $query = new ResultCache(new Cache($data), invalidationCache: new Cache(new RedisLeasableCache())); + $scope = new Scope(namespace: 'durable_state'); + $entry = $query->getEntry($scope, 'users', []); + $this->assertNotNull($entry); + $this->assertTrue($query->set($entry, [new Document(['$id' => 'old'])], $query->getGeneration($entry))); + + $token = $query->createToken(); + $key = $query->getCollectionKey($scope, 'users'); + $query->blockCollection($key, $token); + $data->flush(); + $this->assertNull($query->getEntry($scope, 'users', []), 'Eviction of cached results must not erase the writer barrier'); + + $query->activateCollection($key, $token, $scope->writerTimeout); + $fresh = $query->getEntry($scope, 'users', []); + $this->assertNotNull($fresh); + $this->assertNull($query->get($fresh)); + $this->assertTrue($query->set($fresh, [new Document(['$id' => 'new'])], $query->getGeneration($fresh))); + $documents = $query->get($fresh); + $this->assertNotNull($documents); + $this->assertArrayHasKey(0, $documents); + $this->assertSame('new', $documents[0]->getId()); + } + public function testConstructorWithDefaults(): void { $queryCache = new ResultCache(self::createCache()); diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index eb8382051..307b301ec 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -26,6 +26,54 @@ final class DocumentCacheEpochTest extends TestCase { + public function testDataCacheEvictionCannotLoseInvalidationCounters(): void + { + $data = new RedisLeasableCache(); + $state = new Cache(new RedisLeasableCache()); + $adapter = new CountingMemory(); + $database = new Database($adapter, new Cache($data)); + $database->setInvalidationCache($state)->setDatabase('eviction')->setNamespace('counter_loss'); + $database->getAuthorization()->addRole(Role::any()->toString()); + $database->create(); + $database->createCollection(Collection::create(id: 'users', attributes: [Attribute::string(key: 'name')], permissions: [Permission::read(Role::any()), Permission::create(Role::any()), Permission::update(Role::any())])); + $database->createDocument('users', new Document(['$id' => 'user', 'name' => 'original'])); + + $key = $database->getCacheBaseKeys('users')[0]; + // Reproduce independently lost accounting keys, followed by a cold schema read. + // With one evicting store this strands the collection behind unequal counters. + $data->evict(\strtolower($key.'#finished')); + $data->evict(\strtolower($key.'#epoch')); + $database->getCache()->purge($database->getCacheBaseKeys(Database::METADATA, 'users')[1]); + + $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); + $adapter->reset(); + $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame(0, $adapter->documentReads, 'A lost data-cache key must not disable document caching'); + + $database->updateDocument('users', 'user', new Document(['name' => 'updated'])); + $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); + $adapter->reset(); + $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame(0, $adapter->documentReads, 'Subsequent writes must still activate the cache'); + } + + public function testDataCacheFlushDuringTransactionKeepsTheWriterBarrier(): void + { + $state = new Cache(new RedisLeasableCache()); + [$writer, $reader, $adapter, $path] = $this->createSharedSQLiteDatabases($state); + try { + $adapter->pauseNextCommit(function () use ($reader): void { + $reader->getCache()->flush(); + $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); + }); + $writer->updateDocument('users', 'user', new Document(['name' => 'updated'])); + $this->assertSame('updated', $reader->getDocument('users', 'user')->getAttribute('name')); + } finally { + $this->removeSQLiteFiles($path); + } + } + public function testOuterTransactionCannotPublishAStalePointCacheEntryBeforeCommit(): void { [$writer, $reader, $adapter, $path] = $this->createSharedSQLiteDatabases(); @@ -338,7 +386,7 @@ private function createDatabaseWithCache(CacheAdapter $cache, ?string $namespace /** * @return array{Database, Database, PausedDocumentSQLite, string} */ - private function createSharedSQLiteDatabases(): array + private function createSharedSQLiteDatabases(?Cache $invalidationCache = null): array { $path = \tempnam(\sys_get_temp_dir(), 'document-cache-epoch-'); if ($path === false) { @@ -365,6 +413,7 @@ private function createSharedSQLiteDatabases(): array $namespace = 'shared_epoch_'.\uniqid(); foreach ([$writer, $reader] as $database) { $database + ->setInvalidationCache($invalidationCache) ->setDatabase('utopiaTests') ->setNamespace($namespace); $database->getAuthorization()->addRole(Role::any()->toString()); From 5967f8ee227e7a1562553d0a60208c5cba0ab3e6 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:35:22 +0000 Subject: [PATCH 2/8] fix(cache): recover evicted coordination with atomic epoch records --- composer.json | 2 +- composer.lock | 27 ++- docs/cache-invalidation-state.md | 22 +-- src/Database/Cache/Mutation.php | 93 +++++++++++ src/Database/Cache/Query.php | 68 ++++---- src/Database/Database.php | 21 --- src/Database/Trait/Documents.php | 81 +++++---- tests/unit/Cache/QueryTest.php | 26 --- .../unit/Documents/DocumentCacheEpochTest.php | 51 +----- tests/unit/Documents/EvictionProtocolTest.php | 156 ++++++++++++++++++ tests/unit/Documents/EvictionSQLite.php | 23 +++ 11 files changed, 389 insertions(+), 181 deletions(-) create mode 100644 src/Database/Cache/Mutation.php create mode 100644 tests/unit/Documents/EvictionProtocolTest.php create mode 100644 tests/unit/Documents/EvictionSQLite.php diff --git a/composer.json b/composer.json index ae6e6b043..a73509c37 100755 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "ext-redis": "*", "utopia-php/validators": "^1.0 || ^2.0", "utopia-php/console": "^0.1 || ^0.2", - "utopia-php/cache": "^5.2", + "utopia-php/cache": "dev-codex/eviction-safe-cache-leases as 5.2.999", "utopia-php/pools": "^2.0", "utopia-php/mongo": "^1.5.4", "utopia-php/query": "^0.7.2", diff --git a/composer.lock b/composer.lock index ec33b8a12..5f4402d02 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f0a5261ac168d5060c5bcd2e5f5dacf9", + "content-hash": "1597d70d90cc725c5c09e643fe744dfc", "packages": [ { "name": "brick/math", @@ -2007,16 +2007,16 @@ }, { "name": "utopia-php/cache", - "version": "5.2.0", + "version": "dev-codex/eviction-safe-cache-leases", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "05a631796066a7cf96da289df358909fa2190ebd" + "reference": "6f28c370c04f70ee971c8b241ae226c555732519" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/05a631796066a7cf96da289df358909fa2190ebd", - "reference": "05a631796066a7cf96da289df358909fa2190ebd", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/6f28c370c04f70ee971c8b241ae226c555732519", + "reference": "6f28c370c04f70ee971c8b241ae226c555732519", "shasum": "" }, "require": { @@ -2061,9 +2061,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/5.2.0" + "source": "https://github.com/utopia-php/cache/tree/codex/eviction-safe-cache-leases" }, - "time": "2026-10-19T05:47:09+00:00" + "time": "2026-10-30T14:27:29+00:00" }, { "name": "utopia-php/circuit-breaker", @@ -5224,9 +5224,18 @@ "time": "2026-10-08T14:16:57+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/cache", + "version": "dev-codex/eviction-safe-cache-leases", + "alias": "5.2.999", + "alias_normalized": "5.2.999.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/cache": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { diff --git a/docs/cache-invalidation-state.md b/docs/cache-invalidation-state.md index 1ab80c555..72f2a1933 100644 --- a/docs/cache-invalidation-state.md +++ b/docs/cache-invalidation-state.md @@ -1,21 +1,13 @@ -# Invalidation state and evicting caches +# Eviction-safe invalidation -Document caching keeps a collection epoch, started/finished generations and writer registrations. These are coordination state: independently evicting one key can make the generations unequal even after every writer has finished. A replacement blocked epoch then disables document caching until the writer timeout; further eviction can restart that wait. No key TTL does not mean no eviction. +Redis and Redis multiplexing use one evictable record per collection for the epoch and live writer registrations. Atomic versioned snapshots replace the independent started/finished counter ledger. Eviction drops the whole record; its replacement gets a fresh random identity. A writer finishing after its registration was evicted still rotates the current epoch, retiring pre-commit fills. A newer writer's registration keeps its own barrier in place. -Keep this state in a shared, non-evicting store, separately from disposable document/schema/query values: +Fill leases also use random identities stored with the cached value. A key evicted and recreated cannot reset to generation zero and accidentally accept an old reader's fill. Snapshot and compare-and-swap operations propagate cache transport failures instead of converting them into missing state. -```php -$database = new Database($adapter, $dataCache); -$database->setInvalidationCache($stateCache); -$queryCache = new Utopia\Database\Cache\Query($dataCache, invalidationCache: $stateCache); -``` +The existing cache is sufficient: no separate state cache, non-evicting store or additional Dragonfly is needed. Adapters and older cache versions without atomic versioned operations retain the previous protocol for compatibility. Their counter state is not made eviction-safe by this change. -Configure the same state store on every reader and writer before using the handles. For Redis use `noeviction`; for Dragonfly leave `cache_mode` disabled. Size and monitor this store for the collection inventory; do not add expiry to its accounting keys. Do not wrap it in a circuit breaker that substitutes `0`, empty lists or cache misses for failed reads. A failed accounting operation must remain distinguishable from missing state. Document/schema values and their fill generations remain in the data cache. +## Upgrade -Without an explicit state cache the previous single-store behavior remains for compatibility. `setCache()` changes the data cache; it does not replace an explicitly configured state cache. Flushing data/query results does not clear state or erase in-flight writer barriers. +Deploy matching cache/database library versions together. Drain and stop legacy API/worker/task/migration writers, clear disposable cache data, deploy the new protocol everywhere and resume. Do not mix old and new protocols while writers are running. Rollback requires the same drain/clear procedure. -## Existing installations - -Changing stores is a coordinated cutover, not a rolling environment-variable update. Pause all database writers (API, workers, tasks and migrations), drain in-flight transactions, stop old processes, and configure every process with the same state store. Clear the disposable data cache before starting the new processes and resuming writes, so no cached schema carries an epoch from the previous store. Retain rollback access to the old store. To roll back, repeat the drain/stop/data-cache-clear process with the previous configuration. - -Never flush an invalidation store with active writers. Availability and failover of this store are part of the cache consistency protocol. +The writer timeout still bounds abandoned registrations. A writer completing after that timeout retires any intervening fills. If the cache rejects writes because it is unavailable or out of memory, coordination fails closed; it does not guess an empty owners list. Successful post-commit invalidation is still required. Durable replay of lost invalidations is a separate concern. diff --git a/src/Database/Cache/Mutation.php b/src/Database/Cache/Mutation.php new file mode 100644 index 000000000..63ed6cfaf --- /dev/null +++ b/src/Database/Cache/Mutation.php @@ -0,0 +1,93 @@ +supportsVersioning(); + } + + public static function lease(Cache $cache, string $key): string + { + return \interface_exists(\Utopia\Cache\Feature\Versioned::class) ? $cache->getLease($key) : $cache->getGeneration($key); + } + + public function read(string $key, int $timeout): Epoch + { + return $this->change($key, $timeout); + } + + public function begin(string $key, string $token, int $timeout): void + { + $this->change($key, $timeout, $token, true); + } + + public function finish(string $key, string $token, int $timeout): void + { + $this->change($key, $timeout, $token); + } + + private function change(string $key, int $timeout, ?string $token = null, bool $begin = false): Epoch + { + $key .= '#state'; + for ($attempt = 0; $attempt < 16; $attempt++) { + $snapshot = $this->cache->loadVersioned($key); + $state = $snapshot['value']; + $changed = $state === false; + if ($state === false) { + $state = ['epoch' => \bin2hex(\random_bytes(16)), 'owners' => []]; + } + if (! \is_array($state) || ! \is_string($state['epoch'] ?? null) || ! \is_array($state['owners'] ?? null)) { + throw new Unavailable("Invalid cache mutation record '{$key}'"); + } + $epoch = $state['epoch']; + $owners = []; + $now = \time(); + foreach ($state['owners'] as $owner => $created) { + if (! \is_string($owner) || ! \is_int($created)) { + throw new Unavailable("Invalid cache writer in '{$key}'"); + } + if ($begin || $created + $timeout > $now) { + $owners[$owner] = $created; + } else { + $changed = true; + } + } + if ($token !== null) { + if ($begin) { + $owners[$token] = $now; + } else { + unset($owners[$token]); + } + $changed = true; + } + if ($changed) { + $epoch = \bin2hex(\random_bytes(16)); + $state = ['epoch' => $epoch, 'owners' => $owners]; + if (! $this->cache->compareAndSwap($key, $state, $snapshot['version'])) { + continue; + } + } + + return $owners === [] + ? new Epoch('active:'.$epoch) + : new Epoch(blockedAt: \min($owners)); + } + + throw new Unavailable("Cache mutation record kept changing '{$key}'"); + } +} diff --git a/src/Database/Cache/Query.php b/src/Database/Cache/Query.php index ece39c866..fa36f8914 100644 --- a/src/Database/Cache/Query.php +++ b/src/Database/Cache/Query.php @@ -40,18 +40,13 @@ class Query /** @var array */ private array $regions = []; - private readonly Cache $invalidationCache; - /** * @param int $slots Results a collection scope keeps at most; queries sharing a slot evict each other */ public function __construct( private readonly Cache $cache, private readonly int $slots = self::SLOTS, - ?Cache $invalidationCache = null, ) { - $this->invalidationCache = $invalidationCache ?? $cache; - if ($slots < 1) { throw new InvalidArgumentException('A query cache needs at least one slot'); } @@ -167,7 +162,7 @@ public function get(Entry $entry): ?array public function getGeneration(Entry $entry): string { - return $this->cache->getGeneration($entry->key); + return Mutation::lease($this->cache, $entry->key); } /** @@ -214,15 +209,22 @@ public function createToken(): string */ public function blockCollection(string $key, string $token): void { - if (! (new Owners($this->invalidationCache))->register($key, $token)) { + if (Mutation::supported($this->cache)) { + (new Mutation($this->cache))->begin($key, $token, Scope::WRITER_TIMEOUT); + $this->cache->purge($key, self::BLOCK_FIELD); + + return; + } + + if (! (new Owners($this->cache))->register($key, $token)) { throw new Unavailable("Failed to register query cache owner for '{$key}'"); } - if ($this->invalidationCache->save($this->getEpochKey($key), self::BLOCKED_PREFIX.$token.self::SEPARATOR.\time()) === false) { + if ($this->cache->save($this->getEpochKey($key), self::BLOCKED_PREFIX.$token.self::SEPARATOR.\time()) === false) { throw new Unavailable("Failed to block query cache epoch for '{$key}'"); } - $this->invalidationCache->purge($this->getStartedKey($key)); + $this->cache->purge($this->getStartedKey($key)); $this->cache->purge($key, self::BLOCK_FIELD); } @@ -234,14 +236,20 @@ public function blockCollection(string $key, string $token): void */ public function activateCollection(string $key, string $token, int $writerTimeout): void { - $registration = (new Owners($this->invalidationCache))->find($key, $token); - $owner = $this->invalidationCache->load($registration->key, self::PERMANENT, $registration->field); + if (Mutation::supported($this->cache)) { + (new Mutation($this->cache))->finish($key, $token, $writerTimeout); + + return; + } + + $registration = (new Owners($this->cache))->find($key, $token); + $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid query cache owner for '{$key}'"); } $owned = $owner === $token; - if ($owned && ! $this->invalidationCache->purge($registration->key, $registration->field)) { - $owner = $this->invalidationCache->load($registration->key, self::PERMANENT, $registration->field); + if ($owned && ! $this->cache->purge($registration->key, $registration->field)) { + $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); if ($owner !== false && $owner !== null) { throw new Unavailable("Failed to release query cache owner for '{$key}'"); } @@ -251,9 +259,9 @@ public function activateCollection(string $key, string $token, int $writerTimeou $epochKey = $this->getEpochKey($key); $startedKey = $this->getStartedKey($key); $finishedKey = $this->getFinishedKey($key); - $started = $this->invalidationCache->getGeneration($startedKey); - $finished = $this->invalidationCache->getGeneration($finishedKey); - $current = $this->invalidationCache->load($epochKey, self::PERMANENT); + $started = $this->cache->getGeneration($startedKey); + $finished = $this->cache->getGeneration($finishedKey); + $current = $this->cache->load($epochKey, self::PERMANENT); $ours = $this->isTombstoneOf($current, $token); if ($started === $finished) { @@ -272,9 +280,9 @@ public function activateCollection(string $key, string $token, int $writerTimeou return; } - $this->invalidationCache->purge($finishedKey); - $nextStarted = $this->invalidationCache->getGeneration($startedKey); - $nextFinished = $this->invalidationCache->getGeneration($finishedKey); + $this->cache->purge($finishedKey); + $nextStarted = $this->cache->getGeneration($startedKey); + $nextFinished = $this->cache->getGeneration($finishedKey); if ($nextStarted === $nextFinished) { $this->publish($key, $nextFinished); @@ -282,7 +290,7 @@ public function activateCollection(string $key, string $token, int $writerTimeou return; } - if ($nextFinished === $finished && $this->isTombstoneOf($this->invalidationCache->load($epochKey, self::PERMANENT), $token)) { + if ($nextFinished === $finished && $this->isTombstoneOf($this->cache->load($epochKey, self::PERMANENT), $token)) { throw new Unavailable("Failed to finish query cache invalidation for '{$key}'"); } @@ -307,7 +315,11 @@ public function flush(): void */ private function getEpoch(string $key, string $collection, int $writerTimeout): ?string { - $value = $this->invalidationCache->load($this->getEpochKey($key), self::PERMANENT); + if (Mutation::supported($this->cache)) { + return (new Mutation($this->cache))->read($key, $writerTimeout)->value; + } + + $value = $this->cache->load($this->getEpochKey($key), self::PERMANENT); if ($value === false || $value === null) { return $this->cache->getGeneration($key) === self::NEVER_STARTED @@ -334,8 +346,8 @@ private function getEpoch(string $key, string $collection, int $writerTimeout): return null; } - $started = $this->invalidationCache->getGeneration($this->getStartedKey($key)); - if ($started !== $stamp && $started !== $this->invalidationCache->getGeneration($this->getFinishedKey($key))) { + $started = $this->cache->getGeneration($this->getStartedKey($key)); + if ($started !== $stamp && $started !== $this->cache->getGeneration($this->getFinishedKey($key))) { return null; } @@ -356,8 +368,8 @@ private function getLapsedEpoch(string $key, string $collection, string $tombsto return null; } - $started = $this->invalidationCache->getGeneration($this->getStartedKey($key)); - $finished = $this->invalidationCache->getGeneration($this->getFinishedKey($key)); + $started = $this->cache->getGeneration($this->getStartedKey($key)); + $finished = $this->cache->getGeneration($this->getFinishedKey($key)); if ($stamp + ($started === $finished ? $ttl : $writerTimeout) > $now) { return null; } @@ -373,7 +385,7 @@ private function releaseAbandonedOwners(string $owners, int $writerTimeout): boo { $now = \time(); $abandoned = []; - foreach ($this->invalidationCache->list($owners) as $token) { + foreach ($this->cache->list($owners) as $token) { $created = $this->getTokenTime($token); if ($created === null || $created + $writerTimeout > $now) { return false; @@ -383,7 +395,7 @@ private function releaseAbandonedOwners(string $owners, int $writerTimeout): boo } foreach ($abandoned as $token) { - $this->invalidationCache->purge($owners, $token); + $this->cache->purge($owners, $token); } return true; @@ -416,7 +428,7 @@ private function publish(string $key, string $finished): void { $epoch = self::ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::SEPARATOR.$finished; - if ($this->invalidationCache->save($this->getEpochKey($key), $epoch) === false) { + if ($this->cache->save($this->getEpochKey($key), $epoch) === false) { throw new Unavailable("Failed to activate query cache for '{$key}'"); } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 6e85f4bb0..12d85006b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -206,8 +206,6 @@ class Database protected Cache $cache; - private ?Cache $invalidationCache = null; - protected string $cacheName = Scope::NAME; /** @@ -956,25 +954,6 @@ public function setCache(Cache $cache): static return $this; } - /** - * Store epochs, writer registrations and counters outside an evicting data cache. - * Configure every reader and writer with the same non-evicting store before use. - * Do not wrap this store in a circuit breaker that substitutes missing values. - * - * @return $this - */ - public function setInvalidationCache(?Cache $cache): static - { - $this->invalidationCache = $cache; - - return $this; - } - - private function getInvalidationCache(): Cache - { - return $this->invalidationCache ?? $this->cache; - } - public function getCache(): Cache { return $this->cache; diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 79a03aaa7..5195f67a8 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -16,6 +16,7 @@ use Utopia\Database\Aggregation; use Utopia\Database\Attribute; use Utopia\Database\Cache\Epoch; +use Utopia\Database\Cache\Mutation; use Utopia\Database\Cache\Owners; use Utopia\Database\Cache\Registration; use Utopia\Database\Cache\Scope; @@ -754,7 +755,7 @@ private function readDocument(Collection $collection, string $id, array $queries $generation = '0'; if ($fillEpoch !== null || $fillDefinition) { try { - $generation = $this->cache->getGeneration($documentKey); + $generation = Mutation::lease($this->cache, $documentKey); } catch (Exception $e) { Console::warning('Warning: Failed to get cache generation: '.$e->getMessage()); } @@ -1064,7 +1065,7 @@ private function saveCachedDefinition(string $documentKey, string $field, array ], $field, $generation); if ($saved === false) { - $this->definitionFillsFail = $generation === '0' && $this->cache->getGeneration($documentKey) === '0'; + $this->definitionFillsFail = $generation === '0' && Mutation::lease($this->cache, $documentKey) === '0'; return; } @@ -3687,7 +3688,11 @@ private function loadDocumentCacheState(string $collectionKey): Epoch $now = \time(); try { - $record = $this->getInvalidationCache()->load($collectionKey.'#epoch', self::DOCUMENT_CACHE_PERMANENT); + if (Mutation::supported($this->cache)) { + return (new Mutation($this->cache))->read($collectionKey, $this->cacheWriterTimeout); + } + + $record = $this->cache->load($collectionKey.'#epoch', self::DOCUMENT_CACHE_PERMANENT); if (! \is_string($record) || $record === '') { return $this->restoreDocumentCacheEpoch($collectionKey, $now); } @@ -3703,15 +3708,15 @@ private function loadDocumentCacheState(string $collectionKey): Epoch } $tombstone = \substr($record, \strlen(self::DOCUMENT_CACHE_BLOCKED_PREFIX)); - $finished = $this->getInvalidationCache()->getGeneration($collectionKey.'#finished'); + $finished = $this->cache->getGeneration($collectionKey.'#finished'); return new Epoch(self::DOCUMENT_CACHE_LAPSED_PREFIX.$tombstone.self::DOCUMENT_CACHE_SEPARATOR.$finished); } - $started = $this->getInvalidationCache()->getGeneration($collectionKey.'#started'); + $started = $this->cache->getGeneration($collectionKey.'#started'); if ( ($separator !== false && $started === $stamp) - || $started === $this->getInvalidationCache()->getGeneration($collectionKey.'#finished') + || $started === $this->cache->getGeneration($collectionKey.'#finished') ) { return new Epoch($marker); } @@ -3731,15 +3736,15 @@ private function loadDocumentCacheState(string $collectionKey): Epoch */ private function restoreDocumentCacheEpoch(string $collectionKey, int $now): Epoch { - $started = $this->getInvalidationCache()->getGeneration($collectionKey.'#started'); - if ($started !== $this->getInvalidationCache()->getGeneration($collectionKey.'#finished')) { - $this->getInvalidationCache()->save($collectionKey.'#epoch', self::DOCUMENT_CACHE_BLOCKED_PREFIX.$this->createDocumentCacheToken().self::DOCUMENT_CACHE_SEPARATOR.$now); + $started = $this->cache->getGeneration($collectionKey.'#started'); + if ($started !== $this->cache->getGeneration($collectionKey.'#finished')) { + $this->cache->save($collectionKey.'#epoch', self::DOCUMENT_CACHE_BLOCKED_PREFIX.$this->createDocumentCacheToken().self::DOCUMENT_CACHE_SEPARATOR.$now); return new Epoch(blockedAt: $now); } $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)); - if ($this->getInvalidationCache()->save($collectionKey.'#epoch', $epoch.self::DOCUMENT_CACHE_SEPARATOR.$started) === false) { + if ($this->cache->save($collectionKey.'#epoch', $epoch.self::DOCUMENT_CACHE_SEPARATOR.$started) === false) { return new Epoch(blockedAt: $now); } @@ -3788,23 +3793,30 @@ private function advanceDocumentCacheEpoch(string $collectionKey, string $defini */ private function blockDocumentCacheEpoch(string $collectionKey, string $token, string $definitionKey): bool { + if (Mutation::supported($this->cache)) { + $this->withRetries(fn () => (new Mutation($this->cache))->begin($collectionKey, $token, $this->cacheWriterTimeout)); + $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); + + return true; + } + $epochKey = $collectionKey.'#epoch'; if (! $this->registerDocumentCacheOwner($collectionKey, $token)) { - $epoch = $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); + $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); if ($epoch === false || $epoch === null) { - $this->withRetries(fn () => $this->getInvalidationCache()->purge($epochKey)); + $this->withRetries(fn () => $this->cache->purge($epochKey)); return false; } } $this->withRetries(function () use ($epochKey, $token): void { - if ($this->getInvalidationCache()->save($epochKey, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR.\time()) === false) { + if ($this->cache->save($epochKey, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR.\time()) === false) { throw new CacheUnavailable("Failed to block document cache epoch '{$epochKey}'"); } }); - $this->withRetries(fn () => $this->getInvalidationCache()->purge($collectionKey.'#started')); + $this->withRetries(fn () => $this->cache->purge($collectionKey.'#started')); $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); return true; @@ -3812,7 +3824,7 @@ private function blockDocumentCacheEpoch(string $collectionKey, string $token, s private function registerDocumentCacheOwner(string $collectionKey, string $token): bool { - $owners = new Owners($this->getInvalidationCache()); + $owners = new Owners($this->cache); for ($attempt = 0; $attempt < self::DOCUMENT_CACHE_REGISTRATION_ATTEMPTS; $attempt++) { if ($owners->register($collectionKey, $token)) { return true; @@ -3859,8 +3871,15 @@ protected function activateDocumentInvalidation(array $tokens): void */ private function activateDocumentCacheEpoch(string $collectionKey, string $token, string $definitionKey): void { - $registration = (new Owners($this->getInvalidationCache()))->find($collectionKey, $token); - $owner = $this->getInvalidationCache()->load($registration->key, self::TTL, $registration->field); + if (Mutation::supported($this->cache)) { + $this->withRetries(fn () => (new Mutation($this->cache))->finish($collectionKey, $token, $this->cacheWriterTimeout)); + $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); + + return; + } + + $registration = (new Owners($this->cache))->find($collectionKey, $token); + $owner = $this->cache->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null && $owner !== $token) { throw new RuntimeException("Invalid document cache owner '{$token}' for '{$collectionKey}'"); } @@ -3868,10 +3887,10 @@ private function activateDocumentCacheEpoch(string $collectionKey, string $token $startedKey = $collectionKey.'#started'; $finishedKey = $collectionKey.'#finished'; - $started = $this->getInvalidationCache()->getGeneration($startedKey); - $finished = $this->getInvalidationCache()->getGeneration($finishedKey); + $started = $this->cache->getGeneration($startedKey); + $finished = $this->cache->getGeneration($finishedKey); $epochKey = $collectionKey.'#epoch'; - $epoch = $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); + $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); $blocked = \is_string($epoch) && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX); $ours = $blocked && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR); @@ -3895,7 +3914,7 @@ private function activateDocumentCacheEpoch(string $collectionKey, string $token $nextFinished = $this->finishDocumentCacheWrite($collectionKey, $finished, $epoch); - $nextStarted = $this->getInvalidationCache()->getGeneration($startedKey); + $nextStarted = $this->cache->getGeneration($startedKey); if ($nextStarted === $nextFinished) { $this->publishDocumentCacheEpoch($collectionKey, $nextFinished, $definitionKey); @@ -3920,7 +3939,7 @@ private function publishDocumentCacheEpoch(string $collectionKey, string $starte $epochKey = $collectionKey.'#epoch'; $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::DOCUMENT_CACHE_SEPARATOR.$started; $this->withRetries(function () use ($epochKey, $epoch): void { - if ($this->getInvalidationCache()->save($epochKey, $epoch) === false) { + if ($this->cache->save($epochKey, $epoch) === false) { throw new CacheUnavailable("Failed to activate document cache epoch '{$epochKey}'"); } }); @@ -3936,11 +3955,11 @@ private function releaseDocumentCacheOwner(Registration $registration, string $c { $released = true; $this->withRetries(function () use ($registration, $collectionKey, $token, &$released): void { - if ($this->getInvalidationCache()->purge($registration->key, $registration->field)) { + if ($this->cache->purge($registration->key, $registration->field)) { return; } - $owner = $this->getInvalidationCache()->load($registration->key, self::TTL, $registration->field); + $owner = $this->cache->load($registration->key, self::TTL, $registration->field); if ($owner !== false && $owner !== null) { throw new CacheUnavailable("Failed to release document cache owner '{$token}' for '{$collectionKey}'"); } @@ -3963,18 +3982,18 @@ private function finishDocumentCacheWrite(string $collectionKey, string $finishe $attempted = false; $this->withRetries(function () use ($finishedKey, $epochKey, $finished, $epoch, &$next, &$attempted): void { if ($attempted) { - $next = $this->getInvalidationCache()->getGeneration($finishedKey); + $next = $this->cache->getGeneration($finishedKey); } if ($next === $finished) { $attempted = true; - $this->getInvalidationCache()->purge($finishedKey); + $this->cache->purge($finishedKey); } - $next = $this->getInvalidationCache()->getGeneration($finishedKey); + $next = $this->cache->getGeneration($finishedKey); if ( $next === $finished && \is_string($epoch) - && $this->getInvalidationCache()->load($epochKey, self::DOCUMENT_CACHE_PERMANENT) === $epoch + && $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT) === $epoch ) { throw new CacheUnavailable("Failed to finish document cache invalidation '{$epochKey}'"); } @@ -3991,7 +4010,7 @@ private function releaseAbandonedDocumentCacheOwners(string $owners): bool { $now = \time(); $abandoned = []; - foreach ($this->getInvalidationCache()->list($owners) as $token) { + foreach ($this->cache->list($owners) as $token) { $separator = \strpos($token, self::DOCUMENT_CACHE_TOKEN_SEPARATOR); $created = $separator === false ? '' : \substr($token, 0, $separator); if (! \ctype_digit($created) || (int) $created + $this->cacheWriterTimeout > $now) { @@ -4002,7 +4021,7 @@ private function releaseAbandonedDocumentCacheOwners(string $owners): bool } foreach ($abandoned as $token) { - $this->getInvalidationCache()->purge($owners, $token); + $this->cache->purge($owners, $token); } return true; @@ -4271,7 +4290,7 @@ public function withCache( $generation = '0'; try { - $generation = $this->cache->getGeneration($physicalKey); + $generation = Mutation::lease($this->cache, $physicalKey); } catch (Throwable $error) { Console::warning('Warning: Failed to get cache generation: '.$error->getMessage()); } diff --git a/tests/unit/Cache/QueryTest.php b/tests/unit/Cache/QueryTest.php index 8efe193ec..ac7de9b0a 100644 --- a/tests/unit/Cache/QueryTest.php +++ b/tests/unit/Cache/QueryTest.php @@ -34,32 +34,6 @@ protected function setUp(): void $this->queryCache = new ResultCache($this->cache); } - public function testDataCacheFlushDoesNotEraseAnInFlightWriter(): void - { - $data = new RedisLeasableCache(); - $query = new ResultCache(new Cache($data), invalidationCache: new Cache(new RedisLeasableCache())); - $scope = new Scope(namespace: 'durable_state'); - $entry = $query->getEntry($scope, 'users', []); - $this->assertNotNull($entry); - $this->assertTrue($query->set($entry, [new Document(['$id' => 'old'])], $query->getGeneration($entry))); - - $token = $query->createToken(); - $key = $query->getCollectionKey($scope, 'users'); - $query->blockCollection($key, $token); - $data->flush(); - $this->assertNull($query->getEntry($scope, 'users', []), 'Eviction of cached results must not erase the writer barrier'); - - $query->activateCollection($key, $token, $scope->writerTimeout); - $fresh = $query->getEntry($scope, 'users', []); - $this->assertNotNull($fresh); - $this->assertNull($query->get($fresh)); - $this->assertTrue($query->set($fresh, [new Document(['$id' => 'new'])], $query->getGeneration($fresh))); - $documents = $query->get($fresh); - $this->assertNotNull($documents); - $this->assertArrayHasKey(0, $documents); - $this->assertSame('new', $documents[0]->getId()); - } - public function testConstructorWithDefaults(): void { $queryCache = new ResultCache(self::createCache()); diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index 307b301ec..eb8382051 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -26,54 +26,6 @@ final class DocumentCacheEpochTest extends TestCase { - public function testDataCacheEvictionCannotLoseInvalidationCounters(): void - { - $data = new RedisLeasableCache(); - $state = new Cache(new RedisLeasableCache()); - $adapter = new CountingMemory(); - $database = new Database($adapter, new Cache($data)); - $database->setInvalidationCache($state)->setDatabase('eviction')->setNamespace('counter_loss'); - $database->getAuthorization()->addRole(Role::any()->toString()); - $database->create(); - $database->createCollection(Collection::create(id: 'users', attributes: [Attribute::string(key: 'name')], permissions: [Permission::read(Role::any()), Permission::create(Role::any()), Permission::update(Role::any())])); - $database->createDocument('users', new Document(['$id' => 'user', 'name' => 'original'])); - - $key = $database->getCacheBaseKeys('users')[0]; - // Reproduce independently lost accounting keys, followed by a cold schema read. - // With one evicting store this strands the collection behind unequal counters. - $data->evict(\strtolower($key.'#finished')); - $data->evict(\strtolower($key.'#epoch')); - $database->getCache()->purge($database->getCacheBaseKeys(Database::METADATA, 'users')[1]); - - $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); - $adapter->reset(); - $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); - $this->assertSame(0, $adapter->documentReads, 'A lost data-cache key must not disable document caching'); - - $database->updateDocument('users', 'user', new Document(['name' => 'updated'])); - $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); - $adapter->reset(); - $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); - $this->assertSame(0, $adapter->documentReads, 'Subsequent writes must still activate the cache'); - } - - public function testDataCacheFlushDuringTransactionKeepsTheWriterBarrier(): void - { - $state = new Cache(new RedisLeasableCache()); - [$writer, $reader, $adapter, $path] = $this->createSharedSQLiteDatabases($state); - try { - $adapter->pauseNextCommit(function () use ($reader): void { - $reader->getCache()->flush(); - $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); - $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); - }); - $writer->updateDocument('users', 'user', new Document(['name' => 'updated'])); - $this->assertSame('updated', $reader->getDocument('users', 'user')->getAttribute('name')); - } finally { - $this->removeSQLiteFiles($path); - } - } - public function testOuterTransactionCannotPublishAStalePointCacheEntryBeforeCommit(): void { [$writer, $reader, $adapter, $path] = $this->createSharedSQLiteDatabases(); @@ -386,7 +338,7 @@ private function createDatabaseWithCache(CacheAdapter $cache, ?string $namespace /** * @return array{Database, Database, PausedDocumentSQLite, string} */ - private function createSharedSQLiteDatabases(?Cache $invalidationCache = null): array + private function createSharedSQLiteDatabases(): array { $path = \tempnam(\sys_get_temp_dir(), 'document-cache-epoch-'); if ($path === false) { @@ -413,7 +365,6 @@ private function createSharedSQLiteDatabases(?Cache $invalidationCache = null): $namespace = 'shared_epoch_'.\uniqid(); foreach ([$writer, $reader] as $database) { $database - ->setInvalidationCache($invalidationCache) ->setDatabase('utopiaTests') ->setNamespace($namespace); $database->getAuthorization()->addRole(Role::any()->toString()); diff --git a/tests/unit/Documents/EvictionProtocolTest.php b/tests/unit/Documents/EvictionProtocolTest.php new file mode 100644 index 000000000..0da317173 --- /dev/null +++ b/tests/unit/Documents/EvictionProtocolTest.php @@ -0,0 +1,156 @@ +markTestSkipped('The older-cache compatibility tier exercises the legacy protocol'); + } + $this->redis = new Redis(); + $this->redis->connect(\getenv('DATABASE_TEST_CACHE_HOST') ?: 'redis', (int) (\getenv('DATABASE_TEST_CACHE_PORT') ?: 6379)); + $this->cache = new Cache(new RedisCache($this->redis)); + $this->namespace = 'eviction_'.\bin2hex(\random_bytes(8)); + } + + #[\Override] + protected function tearDown(): void + { + if (! isset($this->redis)) { + return; + } + $cursor = null; + do { + $keys = $this->redis->scan($cursor, '*'.$this->namespace.'*', 1000); + if ($keys !== false && $keys !== []) { + $this->redis->del($keys); + } + } while ($cursor !== 0); + $this->redis->close(); + } + + public function testLostLegacyCountersCannotStrandDocumentCaching(): void + { + $adapter = new CountingMemory(); + $database = $this->database($adapter); + $key = \strtolower($database->getCacheBaseKeys('users')[0]); + $this->cache->save($key.'#epoch', 'blocked:old@'.\time()); + for ($i = 0; $i < 6; $i++) { + $this->cache->purge($key.'#started'); + } + $this->redis->del($key.'#finished'); + $this->cache->purge($database->getCacheBaseKeys(Database::METADATA, 'users')[1]); + + $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); + $adapter->reset(); + $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame(0, $adapter->documentReads); + + $this->redis->del($key.'#state'); + $database->updateDocuments('users', new Document(['name' => 'updated'])); + $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); + $adapter->reset(); + $this->assertSame('updated', $database->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame(0, $adapter->documentReads, 'Caching resumes after a write whose previous state was evicted'); + } + + public function testFinishingAnEvictedWriterCannotUnblockANewerWriter(): void + { + $key = $this->namespace; + $mutation = new Mutation($this->cache); + $mutation->begin($key, 'first', 3600); + $this->redis->del($key.'#state'); + $cold = $mutation->read($key, 3600)->value; + $this->assertNotNull($cold); + $mutation->begin($key, 'second', 3600); + $mutation->finish($key, 'first', 3600); + $this->assertNull($mutation->read($key, 3600)->value, 'A newer live writer still owns its barrier'); + $mutation->finish($key, 'second', 3600); + $active = $mutation->read($key, 3600)->value; + $this->assertNotNull($active); + $this->assertNotSame($cold, $active, 'Finishing lost writers retires cold fills'); + } + + public function testQueryResultsRefilledAfterEvictionAreRetiredWhenTheWriterFinishes(): void + { + $query = new QueryCache($this->cache); + $scope = new Scope(namespace: $this->namespace); + $key = $query->getCollectionKey($scope, 'users'); + $token = $query->createToken(); + $query->blockCollection($key, $token); + $this->redis->del(\strtolower($key.'#state')); + $entry = $query->getEntry($scope, 'users', []); + $this->assertNotNull($entry); + $this->assertTrue($query->set($entry, [new Document(['$id' => 'pre-commit'])], $query->getGeneration($entry))); + $query->activateCollection($key, $token, 3600); + $fresh = $query->getEntry($scope, 'users', []); + $this->assertNotNull($fresh); + $this->assertNull($query->get($fresh), 'A completed writer retires results filled after its registration disappeared'); + } + + public function testPreCommitDocumentFillAfterEvictionCannotSurviveCommit(): void + { + $path = \tempnam(\sys_get_temp_dir(), 'eviction-sqlite-'); + if ($path === false) { + $this->fail('Unable to create SQLite fixture'); + } + try { + $writerConnection = new \PDO('sqlite:'.$path); + $readerConnection = new \PDO('sqlite:'.$path); + $writerConnection->exec('PRAGMA journal_mode=WAL'); + $writerAdapter = new EvictionSQLite($writerConnection); + $writer = $this->database($writerAdapter); + $reader = new Database(new SQLite($readerConnection), $this->cache); + $reader->setDatabase('eviction-tests')->setNamespace($this->namespace); + $key = \strtolower($writer->getCacheBaseKeys('users')[0]); + $writerAdapter->beforeCommit = function () use ($key, $reader): void { + $this->redis->del($key.'#state'); + $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); + }; + $writer->updateDocuments('users', new Document(['name' => 'updated'])); + $this->assertSame('updated', $reader->getDocument('users', 'user')->getAttribute('name')); + $this->assertSame('updated', $reader->getDocument('users', 'user')->getAttribute('name')); + } finally { + foreach ([$path, $path.'-wal', $path.'-shm'] as $file) { + if (\is_file($file)) { + \unlink($file); + } + } + } + } + + private function database(CountingMemory|SQLite $adapter): Database + { + $database = new Database($adapter, $this->cache); + $database->setDatabase('eviction-tests')->setNamespace($this->namespace); + $database->create(); + $database->createCollection(Collection::create(id: 'users', attributes: [Attribute::string(key: 'name')], permissions: [Permission::read(Role::any()), Permission::create(Role::any()), Permission::update(Role::any())])); + $database->createDocument('users', new Document(['$id' => 'user', 'name' => 'original'])); + + return $database; + } +} diff --git a/tests/unit/Documents/EvictionSQLite.php b/tests/unit/Documents/EvictionSQLite.php new file mode 100644 index 000000000..81260e5a6 --- /dev/null +++ b/tests/unit/Documents/EvictionSQLite.php @@ -0,0 +1,23 @@ +inTransaction === 1) { + $callback = $this->beforeCommit; + $this->beforeCommit = null; + $callback?->__invoke(); + } + + return parent::commitTransaction(); + } +} From c4b72bfd99f8b9ded0ab9402bbfcbf04185fce95 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:41:10 +0000 Subject: [PATCH 3/8] test(cache): explicitly install the legacy compatibility dependency --- .github/workflows/tests.yml | 2 +- Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ffc4e094e..24d69338b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -140,7 +140,7 @@ jobs: context: . file: Dockerfile build-args: | - UTOPIA_CACHE_VERSION=^5.1 + UTOPIA_CACHE_VERSION=5.2.0 push: false tags: ${{ env.IMAGE }} load: true diff --git a/Dockerfile b/Dockerfile index 2272ff54f..6ae0e7f54 100755 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,7 @@ RUN composer install \ ARG UTOPIA_CACHE_VERSION="" RUN if [ -n "$UTOPIA_CACHE_VERSION" ]; then \ - composer update utopia-php/cache \ - --with "utopia-php/cache:$UTOPIA_CACHE_VERSION" \ + composer require "utopia-php/cache:$UTOPIA_CACHE_VERSION" \ --with-dependencies \ --ignore-platform-reqs \ --optimize-autoloader \ From 8447521bb778955ff50691ec68315dc0067c2689 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:03:20 +0000 Subject: [PATCH 4/8] fix(cache): skip fills when acquiring a lease fails --- src/Database/Trait/Documents.php | 6 +++++- tests/unit/Documents/GetDocumentFallbackTest.php | 14 +++++++++----- tests/unit/Documents/WithCacheTest.php | 6 ++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 5195f67a8..1c0903760 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -757,6 +757,9 @@ private function readDocument(Collection $collection, string $id, array $queries try { $generation = Mutation::lease($this->cache, $documentKey); } catch (Exception $e) { + $fillEpoch = null; + $fillDefinition = false; + $cacheFailed = true; Console::warning('Warning: Failed to get cache generation: '.$e->getMessage()); } } @@ -4288,11 +4291,12 @@ public function withCache( } } - $generation = '0'; try { $generation = Mutation::lease($this->cache, $physicalKey); } catch (Throwable $error) { Console::warning('Warning: Failed to get cache generation: '.$error->getMessage()); + + return $callback(); } $callbackValue = $callback(); diff --git a/tests/unit/Documents/GetDocumentFallbackTest.php b/tests/unit/Documents/GetDocumentFallbackTest.php index c2f7679ce..17dcd9b8c 100644 --- a/tests/unit/Documents/GetDocumentFallbackTest.php +++ b/tests/unit/Documents/GetDocumentFallbackTest.php @@ -49,7 +49,7 @@ public function testAJoinOnAnAdapterWithoutJoinsIsRefused(): void public function testACacheThatCannotBeReadOrWrittenFallsBackToTheDatabase(): void { - [$cache, $fail] = $this->failingCache(); + [$cache, $fail, $fills] = $this->failingCache(); $database = $this->database(new Memory(), new Cache($cache)); $database->createDocument(self::COLLECTION, new Document([Document::ID => 'session', 'owner' => 'ada'])); $fail(); @@ -67,8 +67,7 @@ public function testACacheThatCannotBeReadOrWrittenFallsBackToTheDatabase(): voi $this->assertTrue($missing->isEmpty()); $this->assertStringContainsString('Warning: Failed to get document from cache: the cache refused load', $log); $this->assertStringContainsString('Warning: Failed to get cache generation: the cache refused getGeneration', $log); - $this->assertStringContainsString('Failed to save document to cache: the cache refused saveWithLease', $log); - $this->assertStringContainsString('Failed to save empty document to cache: the cache refused saveWithLease', $log); + $this->assertSame(0, $fills(), 'A read without a lease must not populate the cache'); } public function testACachedDocumentPastItsTimeToLiveReadsAsEmpty(): void @@ -95,13 +94,15 @@ public function capabilities(): array } /** - * @return array{MemoryCache, Closure(): void} + * @return array{MemoryCache, Closure(): void, Closure(): int} */ private function failingCache(): array { $cache = new class (self::COLLECTION) extends MemoryCache implements Leasable { public bool $failing = false; + public int $fills = 0; + public function __construct(private readonly string $collection) { } @@ -139,6 +140,9 @@ public function getGeneration(string $key): string #[\Override] public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array { + if ($this->failing) { + $this->fills++; + } $this->refuse('saveWithLease', $key); return parent::save($key, $data, $hash); @@ -154,7 +158,7 @@ private function refuse(string $operation, string $key): void return [$cache, static function () use ($cache): void { $cache->failing = true; - }]; + }, static fn (): int => $cache->fills]; } private function database(Memory $adapter, Cache $cache): Database diff --git a/tests/unit/Documents/WithCacheTest.php b/tests/unit/Documents/WithCacheTest.php index a9bc9de10..9938d4c76 100644 --- a/tests/unit/Documents/WithCacheTest.php +++ b/tests/unit/Documents/WithCacheTest.php @@ -136,6 +136,12 @@ public function testACacheFailureFallsBackToTheCallback(string $failure, string $this->assertSame('computed', $value); $this->assertSame(1, $this->calls); + if ($failure === 'getGeneration') { + StderrCapture::during(function () use ($database): void { + $this->cached($database, fn (): string => 'newer'); + }); + $this->assertSame(2, $this->calls, 'A callback result without a lease must not be cached'); + } if ($warning !== '') { $this->assertStringContainsString($warning, $log); } From 492b4a99d5d3cb7633667da9769e46c2055ff098 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:13:33 +0000 Subject: [PATCH 5/8] feat(cache)!: use CAS versions as epochs and remove legacy ledger --- .github/workflows/tests.yml | 74 ---- Dockerfile | 13 - composer.json | 2 +- composer.lock | 14 +- docs/cache-invalidation-state.md | 14 +- src/Database/Cache/Fields.php | 34 -- src/Database/Cache/Mutation.php | 25 +- src/Database/Cache/Owners.php | 67 ---- src/Database/Cache/Query.php | 257 +------------ src/Database/Cache/Registration.php | 12 - src/Database/Trait/Documents.php | 358 ++---------------- tests/unit/Cache/CountingCache.php | 49 ++- tests/unit/Cache/DatabaseTest.php | 16 +- tests/unit/Cache/FailingMemory.php | 6 +- tests/unit/Cache/FailingRegistrationCache.php | 45 ++- tests/unit/Cache/InvalidationCache.php | 40 -- tests/unit/Cache/LeasableHashCache.php | 123 +----- tests/unit/Cache/OwnersTest.php | 103 ----- tests/unit/Cache/OwnershipCache.php | 133 +------ tests/unit/Cache/QueryTest.php | 191 +++++----- tests/unit/Cache/RacedReleaseCache.php | 88 +---- tests/unit/Cache/RedisLeasableCache.php | 90 +++-- tests/unit/Cache/UnavailableCache.php | 43 ++- tests/unit/DocumentFromStorageTest.php | 2 +- tests/unit/Documents/CacheUnavailableTest.php | 6 +- .../unit/Documents/DocumentCacheEpochTest.php | 174 ++++----- .../DocumentCacheInvalidationTest.php | 44 ++- tests/unit/Documents/EvictionProtocolTest.php | 13 +- .../Documents/GetDocumentFallbackTest.php | 27 +- .../InvalidationRestoreFailureTest.php | 2 +- .../MetadataTenantInvalidationTest.php | 48 +-- .../unit/Documents/NegativeCacheEpochTest.php | 49 +-- .../Documents/TransactionCacheReadTest.php | 46 ++- tests/unit/Documents/WithCacheTest.php | 33 +- tests/unit/WithCacheLeaseTest.php | 143 +------ 35 files changed, 513 insertions(+), 1871 deletions(-) delete mode 100644 src/Database/Cache/Fields.php delete mode 100644 src/Database/Cache/Owners.php delete mode 100644 src/Database/Cache/Registration.php delete mode 100644 tests/unit/Cache/OwnersTest.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24d69338b..283f831c0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,6 @@ concurrency: env: IMAGE: databases-dev CACHE_KEY: databases-dev-${{ github.event.pull_request.head.sha || github.sha }} - CACHE_5_KEY: databases-dev-cache-5-${{ github.event.pull_request.head.sha || github.sha }} on: [pull_request, workflow_dispatch] @@ -123,76 +122,3 @@ jobs: - name: Run Tests run: docker compose exec -T -e XDEBUG_MODE=off tests vendor/bin/paratest --configuration phpunit.xml --functional --processes 4 /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php - - setup_cache_5: - name: Setup & Build Docker Image (utopia-php/cache 5.x) - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - - name: Build Docker Image - uses: docker/build-push-action@1104d471370f9806843c095c1db02b5a90c5f8b6 # v3.3.1 - with: - context: . - file: Dockerfile - build-args: | - UTOPIA_CACHE_VERSION=5.2.0 - push: false - tags: ${{ env.IMAGE }} - load: true - cache-from: | - type=gha - type=gha,scope=cache-5 - cache-to: type=gha,mode=max,scope=cache-5 - outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar - - - name: Cache Docker Image - uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 - with: - key: ${{ env.CACHE_5_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - - cache_5_test: - name: "Cache 5.x Tests (${{ matrix.suite }})" - runs-on: ubuntu-latest - needs: setup_cache_5 - strategy: - fail-fast: false - matrix: - include: - - suite: Unit - profiles: "" - path: /usr/src/code/tests/unit - - suite: MariaDB - profiles: "--profile mariadb" - path: /usr/src/code/tests/e2e/Adapter/MariaDBTest.php - - steps: - - name: checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Load Cache - uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 - with: - key: ${{ env.CACHE_5_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Load and Start Services - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose ${{ matrix.profiles }} up -d --wait - - - name: Show utopia-php/cache Version - run: | - docker compose exec -T tests php -r ' - require "vendor/autoload.php"; - echo "utopia-php/cache ", \Composer\InstalledVersions::getPrettyVersion("utopia-php/cache"), PHP_EOL; - ' - - - name: Run Tests - run: docker compose exec -T -e XDEBUG_MODE=off tests vendor/bin/paratest --configuration phpunit.xml --functional --processes 4 ${{ matrix.path }} diff --git a/Dockerfile b/Dockerfile index 6ae0e7f54..15db406a1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -12,19 +12,6 @@ RUN composer install \ --no-scripts \ --prefer-dist -ARG UTOPIA_CACHE_VERSION="" - -RUN if [ -n "$UTOPIA_CACHE_VERSION" ]; then \ - composer require "utopia-php/cache:$UTOPIA_CACHE_VERSION" \ - --with-dependencies \ - --ignore-platform-reqs \ - --optimize-autoloader \ - --no-interaction \ - --no-plugins \ - --no-scripts \ - --prefer-dist; \ - fi - FROM php:8.5.8-cli-alpine AS compile ENV PHP_REDIS_VERSION="6.3.0" \ diff --git a/composer.json b/composer.json index a73509c37..b3e271ac4 100755 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "ext-redis": "*", "utopia-php/validators": "^1.0 || ^2.0", "utopia-php/console": "^0.1 || ^0.2", - "utopia-php/cache": "dev-codex/eviction-safe-cache-leases as 5.2.999", + "utopia-php/cache": "dev-codex/eviction-safe-cache-leases as 6.0.999", "utopia-php/pools": "^2.0", "utopia-php/mongo": "^1.5.4", "utopia-php/query": "^0.7.2", diff --git a/composer.lock b/composer.lock index 5f4402d02..26fce1b8f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1597d70d90cc725c5c09e643fe744dfc", + "content-hash": "3e2516143b11dd8164159b23250268b5", "packages": [ { "name": "brick/math", @@ -2011,12 +2011,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "6f28c370c04f70ee971c8b241ae226c555732519" + "reference": "2fef3be8b1aee62b831625a03b221c8e041c7328" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/6f28c370c04f70ee971c8b241ae226c555732519", - "reference": "6f28c370c04f70ee971c8b241ae226c555732519", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/2fef3be8b1aee62b831625a03b221c8e041c7328", + "reference": "2fef3be8b1aee62b831625a03b221c8e041c7328", "shasum": "" }, "require": { @@ -2063,7 +2063,7 @@ "issues": "https://github.com/utopia-php/cache/issues", "source": "https://github.com/utopia-php/cache/tree/codex/eviction-safe-cache-leases" }, - "time": "2026-10-30T14:27:29+00:00" + "time": "2026-10-30T21:12:57+00:00" }, { "name": "utopia-php/circuit-breaker", @@ -5228,8 +5228,8 @@ { "package": "utopia-php/cache", "version": "dev-codex/eviction-safe-cache-leases", - "alias": "5.2.999", - "alias_normalized": "5.2.999.0" + "alias": "6.0.999", + "alias_normalized": "6.0.999.0" } ], "minimum-stability": "stable", diff --git a/docs/cache-invalidation-state.md b/docs/cache-invalidation-state.md index 72f2a1933..e4039a252 100644 --- a/docs/cache-invalidation-state.md +++ b/docs/cache-invalidation-state.md @@ -1,13 +1,11 @@ -# Eviction-safe invalidation +# Eviction-safe invalidation (breaking change) -Redis and Redis multiplexing use one evictable record per collection for the epoch and live writer registrations. Atomic versioned snapshots replace the independent started/finished counter ledger. Eviction drops the whole record; its replacement gets a fresh random identity. A writer finishing after its registration was evicted still rotates the current epoch, retiring pre-commit fills. A newer writer's registration keeps its own barrier in place. +The collection's CAS version is its epoch. Its single evictable record contains only writer registrations. Atomic snapshots and CAS replace the started/finished counters, owner sidecars and separately maintained epoch. Eviction creates a fresh identity. Every completion changes the version, including a writer whose registration disappeared, retiring pre-commit fills while preserving newer live writers. -Fill leases also use random identities stored with the cached value. A key evicted and recreated cannot reset to generation zero and accidentally accept an old reader's fill. Snapshot and compare-and-swap operations propagate cache transport failures instead of converting them into missing state. +The same opaque versions fence document, definition and query fills. CAS returns the exact identity it committed, or false for a version conflict. Errors and open circuit breakers throw; they are never treated as eviction. Expiry checks use the caller's writer timeout; a reader with a shorter timeout does not delete another caller's live registration. Later writer completion still retires fills made after its timeout. -The existing cache is sufficient: no separate state cache, non-evicting store or additional Dragonfly is needed. Adapters and older cache versions without atomic versioned operations retain the previous protocol for compatibility. Their counter state is not made eviction-safe by this change. +This requires the new cache major release (6.x); the old generation API and compatibility protocol are removed. Redis, Redis multiplexing and Memory support versioning. Database document/query/withCache caching is bypassed on unsupported adapters; ordinary database operations continue. The old cache-5 compatibility CI tier is removed because it is no longer a supported dependency. The temporary development dependency must be replaced with the stable major release before production. -## Upgrade +No additional Dragonfly or coordination DSN is needed. Drain and stop legacy API/worker/task/migration writers, clear disposable cache data, deploy matching library versions everywhere and resume. Do not mix protocols. Rollback requires the same drain/clear procedure. -Deploy matching cache/database library versions together. Drain and stop legacy API/worker/task/migration writers, clear disposable cache data, deploy the new protocol everywhere and resume. Do not mix old and new protocols while writers are running. Rollback requires the same drain/clear procedure. - -The writer timeout still bounds abandoned registrations. A writer completing after that timeout retires any intervening fills. If the cache rejects writes because it is unavailable or out of memory, coordination fails closed; it does not guess an empty owners list. Successful post-commit invalidation is still required. Durable replay of lost invalidations is a separate concern. +A successful post-commit invalidation remains necessary. Durable replay of completions lost during a cache outage is separate work. A cache timeout/OOM must never be interpreted as an empty writer record. diff --git a/src/Database/Cache/Fields.php b/src/Database/Cache/Fields.php deleted file mode 100644 index 73381ad5e..000000000 --- a/src/Database/Cache/Fields.php +++ /dev/null @@ -1,34 +0,0 @@ -|null */ - private static ?WeakMap $caches = null; - - public static function kept(Cache $cache): bool - { - return isset(self::caches()[$cache]); - } - - public static function remember(Cache $cache): void - { - self::caches()[$cache] = true; - } - - /** - * @return WeakMap - */ - private static function caches(): WeakMap - { - return self::$caches ??= new WeakMap(); - } -} diff --git a/src/Database/Cache/Mutation.php b/src/Database/Cache/Mutation.php index 63ed6cfaf..883e2f156 100644 --- a/src/Database/Cache/Mutation.php +++ b/src/Database/Cache/Mutation.php @@ -6,7 +6,7 @@ use Utopia\Cache\Exception\Unavailable; /** - * One evictable record holds both the epoch and every live writer. A cold record + * One evictable record holds every writer; its CAS version is the epoch. A cold record * gets a fresh identity; even a writer whose registration was evicted rotates * that identity after commit, retiring anything filled while it was in flight. */ @@ -16,16 +16,6 @@ public function __construct(private Cache $cache) { } - public static function supported(Cache $cache): bool - { - return \interface_exists(\Utopia\Cache\Feature\Versioned::class) && $cache->supportsVersioning(); - } - - public static function lease(Cache $cache, string $key): string - { - return \interface_exists(\Utopia\Cache\Feature\Versioned::class) ? $cache->getLease($key) : $cache->getGeneration($key); - } - public function read(string $key, int $timeout): Epoch { return $this->change($key, $timeout); @@ -49,12 +39,12 @@ private function change(string $key, int $timeout, ?string $token = null, bool $ $state = $snapshot['value']; $changed = $state === false; if ($state === false) { - $state = ['epoch' => \bin2hex(\random_bytes(16)), 'owners' => []]; + $state = ['owners' => []]; } - if (! \is_array($state) || ! \is_string($state['epoch'] ?? null) || ! \is_array($state['owners'] ?? null)) { + if (! \is_array($state) || ! \is_array($state['owners'] ?? null)) { throw new Unavailable("Invalid cache mutation record '{$key}'"); } - $epoch = $state['epoch']; + $epoch = $snapshot['version']; $owners = []; $now = \time(); foreach ($state['owners'] as $owner => $created) { @@ -63,7 +53,7 @@ private function change(string $key, int $timeout, ?string $token = null, bool $ } if ($begin || $created + $timeout > $now) { $owners[$owner] = $created; - } else { + } elseif ($token !== null) { $changed = true; } } @@ -76,9 +66,8 @@ private function change(string $key, int $timeout, ?string $token = null, bool $ $changed = true; } if ($changed) { - $epoch = \bin2hex(\random_bytes(16)); - $state = ['epoch' => $epoch, 'owners' => $owners]; - if (! $this->cache->compareAndSwap($key, $state, $snapshot['version'])) { + $epoch = $this->cache->compareAndSwap($key, ['owners' => $owners], $snapshot['version']); + if ($epoch === false) { continue; } } diff --git a/src/Database/Cache/Owners.php b/src/Database/Cache/Owners.php deleted file mode 100644 index 88ab65a78..000000000 --- a/src/Database/Cache/Owners.php +++ /dev/null @@ -1,67 +0,0 @@ -getOwnersKey($key); - if ($this->cache->save($owners, $token, $token) === false) { - return false; - } - - if ($this->isField($owners, $token)) { - return true; - } - - return $this->cache->save($this->getOwnerKey($key, $token), $token) !== false; - } - - public function find(string $key, string $token): Registration - { - $owners = $this->getOwnersKey($key); - - return $this->isField($owners, $token) - ? new Registration($owners, $token) - : new Registration($this->getOwnerKey($key, $token)); - } - - private function isField(string $owners, string $token): bool - { - if (Fields::kept($this->cache)) { - return true; - } - - if (! \in_array($token, $this->cache->list($owners), true)) { - return false; - } - - Fields::remember($this->cache); - - return true; - } - - private function getOwnersKey(string $key): string - { - return $key.'#owners'; - } - - private function getOwnerKey(string $key, string $token): string - { - return $key.'#owner:'.$token; - } -} diff --git a/src/Database/Cache/Query.php b/src/Database/Cache/Query.php index fa36f8914..2546f5feb 100644 --- a/src/Database/Cache/Query.php +++ b/src/Database/Cache/Query.php @@ -15,22 +15,6 @@ */ class Query { - private const string ACTIVE_PREFIX = 'active:'; - - private const string INITIAL_EPOCH = self::ACTIVE_PREFIX.'initial'; - - private const string BLOCKED_PREFIX = 'blocked:'; - - private const string LAPSED_PREFIX = 'lapsed:'; - - private const string TOKEN_SEPARATOR = '.'; - - private const string SEPARATOR = '@'; - - private const string NEVER_STARTED = '0'; - - private const int PERMANENT = \PHP_INT_MAX; - private const int VERSION = 2; private const int SLOTS = 1024; @@ -87,12 +71,12 @@ public function getCollectionKey(Scope $scope, string $collection): string */ public function getEntry(Scope $scope, string $collection, array $queries, string $context = ''): ?Entry { - if (! $this->getRegion($collection)->enabled) { + if (! $this->cache->supportsVersioning() || ! $this->getRegion($collection)->enabled) { return null; } $key = $this->getCollectionKey($scope, $collection); - $epoch = $this->getEpoch($key, $collection, $scope->writerTimeout); + $epoch = (new Mutation($this->cache))->read($key, $scope->writerTimeout)->value; if ($epoch === null) { return null; } @@ -160,15 +144,15 @@ public function get(Entry $entry): ?array return $documents; } - public function getGeneration(Entry $entry): string + public function getLease(Entry $entry): string { - return Mutation::lease($this->cache, $entry->key); + return $this->cache->getLease($entry->key); } /** * @param array $results */ - public function set(Entry $entry, array $results, string $generation): bool + public function set(Entry $entry, array $results, string $version): bool { $data = []; foreach ($results as $result) { @@ -184,7 +168,7 @@ public function set(Entry $entry, array $results, string $generation): bool 'epoch' => $entry->epoch, 'field' => $entry->field, 'documents' => $data, - ], $entry->slot, $generation) !== false; + ], $entry->slot, $version) !== false; } public function invalidateCollection(Scope $scope, string $collection): void @@ -200,102 +184,21 @@ public function invalidateCollection(Scope $scope, string $collection): void */ public function createToken(): string { - return \time().self::TOKEN_SEPARATOR.\bin2hex(\random_bytes(16)); + return \bin2hex(\random_bytes(16)); } - /** - * Publish a shared tombstone before a mutation starts. The previous epoch's results stay until - * a fill of their slot replaces them; the purge of one absent field only advances the lease. - */ public function blockCollection(string $key, string $token): void { - if (Mutation::supported($this->cache)) { + if ($this->cache->supportsVersioning()) { (new Mutation($this->cache))->begin($key, $token, Scope::WRITER_TIMEOUT); $this->cache->purge($key, self::BLOCK_FIELD); - - return; - } - - if (! (new Owners($this->cache))->register($key, $token)) { - throw new Unavailable("Failed to register query cache owner for '{$key}'"); - } - - if ($this->cache->save($this->getEpochKey($key), self::BLOCKED_PREFIX.$token.self::SEPARATOR.\time()) === false) { - throw new Unavailable("Failed to block query cache epoch for '{$key}'"); } - - $this->cache->purge($this->getStartedKey($key)); - $this->cache->purge($key, self::BLOCK_FIELD); } - /** - * Replace this mutation's shared tombstone with a fresh usable epoch once no - * other mutation of the collection is in progress. - * - * @param int $writerTimeout Seconds after which another writer's registration is treated as abandoned - */ public function activateCollection(string $key, string $token, int $writerTimeout): void { - if (Mutation::supported($this->cache)) { + if ($this->cache->supportsVersioning()) { (new Mutation($this->cache))->finish($key, $token, $writerTimeout); - - return; - } - - $registration = (new Owners($this->cache))->find($key, $token); - $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); - if ($owner !== false && $owner !== null && $owner !== $token) { - throw new RuntimeException("Invalid query cache owner for '{$key}'"); - } - $owned = $owner === $token; - if ($owned && ! $this->cache->purge($registration->key, $registration->field)) { - $owner = $this->cache->load($registration->key, self::PERMANENT, $registration->field); - if ($owner !== false && $owner !== null) { - throw new Unavailable("Failed to release query cache owner for '{$key}'"); - } - $owned = false; - } - - $epochKey = $this->getEpochKey($key); - $startedKey = $this->getStartedKey($key); - $finishedKey = $this->getFinishedKey($key); - $started = $this->cache->getGeneration($startedKey); - $finished = $this->cache->getGeneration($finishedKey); - $current = $this->cache->load($epochKey, self::PERMANENT); - $ours = $this->isTombstoneOf($current, $token); - - if ($started === $finished) { - if (! $owned && ($ours || ! $this->isTombstone($current))) { - $this->publish($key, $finished); - } - - return; - } - - if (! $owned && ! $ours) { - if ($this->isActive($current)) { - $this->publish($key, $started); - } - - return; - } - - $this->cache->purge($finishedKey); - $nextStarted = $this->cache->getGeneration($startedKey); - $nextFinished = $this->cache->getGeneration($finishedKey); - - if ($nextStarted === $nextFinished) { - $this->publish($key, $nextFinished); - - return; - } - - if ($nextFinished === $finished && $this->isTombstoneOf($this->cache->load($epochKey, self::PERMANENT), $token)) { - throw new Unavailable("Failed to finish query cache invalidation for '{$key}'"); - } - - if ($registration->field !== '' && $this->releaseAbandonedOwners($registration->key, $writerTimeout)) { - $this->publish($key, $nextStarted); } } @@ -306,148 +209,6 @@ public function flush(): void } } - /** - * Epochs never expire in the cache, so one cannot vanish under a transaction that - * outlives the region TTL. An active epoch carries the started generation it was - * published at: while the started generation still equals it, no mutation has - * begun since, so a reader needs one generation read. The initial epoch needs a - * results hash no write has purged, since results filled under it outlive the block. - */ - private function getEpoch(string $key, string $collection, int $writerTimeout): ?string - { - if (Mutation::supported($this->cache)) { - return (new Mutation($this->cache))->read($key, $writerTimeout)->value; - } - - $value = $this->cache->load($this->getEpochKey($key), self::PERMANENT); - - if ($value === false || $value === null) { - return $this->cache->getGeneration($key) === self::NEVER_STARTED - ? self::INITIAL_EPOCH - : null; - } - - if (! \is_string($value) || $value === '') { - return null; - } - - $separator = \strrpos($value, self::SEPARATOR); - if ($separator === false) { - return null; - } - $marker = \substr($value, 0, $separator); - $stamp = \substr($value, $separator + 1); - - if (\str_starts_with($value, self::BLOCKED_PREFIX)) { - return $this->getLapsedEpoch($key, $collection, $value, (int) $stamp, $writerTimeout); - } - - if (! \str_starts_with($value, self::ACTIVE_PREFIX)) { - return null; - } - - $started = $this->cache->getGeneration($this->getStartedKey($key)); - if ($started !== $stamp && $started !== $this->cache->getGeneration($this->getFinishedKey($key))) { - return null; - } - - return $marker; - } - - /** - * A tombstone lapses after the region TTL once no write is counted in flight, and after the writer - * timeout while one is, since its writer may have died before activating. The lapsed epoch belongs - * to this tombstone and the finished generation, so nothing filled before the block, or before a - * later activation, is served under it. - */ - private function getLapsedEpoch(string $key, string $collection, string $tombstone, int $stamp, int $writerTimeout): ?string - { - $now = \time(); - $ttl = $this->getRegion($collection)->ttl; - if ($stamp + \min($ttl, $writerTimeout) > $now) { - return null; - } - - $started = $this->cache->getGeneration($this->getStartedKey($key)); - $finished = $this->cache->getGeneration($this->getFinishedKey($key)); - if ($stamp + ($started === $finished ? $ttl : $writerTimeout) > $now) { - return null; - } - - return self::LAPSED_PREFIX.\substr($tombstone, \strlen(self::BLOCKED_PREFIX)).self::SEPARATOR.$finished; - } - - /** - * Release every other writer still registered when all of them are older than the writer timeout. - * A token without a creation time counts as live. - */ - private function releaseAbandonedOwners(string $owners, int $writerTimeout): bool - { - $now = \time(); - $abandoned = []; - foreach ($this->cache->list($owners) as $token) { - $created = $this->getTokenTime($token); - if ($created === null || $created + $writerTimeout > $now) { - return false; - } - - $abandoned[] = $token; - } - - foreach ($abandoned as $token) { - $this->cache->purge($owners, $token); - } - - return true; - } - - private function getTokenTime(string $token): ?int - { - $separator = \strpos($token, self::TOKEN_SEPARATOR); - $time = $separator === false ? '' : \substr($token, 0, $separator); - - return \ctype_digit($time) ? (int) $time : null; - } - - private function isActive(mixed $value): bool - { - return \is_string($value) && \str_starts_with($value, self::ACTIVE_PREFIX); - } - - private function isTombstone(mixed $value): bool - { - return \is_string($value) && \str_starts_with($value, self::BLOCKED_PREFIX); - } - - private function isTombstoneOf(mixed $value, string $token): bool - { - return \is_string($value) && \str_starts_with($value, self::BLOCKED_PREFIX.$token.self::SEPARATOR); - } - - private function publish(string $key, string $finished): void - { - $epoch = self::ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::SEPARATOR.$finished; - - if ($this->cache->save($this->getEpochKey($key), $epoch) === false) { - throw new Unavailable("Failed to activate query cache for '{$key}'"); - } - } - - private function getEpochKey(string $key): string - { - return $key.'#epoch'; - } - - private function getFinishedKey(string $key): string - { - return $key.'#finished'; - } - - private function getStartedKey(string $key): string - { - return $key.'#started'; - } - private function purgeLoadedEntry(Entry $entry): void { if (! $this->cache->purge($entry->key, $entry->slot)) { diff --git a/src/Database/Cache/Registration.php b/src/Database/Cache/Registration.php deleted file mode 100644 index 6c07723a0..000000000 --- a/src/Database/Cache/Registration.php +++ /dev/null @@ -1,12 +0,0 @@ -adapter->inTransaction(); - $cacheable = ! $forUpdate + $cacheable = $this->cache->supportsVersioning() + && ! $forUpdate && empty($joins) && (! $inTransaction || $this->isCachedInTransaction($documentKey)) && ! isset($this->documentCacheMutations[$this->getEventContext()][$collectionKey]); @@ -752,15 +735,15 @@ private function readDocument(Collection $collection, string $id, array $queries return $document; } - $generation = '0'; + $version = ''; if ($fillEpoch !== null || $fillDefinition) { try { - $generation = Mutation::lease($this->cache, $documentKey); + $version = $this->cache->getLease($documentKey); } catch (Exception $e) { $fillEpoch = null; $fillDefinition = false; $cacheFailed = true; - Console::warning('Warning: Failed to get cache generation: '.$e->getMessage()); + Console::warning('Warning: Failed to get cache version: '.$e->getMessage()); } } @@ -810,9 +793,9 @@ private function readDocument(Collection $collection, string $id, array $queries try { if ($fillEpoch !== null && empty($relationships) && $missing) { - $this->saveCachedDocument($documentKey, $field, $fillEpoch, [self::CACHE_EMPTY_MARKER => true], $generation); + $this->saveCachedDocument($documentKey, $field, $fillEpoch, [self::CACHE_EMPTY_MARKER => true], $version); } elseif ($fillDefinition) { - $this->saveCachedDefinition($documentKey, $field, [self::CACHE_EMPTY_MARKER => true], [], $generation); + $this->saveCachedDefinition($documentKey, $field, [self::CACHE_EMPTY_MARKER => true], [], $version); } } catch (Exception $e) { Console::warning('Failed to save empty document to cache: '.$e->getMessage()); @@ -868,7 +851,7 @@ private function readDocument(Collection $collection, string $id, array $queries try { if ($fillEpoch !== null && empty($relationships)) { - $this->saveCachedDocument($documentKey, $field, $fillEpoch, $document->getArrayCopy(), $generation); + $this->saveCachedDocument($documentKey, $field, $fillEpoch, $document->getArrayCopy(), $version); } elseif ($fillDefinition) { $this->saveCachedDefinition( $documentKey, @@ -879,11 +862,13 @@ private function readDocument(Collection $collection, string $id, array $queries self::DOCUMENT_CACHE_BLOCKED_AT => $collectionState->blockedAt, self::DOCUMENT_CACHE_CHECKED_AT => \time(), ], - $generation, - fn (): bool => $this->loadDocumentCacheState($describedKey)->value === $collectionState->value, + $version, ); } } catch (Exception $e) { + if ($definition) { + $this->definitionFillsFail = true; + } Console::warning('Failed to save document to cache: '.$e->getMessage()); } @@ -954,13 +939,13 @@ private function loadCachedDocument(string $documentKey, string $field, string $ /** * @param array $document */ - private function saveCachedDocument(string $documentKey, string $field, string $epoch, array $document, string $generation): void + private function saveCachedDocument(string $documentKey, string $field, string $epoch, array $document, string $version): void { $this->cache->saveWithLease($documentKey, [ self::DOCUMENT_CACHE_EPOCH => $epoch, self::DOCUMENT_CACHE_FIELD => $field, self::DOCUMENT_CACHE_VALUE => $document, - ], $field, $generation); + ], $field, $version); } /** @@ -1052,32 +1037,20 @@ private function loadCachedDefinition(string $documentKey, string $field): ?arra } /** - * Without generations a fill can land after the purge that should have removed it, so the state it - * was filled under is read again and the fill is dropped when that state has moved on. + * An opaque version rejects definitions filled after a concurrent invalidation. * * @param array $document * @param array $validity - * @param (Closure(): bool)|null $isCurrent */ - private function saveCachedDefinition(string $documentKey, string $field, array $document, array $validity, string $generation, ?Closure $isCurrent = null): void + private function saveCachedDefinition(string $documentKey, string $field, array $document, array $validity, string $version): void { - $saved = $this->cache->saveWithLease($documentKey, [ + $this->cache->saveWithLease($documentKey, [ ...$validity, self::DOCUMENT_CACHE_FIELD => $field, self::DOCUMENT_CACHE_VALUE => $document, - ], $field, $generation); - - if ($saved === false) { - $this->definitionFillsFail = $generation === '0' && Mutation::lease($this->cache, $documentKey) === '0'; - - return; - } + ], $field, $version); $this->definitionFillsFail = false; - - if ($generation === '0' && $isCurrent !== null && ! $isCurrent()) { - $this->cache->purge($documentKey); - } } private function attachCollectionCacheEpoch(Document $definition, ?string $epoch): void @@ -3681,85 +3654,27 @@ protected function purgeWrittenDocuments(array $documents): void } } - /** - * The epoch a collection's documents may be cached under, or null while a write to the collection - * is in flight, with the time that write blocked it. A tombstone older than the writer timeout - * lapses into an epoch of its own, which a later activation changes. - */ + /** Current CAS identity, or a blocked state while writers are registered. */ private function loadDocumentCacheState(string $collectionKey): Epoch { - $now = \time(); - + if (! $this->cache->supportsVersioning()) { + return new Epoch(blockedAt: \time()); + } try { - if (Mutation::supported($this->cache)) { - return (new Mutation($this->cache))->read($collectionKey, $this->cacheWriterTimeout); - } - - $record = $this->cache->load($collectionKey.'#epoch', self::DOCUMENT_CACHE_PERMANENT); - if (! \is_string($record) || $record === '') { - return $this->restoreDocumentCacheEpoch($collectionKey, $now); - } - - $separator = \strrpos($record, self::DOCUMENT_CACHE_SEPARATOR); - $marker = $separator === false ? $record : \substr($record, 0, $separator); - $stamp = $separator === false ? '' : \substr($record, $separator + 1); - - if (\str_starts_with($record, self::DOCUMENT_CACHE_BLOCKED_PREFIX)) { - $blockedAt = \ctype_digit($stamp) ? (int) $stamp : 0; - if ($blockedAt + $this->cacheWriterTimeout > $now) { - return new Epoch(blockedAt: $blockedAt); - } - - $tombstone = \substr($record, \strlen(self::DOCUMENT_CACHE_BLOCKED_PREFIX)); - $finished = $this->cache->getGeneration($collectionKey.'#finished'); - - return new Epoch(self::DOCUMENT_CACHE_LAPSED_PREFIX.$tombstone.self::DOCUMENT_CACHE_SEPARATOR.$finished); - } - - $started = $this->cache->getGeneration($collectionKey.'#started'); - if ( - ($separator !== false && $started === $stamp) - || $started === $this->cache->getGeneration($collectionKey.'#finished') - ) { - return new Epoch($marker); - } - - return $this->restoreDocumentCacheEpoch($collectionKey, $now); + return (new Mutation($this->cache))->read($collectionKey, $this->cacheWriterTimeout); } catch (Throwable $error) { Console::warning('Warning: Failed to load document cache epoch: '.$error->getMessage()); - return new Epoch(blockedAt: $now); + return new Epoch(blockedAt: \time()); } } - /** - * Replace a missing or unusable epoch: with a fresh one when no write is counted in flight, or - * else with a tombstone of its own, so the collection lapses back into the cache after the writer - * timeout even when the write that blocked it left no tombstone behind. - */ - private function restoreDocumentCacheEpoch(string $collectionKey, int $now): Epoch - { - $started = $this->cache->getGeneration($collectionKey.'#started'); - if ($started !== $this->cache->getGeneration($collectionKey.'#finished')) { - $this->cache->save($collectionKey.'#epoch', self::DOCUMENT_CACHE_BLOCKED_PREFIX.$this->createDocumentCacheToken().self::DOCUMENT_CACHE_SEPARATOR.$now); - - return new Epoch(blockedAt: $now); - } - - $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)); - if ($this->cache->save($collectionKey.'#epoch', $epoch.self::DOCUMENT_CACHE_SEPARATOR.$started) === false) { - return new Epoch(blockedAt: $now); - } - - return new Epoch($epoch); - } - /** * A write's token, which records when it was created so a later activation can tell an abandoned write. */ private function createDocumentCacheToken(): string { - return \time().self::DOCUMENT_CACHE_TOKEN_SEPARATOR.\bin2hex(\random_bytes(16)); + return \bin2hex(\random_bytes(16)); } private function advanceDocumentCacheEpoch(string $collectionKey, string $definitionKey): bool @@ -3786,57 +3701,18 @@ private function advanceDocumentCacheEpoch(string $collectionKey, string $defini return true; } - /** - * Publish a tombstone before the write, then drop the definition that carried the previous - * epoch, so readers refill it with the tombstone. A write whose registration keeps failing - * still blocks with its tombstone: its activation finishes it while the tombstone is its own, - * and otherwise retires whatever readers filled while it ran. An epoch that cannot be read is - * purged all the same, so a cache that cannot confirm the purge fails the write. Each step is - * retried first, so a blip shorter than the retries does not cost the write. - */ + /** Register before writing, then retire the definition carrying the old epoch. */ private function blockDocumentCacheEpoch(string $collectionKey, string $token, string $definitionKey): bool { - if (Mutation::supported($this->cache)) { - $this->withRetries(fn () => (new Mutation($this->cache))->begin($collectionKey, $token, $this->cacheWriterTimeout)); - $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); - - return true; - } - - $epochKey = $collectionKey.'#epoch'; - if (! $this->registerDocumentCacheOwner($collectionKey, $token)) { - $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); - if ($epoch === false || $epoch === null) { - $this->withRetries(fn () => $this->cache->purge($epochKey)); - - return false; - } + if (! $this->cache->supportsVersioning()) { + return false; } - - $this->withRetries(function () use ($epochKey, $token): void { - if ($this->cache->save($epochKey, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR.\time()) === false) { - throw new CacheUnavailable("Failed to block document cache epoch '{$epochKey}'"); - } - }); - - $this->withRetries(fn () => $this->cache->purge($collectionKey.'#started')); + $this->withRetries(fn () => (new Mutation($this->cache))->begin($collectionKey, $token, $this->cacheWriterTimeout)); $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); return true; } - private function registerDocumentCacheOwner(string $collectionKey, string $token): bool - { - $owners = new Owners($this->cache); - for ($attempt = 0; $attempt < self::DOCUMENT_CACHE_REGISTRATION_ATTEMPTS; $attempt++) { - if ($owners->register($collectionKey, $token)) { - return true; - } - } - - return false; - } - private function purgeCachedDefinition(string $definitionKey): void { if ($definitionKey !== '') { @@ -3867,169 +3743,13 @@ protected function activateDocumentInvalidation(array $tokens): void } } - /** - * Replace this write's tombstone with a fresh epoch once no other write to the collection is in - * flight. Writes older than the writer timeout no longer count as in flight: their registrations - * are released and the epoch is published. - */ + /** Completion always changes the identity, even if eviction lost this registration. */ private function activateDocumentCacheEpoch(string $collectionKey, string $token, string $definitionKey): void { - if (Mutation::supported($this->cache)) { - $this->withRetries(fn () => (new Mutation($this->cache))->finish($collectionKey, $token, $this->cacheWriterTimeout)); - $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); - - return; - } - - $registration = (new Owners($this->cache))->find($collectionKey, $token); - $owner = $this->cache->load($registration->key, self::TTL, $registration->field); - if ($owner !== false && $owner !== null && $owner !== $token) { - throw new RuntimeException("Invalid document cache owner '{$token}' for '{$collectionKey}'"); - } - $owned = $owner === $token && $this->releaseDocumentCacheOwner($registration, $collectionKey, $token); - - $startedKey = $collectionKey.'#started'; - $finishedKey = $collectionKey.'#finished'; - $started = $this->cache->getGeneration($startedKey); - $finished = $this->cache->getGeneration($finishedKey); - $epochKey = $collectionKey.'#epoch'; - $epoch = $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT); - $blocked = \is_string($epoch) && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX); - $ours = $blocked && \str_starts_with($epoch, self::DOCUMENT_CACHE_BLOCKED_PREFIX.$token.self::DOCUMENT_CACHE_SEPARATOR); - - if ($started === $finished) { - if (! $blocked || $ours) { - $this->publishDocumentCacheEpoch($collectionKey, $finished, $definitionKey); - } - - return; - } - - if (! $owned && ! $ours) { - // This token was cleared by a cache flush or released as abandoned. Leave another - // writer's tombstone fail-closed, but retire whatever readers filled while it ran. - if (\is_string($epoch) && ! $blocked) { - $this->publishDocumentCacheEpoch($collectionKey, $started, $definitionKey); - } - - return; - } - - $nextFinished = $this->finishDocumentCacheWrite($collectionKey, $finished, $epoch); - - $nextStarted = $this->cache->getGeneration($startedKey); - if ($nextStarted === $nextFinished) { - $this->publishDocumentCacheEpoch($collectionKey, $nextFinished, $definitionKey); - - return; - } - - if ($registration->field !== '' && $this->releaseAbandonedDocumentCacheOwners($registration->key)) { - $this->publishDocumentCacheEpoch($collectionKey, $nextStarted, $definitionKey); - - return; - } - - $this->purgeCachedDefinition($definitionKey); - } - - /** - * An epoch carries the started generation it was published at: until the next write starts, it is - * current even while writes judged abandoned still count as unfinished. - */ - private function publishDocumentCacheEpoch(string $collectionKey, string $started, string $definitionKey): void - { - $epochKey = $collectionKey.'#epoch'; - $epoch = self::DOCUMENT_CACHE_ACTIVE_PREFIX.\bin2hex(\random_bytes(16)).self::DOCUMENT_CACHE_SEPARATOR.$started; - $this->withRetries(function () use ($epochKey, $epoch): void { - if ($this->cache->save($epochKey, $epoch) === false) { - throw new CacheUnavailable("Failed to activate document cache epoch '{$epochKey}'"); - } - }); - + $this->withRetries(fn () => (new Mutation($this->cache))->finish($collectionKey, $token, $this->cacheWriterTimeout)); $this->withRetries(fn () => $this->purgeCachedDefinition($definitionKey)); } - /** - * Release this write's registration, and report whether it was still this write's to release. A purge - * that cannot be confirmed is retried while the registration is still in place. - */ - private function releaseDocumentCacheOwner(Registration $registration, string $collectionKey, string $token): bool - { - $released = true; - $this->withRetries(function () use ($registration, $collectionKey, $token, &$released): void { - if ($this->cache->purge($registration->key, $registration->field)) { - return; - } - - $owner = $this->cache->load($registration->key, self::TTL, $registration->field); - if ($owner !== false && $owner !== null) { - throw new CacheUnavailable("Failed to release document cache owner '{$token}' for '{$collectionKey}'"); - } - $released = false; - }); - - return $released; - } - - /** - * Count this write as finished and return the `#finished` generation after it. A cache flush restarts - * generations, so an unchanged `#finished` only proves the purge was lost while the epoch read with it - * is still in place; the purge is retried only then, so a purge that did land is never counted twice. - */ - private function finishDocumentCacheWrite(string $collectionKey, string $finished, mixed $epoch): string - { - $finishedKey = $collectionKey.'#finished'; - $epochKey = $collectionKey.'#epoch'; - $next = $finished; - $attempted = false; - $this->withRetries(function () use ($finishedKey, $epochKey, $finished, $epoch, &$next, &$attempted): void { - if ($attempted) { - $next = $this->cache->getGeneration($finishedKey); - } - if ($next === $finished) { - $attempted = true; - $this->cache->purge($finishedKey); - } - - $next = $this->cache->getGeneration($finishedKey); - if ( - $next === $finished - && \is_string($epoch) - && $this->cache->load($epochKey, self::DOCUMENT_CACHE_PERMANENT) === $epoch - ) { - throw new CacheUnavailable("Failed to finish document cache invalidation '{$epochKey}'"); - } - }); - - return $next; - } - - /** - * Release every other writer still registered when all of them are older than the writer timeout. - * A token without a creation time counts as live. - */ - private function releaseAbandonedDocumentCacheOwners(string $owners): bool - { - $now = \time(); - $abandoned = []; - foreach ($this->cache->list($owners) as $token) { - $separator = \strpos($token, self::DOCUMENT_CACHE_TOKEN_SEPARATOR); - $created = $separator === false ? '' : \substr($token, 0, $separator); - if (! \ctype_digit($created) || (int) $created + $this->cacheWriterTimeout > $now) { - return false; - } - - $abandoned[] = $token; - } - - foreach ($abandoned as $token) { - $this->cache->purge($owners, $token); - } - - return true; - } - /** * Run a cache operation under the document's tenant when tenant-per-document is enabled. * @@ -4163,7 +3883,7 @@ public function purgeCachedQueries(string $collection, ?string $namespace = null } /** - * Execute a callback behind a generation-protected cache-aside lookup. + * Execute a callback behind a version-protected cache-aside lookup. * * @template T * @param callable(): T $callback @@ -4176,7 +3896,7 @@ public function withCache( callable $callback, ?string $hash = '', ): mixed { - if ($hash === null || $this->adapter->inTransaction()) { + if ($hash === null || $this->adapter->inTransaction() || ! $this->cache->supportsVersioning()) { return $callback(); } @@ -4292,9 +4012,9 @@ public function withCache( } try { - $generation = Mutation::lease($this->cache, $physicalKey); + $version = $this->cache->getLease($physicalKey); } catch (Throwable $error) { - Console::warning('Warning: Failed to get cache generation: '.$error->getMessage()); + Console::warning('Warning: Failed to get cache version: '.$error->getMessage()); return $callback(); } @@ -4306,7 +4026,7 @@ public function withCache( $encoded = $this->encodeCacheValue($callbackValue); if ($encoded !== false) { - $this->cache->saveWithLease($physicalKey, $encoded, '', $generation); + $this->cache->saveWithLease($physicalKey, $encoded, '', $version); } } catch (Throwable $error) { Console::warning('Warning: Failed to save cache value: '.$error->getMessage()); @@ -4574,7 +4294,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $adapterQueries = $queries; $cacheEntry = null; - $cacheGeneration = ''; + $cacheVersion = ''; if ( $this->queryCache !== null && $collection->getId() !== self::METADATA @@ -4621,7 +4341,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f $results = $cached; $cacheEntry = null; } else { - $cacheGeneration = $this->queryCache->getGeneration($cacheEntry); + $cacheVersion = $this->queryCache->getLease($cacheEntry); } } } catch (Exception $error) { @@ -4650,7 +4370,7 @@ private function fetch(Collection $collection, array $queries, PermissionType $f if ($cacheEntry !== null && $this->queryCache !== null) { try { if (! $this->isReadFromReplica()) { - $this->queryCache->set($cacheEntry, $results, $cacheGeneration); + $this->queryCache->set($cacheEntry, $results, $cacheVersion); } } catch (Exception $error) { Console::warning('Failed to save query results to cache: '.$error->getMessage()); diff --git a/tests/unit/Cache/CountingCache.php b/tests/unit/Cache/CountingCache.php index e2884a5a0..d58e1f314 100644 --- a/tests/unit/Cache/CountingCache.php +++ b/tests/unit/Cache/CountingCache.php @@ -3,20 +3,47 @@ namespace Tests\Unit\Cache; use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; /** * Counts the calls that each cost one round trip to a networked cache. */ -final class CountingCache implements CacheAdapter, Leasable +final class CountingCache implements CacheAdapter, Versioned { + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); + } + private int $operations = 0; /** @var array */ private array $loads = []; public function __construct( - private readonly CacheAdapter&Leasable $cache, + private readonly CacheAdapter&Versioned $cache, ) { } @@ -92,22 +119,6 @@ public function getName(?string $key = null): string return $this->cache->getName($key); } - #[\Override] - public function getGeneration(string $key): string - { - $this->operations++; - - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - $this->operations++; - - return $this->cache->saveWithLease($key, $data, $hash, $generation); - } - public function getOperations(): int { return $this->operations; diff --git a/tests/unit/Cache/DatabaseTest.php b/tests/unit/Cache/DatabaseTest.php index e20c01bcd..bc2d53c72 100644 --- a/tests/unit/Cache/DatabaseTest.php +++ b/tests/unit/Cache/DatabaseTest.php @@ -82,7 +82,7 @@ public static function queryCacheFailures(): array { return [ 'epoch or entry load' => ['load'], - 'lease' => ['getGeneration'], + 'lease' => ['getLease'], 'fill' => ['saveWithLease'], ]; } @@ -102,11 +102,11 @@ public function load(string $key, int $ttl, string $hash = ''): mixed } #[\Override] - public function getGeneration(string $key): string + public function getLease(string $key): string { - $this->assertAvailable('getGeneration'); + $this->assertAvailable('getLease'); - return parent::getGeneration($key); + return parent::getLease($key); } #[\Override] @@ -166,7 +166,7 @@ public function testSetQueryCacheInvalidatesWhileInstalledAndStopsOnceRemoved(): $scope = $this->scope($database); $before = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'detached'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'detached'])], $queryCache->getLease($before))); $database->createDocument('users', new Document(['$id' => 'b'])); @@ -759,7 +759,7 @@ private function assertMutationRollsBackOnInvalidationFailure(callable $mutation $mutation($database); $this->fail('Mandatory invalidation failure was not propagated'); } catch (\RuntimeException $exception) { - $this->assertStringContainsString('query cache epoch', $exception->getMessage()); + $this->assertInstanceOf(\Utopia\Cache\Exception\Unavailable::class, $exception); } $this->assertSame( @@ -861,7 +861,7 @@ private function assertMutationRetiresQueries(Database $database, callable $muta $scope = $this->scope($database); $before = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getLease($before))); $this->assertSame(['stale'], $this->ids($queryCache->get($before) ?? [])); $mutation($database); @@ -869,7 +869,7 @@ private function assertMutationRetiresQueries(Database $database, callable $muta $after = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($after, 'The mutation must leave the query cache usable'); $this->assertNull($queryCache->get($after), 'The mutation must retire what was cached before it'); - $this->assertTrue($queryCache->set($after, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($after))); + $this->assertTrue($queryCache->set($after, [new Document(['$id' => 'fresh'])], $queryCache->getLease($after))); $this->assertSame(['fresh'], $this->ids($queryCache->get($after) ?? []), 'The mutation must publish a fresh epoch'); } diff --git a/tests/unit/Cache/FailingMemory.php b/tests/unit/Cache/FailingMemory.php index 7cbeb7395..94e8a5e6a 100644 --- a/tests/unit/Cache/FailingMemory.php +++ b/tests/unit/Cache/FailingMemory.php @@ -22,9 +22,9 @@ public function save(string $key, array|string $data, string $hash = '', int $tt { if ( $this->failing - && \str_ends_with($key, '#epoch') - && \is_string($data) - && \str_starts_with($data, 'blocked:') + && \str_ends_with($key, '#state') + && \is_array($data) + && ! empty($data['owners']) ) { return false; } diff --git a/tests/unit/Cache/FailingRegistrationCache.php b/tests/unit/Cache/FailingRegistrationCache.php index c282e3d23..e21d0fdde 100644 --- a/tests/unit/Cache/FailingRegistrationCache.php +++ b/tests/unit/Cache/FailingRegistrationCache.php @@ -3,14 +3,41 @@ namespace Tests\Unit\Cache; use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; /** * A Redis-like cache that refuses a number of owner registrations while every other read and write succeeds, the way * a save that misses its read deadline returns false from an otherwise healthy Redis. */ -final class FailingRegistrationCache implements CacheAdapter, Leasable +final class FailingRegistrationCache implements CacheAdapter, Versioned { + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); + } + private RedisLeasableCache $cache; private int $failures = 0; @@ -41,7 +68,7 @@ public function load(string $key, int $ttl, string $hash = ''): mixed #[\Override] public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array { - if ($hash !== '' && \str_ends_with($key, '#owners')) { + if (\str_ends_with($key, '#state') && \is_array($data) && ! empty($data['owners'])) { $this->registrations++; if ($this->failures > 0) { $this->failures--; @@ -95,16 +122,4 @@ public function getName(?string $key = null): string { return 'failing-registration'; } - - #[\Override] - public function getGeneration(string $key): string - { - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - return $this->cache->saveWithLease($key, $data, $hash, $generation); - } } diff --git a/tests/unit/Cache/InvalidationCache.php b/tests/unit/Cache/InvalidationCache.php index c9d3e397d..ed4466e38 100644 --- a/tests/unit/Cache/InvalidationCache.php +++ b/tests/unit/Cache/InvalidationCache.php @@ -7,48 +7,8 @@ final class InvalidationCache extends Cache { - /** @var array */ - private array $values = []; - - /** @var array */ - private array $generations = []; - public function __construct() { parent::__construct(new Memory()); } - - #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - return $this->values[$key] ?? false; - } - - /** - * @return string|array - */ - #[\Override] - public function save(string $key, mixed $data, string $hash = '', int $ttl = 0): string|array - { - if (\is_string($data)) { - $this->values[$key] = $data; - } - - return $data; - } - - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - unset($this->values[$key]); - - return true; - } } diff --git a/tests/unit/Cache/LeasableHashCache.php b/tests/unit/Cache/LeasableHashCache.php index 50bf5ebd1..15ae0c485 100644 --- a/tests/unit/Cache/LeasableHashCache.php +++ b/tests/unit/Cache/LeasableHashCache.php @@ -2,129 +2,10 @@ namespace Tests\Unit\Cache; -use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; - -final class LeasableHashCache implements CacheAdapter, Leasable +final class LeasableHashCache extends RedisLeasableCache { - /** @var array|string}>> */ - private array $store = []; - - /** @var array */ - private array $generations = []; - - private bool $failActivations = false; - - #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - $hash = $hash === '' ? $key : $hash; - $saved = $this->store[$key][$hash] ?? null; - - return $saved !== null && $saved['time'] + $ttl > \time() ? $saved['data'] : false; - } - - #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array - { - if ($key === '') { - return false; - } - - if ( - $this->failActivations - && \str_ends_with($key, '#epoch') - && \is_string($data) - && \str_starts_with($data, 'active:') - ) { - return false; - } - - $hash = $hash === '' ? $key : $hash; - $this->store[$key][$hash] = ['time' => \time(), 'data' => $data]; - - return $data; - } - - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; - } - - return $this->save($key, $data, $hash); - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - $hash = $hash === '' ? $key : $hash; - if (! isset($this->store[$key][$hash])) { - return false; - } - - $this->store[$key][$hash]['time'] = \time(); - - return true; - } - - /** @return array */ - #[\Override] - public function list(string $key): array - { - return \array_keys($this->store[$key] ?? []); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - - if ($hash === '') { - unset($this->store[$key]); - } else { - unset($this->store[$key][$hash]); - } - - return true; - } - - #[\Override] - public function flush(): bool - { - $this->store = []; - $this->generations = []; - - return true; - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return \count($this->store); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'leasable-hash'; - } - public function failActivations(): void { - $this->failActivations = true; + $this->failCompletions(); } } diff --git a/tests/unit/Cache/OwnersTest.php b/tests/unit/Cache/OwnersTest.php deleted file mode 100644 index 87bc6be13..000000000 --- a/tests/unit/Cache/OwnersTest.php +++ /dev/null @@ -1,103 +0,0 @@ -lists++; - - return parent::list($key); - } - }; - $tokens = ['first', 'second', 'third']; - - foreach ($tokens as $token) { - $this->assertTrue((new Owners($cache))->register(self::KEY, $token)); - } - - foreach ($tokens as $token) { - $registration = (new Owners($cache))->find(self::KEY, $token); - - $this->assertSame([self::KEY.'#owners', $token], [$registration->key, $registration->field]); - $this->assertSame($token, $cache->load($registration->key, self::TTL, $registration->field)); - } - - $this->assertSame(1, $cache->lists, 'Only the first registration should list the owners hash'); - } - - public function testOwnersResolveOnACacheWithoutFields(): void - { - $cache = new class (new Memory()) extends Cache { - public int $lists = 0; - - /** @return string[] */ - #[\Override] - public function list(string $key): array - { - $this->lists++; - - return parent::list($key); - } - }; - - foreach (['first', 'second'] as $token) { - $this->assertTrue((new Owners($cache))->register(self::KEY, $token)); - - $registration = (new Owners($cache))->find(self::KEY, $token); - - $this->assertSame([self::KEY.'#owner:'.$token, ''], [$registration->key, $registration->field]); - $this->assertSame($token, $cache->load($registration->key, self::TTL, $registration->field)); - } - - $this->assertSame(4, $cache->lists, 'A cache that keeps no fields should be asked again on every registration'); - } - - public function testAFlushBeforeTheFirstListDoesNotPinPerTokenKeys(): void - { - $cache = new class (new RedisLeasableCache()) extends Cache { - private bool $flushed = false; - - /** @return string[] */ - #[\Override] - public function list(string $key): array - { - if (! $this->flushed) { - $this->flushed = true; - $this->flush(); - } - - return parent::list($key); - } - }; - - $this->assertTrue((new Owners($cache))->register(self::KEY, 'flushed')); - $flushed = (new Owners($cache))->find(self::KEY, 'flushed'); - $this->assertSame([self::KEY.'#owner:flushed', ''], [$flushed->key, $flushed->field]); - - $this->assertTrue((new Owners($cache))->register(self::KEY, 'kept')); - $kept = (new Owners($cache))->find(self::KEY, 'kept'); - $this->assertSame( - [self::KEY.'#owners', 'kept'], - [$kept->key, $kept->field], - 'A cache that listed no field once should still register later owners as fields', - ); - } -} diff --git a/tests/unit/Cache/OwnershipCache.php b/tests/unit/Cache/OwnershipCache.php index 08e2e74d5..94c0bdeb5 100644 --- a/tests/unit/Cache/OwnershipCache.php +++ b/tests/unit/Cache/OwnershipCache.php @@ -3,132 +3,35 @@ namespace Tests\Unit\Cache; use Closure; -use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Exception\Unavailable; -final class OwnershipCache implements CacheAdapter, Leasable +/** Interleaves another writer or a cache loss before the atomic comparison. */ +final class OwnershipCache extends RedisLeasableCache { - /** @var array|string}> */ - private array $store = []; - - /** @var array */ - private array $generations = []; - private ?Closure $activation = null; - private bool $flushDuringActivation = false; - private bool $failDuringActivation = false; #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false { - $saved = $this->store[$key] ?? null; - - return $saved !== null && $saved['time'] + $ttl > \time() ? $saved['data'] : false; - } - - #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array - { - if ($key === '') { - return false; - } - - if ( - $this->activation !== null - && \str_ends_with($key, '#epoch') - && \is_string($data) - && \str_starts_with($data, 'active:') - ) { - $activation = $this->activation; + $before = $this->load($key, PHP_INT_MAX); + $finishing = \str_ends_with($key, '#state') && \is_array($before) && \is_array($data) && \is_array($data['owners'] ?? null) && \is_array($before['owners'] ?? null) + && \count($data['owners']) < \count($before['owners']); + if ($finishing) { + $callback = $this->activation; $this->activation = null; - $activation(); - } - - $this->store[$key] = ['time' => \time(), 'data' => $data]; - - return $data; - } - - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; + $callback?->__invoke(); + if ($this->flushDuringActivation) { + $this->flushDuringActivation = false; + $this->flush(); + } + if ($this->failDuringActivation) { + throw new Unavailable('Completion unavailable'); + } } - return $this->save($key, $data, $hash); - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - if (! isset($this->store[$key])) { - return false; - } - - $this->store[$key]['time'] = \time(); - - return true; - } - - /** @return array */ - #[\Override] - public function list(string $key): array - { - return []; - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - if ($this->flushDuringActivation && \str_ends_with($key, '#finished')) { - $this->flushDuringActivation = false; - - return $this->flush(); - } - if ($this->failDuringActivation && \str_ends_with($key, '#finished')) { - return false; - } - - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - unset($this->store[$key]); - - return true; - } - - #[\Override] - public function flush(): bool - { - $this->store = []; - $this->generations = []; - - return true; - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return \count($this->store); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'ownership'; + return parent::compareAndSwap($key, $data, $hash, $version); } public function pauseNextActivation(Closure $activation): void diff --git a/tests/unit/Cache/QueryTest.php b/tests/unit/Cache/QueryTest.php index ac7de9b0a..c214f9a40 100644 --- a/tests/unit/Cache/QueryTest.php +++ b/tests/unit/Cache/QueryTest.php @@ -3,7 +3,6 @@ namespace Tests\Unit\Cache; use PHPUnit\Framework\Attributes\DataProvider; -use PHPUnit\Framework\MockObject\Stub; use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\Memory; use Utopia\Cache\Cache; @@ -25,18 +24,18 @@ class QueryTest extends TestCase { private ResultCache $queryCache; - private Cache&Stub $cache; + private Cache $cache; #[\Override] protected function setUp(): void { - $this->cache = self::createCache(); + $this->cache = $this->createCache(); $this->queryCache = new ResultCache($this->cache); } public function testConstructorWithDefaults(): void { - $queryCache = new ResultCache(self::createCache()); + $queryCache = new ResultCache($this->createCache()); $this->assertNotNull($queryCache->getEntry(new Scope(), 'any_collection', [])); } @@ -48,7 +47,7 @@ public function testCacheNamesKeepTheirResultsApartOnTheSameCache(): void $custom = new ResultCache(new Cache($adapter)); $entry = $custom->getEntry(new Scope(name: 'custom'), 'users', []); $this->assertNotNull($entry); - $this->assertTrue($custom->set($entry, [new Document(['$id' => 'custom'])], $custom->getGeneration($entry))); + $this->assertTrue($custom->set($entry, [new Document(['$id' => 'custom'])], $custom->getLease($entry))); $other = $default->getEntry(new Scope(), 'users', []); $this->assertNotNull($other); @@ -66,7 +65,7 @@ public function testOneQueryCacheKeysEachCallByTheNameInItsScope(): void $queryCache = new ResultCache(new Cache(new RedisLeasableCache())); $first = $queryCache->getEntry(new Scope(name: 'first'), 'users', []); $this->assertNotNull($first); - $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'first'])], $queryCache->getGeneration($first))); + $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'first'])], $queryCache->getLease($first))); $second = $queryCache->getEntry(new Scope(name: 'second'), 'users', []); $this->assertNotNull($second); @@ -86,7 +85,7 @@ public function testTheWriterTimeoutComesFromTheScopeOfEachCall(): void $this->assertNull($queryCache->getEntry(new Scope(), 'users', []), 'A write younger than the writer timeout is still in flight'); $this->assertNotNull($queryCache->getEntry(new Scope(writerTimeout: 0), 'users', []), 'Under a zero writer timeout the write is abandoned at once'); - $this->assertNull($queryCache->getEntry(new Scope(), 'users', []), 'A call under a zero writer timeout must not change the timeout of later calls'); + $this->assertNull($queryCache->getEntry(new Scope(), 'users', []), 'A shorter read timeout must preserve live registrations'); } public function testSetRegionAndGetRegion(): void @@ -172,16 +171,15 @@ public function testDifferentCollectionsProduceDifferentEntries(): void public function testGetReturnsNullForCacheMiss(): void { - $this->cache->method('load')->willReturn(false); $this->assertNull($this->queryCache->get(new Entry('some-key', 'users'))); } public function testGetReturnsNullForNullData(): void { - $this->cache->method('load')->willReturn(null); - - $this->assertNull($this->queryCache->get(new Entry('some-key', 'users'))); + $cache = self::createStub(Cache::class); + $cache->method('load')->willReturn(null); + $this->assertNull((new ResultCache($cache))->get(new Entry('some-key', 'users'))); } public function testFilledResultsAreServedByAnotherQueryCacheOnTheSameCache(): void @@ -196,7 +194,7 @@ public function testFilledResultsAreServedByAnotherQueryCacheOnTheSameCache(): v $this->assertTrue($writer->set($entry, [ new Document(['$id' => 'doc1', 'name' => 'Alice']), new Document(['$id' => 'doc2', 'name' => 'Bob']), - ], $writer->getGeneration($entry))); + ], $writer->getLease($entry))); $served = $reader->getEntry($scope, 'users', [Query::limit(2)]); $this->assertNotNull($served); @@ -215,7 +213,7 @@ public function testAFillThatStartedBeforeAnInvalidationIsRejected(): void $scope = new Scope(namespace: 'ns'); $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($entry); - $generation = $queryCache->getGeneration($entry); + $generation = $queryCache->getLease($entry); $queryCache->invalidateCollection($scope, 'users'); @@ -228,7 +226,7 @@ public function testAFillThatStartedBeforeAnInvalidationIsRejected(): void public function testGetHandlesDocumentObjectsInCache(): void { $document = new Document(['$id' => 'doc1', 'name' => 'Alice']); - $this->cache->method('load')->willReturn([ + $this->cache->save('some-key', [ 'version' => 2, 'epoch' => '', 'field' => '', @@ -244,10 +242,16 @@ public function testGetHandlesDocumentObjectsInCache(): void public function testGetPropagatesMalformedPayloadPurgeFailure(): void { - $this->cache->method('load')->willReturn('not-an-array'); - - $this->expectException(\RuntimeException::class); - $this->queryCache->get(new Entry('some-key', 'users')); + $cache = new class (new Memory()) extends Cache { + #[\Override] + public function purge(string $key, string $hash = ''): bool + { + throw new Unavailable('Purge unavailable'); + } + }; + $cache->save('some-key', 'not-an-array'); + $this->expectException(Unavailable::class); + (new ResultCache($cache))->get(new Entry('some-key', 'users')); } public function testEntriesExpireWithTheRegionButEpochsNeverDo(): void @@ -259,7 +263,7 @@ public function testEntriesExpireWithTheRegionButEpochsNeverDo(): void $entry = $queryCache->getEntry($scope, 'users', [Query::limit(10)]); $this->assertNotNull($entry, 'The epoch a write published must stay usable after the region TTL has passed'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'filled'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'filled'])], $queryCache->getLease($entry))); $this->assertNull($queryCache->get($entry), 'A result older than the region TTL must miss'); $queryCache->setRegion('users', new Region(ttl: 60)); @@ -295,7 +299,7 @@ public function testAMalformedResultMissesAndIsReplacedByTheNextFill(\Closure $p $cache->save($entry->key, $payload($entry), $entry->slot); $this->assertNull($queryCache->get($entry), 'A result the query cache cannot read must be a miss, not an error'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getLease($entry))); $this->assertSame(['fresh'], $this->ids($queryCache->get($entry) ?? [])); } @@ -313,9 +317,10 @@ public function testAnInvalidEpochIsAMiss(): void $cache = new Cache(new Memory()); $queryCache = new ResultCache($cache); $scope = new Scope(namespace: 'ns'); - $cache->save($queryCache->getCollectionKey($scope, 'users').'#epoch', ['not' => 'an epoch']); + $cache->save($queryCache->getCollectionKey($scope, 'users').'#state', ['not' => 'a state']); - $this->assertNull($queryCache->getEntry($scope, 'users', []), 'An epoch value the query cache did not write must disable the cache for that read, not fail it'); + $this->expectException(Unavailable::class); + $queryCache->getEntry($scope, 'users', []); } public function testEntriesResolveByDefault(): void @@ -338,8 +343,8 @@ public function testFlushDropsEveryCachedResult(): void $posts = $queryCache->getEntry($scope, 'posts', []); $this->assertNotNull($users); $this->assertNotNull($posts); - $this->assertTrue($queryCache->set($users, [new Document(['$id' => 'user'])], $queryCache->getGeneration($users))); - $this->assertTrue($queryCache->set($posts, [new Document(['$id' => 'post'])], $queryCache->getGeneration($posts))); + $this->assertTrue($queryCache->set($users, [new Document(['$id' => 'user'])], $queryCache->getLease($users))); + $this->assertTrue($queryCache->set($posts, [new Document(['$id' => 'post'])], $queryCache->getLease($posts))); $queryCache->flush(); @@ -434,7 +439,7 @@ public function testInvalidatorInvalidatesTheScopeItIsGiven(): void $scope = new Scope('host', 'database', 'namespace', 7); $untouched = $queryCache->getEntry(new Scope(), 'users', []); $this->assertNotNull($untouched); - $this->assertTrue($queryCache->set($untouched, [new Document(['$id' => 'untouched'])], $queryCache->getGeneration($untouched))); + $this->assertTrue($queryCache->set($untouched, [new Document(['$id' => 'untouched'])], $queryCache->getLease($untouched))); $this->assertRetired($queryCache, $scope, 'users', function () use ($queryCache, $scope): void { (new Invalidator($queryCache))->invalidate(Event::DocumentCreate, 'users', $scope); @@ -473,32 +478,32 @@ public function testMemoryAdapterKeepsPhysicalVariantsIsolated(): void $this->assertNotNull($first); $this->assertNotNull($second); - $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getGeneration($first))); - $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getGeneration($second))); + $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getLease($first))); + $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getLease($second))); $this->assertNull($queryCache->get($first), 'A cache without fields holds one result per collection: the second query took the slot, so the first must miss instead of being served its rows'); $this->assertSame(['private-b'], $this->ids($queryCache->get($second) ?? [])); - $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getGeneration($first))); + $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getLease($first))); $this->assertSame(['private-a'], $this->ids($queryCache->get($first) ?? [])); $this->assertNull($queryCache->get($second)); } - public function testMemoryAdapterStaysBlockedAfterInvalidation(): void + public function testMemoryAdapterResumesAfterInvalidation(): void { $queryCache = new ResultCache(new Cache(new Memory())); $scope = new Scope(namespace: 'ns'); $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($entry); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'old'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'old'])], $queryCache->getLease($entry))); $this->assertSame(['old'], $this->ids($queryCache->get($entry) ?? [])); $queryCache->invalidateCollection($scope, 'users'); - $this->assertNull( + $this->assertNotNull( $queryCache->getEntry($scope, 'users', []), - 'A cache without generations cannot prove a new epoch fresh, so it stays blocked', + 'The last completion makes the versioned cache usable', ); } @@ -517,7 +522,7 @@ public function testConcurrentOwnersCannotEnableCacheEarly(): void $entry = $reader->getEntry($scope, 'users', []); $this->assertNotNull($entry); - $this->assertTrue($reader->set($entry, [new Document(['$id' => 'old'])], $reader->getGeneration($entry))); + $this->assertTrue($reader->set($entry, [new Document(['$id' => 'old'])], $reader->getLease($entry))); $this->assertSame(['old'], $this->ids((new ResultCache(new Cache($adapter)))->get($entry) ?? [])); $first->blockCollection($key, $firstToken); @@ -536,7 +541,7 @@ public function testConcurrentOwnersCannotEnableCacheEarly(): void $fresh = $reader->getEntry($scope, 'users', []); $this->assertNotNull($fresh); $this->assertNull($reader->get($fresh)); - $this->assertTrue($reader->set($fresh, [new Document(['$id' => 'fresh'])], $reader->getGeneration($fresh))); + $this->assertTrue($reader->set($fresh, [new Document(['$id' => 'fresh'])], $reader->getLease($fresh))); $this->assertSame(['fresh'], $this->ids((new ResultCache(new Cache($adapter)))->get($fresh) ?? [])); } } @@ -557,7 +562,7 @@ public function testAnEpochPublishedAfterALaterFinishStaysUsable(): void $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($entry, 'Once every writer has finished, an epoch published late must still be usable'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getLease($entry))); $this->assertSame(['fresh'], $this->ids($queryCache->get($entry) ?? [])); } @@ -568,7 +573,7 @@ public function testATombstoneOlderThanItsRegionStillBlocksWhileItsWriterIsInFli $key = $queryCache->getCollectionKey(new Scope(), 'users'); $stale = $queryCache->getEntry(new Scope(), 'users', []); $this->assertNotNull($stale); - $this->assertTrue($queryCache->set($stale, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($stale))); + $this->assertTrue($queryCache->set($stale, [new Document(['$id' => 'stale'])], $queryCache->getLease($stale))); $queryCache->blockCollection($key, 'writer'); @@ -583,7 +588,7 @@ public function testATombstoneOlderThanItsRegionStillBlocksWhileItsWriterIsInFli $fresh = $queryCache->getEntry(new Scope(), 'users', []); $this->assertNotNull($fresh, 'The writer must still own its tombstone and publish a fresh epoch'); $this->assertNull($queryCache->get($fresh), 'The fresh epoch must not serve what was filled before the write'); - $this->assertTrue($queryCache->set($fresh, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($fresh))); + $this->assertTrue($queryCache->set($fresh, [new Document(['$id' => 'fresh'])], $queryCache->getLease($fresh))); $this->assertSame(['fresh'], $this->ids($queryCache->get($fresh) ?? [])); } @@ -597,12 +602,12 @@ public function testAKilledWriterDoesNotDisableTheQueryCacheForever(): void $this->assertNotNull($before); $queryCache->blockCollection($key, $queryCache->createToken()); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getLease($before))); $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($entry, 'A tombstone whose writer never activates must lapse once it is older than the writer timeout'); $this->assertNull($queryCache->get($entry), 'The lapse must not serve a result filled under the epoch the killed writer retired'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getLease($entry))); $this->assertSame(['fresh'], $this->ids($queryCache->get($entry) ?? [])); } @@ -621,7 +626,7 @@ public function testAWriteAfterAKilledWriterReenablesTheQueryCache(): void $entry = $reader->getEntry($scope, 'users', []); $this->assertNotNull($entry, 'The next write must reconcile a writer whose registration is older than the writer timeout'); - $this->assertTrue($reader->set($entry, [new Document(['$id' => 'fresh'])], $reader->getGeneration($entry))); + $this->assertTrue($reader->set($entry, [new Document(['$id' => 'fresh'])], $reader->getLease($entry))); $this->assertSame(['fresh'], $this->ids($reader->get($entry) ?? [])); $writer->invalidateCollection($writerScope, 'users'); @@ -659,7 +664,7 @@ public function testAWriterWhoseTokenHasNoCreationTimeCountsAsLive(): void $writer->invalidateCollection(new Scope(namespace: 'ns', writerTimeout: 0), 'users'); - $this->assertNull($reader->getEntry($scope, 'users', []), 'Without a creation time a registration cannot be judged abandoned'); + $this->assertNotNull($reader->getEntry($scope, 'users', []), 'Registration time is stored in the record rather than encoded in the token'); } public function testAWriterPastTheTimeoutRetiresWhatReadersFilledWhileItRan(): void @@ -672,7 +677,7 @@ public function testAWriterPastTheTimeoutRetiresWhatReadersFilledWhileItRan(): v $queryCache->blockCollection($key, $token); $during = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($during); - $this->assertTrue($queryCache->set($during, [new Document(['$id' => 'before-commit'])], $queryCache->getGeneration($during))); + $this->assertTrue($queryCache->set($during, [new Document(['$id' => 'before-commit'])], $queryCache->getLease($during))); $queryCache->activateCollection($key, $token, $scope->writerTimeout); @@ -693,7 +698,7 @@ public function testAWriterJudgedAbandonedStillRetiresWhatReadersFilledWhileItRa $writer->invalidateCollection(new Scope(namespace: 'ns', writerTimeout: 0), 'users'); $during = $slow->getEntry($scope, 'users', []); $this->assertNotNull($during); - $this->assertTrue($slow->set($during, [new Document(['$id' => 'before-commit'])], $slow->getGeneration($during))); + $this->assertTrue($slow->set($during, [new Document(['$id' => 'before-commit'])], $slow->getLease($during))); $slow->activateCollection($key, $token, Scope::WRITER_TIMEOUT); @@ -753,7 +758,7 @@ public function testActivationPurgeFailureStillPropagates(): void $queryCache->activateCollection($key, 'owner', Scope::WRITER_TIMEOUT); $this->fail('Query cache activation purge failure was not propagated'); } catch (\RuntimeException $error) { - $this->assertStringContainsString('finish query cache invalidation', $error->getMessage()); + $this->assertInstanceOf(Unavailable::class, $error); } $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); @@ -769,7 +774,7 @@ public function testFillsAndInvalidationsKeepTheCacheSizeBounded(): void $entry = $queryCache->getEntry($scope, 'users', [Query::limit($query)]); $this->assertNotNull($entry); $this->assertNull($queryCache->get($entry)); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'query-'.$query])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'query-'.$query])], $queryCache->getLease($entry))); $this->assertSame(['query-'.$query], $this->ids($queryCache->get($entry) ?? [])); $queryCache->invalidateCollection($scope, 'users'); }; @@ -795,7 +800,7 @@ public function testAnInvalidationRetiresEveryCachedResultOfTheScope(): void for ($query = 1; $query <= 50; $query++) { $entry = $queryCache->getEntry($scope, 'users', [Query::limit($query)]); $this->assertNotNull($entry); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'old-'.$query])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'old-'.$query])], $queryCache->getLease($entry))); $this->assertSame(['old-'.$query], $this->ids($queryCache->get($entry) ?? [])); } @@ -805,7 +810,7 @@ public function testAnInvalidationRetiresEveryCachedResultOfTheScope(): void $entry = $queryCache->getEntry($scope, 'users', [Query::limit($query)]); $this->assertNotNull($entry, 'The invalidation must leave the scope usable'); $this->assertNull($queryCache->get($entry), 'The invalidation must retire every result filled before it'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'new-'.$query])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'new-'.$query])], $queryCache->getLease($entry))); $this->assertSame(['new-'.$query], $this->ids($queryCache->get($entry) ?? []), 'A fill after the invalidation must be served'); } } @@ -820,8 +825,8 @@ public function testQueriesSharingASlotNeverServeEachOther(): void $this->assertNotNull($first); $this->assertNotNull($second); - $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getGeneration($first))); - $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getGeneration($second))); + $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getLease($first))); + $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getLease($second))); $this->assertNull($queryCache->get($first), 'The second query took the only slot, so the first must miss instead of being served its rows'); $this->assertSame(['private-b'], $this->ids($queryCache->get($second) ?? [])); @@ -835,11 +840,10 @@ public function testAResultFilledBeforeTheFirstWriteIsNeverServedAfterTheEpochIs $key = $queryCache->getCollectionKey($scope, 'users'); $before = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getLease($before))); $queryCache->invalidateCollection($scope, 'users'); - $adapter->evict($key.'#epoch'); - $adapter->evict($key.'#started'); + $adapter->evict($key.'#state'); $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNull($entry === null ? null : $queryCache->get($entry), 'An evicted epoch must not bring back the initial epoch a result was filled under before the first write'); @@ -854,7 +858,7 @@ public function testAFillUnderARetiredEpochIsNeverServed(): void $this->assertNotNull($stale); $queryCache->invalidateCollection($scope, 'users'); - $this->assertTrue($queryCache->set($stale, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($stale))); + $this->assertTrue($queryCache->set($stale, [new Document(['$id' => 'stale'])], $queryCache->getLease($stale))); $fresh = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($fresh); @@ -871,8 +875,8 @@ public function testQueriesOfOneCollectionKeepTheirOwnResultsOnACacheWithFields( $this->assertNotNull($first); $this->assertNotNull($second); - $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getGeneration($first))); - $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getGeneration($second))); + $this->assertTrue($queryCache->set($first, [new Document(['$id' => 'private-a'])], $queryCache->getLease($first))); + $this->assertTrue($queryCache->set($second, [new Document(['$id' => 'private-b'])], $queryCache->getLease($second))); $this->assertSame(['private-a'], $this->ids($queryCache->get($first) ?? [])); $this->assertSame(['private-b'], $this->ids($queryCache->get($second) ?? [])); @@ -889,9 +893,9 @@ public function testOverlappingInvalidationsSucceedOnACacheWithoutFields(): void $this->assertNull($queryCache->getEntry(new Scope(), 'users', []), 'The second writer is still in flight'); $queryCache->activateCollection($key, 'second', Scope::WRITER_TIMEOUT); - $this->assertNull( + $this->assertNotNull( $queryCache->getEntry(new Scope(), 'users', []), - 'A cache without generations cannot prove a new epoch fresh, so it stays blocked', + 'The last completion makes the cache usable', ); } @@ -907,10 +911,11 @@ public function testActivationRejectsACorruptedOwnerRegistration(): void $queryCache->activateCollection($key, 'owner', Scope::WRITER_TIMEOUT); $this->fail('A corrupted owner registration was accepted'); } catch (\RuntimeException $error) { - $this->assertStringContainsString('Invalid query cache owner', $error->getMessage()); + $this->assertInstanceOf(Unavailable::class, $error); } - $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); + $this->expectException(Unavailable::class); + $queryCache->getEntry(new Scope(), 'users', []); } public function testActivationPropagatesAnOwnerReleaseFailure(): void @@ -919,13 +924,13 @@ public function testActivationPropagatesAnOwnerReleaseFailure(): void $queryCache = new ResultCache(new Cache($adapter)); $key = $queryCache->getCollectionKey(new Scope(), 'users'); $queryCache->blockCollection($key, 'owner'); - $adapter->failFieldPurges(); + $adapter->failCompletions(); try { $queryCache->activateCollection($key, 'owner', Scope::WRITER_TIMEOUT); $this->fail('An owner release failure was not propagated'); } catch (\RuntimeException $error) { - $this->assertStringContainsString('Failed to release query cache owner', $error->getMessage()); + $this->assertInstanceOf(Unavailable::class, $error); } $this->assertNull($queryCache->getEntry(new Scope(), 'users', [])); @@ -933,17 +938,15 @@ public function testActivationPropagatesAnOwnerReleaseFailure(): void public function testInvalidationPropagatesACacheWriteFailure(): void { - $queryCache = new ResultCache(new class (new RedisLeasableCache()) extends Cache { + $cache = new class (new RedisLeasableCache()) extends Cache { #[\Override] - public function save(string $key, mixed $data, string $hash = '', int $ttl = 0): bool|string|array + public function compareAndSwap(string $key, array|string $data, string $version): string|false { - return $hash === '' ? false : parent::save($key, $data, $hash); + throw new Unavailable('CAS unavailable'); } - }); - + }; $this->expectException(Unavailable::class); - $this->expectExceptionMessage('Failed to block query cache epoch'); - $queryCache->invalidateCollection(new Scope(), 'users'); + (new ResultCache($cache))->invalidateCollection(new Scope(), 'users'); } public function testAFlushedWriterLeavesAnotherWritersTombstoneInPlace(): void @@ -971,59 +974,52 @@ public function testAnOwnerReleasedByAConcurrentFlushIsNotReported(): void $cache = new class (new RedisLeasableCache()) extends Cache { private bool $armed = false; - public function flushOnNextFieldPurge(): void + public function flushOnNextCompletion(): void { $this->armed = true; } #[\Override] - public function purge(string $key, string $hash = ''): bool + public function compareAndSwap(string $key, mixed $data, string $version, string $hash = ''): string|false { - if ($this->armed && $hash !== '') { + if ($this->armed && \str_ends_with($key, '#state')) { $this->armed = false; $this->flush(); } - return parent::purge($key, $hash); + return parent::compareAndSwap($key, $data, $version, $hash); } }; $queryCache = new ResultCache($cache); $scope = new Scope(namespace: 'ns'); $key = $queryCache->getCollectionKey($scope, 'users'); $queryCache->blockCollection($key, 'owner'); - $cache->flushOnNextFieldPurge(); + $cache->flushOnNextCompletion(); $queryCache->activateCollection($key, 'owner', Scope::WRITER_TIMEOUT); $entry = $queryCache->getEntry($scope, 'users', []); $this->assertNotNull($entry, 'An owner that a flush removed before its release must still publish a fresh epoch'); - $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [new Document(['$id' => 'fresh'])], $queryCache->getLease($entry))); $this->assertSame(['fresh'], $this->ids($queryCache->get($entry) ?? [])); } - public function testInvalidationPropagatesAnOwnerRegistrationFailure(): void + public function testRegistrationFailurePreservesPreviouslyCachedResults(): void { - $queryCache = new ResultCache(new class (new RedisLeasableCache()) extends Cache { - #[\Override] - public function save(string $key, mixed $data, string $hash = '', int $ttl = 0): bool|string|array - { - return $hash === '' ? parent::save($key, $data, $hash) : false; - } - }); + $adapter = new FailingRegistrationCache(); + $queryCache = new ResultCache(new Cache($adapter)); $before = $queryCache->getEntry(new Scope(), 'users', []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'cached'])], $queryCache->getGeneration($before))); - + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'cached'])], $queryCache->getLease($before))); + $adapter->failRegistrations(1); try { $queryCache->invalidateCollection(new Scope(), 'users'); - $this->fail('An owner registration failure was not propagated'); - } catch (\RuntimeException $error) { - $this->assertStringContainsString('Failed to register query cache owner', $error->getMessage()); + $this->fail('Unregistered invalidation must fail'); + } catch (Unavailable) { + $after = $queryCache->getEntry(new Scope(), 'users', []); + $this->assertNotNull($after); + $this->assertSame(['cached'], $this->ids($queryCache->get($after) ?? [])); } - - $after = $queryCache->getEntry(new Scope(), 'users', []); - $this->assertNotNull($after, 'A write whose owner was not registered must not block the epoch'); - $this->assertSame(['cached'], $this->ids($queryCache->get($after) ?? [])); } public function testFlushFailureIsReported(): void @@ -1060,7 +1056,7 @@ private function assertRetired(ResultCache $queryCache, Scope $scope, string $co { $before = $queryCache->getEntry($scope, $collection, []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'stale'])], $queryCache->getLease($before))); $this->assertSame(['stale'], $this->ids($queryCache->get($before) ?? [])); $action(); @@ -1068,7 +1064,7 @@ private function assertRetired(ResultCache $queryCache, Scope $scope, string $co $after = $queryCache->getEntry($scope, $collection, []); $this->assertNotNull($after, "The invalidation must leave '{$collection}' usable"); $this->assertNull($queryCache->get($after), "The invalidation must retire what '{$collection}' filled before it"); - $this->assertTrue($queryCache->set($after, [new Document(['$id' => 'fresh'])], $queryCache->getGeneration($after))); + $this->assertTrue($queryCache->set($after, [new Document(['$id' => 'fresh'])], $queryCache->getLease($after))); $this->assertSame(['fresh'], $this->ids($queryCache->get($after) ?? []), "The invalidation must publish a fresh epoch for '{$collection}'"); } @@ -1080,7 +1076,7 @@ private function assertKept(ResultCache $queryCache, array $collections, callabl foreach ($collections as $collection) { $before = $queryCache->getEntry(new Scope(), $collection, []); $this->assertNotNull($before); - $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'cached'])], $queryCache->getGeneration($before))); + $this->assertTrue($queryCache->set($before, [new Document(['$id' => 'cached'])], $queryCache->getLease($before))); } $action(); @@ -1092,12 +1088,9 @@ private function assertKept(ResultCache $queryCache, array $collections, callabl } } - private static function createCache(): Cache&Stub + private function createCache(): Cache { - $cache = self::createStub(Cache::class); - $cache->method('getGeneration')->willReturn('0'); - - return $cache; + return new Cache(new RedisLeasableCache()); } /** diff --git a/tests/unit/Cache/RacedReleaseCache.php b/tests/unit/Cache/RacedReleaseCache.php index 5b6c37141..f3ad863f4 100644 --- a/tests/unit/Cache/RacedReleaseCache.php +++ b/tests/unit/Cache/RacedReleaseCache.php @@ -2,98 +2,26 @@ namespace Tests\Unit\Cache; -use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; - -/** - * A Redis-like cache where another worker releases a write's owner registration between the write's read of it and - * its own release, so that release finds nothing left to remove. - */ -final class RacedReleaseCache implements CacheAdapter, Leasable +/** Evicts a registration immediately before its owner's completion CAS. */ +final class RacedReleaseCache extends RedisLeasableCache { - private RedisLeasableCache $cache; - private bool $racing = false; - public function __construct() - { - $this->cache = new RedisLeasableCache(); - } - public function releaseBeforeNextOwnerRelease(): void { $this->racing = true; } #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - return $this->cache->load($key, $ttl, $hash); - } - - #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false { - return $this->cache->save($key, $data, $hash); - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - return $this->cache->touch($key, $hash); - } - - /** @return array */ - #[\Override] - public function list(string $key): array - { - return $this->cache->list($key); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - if ($this->racing && $hash !== '' && \str_ends_with($key, '#owners')) { + $before = $this->load($key, PHP_INT_MAX); + if ($this->racing && \str_ends_with($key, '#state') && \is_array($before) && \is_array($data) && \is_array($data['owners'] ?? null) && \is_array($before['owners'] ?? null) + && \count($data['owners']) < \count($before['owners'])) { $this->racing = false; - $this->cache->purge($key, $hash); + $this->evict($key); } - return $this->cache->purge($key, $hash); - } - - #[\Override] - public function flush(): bool - { - return $this->cache->flush(); - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return $this->cache->getSize(); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'raced-release'; - } - - #[\Override] - public function getGeneration(string $key): string - { - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - return $this->cache->saveWithLease($key, $data, $hash, $generation); + return parent::compareAndSwap($key, $data, $hash, $version); } } diff --git a/tests/unit/Cache/RedisLeasableCache.php b/tests/unit/Cache/RedisLeasableCache.php index 5dc00f59b..b62015437 100644 --- a/tests/unit/Cache/RedisLeasableCache.php +++ b/tests/unit/Cache/RedisLeasableCache.php @@ -3,24 +3,52 @@ namespace Tests\Unit\Cache; use Utopia\Cache\Adapter as CacheAdapter; -use Utopia\Cache\Feature\Leasable; - -/** - * Mirrors how utopia-php/cache's Redis adapters store keys: every key is a hash whose generation is a reserved - * field, and purging a key or one of its fields advances that generation instead of deleting the key - * (LUA_PURGE_BUMP and LUA_PURGE_FIELD). A key, once written, stays behind for good, holding at least its - * generation, because the adapters never set an expiry. - */ -final class RedisLeasableCache implements CacheAdapter, Leasable +use Utopia\Cache\Feature\Versioned; + +/** Hash-backed versioned cache with explicit eviction and transport fault injection. */ +class RedisLeasableCache implements CacheAdapter, Versioned { + /** @var array */ + private array $versions = []; + + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->versions[$key] ??= \bin2hex(\random_bytes(16)); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + $before = $this->load($key, PHP_INT_MAX); + if ($this->failingCompletions && \str_ends_with($key, '#state') && \is_array($before) && \is_array($data) && \is_array($data['owners'] ?? null) && \is_array($before['owners'] ?? null) && \count($data['owners']) < \count($before['owners'])) { + throw new \Utopia\Cache\Exception\Unavailable('Completion unavailable'); + } + if ($this->getVersion($key) !== $version || ($this->tombstones[$key] ?? 0.0) > \microtime(true)) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->versions[$key] = \bin2hex(\random_bytes(16)); + } + /** @var array|string}>> */ private array $fields = []; - /** @var array */ - private array $generations = []; - private bool $failingFieldPurges = false; + private bool $failingCompletions = false; + private bool $corruptingFieldWrites = false; private int $leaseGraceWindow = 0; @@ -43,37 +71,16 @@ public function save(string $key, array|string $data, string $hash = '', int $tt return false; } - if ($hash !== '' && $this->corruptingFieldWrites) { + if ($this->corruptingFieldWrites && ($hash !== '' || \str_ends_with($key, '#state'))) { $data = 'corrupted'; } + $this->versions[$key] = \bin2hex(\random_bytes(16)); $this->fields[$key][$this->field($key, $hash)] = ['time' => \time(), 'data' => $data]; return $data; } - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; - } - - if (($this->tombstones[$key] ?? 0.0) > \microtime(true)) { - return false; - } - - unset($this->tombstones[$key]); - - return $this->save($key, $data, $hash); - } - #[\Override] public function touch(string $key, string $hash = ''): bool { @@ -101,7 +108,7 @@ public function purge(string $key, string $hash = ''): bool return false; } - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; + unset($this->versions[$key]); if ($this->leaseGraceWindow > 0) { $this->tombstones[$key] = \microtime(true) + $this->leaseGraceWindow / 1000; } @@ -126,7 +133,7 @@ public function purge(string $key, string $hash = ''): bool public function flush(): bool { $this->fields = []; - $this->generations = []; + $this->versions = []; $this->tombstones = []; return true; @@ -157,7 +164,7 @@ public function getName(?string $key = null): string */ public function keys(): array { - $keys = \array_map(\strval(...), \array_keys($this->fields + $this->generations)); + $keys = \array_map(\strval(...), \array_keys($this->fields + $this->versions)); \sort($keys); return $keys; @@ -180,6 +187,11 @@ public function expireTombstones(): void /** * Fail every purge of a single field, leaving the field in place. */ + public function failCompletions(): void + { + $this->failingCompletions = true; + } + public function failFieldPurges(): void { $this->failingFieldPurges = true; @@ -198,7 +210,7 @@ public function corruptFieldWrites(bool $corrupting = true): void */ public function evict(string $key): void { - unset($this->fields[$key], $this->generations[$key], $this->tombstones[$key]); + unset($this->fields[$key], $this->versions[$key], $this->tombstones[$key]); } /** diff --git a/tests/unit/Cache/UnavailableCache.php b/tests/unit/Cache/UnavailableCache.php index d3341684c..3c668b03c 100644 --- a/tests/unit/Cache/UnavailableCache.php +++ b/tests/unit/Cache/UnavailableCache.php @@ -4,15 +4,42 @@ use Utopia\Cache\Adapter as CacheAdapter; use Utopia\Cache\Exception\Unavailable; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; /** * A Redis-like cache behind utopia-php/cache's circuit breaker. While the circuit is open every call takes its * fallback: reads miss and saves report false. A purge the breaker cannot confirm throws instead of reporting * nothing to purge. With only writes failing, reads are still served. */ -final class UnavailableCache implements CacheAdapter, Leasable +final class UnavailableCache implements CacheAdapter, Versioned { + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); + } + private RedisLeasableCache $cache; private bool $open = false; @@ -140,16 +167,4 @@ public function getName(?string $key = null): string { return 'unavailable'; } - - #[\Override] - public function getGeneration(string $key): string - { - return $this->open ? '0' : $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - return $this->writable() ? $this->cache->saveWithLease($key, $data, $hash, $generation) : false; - } } diff --git a/tests/unit/DocumentFromStorageTest.php b/tests/unit/DocumentFromStorageTest.php index 29d30449a..7399cdaaf 100644 --- a/tests/unit/DocumentFromStorageTest.php +++ b/tests/unit/DocumentFromStorageTest.php @@ -160,7 +160,7 @@ public function testQueryCacheRebuildsCachedDocumentsWithANonStringPermission(): $this->assertNotNull($entry); $stored = new Document(); $stored->exchangeArray([Document::ID => 'legacy', Document::PERMISSIONS => [Permission::read(Role::any()), 42]]); - $this->assertTrue($queryCache->set($entry, [$stored], $queryCache->getGeneration($entry))); + $this->assertTrue($queryCache->set($entry, [$stored], $queryCache->getLease($entry))); $cached = $queryCache->get($entry); diff --git a/tests/unit/Documents/CacheUnavailableTest.php b/tests/unit/Documents/CacheUnavailableTest.php index 2bb1ccc3d..223afde1d 100644 --- a/tests/unit/Documents/CacheUnavailableTest.php +++ b/tests/unit/Documents/CacheUnavailableTest.php @@ -179,9 +179,7 @@ public function testAnEpochPublishBlipAfterTheCommitIsRetried(): void $database->updateDocuments('webhooks', new Document(['name' => 'updated']), [Query::equal('$id', ['hook'])]); $collectionKey = $database->getCacheBaseKeys('webhooks')[0]; - $epoch = $database->getCache()->load($collectionKey.'#epoch', Database::TTL); - $this->assertIsString($epoch); - $this->assertStringStartsNotWith('blocked', $epoch, 'A blip shorter than the retries must still publish a fresh epoch'); + $this->assertNotNull((new \Utopia\Database\Cache\Mutation($database->getCache()))->read($collectionKey, 3600)->value); $this->assertSame('updated', $this->cached($database)); } @@ -191,7 +189,7 @@ public function testPurgeCachedCollectionReportsACacheThatCannotConfirmIt(): voi $database = $this->createDatabase($cache); $cache->open(); - $this->expectExceptionMessage('Cache purge could not be confirmed'); + $this->expectException(\Utopia\Cache\Exception\Unavailable::class); $database->purgeCachedCollection('webhooks'); } diff --git a/tests/unit/Documents/DocumentCacheEpochTest.php b/tests/unit/Documents/DocumentCacheEpochTest.php index eb8382051..d959ebd1b 100644 --- a/tests/unit/Documents/DocumentCacheEpochTest.php +++ b/tests/unit/Documents/DocumentCacheEpochTest.php @@ -12,7 +12,7 @@ use Utopia\Cache\Adapter as CacheAdapter; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Adapter\SQLite; use Utopia\Database\Attribute; @@ -81,7 +81,7 @@ public function testBlockFailureRollsBackTheMutation(): void $this->renameDocument($database, 'webhooks', 'hook', 'updated'); $this->fail('Document cache block failure was not propagated'); } catch (\RuntimeException $error) { - $this->assertStringContainsString('block document cache epoch', $error->getMessage()); + $this->assertStringContainsString('Unable to save cache snapshot', $error->getMessage()); } $this->assertSame('original', $database->getDocument('webhooks', 'hook')->getAttribute('name')); @@ -127,7 +127,7 @@ public function testActivationFailureDoesNotStrandOtherCollectionEpochs(): void ])); $this->assertSame('original', $database->getDocument('webhooks', 'hook')->getAttribute('name')); $this->assertSame('original', $database->getDocument('logs', 'log')->getAttribute('name')); - $cache->failActivations('collection:webhooks#epoch'); + $cache->failActivations('collection:webhooks#state'); $database->withTransaction(function () use ($database): void { $this->renameDocument($database, 'webhooks', 'hook', 'updated'); @@ -185,13 +185,13 @@ public function testCacheFlushDuringActivationDoesNotFailTheCommittedMutation(): $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); } - public function testCacheFlushAfterActivationReadsGenerationsDoesNotFailTheCommittedMutation(): void + public function testCacheFlushAfterActivationReadsVersionDoesNotFailTheCommittedMutation(): void { $cache = new FlushDuringActivationMemory(); $database = $this->createDatabaseWithCache($cache); $this->assertTrue($cache->flush()); $database->purgeCachedCollection('webhooks'); - $cache->flushAfterReading('collection:webhooks#finished'); + $cache->flushAfterReading('collection:webhooks#state'); $database->deleteCollection('webhooks'); $this->assertNull($database->findCollection('webhooks')); @@ -268,7 +268,7 @@ public function testOverlappingWritesSucceedOnACacheWithoutFields(): void $this->renameDocument($other, 'webhooks', 'hook', 'updated'); $this->fail('The other writer\'s block did not fail'); } catch (\RuntimeException $error) { - $this->assertStringContainsString('block document cache epoch', $error->getMessage()); + $this->assertStringContainsString('Unable to save cache snapshot', $error->getMessage()); } finally { $cache->failBlocks(false); } @@ -303,7 +303,7 @@ public function testAnOwnerReleaseFailureDoesNotFailTheCommittedWrite(): void '$id' => 'hook', 'name' => 'original', ])); - $cache->failFieldPurges(); + $cache->failCompletions(); $this->assertSame(1, $this->renameDocument($database, 'webhooks', 'hook', 'updated'), 'A stored write must not be reported as failed because its cache activation failed'); @@ -586,40 +586,24 @@ public function testARegistrationReleasedJustBeforeItsOwnReleaseFinishesTheWrite $this->assertSame(0, $adapter->documentReads, 'A registration another worker already released is not a failed release: the write activates and reads are cached again'); } - public function testAWriteWhoseOwnerRegistrationKeepsFailingStillBlocksReaders(): void - { - $cache = new FailingRegistrationCache(); - [$writer, $reader, $path] = $this->createSQLiteDatabasesSharing($cache); - - try { - $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); - $cache->failRegistrations(\PHP_INT_MAX); - - $writer->withTransaction(function () use ($writer, $reader): void { - $this->renameDocument($writer, 'users', 'user', 'updated'); - $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); - $this->assertSame('original', $reader->getDocument('users', 'user')->getAttribute('name')); - }); - - $this->assertSame('updated', $reader->getDocument('users', 'user')->getAttribute('name'), 'A write whose owner registration failed must still keep readers from caching what they read while it ran'); - } finally { - $this->removeSQLiteFiles($path); - } - } - - public function testAWriteWhoseOwnerRegistrationKeepsFailingReenablesTheCollection(): void + public function testUnregisterableWritesRollBackAndCachingRecovers(): void { $cache = new FailingRegistrationCache(); $adapter = new CountingMemory(); $database = $this->createCountedDatabase($adapter, $cache); - $cache->failRegistrations(\PHP_INT_MAX); - + $cache->failRegistrations(PHP_INT_MAX); + try { + $this->renameDocument($database, 'webhooks', 'hook', 'updated'); + $this->fail('A write must not proceed without registering its invalidation'); + } catch (\Utopia\Cache\Exception\Unavailable) { + $this->assertSame('original', UncachedTwin::of($database)->getDocument('webhooks', 'hook')->getAttribute('name')); + } + $cache->failRegistrations(0); $this->assertSame(1, $this->renameDocument($database, 'webhooks', 'hook', 'updated')); - $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); $adapter->reset(); $this->assertSame('updated', $database->getDocument('webhooks', 'hook')->getAttribute('name')); - $this->assertSame(0, $adapter->documentReads, 'A write that blocked its collection without an owner registration re-enables it once it activates'); + $this->assertSame(0, $adapter->documentReads); } public function testAFailedOwnerRegistrationIsRetried(): void @@ -736,6 +720,8 @@ final class FailDocumentEpochMemory extends MemoryCache private ?string $activationFailure = null; + private int $activationFailures = 0; + public function failBlocks(bool $failing = true): void { $this->failingBlocks = $failing; @@ -743,21 +729,24 @@ public function failBlocks(bool $failing = true): void public function failActivations(?string $key = null): void { - $this->activationFailure = $key ?? '#epoch'; + $this->activationFailure = $key ?? '#state'; + $this->activationFailures = 3; } #[\Override] public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array { - if (\str_ends_with($key, '#epoch') && \is_string($data)) { - if ($this->failingBlocks && \str_starts_with($data, 'blocked:')) { + if (\str_ends_with($key, '#state') && \is_array($data)) { + if ($this->failingBlocks && ! empty($data['owners'])) { return false; } if ( $this->activationFailure !== null + && $this->activationFailures > 0 && \str_contains($key, $this->activationFailure) - && ! \str_starts_with($data, 'blocked:') + && empty($data['owners']) ) { + $this->activationFailures--; return false; } } @@ -766,15 +755,10 @@ public function save(string $key, array|string $data, string $hash = '', int $tt } } -final class FlushDuringActivationMemory extends MemoryCache implements Leasable +final class FlushDuringActivationMemory extends MemoryCache { - /** @var array */ - private array $generations = []; - private bool $flushDuringActivation = false; - private bool $failDuringActivation = false; - private ?string $flushAfterReading = null; public function flushDuringActivation(): void @@ -793,60 +777,79 @@ public function flushAfterReading(string $key): void } #[\Override] - public function getGeneration(string $key): string + public function loadVersioned(string $key): array { - $generation = (string) ($this->generations[$key] ?? 0); + $snapshot = parent::loadVersioned($key); if ($this->flushAfterReading !== null && \str_ends_with($key, $this->flushAfterReading)) { $this->flushAfterReading = null; $this->flush(); } - - return $generation; + return $snapshot; } #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + $before = $this->load($key, PHP_INT_MAX); + if (\str_ends_with($key, '#state') && \is_array($before) && \is_array($data) + && \is_array($before['owners'] ?? null) && \is_array($data['owners'] ?? null) + && \count($data['owners']) < \count($before['owners'])) { + if ($this->flushDuringActivation) { + $this->flushDuringActivation = false; + $this->flush(); + } + if ($this->failDuringActivation) { + throw new \Utopia\Cache\Exception\Unavailable('Completion unavailable'); + } } - - return $this->save($key, $data, $hash); + return parent::compareAndSwap($key, $data, $hash, $version); } +} +/** + * A Redis-like cache that can abandon a write the way a killed worker does: the write stays registered and its + * collection stays blocked, because the release of its registration is refused and its activation stops there. + */ +final class AbandoningCache implements CacheAdapter, Versioned +{ #[\Override] - public function purge(string $key, string $hash = ''): bool + public function supportsVersioning(): bool { - if ($this->flushDuringActivation && \str_ends_with($key, '#finished')) { - $this->flushDuringActivation = false; - - return $this->flush(); - } - if ($this->failDuringActivation && \str_ends_with($key, '#finished')) { - return false; - } - - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - parent::purge($key, $hash); - return true; } - #[\Override] - public function flush(): bool + public function getVersion(string $key): string { - $this->generations = []; - - return parent::flush(); + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if (\str_ends_with($key, '#state') && \is_array($data)) { + $before = $this->cache->load($key, PHP_INT_MAX); + $removed = \is_array($before) && \is_array($before['owners'] ?? null) && \is_array($data['owners'] ?? null) ? \array_diff_key($before['owners'], $data['owners']) : []; + if ($removed !== [] && $this->abandoning) { + $this->abandoning = false; + $this->abandoned = (string) \array_key_first($removed); + } + if ($this->abandoned !== '' && \count($removed) === 1 && isset($removed[$this->abandoned])) { + throw new \Utopia\Cache\Exception\Unavailable('Completion unavailable'); + } + } + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); } -} -/** - * A Redis-like cache that can abandon a write the way a killed worker does: the write stays registered and its - * collection stays blocked, because the release of its registration is refused and its activation stops there. - */ -final class AbandoningCache implements CacheAdapter, Leasable -{ private RedisLeasableCache $cache; private bool $abandoning = false; @@ -928,17 +931,6 @@ public function getName(?string $key = null): string return 'abandoning'; } - #[\Override] - public function getGeneration(string $key): string - { - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - return $this->cache->saveWithLease($key, $data, $hash, $generation); - } } /** @@ -958,7 +950,7 @@ public function beforeNextSave(string $fragment, Closure $callback): void } #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false { $callback = $this->callback; if ($callback !== null && $this->fragment !== null && \str_contains($key, $this->fragment)) { @@ -966,6 +958,6 @@ public function save(string $key, array|string $data, string $hash = '', int $tt $callback(); } - return parent::save($key, $data, $hash); + return parent::compareAndSwap($key, $data, $hash, $version); } } diff --git a/tests/unit/Documents/DocumentCacheInvalidationTest.php b/tests/unit/Documents/DocumentCacheInvalidationTest.php index edc4aa627..5b39b1444 100644 --- a/tests/unit/Documents/DocumentCacheInvalidationTest.php +++ b/tests/unit/Documents/DocumentCacheInvalidationTest.php @@ -13,7 +13,7 @@ use Utopia\Cache\Adapter as CacheAdapter; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; use Utopia\Database\Adapter as DatabaseAdapter; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Adapter\SQLite; @@ -326,9 +326,36 @@ protected function documentKey(string $id, int|string|null $tenant = null): stri /** * @param Closure(): bool $refuses Whether the purge throws, asked once per purge */ - private function purgeRefusingCache(Closure $refuses): CacheAdapter&Leasable + private function purgeRefusingCache(Closure $refuses): CacheAdapter&Versioned { - return new class ($refuses) implements CacheAdapter, Leasable { + return new class ($refuses) implements CacheAdapter, Versioned { + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); + } + private RedisLeasableCache $cache; /** @@ -398,17 +425,6 @@ public function getName(?string $key = null): string return 'purge-failing'; } - #[\Override] - public function getGeneration(string $key): string - { - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - return $this->cache->saveWithLease($key, $data, $hash, $generation); - } }; } diff --git a/tests/unit/Documents/EvictionProtocolTest.php b/tests/unit/Documents/EvictionProtocolTest.php index 0da317173..bbed97185 100644 --- a/tests/unit/Documents/EvictionProtocolTest.php +++ b/tests/unit/Documents/EvictionProtocolTest.php @@ -28,9 +28,6 @@ final class EvictionProtocolTest extends TestCase #[\Override] protected function setUp(): void { - if (! \interface_exists(\Utopia\Cache\Feature\Versioned::class)) { - $this->markTestSkipped('The older-cache compatibility tier exercises the legacy protocol'); - } $this->redis = new Redis(); $this->redis->connect(\getenv('DATABASE_TEST_CACHE_HOST') ?: 'redis', (int) (\getenv('DATABASE_TEST_CACHE_PORT') ?: 6379)); $this->cache = new Cache(new RedisCache($this->redis)); @@ -53,16 +50,12 @@ protected function tearDown(): void $this->redis->close(); } - public function testLostLegacyCountersCannotStrandDocumentCaching(): void + public function testLostCollectionStateCannotStrandDocumentCaching(): void { $adapter = new CountingMemory(); $database = $this->database($adapter); $key = \strtolower($database->getCacheBaseKeys('users')[0]); - $this->cache->save($key.'#epoch', 'blocked:old@'.\time()); - for ($i = 0; $i < 6; $i++) { - $this->cache->purge($key.'#started'); - } - $this->redis->del($key.'#finished'); + $this->redis->del($key.'#state'); $this->cache->purge($database->getCacheBaseKeys(Database::METADATA, 'users')[1]); $this->assertSame('original', $database->getDocument('users', 'user')->getAttribute('name')); @@ -105,7 +98,7 @@ public function testQueryResultsRefilledAfterEvictionAreRetiredWhenTheWriterFini $this->redis->del(\strtolower($key.'#state')); $entry = $query->getEntry($scope, 'users', []); $this->assertNotNull($entry); - $this->assertTrue($query->set($entry, [new Document(['$id' => 'pre-commit'])], $query->getGeneration($entry))); + $this->assertTrue($query->set($entry, [new Document(['$id' => 'pre-commit'])], $query->getLease($entry))); $query->activateCollection($key, $token, 3600); $fresh = $query->getEntry($scope, 'users', []); $this->assertNotNull($fresh); diff --git a/tests/unit/Documents/GetDocumentFallbackTest.php b/tests/unit/Documents/GetDocumentFallbackTest.php index 17dcd9b8c..79f7844dd 100644 --- a/tests/unit/Documents/GetDocumentFallbackTest.php +++ b/tests/unit/Documents/GetDocumentFallbackTest.php @@ -8,7 +8,7 @@ use Tests\Unit\Support\StderrCapture; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; use Utopia\Database\Adapter\Memory; use Utopia\Database\Attribute; use Utopia\Database\Capability; @@ -66,7 +66,7 @@ public function testACacheThatCannotBeReadOrWrittenFallsBackToTheDatabase(): voi $this->assertInstanceOf(Document::class, $missing); $this->assertTrue($missing->isEmpty()); $this->assertStringContainsString('Warning: Failed to get document from cache: the cache refused load', $log); - $this->assertStringContainsString('Warning: Failed to get cache generation: the cache refused getGeneration', $log); + $this->assertStringContainsString('Warning: Failed to get cache version: the cache refused getVersion', $log); $this->assertSame(0, $fills(), 'A read without a lease must not populate the cache'); } @@ -98,7 +98,7 @@ public function capabilities(): array */ private function failingCache(): array { - $cache = new class (self::COLLECTION) extends MemoryCache implements Leasable { + $cache = new class (self::COLLECTION) extends MemoryCache implements Versioned { public bool $failing = false; public int $fills = 0; @@ -126,26 +126,11 @@ public function save(string $key, array|string $data, string $hash = '', int $tt } #[\Override] - public function getGeneration(string $key): string + public function getVersion(string $key): string { - $this->refuse('getGeneration', $key); + $this->refuse('getVersion', $key); - return '0'; - } - - /** - * @param array|string $data - * @return bool|string|array - */ - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->failing) { - $this->fills++; - } - $this->refuse('saveWithLease', $key); - - return parent::save($key, $data, $hash); + return parent::getVersion($key); } private function refuse(string $operation, string $key): void diff --git a/tests/unit/Documents/InvalidationRestoreFailureTest.php b/tests/unit/Documents/InvalidationRestoreFailureTest.php index 065460662..9848cd92d 100644 --- a/tests/unit/Documents/InvalidationRestoreFailureTest.php +++ b/tests/unit/Documents/InvalidationRestoreFailureTest.php @@ -77,7 +77,7 @@ public function __construct(private readonly \Closure $failing) #[\Override] public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array { - if (($this->failing)() && \str_ends_with($key, '#epoch') && \is_string($data) && ! \str_starts_with($data, 'blocked:')) { + if (($this->failing)() && \str_ends_with($key, '#state') && \is_array($data) && empty($data['owners'])) { throw new RuntimeException('the cache refused the epoch'); } diff --git a/tests/unit/Documents/MetadataTenantInvalidationTest.php b/tests/unit/Documents/MetadataTenantInvalidationTest.php index ff3c4d2f8..7bd540ac7 100644 --- a/tests/unit/Documents/MetadataTenantInvalidationTest.php +++ b/tests/unit/Documents/MetadataTenantInvalidationTest.php @@ -2,11 +2,9 @@ namespace Tests\Unit\Documents; -use Override; use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Attribute; use Utopia\Database\Collection; @@ -86,7 +84,7 @@ public function testACollectionIsReadableByItsOwnWriterAfterCreation(): void public function testACollectionIsReadableAfterCreationWithoutSharedTables(): void { $adapter = new DatabaseMemory(); - $cache = new Cache(new LeasedMemoryCacheAdapter()); + $cache = new Cache(new MemoryCache()); $namespace = '_'.\uniqid(); $databases = []; @@ -141,7 +139,7 @@ private function attributesOf(Database $database): array private function databases(): array { $adapter = new DatabaseMemory(); - $cache = new Cache(new LeasedMemoryCacheAdapter()); + $cache = new Cache(new MemoryCache()); $namespace = 'metadata_tenant_'.\uniqid(); $databases = []; @@ -162,45 +160,3 @@ private function databases(): array return [$databases[0], $databases[1]]; } } - -final class LeasedMemoryCacheAdapter extends MemoryCache implements Leasable -{ - /** @var array */ - private array $generations = []; - - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - /** - * @param array|string $data - * @return bool|string|array - */ - #[Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; - } - - return $this->save($key, $data, $hash); - } - - #[Override] - public function purge(string $key, string $hash = ''): bool - { - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - - return parent::purge($key, $hash); - } - - #[Override] - public function flush(): bool - { - $this->generations = []; - - return parent::flush(); - } -} diff --git a/tests/unit/Documents/NegativeCacheEpochTest.php b/tests/unit/Documents/NegativeCacheEpochTest.php index 19273d951..b8ae8d87d 100644 --- a/tests/unit/Documents/NegativeCacheEpochTest.php +++ b/tests/unit/Documents/NegativeCacheEpochTest.php @@ -8,7 +8,6 @@ use Tests\Unit\Support\CountingMemory; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; use Utopia\Database\Attribute; use Utopia\Database\Collection; use Utopia\Database\Database; @@ -78,7 +77,7 @@ private function hook(): Document private function createDatabase(InterceptingMemory $adapter): Database { - $database = new Database($adapter, new Cache(new LeasedMemoryCache())); + $database = new Database($adapter, new Cache(new MemoryCache())); $database ->setDatabase('utopiaTests') ->setNamespace('negative_cache_'.\uniqid()); @@ -134,49 +133,3 @@ public function getDocument(Document $collection, string $id, array $queries = [ return $document; } } - -/** - * A memory cache with the generations of the Leasable contract, so a save that - * raced a purge is refused as it is on Redis. - */ -final class LeasedMemoryCache extends MemoryCache implements Leasable -{ - /** @var array */ - private array $generations = []; - - #[\Override] - public function getGeneration(string $key): string - { - return (string) ($this->generations[$key] ?? 0); - } - - /** - * @param array|string $data - * @return bool|string|array - */ - #[Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->getGeneration($key) !== $generation) { - return false; - } - - return $this->save($key, $data, $hash); - } - - #[Override] - public function purge(string $key, string $hash = ''): bool - { - $this->generations[$key] = ($this->generations[$key] ?? 0) + 1; - - return parent::purge($key, $hash); - } - - #[Override] - public function flush(): bool - { - $this->generations = []; - - return parent::flush(); - } -} diff --git a/tests/unit/Documents/TransactionCacheReadTest.php b/tests/unit/Documents/TransactionCacheReadTest.php index 65caa35ae..596e019e5 100644 --- a/tests/unit/Documents/TransactionCacheReadTest.php +++ b/tests/unit/Documents/TransactionCacheReadTest.php @@ -9,7 +9,7 @@ use Tests\Unit\Support\CountingMemory; use Utopia\Cache\Adapter as CacheAdapter; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; use Utopia\Database\Adapter as DatabaseAdapter; use Utopia\Database\Adapter\SQLite; use Utopia\Database\Attribute; @@ -294,9 +294,36 @@ private function library(string $id, string $prefix): Document /** * @param Closure(): void $onFill Called on every save to the cache */ - private function fillCountingCache(Closure $onFill): CacheAdapter&Leasable + private function fillCountingCache(Closure $onFill): CacheAdapter&Versioned { - return new class ($onFill) implements CacheAdapter, Leasable { + return new class ($onFill) implements CacheAdapter, Versioned { + #[\Override] + public function supportsVersioning(): bool + { + return true; + } + #[\Override] + public function getVersion(string $key): string + { + return $this->cache->getVersion($key); + } + #[\Override] + public function loadVersioned(string $key): array + { + return ['version' => $this->getVersion($key), 'value' => $this->load($key, PHP_INT_MAX)]; + } + #[\Override] + public function compareAndSwap(string $key, array|string $data, string $hash, string $version): string|false + { + if ($this->getVersion($key) !== $version) { + return false; + } + if ($this->save($key, $data, $hash) === false) { + throw new \Utopia\Cache\Exception\Unavailable('Cache write failed'); + } + return $this->cache->getVersion($key); + } + private RedisLeasableCache $cache; /** @@ -364,19 +391,6 @@ public function getName(?string $key = null): string return 'fill-counting'; } - #[\Override] - public function getGeneration(string $key): string - { - return $this->cache->getGeneration($key); - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - ($this->onFill)(); - - return $this->cache->saveWithLease($key, $data, $hash, $generation); - } }; } diff --git a/tests/unit/Documents/WithCacheTest.php b/tests/unit/Documents/WithCacheTest.php index 9938d4c76..4bf8b6655 100644 --- a/tests/unit/Documents/WithCacheTest.php +++ b/tests/unit/Documents/WithCacheTest.php @@ -8,7 +8,7 @@ use Tests\Unit\Support\StderrCapture; use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; +use Utopia\Cache\Feature\Versioned; use Utopia\Database\Adapter\Memory; use Utopia\Database\Attribute; use Utopia\Database\Capability; @@ -111,7 +111,7 @@ public static function cacheFailures(): array 'the epoch is not a string' => ['listEpoch', ''], 'the value cannot be read' => ['loadValue', 'Warning: Failed to load cache value: the cache refused loadValue'], 'a rejected value cannot be purged' => ['purge', 'Warning: Failed to purge rejected cache value: the cache refused purge'], - 'the generation cannot be read' => ['getGeneration', 'Warning: Failed to get cache generation: the cache refused getGeneration'], + 'the generation cannot be read' => ['getVersion', 'Warning: Failed to get cache version: the cache refused getVersion'], 'the value cannot be written' => ['saveWithLease', 'Warning: Failed to save cache value: the cache refused saveWithLease'], ]; } @@ -136,7 +136,7 @@ public function testACacheFailureFallsBackToTheCallback(string $failure, string $this->assertSame('computed', $value); $this->assertSame(1, $this->calls); - if ($failure === 'getGeneration') { + if ($failure === 'getVersion') { StderrCapture::during(function () use ($database): void { $this->cached($database, fn (): string => 'newer'); }); @@ -164,7 +164,7 @@ private function cached(Database $database, callable $callback): mixed private function failingCache(string $failure): MemoryCache { - return new class ($failure) extends MemoryCache implements Leasable { + return new class ($failure) extends MemoryCache implements Versioned { public function __construct(private readonly string $failure) { } @@ -191,6 +191,10 @@ public function save(string $key, array|string $data, string $hash = '', int $tt return false; } + if ($this->failure === 'saveWithLease' && \str_starts_with($key, 'reports:summary#') && ! \str_ends_with($key, '#epoch')) { + throw new RuntimeException('the cache refused saveWithLease'); + } + return parent::save($key, $data, $hash); } @@ -205,28 +209,15 @@ public function purge(string $key, string $hash = ''): bool } #[\Override] - public function getGeneration(string $key): string + public function getVersion(string $key): string { - if ($this->failure === 'getGeneration' && \str_starts_with($key, 'reports:summary#')) { - throw new RuntimeException('the cache refused getGeneration'); + if ($this->failure === 'getVersion' && \str_starts_with($key, 'reports:summary#')) { + throw new RuntimeException('the cache refused getVersion'); } - return '0'; + return parent::getVersion($key); } - /** - * @param array|string $data - * @return bool|string|array - */ - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if ($this->failure === 'saveWithLease' && \str_starts_with($key, 'reports:summary#')) { - throw new RuntimeException('the cache refused saveWithLease'); - } - - return parent::save($key, $data, $hash); - } }; } diff --git a/tests/unit/WithCacheLeaseTest.php b/tests/unit/WithCacheLeaseTest.php index d59de806c..ae108fe0b 100644 --- a/tests/unit/WithCacheLeaseTest.php +++ b/tests/unit/WithCacheLeaseTest.php @@ -6,7 +6,6 @@ use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter; use Utopia\Cache\Cache; -use Utopia\Cache\Feature\Leasable; use Utopia\Database\Adapter\Memory as DatabaseMemory; use Utopia\Database\Attribute; use Utopia\Database\Collection; @@ -29,7 +28,7 @@ class WithCacheLeaseTest extends TestCase #[\Override] protected function setUp(): void { - $this->cache = new Cache(new LeasableMemoryCache()); + $this->cache = new Cache(new \Utopia\Cache\Adapter\Memory()); $this->adapter = new DatabaseMemory(); $this->database = new Database($this->adapter, $this->cache); $this->database @@ -103,7 +102,7 @@ public function getDocument(Document $collection, string $id, array $queries = [ return $document; } }; - $database = new Database($adapter, new Cache(new LeasableMemoryCache())); + $database = new Database($adapter, new Cache(new \Utopia\Cache\Adapter\Memory())); $database ->setDatabase('utopiaTests') ->setNamespace('with_cache_in_flight_' . \uniqid()); @@ -182,141 +181,3 @@ public function testListWriteLandsWhenNoConcurrentPurge(): void $this->assertSame(0, $callbackCalls); } } - -class LeasableMemoryCache implements Adapter, Leasable -{ - private const string GENERATION_FIELD = '__utopia_gen__'; - - /** - * @var array|string}>> - */ - private array $store = []; - - #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - if ($hash === '') { - $hash = $key; - } - - if (! isset($this->store[$key][$hash])) { - return false; - } - - $saved = $this->store[$key][$hash]; - - return ($saved['time'] + $ttl > \time()) ? $saved['data'] : false; - } - - #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array - { - if (empty($key) || empty($data)) { - return false; - } - - if ($hash === '') { - $hash = $key; - } - - if ($hash === self::GENERATION_FIELD) { - return false; - } - - $this->store[$key][$hash] = ['time' => \time(), 'data' => $data]; - - return $data; - } - - #[\Override] - public function getGeneration(string $key): string - { - $generation = $this->store[$key][self::GENERATION_FIELD]['data'] ?? '0'; - - return \is_string($generation) ? $generation : '0'; - } - - #[\Override] - public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array - { - if (empty($key) || empty($data)) { - return false; - } - - if ($this->getGeneration($key) !== $generation) { - return false; - } - - return $this->save($key, $data, $hash); - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - if ($hash === '') { - $hash = $key; - } - - if (! isset($this->store[$key][$hash])) { - return false; - } - - $this->store[$key][$hash]['time'] = \time(); - - return true; - } - - /** - * @return string[] - */ - #[\Override] - public function list(string $key): array - { - return \array_values(\array_filter( - \array_keys($this->store[$key] ?? []), - fn (string $field): bool => $field !== self::GENERATION_FIELD - )); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - $generation = (string) (((int) $this->getGeneration($key)) + 1); - - if ($hash !== '' && $hash !== self::GENERATION_FIELD) { - unset($this->store[$key][$hash]); - } else { - $this->store[$key] = []; - } - - $this->store[$key][self::GENERATION_FIELD] = ['time' => \time(), 'data' => $generation]; - - return true; - } - - #[\Override] - public function flush(): bool - { - $this->store = []; - - return true; - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return \count($this->store); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'leasable-memory'; - } -} From 8389a6e5708ce748ec704b5c3aacfdb5328880c1 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:14:42 +0000 Subject: [PATCH 6/8] test(cache): match the CAS facade signature in flush race --- tests/unit/Cache/QueryTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/Cache/QueryTest.php b/tests/unit/Cache/QueryTest.php index c214f9a40..9a98c7f75 100644 --- a/tests/unit/Cache/QueryTest.php +++ b/tests/unit/Cache/QueryTest.php @@ -980,14 +980,14 @@ public function flushOnNextCompletion(): void } #[\Override] - public function compareAndSwap(string $key, mixed $data, string $version, string $hash = ''): string|false + public function compareAndSwap(string $key, array|string $data, string $version): string|false { if ($this->armed && \str_ends_with($key, '#state')) { $this->armed = false; $this->flush(); } - return parent::compareAndSwap($key, $data, $version, $hash); + return parent::compareAndSwap($key, $data, $version); } }; $queryCache = new ResultCache($cache); From 77990f1f1747623ee59ea51379bb8a60fa52a218 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:16:02 +0000 Subject: [PATCH 7/8] test(cache): migrate callback fixtures to the versioned contract --- .github/workflows/claude.yml | 40 ------- tests/unit/SwooleAbsentTest.php | 2 +- tests/unit/WithCacheLeaseTest.php | 4 +- tests/unit/WithCacheTest.php | 189 +----------------------------- 4 files changed, 9 insertions(+), 226 deletions(-) delete mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index 0dd9e1188..000000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,40 +0,0 @@ -# name: Claude - -# on: -# pull_request: -# types: [opened, synchronize, ready_for_review, reopened] -# pull_request_review: -# types: [submitted] -# issue_comment: -# types: [created] -# pull_request_review_comment: -# types: [created] -# issues: -# types: [opened, assigned] -# workflow_run: -# workflows: [Tests] -# types: [completed] - -# jobs: -# claude: -# # Caller must grant the union of every permission the callee's jobs ask -# # for; reusable workflows can't exceed the caller's ceiling. -# permissions: -# contents: write -# pull-requests: write -# issues: write -# actions: read -# id-token: write -# uses: abnegate/claude-pr-owner/.github/workflows/orchestrator.yml@7138fd26ee7317ac1f05001eafeb41d371ba58a6 # v0.6.0 -# secrets: -# oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} -# # PAT with `contents: write` on this repo. Without this, commits pushed -# # by the consolidator (authenticated with GITHUB_TOKEN) do not trigger -# # workflow_run/synchronize events, so Tests/CodeQL/Linter stay dormant -# # on Claude's fixes. -# push_token: ${{ secrets.CLAUDE_PUSH_TOKEN }} -# with: -# # improvement: true -# # healing: true -# bots: true -# comments: true diff --git a/tests/unit/SwooleAbsentTest.php b/tests/unit/SwooleAbsentTest.php index 6a3e79177..b78121b61 100644 --- a/tests/unit/SwooleAbsentTest.php +++ b/tests/unit/SwooleAbsentTest.php @@ -61,7 +61,7 @@ public function testEveryClassLoadsWithoutSwoole(): void $directory = (string) \ini_get('extension_dir'); foreach (['pdo_mysql', 'pdo_pgsql', 'mongodb', 'redis'] as $extension) { if (\is_file($directory.'/'.$extension.'.so')) { - $flags[] = '-d extension='.$extension; + $flags[] = '-d extension='.\escapeshellarg($directory.'/'.$extension.'.so'); } } diff --git a/tests/unit/WithCacheLeaseTest.php b/tests/unit/WithCacheLeaseTest.php index ae108fe0b..6d7f14a3b 100644 --- a/tests/unit/WithCacheLeaseTest.php +++ b/tests/unit/WithCacheLeaseTest.php @@ -28,7 +28,7 @@ class WithCacheLeaseTest extends TestCase #[\Override] protected function setUp(): void { - $this->cache = new Cache(new \Utopia\Cache\Adapter\Memory()); + $this->cache = new Cache(new \Tests\Unit\Cache\RedisLeasableCache()); $this->adapter = new DatabaseMemory(); $this->database = new Database($this->adapter, $this->cache); $this->database @@ -102,7 +102,7 @@ public function getDocument(Document $collection, string $id, array $queries = [ return $document; } }; - $database = new Database($adapter, new Cache(new \Utopia\Cache\Adapter\Memory())); + $database = new Database($adapter, new Cache(new \Tests\Unit\Cache\RedisLeasableCache())); $database ->setDatabase('utopiaTests') ->setNamespace('with_cache_in_flight_' . \uniqid()); diff --git a/tests/unit/WithCacheTest.php b/tests/unit/WithCacheTest.php index cb86c052e..357bf84ec 100644 --- a/tests/unit/WithCacheTest.php +++ b/tests/unit/WithCacheTest.php @@ -851,196 +851,19 @@ private function storeCacheValue(HashMemoryCache $cache, string $key, string $ha } } -class HashMemoryCache implements Adapter +class HashMemoryCache extends \Tests\Unit\Cache\RedisLeasableCache { - /** - * @var array|string}>> - */ - private array $store = []; - - #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - $hash = $hash === '' ? $key : $hash; - $saved = $this->store[$key][$hash] ?? null; - if ($saved === null) { - return false; - } - - return ($saved['time'] + $ttl > \time()) ? $saved['data'] : false; - } - - #[\Override] - public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array - { - if ($key === '' || empty($data)) { - return false; - } - - $hash = $hash === '' ? $key : $hash; - $this->store[$key][$hash] = [ - 'time' => \time(), - 'data' => $data, - ]; - - return $data; - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - $hash = $hash === '' ? $key : $hash; - if (!isset($this->store[$key][$hash])) { - return false; - } - - $this->store[$key][$hash]['time'] = \time(); - - return true; - } - - /** - * @return array - */ - #[\Override] - public function list(string $key): array - { - return \array_keys($this->store[$key] ?? []); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - if ($hash !== '') { - unset($this->store[$key][$hash]); - return true; - } - - unset($this->store[$key]); - - return true; - } - - #[\Override] - public function flush(): bool - { - $this->store = []; - - return true; - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return \count($this->store); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'hash-memory'; - } } -class JsonHashMemoryCache implements Adapter +class JsonHashMemoryCache extends HashMemoryCache { - /** - * @var array> - */ - private array $store = []; - - #[\Override] - public function load(string $key, int $ttl, string $hash = ''): mixed - { - $hash = $hash === '' ? $key : $hash; - $saved = $this->store[$key][$hash] ?? null; - if ($saved === null || $saved['time'] + $ttl <= \time()) { - return false; - } - - return \json_decode($saved['data'], true); - } - #[\Override] public function save(string $key, array|string $data, string $hash = '', int $ttl = 0): bool|string|array { - if ($key === '' || empty($data)) { - return false; - } - - $hash = $hash === '' ? $key : $hash; - $this->store[$key][$hash] = [ - 'time' => \time(), - 'data' => \json_encode($data) ?: '', - ]; - - return $data; - } - - #[\Override] - public function touch(string $key, string $hash = ''): bool - { - $hash = $hash === '' ? $key : $hash; - if (!isset($this->store[$key][$hash])) { - return false; + $decoded = \json_decode(\json_encode($data, JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR); + if (! \is_array($decoded) && ! \is_string($decoded)) { + throw new \RuntimeException('Invalid serialized cache value'); } - - $this->store[$key][$hash]['time'] = \time(); - - return true; - } - - /** - * @return array - */ - #[\Override] - public function list(string $key): array - { - return \array_keys($this->store[$key] ?? []); - } - - #[\Override] - public function purge(string $key, string $hash = ''): bool - { - if ($hash !== '') { - unset($this->store[$key][$hash]); - return true; - } - - unset($this->store[$key]); - - return true; - } - - #[\Override] - public function flush(): bool - { - $this->store = []; - - return true; - } - - #[\Override] - public function ping(): bool - { - return true; - } - - #[\Override] - public function getSize(): int - { - return \count($this->store); - } - - #[\Override] - public function getName(?string $key = null): string - { - return 'json-hash-memory'; + return parent::save($key, $decoded, $hash, $ttl); } } From c0c72c75a5c47730759ddac169973826e0c05fd5 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:00:11 +0000 Subject: [PATCH 8/8] build: require stable cache 6.0 --- composer.json | 2 +- composer.lock | 19 +++++-------------- docs/cache-invalidation-state.md | 2 +- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/composer.json b/composer.json index b3e271ac4..c393ab2aa 100755 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "ext-redis": "*", "utopia-php/validators": "^1.0 || ^2.0", "utopia-php/console": "^0.1 || ^0.2", - "utopia-php/cache": "dev-codex/eviction-safe-cache-leases as 6.0.999", + "utopia-php/cache": "^6.0", "utopia-php/pools": "^2.0", "utopia-php/mongo": "^1.5.4", "utopia-php/query": "^0.7.2", diff --git a/composer.lock b/composer.lock index 26fce1b8f..2243fae73 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3e2516143b11dd8164159b23250268b5", + "content-hash": "e95611681420d2fa5709ac90a368e813", "packages": [ { "name": "brick/math", @@ -2007,7 +2007,7 @@ }, { "name": "utopia-php/cache", - "version": "dev-codex/eviction-safe-cache-leases", + "version": "6.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", @@ -2061,7 +2061,7 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/codex/eviction-safe-cache-leases" + "source": "https://github.com/utopia-php/cache/tree/6.0.0" }, "time": "2026-10-30T21:12:57+00:00" }, @@ -5224,18 +5224,9 @@ "time": "2026-10-08T14:16:57+00:00" } ], - "aliases": [ - { - "package": "utopia-php/cache", - "version": "dev-codex/eviction-safe-cache-leases", - "alias": "6.0.999", - "alias_normalized": "6.0.999.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/cache": 20 - }, + "stability-flags": {}, "prefer-stable": true, "prefer-lowest": false, "platform": { diff --git a/docs/cache-invalidation-state.md b/docs/cache-invalidation-state.md index e4039a252..c8491ccb9 100644 --- a/docs/cache-invalidation-state.md +++ b/docs/cache-invalidation-state.md @@ -4,7 +4,7 @@ The collection's CAS version is its epoch. Its single evictable record contains The same opaque versions fence document, definition and query fills. CAS returns the exact identity it committed, or false for a version conflict. Errors and open circuit breakers throw; they are never treated as eviction. Expiry checks use the caller's writer timeout; a reader with a shorter timeout does not delete another caller's live registration. Later writer completion still retires fills made after its timeout. -This requires the new cache major release (6.x); the old generation API and compatibility protocol are removed. Redis, Redis multiplexing and Memory support versioning. Database document/query/withCache caching is bypassed on unsupported adapters; ordinary database operations continue. The old cache-5 compatibility CI tier is removed because it is no longer a supported dependency. The temporary development dependency must be replaced with the stable major release before production. +This requires cache 6.x; the old generation API and compatibility protocol are removed. Redis, Redis multiplexing and Memory support versioning. Database document/query/withCache caching is bypassed on unsupported adapters; ordinary database operations continue. The old cache-5 compatibility CI tier is removed because it is no longer a supported dependency. No additional Dragonfly or coordination DSN is needed. Drain and stop legacy API/worker/task/migration writers, clear disposable cache data, deploy matching library versions everywhere and resume. Do not mix protocols. Rollback requires the same drain/clear procedure.