Nodarx/provider_imap
1
0
Fork 0
generated from Nodarx/template
Code Issues 1 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

feat: track cache changes with sequence numbers and tombstones

Browse Source 
Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>
This commit is contained in:
Sebastian Krupinski
2026-10-07 21:21:14 -04:00
parent c20dde6633
commit eea9b5630a
10 changed files with 511 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/php/Unit/CacheSerializationTest.php
+4 -2
View File
@@ -128,16 +128,18 @@ final class CacheSerializationTest extends TestCase
->with(
['sid' => 'svc', 'mailbox' => 'INBOX', 'uidValidity' => 7, 'uid' => 42],
$this->callback(function (array $update): bool {
$this->assertSame(['$set'], array_keys($update));
$this->assertSame(['$set', '$setOnInsert'], array_keys($update));
$this->assertSame('tenant', $update['$set']['tid']);
$this->assertSame(7, $update['$set']['uidValidity']);
$this->assertSame(5, $update['$set']['modSeq']);
$this->assertSame(['addedSeq' => 5], $update['$setOnInsert']);
$this->assertArrayNotHasKey('blobs', $update['$set']);
return true;
}),
['upsert' => true],
);
(new MessageStore($this->dataStore($collection)))->upsert('tenant', 'svc', 7, $this->entity());
(new MessageStore($this->dataStore($collection)))->upsert('tenant', 'svc', 7, $this->entity(), 5);
}
public function testMailboxStoreUpsertLeavesHarmonizationState(): void
tests/php/Unit/MessageHarmonizationTest.php
+149 -9
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use Generator;
use KTXF\Resource\Delta\Delta;
use KTXF\Resource\Filter\IFilter;
use KTXF\Resource\Sort\ISort;
use KTXC\Db\DataStore;
@@ -15,6 +16,7 @@ use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
use KTXM\ProviderImap\Service\Cache\MessageDeltaService;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
@@ -29,6 +31,7 @@ final class MessageHarmonizationTest extends TestCase
private FakeMessageFileStore $files;
private MessageHarmonizationLiveStub $live;
private HarmonizationService $harmonizer;
private MessageDeltaService $deltas;
protected function setUp(): void
{
@@ -46,8 +49,9 @@ final class MessageHarmonizationTest extends TestCase
$this->mailboxes,
$this->messages,
$this->files,
new MessageIngestor($this->files, $this->messages),
new MessageIngestor($this->files, $this->messages, $this->mailboxes),
))->for($service, $this->live);
$this->deltas = new MessageDeltaService($this->mailboxes, $this->messages);
$this->mailboxes->upsert('tenant', 'svc', (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', [])));
}
@@ -159,6 +163,87 @@ final class MessageHarmonizationTest extends TestCase
$this->assertNull($this->mailboxes->lockOwner);
}
public function testEmptySignatureReturnsTheCurrentSignature(): void
{
$this->live->flags = [1 => [], 2 => [], 3 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$delta = $this->deltas->delta('svc', 'INBOX', '');
$this->assertSame('7:3', $delta->signature);
$this->assertSame([[], [], []], $this->changeLists($delta));
}
public function testEmptySignatureWhileIncompleteStartsFromZero(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->live->unfetchable = [2];
$this->harmonizer->harmonizeMessages('INBOX');
$delta = $this->deltas->delta('svc', 'INBOX', '');
$this->assertSame('7:0', $delta->signature);
$this->assertSame([['1'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:0')));
}
public function testDeltaReportsAdditionsModificationsAndDeletions(): void
{
$this->live->flags = [1 => [], 2 => [], 3 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$signature = $this->deltas->delta('svc', 'INBOX', '')->signature;
$this->live->flags = [2 => ['\\Seen'], 3 => [], 4 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$delta = $this->deltas->delta('svc', 'INBOX', $signature);
$this->assertSame([['4'], ['2'], ['1']], $this->changeLists($delta));
$this->assertSame([[], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $delta->signature)));
}
public function testMessagesAddedSinceTheSignatureAreOnlyAdditions(): void
{
$this->live->flags = [1 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$signature = $this->deltas->delta('svc', 'INBOX', '')->signature;
$this->live->flags = [1 => [], 2 => [], 3 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$this->live->flags = [1 => [], 2 => ['\\Flagged']];
$this->harmonizer->harmonizeMessages('INBOX');
// 2: added then flagged -> addition; 3: added then expunged -> not reported
$this->assertSame([['2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $signature)));
}
public function testSignatureThatCannotBeAnsweredIsReset(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$old = $this->deltas->delta('svc', 'INBOX', '')->signature;
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', 'garbage')));
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:999')));
$this->mailboxes->updateState('svc', 'INBOX', ['purgedSeq' => 2]);
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:1')));
$this->live->uidValidity = 8;
$this->live->flags = [5 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$delta = $this->deltas->delta('svc', 'INBOX', $old);
$this->assertSame([['5'], [], []], $this->changeLists($delta));
$this->assertStringStartsWith('8:', $delta->signature);
}
public function testMailboxThatIsNotCachedHasAnEmptyDelta(): void
{
$delta = $this->deltas->delta('svc', 'Unknown', '7:1');
$this->assertSame('', $delta->signature);
$this->assertSame([[], [], []], $this->changeLists($delta));
}
public function testLockIsReleasedWhenTheRunFails(): void
{
$this->live->uidValidity = null;
@@ -171,6 +256,18 @@ final class MessageHarmonizationTest extends TestCase
$this->assertNull($this->mailboxes->lockOwner);
}
/**
* @return array{0: string[], 1: string[], 2: string[]}
*/
private function changeLists(Delta $delta): array
{
return [
json_decode(json_encode($delta->additions), true),
json_decode(json_encode($delta->modifications), true),
json_decode(json_encode($delta->deletions), true),
];
}
}
final class MessageHarmonizationLiveStub extends LiveMailService
@@ -272,37 +369,80 @@ final class FakeMailboxStore extends MailboxStore
$this->lockOwner = null;
}
}
public function reserveSequence(string $serviceId, string $name, int $count = 1): int
{
$this->documents[$name]['changeSeq'] += $count;
return $this->documents[$name]['changeSeq'] - $count + 1;
}
public function state(string $serviceId, string $name): array
{
return array_replace(self::STATE_DEFAULTS, array_intersect_key($this->documents[$name] ?? [], self::STATE_DEFAULTS));
}
}
final class FakeMessageStore extends MessageStore
{
/** @var array<int, array<int, list<string>>> flags by UIDVALIDITY and UID */
/** @var array<int, array<int, array{flags: list<string>, addedSeq: int, modSeq: int, removedSeq: ?int}>> by UIDVALIDITY and UID */
private array $documents = [];
public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity): void
public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity, int $seq): void
{
$meta = $entity->toCacheMeta();
$this->documents[$uidValidity][$meta['uid']] = $meta['flags'];
$existing = $this->documents[$uidValidity][$meta['uid']] ?? null;
$this->documents[$uidValidity][$meta['uid']] = [
'flags' => $meta['flags'],
'addedSeq' => $existing['addedSeq'] ?? $seq,
'modSeq' => $seq,
'removedSeq' => null,
];
ksort($this->documents[$uidValidity]);
}
public function flags(string $serviceId, string $mailbox, int $uidValidity): array
{
return $this->documents[$uidValidity] ?? [];
$flags = [];
foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) {
if ($document['removedSeq'] === null) {
$flags[$uid] = $document['flags'];
}
}
return $flags;
}
public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags): void
public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags, int $seq): void
{
$this->documents[$uidValidity][$uid] = array_values($flags);
$this->documents[$uidValidity][$uid]['flags'] = array_values($flags);
$this->documents[$uidValidity][$uid]['modSeq'] = $seq;
}
public function delete(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
public function tombstone(string $serviceId, string $mailbox, int $uidValidity, int $seq, int ...$uids): void
{
foreach ($uids as $uid) {
unset($this->documents[$uidValidity][$uid]);
$this->documents[$uidValidity][$uid]['removedSeq'] = $seq;
$this->documents[$uidValidity][$uid]['modSeq'] = $seq;
$this->documents[$uidValidity][$uid]['flags'] = [];
}
}
public function changes(string $serviceId, string $mailbox, int $uidValidity, int $since): array
{
$changes = ['additions' => [], 'modifications' => [], 'deletions' => []];
foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) {
if ($document['removedSeq'] !== null) {
if ($document['addedSeq'] <= $since && $document['removedSeq'] > $since) {
$changes['deletions'][] = $uid;
}
} elseif ($document['addedSeq'] > $since) {
$changes['additions'][] = $uid;
} elseif ($document['modSeq'] > $since) {
$changes['modifications'][] = $uid;
}
}
return $changes;
}
public function deleteByMailbox(string $serviceId, string $mailbox): void
{
$this->documents = [];
tests/php/Unit/MessageIngestorTest.php
+9 -6
View File
@@ -7,6 +7,7 @@ namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\MessageProperties;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
use KTXM\ProviderImap\Stores\MessageStore;
use PHPUnit\Framework\TestCase;
@@ -26,20 +27,22 @@ final class MessageIngestorTest extends TestCase
);
$messageStore = $this->createStub(MessageStore::class);
$messageStore->method('upsert')->willReturnCallback(
function (string $tid, string $sid, int $uidValidity, EntityResource $entity) use (&$calls): void {
$calls[] = ['meta', $tid, $sid, $entity->collection(), $uidValidity, $entity->identifier()];
function (string $tid, string $sid, int $uidValidity, EntityResource $entity, int $seq) use (&$calls): void {
$calls[] = ['meta', $tid, $sid, $entity->collection(), $uidValidity, $entity->identifier(), $seq];
},
);
$mailboxStore = $this->createStub(MailboxStore::class);
$mailboxStore->method('reserveSequence')->willReturn(10);
$ingested = (new MessageIngestor($fileStore, $messageStore))
$ingested = (new MessageIngestor($fileStore, $messageStore, $mailboxStore))
->ingest('tenant', 'svc', 7, $this->entity(41), $this->entity(42));
$this->assertSame([41, 42], $ingested);
$this->assertSame([
['file', 'tenant', 'svc', 'INBOX', 7, 41, EntityResource::CACHE_SCHEMA_VERSION],
['meta', 'tenant', 'svc', 'INBOX', 7, 41],
['meta', 'tenant', 'svc', 'INBOX', 7, 41, 10],
['file', 'tenant', 'svc', 'INBOX', 7, 42, EntityResource::CACHE_SCHEMA_VERSION],
['meta', 'tenant', 'svc', 'INBOX', 7, 42],
['meta', 'tenant', 'svc', 'INBOX', 7, 42, 11],
], $calls);
}
@@ -51,7 +54,7 @@ final class MessageIngestorTest extends TestCase
$messageStore->expects($this->never())->method('upsert');
$this->expectException(RuntimeException::class);
(new MessageIngestor($fileStore, $messageStore))->ingest('tenant', 'svc', 7, $this->entity(41));
(new MessageIngestor($fileStore, $messageStore, $this->createStub(MailboxStore::class)))->ingest('tenant', 'svc', 7, $this->entity(41));
}
private function entity(int $uid): EntityResource
tests/php/Unit/MessageStoreChangesTest.php
+92
View File
@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use ArrayIterator;
use KTXC\Db\Collection;
use KTXC\Db\Cursor;
use KTXC\Db\DataStore;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageStore;
use MongoDB\Collection as MongoCollection;
use MongoDB\Operation\FindOneAndUpdate;
use PHPUnit\Framework\TestCase;
final class MessageStoreChangesTest extends TestCase
{
public function testChangesQueriesAdditionsModificationsAndDeletions(): void
{
$filters = [];
$collection = $this->createStub(Collection::class);
$collection->method('find')->willReturnCallback(function (array $filter) use (&$filters): Cursor {
$filters[] = $filter;
return new Cursor(new ArrayIterator([['uid' => count($filters)]]));
});
$changes = (new MessageStore($this->dataStore($collection)))->changes('svc', 'INBOX', 7, 10);
$key = ['sid' => 'svc', 'mailbox' => 'INBOX', 'uidValidity' => 7];
$this->assertSame([
[...$key, 'removedSeq' => null, 'addedSeq' => ['$gt' => 10]],
[...$key, 'removedSeq' => null, 'addedSeq' => ['$lte' => 10], 'modSeq' => ['$gt' => 10]],
[...$key, 'addedSeq' => ['$lte' => 10], 'removedSeq' => ['$gt' => 10]],
], $filters);
$this->assertSame(['additions' => [1], 'modifications' => [2], 'deletions' => [3]], $changes);
}
public function testTombstoneKeepsKeyAndSequencesOnly(): void
{
$collection = $this->createMock(Collection::class);
$collection->expects($this->once())
->method('updateMany')
->with(
['sid' => 'svc', 'mailbox' => 'INBOX', 'uidValidity' => 7, 'uid' => ['$in' => [3, 4]], 'removedSeq' => null],
$this->callback(function (array $update): bool {
$this->assertSame(['removedSeq' => 12, 'modSeq' => 12], $update['$set']);
$this->assertArrayHasKey('subject', $update['$unset']);
$this->assertArrayHasKey('flags', $update['$unset']);
$this->assertArrayNotHasKey('addedSeq', $update['$unset']);
return true;
}),
);
(new MessageStore($this->dataStore($collection)))->tombstone('svc', 'INBOX', 7, 12, 3, 4);
}
public function testLiveReadsSkipTombstones(): void
{
$collection = $this->createMock(Collection::class);
$collection->expects($this->once())
->method('find')
->with($this->callback(fn (array $filter): bool => array_key_exists('removedSeq', $filter) && $filter['removedSeq'] === null))
->willReturn(new Cursor(new ArrayIterator([])));
(new MessageStore($this->dataStore($collection)))->flags('svc', 'INBOX', 7);
}
public function testReserveSequenceReturnsTheFirstNumberOfTheRange(): void
{
$mongo = $this->createMock(MongoCollection::class);
$mongo->expects($this->once())
->method('findOneAndUpdate')
->with(
['sid' => 'svc', 'name' => 'INBOX'],
['$inc' => ['changeSeq' => 3]],
$this->callback(fn (array $options): bool => $options['returnDocument'] === FindOneAndUpdate::RETURN_DOCUMENT_AFTER),
)
->willReturn(['changeSeq' => 12]);
$collection = $this->createStub(Collection::class);
$collection->method('getMongoCollection')->willReturn($mongo);
$this->assertSame(10, (new MailboxStore($this->dataStore($collection)))->reserveSequence('svc', 'INBOX', 3));
}
private function dataStore(Collection $collection): DataStore
{
$dataStore = $this->createStub(DataStore::class);
$dataStore->method('selectCollection')->willReturn($collection);
return $dataStore;
}
}