Nodarx/provider_imap
1
0
Fork 0
generated from Nodarx/template
Code Issues 1 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

feat: harmonize mailbox messages

Browse Source 
Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>
This commit is contained in:
Sebastian Krupinski
2026-10-05 20:34:02 -04:00
parent b02ca63208
commit 65fdb23fa4
12 changed files with 879 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/php/Unit/CollectionFetchTest.php
+95
View File
@@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig};
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use PHPUnit\Framework\TestCase;
final class CollectionFetchTest extends TestCase
{
private array $commands = [];
public function testWithListStatusUsesOneRoundTrip(): void
{
$mailbox = $this->service(listStatus: true)->collectionFetch('INBOX');
$this->assertCount(1, $this->commands);
$this->assertStringContainsString('RETURN (STATUS', $this->commands[0]);
$this->assertSame(3, $mailbox?->messages());
$this->assertSame(7, $mailbox?->uidValidity());
$this->assertSame(10, $mailbox?->uidNext());
}
public function testWithoutListStatusFallsBackToStatus(): void
{
$mailbox = $this->service(listStatus: false)->collectionFetch('INBOX');
$this->assertCount(2, $this->commands);
$this->assertStringNotContainsString('RETURN', $this->commands[0]);
$this->assertStringStartsWith('STATUS', $this->commands[1]);
$this->assertSame(3, $mailbox?->messages());
$this->assertSame(7, $mailbox?->uidValidity());
$this->assertSame(10, $mailbox?->uidNext());
}
public function testUnknownMailboxIsNull(): void
{
$this->assertNull($this->service(listStatus: false, exists: false)->collectionFetch('Missing'));
}
private function service(bool $listStatus, bool $exists = true): LiveMailService
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(function (string $wire) use (&$lines, $listStatus, $exists): void {
[$tag, $command] = explode(' ', trim($wire), 2);
$operation = explode(' ', $command)[0];
if ($operation === 'CAPABILITY') {
$lines[] = '* CAPABILITY IMAP4rev1' . ($listStatus ? ' LIST-STATUS' : '') . "\r\n";
} else {
$this->commands[] = $command;
}
if ($operation === 'LIST') {
if (!$listStatus && str_contains($command, 'RETURN')) {
$lines[] = "$tag BAD Unknown RETURN option\r\n";
return;
}
if ($exists) {
$lines[] = "* LIST (\\HasNoChildren) \"/\" \"INBOX\"\r\n";
if ($listStatus) {
$lines[] = "* STATUS \"INBOX\" (MESSAGES 3 UNSEEN 1 UIDNEXT 10 UIDVALIDITY 7)\r\n";
}
}
}
if ($operation === 'STATUS') {
$lines[] = "* STATUS \"INBOX\" (MESSAGES 3 UNSEEN 1 UIDNEXT 10 UIDVALIDITY 7)\r\n";
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return new class($this->createStub(Service::class), $client) extends LiveMailService {
public function __construct(Service $service, private readonly Client $client)
{
parent::__construct($service);
}
public function imapClient(): Client
{
return $this->client;
}
};
}
}
tests/php/Unit/HarmonizationServiceTest.php
+3 -1
View File
@@ -13,6 +13,7 @@ use KTXM\ProviderImap\Client\Mailbox;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
@@ -63,6 +64,7 @@ final class HarmonizationServiceTest extends TestCase
$this->createStub(MailboxStore::class),
$this->createStub(MessageStore::class),
$this->createStub(MessageFileStore::class),
$this->createStub(MessageIngestor::class),
);
$this->expectException(LogicException::class);
@@ -148,7 +150,7 @@ final class HarmonizationServiceTest extends TestCase
$live = new HarmonizationServiceTestLiveStub($service);
$live->mailboxes = array_map(static fn (string $name): Mailbox => new Mailbox($name, '/', []), $remote);
return (new HarmonizationService($mailboxes, $messages, $files))->for($service, $live);
return (new HarmonizationService($mailboxes, $messages, $files, $this->createStub(MessageIngestor::class)))->for($service, $live);
}
private function dataStore(Collection $collection): DataStore
tests/php/Unit/MailboxSelectionTest.php
+116
View File
@@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig, ImapException};
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use PHPUnit\Framework\TestCase;
final class MailboxSelectionTest extends TestCase
{
private array $commands = [];
private array $failing = [];
public function testSameMailboxIsSelectedOnce(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityExtant('INBOX', 2);
$this->assertSame(1, $this->sent('EXAMINE "INBOX"'));
}
public function testOtherMailboxIsSelected(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityExtant('Sent', 1);
$service->entityExtant('INBOX', 1);
$this->assertSame(2, $this->sent('EXAMINE "INBOX"'));
$this->assertSame(1, $this->sent('EXAMINE "Sent"'));
}
public function testWriteAfterReadOnlySelectionSelectsReadWrite(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityPatch('INBOX', ['\\Seen'], [], 1);
$service->entityExtant('INBOX', 1);
$this->assertSame(1, $this->sent('EXAMINE "INBOX"'));
$this->assertSame(1, $this->sent('SELECT "INBOX"'));
}
public function testFailedSelectionIsNotReused(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$this->failing = ['EXAMINE "Gone"'];
try {
$service->entityExtant('Gone', 1);
$this->fail('Expected the selection to fail');
} catch (ImapException) {
}
$service->entityExtant('INBOX', 1);
$this->assertSame(2, $this->sent('EXAMINE "INBOX"'));
}
private function sent(string $command): int
{
return count(array_filter($this->commands, static fn (string $sent): bool => $sent === $command));
}
private function service(): LiveMailService
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(function (string $wire) use (&$lines): void {
[$tag, $command] = explode(' ', trim($wire), 2);
$operation = explode(' ', $command)[0];
if ($operation === 'CAPABILITY') {
$lines[] = "* CAPABILITY IMAP4rev1\r\n";
} else {
$this->commands[] = $command;
}
if (in_array($command, $this->failing, true)) {
$lines[] = "$tag NO Mailbox does not exist\r\n";
return;
}
if (in_array($operation, ['SELECT', 'EXAMINE'], true)) {
$lines[] = "* 3 EXISTS\r\n";
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return new class($this->createStub(Service::class), $client) extends LiveMailService {
public function __construct(Service $service, private readonly Client $client)
{
parent::__construct($service);
}
public function imapClient(): Client
{
return $this->client;
}
};
}
}
tests/php/Unit/MessageHarmonizationTest.php
+303
View File
@@ -0,0 +1,303 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use Generator;
use KTXC\Db\DataStore;
use KTXM\ProviderImap\Client\Mailbox;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
use KTXM\ProviderImap\Client\Protocol\Parser\FetchMessageParser;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
use KTXM\ProviderImap\Stores\MessageStore;
use PHPUnit\Framework\TestCase;
final class MessageHarmonizationTest extends TestCase
{
private FakeMailboxStore $mailboxes;
private FakeMessageStore $messages;
private FakeMessageFileStore $files;
private MessageHarmonizationLiveStub $live;
private HarmonizationService $harmonizer;
protected function setUp(): void
{
$dataStore = $this->createStub(DataStore::class);
$this->mailboxes = new FakeMailboxStore($dataStore);
$this->messages = new FakeMessageStore($dataStore);
$this->files = new FakeMessageFileStore('/nonexistent');
$service = (new Service())->fromStore(['tid' => 'tenant', 'sid' => 'svc']);
$this->live = new MessageHarmonizationLiveStub($service);
$this->live->uidValidity = 7;
$this->live->uidNext = 100;
$this->harmonizer = (new HarmonizationService(
$this->mailboxes,
$this->messages,
$this->files,
new MessageIngestor($this->files, $this->messages),
))->for($service, $this->live);
$this->mailboxes->upsert('tenant', 'svc', (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', [])));
}
public function testFirstRunIngestsEverythingNewestFirst(): void
{
$this->live->flags = [1 => ['\\Seen'], 2 => [], 3 => ['\\Flagged']];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 3, 'updated' => 0, 'removed' => 0, 'complete' => true], $result);
$this->assertSame([[3, 2, 1]], $this->live->fetchBatches);
$this->assertSame([1 => ['seen'], 2 => [], 3 => ['flagged']], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([1, 2, 3], $this->files->uids(7));
$this->assertSame(262144, $this->live->bodyTextLimit);
$state = $this->mailboxes->state('svc', 'INBOX');
$this->assertSame(7, $state['uidValidity']);
$this->assertSame(100, $state['uidNext']);
$this->assertTrue($state['harmonizationComplete']);
$this->assertIsInt($state['harmonizedAt']);
$this->assertNull($this->mailboxes->lockOwner);
}
public function testIncrementalRunAddsUpdatesAndRemoves(): void
{
$this->live->flags = [1 => ['\\Seen'], 2 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$this->live->flags = [2 => ['\\Seen', '$Forwarded'], 3 => []];
$this->live->fetchBatches = [];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 1, 'updated' => 1, 'removed' => 1, 'complete' => true], $result);
$this->assertSame([[3]], $this->live->fetchBatches);
$this->assertSame([2 => ['seen', '$forwarded'], 3 => []], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([2, 3], $this->files->uids(7));
}
public function testChangedUidValidityDiscardsTheCache(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$this->live->uidValidity = 8;
$this->live->flags = [1 => []];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertTrue($result['reset']);
$this->assertSame(1, $result['added']);
$this->assertSame([], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([1 => []], $this->messages->flags('svc', 'INBOX', 8));
$this->assertSame([], $this->files->uids(7));
$this->assertSame(8, $this->mailboxes->state('svc', 'INBOX')['uidValidity']);
}
public function testHeldLockSkipsTheRun(): void
{
$this->live->flags = [1 => []];
$this->mailboxes->lockOwner = 'someone-else';
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame('skipped', $result['status']);
$this->assertSame(0, $this->live->selects);
$this->assertSame('someone-else', $this->mailboxes->lockOwner);
}
public function testMessagesThatCannotBeFetchedLeaveTheMailboxIncomplete(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->live->unfetchable = [2];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(1, $result['added']);
$this->assertFalse($result['complete']);
$this->assertFalse($this->mailboxes->state('svc', 'INBOX')['harmonizationComplete']);
}
public function testNewMessagesAreFetchedInBatches(): void
{
$this->live->flags = array_fill_keys(range(1, 450), []);
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(450, $result['added']);
$this->assertSame([200, 200, 50], array_map('count', $this->live->fetchBatches));
$this->assertSame(450, $this->live->fetchBatches[0][0]);
$this->assertSame(1, $this->live->fetchBatches[2][49]);
}
public function testLockIsReleasedWhenTheRunFails(): void
{
$this->live->uidValidity = null;
try {
$this->harmonizer->harmonizeMessages('INBOX');
$this->fail('Expected missing UIDVALIDITY to fail the run');
} catch (\RuntimeException) {
}
$this->assertNull($this->mailboxes->lockOwner);
}
}
final class MessageHarmonizationLiveStub extends LiveMailService
{
public ?int $uidValidity = null;
public ?int $uidNext = null;
/** @var array<int, list<string>> */
public array $flags = [];
/** @var int[] */
public array $unfetchable = [];
public array $fetchBatches = [];
public int $selects = 0;
public ?int $bodyTextLimit = null;
public function collectionFetch(string $identifier): ?Mailbox
{
$this->selects++;
return new Mailbox($identifier, '/', [], count($this->flags), 0, $this->uidValidity, 0, [], true, $this->uidNext);
}
public function entityFlags(string $collection): Generator
{
yield from $this->flags;
}
public function entityFetch(string $collection, ?FetchOptions $options = null, int ...$uids): Generator
{
$this->fetchBatches[] = $uids;
if (preg_match('/BODY\.PEEK\[TEXT\]<0\.(\d+)>/', (string) $options?->toCommand(), $matches) === 1) {
$this->bodyTextLimit = (int) $matches[1];
}
foreach ($uids as $uid) {
if (in_array($uid, $this->unfetchable, true)) {
continue;
}
$flags = implode(' ', $this->flags[$uid] ?? []);
yield $uid => FetchMessageParser::parse("* {$uid} FETCH (UID {$uid} FLAGS ({$flags}))");
}
}
}
final class FakeMailboxStore extends MailboxStore
{
/** @var array<string, array> */
public array $documents = [];
public ?string $lockOwner = null;
public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void
{
$name = (string) $collection->identifier();
$this->documents[$name] = ($this->documents[$name] ?? self::STATE_DEFAULTS) + ['name' => $name];
}
public function fetch(string $serviceId, string $name): ?array
{
return $this->documents[$name] ?? null;
}
public function list(string $serviceId): array
{
return $this->documents;
}
public function updateState(string $serviceId, string $name, array $state): void
{
$this->documents[$name] = array_replace($this->documents[$name], array_intersect_key($state, self::STATE_DEFAULTS));
}
public function acquireLock(string $serviceId, string $name, string $owner, int $ttl): bool
{
if ($this->lockOwner !== null && $this->lockOwner !== $owner) {
return false;
}
$this->lockOwner = $owner;
return true;
}
public function releaseLock(string $serviceId, string $name, string $owner): void
{
if ($this->lockOwner === $owner) {
$this->lockOwner = null;
}
}
}
final class FakeMessageStore extends MessageStore
{
/** @var array<int, array<int, list<string>>> flags by UIDVALIDITY and UID */
private array $documents = [];
public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity): void
{
$meta = $entity->toCacheMeta();
$this->documents[$uidValidity][$meta['uid']] = $meta['flags'];
ksort($this->documents[$uidValidity]);
}
public function flags(string $serviceId, string $mailbox, int $uidValidity): array
{
return $this->documents[$uidValidity] ?? [];
}
public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags): void
{
$this->documents[$uidValidity][$uid] = array_values($flags);
}
public function delete(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
{
foreach ($uids as $uid) {
unset($this->documents[$uidValidity][$uid]);
}
}
public function deleteByMailbox(string $serviceId, string $mailbox): void
{
$this->documents = [];
}
}
final class FakeMessageFileStore extends MessageFileStore
{
/** @var array<int, array<int, array>> content by UIDVALIDITY and UID */
private array $files = [];
public function write(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid, array $content): void
{
$this->files[$uidValidity][$uid] = $content;
}
public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
{
foreach ($uids as $uid) {
unset($this->files[$uidValidity][$uid]);
}
}
public function deleteByMailbox(string $tenantId, string $serviceId, string $mailbox): void
{
$this->files = [];
}
/** @return int[] */
public function uids(int $uidValidity): array
{
$uids = array_keys($this->files[$uidValidity] ?? []);
sort($uids);
return $uids;
}
}
tests/php/Unit/SelectCommandTest.php
+69
View File
@@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig};
use KTXM\ProviderImap\Client\Protocol\Command\SelectCommand;
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use PHPUnit\Framework\TestCase;
final class SelectCommandTest extends TestCase
{
public function testResponseCodesAreParsed(): void
{
$mailbox = $this->client([
"* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n",
"* OK [PERMANENTFLAGS ()] Read-only mailbox\r\n",
"* 172 EXISTS\r\n",
"* 1 RECENT\r\n",
"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n",
"* OK [UIDNEXT 4392] Predicted next UID\r\n",
"* OK [HIGHESTMODSEQ 715194045007] Highest\r\n",
])->perform(new SelectCommand('INBOX', true));
$this->assertSame(172, $mailbox->messages());
$this->assertSame(1, $mailbox->recent());
$this->assertSame(3857529045, $mailbox->uidValidity());
$this->assertSame(4392, $mailbox->uidNext());
$this->assertSame(715194045007, $mailbox->highestModSeq());
}
public function testMissingResponseCodesAreNull(): void
{
$mailbox = $this->client(["* 0 EXISTS\r\n"])->perform(new SelectCommand('INBOX', true));
$this->assertNull($mailbox->uidValidity());
$this->assertNull($mailbox->uidNext());
$this->assertNull($mailbox->highestModSeq());
}
/**
* @param string[] $untagged untagged lines the server sends before completing SELECT
*/
private function client(array $untagged): Client
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(static function (string $wire) use (&$lines, $untagged): void {
[$tag, $command] = explode(' ', trim($wire), 2);
if (str_starts_with($command, 'CAPABILITY')) {
$lines[] = "* CAPABILITY IMAP4rev1\r\n";
}
if (str_starts_with($command, 'EXAMINE') || str_starts_with($command, 'SELECT')) {
array_push($lines, ...$untagged);
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return $client;
}
}