Nodarx/provider_imap
1
0
Fork 0
generated from Nodarx/template
Code Issues 1 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

feat: harmonize mailbox messages

Browse Source 
Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>
This commit is contained in:
Sebastian Krupinski
2026-10-05 20:34:02 -04:00
parent b02ca63208
commit 65fdb23fa4
12 changed files with 879 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Client/Mailbox.php
+14
View File
@@ -22,6 +22,8 @@ final class Mailbox
private readonly int $recent = 0,
private readonly array $flags = [],
private readonly bool $readOnly = true,
private readonly ?int $uidNext = null,
private readonly ?int $highestModSeq = null,
) {}
public function fromStatus(MailboxStatusResult $status): self
@@ -38,6 +40,8 @@ final class Mailbox
$this->recent,
$this->flags,
$this->readOnly,
$items['UIDNEXT'] ?? $this->uidNext,
$items['HIGHESTMODSEQ'] ?? $this->highestModSeq,
);
}
@@ -64,6 +68,16 @@ final class Mailbox
return $this->uidValidity;
}
public function uidNext(): ?int
{
return $this->uidNext;
}
public function highestModSeq(): ?int
{
return $this->highestModSeq;
}
public function messages(): int
{
return $this->messages;
lib/Client/Protocol/Command/ListCommand.php
+2 -11
View File
@@ -13,6 +13,7 @@ use KTXM\ProviderImap\Client\ImapException;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\ListReturnOptions;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\ListSelectionOptions;
use KTXM\ProviderImap\Client\Mailbox;
use KTXM\ProviderImap\Client\Result\MailboxStatusResult;
use KTXM\ProviderImap\Client\Protocol\RequestFrame;
use KTXM\ProviderImap\Client\Protocol\Response\TaggedResponse;
use KTXM\ProviderImap\Client\Protocol\Response\UntaggedResponse;
@@ -144,16 +145,6 @@ final class ListCommand implements CommandInterface
*/
private function applyStatus(Mailbox $mailbox, array $status): Mailbox
{
return new Mailbox(
$mailbox->name(),
$mailbox->delimiter(),
$mailbox->attributes(),
$status['MESSAGES'] ?? $mailbox->messages(),
$status['UNSEEN'] ?? $mailbox->unread(),
$mailbox->uidValidity(),
$mailbox->recent(),
$mailbox->flags(),
$mailbox->readOnly(),
);
return $mailbox->fromStatus(new MailboxStatusResult($mailbox->name(), $status));
}
}
lib/Client/Protocol/Command/SelectCommand.php
+20 -1
View File
@@ -53,6 +53,9 @@ final class SelectCommand implements CommandInterface
{
$exists = 0;
$recent = 0;
$uidValidity = null;
$uidNext = null;
$highestModSeq = null;
$flags = [];
$readOnly = $this->readOnly;
@@ -70,6 +73,16 @@ final class SelectCommand implements CommandInterface
continue;
}
// response codes, e.g. "* OK [UIDVALIDITY 3857529045] UIDs valid"
if (preg_match('/^\*\s+OK\s+\[(UIDVALIDITY|UIDNEXT|HIGHESTMODSEQ)\s+(\d+)\]/i', $raw, $matches)) {
match (strtoupper($matches[1])) {
'UIDVALIDITY' => $uidValidity = (int) $matches[2],
'UIDNEXT' => $uidNext = (int) $matches[2],
'HIGHESTMODSEQ' => $highestModSeq = (int) $matches[2],
};
continue;
}
if ($response->label() === 'FLAGS' && preg_match('/\(([^)]*)\)/', $response->payload(), $matches)) {
$flags = $this->parseFlags($matches[1]);
continue;
@@ -77,6 +90,10 @@ final class SelectCommand implements CommandInterface
}
if ($response instanceof TaggedResponse) {
// a failed SELECT leaves no mailbox selected (RFC 3501 6.3.1)
if ($response->status() !== 'OK') {
$context->setSelectedMailbox(null);
}
CompletionChecker::assertSuccess($this->name(), $response);
if (str_contains(strtoupper($response->text()), 'READ-ONLY')) {
@@ -92,10 +109,12 @@ final class SelectCommand implements CommandInterface
[],
$exists,
0,
null,
$uidValidity,
$recent,
$flags,
$readOnly,
$uidNext,
$highestModSeq,
);
}
}
lib/Providers/MessageProperties.php
+17 -13
View File
@@ -100,23 +100,27 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
}
}
$this->data[static::PROPERTY_FLAGS] = [];
foreach ($message->flags() as $flag) {
$flag = ltrim($flag, '\\');
$normalized = match (strtolower($flag)) {
'seen' => 'seen',
'flagged' => 'flagged',
'answered' => 'answered',
'draft' => 'draft',
'deleted' => 'deleted',
default => strtolower($flag),
};
$this->data[static::PROPERTY_FLAGS][$normalized] = true;
}
$this->data[static::PROPERTY_FLAGS] = array_fill_keys(self::normalizeFlags($message->flags()), true);
return $this;
}
/**
* Normalise IMAP flags (e.g. "\\Seen", "$Forwarded") to flag names (e.g. "seen", "$forwarded").
*
* @param string[] $flags
* @return list<string>
*/
public static function normalizeFlags(array $flags): array
{
$normalized = [];
foreach ($flags as $flag) {
$normalized[] = strtolower(ltrim($flag, '\\'));
}
return array_values(array_unique($normalized));
}
// ── Cache (meta store / content store) ───────────────────────────────────
/**
lib/Service/Cache/HarmonizationService.php
+143
View File
@@ -9,13 +9,17 @@ declare(strict_types=1);
namespace KTXM\ProviderImap\Service\Cache;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\MessageProperties;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
use KTXM\ProviderImap\Stores\MessageStore;
use LogicException;
use RuntimeException;
/**
* Brings the cache of one service in line with its IMAP server.
@@ -25,6 +29,12 @@ use LogicException;
*/
class HarmonizationService
{
/** Messages fetched per UID FETCH when ingesting; each is cached as it streams in */
public const INGEST_BATCH_SIZE = 200;
/** Lifetime of a mailbox harmonization lock in seconds; long enough for a large first run */
public const LOCK_TTL = 900;
private ?Service $service = null;
private ?LiveMailService $live = null;
@@ -32,6 +42,7 @@ class HarmonizationService
private readonly MailboxStore $mailboxStore,
private readonly MessageStore $messageStore,
private readonly MessageFileStore $fileStore,
private readonly MessageIngestor $ingestor,
) {}
/**
@@ -87,6 +98,138 @@ class HarmonizationService
return $result;
}
/**
* Harmonize the messages of one mailbox with the server.
*
* Ingests messages that are not cached yet (newest first), updates changed
* flags and removes messages expunged on the server. A changed UIDVALIDITY
* discards the mailbox's cache first. Never waits: when another run holds the
* mailbox lock, nothing is done.
*
* @return array{status: 'harmonized'|'skipped', reset: bool, added: int, updated: int, removed: int, complete: bool}
*/
public function harmonizeMessages(string $mailbox): array
{
[$service, $live] = $this->bound();
$tenantId = (string) $service->tenantIdentifier();
$serviceId = (string) $service->identifier();
$result = ['status' => 'skipped', 'reset' => false, 'added' => 0, 'updated' => 0, 'removed' => 0, 'complete' => false];
// the lock lives on the mailbox document, so it has to exist first
if ($this->mailboxStore->fetch($serviceId, $mailbox) === null) {
$this->harmonizeMailboxes();
if ($this->mailboxStore->fetch($serviceId, $mailbox) === null) {
throw new RuntimeException("Mailbox not found on server: {$mailbox}");
}
}
$owner = bin2hex(random_bytes(8));
if (!$this->mailboxStore->acquireLock($serviceId, $mailbox, $owner, self::LOCK_TTL)) {
return $result;
}
try {
$state = $this->mailboxStore->state($serviceId, $mailbox);
$selected = $live->collectionFetch($mailbox)
?? throw new RuntimeException("Mailbox not found on server: {$mailbox}");
$uidValidity = $selected->uidValidity()
?? throw new RuntimeException("Server did not report UIDVALIDITY for mailbox: {$mailbox}");
// a new UIDVALIDITY invalidates every cached UID of the mailbox
if ($state['uidValidity'] !== null && (int) $state['uidValidity'] !== $uidValidity) {
$this->messageStore->deleteByMailbox($serviceId, $mailbox);
$this->fileStore->deleteByMailbox($tenantId, $serviceId, $mailbox);
$result['reset'] = true;
}
$cachedFlags = $this->messageStore->flags($serviceId, $mailbox, $uidValidity);
// stream every UID and its flags from the server: cached messages get their flags
// compared (and are dropped from $cachedFlags), unknown UIDs are collected as new.
// No other IMAP command may run until the stream is consumed.
$new = [];
foreach ($live->entityFlags($mailbox) as $uid => $flags) {
if (!isset($cachedFlags[$uid])) {
$new[] = $uid;
continue;
}
$flags = MessageProperties::normalizeFlags($flags);
if (!self::sameFlags($flags, $cachedFlags[$uid])) {
$this->messageStore->updateFlags($serviceId, $mailbox, $uidValidity, $uid, $flags);
$result['updated']++;
}
unset($cachedFlags[$uid]);
}
// new: newest first so recent mail is available soonest
rsort($new);
$missing = [];
foreach (array_chunk($new, self::INGEST_BATCH_SIZE) as $batch) {
$ingested = $this->ingestBatch($tenantId, $serviceId, $mailbox, $uidValidity, $batch);
$result['added'] += count($ingested);
array_push($missing, ...array_diff($batch, $ingested));
}
// expunged: cached UIDs the server did not list; meta documents first, then files
$expunged = array_keys($cachedFlags);
if ($expunged !== []) {
$this->messageStore->delete($serviceId, $mailbox, $uidValidity, ...$expunged);
$this->fileStore->delete($tenantId, $serviceId, $mailbox, $uidValidity, ...$expunged);
$result['removed'] = count($expunged);
}
// UIDs that could not be fetched (e.g. expunged meanwhile) are retried next run
$result['complete'] = $missing === [];
$result['status'] = 'harmonized';
$this->mailboxStore->updateState($serviceId, $mailbox, [
'uidValidity' => $uidValidity,
'uidNext' => $selected->uidNext(),
'highestModSeq' => $selected->highestModSeq(),
'harmonizedAt' => time(),
'harmonizationComplete' => $result['complete'],
]);
} finally {
$this->mailboxStore->releaseLock($serviceId, $mailbox, $owner);
}
return $result;
}
/**
* Fetch a batch of messages and cache each one as it streams in.
*
* @param int[] $uids
* @return int[] UIDs that were cached
*/
private function ingestBatch(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, array $uids): array
{
[$service, $live] = $this->bound();
$options = FetchOptions::message()->withBodyText(MessageIngestor::BODY_TEXT_LIMIT);
$ingested = [];
foreach ($live->entityFetch($mailbox, $options, ...$uids) as $message) {
$entity = (new EntityResource($service->provider(), $serviceId))->fromImap($message, $mailbox);
array_push($ingested, ...$this->ingestor->ingest($tenantId, $serviceId, $uidValidity, $entity));
}
return $ingested;
}
/**
* @param string[] $left
* @param string[] $right
*/
private static function sameFlags(array $left, array $right): bool
{
sort($left);
sort($right);
return $left === $right;
}
/**
* Remove a mailbox and everything cached for it: meta documents first, then files, then the mailbox.
*/
lib/Service/Live/LiveMailService.php
+78 -17
View File
@@ -65,6 +65,8 @@ class LiveMailService
private ?ImapClient $imapClient = null;
private ?SmtpClient $smtpClient = null;
/** Result of the last SELECT on the current connection, reused while still selected */
private ?Mailbox $selected = null;
public function __construct(
private readonly Service $service,
@@ -200,16 +202,25 @@ class LiveMailService
*/
public function collectionFetch(string $identifier): ?Mailbox
{
// LIST-STATUS (RFC 5819) returns the status with the LIST response in one round trip
$listStatus = $this->imapClient()->hasCapability('LIST-STATUS');
$command = $listStatus
? new ListCommand('', $identifier, null, ListReturnOptions::status(...self::DEFAULT_MAILBOX_STATUS_ITEMS))
: new ListCommand('', $identifier);
// retrieve mailbox from remote
$mailbox = iterator_to_array($this->imapClient()->perform(new ListCommand('', $identifier, null, ListReturnOptions::status(...self::DEFAULT_MAILBOX_STATUS_ITEMS))));
$mailbox = iterator_to_array($this->imapClient()->perform($command));
if (empty($mailbox)) {
return null;
}
$mailbox = reset($mailbox);
// enrich with STATUS
$status = $this->imapClient()->perform(new StatusCommand($mailbox->name(), self::DEFAULT_MAILBOX_STATUS_ITEMS));
$mailbox = $mailbox->fromStatus($status);
// enrich with STATUS when LIST could not provide it
if (!$listStatus && $mailbox->isSelectable()) {
$status = $this->imapClient()->perform(new StatusCommand($mailbox->name(), self::DEFAULT_MAILBOX_STATUS_ITEMS));
$mailbox = $mailbox->fromStatus($status);
}
return $mailbox;
}
@@ -270,7 +281,7 @@ class LiveMailService
$nativeFilter = $this->buildEntitySearchCriteria($filter);
$nativeSort = $sort !== null ? $this->entitySortCriteria($sort) : [];
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$rfc5258 = $this->imapClient()->hasCapability('SORT');
$uids = [];
@@ -298,6 +309,31 @@ class LiveMailService
return $this->entityApplyRange($uids, $range);
}
/**
* Stream the flags of every message in a mailbox.
*
* The stream also lists every UID that exists in the mailbox.
*
* @return Generator<int, list<string>> IMAP flags keyed by UID
*/
public function entityFlags(string $collection): Generator
{
// fresh SELECT: the message count decides whether "1:*" may be sent
$mailbox = $this->select($collection, refresh: true);
// "1:*" on an empty mailbox is rejected by some servers
if ($mailbox->messages() === 0) {
return;
}
foreach ($this->imapClient()->perform(new FetchManyCommand(
MessageTarget::uid('1:*'),
FetchOptions::of('FLAGS'),
)) as $message) {
yield $message->uid() => $message->flags();
}
}
/**
* Determine which of the given UIDs exist in a mailbox.
*
@@ -309,7 +345,7 @@ class LiveMailService
return [];
}
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
return $this->imapClient()->perform(new SearchCommand(
SearchCriteriaBuilder::create()->uid(SequenceSet::items(...$uids)),
@@ -329,9 +365,10 @@ class LiveMailService
// fast path: fetch all messages without filtering, sorting or pagination
if ($filter === null && $sort === null && $range === null) {
$mailbox = $this->imapClient()->perform(new SelectCommand($collection, true));
$mailbox = $this->select($collection, refresh: true);
if ($mailbox === null) {
// "1:*" on an empty mailbox is rejected by some servers
if ($mailbox->messages() === 0) {
return [];
}
@@ -366,7 +403,7 @@ class LiveMailService
}
$options ??= FetchOptions::message()->withBodyText();
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$request = new FetchManyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
@@ -391,7 +428,7 @@ class LiveMailService
*/
public function entityDownload(string $collection, int $uid, ?string $partId = null): BinaryResource
{
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$encoding = null;
@@ -550,7 +587,7 @@ class LiveMailService
return;
}
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$this->imapClient()->perform(new StoreCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$flags,
@@ -569,7 +606,7 @@ class LiveMailService
$target = MessageTarget::uid(SequenceSet::items(...array_values($uids)));
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$this->imapClient()->perform(new StoreCommand($target, ['\\Deleted'], '+'));
$this->imapClient()->perform(new ExpungeCommand($target));
@@ -583,7 +620,7 @@ class LiveMailService
return;
}
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$flagsToAdd = $this->normalizeFlags($flagsToAdd);
$flagsToRemove = $this->normalizeFlags($flagsToRemove);
@@ -615,13 +652,13 @@ class LiveMailService
// if MOVE is supported, use it; otherwise, fall back to COPY + EXPUNGE
if ($rfc6851) {
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new MoveCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
));
} else {
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new CopyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
@@ -657,7 +694,7 @@ class LiveMailService
return [];
}
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new CopyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
@@ -1143,4 +1180,28 @@ class LiveMailService
}
return $normalized;
}
/**
* Select a mailbox, reusing the current selection when possible.
*
* The current selection is reused when the same mailbox is still selected on the
* session and its access mode suffices (read-write covers read-only). A refresh
* forces a new SELECT, e.g. when an up-to-date message count is needed.
*/
private function select(string $collection, bool $readOnly = true, bool $refresh = false): Mailbox
{
if (!$refresh
&& $this->selected !== null
&& $this->selected->name() === $collection
&& $this->imapClient()->session()->selectedMailbox() === $collection
&& ($readOnly || !$this->selected->readOnly())
) {
return $this->selected;
}
$this->selected = null;
$this->selected = $this->imapClient()->perform(new SelectCommand($collection, $readOnly));
return $this->selected;
}
}
lib/Stores/MessageStore.php
+19
View File
@@ -145,6 +145,25 @@ class MessageStore
return $uids;
}
/**
* List the flags of every message cached for a mailbox.
*
* @return array<int, list<string>> set flags keyed by UID
*/
public function flags(string $serviceId, string $mailbox, int $uidValidity): array
{
$cursor = $this->dataStore->selectCollection(self::COLLECTION_NAME)->find(
['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity],
['projection' => ['uid' => 1, 'flags' => 1]],
);
$flags = [];
foreach ($cursor as $document) {
$flags[(int) $document['uid']] = array_values(array_map('strval', (array) ($document['flags'] ?? [])));
}
return $flags;
}
/**
* Replace the flags of a message.
*
tests/php/Unit/CollectionFetchTest.php
+95
View File
@@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig};
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use PHPUnit\Framework\TestCase;
final class CollectionFetchTest extends TestCase
{
private array $commands = [];
public function testWithListStatusUsesOneRoundTrip(): void
{
$mailbox = $this->service(listStatus: true)->collectionFetch('INBOX');
$this->assertCount(1, $this->commands);
$this->assertStringContainsString('RETURN (STATUS', $this->commands[0]);
$this->assertSame(3, $mailbox?->messages());
$this->assertSame(7, $mailbox?->uidValidity());
$this->assertSame(10, $mailbox?->uidNext());
}
public function testWithoutListStatusFallsBackToStatus(): void
{
$mailbox = $this->service(listStatus: false)->collectionFetch('INBOX');
$this->assertCount(2, $this->commands);
$this->assertStringNotContainsString('RETURN', $this->commands[0]);
$this->assertStringStartsWith('STATUS', $this->commands[1]);
$this->assertSame(3, $mailbox?->messages());
$this->assertSame(7, $mailbox?->uidValidity());
$this->assertSame(10, $mailbox?->uidNext());
}
public function testUnknownMailboxIsNull(): void
{
$this->assertNull($this->service(listStatus: false, exists: false)->collectionFetch('Missing'));
}
private function service(bool $listStatus, bool $exists = true): LiveMailService
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(function (string $wire) use (&$lines, $listStatus, $exists): void {
[$tag, $command] = explode(' ', trim($wire), 2);
$operation = explode(' ', $command)[0];
if ($operation === 'CAPABILITY') {
$lines[] = '* CAPABILITY IMAP4rev1' . ($listStatus ? ' LIST-STATUS' : '') . "\r\n";
} else {
$this->commands[] = $command;
}
if ($operation === 'LIST') {
if (!$listStatus && str_contains($command, 'RETURN')) {
$lines[] = "$tag BAD Unknown RETURN option\r\n";
return;
}
if ($exists) {
$lines[] = "* LIST (\\HasNoChildren) \"/\" \"INBOX\"\r\n";
if ($listStatus) {
$lines[] = "* STATUS \"INBOX\" (MESSAGES 3 UNSEEN 1 UIDNEXT 10 UIDVALIDITY 7)\r\n";
}
}
}
if ($operation === 'STATUS') {
$lines[] = "* STATUS \"INBOX\" (MESSAGES 3 UNSEEN 1 UIDNEXT 10 UIDVALIDITY 7)\r\n";
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return new class($this->createStub(Service::class), $client) extends LiveMailService {
public function __construct(Service $service, private readonly Client $client)
{
parent::__construct($service);
}
public function imapClient(): Client
{
return $this->client;
}
};
}
}
tests/php/Unit/HarmonizationServiceTest.php
+3 -1
View File
@@ -13,6 +13,7 @@ use KTXM\ProviderImap\Client\Mailbox;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
@@ -63,6 +64,7 @@ final class HarmonizationServiceTest extends TestCase
$this->createStub(MailboxStore::class),
$this->createStub(MessageStore::class),
$this->createStub(MessageFileStore::class),
$this->createStub(MessageIngestor::class),
);
$this->expectException(LogicException::class);
@@ -148,7 +150,7 @@ final class HarmonizationServiceTest extends TestCase
$live = new HarmonizationServiceTestLiveStub($service);
$live->mailboxes = array_map(static fn (string $name): Mailbox => new Mailbox($name, '/', []), $remote);
return (new HarmonizationService($mailboxes, $messages, $files))->for($service, $live);
return (new HarmonizationService($mailboxes, $messages, $files, $this->createStub(MessageIngestor::class)))->for($service, $live);
}
private function dataStore(Collection $collection): DataStore
tests/php/Unit/MailboxSelectionTest.php
+116
View File
@@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig, ImapException};
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use PHPUnit\Framework\TestCase;
final class MailboxSelectionTest extends TestCase
{
private array $commands = [];
private array $failing = [];
public function testSameMailboxIsSelectedOnce(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityExtant('INBOX', 2);
$this->assertSame(1, $this->sent('EXAMINE "INBOX"'));
}
public function testOtherMailboxIsSelected(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityExtant('Sent', 1);
$service->entityExtant('INBOX', 1);
$this->assertSame(2, $this->sent('EXAMINE "INBOX"'));
$this->assertSame(1, $this->sent('EXAMINE "Sent"'));
}
public function testWriteAfterReadOnlySelectionSelectsReadWrite(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$service->entityPatch('INBOX', ['\\Seen'], [], 1);
$service->entityExtant('INBOX', 1);
$this->assertSame(1, $this->sent('EXAMINE "INBOX"'));
$this->assertSame(1, $this->sent('SELECT "INBOX"'));
}
public function testFailedSelectionIsNotReused(): void
{
$service = $this->service();
$service->entityExtant('INBOX', 1);
$this->failing = ['EXAMINE "Gone"'];
try {
$service->entityExtant('Gone', 1);
$this->fail('Expected the selection to fail');
} catch (ImapException) {
}
$service->entityExtant('INBOX', 1);
$this->assertSame(2, $this->sent('EXAMINE "INBOX"'));
}
private function sent(string $command): int
{
return count(array_filter($this->commands, static fn (string $sent): bool => $sent === $command));
}
private function service(): LiveMailService
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(function (string $wire) use (&$lines): void {
[$tag, $command] = explode(' ', trim($wire), 2);
$operation = explode(' ', $command)[0];
if ($operation === 'CAPABILITY') {
$lines[] = "* CAPABILITY IMAP4rev1\r\n";
} else {
$this->commands[] = $command;
}
if (in_array($command, $this->failing, true)) {
$lines[] = "$tag NO Mailbox does not exist\r\n";
return;
}
if (in_array($operation, ['SELECT', 'EXAMINE'], true)) {
$lines[] = "* 3 EXISTS\r\n";
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return new class($this->createStub(Service::class), $client) extends LiveMailService {
public function __construct(Service $service, private readonly Client $client)
{
parent::__construct($service);
}
public function imapClient(): Client
{
return $this->client;
}
};
}
}
tests/php/Unit/MessageHarmonizationTest.php
+303
View File
@@ -0,0 +1,303 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use Generator;
use KTXC\Db\DataStore;
use KTXM\ProviderImap\Client\Mailbox;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
use KTXM\ProviderImap\Client\Protocol\Parser\FetchMessageParser;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
use KTXM\ProviderImap\Stores\MessageStore;
use PHPUnit\Framework\TestCase;
final class MessageHarmonizationTest extends TestCase
{
private FakeMailboxStore $mailboxes;
private FakeMessageStore $messages;
private FakeMessageFileStore $files;
private MessageHarmonizationLiveStub $live;
private HarmonizationService $harmonizer;
protected function setUp(): void
{
$dataStore = $this->createStub(DataStore::class);
$this->mailboxes = new FakeMailboxStore($dataStore);
$this->messages = new FakeMessageStore($dataStore);
$this->files = new FakeMessageFileStore('/nonexistent');
$service = (new Service())->fromStore(['tid' => 'tenant', 'sid' => 'svc']);
$this->live = new MessageHarmonizationLiveStub($service);
$this->live->uidValidity = 7;
$this->live->uidNext = 100;
$this->harmonizer = (new HarmonizationService(
$this->mailboxes,
$this->messages,
$this->files,
new MessageIngestor($this->files, $this->messages),
))->for($service, $this->live);
$this->mailboxes->upsert('tenant', 'svc', (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', [])));
}
public function testFirstRunIngestsEverythingNewestFirst(): void
{
$this->live->flags = [1 => ['\\Seen'], 2 => [], 3 => ['\\Flagged']];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 3, 'updated' => 0, 'removed' => 0, 'complete' => true], $result);
$this->assertSame([[3, 2, 1]], $this->live->fetchBatches);
$this->assertSame([1 => ['seen'], 2 => [], 3 => ['flagged']], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([1, 2, 3], $this->files->uids(7));
$this->assertSame(262144, $this->live->bodyTextLimit);
$state = $this->mailboxes->state('svc', 'INBOX');
$this->assertSame(7, $state['uidValidity']);
$this->assertSame(100, $state['uidNext']);
$this->assertTrue($state['harmonizationComplete']);
$this->assertIsInt($state['harmonizedAt']);
$this->assertNull($this->mailboxes->lockOwner);
}
public function testIncrementalRunAddsUpdatesAndRemoves(): void
{
$this->live->flags = [1 => ['\\Seen'], 2 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$this->live->flags = [2 => ['\\Seen', '$Forwarded'], 3 => []];
$this->live->fetchBatches = [];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 1, 'updated' => 1, 'removed' => 1, 'complete' => true], $result);
$this->assertSame([[3]], $this->live->fetchBatches);
$this->assertSame([2 => ['seen', '$forwarded'], 3 => []], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([2, 3], $this->files->uids(7));
}
public function testChangedUidValidityDiscardsTheCache(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->harmonizer->harmonizeMessages('INBOX');
$this->live->uidValidity = 8;
$this->live->flags = [1 => []];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertTrue($result['reset']);
$this->assertSame(1, $result['added']);
$this->assertSame([], $this->messages->flags('svc', 'INBOX', 7));
$this->assertSame([1 => []], $this->messages->flags('svc', 'INBOX', 8));
$this->assertSame([], $this->files->uids(7));
$this->assertSame(8, $this->mailboxes->state('svc', 'INBOX')['uidValidity']);
}
public function testHeldLockSkipsTheRun(): void
{
$this->live->flags = [1 => []];
$this->mailboxes->lockOwner = 'someone-else';
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame('skipped', $result['status']);
$this->assertSame(0, $this->live->selects);
$this->assertSame('someone-else', $this->mailboxes->lockOwner);
}
public function testMessagesThatCannotBeFetchedLeaveTheMailboxIncomplete(): void
{
$this->live->flags = [1 => [], 2 => []];
$this->live->unfetchable = [2];
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(1, $result['added']);
$this->assertFalse($result['complete']);
$this->assertFalse($this->mailboxes->state('svc', 'INBOX')['harmonizationComplete']);
}
public function testNewMessagesAreFetchedInBatches(): void
{
$this->live->flags = array_fill_keys(range(1, 450), []);
$result = $this->harmonizer->harmonizeMessages('INBOX');
$this->assertSame(450, $result['added']);
$this->assertSame([200, 200, 50], array_map('count', $this->live->fetchBatches));
$this->assertSame(450, $this->live->fetchBatches[0][0]);
$this->assertSame(1, $this->live->fetchBatches[2][49]);
}
public function testLockIsReleasedWhenTheRunFails(): void
{
$this->live->uidValidity = null;
try {
$this->harmonizer->harmonizeMessages('INBOX');
$this->fail('Expected missing UIDVALIDITY to fail the run');
} catch (\RuntimeException) {
}
$this->assertNull($this->mailboxes->lockOwner);
}
}
final class MessageHarmonizationLiveStub extends LiveMailService
{
public ?int $uidValidity = null;
public ?int $uidNext = null;
/** @var array<int, list<string>> */
public array $flags = [];
/** @var int[] */
public array $unfetchable = [];
public array $fetchBatches = [];
public int $selects = 0;
public ?int $bodyTextLimit = null;
public function collectionFetch(string $identifier): ?Mailbox
{
$this->selects++;
return new Mailbox($identifier, '/', [], count($this->flags), 0, $this->uidValidity, 0, [], true, $this->uidNext);
}
public function entityFlags(string $collection): Generator
{
yield from $this->flags;
}
public function entityFetch(string $collection, ?FetchOptions $options = null, int ...$uids): Generator
{
$this->fetchBatches[] = $uids;
if (preg_match('/BODY\.PEEK\[TEXT\]<0\.(\d+)>/', (string) $options?->toCommand(), $matches) === 1) {
$this->bodyTextLimit = (int) $matches[1];
}
foreach ($uids as $uid) {
if (in_array($uid, $this->unfetchable, true)) {
continue;
}
$flags = implode(' ', $this->flags[$uid] ?? []);
yield $uid => FetchMessageParser::parse("* {$uid} FETCH (UID {$uid} FLAGS ({$flags}))");
}
}
}
final class FakeMailboxStore extends MailboxStore
{
/** @var array<string, array> */
public array $documents = [];
public ?string $lockOwner = null;
public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void
{
$name = (string) $collection->identifier();
$this->documents[$name] = ($this->documents[$name] ?? self::STATE_DEFAULTS) + ['name' => $name];
}
public function fetch(string $serviceId, string $name): ?array
{
return $this->documents[$name] ?? null;
}
public function list(string $serviceId): array
{
return $this->documents;
}
public function updateState(string $serviceId, string $name, array $state): void
{
$this->documents[$name] = array_replace($this->documents[$name], array_intersect_key($state, self::STATE_DEFAULTS));
}
public function acquireLock(string $serviceId, string $name, string $owner, int $ttl): bool
{
if ($this->lockOwner !== null && $this->lockOwner !== $owner) {
return false;
}
$this->lockOwner = $owner;
return true;
}
public function releaseLock(string $serviceId, string $name, string $owner): void
{
if ($this->lockOwner === $owner) {
$this->lockOwner = null;
}
}
}
final class FakeMessageStore extends MessageStore
{
/** @var array<int, array<int, list<string>>> flags by UIDVALIDITY and UID */
private array $documents = [];
public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity): void
{
$meta = $entity->toCacheMeta();
$this->documents[$uidValidity][$meta['uid']] = $meta['flags'];
ksort($this->documents[$uidValidity]);
}
public function flags(string $serviceId, string $mailbox, int $uidValidity): array
{
return $this->documents[$uidValidity] ?? [];
}
public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags): void
{
$this->documents[$uidValidity][$uid] = array_values($flags);
}
public function delete(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
{
foreach ($uids as $uid) {
unset($this->documents[$uidValidity][$uid]);
}
}
public function deleteByMailbox(string $serviceId, string $mailbox): void
{
$this->documents = [];
}
}
final class FakeMessageFileStore extends MessageFileStore
{
/** @var array<int, array<int, array>> content by UIDVALIDITY and UID */
private array $files = [];
public function write(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid, array $content): void
{
$this->files[$uidValidity][$uid] = $content;
}
public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
{
foreach ($uids as $uid) {
unset($this->files[$uidValidity][$uid]);
}
}
public function deleteByMailbox(string $tenantId, string $serviceId, string $mailbox): void
{
$this->files = [];
}
/** @return int[] */
public function uids(int $uidValidity): array
{
$uids = array_keys($this->files[$uidValidity] ?? []);
sort($uids);
return $uids;
}
}
tests/php/Unit/SelectCommandTest.php
+69
View File
@@ -0,0 +1,69 @@
<?php
declare(strict_types=1);
namespace KTXT\ProviderImap\Tests\Unit;
use KTXM\ProviderImap\Client\{Client, ConnectionConfig};
use KTXM\ProviderImap\Client\Protocol\Command\SelectCommand;
use KTXM\ProviderImap\Client\Transport\{ConnectionInterface, ConnectionFactoryInterface};
use PHPUnit\Framework\TestCase;
final class SelectCommandTest extends TestCase
{
public function testResponseCodesAreParsed(): void
{
$mailbox = $this->client([
"* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n",
"* OK [PERMANENTFLAGS ()] Read-only mailbox\r\n",
"* 172 EXISTS\r\n",
"* 1 RECENT\r\n",
"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n",
"* OK [UIDNEXT 4392] Predicted next UID\r\n",
"* OK [HIGHESTMODSEQ 715194045007] Highest\r\n",
])->perform(new SelectCommand('INBOX', true));
$this->assertSame(172, $mailbox->messages());
$this->assertSame(1, $mailbox->recent());
$this->assertSame(3857529045, $mailbox->uidValidity());
$this->assertSame(4392, $mailbox->uidNext());
$this->assertSame(715194045007, $mailbox->highestModSeq());
}
public function testMissingResponseCodesAreNull(): void
{
$mailbox = $this->client(["* 0 EXISTS\r\n"])->perform(new SelectCommand('INBOX', true));
$this->assertNull($mailbox->uidValidity());
$this->assertNull($mailbox->uidNext());
$this->assertNull($mailbox->highestModSeq());
}
/**
* @param string[] $untagged untagged lines the server sends before completing SELECT
*/
private function client(array $untagged): Client
{
$lines = ["* PREAUTH Ready\r\n"];
$connection = $this->createStub(ConnectionInterface::class);
$connection->method('readLine')->willReturnCallback(static function () use (&$lines): string {
return array_shift($lines) ?? throw new \RuntimeException('Unexpected response read');
});
$connection->method('write')->willReturnCallback(static function (string $wire) use (&$lines, $untagged): void {
[$tag, $command] = explode(' ', trim($wire), 2);
if (str_starts_with($command, 'CAPABILITY')) {
$lines[] = "* CAPABILITY IMAP4rev1\r\n";
}
if (str_starts_with($command, 'EXAMINE') || str_starts_with($command, 'SELECT')) {
array_push($lines, ...$untagged);
}
$lines[] = "$tag OK Completed\r\n";
});
$factory = $this->createStub(ConnectionFactoryInterface::class);
$factory->method('create')->willReturn($connection);
$client = new Client($factory);
$client->connect(new ConnectionConfig('localhost'));
return $client;
}
}