generated from Nodarx/template
484 lines
18 KiB
PHP
484 lines
18 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace KTXT\ProviderImap\Tests\Unit;
|
|
|
|
use Generator;
|
|
use KTXF\Resource\Delta\Delta;
|
|
use KTXF\Resource\Filter\IFilter;
|
|
use KTXF\Resource\Sort\ISort;
|
|
use KTXC\Db\DataStore;
|
|
use KTXM\ProviderImap\Client\Mailbox;
|
|
use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
|
|
use KTXM\ProviderImap\Client\Protocol\Parser\FetchMessageParser;
|
|
use KTXM\ProviderImap\Providers\CollectionResource;
|
|
use KTXM\ProviderImap\Providers\EntityResource;
|
|
use KTXM\ProviderImap\Providers\LiveService;
|
|
use KTXM\ProviderImap\Service\Cache\HarmonizationService;
|
|
use KTXM\ProviderImap\Service\Cache\MessageDeltaService;
|
|
use KTXM\ProviderImap\Service\Cache\MessageIngestor;
|
|
use KTXM\ProviderImap\Service\Live\LiveMailService;
|
|
use KTXM\ProviderImap\Stores\MailboxStore;
|
|
use KTXM\ProviderImap\Stores\MessageFileStore;
|
|
use KTXM\ProviderImap\Stores\MessageStore;
|
|
use PHPUnit\Framework\TestCase;
|
|
|
|
final class MessageHarmonizationTest extends TestCase
|
|
{
|
|
private FakeMailboxStore $mailboxes;
|
|
private FakeMessageStore $messages;
|
|
private FakeMessageFileStore $files;
|
|
private MessageHarmonizationLiveStub $live;
|
|
private HarmonizationService $harmonizer;
|
|
private MessageDeltaService $deltas;
|
|
|
|
protected function setUp(): void
|
|
{
|
|
$dataStore = $this->createStub(DataStore::class);
|
|
$this->mailboxes = new FakeMailboxStore($dataStore);
|
|
$this->messages = new FakeMessageStore($dataStore);
|
|
$this->files = new FakeMessageFileStore('/nonexistent');
|
|
|
|
$service = (new LiveService())->fromStore(['tid' => 'tenant', 'sid' => 'svc']);
|
|
$this->live = new MessageHarmonizationLiveStub($service);
|
|
$this->live->uidValidity = 7;
|
|
$this->live->uidNext = 100;
|
|
|
|
$this->harmonizer = (new HarmonizationService(
|
|
$this->mailboxes,
|
|
$this->messages,
|
|
$this->files,
|
|
new MessageIngestor($this->files, $this->messages, $this->mailboxes),
|
|
))->for($service, $this->live);
|
|
$this->deltas = new MessageDeltaService($this->mailboxes, $this->messages);
|
|
|
|
$this->mailboxes->upsert('tenant', 'svc', (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', [])));
|
|
}
|
|
|
|
public function testFirstRunIngestsEverythingNewestFirst(): void
|
|
{
|
|
$this->live->flags = [1 => ['\\Seen'], 2 => [], 3 => ['\\Flagged']];
|
|
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 3, 'updated' => 0, 'removed' => 0, 'complete' => true], $result);
|
|
$this->assertSame([[3, 2, 1]], $this->live->fetchBatches);
|
|
$this->assertSame([1 => ['seen'], 2 => [], 3 => ['flagged']], $this->messages->flags('svc', 'INBOX', 7));
|
|
$this->assertSame([1, 2, 3], $this->files->uids(7));
|
|
$this->assertSame(262144, $this->live->bodyTextLimit);
|
|
|
|
$state = $this->mailboxes->state('svc', 'INBOX');
|
|
$this->assertSame(7, $state['uidValidity']);
|
|
$this->assertSame(100, $state['uidNext']);
|
|
$this->assertTrue($state['harmonizationComplete']);
|
|
$this->assertIsInt($state['harmonizedAt']);
|
|
$this->assertNull($this->mailboxes->lockOwner);
|
|
}
|
|
|
|
public function testIncrementalRunAddsUpdatesAndRemoves(): void
|
|
{
|
|
$this->live->flags = [1 => ['\\Seen'], 2 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->live->flags = [2 => ['\\Seen', '$Forwarded'], 3 => []];
|
|
$this->live->fetchBatches = [];
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 1, 'updated' => 1, 'removed' => 1, 'complete' => true], $result);
|
|
$this->assertSame([[3]], $this->live->fetchBatches);
|
|
$this->assertSame([2 => ['seen', '$forwarded'], 3 => []], $this->messages->flags('svc', 'INBOX', 7));
|
|
$this->assertSame([2, 3], $this->files->uids(7));
|
|
}
|
|
|
|
public function testChangedUidValidityDiscardsTheCache(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->live->uidValidity = 8;
|
|
$this->live->flags = [1 => []];
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertTrue($result['reset']);
|
|
$this->assertSame(1, $result['added']);
|
|
$this->assertSame([], $this->messages->flags('svc', 'INBOX', 7));
|
|
$this->assertSame([1 => []], $this->messages->flags('svc', 'INBOX', 8));
|
|
$this->assertSame([], $this->files->uids(7));
|
|
$this->assertSame(8, $this->mailboxes->state('svc', 'INBOX')['uidValidity']);
|
|
}
|
|
|
|
public function testHeldLockSkipsTheRun(): void
|
|
{
|
|
$this->live->flags = [1 => []];
|
|
$this->mailboxes->lockOwner = 'someone-else';
|
|
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertSame('skipped', $result['status']);
|
|
$this->assertSame(0, $this->live->selects);
|
|
$this->assertSame('someone-else', $this->mailboxes->lockOwner);
|
|
}
|
|
|
|
public function testMessagesThatCannotBeFetchedLeaveTheMailboxIncomplete(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => []];
|
|
$this->live->unfetchable = [2];
|
|
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertSame(1, $result['added']);
|
|
$this->assertFalse($result['complete']);
|
|
$this->assertFalse($this->mailboxes->state('svc', 'INBOX')['harmonizationComplete']);
|
|
}
|
|
|
|
public function testNewMessagesAreFetchedInBatches(): void
|
|
{
|
|
$this->live->flags = array_fill_keys(range(1, 450), []);
|
|
|
|
$result = $this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$this->assertSame(450, $result['added']);
|
|
$this->assertSame([200, 200, 50], array_map('count', $this->live->fetchBatches));
|
|
$this->assertSame(450, $this->live->fetchBatches[0][0]);
|
|
$this->assertSame(1, $this->live->fetchBatches[2][49]);
|
|
}
|
|
|
|
public function testHarmonizeAllSkipsUnselectableAndIsolatesFailures(): void
|
|
{
|
|
$this->live->flags = [1 => []];
|
|
$this->live->mailboxList = [
|
|
new Mailbox('INBOX', '/', []),
|
|
new Mailbox('Archive', '/', ['\\Noselect']),
|
|
new Mailbox('Broken', '/', []),
|
|
];
|
|
$this->live->failing = ['Broken'];
|
|
|
|
$result = $this->harmonizer->harmonizeAll();
|
|
|
|
$this->assertSame(['Archive', 'Broken'], $result['mailboxes']['added']);
|
|
$this->assertSame(['INBOX', 'Broken'], array_keys($result['messages']));
|
|
$this->assertSame('harmonized', $result['messages']['INBOX']['status']);
|
|
$this->assertSame('Mailbox not found on server: Broken', $result['messages']['Broken']);
|
|
$this->assertNull($this->mailboxes->lockOwner);
|
|
}
|
|
|
|
public function testEmptySignatureReturnsTheCurrentSignature(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => [], 3 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$delta = $this->deltas->delta('svc', 'INBOX', '');
|
|
|
|
$this->assertSame('7:3', $delta->signature);
|
|
$this->assertSame([[], [], []], $this->changeLists($delta));
|
|
}
|
|
|
|
public function testEmptySignatureWhileIncompleteStartsFromZero(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => []];
|
|
$this->live->unfetchable = [2];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
$delta = $this->deltas->delta('svc', 'INBOX', '');
|
|
$this->assertSame('7:0', $delta->signature);
|
|
|
|
$this->assertSame([['1'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:0')));
|
|
}
|
|
|
|
public function testDeltaReportsAdditionsModificationsAndDeletions(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => [], 3 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$signature = $this->deltas->delta('svc', 'INBOX', '')->signature;
|
|
|
|
$this->live->flags = [2 => ['\\Seen'], 3 => [], 4 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$delta = $this->deltas->delta('svc', 'INBOX', $signature);
|
|
|
|
$this->assertSame([['4'], ['2'], ['1']], $this->changeLists($delta));
|
|
$this->assertSame([[], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $delta->signature)));
|
|
}
|
|
|
|
public function testMessagesAddedSinceTheSignatureAreOnlyAdditions(): void
|
|
{
|
|
$this->live->flags = [1 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$signature = $this->deltas->delta('svc', 'INBOX', '')->signature;
|
|
|
|
$this->live->flags = [1 => [], 2 => [], 3 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$this->live->flags = [1 => [], 2 => ['\\Flagged']];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
|
|
// 2: added then flagged -> addition; 3: added then expunged -> not reported
|
|
$this->assertSame([['2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $signature)));
|
|
}
|
|
|
|
public function testSignatureThatCannotBeAnsweredIsReset(): void
|
|
{
|
|
$this->live->flags = [1 => [], 2 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$old = $this->deltas->delta('svc', 'INBOX', '')->signature;
|
|
|
|
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', 'garbage')));
|
|
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:999')));
|
|
|
|
$this->mailboxes->updateState('svc', 'INBOX', ['purgedSeq' => 2]);
|
|
$this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:1')));
|
|
|
|
$this->live->uidValidity = 8;
|
|
$this->live->flags = [5 => []];
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$delta = $this->deltas->delta('svc', 'INBOX', $old);
|
|
|
|
$this->assertSame([['5'], [], []], $this->changeLists($delta));
|
|
$this->assertStringStartsWith('8:', $delta->signature);
|
|
}
|
|
|
|
public function testMailboxThatIsNotCachedHasAnEmptyDelta(): void
|
|
{
|
|
$delta = $this->deltas->delta('svc', 'Unknown', '7:1');
|
|
|
|
$this->assertSame('', $delta->signature);
|
|
$this->assertSame([[], [], []], $this->changeLists($delta));
|
|
}
|
|
|
|
public function testLockIsReleasedWhenTheRunFails(): void
|
|
{
|
|
$this->live->uidValidity = null;
|
|
|
|
try {
|
|
$this->harmonizer->harmonizeMessages('INBOX');
|
|
$this->fail('Expected missing UIDVALIDITY to fail the run');
|
|
} catch (\RuntimeException) {
|
|
}
|
|
|
|
$this->assertNull($this->mailboxes->lockOwner);
|
|
}
|
|
|
|
/**
|
|
* @return array{0: string[], 1: string[], 2: string[]}
|
|
*/
|
|
private function changeLists(Delta $delta): array
|
|
{
|
|
return [
|
|
json_decode(json_encode($delta->additions), true),
|
|
json_decode(json_encode($delta->modifications), true),
|
|
json_decode(json_encode($delta->deletions), true),
|
|
];
|
|
}
|
|
}
|
|
|
|
final class MessageHarmonizationLiveStub extends LiveMailService
|
|
{
|
|
public ?int $uidValidity = null;
|
|
public ?int $uidNext = null;
|
|
/** @var array<int, list<string>> */
|
|
public array $flags = [];
|
|
/** @var int[] */
|
|
public array $unfetchable = [];
|
|
public array $fetchBatches = [];
|
|
public int $selects = 0;
|
|
public ?int $bodyTextLimit = null;
|
|
/** @var Mailbox[] */
|
|
public array $mailboxList = [];
|
|
/** @var string[] */
|
|
public array $failing = [];
|
|
|
|
public function collectionList(?string $location = null, ?IFilter $filter = null, ?ISort $sort = null): array
|
|
{
|
|
$list = [];
|
|
foreach ($this->mailboxList as $mailbox) {
|
|
$list[$mailbox->name()] = (new CollectionResource('imap', 'svc'))->fromImap($mailbox);
|
|
}
|
|
return $list;
|
|
}
|
|
|
|
public function mailboxFetch(string $identifier): ?Mailbox
|
|
{
|
|
if (in_array($identifier, $this->failing, true)) {
|
|
return null;
|
|
}
|
|
$this->selects++;
|
|
return new Mailbox($identifier, '/', [], count($this->flags), 0, $this->uidValidity, 0, [], true, $this->uidNext);
|
|
}
|
|
|
|
public function entityFlags(string $collection): Generator
|
|
{
|
|
yield from $this->flags;
|
|
}
|
|
|
|
public function messageFetch(string $collection, ?FetchOptions $options = null, int ...$uids): Generator
|
|
{
|
|
$this->fetchBatches[] = $uids;
|
|
if (preg_match('/BODY\.PEEK\[TEXT\]<0\.(\d+)>/', (string) $options?->toCommand(), $matches) === 1) {
|
|
$this->bodyTextLimit = (int) $matches[1];
|
|
}
|
|
|
|
foreach ($uids as $uid) {
|
|
if (in_array($uid, $this->unfetchable, true)) {
|
|
continue;
|
|
}
|
|
$flags = implode(' ', $this->flags[$uid] ?? []);
|
|
yield $uid => FetchMessageParser::parse("* {$uid} FETCH (UID {$uid} FLAGS ({$flags}))");
|
|
}
|
|
}
|
|
}
|
|
|
|
final class FakeMailboxStore extends MailboxStore
|
|
{
|
|
/** @var array<string, array> */
|
|
public array $documents = [];
|
|
public ?string $lockOwner = null;
|
|
|
|
public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void
|
|
{
|
|
$name = (string) $collection->identifier();
|
|
$this->documents[$name] = [
|
|
...($this->documents[$name] ?? self::STATE_DEFAULTS),
|
|
...$collection->toCacheMeta(),
|
|
];
|
|
}
|
|
|
|
public function fetch(string $serviceId, string $name): ?array
|
|
{
|
|
return $this->documents[$name] ?? null;
|
|
}
|
|
|
|
public function list(string $serviceId): array
|
|
{
|
|
return $this->documents;
|
|
}
|
|
|
|
public function updateState(string $serviceId, string $name, array $state): void
|
|
{
|
|
$this->documents[$name] = array_replace($this->documents[$name], array_intersect_key($state, self::STATE_DEFAULTS));
|
|
}
|
|
|
|
public function acquireLock(string $serviceId, string $name, string $owner, int $ttl): bool
|
|
{
|
|
if ($this->lockOwner !== null && $this->lockOwner !== $owner) {
|
|
return false;
|
|
}
|
|
$this->lockOwner = $owner;
|
|
return true;
|
|
}
|
|
|
|
public function releaseLock(string $serviceId, string $name, string $owner): void
|
|
{
|
|
if ($this->lockOwner === $owner) {
|
|
$this->lockOwner = null;
|
|
}
|
|
}
|
|
|
|
public function reserveSequence(string $serviceId, string $name, int $count = 1): int
|
|
{
|
|
$this->documents[$name]['changeSeq'] += $count;
|
|
return $this->documents[$name]['changeSeq'] - $count + 1;
|
|
}
|
|
|
|
public function state(string $serviceId, string $name): array
|
|
{
|
|
return array_replace(self::STATE_DEFAULTS, array_intersect_key($this->documents[$name] ?? [], self::STATE_DEFAULTS));
|
|
}
|
|
}
|
|
|
|
final class FakeMessageStore extends MessageStore
|
|
{
|
|
/** @var array<int, array<int, array{flags: list<string>, addedSeq: int, modSeq: int, removedSeq: ?int}>> by UIDVALIDITY and UID */
|
|
private array $documents = [];
|
|
|
|
public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity, int $seq): void
|
|
{
|
|
$meta = $entity->toCacheMeta();
|
|
$existing = $this->documents[$uidValidity][$meta['uid']] ?? null;
|
|
$this->documents[$uidValidity][$meta['uid']] = [
|
|
'flags' => $meta['flags'],
|
|
'addedSeq' => $existing['addedSeq'] ?? $seq,
|
|
'modSeq' => $seq,
|
|
'removedSeq' => null,
|
|
];
|
|
ksort($this->documents[$uidValidity]);
|
|
}
|
|
|
|
public function flags(string $serviceId, string $mailbox, int $uidValidity): array
|
|
{
|
|
$flags = [];
|
|
foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) {
|
|
if ($document['removedSeq'] === null) {
|
|
$flags[$uid] = $document['flags'];
|
|
}
|
|
}
|
|
return $flags;
|
|
}
|
|
|
|
public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags, int $seq): void
|
|
{
|
|
$this->documents[$uidValidity][$uid]['flags'] = array_values($flags);
|
|
$this->documents[$uidValidity][$uid]['modSeq'] = $seq;
|
|
}
|
|
|
|
public function tombstone(string $serviceId, string $mailbox, int $uidValidity, int $seq, int ...$uids): void
|
|
{
|
|
foreach ($uids as $uid) {
|
|
$this->documents[$uidValidity][$uid]['removedSeq'] = $seq;
|
|
$this->documents[$uidValidity][$uid]['modSeq'] = $seq;
|
|
$this->documents[$uidValidity][$uid]['flags'] = [];
|
|
}
|
|
}
|
|
|
|
public function changes(string $serviceId, string $mailbox, int $uidValidity, int $since): array
|
|
{
|
|
$changes = ['additions' => [], 'modifications' => [], 'deletions' => []];
|
|
foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) {
|
|
if ($document['removedSeq'] !== null) {
|
|
if ($document['addedSeq'] <= $since && $document['removedSeq'] > $since) {
|
|
$changes['deletions'][] = $uid;
|
|
}
|
|
} elseif ($document['addedSeq'] > $since) {
|
|
$changes['additions'][] = $uid;
|
|
} elseif ($document['modSeq'] > $since) {
|
|
$changes['modifications'][] = $uid;
|
|
}
|
|
}
|
|
return $changes;
|
|
}
|
|
|
|
public function deleteByMailbox(string $serviceId, string $mailbox): void
|
|
{
|
|
$this->documents = [];
|
|
}
|
|
}
|
|
|
|
final class FakeMessageFileStore extends MessageFileStore
|
|
{
|
|
/** @var array<int, array<int, array>> content by UIDVALIDITY and UID */
|
|
private array $files = [];
|
|
|
|
public function write(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid, array $content): void
|
|
{
|
|
$this->files[$uidValidity][$uid] = $content;
|
|
}
|
|
|
|
public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
|
|
{
|
|
foreach ($uids as $uid) {
|
|
unset($this->files[$uidValidity][$uid]);
|
|
}
|
|
}
|
|
|
|
public function deleteByMailbox(string $tenantId, string $serviceId, string $mailbox): void
|
|
{
|
|
$this->files = [];
|
|
}
|
|
|
|
/** @return int[] */
|
|
public function uids(int $uidValidity): array
|
|
{
|
|
$uids = array_keys($this->files[$uidValidity] ?? []);
|
|
sort($uids);
|
|
return $uids;
|
|
}
|
|
}
|