diff --git a/composer.json b/composer.json index a198d5c..2d884fb 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,8 @@ "vendor-dir": "lib/vendor" }, "require": { - "php": ">=8.2 <=8.5" + "php": ">=8.2 <=8.5", + "sabre/vobject": "^4.5" }, "autoload": { "psr-4": { diff --git a/composer.lock b/composer.lock index 45d98d3..db4d0dc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,15 +4,259 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b932f96922722e0403842d3975e00a91", - "packages": [], + "content-hash": "76cf61017e98587f1cc5c8c148137bb9", + "packages": [ + { + "name": "sabre/uri", + "version": "3.1.0", + "source": { + "type": "git", + "url": "https://github.com/sabre-io/uri.git", + "reference": "a926c749dddfb289b8a9b5218d16ac06affdc631" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sabre-io/uri/zipball/a926c749dddfb289b8a9b5218d16ac06affdc631", + "reference": "a926c749dddfb289b8a9b5218d16ac06affdc631", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.95", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpstan/phpstan-strict-rules": "^2.0", + "phpunit/phpunit": "^10.5", + "rector/rector": "^2.4" + }, + "type": "library", + "autoload": { + "files": [ + "lib/functions.php" + ], + "psr-4": { + "Sabre\\Uri\\": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Evert Pot", + "email": "me@evertpot.com", + "homepage": "http://evertpot.com/", + "role": "Developer" + } + ], + "description": "Functions for making sense out of URIs.", + "homepage": "http://sabre.io/uri/", + "keywords": [ + "rfc3986", + "uri", + "url" + ], + "support": { + "forum": "https://groups.google.com/group/sabredav-discuss", + "issues": "https://github.com/sabre-io/uri/issues", + "source": "https://github.com/fruux/sabre-uri" + }, + "time": "2026-04-26T04:19:03+00:00" + }, + { + "name": "sabre/vobject", + "version": "4.6.0", + "source": { + "type": "git", + "url": "https://github.com/sabre-io/vobject.git", + "reference": "9432544fc369851fb8202c5d91159b2e669f0c88" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sabre-io/vobject/zipball/9432544fc369851fb8202c5d91159b2e669f0c88", + "reference": "9432544fc369851fb8202c5d91159b2e669f0c88", + "shasum": "" + }, + "require": { + "ext-mbstring": "*", + "php": "^7.1 || ^8.0", + "sabre/xml": "^2.1 || ^3.0 || ^4.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "~2.17.1", + "phpstan/phpstan": "^0.12 || ^1.12 || ^2.0", + "phpunit/php-invoker": "^2.0 || ^3.1", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.6" + }, + "suggest": { + "hoa/bench": "If you would like to run the benchmark scripts" + }, + "bin": [ + "bin/vobject", + "bin/generate_vcards" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Sabre\\VObject\\": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Evert Pot", + "email": "me@evertpot.com", + "homepage": "http://evertpot.com/", + "role": "Developer" + }, + { + "name": "Dominik Tobschall", + "email": "dominik@fruux.com", + "homepage": "http://tobschall.de/", + "role": "Developer" + }, + { + "name": "Ivan Enderlin", + "email": "ivan.enderlin@hoa-project.net", + "homepage": "http://mnt.io/", + "role": "Developer" + } + ], + "description": "The VObject library for PHP allows you to easily parse and manipulate iCalendar and vCard objects", + "homepage": "http://sabre.io/vobject/", + "keywords": [ + "availability", + "freebusy", + "iCalendar", + "ical", + "ics", + "jCal", + "jCard", + "recurrence", + "rfc2425", + "rfc2426", + "rfc2739", + "rfc4770", + "rfc5545", + "rfc5546", + "rfc6321", + "rfc6350", + "rfc6351", + "rfc6474", + "rfc6638", + "rfc6715", + "rfc6868", + "vCalendar", + "vCard", + "vcf", + "xCal", + "xCard" + ], + "support": { + "forum": "https://groups.google.com/group/sabredav-discuss", + "issues": "https://github.com/sabre-io/vobject/issues", + "source": "https://github.com/fruux/sabre-vobject" + }, + "time": "2026-05-31T13:04:55+00:00" + }, + { + "name": "sabre/xml", + "version": "4.1.0", + "source": { + "type": "git", + "url": "https://github.com/sabre-io/xml.git", + "reference": "bec83cbe2f4e1e1ca4254ed8d7caa7e32cc74b05" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sabre-io/xml/zipball/bec83cbe2f4e1e1ca4254ed8d7caa7e32cc74b05", + "reference": "bec83cbe2f4e1e1ca4254ed8d7caa7e32cc74b05", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-xmlreader": "*", + "ext-xmlwriter": "*", + "lib-libxml": ">=2.6.20", + "php": "^8.2", + "sabre/uri": ">=2.0,<4.0.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.95", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpstan/phpstan-strict-rules": "^2.0", + "phpunit/phpunit": "^10.5", + "rector/rector": "^2.4" + }, + "type": "library", + "autoload": { + "files": [ + "lib/Deserializer/functions.php", + "lib/Serializer/functions.php" + ], + "psr-4": { + "Sabre\\Xml\\": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Evert Pot", + "email": "me@evertpot.com", + "homepage": "http://evertpot.com/", + "role": "Developer" + }, + { + "name": "Markus Staab", + "email": "markus.staab@redaxo.de", + "role": "Developer" + } + ], + "description": "sabre/xml is an XML library that you may not hate.", + "homepage": "https://sabre.io/xml/", + "keywords": [ + "XMLReader", + "XMLWriter", + "dom", + "xml" + ], + "support": { + "forum": "https://groups.google.com/group/sabredav-discuss", + "issues": "https://github.com/sabre-io/xml/issues", + "source": "https://github.com/fruux/sabre-xml" + }, + "time": "2026-04-27T10:56:01+00:00" + } + ], "packages-dev": [], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, - "platform": [], - "platform-dev": [], - "plugin-api-version": "2.3.0" + "platform": { + "php": ">=8.2 <=8.5" + }, + "platform-dev": {}, + "platform-overrides": { + "php": "8.2" + }, + "plugin-api-version": "2.6.0" } diff --git a/lib/Controllers/DefaultController.php b/lib/Controllers/DefaultController.php index 58f5219..f117d36 100644 --- a/lib/Controllers/DefaultController.php +++ b/lib/Controllers/DefaultController.php @@ -23,7 +23,10 @@ use KTXF\Resource\Identifier\ResourceIdentifier; use KTXF\Resource\Identifier\ResourceIdentifiers; use KTXF\Resource\Identifier\ServiceIdentifier; use KTXF\Routing\Attributes\AuthenticatedRoute; +use KTXM\PeopleManager\Import\ImportOptions; +use KTXM\PeopleManager\Import\ImportService; use KTXM\PeopleManager\Manager; +use KTXM\PeopleManager\Stream\ExpectedTotal; use Psr\Log\LoggerInterface; use Throwable; @@ -50,6 +53,7 @@ class DefaultController extends ControllerAbstract { private readonly SessionTenant $tenantIdentity, private readonly SessionIdentity $userIdentity, private readonly Manager $manager, + private readonly ImportService $importService, private readonly LoggerInterface $logger, ) {} @@ -153,7 +157,8 @@ class DefaultController extends ControllerAbstract { 'entity.delete' => $this->entityDelete($tenantId, $userId, $data), 'entity.move' => $this->entityMove($tenantId, $userId, $data), 'entity.copy' => $this->entityCopy($tenantId, $userId, $data), - + 'entity.import' => $this->entityImport($tenantId, $userId, $data, $version, $transaction), + default => throw new InvalidArgumentException(self::ERR_INVALID_OPERATION . $operation) }; } @@ -540,39 +545,117 @@ class DefaultController extends ControllerAbstract { $sources = null; } - $filter = $data['filter'] ?? null; $sort = $data['sort'] ?? null; $range = $data['range'] ?? null; - $entityGenerator = $this->manager->entityListStream($tenantId, $userId, $sources, $filter, $sort, $range); - $logger = $this->logger; + $entities = $this->manager->entityListStream($tenantId, $userId, $sources, $filter, $sort, $range); - $responseGenerator = (function () use ($entityGenerator, $version, $transaction, $logger): \Generator { - yield ['type' => 'control', 'status' => 'start', 'version' => $version, 'transaction' => $transaction]; + return new StreamedNdJsonResponse( + $this->streamEnvelope($entities, $version, $transaction), + 1, + 200, + ['Content-Type' => 'application/json'], + ); + } - $total = 0; + /** + * Import vCards into a collection, streaming one NDJSON event per contact. + * + * Request data: { target: "provider:service:collection", data: "", options?: {...} } + */ + private function entityImport(string $tenantId, string $userId, array $data, int $version, string $transaction): StreamedNdJsonResponse { + + if (!isset($data['target'])) { + throw new InvalidArgumentException(self::ERR_MISSING_TARGET); + } + if (!is_string($data['target'])) { + throw new InvalidArgumentException(self::ERR_INVALID_TARGET); + } + if (!isset($data['data']) || !is_string($data['data']) || trim($data['data']) === '') { + throw new InvalidArgumentException('Invalid parameter: data must be a non-empty string'); + } + + $target = ResourceIdentifier::fromString($data['target']); + if (!$target instanceof CollectionIdentifier) { + throw new InvalidArgumentException('Invalid parameter: target must be provider:service:collection'); + } + + $options = ImportOptions::fromArray($data['options'] ?? []); + + // Spill the payload to a temp file and release the in-memory copy before the + // (potentially long) streaming parse, so peak memory stays at one contact object. + $tempFile = tmpfile(); + if ($tempFile === false) { + throw new \RuntimeException('Unable to allocate temporary file for import'); + } + fwrite($tempFile, $data['data']); + unset($data); + rewind($tempFile); + + $events = $this->importService->import($tempFile, $target, $options, $tenantId, $userId); + $frames = $this->streamEnvelope($events, $version, $transaction); + + // Stream the envelope, releasing the spilled temp file once it is fully + // drained (or the client disconnects) so it never outlives its stream. + $response = (function () use ($frames, $tempFile): \Generator { try { - foreach ($entityGenerator as $entity) { - if (!$entity instanceof JsonSerializable) { - continue; - } - yield [ - 'type' => 'data', - 'data' => $entity->jsonSerialize() - ]; - $total++; - } - } catch (\Throwable $t) { - $logger->error('Error streaming entities', ['exception' => $t]); - yield ['type' => 'error', 'message' => $t->getMessage()]; - return; + yield from $frames; + } finally { + fclose($tempFile); } - - yield ['type' => 'control', 'status' => 'end', 'total' => $total]; })(); - return new StreamedNdJsonResponse($responseGenerator, 1, 200, ['Content-Type' => 'application/json']); + return new StreamedNdJsonResponse($response, 1, 200); + } + + /** + * Wrap a generator of JsonSerializable domain objects in the canonical NDJSON + * stream envelope shared by every streaming operation: + * + * control:start {version, transaction, total?} — total? = expected count + * data {data} — one per domain object + * error {message} — on failure, then stop + * control:end {total} — total = objects emitted + * + * If the generator leads with an {@see ExpectedTotal} event, its value is + * folded into the start frame's `total` (the progress denominator) rather + * than emitted as a data frame. + * + * @param \Generator<\JsonSerializable> $items + */ + private function streamEnvelope(\Generator $items, int $version, string $transaction): \Generator { + // Peek the first event: an expected-total marker rides on the start frame. + $expected = null; + $items->rewind(); + if ($items->valid() && $items->current() instanceof ExpectedTotal) { + $expected = $items->current()->expectedTotal(); + $items->next(); + } + + $start = ['type' => 'control', 'status' => 'start', 'version' => $version, 'transaction' => $transaction]; + if ($expected !== null) { + $start['total'] = $expected; + } + yield $start; + + $total = 0; + try { + for (; $items->valid(); $items->next()) { + $item = $items->current(); + if (!$item instanceof \JsonSerializable) { + continue; + } + yield ['type' => 'data', 'data' => $item->jsonSerialize()]; + $total++; + } + } catch (\Throwable $t) { + $this->logger->error('Error streaming response', ['exception' => $t]); + yield ['type' => 'error', 'message' => $t->getMessage()]; + return; + } + + yield ['type' => 'control', 'status' => 'end', 'total' => $total]; } private function entityFetch(string $tenantId, string $userId, array $data): mixed { diff --git a/lib/Import/ImportCountEvent.php b/lib/Import/ImportCountEvent.php new file mode 100644 index 0000000..22a0ce1 --- /dev/null +++ b/lib/Import/ImportCountEvent.php @@ -0,0 +1,38 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +use KTXM\PeopleManager\Stream\ExpectedTotal; + +/** + * Carries the total number of vCards discovered in the source. + * Yielded once, before any object events, when counting is enabled. + * + * The stream envelope consumes this as the `control:start` total (the progress + * denominator); it is never emitted as a `data` frame, so it carries no payload + * of its own. + */ +final readonly class ImportCountEvent implements ImportEvent, ExpectedTotal { + + public function __construct( + public int $total, + ) {} + + public function expectedTotal(): int { + return $this->total; + } + + /** + * @return array{total: int} + */ + public function jsonSerialize(): array { + return ['total' => $this->total]; + } +} diff --git a/lib/Import/ImportDisposition.php b/lib/Import/ImportDisposition.php new file mode 100644 index 0000000..4b70c14 --- /dev/null +++ b/lib/Import/ImportDisposition.php @@ -0,0 +1,17 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +enum ImportDisposition: string { + case Created = 'created'; + case Updated = 'updated'; + case Exists = 'exists'; + case Error = 'error'; +} diff --git a/lib/Import/ImportEvent.php b/lib/Import/ImportEvent.php new file mode 100644 index 0000000..cffe149 --- /dev/null +++ b/lib/Import/ImportEvent.php @@ -0,0 +1,21 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +use JsonSerializable; + +/** + * Marker interface for events yielded by the contact import generator. + * + * Each event serializes to a flat NDJSON frame (no nested envelope); the + * controller appends the transaction id and streams it as one line. + */ +interface ImportEvent extends JsonSerializable { +} diff --git a/lib/Import/ImportObjectEvent.php b/lib/Import/ImportObjectEvent.php new file mode 100644 index 0000000..27bc8fd --- /dev/null +++ b/lib/Import/ImportObjectEvent.php @@ -0,0 +1,40 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +/** + * Yielded once per vCard processed, carrying its disposition and any errors. + */ +final readonly class ImportObjectEvent implements ImportEvent { + + /** + * @param list $errors + */ + public function __construct( + public ?string $identifier, + public ImportDisposition $disposition, + public array $errors = [], + ) {} + + public function isError(): bool { + return $this->disposition === ImportDisposition::Error; + } + + /** + * @return array{identifier: ?string, disposition: string, errors: list} + */ + public function jsonSerialize(): array { + return [ + 'identifier' => $this->identifier, + 'disposition' => $this->disposition->value, + 'errors' => $this->errors, + ]; + } +} diff --git a/lib/Import/ImportOptions.php b/lib/Import/ImportOptions.php new file mode 100644 index 0000000..75504ff --- /dev/null +++ b/lib/Import/ImportOptions.php @@ -0,0 +1,77 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +use InvalidArgumentException; + +/** + * Configuration for a contact import run. + */ +final class ImportOptions { + + public const ERROR_CONTINUE = 0; + public const ERROR_FAIL = 1; + public const ERROR_OPTIONS = [self::ERROR_CONTINUE, self::ERROR_FAIL]; + + /** Overwrite an existing entity when the same UID is already present. */ + private bool $supersede = false; + + /** Emit an ImportCountEvent (discovered total) before the object stream. */ + private bool $counts = true; + + /** How to handle per-object errors. */ + private int $errors = self::ERROR_CONTINUE; + + public function getSupersede(): bool { + return $this->supersede; + } + + public function setSupersede(bool $value): void { + $this->supersede = $value; + } + + public function getCounts(): bool { + return $this->counts; + } + + public function setCounts(bool $value): void { + $this->counts = $value; + } + + public function getErrors(): int { + return $this->errors; + } + + public function setErrors(int $value): void { + if (!in_array($value, self::ERROR_OPTIONS, true)) { + throw new InvalidArgumentException('Invalid errors option specified'); + } + $this->errors = $value; + } + + /** + * Build options from the raw `options` array carried in the request. + * + * @param array $data + */ + public static function fromArray(array $data): self { + $options = new self(); + if (isset($data['supersede'])) { + $options->setSupersede((bool)$data['supersede']); + } + if (isset($data['counts'])) { + $options->setCounts((bool)$data['counts']); + } + if (isset($data['errors'])) { + $options->setErrors((int)$data['errors']); + } + return $options; + } +} diff --git a/lib/Import/ImportService.php b/lib/Import/ImportService.php new file mode 100644 index 0000000..afb6bd7 --- /dev/null +++ b/lib/Import/ImportService.php @@ -0,0 +1,372 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Import; + +use Generator; +use InvalidArgumentException; +use KTXF\People\Entity as E; +use KTXF\People\Entity\Property as P; +use KTXF\Resource\Identifier\CollectionIdentifier; +use KTXM\PeopleManager\Manager; +use Sabre\VObject\Component\VCard as VCardComponent; +use Sabre\VObject\ParseException; +use Sabre\VObject\Parser\MimeDir; +use Sabre\VObject\Property; +use Sabre\VObject\Splitter\VCard as VCardSplitter; +use Throwable; + +/** + * Streams vCards from a file resource, maps each to a people entity, and creates + * it in the target collection — yielding one ImportEvent per card. + * + * Memory profile: one vCard is held at a time (sabre's streaming splitter), the + * generator keeps no running collection, and the discovered count is obtained by a + * cheap byte scan rather than a second parse. See modules/people/.ideas/vcf-import.md. + * + * v1 scope: every card is created (disposition `created`) or fails (`error`). The + * vCard UID is mapped to the entity `urid` so future supersede/exists detection is + * possible, but the local provider keys entities on a fresh server id and exposes no + * UID lookup, so supersede/updated/exists are deferred. Individual ORG affiliation + * mapping is also deferred (the ORG property on individual cards is not yet mapped). + */ +class ImportService { + + public function __construct( + private readonly Manager $manager, + ) {} + + /** + * @param resource $source readable, seekable file resource containing one or more vCards + * + * @return Generator + */ + public function import($source, CollectionIdentifier $target, ImportOptions $options, string $tenantId, string $userId): Generator { + if (!is_resource($source)) { + throw new InvalidArgumentException('Invalid import source: must be a file resource'); + } + + // Discovered count — cheap O(1)-memory byte scan, no second parse. + if ($options->getCounts()) { + yield new ImportCountEvent($this->countCards($source)); + } + + rewind($source); + $splitter = new VCardSplitter($source, MimeDir::OPTION_FORGIVING | MimeDir::OPTION_IGNORE_INVALID_LINES); + + while (true) { + try { + $vCard = $splitter->getNext(); + } catch (ParseException $e) { + // The parser position is unreliable after a parse error, so stop here. + yield new ImportObjectEvent(null, ImportDisposition::Error, ['Malformed vCard: ' . $e->getMessage()]); + return; + } + + if ($vCard === null) { + break; + } + + $uid = isset($vCard->UID) ? (string)$vCard->UID : null; + + try { + $properties = $this->mapVCard($vCard); + $this->manager->entityCreate($tenantId, $userId, $target, $properties); + yield new ImportObjectEvent($uid, ImportDisposition::Created); + } catch (Throwable $e) { + yield new ImportObjectEvent($uid, ImportDisposition::Error, [$e->getMessage()]); + if ($options->getErrors() === ImportOptions::ERROR_FAIL) { + return; + } + } + + unset($vCard); + } + } + + /** + * Count vCards by scanning for BEGIN:VCARD lines — no parsing, O(1) memory. + * + * @param resource $source + */ + private function countCards($source): int { + rewind($source); + $count = 0; + while (($line = fgets($source)) !== false) { + if (stripos($line, 'BEGIN:VCARD') === 0) { + $count++; + } + } + rewind($source); + return $count; + } + + /** + * Map a sabre vCard to an entity property array (the shape entity.create consumes). + * + * @return array + */ + private function mapVCard(VCardComponent $vCard): array { + $kind = isset($vCard->KIND) ? strtolower((string)$vCard->KIND) : 'individual'; + + return match ($kind) { + 'org', 'organization' => $this->mapOrganization($vCard)->jsonSerialize(), + 'group' => $this->mapGroup($vCard)->jsonSerialize(), + default => $this->mapIndividual($vCard)->jsonSerialize(), + }; + } + + private function mapIndividual(VCardComponent $vCard): E\IndividualObject { + $object = new E\IndividualObject(); + $object->urid = $this->uid($vCard); + $object->label = $this->text($vCard, 'FN'); + + // Structured name: N = family;given;additional;prefix;suffix + if (isset($vCard->N)) { + $parts = $vCard->N->getParts(); + $object->names->family = $this->part($parts, 0); + $object->names->given = $this->part($parts, 1); + $object->names->additional = $this->part($parts, 2); + $object->names->prefix = $this->part($parts, 3); + $object->names->suffix = $this->part($parts, 4); + } + + foreach ($vCard->select('EMAIL') as $prop) { + $email = new P\EmailObject(); + $email->address = (string)$prop; + $email->context = $this->firstType($prop); + $object->emails->add($email, $this->key()); + } + + foreach ($vCard->select('TEL') as $prop) { + $phone = new P\PhoneObject(); + $phone->number = (string)$prop; + $phone->context = $this->firstType($prop); + $object->phones->add($phone, $this->key()); + } + + foreach ($vCard->select('ADR') as $prop) { + $object->physicalLocations->add($this->address(new P\PhysicalLocationObject(), $prop), $this->key()); + } + + foreach ($vCard->select('URL') as $prop) { + $location = new P\VirtualLocationObject(); + $location->location = (string)$prop; + $location->context = $this->firstType($prop); + $object->virtualLocations->add($location, $this->key()); + } + + foreach ($vCard->select('NOTE') as $prop) { + $note = new P\NoteObject(); + $note->content = (string)$prop; + $object->notes->add($note, $this->key()); + } + + foreach ($vCard->select('TITLE') as $prop) { + $title = new P\TitleObject(); + $title->kind = P\TitleTypes::Title; + $title->label = (string)$prop; + $object->titles->add($title, $this->key()); + } + foreach ($vCard->select('ROLE') as $prop) { + $title = new P\TitleObject(); + $title->kind = P\TitleTypes::Role; + $title->label = (string)$prop; + $object->titles->add($title, $this->key()); + } + + if (isset($vCard->BDAY)) { + $this->anniversary($object, P\AnniversaryTypes::Birth, (string)$vCard->BDAY); + } + if (isset($vCard->ANNIVERSARY)) { + $this->anniversary($object, P\AnniversaryTypes::Nuptial, (string)$vCard->ANNIVERSARY); + } + + foreach ($this->categories($vCard) as $tag) { + $object->tags->add($tag); + } + + return $object; + } + + private function mapOrganization(VCardComponent $vCard): E\OrganizationObject { + $object = new E\OrganizationObject(); + $object->urid = $this->uid($vCard); + $object->label = $this->text($vCard, 'FN') ?? $this->text($vCard, 'ORG'); + $object->names->full = $object->label; + $object->names->sort = $this->text($vCard, 'SORT-STRING'); + + foreach ($vCard->select('EMAIL') as $prop) { + $email = new P\EmailObject(); + $email->address = (string)$prop; + $email->context = $this->firstType($prop); + $object->emails->add($email, $this->key()); + } + + foreach ($vCard->select('TEL') as $prop) { + $phone = new P\PhoneObject(); + $phone->number = (string)$prop; + $phone->context = $this->firstType($prop); + $object->phones->add($phone, $this->key()); + } + + foreach ($vCard->select('ADR') as $prop) { + $object->physicalLocations->add($this->address(new P\PhysicalLocationObject(), $prop), $this->key()); + } + + foreach ($vCard->select('URL') as $prop) { + $location = new P\VirtualLocationObject(); + $location->location = (string)$prop; + $location->context = $this->firstType($prop); + $object->virtualLocations->add($location, $this->key()); + } + + foreach ($vCard->select('NOTE') as $prop) { + $note = new P\NoteObject(); + $note->content = (string)$prop; + $object->notes->add($note, $this->key()); + } + + return $object; + } + + private function mapGroup(VCardComponent $vCard): E\GroupObject { + $object = new E\GroupObject(); + $object->urid = $this->uid($vCard); + $object->label = $this->text($vCard, 'FN'); + $object->names->full = $object->label; + + foreach ($vCard->select('MEMBER') as $prop) { + $member = new P\MemberObject(); + // MEMBER values are URIs, commonly urn:uuid:. + $member->entityId = preg_replace('/^urn:uuid:/i', '', (string)$prop); + $object->members->add($member, $this->key()); + } + + foreach ($vCard->select('URL') as $prop) { + $location = new P\VirtualLocationObject(); + $location->location = (string)$prop; + $location->context = $this->firstType($prop); + $object->virtualLocations->add($location, $this->key()); + } + + foreach ($vCard->select('NOTE') as $prop) { + $note = new P\NoteObject(); + $note->content = (string)$prop; + $object->notes->add($note, $this->key()); + } + + return $object; + } + + // ==================== Helpers ==================== + + /** + * Populate a physical-location object from an ADR property. + * ADR = pobox;ext(unit);street;locality;region;code;country + */ + private function address(P\PhysicalLocationObject $location, Property $prop): P\PhysicalLocationObject { + $parts = $prop->getParts(); + $location->box = $this->part($parts, 0); + $location->unit = $this->part($parts, 1); + $location->street = $this->part($parts, 2); + $location->locality = $this->part($parts, 3); + $location->region = $this->part($parts, 4); + $location->code = $this->part($parts, 5); + $location->country = $this->part($parts, 6); + $location->context = $this->firstType($prop); + return $location; + } + + private function anniversary(E\IndividualObject $object, P\AnniversaryTypes $type, string $value): void { + $date = $this->date($value); + if ($date === null) { + return; + } + $anniversary = new P\AnniversaryObject(); + $anniversary->type = $type; + $anniversary->when = $date; + $object->anniversaries->add($anniversary); + } + + private function uid(VCardComponent $vCard): ?string { + return isset($vCard->UID) ? (string)$vCard->UID : null; + } + + private function text(VCardComponent $vCard, string $name): ?string { + if (!isset($vCard->$name)) { + return null; + } + $value = trim((string)$vCard->$name); + return $value === '' ? null : $value; + } + + /** + * First TYPE parameter value, lowercased (used as `context`). + */ + private function firstType(Property $prop): ?string { + $type = $prop['TYPE'] ?? null; + if ($type === null) { + return null; + } + $parts = $type->getParts(); + return isset($parts[0]) && $parts[0] !== '' ? strtolower((string)$parts[0]) : null; + } + + /** + * @param array $parts + */ + private function part(array $parts, int $index): ?string { + if (!isset($parts[$index])) { + return null; + } + $value = trim((string)$parts[$index]); + return $value === '' ? null : $value; + } + + /** + * @return list + */ + private function categories(VCardComponent $vCard): array { + $tags = []; + foreach ($vCard->select('CATEGORIES') as $prop) { + foreach ($prop->getParts() as $part) { + $part = trim((string)$part); + if ($part !== '') { + $tags[] = $part; + } + } + } + return $tags; + } + + /** + * Best-effort parse of a vCard date value; null if unparseable. + */ + private function date(string $value): ?\DateTimeImmutable { + $value = trim($value); + if ($value === '') { + return null; + } + // Common compact form YYYYMMDD. + if (preg_match('/^\d{8}$/', $value)) { + $date = \DateTimeImmutable::createFromFormat('!Ymd', $value); + return $date ?: null; + } + try { + return new \DateTimeImmutable($value); + } catch (Throwable) { + return null; + } + } + + private function key(): string { + return bin2hex(random_bytes(8)); + } +} diff --git a/lib/Module.php b/lib/Module.php index 30483fb..3390a9b 100644 --- a/lib/Module.php +++ b/lib/Module.php @@ -14,6 +14,15 @@ class Module extends ModuleInstanceAbstract implements ModuleBrowserInterface public function __construct() { } + public function boot(): void + { + // Load module-local vendored dependencies (e.g. sabre/vobject for VCF import). + $vendorAutoload = __DIR__ . '/vendor/autoload.php'; + if (file_exists($vendorAutoload)) { + require_once $vendorAutoload; + } + } + public function handle(): string { return 'people_manager'; diff --git a/lib/Stream/ExpectedTotal.php b/lib/Stream/ExpectedTotal.php new file mode 100644 index 0000000..d2864c4 --- /dev/null +++ b/lib/Stream/ExpectedTotal.php @@ -0,0 +1,20 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\PeopleManager\Stream; + +/** + * Implemented by a stream event that declares, up front, how many data frames + * are expected to follow. When such an event leads a stream generator, the + * envelope writer folds its value into the `control:start` frame's `total` + * (the progress denominator) instead of emitting it as a `data` frame. + */ +interface ExpectedTotal { + public function expectedTotal(): int; +} diff --git a/src/services/entityService.ts b/src/services/entityService.ts index c47442f..24af93f 100644 --- a/src/services/entityService.ts +++ b/src/services/entityService.ts @@ -22,8 +22,8 @@ import type { EntityDeltaResponse, EntityMoveRequest, EntityMoveResponse, - EntityCopyRequest, - EntityCopyResponse, + EntityImportRequest, + EntityImportResponse, EntityInterface, } from '../types/entity'; import { useIntegrationStore } from '@KTXC/stores/integrationStore'; @@ -184,14 +184,25 @@ export const entityService = { }, /** - * Copy entities to a target collection + * Import vCards into a collection, streaming progress as it arrives. * - * @param request - copy request parameters + * @param request - import request (target collection, raw data, options) + * @param onObject - called synchronously for each per-contact result frame + * @param onDiscovered - called once with the discovered total (progress denominator) * - * @returns Promise with copy results keyed by source entity identifier + * @returns Promise resolving to { total } (objects processed) when the stream completes */ - async copy(request: EntityCopyRequest): Promise { - return await transceivePost('entity.copy', request); + async import( + request: EntityImportRequest, + onObject: (object: EntityImportResponse) => void, + onDiscovered: (expected: number) => void, + ): Promise<{ total: number }> { + return await transceiveStream( + 'entity.import', + request, + onObject, + { onStart: (expected) => { if (expected !== undefined) onDiscovered(expected); } }, + ); }, }; diff --git a/src/services/transceive.ts b/src/services/transceive.ts index 53d2aec..7dab875 100644 --- a/src/services/transceive.ts +++ b/src/services/transceive.ts @@ -53,32 +53,55 @@ export async function transceivePost( * Stream an NDJSON API response, unwrapping data frames for the caller. * * The server emits one JSON object per line with a transport-level `type` - * discriminant. This helper consumes control and error frames, forwards only - * unwrapped `data` payloads to the caller, and returns the final stream total. + * discriminant. This consumes the chunked body, splits it into lines, forwards + * only unwrapped `data` payloads to the caller, and returns the final total. * - * @param operation - Operation name, e.g. 'entity.listStream' + * @param operation - Operation name, e.g. 'entity.listStream', 'entity.import' * @param data - Operation-specific request data * @param onData - Synchronous callback invoked for every unwrapped data payload. * May throw to abort the stream. - * @param user - Optional user identifier override + * @param options - Optional `user` override and an `onStart` hook, invoked once + * with the expected total (the progress denominator) when the + * server declares one on the start frame. * @returns Promise resolving to the final stream total from the control/end frame */ export async function transceiveStream( operation: string, data: TRequest, onData: (data: TData) => void, - user?: string + options?: { user?: string; onStart?: (expected?: number) => void } ): Promise<{ total: number }> { const request: ApiRequest = { version: API_VERSION, transaction: generateTransactionId(), operation, data, - user, + user: options?.user, }; let total = 0; + // Interpret one NDJSON line: control frames carry start/end metadata, error + // frames abort, data frames are unwrapped to the caller. + const dispatch = (line: string): void => { + const message = JSON.parse(line) as ApiStreamResponse; + + if (message.type === 'control') { + if (message.status === 'start') { + options?.onStart?.(message.total); + } else if (message.status === 'end') { + total = message.total; + } + return; + } + + if (message.type === 'error') { + throw new Error(`[${operation}] ${message.message}`); + } + + onData(message.data); + }; + await fetchWrapper.post(API_URL, request, { headers: { 'Accept': 'application/json' }, onStream: async (response: Response) => { @@ -100,38 +123,12 @@ export async function transceiveStream( buffer = lines.pop()!; // retain any incomplete trailing chunk for (const line of lines) { - if (!line.trim()) continue; - const message = JSON.parse(line) as ApiStreamResponse; - - if (message.type === 'control') { - if (message.status === 'end') { - total = message.total; - } - continue; - } - - if (message.type === 'error') { - throw new Error(`[${operation}] ${message.message}`); - } - - onData(message.data); + if (line.trim()) dispatch(line); } } // flush any remaining bytes still in the buffer - if (buffer.trim()) { - const message = JSON.parse(buffer) as ApiStreamResponse; - - if (message.type === 'control') { - if (message.status === 'end') { - total = message.total; - } - } else if (message.type === 'error') { - throw new Error(`[${operation}] ${message.message}`); - } else { - onData(message.data); - } - } + if (buffer.trim()) dispatch(buffer); } finally { reader.releaseLock(); } diff --git a/src/stores/importStore.ts b/src/stores/importStore.ts new file mode 100644 index 0000000..7652ab7 --- /dev/null +++ b/src/stores/importStore.ts @@ -0,0 +1,222 @@ +/** + * Contact Import Store + * + * Drives the VCF import flow: a queue of files, each targeting a collection, imported + * sequentially while live counters stream in. Memory stays flat regardless of contact + * count — per-session counters are plain integers and recent results are capped. + */ + +import { ref, computed } from 'vue' +import { defineStore } from 'pinia' +import { entityService } from '../services/entityService' +import type { CollectionIdentifier } from '../types/common' +import type { EntityImportResponse } from '../types/entity' +import type { + ImportCounters, + ImportFileAdd, + ImportFileEntry, + ImportFileOptions, + ImportSession, + ImportSessionStage, +} from '../types/import' + +/** Max object results retained per session for UI display. */ +const RECENT_RESULTS_CAP = 25 + +function createEmptyCounters(): ImportCounters { + return { discovered: 0, processed: 0, created: 0, updated: 0, exists: 0, error: 0 } +} + +function defaultOptions(): ImportFileOptions { + return { supersede: false } +} + +export const useImportStore = defineStore('peopleImportStore', () => { + // State + const lastFileInsertId = ref(-1) + const files = ref([]) + const stage = ref('idle') + const running = ref(false) + const activeFileId = ref(null) + const lastError = ref(null) + const sessions = ref>({}) + const order = ref([]) + + // Computed + + /** Aggregate counters across every session. */ + const totals = computed(() => { + const aggregate = createEmptyCounters() + for (const id of order.value) { + const session = sessions.value[id] + if (!session) continue + aggregate.discovered += session.counters.discovered + aggregate.processed += session.counters.processed + aggregate.created += session.counters.created + aggregate.updated += session.counters.updated + aggregate.exists += session.counters.exists + aggregate.error += session.counters.error + } + return aggregate + }) + + const activeSession = computed(() => + activeFileId.value !== null ? sessions.value[activeFileId.value] ?? null : null, + ) + + // Actions + + /** Queue a file for import; returns its assigned id. */ + function addFile(file: ImportFileAdd): number { + const id = ++lastFileInsertId.value + files.value.push({ + file: { id, ...file }, + collectionId: null, + options: defaultOptions(), + }) + return id + } + + /** Clear the queue, retaining the insert counter. */ + function removeAllFiles(): void { + files.value = [] + } + + /** Reset run state, preserving queued files. */ + function reset(): void { + stage.value = 'idle' + running.value = false + activeFileId.value = null + lastError.value = null + sessions.value = {} + order.value = [] + } + + function entryFor(fileId: number): ImportFileEntry | undefined { + return files.value.find((entry) => entry.file.id === fileId) + } + + /** Set the destination collection for a queued file. */ + function setCollectionForFile(fileId: number, collectionId: CollectionIdentifier | null): void { + const entry = entryFor(fileId) + if (entry) entry.collectionId = collectionId + } + + /** Merge option changes for a queued file without dropping untouched keys. */ + function setOptionsForFile(fileId: number, options: Partial): void { + const entry = entryFor(fileId) + if (entry) entry.options = { ...entry.options, ...options } + } + + /** Record the discovered total (progress denominator) for a session. */ + function setDiscovered(fileId: number, expected: number): void { + const session = sessions.value[fileId] + if (session) session.counters.discovered = expected + } + + /** Fold one per-contact result into a session's counters and recent window. */ + function recordObject(fileId: number, object: EntityImportResponse): void { + const session = sessions.value[fileId] + if (!session) return + + session.recentResults = [object, ...session.recentResults].slice(0, RECENT_RESULTS_CAP) + session.counters.processed += 1 + session.counters[object.disposition] += 1 + } + + /** + * Import every queued file sequentially, streaming live counters into each session. + * + * @returns aggregated counters across all files + * @throws if a queued file has no destination collection + */ + async function startImport(): Promise { + const entries = files.value.slice() + if (entries.length === 0) { + stage.value = 'completed' + return createEmptyCounters() + } + + // Initialise sessions up front so the UI can render the full set. + sessions.value = {} + order.value = [] + for (const entry of entries) { + sessions.value[entry.file.id] = { + fileId: entry.file.id, + fileName: entry.file.name, + targetDisplayName: entry.collectionId ?? '', + targetIdentifier: entry.collectionId, + status: 'pending', + counters: createEmptyCounters(), + recentResults: [], + lastError: null, + } + order.value.push(entry.file.id) + } + + running.value = true + stage.value = 'importing' + lastError.value = null + + try { + for (const entry of entries) { + const session = sessions.value[entry.file.id] + if (!entry.collectionId) { + session.status = 'error' + session.lastError = 'No destination collection selected' + throw new Error(`Selected collection not found for "${entry.file.name}"`) + } + + activeFileId.value = entry.file.id + session.status = 'importing' + + try { + await entityService.import( + { target: entry.collectionId, data: entry.file.contents, options: entry.options }, + (object) => recordObject(entry.file.id, object), + (expected) => setDiscovered(entry.file.id, expected), + ) + session.status = session.counters.error > 0 ? 'error' : 'completed' + } catch (error) { + session.status = 'error' + session.lastError = error instanceof Error ? error.message : String(error) + throw error + } + } + + stage.value = 'completed' + return totals.value + } catch (error) { + stage.value = 'error' + lastError.value = error instanceof Error ? error.message : String(error) + throw error + } finally { + running.value = false + activeFileId.value = null + } + } + + return { + // state + lastFileInsertId, + files, + stage, + running, + activeFileId, + lastError, + sessions, + order, + // computed + totals, + activeSession, + // actions + addFile, + removeAllFiles, + reset, + setCollectionForFile, + setOptionsForFile, + startImport, + } +}) + +export default useImportStore diff --git a/src/stores/index.ts b/src/stores/index.ts index 800c024..9a4c988 100644 --- a/src/stores/index.ts +++ b/src/stores/index.ts @@ -2,3 +2,4 @@ export { useProvidersStore } from './providersStore'; export { useServicesStore } from './servicesStore'; export { useCollectionsStore } from './collectionsStore'; export { useEntitiesStore } from './entitiesStore'; +export { useImportStore } from './importStore'; diff --git a/src/types/common.ts b/src/types/common.ts index 6dd33be..b106f9a 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -44,13 +44,17 @@ export interface ApiErrorResponse { export type ApiResponse = ApiSuccessResponse | ApiErrorResponse; /** - * Stream control start line + * Stream control start line. + * + * `total`, when present, is the expected number of data frames (the progress + * denominator). Streams that cannot cheaply know their size up front omit it. */ export interface ApiStreamStartResponse { type: 'control'; status: 'start'; version: number; transaction: string; + total?: number; } /** diff --git a/src/types/entity.ts b/src/types/entity.ts index 5a815a1..fa54cd1 100644 --- a/src/types/entity.ts +++ b/src/types/entity.ts @@ -11,6 +11,7 @@ import type { import type { GroupInterface } from './group'; import type { IndividualInterface } from './individual'; import type { OrganizationInterface } from './organization'; +import type { ImportDisposition, ImportFileOptions } from './import'; export type EntityPropertiesInterface = IndividualInterface | OrganizationInterface | GroupInterface; @@ -167,3 +168,18 @@ export interface EntityMoveResponse { error?: string; }; } + +/** + * Entity import + */ +export interface EntityImportRequest { + target: CollectionIdentifier; + data: string; + options: ImportFileOptions; +} + +export interface EntityImportResponse { + identifier: string | null; + disposition: ImportDisposition; + errors: string[]; +} diff --git a/src/types/import.ts b/src/types/import.ts new file mode 100644 index 0000000..6f58aee --- /dev/null +++ b/src/types/import.ts @@ -0,0 +1,74 @@ +/** + * Types for the VCF / vCard contact import UI flow (file queue, options, and + * live session state). + * + * The wire request/response for the `entity.import` operation live alongside the + * other entity operations in `./entity` ({@link EntityImportRequest}, + * {@link EntityImportResponse}). + */ + +import type { CollectionIdentifier } from './common'; +import type { EntityImportResponse } from './entity'; + +export type ImportDisposition = 'created' | 'updated' | 'exists' | 'error'; + +export type ImportSessionStage = 'idle' | 'preparing' | 'selecting' | 'importing' | 'completed' | 'error'; + +/** + * Per-file import options sent to the backend. + */ +export interface ImportFileOptions { + supersede: boolean; +} + +/** + * A file queued for import (raw contents read from disk). + */ +export interface ImportFileSource { + id: number; + name: string; + contents: string; + size: number; + type: string; +} + +/** + * Payload for queueing a file (id is assigned by the store). + */ +export type ImportFileAdd = Omit; + +/** + * A queued file paired with its chosen target collection and options. + */ +export interface ImportFileEntry { + file: ImportFileSource; + collectionId: CollectionIdentifier | null; + options: ImportFileOptions; +} + +/** + * Aggregate counters for an import run. + */ +export interface ImportCounters { + discovered: number; + processed: number; + created: number; + updated: number; + exists: number; + error: number; +} + +/** + * Live state for one file's import session. + */ +export interface ImportSession { + fileId: number; + fileName: string; + targetDisplayName: string; + targetIdentifier: CollectionIdentifier | null; + status: 'pending' | 'importing' | 'completed' | 'error'; + counters: ImportCounters; + /** Bounded rolling window of recent object results, capped (see RECENT_RESULTS_CAP). */ + recentResults: EntityImportResponse[]; + lastError: string | null; +} diff --git a/src/types/index.ts b/src/types/index.ts index b73ea6c..0b33c8b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -2,6 +2,7 @@ export type * from './collection'; export type * from './common'; export type * from './entity'; export type * from './group'; +export type * from './import'; export type * from './individual'; export type * from './organization'; export type * from './provider';