Update search service

Trying to load custom Doctrine config

.editorconfig

@@ -0,0 +1,14 @@
+# This is the top-most .editorconfig file; do not search in parent directories.
+root = true
+
+# All files.
+[*]
+end_of_line = LF
+indent_style = space
+indent_size = 4
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_style = tab

.gitignore

@@ -1,2 +1,3 @@
 /vendor
 composer.lock
+.phpunit.result.cache

Containerfile

@@ -1,4 +1,4 @@
-FROM php:8.1-alpine
+FROM php:8.4-alpine
 
 WORKDIR /code
 COPY --from=composer:2 /usr/bin/composer /usr/bin/composer

Makefile

@@ -1,8 +0,0 @@
-PHP = docker compose run php
-
-
-composer_install:
-	@$(PHP) composer install
-
-static_analysis:
-	@$(PHP) vendor/bin/psalm

README.md

@@ -0,0 +1,86 @@
+# PCM Search Bundle
+
+Provides a search system for Symfony projects.
+
+## Installation
+
+Add the repo to your `composer.json` file:
+```json
+
+"repositories": [
+    {
+        "type": "vcs",
+        "url": "https://git.pcmdev.co.uk/pcm-libraries/pcm-search-bundle.git"
+    }
+]
+```
+
+Then install the bundle via Composer:
+
+```sh
+composer require pcm/search-bundle
+```
+
+## Usage
+
+Make any entity searchable by implementing `SearchableInterface`. The bundle's `SearchService` will then index, re-index, and remove entries automatically via Doctrine event subscribers.
+
+To (re)build the index for all searchable entities, run:
+
+```sh
+php bin/console pcm:search:reindex
+```
+
+Inject `Pcm\SearchBundle\Service\SearchService` into your own services to query, index, or un-index entities programmatically.
+
+### Adding a search bar
+
+Make your entity searchable by implementing `SearchableInterface`:
+
+```php
+use Pcm\SearchBundle\Interface\SearchableInterface;
+
+#[ORM\Entity]
+class Company implements SearchableInterface
+{
+    public function getSearchTitle(): string
+    {
+        return $this->name;
+    }
+
+    public function getSearchValues(): array
+    {
+        return [
+            $this->name,
+            $this->website
+        ];
+    }
+}
+```
+
+Query the index from a controller and resolve each hit to a URL:
+
+```php
+#[Route('/search', name: 'app_search', methods: ['GET'])]
+public function search(Request $request, SearchIndexRepository $repo): Response
+{
+    $routes = [Company::class => 'app_company_show'];
+    $rows = $repo->findAllPagination($request->query->get('q', ''))->getResult();
+
+    $results = [];
+    foreach ($rows as [$index]) {
+        $route = $routes[$index->getEntityClass()] ?? null;
+        if ($route === null) {
+            continue;
+        }
+        $results[] = [
+            'title' => $index->getTitle(),
+            'url'   => $this->generateUrl($route, ['id' => $index->getEntityId()]),
+        ];
+    }
+
+    return $this->render('search/_results.html.twig', ['results' => $results]);
+}
+```
+
+Each row from `findAllPagination()` is `[SearchIndex $hit, 'score' => float]`.

composer.json

@@ -3,17 +3,28 @@
     "description": "Provides a search system for Symfony projects",
     "type": "symfony-bundle",
     "require": {
-        "php": ">=8.0",
+        "php": ">=8.2",
-        "beberlei/doctrineextensions": "^1.3",
+        "beberlei/doctrineextensions": "^1.5",
-        "doctrine/doctrine-bundle": "^2.7",
+        "doctrine/doctrine-bundle": "^2.13",
-        "doctrine/orm": "^2.12",
+        "doctrine/orm": "^2.20 || ^3.3",
-        "symfony/framework-bundle": "*"
+        "doctrine/persistence": "^3.3 || ^4.0",
+        "symfony/framework-bundle": "^6.4 || ^7.0"
+    },
+    "require-dev": {
+        "vimeo/psalm": "^6.0",
+        "psalm/plugin-symfony": "^5.2",
+        "phpunit/phpunit": "^11.0 || ^12.0"
     },
     "autoload": {
         "psr-4": {
             "Pcm\\SearchBundle\\": "src/"
         }
     },
+    "autoload-dev": {
+        "psr-4": {
+            "Pcm\\SearchBundle\\": "tests/"
+        }
+    },
     "authors": [
         {
             "name": "Matt Feeny",
@@ -23,9 +34,5 @@
             "name": "Bradley Goode",
             "email": "bg@pcmsystems.co.uk"
         }
-    ],
+    ]
-    "require-dev": {
-        "vimeo/psalm": "^4.24",
-        "psalm/plugin-symfony": "^3.1"
-    }
 }

config/services.yaml

@@ -0,0 +1,21 @@
+services:
+    _defaults:
+        autowire: true
+        autoconfigure: true
+
+    pcm_search.command.index:
+        class: Pcm\SearchBundle\Command\SearchIndexCommand
+        tags:
+            - { name: "console.command", command: "pcm:search:reindex" }
+
+    Pcm\SearchBundle\EventSubscriber\SearchableListener: ~
+
+    pcm_search.search_service:
+        alias: Pcm\SearchBundle\Service\SearchService
+        public: true
+
+    Pcm\SearchBundle\Service\SearchService: ~
+
+    Pcm\SearchBundle\Repository\SearchIndexRepository:
+        autowire: true
+        tags: ["doctrine.repository_service"]

justfile

@@ -0,0 +1,17 @@
+php := "docker compose run php"
+
+composer-install:
+    @{{php}} composer install
+
+composer-update:
+    @{{php}} composer update
+
+composer *args:
+    @{{php}} composer {{args}}
+
+static-analysis:
+    @{{php}} vendor/bin/psalm
+
+tests:
+    @{{php}} rm -rf var/cache
+    @{{php}} vendor/bin/phpunit

phpunit.xml.dist

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd" backupGlobals="false" colors="true" bootstrap="./vendor/autoload.php">
+    <coverage>
+        <include>
+            <directory>./src</directory>
+        </include>
+    </coverage>
+    <php>
+        <ini name="error_reporting" value="-1"/>
+        <ini name="intl.default_locale" value="en"/>
+        <ini name="intl.error_level" value="0"/>
+        <ini name="memory_limit" value="-1"/>
+    </php>
+    <testsuites>
+        <testsuite name="Test suite">
+            <directory suffix="Test.php">./tests</directory>
+        </testsuite>
+    </testsuites>
+</phpunit>

src/Command/SearchIndexCommand.php

@@ -2,10 +2,12 @@
 
 namespace Pcm\SearchBundle\Command;
 
+use Doctrine\ORM\EntityManagerInterface;
 use Pcm\SearchBundle\Entity\SearchIndex;
 use Pcm\SearchBundle\Service\SearchService;
-use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Style\SymfonyStyle;
 use Symfony\Component\Console\Attribute\AsCommand;
@@ -23,7 +25,16 @@ class SearchIndexCommand extends Command
         parent::__construct();
     }
 
-    protected function configure(): void {}
+    protected function configure(): void
+    {
+        $this->addOption(
+            'batchsize',
+            'b',
+            InputOption::VALUE_REQUIRED,
+            'Number of rows to process before flushing',
+            2000
+        );
+    }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -42,29 +53,47 @@ class SearchIndexCommand extends Command
         $progress_bar->setProgressCharacter("<fg=green>➤</>");
         $progress_bar->start();
 
+        $batchSize = $input->getOption('batchsize');
+        $searchEntities = null;
+
         foreach ($searchables as $searchable) {
             $entities = $this
                 ->em
                 ->getRepository($searchable)
                 ->findAll();
 
+            $searchEntities[] = [$searchable, count($entities)];
+
+            $this->em->clear();
+            $counter = 1;
+
             foreach ($entities as $entity) {
                 $search_result = $this->searchService->createSearchResult($entity);
-
                 $this->em->persist($search_result);
 
+                if (($counter % $batchSize) === 0) {
+                    $this->em->flush();
+                    $this->em->clear();
+                }
+                $counter++;
             }
 
             $progress_bar->advance();
+            $this->em->flush();
+            $this->em->clear();
         }
 
-        $this->em->flush();
-
         $progress_bar->finish();
-
-        $io->writeln('');
         $io->writeln('');
 
+        $table = new Table($output);
+        $table
+            ->setHeaders(['Class', 'Count'])
+            ->setRows($searchEntities)
+        ;
+        $table->render();
+
+        $io->writeln('');
         $io->success('Index updated');
 
         return Command::SUCCESS;

src/DependencyInjection/PcmSearchExtension.php

@@ -0,0 +1,37 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Pcm\SearchBundle\DependencyInjection;
+
+use Symfony\Component\Config\FileLocator;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
+use Symfony\Component\DependencyInjection\Extension\PrependExtensionInterface;
+use Symfony\Component\HttpKernel\DependencyInjection\Extension;
+use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
+
+final class PcmSearchExtension extends Extension implements PrependExtensionInterface
+{
+    public function prepend(ContainerBuilder $container): void
+    {
+        $container->loadFromExtension(
+            'doctrine',
+            [
+                'orm' => [
+                    'dql' => [
+                        'string_functions' => [
+                            'match' => 'DoctrineExtensions\Query\Mysql\MatchAgainst'
+                        ]
+                    ]
+                ]
+            ]
+        );
+    }
+
+    public function load(array $configs, ContainerBuilder $container): void
+    {
+        $loader = new YamlFileLoader($container, new FileLocator(__DIR__.'/../../config'));
+
+        $loader->load('services.yaml');
+    }
+}

src/Entity/Interface/SearchableInterface.php

@@ -1,11 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Pcm\SearchBundle\Entity\Interface;
-
-interface SearchableInterface
-{
-    public function getSearchTitle(): string;
-    public function getSearchValues(): array;
-}

src/Entity/SearchIndex.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 namespace Pcm\SearchBundle\Entity;
 
 use Pcm\SearchBundle\Repository\SearchIndexRepository;
@@ -7,67 +9,71 @@ use Doctrine\ORM\Mapping as ORM;
 
 #[ORM\Entity(repositoryClass: SearchIndexRepository::class)]
 #[ORM\Index(name: "idx_data_fulltext", columns: ["data"], flags: ["fulltext"])]
-class SearchIndex
+final class SearchIndex
 {
     #[ORM\Id]
     #[ORM\GeneratedValue]
     #[ORM\Column(type: 'integer')]
-    private $id;
+    private ?int $id = null;
+
+    #[ORM\Column(name: 'data', type: 'text')]
+    private string $searchText;
 
     #[ORM\Column(type: 'string', length: 255)]
-    private $entityClass;
+    private string $title;
 
-    #[ORM\Column(type: 'integer')]
+    public function __construct(
-    private $entityId;
+        #[ORM\Column(type: 'string', length: 512)]
+        private readonly string $entityClass,
 
-    #[ORM\Column(type: 'text')]
+        #[ORM\Column(type: 'integer')]
-    private $data;
+        private readonly int $entityId,
-
+    ) {}
-    #[ORM\Column(type: 'string', length: 255)]
-    private $title;
 
+    /**
+     * The primary key of this index row. Null until the row is persisted.
+     */
     public function getId(): ?int
     {
         return $this->id;
     }
 
-    public function getEntityClass(): ?string
+    /**
+     * The fully-qualified class name of the indexed entity.
+     */
+    public function getEntityClass(): string
     {
         return $this->entityClass;
     }
 
-    public function setEntityClass(string $entityClass): self
+    /**
-    {
+     * The primary key of the indexed entity within its own table.
-        $this->entityClass = $entityClass;
+     */
-
+    public function getEntityId(): int
-        return $this;
-    }
-
-    public function getEntityId(): ?int
     {
         return $this->entityId;
     }
 
-    public function setEntityId(int $entityId): self
+    /**
+     * The concatenated searchable text for this entity. This is the value
+     * the fulltext index is matched against.
+     */
+    public function getSearchText(): string
     {
-        $this->entityId = $entityId;
+        return $this->searchText;
+    }
+
+    public function setSearchText(string $searchText): self
+    {
+        $this->searchText = $searchText;
 
         return $this;
     }
 
-    public function getData(): ?string
+    /**
-    {
+     * The human-readable title to display for this hit in search results.
-        return $this->data;
+     */
-    }
+    public function getTitle(): string
-
-    public function setData(string $data): self
-    {
-        $this->data = $data;
-
-        return $this;
-    }
-
-    public function getTitle(): ?string
     {
         return $this->title;
     }

src/EventSubscriber/SearchableListener.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Pcm\SearchBundle\EventSubscriber;
+
+use Pcm\SearchBundle\Interface\SearchableInterface;
+use Pcm\SearchBundle\Service\SearchService;
+use Doctrine\Bundle\DoctrineBundle\Attribute\AsDoctrineListener;
+use Doctrine\ORM\Event\PostPersistEventArgs;
+use Doctrine\ORM\Event\PostUpdateEventArgs;
+use Doctrine\ORM\Event\PreRemoveEventArgs;
+use Doctrine\ORM\Events;
+
+#[AsDoctrineListener(event: Events::postPersist)]
+#[AsDoctrineListener(event: Events::postUpdate)]
+#[AsDoctrineListener(event: Events::preRemove)]
+final class SearchableListener
+{
+    public function __construct(private SearchService $searchService)
+    {
+    }
+
+    public function postPersist(PostPersistEventArgs $args): void
+    {
+        $this->reindex($args->getObject());
+    }
+
+    public function postUpdate(PostUpdateEventArgs $args): void
+    {
+        $this->reindex($args->getObject());
+    }
+
+    public function preRemove(PreRemoveEventArgs $args): void
+    {
+        $entity = $args->getObject();
+
+        if ($entity instanceof SearchableInterface) {
+            $this->searchService->unIndex($entity);
+        }
+    }
+
+    private function reindex(object $entity): void
+    {
+        if ($entity instanceof SearchableInterface) {
+            $this->searchService->index($entity);
+        }
+    }
+}

src/EventSubscriber/SearchableSubscriber.php

@@ -1,52 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Pcm\SearchBundle\EventSubscriber;
-
-use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
-use Pcm\SearchBundle\Service\SearchService;
-use Doctrine\Common\EventSubscriber;
-use Doctrine\Persistence\Event\LifecycleEventArgs;
-use Doctrine\ORM\Events;
-
-class SearchableSubscriber implements EventSubscriber
-{
-    public function __construct(private SearchService $searchService) {}
-
-    public function getSubscribedEvents()
-    {
-        return [
-            Events::postPersist,
-            Events::postUpdate,
-            Events::preRemove
-        ];
-    }
-
-    public function postUpdate(LifecycleEventArgs $args)
-    {
-        $entity = $args->getObject();
-
-        if ($entity instanceof SearchableInterface) {
-            $this->searchService->index($entity);
-        }
-    }
-
-    public function postPersist(LifecycleEventArgs $args)
-    {
-        $entity = $args->getObject();
-
-        if ($entity instanceof SearchableInterface) {
-            $this->searchService->index($entity);
-        }
-    }
-
-    public function preRemove(LifecycleEventArgs $args)
-    {
-        $entity = $args->getObject();
-
-        if ($entity instanceof SearchableInterface) {
-            $this->searchService->unIndex($entity);
-        }
-    }
-}

src/Interface/SearchableInterface.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Pcm\SearchBundle\Interface;
+
+interface SearchableInterface
+{
+    /**
+     * This entity's ID.
+     */
+    public function getId(): ?int;
+
+    /**
+     * This entity's human-readable title to be shown in search results.
+     */
+    public function getSearchTitle(): string;
+
+    /**
+     * The string values to index for this entity
+     *
+     * Each value is concatenated into the fulltext index, so include any field a user might search by.
+     *
+     * @return string[]
+     */
+    public function getSearchValues(): array;
+}

src/Repository/SearchIndexRepository.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Pcm\SearchBundle\Repository;
+
+use Pcm\SearchBundle\Entity\SearchIndex;
+use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
+use Doctrine\ORM\Query;
+use Doctrine\Persistence\ManagerRegistry;
+
+final class SearchIndexRepository extends ServiceEntityRepository
+{
+    /**
+     * @param ManagerRegistry $registry
+     */
+    public function __construct(ManagerRegistry $registry)
+    {
+        parent::__construct($registry, SearchIndex::class);
+    }
+
+    /**
+     * Returns a Query object ready to be paginated or used to present results.
+     */
+    public function findAllPagination(string $query, int $minScore = 0): Query
+    {
+        return $this->createQueryBuilder('r')
+            ->addSelect('MATCH(r.searchText) AGAINST(:searchText boolean) AS score')
+            ->where('MATCH(r.searchText) AGAINST(:searchText boolean) > :minScore')
+            ->orderBy('score', 'DESC')
+            ->setParameter('searchText', $this->convertSearchTerm($query))
+            ->setParameter('minScore', $minScore)
+            ->getQuery();
+    }
+
+    /**
+     * Splits the query into words and wraps each as `+WORD*`, so the fulltext
+     * search requires every word and matches as a wildcard on each word's start.
+     */
+    private function convertSearchTerm(string $query): string
+    {
+        $sanitised = preg_replace('/[^\w\d]/', ' ', $query) ?? '';
+        $words = preg_split('/\s+/', $sanitised, -1, PREG_SPLIT_NO_EMPTY) ?: [];
+
+        $extractedWords = [];
+        foreach ($words as $word) {
+            $extractedWords[$word] = '+' . $word . '*';
+        }
+
+        return implode(' ', $extractedWords);
+    }
+
+    /**
+     * Clears the index table of all results
+     */
+    public function clearIndex(): void
+    {
+        $this
+            ->createQueryBuilder('s')
+            ->delete()
+            ->getQuery()
+            ->execute();
+    }
+}

src/Service/SearchService.php

@@ -5,90 +5,95 @@ declare(strict_types=1);
 namespace Pcm\SearchBundle\Service;
 
 use Pcm\SearchBundle\Entity\SearchIndex;
-use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
+use Pcm\SearchBundle\Interface\SearchableInterface;
-use Doctrine\Common\Util\ClassUtils;
 use Doctrine\ORM\EntityManagerInterface;
-use Doctrine\ORM\Proxy\Proxy;
 
-class SearchService
+final class SearchService
 {
-    public function __construct(private EntityManagerInterface $em) {}
+    public function __construct(private EntityManagerInterface $em)
-
-    public function index($entity)
     {
-        $searchIndex = $this->createSearchResult($entity);
+    }
 
-        $this->em->persist($searchIndex);
+    public function index(SearchableInterface $entity): void
+    {
+        $index = $this->createSearchResult($entity);
+
+        $this->em->persist($index);
         $this->em->flush();
     }
 
-    public function unIndex($entity)
+    public function unIndex(SearchableInterface $entity): void
     {
-        $class = get_class($entity);
+        $entityId = $this->requireEntityId($entity);
+        $index = $this->findIndex($this->resolveClass($entity), $entityId);
 
-        $search_result = $this
+        if ($index === null) {
-            ->em
+            return;
-            ->getRepository(SearchIndex::class)
-            ->findOneBy(['entityClass' => $class, 'entityId' => $entity->getId()]);
-
-        if ($search_result) {
-            $this->em->remove($search_result);
-            $this->em->flush();
-        }
-    }
-
-    public function createSearchResult(SearchableInterface $entity): SearchIndex
-    {
-        $values = [];
-
-        foreach ($entity->getSearchValues() as $value) {
-            $values[] = $value;
         }
 
-        $data = implode(' ', $values);
+        $this->em->remove($index);
-
+        $this->em->flush();
-        $class = get_class($entity);
-
-        if ($entity instanceof Proxy) {
-            $class = ClassUtils::getRealClass($class);
-        }
-
-        $searchResult = $this
-            ->em
-            ->getRepository(SearchIndex::class)
-            ->findOneBy(
-                [
-                    'entityClass' => $class,
-                    'entityId' => $entity->getId()
-                ]
-            );
-
-        if (!$searchResult) {
-            $searchResult = new SearchIndex();
-            $searchResult->setEntityClass($class);
-            $searchResult->setEntityId($entity->getId());
-        }
-
-        $searchResult->setTitle($entity->getSearchTitle());
-        $searchResult->setData($data);
-
-        return $searchResult;
     }
 
     /**
-     * Finds all searchable Doctrine entities.
+     * Finds or creates the SearchIndex row for the given entity and populates
+     * its title and search text from the entity. The returned row is unflushed.
+     */
+    public function createSearchResult(SearchableInterface $entity): SearchIndex
+    {
+        $entityId = $this->requireEntityId($entity);
+        $class = $this->resolveClass($entity);
+        $index = $this->findIndex($class, $entityId) ?? new SearchIndex($class, $entityId);
+
+        $index->setTitle($entity->getSearchTitle());
+        $index->setSearchText(implode(' ', $entity->getSearchValues()));
+
+        return $index;
+    }
+
+    /**
+     * @return class-string[]
      */
     public function getSearchableClasses(): array
     {
-        $metadata = $this->em->getMetadataFactory()->getAllMetadata();
+        $classes = [];
-        $searchables = [];
 
-        foreach ($metadata as $meta) {
+        foreach ($this->em->getMetadataFactory()->getAllMetadata() as $meta) {
             if ($meta->reflClass->implementsInterface(SearchableInterface::class)) {
-                $searchables[] = $meta->name;
+                $classes[] = $meta->name;
             }
         }
 
-        return $searchables;
+        return $classes;
+    }
+
+    private function findIndex(string $class, int $entityId): ?SearchIndex
+    {
+        return $this->em
+            ->getRepository(SearchIndex::class)
+            ->findOneBy(['entityClass' => $class, 'entityId' => $entityId]);
+    }
+
+    private function requireEntityId(SearchableInterface $entity): int
+    {
+        $entityId = $entity->getId();
+
+        if ($entityId === null) {
+            throw new \LogicException('Entity must be persisted before indexing.');
+        }
+
+        return $entityId;
+    }
+
+    /**
+     * @template T of object
+     *
+     * @param T $entity
+     *
+     * @return class-string<T>
+     */
+    private function resolveClass(object $entity): string
+    {
+        return $this->em->getClassMetadata($entity::class)->getName();
     }
 }