Compare commits
46 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| efc9b17136 | |||
| d3de91ea0b | |||
| f9700e1959 | |||
| 92bd22ad99 | |||
| 5548df12bd | |||
| 6e2c241349 | |||
| 802413f754 | |||
| e56ea29018 | |||
| 3faec6370c | |||
| 361cd7baf8 | |||
| 6111bd07e5 | |||
| 76ab724d1a | |||
| d7a7d0b6e8 | |||
| 3f57c78d01 | |||
| e88da82267 | |||
| 709c641fc9 | |||
| ff506e2aef | |||
| 702fe24be4 | |||
| 319334834b | |||
| 9bad1cb2b9 | |||
| df927a8197 | |||
| 5c98989631 | |||
| 373087a5d5 | |||
| 205b29e07c | |||
| e1d5f094de | |||
| 8145b87242 | |||
| d027933ad9 | |||
| cbad86a01c | |||
| 15bf7cb324 | |||
| 1b9787aa49 | |||
| 9d4297659d | |||
| 92842c552b | |||
| 80e584e29f | |||
| c03d2ae9ad | |||
| c27ae18aa0 | |||
| 89c88520d9 | |||
| c19a755ed1 | |||
| 12925cc7d7 | |||
| 393a704f1c | |||
| 398e2903ea | |||
| 95db65f56d | |||
| 1ec9b05413 | |||
| 6a4ce3bef3 | |||
| bf89b962ec | |||
| 7de30e78ac | |||
| 6d65df0b9c |
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM php:8.1-alpine
|
FROM php:8.4-alpine
|
||||||
|
|
||||||
WORKDIR /code
|
WORKDIR /code
|
||||||
COPY --from=composer:2 /usr/bin/composer /usr/bin/composer
|
COPY --from=composer:2 /usr/bin/composer /usr/bin/composer
|
||||||
|
|||||||
@@ -1,8 +0,0 @@
|
|||||||
PHP = docker compose run php
|
|
||||||
|
|
||||||
|
|
||||||
composer_install:
|
|
||||||
@$(PHP) composer install
|
|
||||||
|
|
||||||
static_analysis:
|
|
||||||
@$(PHP) vendor/bin/psalm
|
|
||||||
+9
-8
@@ -3,16 +3,17 @@
|
|||||||
"description": "Provides a search system for Symfony projects",
|
"description": "Provides a search system for Symfony projects",
|
||||||
"type": "symfony-bundle",
|
"type": "symfony-bundle",
|
||||||
"require": {
|
"require": {
|
||||||
"php": ">=8.0",
|
"php": ">=8.2",
|
||||||
"beberlei/doctrineextensions": "^1.3",
|
"beberlei/doctrineextensions": "^1.5",
|
||||||
"doctrine/doctrine-bundle": "^2.7",
|
"doctrine/doctrine-bundle": "^2.13",
|
||||||
"doctrine/orm": "^2.12",
|
"doctrine/orm": "^2.20 || ^3.3",
|
||||||
"symfony/framework-bundle": "*"
|
"doctrine/persistence": "^3.3 || ^4.0",
|
||||||
|
"symfony/framework-bundle": "^6.4 || ^7.0"
|
||||||
},
|
},
|
||||||
"require-dev": {
|
"require-dev": {
|
||||||
"vimeo/psalm": "^4.24",
|
"vimeo/psalm": "^6.0",
|
||||||
"psalm/plugin-symfony": "^3.1",
|
"psalm/plugin-symfony": "^5.2",
|
||||||
"phpunit/phpunit": "^9.5"
|
"phpunit/phpunit": "^11.0 || ^12.0"
|
||||||
},
|
},
|
||||||
"autoload": {
|
"autoload": {
|
||||||
"psr-4": {
|
"psr-4": {
|
||||||
|
|||||||
+19
-4
@@ -1,6 +1,21 @@
|
|||||||
services:
|
services:
|
||||||
pcm_search.searchable_subscriber:
|
_defaults:
|
||||||
class: Pcm\SearchBundle\EventSubscriber\SearchableSubscriber
|
autowire: true
|
||||||
public: true
|
autoconfigure: true
|
||||||
|
|
||||||
|
pcm_search.command.index:
|
||||||
|
class: Pcm\SearchBundle\Command\SearchIndexCommand
|
||||||
tags:
|
tags:
|
||||||
- { name: doctrine.event_subscriber }
|
- { name: 'console.command', command: 'pcm:search:reindex' }
|
||||||
|
|
||||||
|
Pcm\SearchBundle\EventSubscriber\SearchableSubscriber: ~
|
||||||
|
|
||||||
|
pcm_search.search_service:
|
||||||
|
alias: Pcm\SearchBundle\Service\SearchService
|
||||||
|
public: true
|
||||||
|
|
||||||
|
Pcm\SearchBundle\Service\SearchService: ~
|
||||||
|
|
||||||
|
Pcm\SearchBundle\Repository\SearchIndexRepository:
|
||||||
|
autowire: true
|
||||||
|
tags: ['doctrine.repository_service']
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
php := "docker compose run php"
|
||||||
|
|
||||||
|
composer-install:
|
||||||
|
@{{php}} composer install
|
||||||
|
|
||||||
|
composer-update:
|
||||||
|
@{{php}} composer update
|
||||||
|
|
||||||
|
composer *args:
|
||||||
|
@{{php}} composer {{args}}
|
||||||
|
|
||||||
|
static-analysis:
|
||||||
|
@{{php}} vendor/bin/psalm
|
||||||
|
|
||||||
|
tests:
|
||||||
|
@{{php}} rm -rf var/cache
|
||||||
|
@{{php}} vendor/bin/phpunit
|
||||||
-13
@@ -1,13 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<phpunit colors="true" bootstrap="vendor/autoload.php">
|
|
||||||
<testsuites>
|
|
||||||
<testsuite name="Test Suite">
|
|
||||||
<directory suffix="Test.php">./tests/</directory>
|
|
||||||
</testsuite>
|
|
||||||
</testsuites>
|
|
||||||
<filter>
|
|
||||||
<whitelist>
|
|
||||||
<directory>./src</directory>
|
|
||||||
</whitelist>
|
|
||||||
</filter>
|
|
||||||
</phpunit>
|
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd" backupGlobals="false" colors="true" bootstrap="./vendor/autoload.php">
|
||||||
|
<coverage>
|
||||||
|
<include>
|
||||||
|
<directory>./src</directory>
|
||||||
|
</include>
|
||||||
|
</coverage>
|
||||||
|
<php>
|
||||||
|
<ini name="error_reporting" value="-1"/>
|
||||||
|
<ini name="intl.default_locale" value="en"/>
|
||||||
|
<ini name="intl.error_level" value="0"/>
|
||||||
|
<ini name="memory_limit" value="-1"/>
|
||||||
|
</php>
|
||||||
|
<testsuites>
|
||||||
|
<testsuite name="Test suite">
|
||||||
|
<directory suffix="Test.php">./tests</directory>
|
||||||
|
</testsuite>
|
||||||
|
</testsuites>
|
||||||
|
</phpunit>
|
||||||
@@ -2,10 +2,12 @@
|
|||||||
|
|
||||||
namespace Pcm\SearchBundle\Command;
|
namespace Pcm\SearchBundle\Command;
|
||||||
|
|
||||||
|
use Doctrine\ORM\EntityManagerInterface;
|
||||||
use Pcm\SearchBundle\Entity\SearchIndex;
|
use Pcm\SearchBundle\Entity\SearchIndex;
|
||||||
use Pcm\SearchBundle\Service\SearchService;
|
use Pcm\SearchBundle\Service\SearchService;
|
||||||
use Doctrine\ORM\EntityManagerInterface;
|
use Symfony\Component\Console\Helper\Table;
|
||||||
use Symfony\Component\Console\Command\Command;
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Helper\ProgressBar;
|
use Symfony\Component\Console\Helper\ProgressBar;
|
||||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||||
use Symfony\Component\Console\Attribute\AsCommand;
|
use Symfony\Component\Console\Attribute\AsCommand;
|
||||||
@@ -23,7 +25,16 @@ class SearchIndexCommand extends Command
|
|||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function configure(): void {}
|
protected function configure(): void
|
||||||
|
{
|
||||||
|
$this->addOption(
|
||||||
|
'batchsize',
|
||||||
|
'b',
|
||||||
|
InputOption::VALUE_REQUIRED,
|
||||||
|
'Number of rows to process before flushing',
|
||||||
|
2000
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||||
{
|
{
|
||||||
@@ -42,29 +53,47 @@ class SearchIndexCommand extends Command
|
|||||||
$progress_bar->setProgressCharacter("<fg=green>➤</>");
|
$progress_bar->setProgressCharacter("<fg=green>➤</>");
|
||||||
$progress_bar->start();
|
$progress_bar->start();
|
||||||
|
|
||||||
|
$batchSize = $input->getOption('batchsize');
|
||||||
|
$searchEntities = null;
|
||||||
|
|
||||||
foreach ($searchables as $searchable) {
|
foreach ($searchables as $searchable) {
|
||||||
$entities = $this
|
$entities = $this
|
||||||
->em
|
->em
|
||||||
->getRepository($searchable)
|
->getRepository($searchable)
|
||||||
->findAll();
|
->findAll();
|
||||||
|
|
||||||
|
$searchEntities[] = [$searchable, count($entities)];
|
||||||
|
|
||||||
|
$this->em->clear();
|
||||||
|
$counter = 1;
|
||||||
|
|
||||||
foreach ($entities as $entity) {
|
foreach ($entities as $entity) {
|
||||||
$search_result = $this->searchService->createSearchResult($entity);
|
$search_result = $this->searchService->createSearchResult($entity);
|
||||||
|
|
||||||
$this->em->persist($search_result);
|
$this->em->persist($search_result);
|
||||||
|
|
||||||
|
if (($counter % $batchSize) === 0) {
|
||||||
|
$this->em->flush();
|
||||||
|
$this->em->clear();
|
||||||
|
}
|
||||||
|
$counter++;
|
||||||
}
|
}
|
||||||
|
|
||||||
$progress_bar->advance();
|
$progress_bar->advance();
|
||||||
|
$this->em->flush();
|
||||||
|
$this->em->clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->em->flush();
|
|
||||||
|
|
||||||
$progress_bar->finish();
|
$progress_bar->finish();
|
||||||
|
|
||||||
$io->writeln('');
|
|
||||||
$io->writeln('');
|
$io->writeln('');
|
||||||
|
|
||||||
|
$table = new Table($output);
|
||||||
|
$table
|
||||||
|
->setHeaders(['Class', 'Count'])
|
||||||
|
->setRows($searchEntities)
|
||||||
|
;
|
||||||
|
$table->render();
|
||||||
|
|
||||||
|
$io->writeln('');
|
||||||
$io->success('Index updated');
|
$io->success('Index updated');
|
||||||
|
|
||||||
return Command::SUCCESS;
|
return Command::SUCCESS;
|
||||||
|
|||||||
@@ -6,11 +6,28 @@ namespace Pcm\SearchBundle\DependencyInjection;
|
|||||||
|
|
||||||
use Symfony\Component\Config\FileLocator;
|
use Symfony\Component\Config\FileLocator;
|
||||||
use Symfony\Component\DependencyInjection\ContainerBuilder;
|
use Symfony\Component\DependencyInjection\ContainerBuilder;
|
||||||
|
use Symfony\Component\DependencyInjection\Extension\PrependExtensionInterface;
|
||||||
use Symfony\Component\HttpKernel\DependencyInjection\Extension;
|
use Symfony\Component\HttpKernel\DependencyInjection\Extension;
|
||||||
use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
|
use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
|
||||||
|
|
||||||
class PcmSearchExtension extends Extension
|
class PcmSearchExtension extends Extension implements PrependExtensionInterface
|
||||||
{
|
{
|
||||||
|
public function prepend(ContainerBuilder $container)
|
||||||
|
{
|
||||||
|
$container->loadFromExtension(
|
||||||
|
'doctrine',
|
||||||
|
[
|
||||||
|
'orm' => [
|
||||||
|
'dql' => [
|
||||||
|
'string_functions' => [
|
||||||
|
'match' => 'DoctrineExtensions\Query\Mysql\MatchAgainst'
|
||||||
|
]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
public function load(array $configs, ContainerBuilder $container)
|
public function load(array $configs, ContainerBuilder $container)
|
||||||
{
|
{
|
||||||
$loader = new YamlFileLoader(
|
$loader = new YamlFileLoader(
|
||||||
|
|||||||
@@ -6,24 +6,20 @@ namespace Pcm\SearchBundle\EventSubscriber;
|
|||||||
|
|
||||||
use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
|
use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
|
||||||
use Pcm\SearchBundle\Service\SearchService;
|
use Pcm\SearchBundle\Service\SearchService;
|
||||||
use Doctrine\Common\EventSubscriber;
|
use Doctrine\Bundle\DoctrineBundle\Attribute\AsDoctrineListener;
|
||||||
use Doctrine\Persistence\Event\LifecycleEventArgs;
|
use Doctrine\ORM\Event\PostPersistEventArgs;
|
||||||
|
use Doctrine\ORM\Event\PostUpdateEventArgs;
|
||||||
|
use Doctrine\ORM\Event\PreRemoveEventArgs;
|
||||||
use Doctrine\ORM\Events;
|
use Doctrine\ORM\Events;
|
||||||
|
|
||||||
class SearchableSubscriber implements EventSubscriber
|
#[AsDoctrineListener(event: Events::postPersist)]
|
||||||
|
#[AsDoctrineListener(event: Events::postUpdate)]
|
||||||
|
#[AsDoctrineListener(event: Events::preRemove)]
|
||||||
|
class SearchableSubscriber
|
||||||
{
|
{
|
||||||
public function __construct(private SearchService $searchService) {}
|
public function __construct(private SearchService $searchService) {}
|
||||||
|
|
||||||
public function getSubscribedEvents()
|
public function postUpdate(PostUpdateEventArgs $args): void
|
||||||
{
|
|
||||||
return [
|
|
||||||
Events::postPersist,
|
|
||||||
Events::postUpdate,
|
|
||||||
Events::preRemove
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
public function postUpdate(LifecycleEventArgs $args)
|
|
||||||
{
|
{
|
||||||
$entity = $args->getObject();
|
$entity = $args->getObject();
|
||||||
|
|
||||||
@@ -32,7 +28,7 @@ class SearchableSubscriber implements EventSubscriber
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function postPersist(LifecycleEventArgs $args)
|
public function postPersist(PostPersistEventArgs $args): void
|
||||||
{
|
{
|
||||||
$entity = $args->getObject();
|
$entity = $args->getObject();
|
||||||
|
|
||||||
@@ -41,7 +37,7 @@ class SearchableSubscriber implements EventSubscriber
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function preRemove(LifecycleEventArgs $args)
|
public function preRemove(PreRemoveEventArgs $args): void
|
||||||
{
|
{
|
||||||
$entity = $args->getObject();
|
$entity = $args->getObject();
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Pcm\SearchBundle\Repository;
|
||||||
|
|
||||||
|
use Pcm\SearchBundle\Entity\SearchIndex;
|
||||||
|
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
|
||||||
|
use Doctrine\ORM\Query;
|
||||||
|
use Doctrine\Persistence\ManagerRegistry;
|
||||||
|
|
||||||
|
class SearchIndexRepository extends ServiceEntityRepository
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* @param ManagerRegistry $registry
|
||||||
|
*/
|
||||||
|
public function __construct(ManagerRegistry $registry)
|
||||||
|
{
|
||||||
|
parent::__construct($registry, SearchIndex::class);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves a SearchIndex entity
|
||||||
|
*
|
||||||
|
* @param SearchIndex $entity
|
||||||
|
* @param boolean $flush
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function add(SearchIndex $entity, bool $flush = false): void
|
||||||
|
{
|
||||||
|
$this->getEntityManager()->persist($entity);
|
||||||
|
|
||||||
|
if ($flush) {
|
||||||
|
$this->getEntityManager()->flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Removes a SearchIndex entity
|
||||||
|
*
|
||||||
|
* @param SearchIndex $entity
|
||||||
|
* @param boolean $flush
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function remove(SearchIndex $entity, bool $flush = false): void
|
||||||
|
{
|
||||||
|
$this->getEntityManager()->remove($entity);
|
||||||
|
|
||||||
|
if ($flush) {
|
||||||
|
$this->getEntityManager()->flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a Query object ready to be paginated or used to present results.
|
||||||
|
*
|
||||||
|
* @param string $query
|
||||||
|
* @param integer $minScore
|
||||||
|
* @return Query
|
||||||
|
*/
|
||||||
|
public function findAllPagination(string $query, int $minScore = 0): Query
|
||||||
|
{
|
||||||
|
$qb = $this->createQueryBuilder('r')
|
||||||
|
->addSelect('MATCH(r.data) AGAINST(:searchText boolean) AS score')
|
||||||
|
->where(
|
||||||
|
sprintf(
|
||||||
|
'MATCH(r.data) AGAINST(:searchText boolean) > %f',
|
||||||
|
$minScore
|
||||||
|
)
|
||||||
|
)->orderBy('score', 'DESC')
|
||||||
|
->setParameter(
|
||||||
|
'searchText',
|
||||||
|
$this->convertSearchTerm($query)
|
||||||
|
);
|
||||||
|
return $qb->getQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes a string $query and explodes into individual words. Each word
|
||||||
|
* is then prefixed with + and ends with *, making the full text search
|
||||||
|
* operate as wildcard on all words
|
||||||
|
*
|
||||||
|
* @param string $query
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
private function convertSearchTerm(string $query): string
|
||||||
|
{
|
||||||
|
$extractedWords = [];
|
||||||
|
$sanitisedString = preg_replace('/[^\w^\d]/', ' ', $query);
|
||||||
|
$words = mb_split('\s', preg_replace(['/([^\w+])/','/(\s+)/'], ' ', $sanitisedString));
|
||||||
|
|
||||||
|
foreach ($words as $word) {
|
||||||
|
if (strlen($word)< 1) {
|
||||||
|
//
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$word = strtoupper($word);
|
||||||
|
$extractedWords[$word] = $word;
|
||||||
|
}
|
||||||
|
|
||||||
|
array_walk(
|
||||||
|
$extractedWords,
|
||||||
|
function(&$word) {
|
||||||
|
// require every word but allow matching just the start
|
||||||
|
$word = '+' . $word . '*';
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return implode(' ', $extractedWords);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clears the index table of all results
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function clearIndex(): void
|
||||||
|
{
|
||||||
|
$this
|
||||||
|
->createQueryBuilder('s')
|
||||||
|
->delete()
|
||||||
|
->getQuery()
|
||||||
|
->execute();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,9 +6,7 @@ namespace Pcm\SearchBundle\Service;
|
|||||||
|
|
||||||
use Pcm\SearchBundle\Entity\SearchIndex;
|
use Pcm\SearchBundle\Entity\SearchIndex;
|
||||||
use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
|
use Pcm\SearchBundle\Entity\Interface\SearchableInterface;
|
||||||
use Doctrine\Common\Util\ClassUtils;
|
|
||||||
use Doctrine\ORM\EntityManagerInterface;
|
use Doctrine\ORM\EntityManagerInterface;
|
||||||
use Doctrine\ORM\Proxy\Proxy;
|
|
||||||
|
|
||||||
class SearchService
|
class SearchService
|
||||||
{
|
{
|
||||||
@@ -38,7 +36,7 @@ class SearchService
|
|||||||
*/
|
*/
|
||||||
public function unIndex(SearchableInterface $entity)
|
public function unIndex(SearchableInterface $entity)
|
||||||
{
|
{
|
||||||
$class = get_class($entity);
|
$class = $this->em->getClassMetadata($entity::class)->getName();
|
||||||
|
|
||||||
$search_result = $this
|
$search_result = $this
|
||||||
->em
|
->em
|
||||||
@@ -69,11 +67,7 @@ class SearchService
|
|||||||
|
|
||||||
$data = implode(' ', $values);
|
$data = implode(' ', $values);
|
||||||
|
|
||||||
$class = get_class($entity);
|
$class = $this->em->getClassMetadata($entity::class)->getName();
|
||||||
|
|
||||||
if ($entity instanceof Proxy) {
|
|
||||||
$class = ClassUtils::getRealClass($class);
|
|
||||||
}
|
|
||||||
|
|
||||||
$searchResult = $this
|
$searchResult = $this
|
||||||
->em
|
->em
|
||||||
|
|||||||
Reference in New Issue
Block a user