From e56ea290181a5757e77ddb6f84c58ea77632f958 Mon Sep 17 00:00:00 2001
From: Matt Feeny <mf@pcmsystems.co.uk>
Date: Mon, 9 Jan 2023 16:06:37 +0000
Subject: [PATCH] Adding batching to index command for performance

---
 src/Command/SearchIndexCommand.php | 42 +++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/src/Command/SearchIndexCommand.php b/src/Command/SearchIndexCommand.php
index 74376ad..a38ac7a 100644
--- a/src/Command/SearchIndexCommand.php
+++ b/src/Command/SearchIndexCommand.php
@@ -2,10 +2,12 @@
 
 namespace Pcm\SearchBundle\Command;
 
+use Doctrine\ORM\EntityManagerInterface;
 use Pcm\SearchBundle\Entity\SearchIndex;
 use Pcm\SearchBundle\Service\SearchService;
-use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Style\SymfonyStyle;
 use Symfony\Component\Console\Attribute\AsCommand;
@@ -23,7 +25,16 @@ class SearchIndexCommand extends Command
         parent::__construct();
     }
 
-    protected function configure(): void {}
+    protected function configure(): void
+    {
+        $this->addOption(
+            'batchsize',
+            'b',
+            InputOption::VALUE_REQUIRED,
+            'Number of rows to process before flushing',
+            2000
+        );
+    }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -42,29 +53,46 @@ class SearchIndexCommand extends Command
         $progress_bar->setProgressCharacter("<fg=green>➤</>");
         $progress_bar->start();
 
+        $batchSize = $input->getOption('batchsize');
+        $searchEntities = null;
+
         foreach ($searchables as $searchable) {
             $entities = $this
                 ->em
                 ->getRepository($searchable)
                 ->findAll();
 
+            $searchEntities[] = [$searchable, count($entities)];
+
+            $this->em->clear();
+            $counter = 1;
+
             foreach ($entities as $entity) {
                 $search_result = $this->searchService->createSearchResult($entity);
-
                 $this->em->persist($search_result);
 
+                if (($counter % $batchSize) === 0) {
+                    $this->em->flush();
+                    $this->em->clear();
+                }
+                $counter++;
             }
 
             $progress_bar->advance();
+            $this->em->flush();
+            $this->em->clear();
         }
 
-        $this->em->flush();
-
         $progress_bar->finish();
 
-        $io->writeln('');
-        $io->writeln('');
+        $table = new Table($output);
+        $table
+            ->setHeaders(['Class', 'Count'])
+            ->setRows($searchEntities)
+        ;
+        $table->render();
 
+        $io->writeln('');
         $io->success('Index updated');
 
         return Command::SUCCESS;