From 3f784b0a5b889f160b2c884bac4e29e1edebef94 Mon Sep 17 00:00:00 2001 From: Matyas Somfai Date: Tue, 8 Oct 2013 12:00:04 +0200 Subject: [PATCH] added new options to fos:elastica:populate command and options array parameter to ProviderInterface removed assume-yes option as the no-interaction option has the same purpose --- Command/PopulateCommand.php | 36 ++++++++++++++++++++++++---------- Doctrine/AbstractProvider.php | 11 ++++++++--- Propel/Provider.php | 11 ++++++++--- Provider/ProviderInterface.php | 3 ++- README.md | 15 +++++++------- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 4f88b75..237b223 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; +use Symfony\Component\Console\Helper\DialogHelper; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputInterface; @@ -43,6 +44,9 @@ class PopulateCommand extends ContainerAwareCommand ->addOption('index', null, InputOption::VALUE_OPTIONAL, 'The index to repopulate') ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'The type to repopulate') ->addOption('no-reset', null, InputOption::VALUE_NONE, 'Do not reset index before populating') + ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0) + ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0) + ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)') ->setDescription('Populates search indexes from providers') ; } @@ -62,9 +66,19 @@ class PopulateCommand extends ContainerAwareCommand */ protected function execute(InputInterface $input, OutputInterface $output) { - $index = $input->getOption('index'); - $type = $input->getOption('type'); - $reset = $input->getOption('no-reset') ? false : true; + $index = $input->getOption('index'); + $type = $input->getOption('type'); + $reset = $input->getOption('no-reset') ? false : true; + $noInteraction = $input->getOption('no-interaction'); + $options = $input->getOptions(); + + if (!$noInteraction && $reset && $input->getOption('offset')) { + /** @var DialogHelper $dialog */ + $dialog = $this->getHelperSet()->get('dialog'); + if (!$dialog->askConfirmation($output, 'You chose to reset the index and start indexing with an offset. Do you really want to do that?', true)) { + return; + } + } if (null === $index && null !== $type) { throw new \InvalidArgumentException('Cannot specify type option without an index.'); @@ -72,15 +86,15 @@ class PopulateCommand extends ContainerAwareCommand if (null !== $index) { if (null !== $type) { - $this->populateIndexType($output, $index, $type, $reset); + $this->populateIndexType($output, $index, $type, $reset, $options); } else { - $this->populateIndex($output, $index, $reset); + $this->populateIndex($output, $index, $reset, $options); } } else { $indexes = array_keys($this->indexManager->getAllIndexes()); foreach ($indexes as $index) { - $this->populateIndex($output, $index, $reset); + $this->populateIndex($output, $index, $reset, $options); } } } @@ -91,8 +105,9 @@ class PopulateCommand extends ContainerAwareCommand * @param OutputInterface $output * @param string $index * @param boolean $reset + * @param array $options */ - private function populateIndex(OutputInterface $output, $index, $reset) + private function populateIndex(OutputInterface $output, $index, $reset, $options) { if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); @@ -107,7 +122,7 @@ class PopulateCommand extends ContainerAwareCommand $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); }; - $provider->populate($loggerClosure); + $provider->populate($loggerClosure, $options); } $output->writeln(sprintf('Refreshing %s', $index)); @@ -121,8 +136,9 @@ class PopulateCommand extends ContainerAwareCommand * @param string $index * @param string $type * @param boolean $reset + * @param array $options */ - private function populateIndexType(OutputInterface $output, $index, $type, $reset) + private function populateIndexType(OutputInterface $output, $index, $type, $reset, $options) { if ($reset) { $output->writeln(sprintf('Resetting: %s/%s', $index, $type)); @@ -134,7 +150,7 @@ class PopulateCommand extends ContainerAwareCommand }; $provider = $this->providerRegistry->getProvider($index, $type); - $provider->populate($loggerClosure); + $provider->populate($loggerClosure, $options); $output->writeln(sprintf('Refreshing: %s', $index)); $this->indexManager->getIndex($index)->refresh(); diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 43e8c20..1fb41f5 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -31,16 +31,19 @@ abstract class AbstractProvider extends BaseAbstractProvider /** * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() */ - public function populate(\Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { $queryBuilder = $this->createQueryBuilder(); $nbObjects = $this->countObjects($queryBuilder); + $offset = isset($options['offset']) ? intval($options['offset']) : 0; + $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; + $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; - for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) { + for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { $stepStartTime = microtime(true); } - $objects = $this->fetchSlice($queryBuilder, $this->options['batch_size'], $offset); + $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); $this->objectPersister->insertMany($objects); @@ -48,6 +51,8 @@ abstract class AbstractProvider extends BaseAbstractProvider $this->managerRegistry->getManagerForClass($this->objectClass)->clear(); } + usleep($sleep); + if ($loggerClosure) { $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; diff --git a/Propel/Provider.php b/Propel/Provider.php index f173e72..c319691 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -14,23 +14,28 @@ class Provider extends AbstractProvider /** * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() */ - public function populate(\Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { $queryClass = $this->objectClass . 'Query'; $nbObjects = $queryClass::create()->count(); + $offset = isset($options['offset']) ? intval($options['offset']) : 0; + $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; + $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; - for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) { + for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { $stepStartTime = microtime(true); } $objects = $queryClass::create() - ->limit($this->options['batch_size']) + ->limit($batchSize) ->offset($offset) ->find(); $this->objectPersister->insertMany($objects->getArrayCopy()); + usleep($sleep); + if ($loggerClosure) { $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; diff --git a/Provider/ProviderInterface.php b/Provider/ProviderInterface.php index 710fd4b..e8d7ea4 100644 --- a/Provider/ProviderInterface.php +++ b/Provider/ProviderInterface.php @@ -13,7 +13,8 @@ interface ProviderInterface * Persists all domain objects to ElasticSearch for this provider. * * @param \Closure $loggerClosure + * @param array $options * @return */ - function populate(\Closure $loggerClosure = null); + function populate(\Closure $loggerClosure = null, array $options = array()); } diff --git a/README.md b/README.md index 28359f6..914150a 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,9 @@ Here we created a "website" index, that uses our "default" client. Our index is now available as a service: `fos_elastica.index.website`. It is an instance of `Elastica_Index`. -If you need to have different index name from the service name, for example, -in order to have different indexes for different environments then you can -use the ```index_name``` key to change the index name. The service name will +If you need to have different index name from the service name, for example, +in order to have different indexes for different environments then you can +use the ```index_name``` key to change the index name. The service name will remain the same across the environments: fos_elastica: @@ -86,8 +86,8 @@ remain the same across the environments: website: client: default index_name: website_qa - -The service id will be `fos_elastica.index.website` but the underlying index name is website_qa. + +The service id will be `fos_elastica.index.website` but the underlying index name is website_qa. #### Declare a type @@ -246,9 +246,10 @@ Its class must implement `FOS\ElasticaBundle\Provider\ProviderInterface`. /** * Insert the repository objects in the type index * - * @param Closure $loggerClosure + * @param \Closure $loggerClosure + * @param array $options */ - public function populate(Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { if ($loggerClosure) { $loggerClosure('Indexing users');