From cb7b4c1dcaddde70d5937fd9ea7905497d36c533 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 11 Mar 2015 15:09:48 +1100 Subject: [PATCH 1/5] Configurable ProgressBar format definition overrides --- Command/PopulateCommand.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 5817c4c..ceaef64 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -47,18 +47,26 @@ class PopulateCommand extends ContainerAwareCommand ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0) ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)') ->addOption('ignore-errors', null, InputOption::VALUE_NONE, 'Do not stop on errors') + ->addOption('no-overwrite-format', null, InputOption::VALUE_NONE, 'Prevent this command from overwriting ProgressBar\'s formats') ->setDescription('Populates search indexes from providers') ; } /** - * @see Symfony\Component\Console\Command\Command::initialize() + * {@inheritDoc} */ protected function initialize(InputInterface $input, OutputInterface $output) { $this->indexManager = $this->getContainer()->get('fos_elastica.index_manager'); $this->providerRegistry = $this->getContainer()->get('fos_elastica.provider_registry'); $this->resetter = $this->getContainer()->get('fos_elastica.resetter'); + + if (!$input->getOption('no-overwrite-format')) { + ProgressBar::setFormatDefinition('normal', " %current%/%max% [%bar%] %percent:3s%%\n%message%"); + ProgressBar::setFormatDefinition('verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%\n%message%"); + ProgressBar::setFormatDefinition('very_verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s%\n%message%"); + ProgressBar::setFormatDefinition('debug', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%\n%message%"); + } } /** @@ -155,10 +163,6 @@ class PopulateCommand extends ContainerAwareCommand }; } - ProgressBar::setFormatDefinition('normal', " %current%/%max% [%bar%] %percent:3s%%\n%message%"); - ProgressBar::setFormatDefinition('verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%\n%message%"); - ProgressBar::setFormatDefinition('very_verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s%\n%message%"); - ProgressBar::setFormatDefinition('debug', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%\n%message%"); $progress = null; return function ($increment, $totalObjects) use (&$progress, $output, $index, $type) { From 47785260a46b482ffa75fd65c44e12173ddac95b Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 11 Mar 2015 15:11:13 +1100 Subject: [PATCH 2/5] Fix an error that PopulateCommand would always ignore errors --- CHANGELOG-3.1.md | 2 ++ Command/PopulateCommand.php | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index 19bec1f..a5937c7 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -31,3 +31,5 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 that property while transforming. Combined with the above POST_TRANSFORM event developers can now create calculated dynamic properties on Elastica documents for indexing. #794 + * Fixed a case where ProgressCommand would always ignore errors regardless of + --ignore-errors being passed or not. diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index ceaef64..3bb6290 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -70,16 +70,15 @@ class PopulateCommand extends ContainerAwareCommand } /** - * @see Symfony\Component\Console\Command\Command::execute() + * {@inheritDoc} */ protected function execute(InputInterface $input, OutputInterface $output) { - $index = $input->getOption('index'); - $type = $input->getOption('type'); - $reset = !$input->getOption('no-reset'); - $options = $input->getOptions(); - - $options['ignore-errors'] = $input->hasOption('ignore-errors'); + $index = $input->getOption('index'); + $type = $input->getOption('type'); + $reset = !$input->getOption('no-reset'); + $options = $input->getOptions(); + $options['ignore-errors'] = $input->getOption('ignore-errors'); if ($input->isInteractive() && $reset && $input->getOption('offset')) { /** @var DialogHelper $dialog */ From ef2671dd36e69d7343091ec6a2a8a102b875fac4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 11 Mar 2015 15:11:42 +1100 Subject: [PATCH 3/5] Moved the progress helper closure building to a dedicated class --- Command/PopulateCommand.php | 66 +++------------------ Command/ProgressClosureBuilder.php | 94 ++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 59 deletions(-) create mode 100644 Command/ProgressClosureBuilder.php diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 3bb6290..e4ed4c9 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -23,6 +23,11 @@ class PopulateCommand extends ContainerAwareCommand */ private $indexManager; + /** + * @var ProgressClosureBuilder + */ + private $progressClosureBuilder; + /** * @var ProviderRegistry */ @@ -60,6 +65,7 @@ class PopulateCommand extends ContainerAwareCommand $this->indexManager = $this->getContainer()->get('fos_elastica.index_manager'); $this->providerRegistry = $this->getContainer()->get('fos_elastica.provider_registry'); $this->resetter = $this->getContainer()->get('fos_elastica.resetter'); + $this->progressClosureBuilder = new ProgressClosureBuilder(); if (!$input->getOption('no-overwrite-format')) { ProgressBar::setFormatDefinition('normal', " %current%/%max% [%bar%] %percent:3s%%\n%message%"); @@ -116,69 +122,11 @@ class PopulateCommand extends ContainerAwareCommand */ private function doPopulateType(ProviderInterface $provider, OutputInterface $output, $input, $type, $options) { - $loggerClosure = $this->getLoggerClosure($output, $input, $type); + $loggerClosure = $this->progressClosureBuilder->build($output, 'Populating', $input, $type); $provider->populate($loggerClosure, $options); } - /** - * Builds a loggerClosure to be called from inside the Provider to update the command - * line. - * - * @param OutputInterface $output - * @param string $index - * @param string $type - * @return callable - */ - private function getLoggerClosure(OutputInterface $output, $index, $type) - { - if (!class_exists('Symfony\Component\Console\Helper\ProgressBar')) { - $lastStep = null; - $current = 0; - - return function ($increment, $totalObjects) use ($output, $index, $type, &$lastStep, &$current) { - if ($current + $increment > $totalObjects) { - $increment = $totalObjects - $current; - } - - $currentTime = microtime(true); - $timeDifference = $currentTime - $lastStep; - $objectsPerSecond = $lastStep ? ($increment / $timeDifference) : $increment; - $lastStep = $currentTime; - $current += $increment; - $percent = 100 * $current / $totalObjects; - - $output->writeln(sprintf( - 'Populating %s/%s %0.1f%% (%d/%d), %d objects/s (RAM: current=%uMo peak=%uMo)', - $index, - $type, - $percent, - $current, - $totalObjects, - $objectsPerSecond, - round(memory_get_usage() / (1024 * 1024)), - round(memory_get_peak_usage() / (1024 * 1024)) - )); - }; - } - - $progress = null; - - return function ($increment, $totalObjects) use (&$progress, $output, $index, $type) { - if (null === $progress) { - $progress = new ProgressBar($output, $totalObjects); - $progress->start(); - } - - $progress->setMessage(sprintf('Populating %s/%s', $index, $type)); - $progress->advance($increment); - - if ($progress->getProgressPercent() >= 1.0) { - $progress->finish(); - } - }; - } - /** * Recreates an index, populates its types, and refreshes the index. * diff --git a/Command/ProgressClosureBuilder.php b/Command/ProgressClosureBuilder.php new file mode 100644 index 0000000..c312899 --- /dev/null +++ b/Command/ProgressClosureBuilder.php @@ -0,0 +1,94 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Command; + +use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Output\OutputInterface; + +class ProgressClosureBuilder +{ + /** + * Builds a loggerClosure to be called from inside the Provider to update the command + * line. + * + * @param OutputInterface $output + * @param string $action + * @param string $index + * @param string $type + * @return callable + */ + public function build(OutputInterface $output, $action, $index, $type) + { + if (!class_exists('Symfony\Component\Console\Helper\ProgressBar')) { + return $this->buildLegacy($output, $action, $index, $type); + } + + $progress = null; + + return function ($increment, $totalObjects) use (&$progress, $output, $action, $index, $type) { + if (null === $progress) { + $progress = new ProgressBar($output, $totalObjects); + $progress->start(); + } + + $progress->setMessage(sprintf('%s %s/%s', $action, $index, $type)); + $progress->advance($increment); + + if ($progress->getProgressPercent() >= 1.0) { + $progress->finish(); + } + }; + } + + /** + * Builds a legacy closure that outputs lines for each step. Used in cases + * where the ProgressBar component doesnt exist or does not have the correct + * methods to support what we need. + * + * @param OutputInterface $output + * @param string $action + * @param string $index + * @param string $type + * @return callable + */ + private function buildLegacy(OutputInterface $output, $action, $index, $type) + { + $lastStep = null; + $current = 0; + + return function ($increment, $totalObjects) use ($output, $action, $index, $type, &$lastStep, &$current) { + if ($current + $increment > $totalObjects) { + $increment = $totalObjects - $current; + } + + $currentTime = microtime(true); + $timeDifference = $currentTime - $lastStep; + $objectsPerSecond = $lastStep ? ($increment / $timeDifference) : $increment; + $lastStep = $currentTime; + $current += $increment; + $percent = 100 * $current / $totalObjects; + + $output->writeln(sprintf( + '%s %s/%s %0.1f%% (%d/%d), %d objects/s (RAM: current=%uMo peak=%uMo)', + $action, + $index, + $type, + $percent, + $current, + $totalObjects, + $objectsPerSecond, + round(memory_get_usage() / (1024 * 1024)), + round(memory_get_peak_usage() / (1024 * 1024)) + )); + }; + } +} From 4a564401b417cab6cf163a34e8a9a9d8b1699a7d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 11 Mar 2015 15:17:06 +1100 Subject: [PATCH 4/5] Symfony <=2.5 uses the legacy progress closure --- Command/ProgressClosureBuilder.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Command/ProgressClosureBuilder.php b/Command/ProgressClosureBuilder.php index c312899..b918511 100644 --- a/Command/ProgressClosureBuilder.php +++ b/Command/ProgressClosureBuilder.php @@ -28,7 +28,8 @@ class ProgressClosureBuilder */ public function build(OutputInterface $output, $action, $index, $type) { - if (!class_exists('Symfony\Component\Console\Helper\ProgressBar')) { + if (!class_exists('Symfony\Component\Console\Helper\ProgressBar') || + !is_callable(array('Symfony\Component\Console\Helper\ProgressBar', 'getProgress'))) { return $this->buildLegacy($output, $action, $index, $type); } @@ -42,10 +43,6 @@ class ProgressClosureBuilder $progress->setMessage(sprintf('%s %s/%s', $action, $index, $type)); $progress->advance($increment); - - if ($progress->getProgressPercent() >= 1.0) { - $progress->finish(); - } }; } From 0009c858a783c929f507376b3ebec7bad4574948 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 11 Mar 2015 21:31:23 +1100 Subject: [PATCH 5/5] Fixes for errors when with the progress closure --- Command/ProgressClosureBuilder.php | 14 ++++++++++++-- Doctrine/AbstractProvider.php | 2 +- Provider/ProviderInterface.php | 5 +++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Command/ProgressClosureBuilder.php b/Command/ProgressClosureBuilder.php index b918511..53bcb18 100644 --- a/Command/ProgressClosureBuilder.php +++ b/Command/ProgressClosureBuilder.php @@ -35,12 +35,18 @@ class ProgressClosureBuilder $progress = null; - return function ($increment, $totalObjects) use (&$progress, $output, $action, $index, $type) { + return function ($increment, $totalObjects, $message = null) use (&$progress, $output, $action, $index, $type) { if (null === $progress) { $progress = new ProgressBar($output, $totalObjects); $progress->start(); } + if (null !== $message) { + $progress->clear(); + $output->writeln(sprintf('%s %s', $action, $message)); + $progress->display(); + } + $progress->setMessage(sprintf('%s %s/%s', $action, $index, $type)); $progress->advance($increment); }; @@ -62,11 +68,15 @@ class ProgressClosureBuilder $lastStep = null; $current = 0; - return function ($increment, $totalObjects) use ($output, $action, $index, $type, &$lastStep, &$current) { + return function ($increment, $totalObjects, $message = null) use ($output, $action, $index, $type, &$lastStep, &$current) { if ($current + $increment > $totalObjects) { $increment = $totalObjects - $current; } + if (null !== $message) { + $output->writeln(sprintf('%s %s', $action, $message)); + } + $currentTime = microtime(true); $timeDifference = $currentTime - $lastStep; $objectsPerSecond = $lastStep ? ($increment / $timeDifference) : $increment; diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 80d0716..2d5d264 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -67,7 +67,7 @@ abstract class AbstractProvider extends BaseAbstractProvider $this->objectPersister->insertMany($objects); } catch(BulkResponseException $e) { if ($loggerClosure) { - $loggerClosure(sprintf('%s',$e->getMessage())); + $loggerClosure($batchSize, $nbObjects, sprintf('%s', $e->getMessage())); } } } diff --git a/Provider/ProviderInterface.php b/Provider/ProviderInterface.php index e8d7ea4..188f2a1 100644 --- a/Provider/ProviderInterface.php +++ b/Provider/ProviderInterface.php @@ -12,6 +12,11 @@ interface ProviderInterface /** * Persists all domain objects to ElasticSearch for this provider. * + * The closure can expect 2 or 3 arguments: + * * The step size + * * The total number of objects + * * A message to output in error conditions (not normally provided) + * * @param \Closure $loggerClosure * @param array $options * @return