deblan/FOSElasticaBundle
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

added new options to fos:elastica:populate command and options array parameter to ProviderInterface

Browse source 
removed assume-yes option as the no-interaction option has the same purpose
This commit is contained in:
Matyas Somfai 2013-10-08 12:00:04 +02:00 committed by Tim Nagel
parent 52d5d0d55f
commit 3f784b0a5b
5 changed files with 52 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
Command/PopulateCommand.php
View file

@ -3,6 +3,7 @@
namespace FOS\ElasticaBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Helper\DialogHelper;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputInterface;
@ -43,6 +44,9 @@ class PopulateCommand extends ContainerAwareCommand
->addOption('index', null, InputOption::VALUE_OPTIONAL, 'The index to repopulate')
->addOption('type', null, InputOption::VALUE_OPTIONAL, 'The type to repopulate')
->addOption('no-reset', null, InputOption::VALUE_NONE, 'Do not reset index before populating')
->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0)
->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0)
->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)')
->setDescription('Populates search indexes from providers')
;
}
@ -62,9 +66,19 @@ class PopulateCommand extends ContainerAwareCommand
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$index = $input->getOption('index');
$type = $input->getOption('type');
$reset = $input->getOption('no-reset') ? false : true;
$index = $input->getOption('index');
$type = $input->getOption('type');
$reset = $input->getOption('no-reset') ? false : true;
$noInteraction = $input->getOption('no-interaction');
$options = $input->getOptions();
if (!$noInteraction && $reset && $input->getOption('offset')) {
/** @var DialogHelper $dialog */
$dialog = $this->getHelperSet()->get('dialog');
if (!$dialog->askConfirmation($output, '<question>You chose to reset the index and start indexing with an offset. Do you really want to do that?</question>', true)) {
return;
}
}
if (null === $index && null !== $type) {
throw new \InvalidArgumentException('Cannot specify type option without an index.');
@ -72,15 +86,15 @@ class PopulateCommand extends ContainerAwareCommand
if (null !== $index) {
if (null !== $type) {
$this->populateIndexType($output, $index, $type, $reset);
$this->populateIndexType($output, $index, $type, $reset, $options);
} else {
$this->populateIndex($output, $index, $reset);
$this->populateIndex($output, $index, $reset, $options);
}
} else {
$indexes = array_keys($this->indexManager->getAllIndexes());
foreach ($indexes as $index) {
$this->populateIndex($output, $index, $reset);
$this->populateIndex($output, $index, $reset, $options);
}
}
}
@ -91,8 +105,9 @@ class PopulateCommand extends ContainerAwareCommand
* @param OutputInterface $output
* @param string $index
* @param boolean $reset
* @param array $options
*/
private function populateIndex(OutputInterface $output, $index, $reset)
private function populateIndex(OutputInterface $output, $index, $reset, $options)
{
if ($reset) {
$output->writeln(sprintf('<info>Resetting</info> <comment>%s</comment>', $index));
@ -107,7 +122,7 @@ class PopulateCommand extends ContainerAwareCommand
$output->writeln(sprintf('<info>Populating</info> %s/%s, %s', $index, $type, $message));
};
$provider->populate($loggerClosure);
$provider->populate($loggerClosure, $options);
}
$output->writeln(sprintf('<info>Refreshing</info> <comment>%s</comment>', $index));
@ -121,8 +136,9 @@ class PopulateCommand extends ContainerAwareCommand
* @param string $index
* @param string $type
* @param boolean $reset
* @param array $options
*/
private function populateIndexType(OutputInterface $output, $index, $type, $reset)
private function populateIndexType(OutputInterface $output, $index, $type, $reset, $options)
{
if ($reset) {
$output->writeln(sprintf('Resetting: %s/%s', $index, $type));
@ -134,7 +150,7 @@ class PopulateCommand extends ContainerAwareCommand
};
$provider = $this->providerRegistry->getProvider($index, $type);
$provider->populate($loggerClosure);
$provider->populate($loggerClosure, $options);
$output->writeln(sprintf('Refreshing: %s', $index));
$this->indexManager->getIndex($index)->refresh();

11
Doctrine/AbstractProvider.php
View file

@ -31,16 +31,19 @@ abstract class AbstractProvider extends BaseAbstractProvider
/**
* @see FOS\ElasticaBundle\Provider\ProviderInterface::populate()
*/
public function populate(\Closure $loggerClosure = null)
public function populate(\Closure $loggerClosure = null, array $options = array())
{
$queryBuilder = $this->createQueryBuilder();
$nbObjects = $this->countObjects($queryBuilder);
$offset = isset($options['offset']) ? intval($options['offset']) : 0;
$sleep = isset($options['sleep']) ? intval($options['sleep']) : 0;
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) {
for (; $offset < $nbObjects; $offset += $batchSize) {
if ($loggerClosure) {
$stepStartTime = microtime(true);
}
$objects = $this->fetchSlice($queryBuilder, $this->options['batch_size'], $offset);
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
$this->objectPersister->insertMany($objects);
@ -48,6 +51,8 @@ abstract class AbstractProvider extends BaseAbstractProvider
$this->managerRegistry->getManagerForClass($this->objectClass)->clear();
}
usleep($sleep);
if ($loggerClosure) {
$stepNbObjects = count($objects);
$stepCount = $stepNbObjects + $offset;

11
Propel/Provider.php
View file

@ -14,23 +14,28 @@ class Provider extends AbstractProvider
/**
* @see FOS\ElasticaBundle\Provider\ProviderInterface::populate()
*/
public function populate(\Closure $loggerClosure = null)
public function populate(\Closure $loggerClosure = null, array $options = array())
{
$queryClass = $this->objectClass . 'Query';
$nbObjects = $queryClass::create()->count();
$offset = isset($options['offset']) ? intval($options['offset']) : 0;
$sleep = isset($options['sleep']) ? intval($options['sleep']) : 0;
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) {
for (; $offset < $nbObjects; $offset += $batchSize) {
if ($loggerClosure) {
$stepStartTime = microtime(true);
}
$objects = $queryClass::create()
->limit($this->options['batch_size'])
->limit($batchSize)
->offset($offset)
->find();
$this->objectPersister->insertMany($objects->getArrayCopy());
usleep($sleep);
if ($loggerClosure) {
$stepNbObjects = count($objects);
$stepCount = $stepNbObjects + $offset;

3
Provider/ProviderInterface.php
View file

@ -13,7 +13,8 @@ interface ProviderInterface
* Persists all domain objects to ElasticSearch for this provider.
*
* @param \Closure $loggerClosure
* @param array $options
* @return
*/
function populate(\Closure $loggerClosure = null);
function populate(\Closure $loggerClosure = null, array $options = array());
}

15
README.md
View file

@ -74,9 +74,9 @@ Here we created a "website" index, that uses our "default" client.
Our index is now available as a service: `fos_elastica.index.website`. It is an instance of `Elastica_Index`.
If you need to have different index name from the service name, for example,
in order to have different indexes for different environments then you can
use the ```index_name``` key to change the index name. The service name will
If you need to have different index name from the service name, for example,
in order to have different indexes for different environments then you can
use the ```index_name``` key to change the index name. The service name will
remain the same across the environments:
fos_elastica:
@ -86,8 +86,8 @@ remain the same across the environments:
website:
client: default
index_name: website_qa
The service id will be `fos_elastica.index.website` but the underlying index name is website_qa.
The service id will be `fos_elastica.index.website` but the underlying index name is website_qa.
#### Declare a type
@ -246,9 +246,10 @@ Its class must implement `FOS\ElasticaBundle\Provider\ProviderInterface`.
/**
* Insert the repository objects in the type index
*
* @param Closure $loggerClosure
* @param \Closure $loggerClosure
* @param array $options
*/
public function populate(Closure $loggerClosure = null)
public function populate(\Closure $loggerClosure = null, array $options = array())
{
if ($loggerClosure) {
$loggerClosure('Indexing users');