From c68bb411aca4ab58edd9f1ba4cadd4007a3509fd Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Mon, 28 Oct 2013 16:15:30 +0100 Subject: [PATCH 1/4] Add support for dynamic templates --- DependencyInjection/Configuration.php | 32 +++++++++++++++ DependencyInjection/FOSElasticaExtension.php | 6 +++ README.md | 30 ++++++++++++++ Resetter.php | 4 ++ .../DependencyInjection/ConfigurationTest.php | 41 +++++++++++++++++++ Tests/ResetterTest.php | 6 ++- 6 files changed, 118 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index d44e714..eb6abd2 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -265,6 +265,7 @@ class Configuration implements ConfigurationInterface ->end() ->end() ->append($this->getMappingsNode()) + ->append($this->getDynamicTemplateNode()) ->append($this->getSourceNode()) ->append($this->getBoostNode()) ->append($this->getRoutingNode()) @@ -296,6 +297,37 @@ class Configuration implements ConfigurationInterface return $node; } + /** + * Returns the array node used for "dynamic_templates". + */ + public function getDynamicTemplateNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('dynamic_templates'); + + $node + ->useAttributeAsKey('name') + ->prototype('array') + ->children() + ->scalarNode('match')->isRequired()->end() + ->scalarNode('match_mapping_type')->end() + ->arrayNode('mapping') + ->isRequired() + ->children() + ->scalarNode('type')->end() + ->scalarNode('index')->end() + ->arrayNode('fields') + ->children() + ->end() + ->end() + ->end() + ->end() + ->end() + ; + + return $node; + } + /** * @param \Symfony\Component\Config\Definition\Builder\NodeBuilder $node The node to which to attach the field config to * @param array $nestings the nested mappings for the current field level diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 6d38714..26cb209 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -205,6 +205,12 @@ class FOSElasticaExtension extends Extension if (isset($type['index'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['index'] = $type['index']; } + if (!empty($type['dynamic_templates'])) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'] = array(); + foreach ($type['dynamic_templates'] as $templateName => $templateData) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'][] = array($templateName => $templateData); + } + } } } diff --git a/README.md b/README.md index 1c5c5f3..28359f6 100644 --- a/README.md +++ b/README.md @@ -684,3 +684,33 @@ fos_elastica: lastlogin: { type: date, format: basic_date_time } birthday: { type: date, format: "yyyy-MM-dd" } ``` + +#### Dynamic templates + +Dynamic templates allow to define mapping templates that will be +applied when dynamic introduction of fields / objects happens. + +[Documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-root-object-type.html#_dynamic_templates) + +```yaml +fos_elastica: + clients: + default: { host: localhost, port: 9200 } + indexes: + site: + types: + user: + dynamic_templates: + my_template_1: + match: apples_* + mapping: + type: float + my_template_2: + match: * + match_mapping_type: string + mapping: + type: string + index: not_analyzed + mappings: + username: { type: string } +``` diff --git a/Resetter.php b/Resetter.php index 7614675..5929cc8 100644 --- a/Resetter.php +++ b/Resetter.php @@ -78,6 +78,10 @@ class Resetter } } + if (isset($indexConfig['dynamic_templates'])) { + $mapping->setParam('dynamic_templates', $indexConfig['dynamic_templates']); + } + return $mapping; } diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index f410621..ead9977 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -31,4 +31,45 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mappings['format']); $this->assertNull($mappings['format']->getDefaultValue()); } + + public function testDynamicTemplateNodes() + { + $tree = $this->configuration->getConfigTree(); + $children = $tree->getChildren(); + $children = $children['indexes']->getPrototype()->getChildren(); + $typeNodes = $children['types']->getPrototype()->getChildren(); + $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren(); + + $this->assertArrayHasKey('match', $dynamicTemplates); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match']); + $this->assertNull($dynamicTemplates['match']->getDefaultValue()); + + $this->assertArrayHasKey('match_mapping_type', $dynamicTemplates); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match_mapping_type']); + $this->assertNull($dynamicTemplates['match_mapping_type']->getDefaultValue()); + + $this->assertArrayHasKey('mapping', $dynamicTemplates); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ArrayNode', $dynamicTemplates['mapping']); + } + + public function testDynamicTemplateMappingNodes() + { + $tree = $this->configuration->getConfigTree(); + $children = $tree->getChildren(); + $children = $children['indexes']->getPrototype()->getChildren(); + $typeNodes = $children['types']->getPrototype()->getChildren(); + $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren(); + $mapping = $dynamicTemplates['mapping']->getChildren(); + + $this->assertArrayHasKey('type', $mapping); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['type']); + $this->assertNull($mapping['type']->getDefaultValue()); + + $this->assertArrayHasKey('index', $mapping); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['index']); + $this->assertNull($mapping['index']->getDefaultValue()); + + $this->assertArrayHasKey('fields', $mapping); + $this->assertInstanceOf('Symfony\Component\Config\Definition\ArrayNode', $mapping['fields']); + } } diff --git a/Tests/ResetterTest.php b/Tests/ResetterTest.php index 61a18a9..cc9154b 100644 --- a/Tests/ResetterTest.php +++ b/Tests/ResetterTest.php @@ -15,7 +15,10 @@ class ResetterTest extends \PHPUnit_Framework_TestCase 'index' => $this->getMockElasticaIndex(), 'config' => array( 'mappings' => array( - 'a' => array('properties' => array()), + 'a' => array( + 'dynamic_templates' => array(), + 'properties' => array(), + ), 'b' => array('properties' => array()), ), ), @@ -92,6 +95,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase ->method('delete'); $mapping = \Elastica_Type_Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); + $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']); $type->expects($this->once()) ->method('setMapping') ->with($mapping); From 9b6b0b01482325eeea80c098b37852f60ce25127 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 23 Sep 2013 09:57:58 +1000 Subject: [PATCH 2/4] Add query_builder_method option to the transformers --- DependencyInjection/Configuration.php | 2 + DependencyInjection/FOSElasticaExtension.php | 3 +- .../AbstractElasticaToModelTransformer.php | 1 + .../MongoDB/ElasticaToModelTransformer.php | 2 +- Doctrine/ORM/ElasticaToModelTransformer.php | 25 +++- .../ORM/ElasticaToModelTransformerTest.php | 130 ++++++++++++++++++ 6 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index eb6abd2..0fea69a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -171,6 +171,7 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('hydrate')->defaultTrue()->end() ->scalarNode('ignore_missing')->defaultFalse()->end() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() ->scalarNode('service')->end() ->end() ->end() @@ -252,6 +253,7 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('hydrate')->defaultTrue()->end() ->scalarNode('ignore_missing')->defaultFalse()->end() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() ->scalarNode('service')->end() ->end() ->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 7005779..7fbbe7f 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -283,7 +283,8 @@ class FOSElasticaExtension extends Extension $serviceDef->replaceArgument($argPos + 1, array( 'hydrate' => $typeConfig['elastica_to_model_transformer']['hydrate'], 'identifier' => $typeConfig['identifier'], - 'ignore_missing' => $typeConfig['elastica_to_model_transformer']['ignore_missing'] + 'ignore_missing' => $typeConfig['elastica_to_model_transformer']['ignore_missing'], + 'query_builder_method' => $typeConfig['elastica_to_model_transformer']['query_builder_method'] )); $container->setDefinition($serviceId, $serviceDef); diff --git a/Doctrine/AbstractElasticaToModelTransformer.php b/Doctrine/AbstractElasticaToModelTransformer.php index e8f9472..147067d 100755 --- a/Doctrine/AbstractElasticaToModelTransformer.php +++ b/Doctrine/AbstractElasticaToModelTransformer.php @@ -35,6 +35,7 @@ abstract class AbstractElasticaToModelTransformer implements ElasticaToModelTran 'hydrate' => true, 'identifier' => 'id', 'ignore_missing' => false, + 'query_builder_method' => 'createQueryBuilder', ); /** diff --git a/Doctrine/MongoDB/ElasticaToModelTransformer.php b/Doctrine/MongoDB/ElasticaToModelTransformer.php index 4c35a0c..855a093 100644 --- a/Doctrine/MongoDB/ElasticaToModelTransformer.php +++ b/Doctrine/MongoDB/ElasticaToModelTransformer.php @@ -22,7 +22,7 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer { return $this->registry ->getManagerForClass($this->objectClass) - ->createQueryBuilder($this->objectClass) + ->{$this->options['query_builder_method']}($this->objectClass) ->field($this->options['identifier'])->in($identifierValues) ->hydrate($hydrate) ->getQuery() diff --git a/Doctrine/ORM/ElasticaToModelTransformer.php b/Doctrine/ORM/ElasticaToModelTransformer.php index 0a889a3..20ec6e8 100644 --- a/Doctrine/ORM/ElasticaToModelTransformer.php +++ b/Doctrine/ORM/ElasticaToModelTransformer.php @@ -12,6 +12,8 @@ use Doctrine\ORM\Query; */ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer { + const ENTITY_ALIAS = 'o'; + /** * Fetch objects for theses identifier values * @@ -25,14 +27,25 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer return array(); } $hydrationMode = $hydrate ? Query::HYDRATE_OBJECT : Query::HYDRATE_ARRAY; - $qb = $this->registry - ->getManagerForClass($this->objectClass) - ->getRepository($this->objectClass) - ->createQueryBuilder('o'); - /* @var $qb \Doctrine\ORM\QueryBuilder */ - $qb->where($qb->expr()->in('o.'.$this->options['identifier'], ':values')) + + $qb = $this->getEntityQueryBuilder(); + $qb->where($qb->expr()->in(static::ENTITY_ALIAS.'.'.$this->options['identifier'], ':values')) ->setParameter('values', $identifierValues); return $qb->getQuery()->setHydrationMode($hydrationMode)->execute(); } + + /** + * Retrieves a query builder to be used for querying by identifiers + * + * @return \Doctrine\ORM\QueryBuilder + */ + protected function getEntityQueryBuilder() + { + $repository = $this->registry + ->getManagerForClass($this->objectClass) + ->getRepository($this->objectClass); + + return $repository->{$this->options['query_builder_method']}(static::ENTITY_ALIAS); + } } diff --git a/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php b/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php new file mode 100644 index 0000000..c35b9fc --- /dev/null +++ b/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php @@ -0,0 +1,130 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Tests\Doctrine\ORM; + +use Doctrine\ORM\QueryBuilder; +use FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer; + +class ElasticaToModelTransformerTest extends \PHPUnit_Framework_TestCase +{ + /** + * @var \Doctrine\Common\Persistence\ManagerRegistry|\PHPUnit_Framework_MockObject_MockObject + */ + protected $registry; + + /** + * @var \Doctrine\ORM\EntityManager|\PHPUnit_Framework_MockObject_MockObject + */ + protected $manager; + + /** + * @var \Doctrine\Common\Persistence\ObjectRepository|\PHPUnit_Framework_MockObject_MockObject + */ + protected $repository; + + /** + * @var string + */ + protected $objectClass = 'stdClass'; + + /** + * Tests that the Transformer uses the query_builder_method configuration option + * allowing configuration of createQueryBuilder call. + */ + public function testTransformUsesQueryBuilderMethodConfiguration() + { + $qb = $this->getMockBuilder('Doctrine\ORM\QueryBuilder') + ->disableOriginalConstructor() + ->getMock(); + + $this->repository->expects($this->once()) + ->method('customQueryBuilderCreator') + ->with($this->equalTo(ElasticaToModelTransformer::ENTITY_ALIAS)) + ->will($this->returnValue($qb)); + $this->repository->expects($this->never()) + ->method('createQueryBuilder'); + + $transformer = new ElasticaToModelTransformer($this->registry, $this->objectClass, array( + 'query_builder_method' => 'customQueryBuilderCreator', + )); + + $class = new \ReflectionClass('FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer'); + $method = $class->getMethod('getEntityQueryBuilder'); + $method->setAccessible(true); + + $method->invokeArgs($transformer, array()); + } + + /** + * Tests that the Transformer uses the query_builder_method configuration option + * allowing configuration of createQueryBuilder call. + */ + public function testTransformUsesDefaultQueryBuilderMethodConfiguration() + { + $qb = $this->getMockBuilder('Doctrine\ORM\QueryBuilder') + ->disableOriginalConstructor() + ->getMock(); + + $this->repository->expects($this->never()) + ->method('customQueryBuilderCreator'); + $this->repository->expects($this->once()) + ->method('createQueryBuilder') + ->with($this->equalTo(ElasticaToModelTransformer::ENTITY_ALIAS)) + ->will($this->returnValue($qb)); + + $transformer = new ElasticaToModelTransformer($this->registry, $this->objectClass); + + $class = new \ReflectionClass('FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer'); + $method = $class->getMethod('getEntityQueryBuilder'); + $method->setAccessible(true); + + $method->invokeArgs($transformer, array()); + } + + protected function setUp() + { + if (!interface_exists('Doctrine\Common\Persistence\ManagerRegistry')) { + $this->markTestSkipped('Doctrine Common is not present'); + } + if (!class_exists('Doctrine\ORM\EntityManager')) { + $this->markTestSkipped('Doctrine Common is not present'); + } + + $this->registry = $this->getMockBuilder('Doctrine\Common\Persistence\ManagerRegistry') + ->disableOriginalConstructor() + ->getMock(); + + $this->manager = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->registry->expects($this->any()) + ->method('getManagerForClass') + ->with($this->objectClass) + ->will($this->returnValue($this->manager)); + + $this->repository = $this->getMock('Doctrine\Common\Persistence\ObjectRepository', array( + 'customQueryBuilderCreator', + 'createQueryBuilder', + 'find', + 'findAll', + 'findBy', + 'findOneBy', + 'getClassName' + )); + + $this->manager->expects($this->any()) + ->method('getRepository') + ->with($this->objectClass) + ->will($this->returnValue($this->repository)); + } +} From 52d5d0d55f6f979e66f1df045b825e8ae263aca4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 11 Nov 2013 13:21:36 +1100 Subject: [PATCH 3/4] Remove incorrect copyright header --- Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php b/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php index c35b9fc..14f3ffb 100644 --- a/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php +++ b/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php @@ -1,17 +1,7 @@ - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - namespace FOS\ElasticaBundle\Tests\Doctrine\ORM; -use Doctrine\ORM\QueryBuilder; use FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer; class ElasticaToModelTransformerTest extends \PHPUnit_Framework_TestCase From 3f784b0a5b889f160b2c884bac4e29e1edebef94 Mon Sep 17 00:00:00 2001 From: Matyas Somfai Date: Tue, 8 Oct 2013 12:00:04 +0200 Subject: [PATCH 4/4] added new options to fos:elastica:populate command and options array parameter to ProviderInterface removed assume-yes option as the no-interaction option has the same purpose --- Command/PopulateCommand.php | 36 ++++++++++++++++++++++++---------- Doctrine/AbstractProvider.php | 11 ++++++++--- Propel/Provider.php | 11 ++++++++--- Provider/ProviderInterface.php | 3 ++- README.md | 15 +++++++------- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 4f88b75..237b223 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; +use Symfony\Component\Console\Helper\DialogHelper; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputInterface; @@ -43,6 +44,9 @@ class PopulateCommand extends ContainerAwareCommand ->addOption('index', null, InputOption::VALUE_OPTIONAL, 'The index to repopulate') ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'The type to repopulate') ->addOption('no-reset', null, InputOption::VALUE_NONE, 'Do not reset index before populating') + ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0) + ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0) + ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)') ->setDescription('Populates search indexes from providers') ; } @@ -62,9 +66,19 @@ class PopulateCommand extends ContainerAwareCommand */ protected function execute(InputInterface $input, OutputInterface $output) { - $index = $input->getOption('index'); - $type = $input->getOption('type'); - $reset = $input->getOption('no-reset') ? false : true; + $index = $input->getOption('index'); + $type = $input->getOption('type'); + $reset = $input->getOption('no-reset') ? false : true; + $noInteraction = $input->getOption('no-interaction'); + $options = $input->getOptions(); + + if (!$noInteraction && $reset && $input->getOption('offset')) { + /** @var DialogHelper $dialog */ + $dialog = $this->getHelperSet()->get('dialog'); + if (!$dialog->askConfirmation($output, 'You chose to reset the index and start indexing with an offset. Do you really want to do that?', true)) { + return; + } + } if (null === $index && null !== $type) { throw new \InvalidArgumentException('Cannot specify type option without an index.'); @@ -72,15 +86,15 @@ class PopulateCommand extends ContainerAwareCommand if (null !== $index) { if (null !== $type) { - $this->populateIndexType($output, $index, $type, $reset); + $this->populateIndexType($output, $index, $type, $reset, $options); } else { - $this->populateIndex($output, $index, $reset); + $this->populateIndex($output, $index, $reset, $options); } } else { $indexes = array_keys($this->indexManager->getAllIndexes()); foreach ($indexes as $index) { - $this->populateIndex($output, $index, $reset); + $this->populateIndex($output, $index, $reset, $options); } } } @@ -91,8 +105,9 @@ class PopulateCommand extends ContainerAwareCommand * @param OutputInterface $output * @param string $index * @param boolean $reset + * @param array $options */ - private function populateIndex(OutputInterface $output, $index, $reset) + private function populateIndex(OutputInterface $output, $index, $reset, $options) { if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); @@ -107,7 +122,7 @@ class PopulateCommand extends ContainerAwareCommand $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); }; - $provider->populate($loggerClosure); + $provider->populate($loggerClosure, $options); } $output->writeln(sprintf('Refreshing %s', $index)); @@ -121,8 +136,9 @@ class PopulateCommand extends ContainerAwareCommand * @param string $index * @param string $type * @param boolean $reset + * @param array $options */ - private function populateIndexType(OutputInterface $output, $index, $type, $reset) + private function populateIndexType(OutputInterface $output, $index, $type, $reset, $options) { if ($reset) { $output->writeln(sprintf('Resetting: %s/%s', $index, $type)); @@ -134,7 +150,7 @@ class PopulateCommand extends ContainerAwareCommand }; $provider = $this->providerRegistry->getProvider($index, $type); - $provider->populate($loggerClosure); + $provider->populate($loggerClosure, $options); $output->writeln(sprintf('Refreshing: %s', $index)); $this->indexManager->getIndex($index)->refresh(); diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 43e8c20..1fb41f5 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -31,16 +31,19 @@ abstract class AbstractProvider extends BaseAbstractProvider /** * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() */ - public function populate(\Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { $queryBuilder = $this->createQueryBuilder(); $nbObjects = $this->countObjects($queryBuilder); + $offset = isset($options['offset']) ? intval($options['offset']) : 0; + $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; + $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; - for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) { + for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { $stepStartTime = microtime(true); } - $objects = $this->fetchSlice($queryBuilder, $this->options['batch_size'], $offset); + $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); $this->objectPersister->insertMany($objects); @@ -48,6 +51,8 @@ abstract class AbstractProvider extends BaseAbstractProvider $this->managerRegistry->getManagerForClass($this->objectClass)->clear(); } + usleep($sleep); + if ($loggerClosure) { $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; diff --git a/Propel/Provider.php b/Propel/Provider.php index f173e72..c319691 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -14,23 +14,28 @@ class Provider extends AbstractProvider /** * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() */ - public function populate(\Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { $queryClass = $this->objectClass . 'Query'; $nbObjects = $queryClass::create()->count(); + $offset = isset($options['offset']) ? intval($options['offset']) : 0; + $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; + $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; - for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) { + for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { $stepStartTime = microtime(true); } $objects = $queryClass::create() - ->limit($this->options['batch_size']) + ->limit($batchSize) ->offset($offset) ->find(); $this->objectPersister->insertMany($objects->getArrayCopy()); + usleep($sleep); + if ($loggerClosure) { $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; diff --git a/Provider/ProviderInterface.php b/Provider/ProviderInterface.php index 710fd4b..e8d7ea4 100644 --- a/Provider/ProviderInterface.php +++ b/Provider/ProviderInterface.php @@ -13,7 +13,8 @@ interface ProviderInterface * Persists all domain objects to ElasticSearch for this provider. * * @param \Closure $loggerClosure + * @param array $options * @return */ - function populate(\Closure $loggerClosure = null); + function populate(\Closure $loggerClosure = null, array $options = array()); } diff --git a/README.md b/README.md index 28359f6..914150a 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,9 @@ Here we created a "website" index, that uses our "default" client. Our index is now available as a service: `fos_elastica.index.website`. It is an instance of `Elastica_Index`. -If you need to have different index name from the service name, for example, -in order to have different indexes for different environments then you can -use the ```index_name``` key to change the index name. The service name will +If you need to have different index name from the service name, for example, +in order to have different indexes for different environments then you can +use the ```index_name``` key to change the index name. The service name will remain the same across the environments: fos_elastica: @@ -86,8 +86,8 @@ remain the same across the environments: website: client: default index_name: website_qa - -The service id will be `fos_elastica.index.website` but the underlying index name is website_qa. + +The service id will be `fos_elastica.index.website` but the underlying index name is website_qa. #### Declare a type @@ -246,9 +246,10 @@ Its class must implement `FOS\ElasticaBundle\Provider\ProviderInterface`. /** * Insert the repository objects in the type index * - * @param Closure $loggerClosure + * @param \Closure $loggerClosure + * @param array $options */ - public function populate(Closure $loggerClosure = null) + public function populate(\Closure $loggerClosure = null, array $options = array()) { if ($loggerClosure) { $loggerClosure('Indexing users');