Merge remote-tracking branch 'origin/master' into all-field-config

Conflicts: DependencyInjection/FOSElasticaExtension.php
2013-12-06 08:25:02 +01:00 · 2013-12-06 08:25:02 +01:00 · 8f1b52328f
parent 08193ec2fd a227a3f076
commit 8f1b52328f
20 changed files with 488 additions and 68 deletions
--- a/Command/PopulateCommand.php
+++ b/Command/PopulateCommand.php
@ -3,6 +3,7 @@
 namespace FOS\ElasticaBundle\Command;

 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Helper\DialogHelper;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Input\InputInterface;
@ -43,6 +44,9 @@ class PopulateCommand extends ContainerAwareCommand
            ->addOption('index', null, InputOption::VALUE_OPTIONAL, 'The index to repopulate')
            ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'The type to repopulate')
            ->addOption('no-reset', null, InputOption::VALUE_NONE, 'Do not reset index before populating')
+            ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0)
+            ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0)
+            ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)')
            ->setDescription('Populates search indexes from providers')
        ;
    }
@ -62,9 +66,19 @@ class PopulateCommand extends ContainerAwareCommand
     */
    protected function execute(InputInterface $input, OutputInterface $output)
    {
-        $index  = $input->getOption('index');
-        $type   = $input->getOption('type');
-        $reset  = $input->getOption('no-reset') ? false : true;
+        $index         = $input->getOption('index');
+        $type          = $input->getOption('type');
+        $reset         = $input->getOption('no-reset') ? false : true;
+        $noInteraction = $input->getOption('no-interaction');
+        $options       = $input->getOptions();
+
+        if (!$noInteraction && $reset && $input->getOption('offset')) {
+            /** @var DialogHelper $dialog */
+            $dialog = $this->getHelperSet()->get('dialog');
+            if (!$dialog->askConfirmation($output, '<question>You chose to reset the index and start indexing with an offset. Do you really want to do that?</question>', true)) {
+                return;
+            }
+        }

        if (null === $index && null !== $type) {
            throw new \InvalidArgumentException('Cannot specify type option without an index.');
@ -72,15 +86,15 @@ class PopulateCommand extends ContainerAwareCommand

        if (null !== $index) {
            if (null !== $type) {
-                $this->populateIndexType($output, $index, $type, $reset);
+                $this->populateIndexType($output, $index, $type, $reset, $options);
            } else {
-                $this->populateIndex($output, $index, $reset);
+                $this->populateIndex($output, $index, $reset, $options);
            }
        } else {
            $indexes = array_keys($this->indexManager->getAllIndexes());

            foreach ($indexes as $index) {
-                $this->populateIndex($output, $index, $reset);
+                $this->populateIndex($output, $index, $reset, $options);
            }
        }
    }
@ -91,8 +105,9 @@ class PopulateCommand extends ContainerAwareCommand
     * @param OutputInterface $output
     * @param string          $index
     * @param boolean         $reset
+     * @param array           $options
     */
-    private function populateIndex(OutputInterface $output, $index, $reset)
+    private function populateIndex(OutputInterface $output, $index, $reset, $options)
    {
        if ($reset) {
            $output->writeln(sprintf('<info>Resetting</info> <comment>%s</comment>', $index));
@ -107,7 +122,7 @@ class PopulateCommand extends ContainerAwareCommand
                $output->writeln(sprintf('<info>Populating</info> %s/%s, %s', $index, $type, $message));
            };

-            $provider->populate($loggerClosure);
+            $provider->populate($loggerClosure, $options);
        }

        $output->writeln(sprintf('<info>Refreshing</info> <comment>%s</comment>', $index));
@ -121,8 +136,9 @@ class PopulateCommand extends ContainerAwareCommand
     * @param string          $index
     * @param string          $type
     * @param boolean         $reset
+     * @param array           $options
     */
-    private function populateIndexType(OutputInterface $output, $index, $type, $reset)
+    private function populateIndexType(OutputInterface $output, $index, $type, $reset, $options)
    {
        if ($reset) {
            $output->writeln(sprintf('<info>Resetting</info> <comment>%s/%s</comment>', $index, $type));
@ -134,7 +150,7 @@ class PopulateCommand extends ContainerAwareCommand
        };

        $provider = $this->providerRegistry->getProvider($index, $type);
-        $provider->populate($loggerClosure);
+        $provider->populate($loggerClosure, $options);

        $output->writeln(sprintf('<info>Refreshing</info> <comment>%s</comment>', $index));
        $this->indexManager->getIndex($index)->refresh();
--- a/DependencyInjection/Compiler/TransformerPass.php
+++ b/DependencyInjection/Compiler/TransformerPass.php
@ -19,7 +19,7 @@ class TransformerPass implements CompilerPassInterface
     */
    public function process(ContainerBuilder $container)
    {
-        if (!$container->hasDefinition('fos_elastica.elastica_to_model_transformer.collection.prototype')) {
+        if (!$container->hasDefinition('fos_elastica.elastica_to_model_transformer.collection')) {
            return;
        }

@ -44,4 +44,4 @@ class TransformerPass implements CompilerPassInterface
            $index->replaceArgument(0, $indexTransformers);
        }
    }
-}
+}
--- a/DependencyInjection/Configuration.php
+++ b/DependencyInjection/Configuration.php
@ -179,6 +179,7 @@ class Configuration implements ConfigurationInterface
                                                ->children()
                                                    ->scalarNode('hydrate')->defaultTrue()->end()
                                                    ->scalarNode('ignore_missing')->defaultFalse()->end()
+                                                    ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end()
                                                    ->scalarNode('service')->end()
                                                ->end()
                                            ->end()
@ -270,6 +271,7 @@ class Configuration implements ConfigurationInterface
                                ->children()
                                    ->scalarNode('hydrate')->defaultTrue()->end()
                                    ->scalarNode('ignore_missing')->defaultFalse()->end()
+                                    ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end()
                                    ->scalarNode('service')->end()
                                ->end()
                            ->end()
@ -284,6 +286,7 @@ class Configuration implements ConfigurationInterface
                ->end()
                ->append($this->getIdNode())
                ->append($this->getMappingsNode())
+                ->append($this->getDynamicTemplateNode())
                ->append($this->getSourceNode())
                ->append($this->getBoostNode())
                ->append($this->getRoutingNode())
@ -317,6 +320,37 @@ class Configuration implements ConfigurationInterface
        return $node;
    }

+    /**
+     * Returns the array node used for "dynamic_templates".
+     */
+    public function getDynamicTemplateNode()
+    {
+        $builder = new TreeBuilder();
+        $node = $builder->root('dynamic_templates');
+
+        $node
+            ->useAttributeAsKey('name')
+            ->prototype('array')
+                ->children()
+                    ->scalarNode('match')->isRequired()->end()
+                    ->scalarNode('match_mapping_type')->end()
+                    ->arrayNode('mapping')
+                        ->isRequired()
+                        ->children()
+                            ->scalarNode('type')->end()
+                            ->scalarNode('index')->end()
+                            ->arrayNode('fields')
+                                ->children()
+                            ->end()
+                        ->end()
+                    ->end()
+                ->end()
+            ->end()
+        ;
+
+        return $node;
+    }
+
    /**
     * @param \Symfony\Component\Config\Definition\Builder\NodeBuilder $node The node to which to attach the field config to
     * @param array $nestings the nested mappings for the current field level
@ -334,12 +368,21 @@ class Configuration implements ConfigurationInterface
            ->scalarNode('term_vector')->end()
            ->scalarNode('null_value')->end()
            ->booleanNode('include_in_all')->defaultValue(true)->end()
+            ->booleanNode('enabled')->defaultValue(true)->end()
            ->scalarNode('lat_lon')->end()
            ->scalarNode('index_name')->end()
            ->booleanNode('omit_norms')->end()
            ->scalarNode('index_options')->end()
            ->scalarNode('ignore_above')->end()
            ->scalarNode('position_offset_gap')->end()
+            ->arrayNode('_parent')
+                ->treatNullLike(array())
+                ->children()
+                    ->scalarNode('type')->end()
+                    ->scalarNode('identifier')->defaultValue('id')->end()
+                ->end()
+            ->end()
+            ->scalarNode('format')->end();
        ;

        if (isset($nestings['fields'])) {
--- a/DependencyInjection/FOSElasticaExtension.php
+++ b/DependencyInjection/FOSElasticaExtension.php
@ -71,10 +71,10 @@ class FOSElasticaExtension extends Extension
    {
        $clientIds = array();
        foreach ($clients as $name => $clientConfig) {
-            $clientDef = $container->getDefinition('fos_elastica.client');
-            $clientDef->replaceArgument(0, $clientConfig);
-
            $clientId = sprintf('fos_elastica.client.%s', $name);
+            $clientDef = new Definition('%fos_elastica.client.class%', array($clientConfig));
+            $clientDef->addMethodCall('setLogger', array(new Reference('fos_elastica.logger')));
+            $clientDef->addTag('fos_elastica.client');

            $container->setDefinition($clientId, $clientDef);

@ -145,14 +145,15 @@ class FOSElasticaExtension extends Extension
     */
    protected function loadIndexFinder(ContainerBuilder $container, $name, $indexId)
    {
-        $abstractTransformerId = 'fos_elastica.elastica_to_model_transformer.collection.prototype';
+        /* Note: transformer services may conflict with "collection.index", if
+         * an index and type names were "collection" and an index, respectively.
+         */
        $transformerId = sprintf('fos_elastica.elastica_to_model_transformer.collection.%s', $name);
-        $transformerDef = new DefinitionDecorator($abstractTransformerId);
+        $transformerDef = new DefinitionDecorator('fos_elastica.elastica_to_model_transformer.collection');
        $container->setDefinition($transformerId, $transformerDef);

-        $abstractFinderId = 'fos_elastica.finder.prototype';
        $finderId = sprintf('fos_elastica.finder.%s', $name);
-        $finderDef = new DefinitionDecorator($abstractFinderId);
+        $finderDef = new DefinitionDecorator('fos_elastica.finder');
        $finderDef->replaceArgument(0, new Reference($indexId));
        $finderDef->replaceArgument(1, new Reference($transformerId));

@ -234,6 +235,12 @@ class FOSElasticaExtension extends Extension
            if (isset($type['_all'])) {
                $this->indexConfigs[$indexName]['config']['mappings'][$name]['_all'] = $type['_all'];
            }
+            if (!empty($type['dynamic_templates'])) {
+                $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'] = array();
+                foreach ($type['dynamic_templates'] as $templateName => $templateData) {
+                    $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'][] = array($templateName => $templateData);
+                }
+            }
        }
    }

@ -291,6 +298,9 @@ class FOSElasticaExtension extends Extension
        if (isset($typeConfig['elastica_to_model_transformer']['service'])) {
            return $typeConfig['elastica_to_model_transformer']['service'];
        }
+        /* Note: transformer services may conflict with "prototype.driver", if
+         * the index and type names were "prototype" and a driver, respectively.
+         */
        $abstractId = sprintf('fos_elastica.elastica_to_model_transformer.prototype.%s', $typeConfig['driver']);
        $serviceId = sprintf('fos_elastica.elastica_to_model_transformer.%s.%s', $indexName, $typeName);
        $serviceDef = new DefinitionDecorator($abstractId);
@ -303,7 +313,8 @@ class FOSElasticaExtension extends Extension
        $serviceDef->replaceArgument($argPos + 1, array(
            'hydrate'        => $typeConfig['elastica_to_model_transformer']['hydrate'],
            'identifier'     => $typeConfig['identifier'],
-            'ignore_missing' => $typeConfig['elastica_to_model_transformer']['ignore_missing']
+            'ignore_missing' => $typeConfig['elastica_to_model_transformer']['ignore_missing'],
+            'query_builder_method' => $typeConfig['elastica_to_model_transformer']['query_builder_method']
        ));
        $container->setDefinition($serviceId, $serviceDef);

@ -315,9 +326,9 @@ class FOSElasticaExtension extends Extension
        if (isset($typeConfig['model_to_elastica_transformer']['service'])) {
            return $typeConfig['model_to_elastica_transformer']['service'];
        }
-        $abstractId = sprintf('fos_elastica.model_to_elastica_transformer.prototype.auto');
+
        $serviceId = sprintf('fos_elastica.model_to_elastica_transformer.%s.%s', $indexName, $typeName);
-        $serviceDef = new DefinitionDecorator($abstractId);
+        $serviceDef = new DefinitionDecorator('fos_elastica.model_to_elastica_transformer');
        $serviceDef->replaceArgument(0, array(
            'identifier' => $typeConfig['identifier']
        ));
@ -328,9 +339,8 @@ class FOSElasticaExtension extends Extension

    protected function loadObjectPersister(array $typeConfig, Definition $typeDef, ContainerBuilder $container, $indexName, $typeName, $transformerId)
    {
-        $abstractId = sprintf('fos_elastica.object_persister.prototype');
        $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName);
-        $serviceDef = new DefinitionDecorator($abstractId);
+        $serviceDef = new DefinitionDecorator('fos_elastica.object_persister');
        $serviceDef->replaceArgument(0, $typeDef);
        $serviceDef->replaceArgument(1, new Reference($transformerId));
        $serviceDef->replaceArgument(2, $typeConfig['model']);
@ -345,7 +355,9 @@ class FOSElasticaExtension extends Extension
        if (isset($typeConfig['provider']['service'])) {
            return $typeConfig['provider']['service'];
        }
-
+        /* Note: provider services may conflict with "prototype.driver", if the
+         * index and type names were "prototype" and a driver, respectively.
+         */
        $providerId = sprintf('fos_elastica.provider.%s.%s', $indexName, $typeName);
        $providerDef = new DefinitionDecorator('fos_elastica.provider.prototype.' . $typeConfig['driver']);
        $providerDef->addTag('fos_elastica.provider', array('index' => $indexName, 'type' => $typeName));
@ -363,6 +375,9 @@ class FOSElasticaExtension extends Extension
        if (isset($typeConfig['listener']['service'])) {
            return $typeConfig['listener']['service'];
        }
+        /* Note: listener services may conflict with "prototype.driver", if the
+         * index and type names were "prototype" and a driver, respectively.
+         */
        $abstractListenerId = sprintf('fos_elastica.listener.prototype.%s', $typeConfig['driver']);
        $listenerId = sprintf('fos_elastica.listener.%s.%s', $indexName, $typeName);
        $listenerDef = new DefinitionDecorator($abstractListenerId);
@ -414,9 +429,8 @@ class FOSElasticaExtension extends Extension
        if (isset($typeConfig['finder']['service'])) {
            $finderId = $typeConfig['finder']['service'];
        } else {
-            $abstractFinderId = 'fos_elastica.finder.prototype';
            $finderId = sprintf('fos_elastica.finder.%s.%s', $indexName, $typeName);
-            $finderDef = new DefinitionDecorator($abstractFinderId);
+            $finderDef = new DefinitionDecorator('fos_elastica.finder');
            $finderDef->replaceArgument(0, $typeDef);
            $finderDef->replaceArgument(1, new Reference($elasticaToModelId));
            $container->setDefinition($finderId, $finderDef);
--- a/Doctrine/AbstractElasticaToModelTransformer.php
+++ b/Doctrine/AbstractElasticaToModelTransformer.php
@ -35,6 +35,7 @@ abstract class AbstractElasticaToModelTransformer implements ElasticaToModelTran
        'hydrate'        => true,
        'identifier'     => 'id',
        'ignore_missing' => false,
+        'query_builder_method' => 'createQueryBuilder',
    );

    /**
--- a/Doctrine/AbstractListener.php
+++ b/Doctrine/AbstractListener.php
@ -6,6 +6,9 @@ use Doctrine\Common\EventSubscriber;
 use Doctrine\Common\Persistence\ObjectManager;
 use FOS\ElasticaBundle\Persister\ObjectPersisterInterface;
 use FOS\ElasticaBundle\Persister\ObjectPersister;
+use Symfony\Component\ExpressionLanguage\Expression;
+use Symfony\Component\ExpressionLanguage\ExpressionLanguage;
+use Symfony\Component\ExpressionLanguage\SyntaxError;

 abstract class AbstractListener implements EventSubscriber
 {
@ -51,6 +54,13 @@ abstract class AbstractListener implements EventSubscriber
     */
    private $scheduledForRemoval = array();

+    /**
+     * An instance of ExpressionLanguage
+     *
+     * @var ExpressionLanguage
+     */
+    protected $expressionLanguage;
+
    /**
     * Constructor.
     *
@ -83,14 +93,23 @@ abstract class AbstractListener implements EventSubscriber
     * should expect the object for consideration as its only argument and
     * return a boolean.
     *
-     * @param callback $callback
+     * @param  callback          $callback
     * @throws \RuntimeException if the callback is not callable
     */
    public function setIsIndexableCallback($callback)
    {
        if (is_string($callback)) {
            if (!is_callable(array($this->objectClass, $callback))) {
-                throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback));
+                if (false !== ($expression = $this->getExpressionLanguage())) {
+                    $callback = new Expression($callback);
+                    try {
+                        $expression->compile($callback, array($this->getExpressionVar()));
+                    } catch (SyntaxError $e) {
+                        throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e);
+                    }
+                } else {
+                    throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback));
+                }
            }
        } elseif (!is_callable($callback)) {
            if (is_array($callback)) {
@ -98,6 +117,7 @@ abstract class AbstractListener implements EventSubscriber
                if (is_object($class)) {
                    $class = get_class($class);
                }
+
                if ($class && $method) {
                    throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method));
                }
@ -111,7 +131,7 @@ abstract class AbstractListener implements EventSubscriber
    /**
     * Return whether the object is indexable with respect to the callback.
     *
-     * @param object $object
+     * @param  object  $object
     * @return boolean
     */
    protected function isObjectIndexable($object)
@ -120,6 +140,10 @@ abstract class AbstractListener implements EventSubscriber
            return true;
        }

+        if ($this->isIndexableCallback instanceof Expression) {
+            return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object));
+        }
+
        return is_string($this->isIndexableCallback)
            ? call_user_func(array($object, $this->isIndexableCallback))
            : call_user_func($this->isIndexableCallback, $object);
@ -155,4 +179,32 @@ abstract class AbstractListener implements EventSubscriber
            unset($this->scheduledForRemoval[$objectHash]);
        }
    }
+
+    /**
+     * @param  mixed  $object
+     * @return string
+     */
+    private function getExpressionVar($object = null)
+    {
+        $class = $object ?: $this->objectClass;
+        $ref = new \ReflectionClass($class);
+
+        return strtolower($ref->getShortName());
+    }
+
+    /**
+     * @return bool|ExpressionLanguage
+     */
+    private function getExpressionLanguage()
+    {
+        if (null === $this->expressionLanguage) {
+            if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) {
+                return false;
+            }
+
+            $this->expressionLanguage = new ExpressionLanguage();
+        }
+
+        return $this->expressionLanguage;
+    }
 }
--- a/Doctrine/AbstractProvider.php
+++ b/Doctrine/AbstractProvider.php
@ -31,16 +31,19 @@ abstract class AbstractProvider extends BaseAbstractProvider
    /**
     * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate()
     */
-    public function populate(\Closure $loggerClosure = null)
+    public function populate(\Closure $loggerClosure = null, array $options = array())
    {
        $queryBuilder = $this->createQueryBuilder();
        $nbObjects = $this->countObjects($queryBuilder);
+        $offset = isset($options['offset']) ? intval($options['offset']) : 0;
+        $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0;
+        $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];

-        for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) {
+        for (; $offset < $nbObjects; $offset += $batchSize) {
            if ($loggerClosure) {
                $stepStartTime = microtime(true);
            }
-            $objects = $this->fetchSlice($queryBuilder, $this->options['batch_size'], $offset);
+            $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);

            $this->objectPersister->insertMany($objects);

@ -48,6 +51,8 @@ abstract class AbstractProvider extends BaseAbstractProvider
                $this->managerRegistry->getManagerForClass($this->objectClass)->clear();
            }

+            usleep($sleep);
+
            if ($loggerClosure) {
                $stepNbObjects = count($objects);
                $stepCount = $stepNbObjects + $offset;
--- a/Doctrine/MongoDB/ElasticaToModelTransformer.php
+++ b/Doctrine/MongoDB/ElasticaToModelTransformer.php
@ -22,7 +22,7 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer
    {
        return $this->registry
            ->getManagerForClass($this->objectClass)
-            ->createQueryBuilder($this->objectClass)
+            ->{$this->options['query_builder_method']}($this->objectClass)
            ->field($this->options['identifier'])->in($identifierValues)
            ->hydrate($hydrate)
            ->getQuery()
--- a/Doctrine/ORM/ElasticaToModelTransformer.php
+++ b/Doctrine/ORM/ElasticaToModelTransformer.php
@ -12,6 +12,8 @@ use Doctrine\ORM\Query;
 */
 class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer
 {
+    const ENTITY_ALIAS = 'o';
+
    /**
     * Fetch objects for theses identifier values
     *
@ -25,14 +27,25 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer
            return array();
        }
        $hydrationMode = $hydrate ? Query::HYDRATE_OBJECT : Query::HYDRATE_ARRAY;
-        $qb = $this->registry
-            ->getManagerForClass($this->objectClass)
-            ->getRepository($this->objectClass)
-            ->createQueryBuilder('o');
-        /* @var $qb \Doctrine\ORM\QueryBuilder */
-        $qb->where($qb->expr()->in('o.'.$this->options['identifier'], ':values'))
+
+        $qb = $this->getEntityQueryBuilder();
+        $qb->where($qb->expr()->in(static::ENTITY_ALIAS.'.'.$this->options['identifier'], ':values'))
            ->setParameter('values', $identifierValues);

        return $qb->getQuery()->setHydrationMode($hydrationMode)->execute();
    }
+
+    /**
+     * Retrieves a query builder to be used for querying by identifiers
+     *
+     * @return \Doctrine\ORM\QueryBuilder
+     */
+    protected function getEntityQueryBuilder()
+    {
+        $repository = $this->registry
+            ->getManagerForClass($this->objectClass)
+            ->getRepository($this->objectClass);
+
+        return $repository->{$this->options['query_builder_method']}(static::ENTITY_ALIAS);
+    }
 }
--- a/Finder/TransformedFinder.php
+++ b/Finder/TransformedFinder.php
@ -2,7 +2,7 @@

 namespace FOS\ElasticaBundle\Finder;

-use FOS\ElasticaBundle\Finder\PaginatedFinderInterface;
+use Elastica\Document;
 use FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface;
 use FOS\ElasticaBundle\Paginator\TransformedPaginatorAdapter;
 use FOS\ElasticaBundle\Paginator\FantaPaginatorAdapter;
@ -45,6 +45,22 @@ class TransformedFinder implements PaginatedFinderInterface
        return $this->transformer->hybridTransform($results);
    }

+    /**
+     * Find documents similar to one with passed id.
+     *
+     * @param integer $id
+     * @param array $params
+     * @param array $query
+     * @return array of model objects
+     **/
+    public function moreLikeThis($id, $params = array(), $query = array())
+    {
+        $doc = new Document($id);
+        $results = $this->searchable->moreLikeThis($doc, $params, $query)->getResults();
+
+        return $this->transformer->transform($results);
+    }
+
    /**
     * @param $query
     * @param null|int $limit
@ -61,7 +77,6 @@ class TransformedFinder implements PaginatedFinderInterface
        return $results;
    }

-
    /**
     * Gets a paginator wrapping the result of a search
     *
--- a/Propel/Provider.php
+++ b/Propel/Provider.php
@ -14,23 +14,28 @@ class Provider extends AbstractProvider
    /**
     * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate()
     */
-    public function populate(\Closure $loggerClosure = null)
+    public function populate(\Closure $loggerClosure = null, array $options = array())
    {
        $queryClass = $this->objectClass . 'Query';
        $nbObjects = $queryClass::create()->count();
+        $offset = isset($options['offset']) ? intval($options['offset']) : 0;
+        $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0;
+        $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];

-        for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) {
+        for (; $offset < $nbObjects; $offset += $batchSize) {
            if ($loggerClosure) {
                $stepStartTime = microtime(true);
            }

            $objects = $queryClass::create()
-                ->limit($this->options['batch_size'])
+                ->limit($batchSize)
                ->offset($offset)
                ->find();

            $this->objectPersister->insertMany($objects->getArrayCopy());

+            usleep($sleep);
+
            if ($loggerClosure) {
                $stepNbObjects = count($objects);
                $stepCount = $stepNbObjects + $offset;
--- a/Provider/ProviderInterface.php
+++ b/Provider/ProviderInterface.php
@ -13,7 +13,8 @@ interface ProviderInterface
     * Persists all domain objects to ElasticSearch for this provider.
     *
     * @param \Closure $loggerClosure
+     * @param array    $options
     * @return
     */
-    function populate(\Closure $loggerClosure = null);
+    function populate(\Closure $loggerClosure = null, array $options = array());
 }
--- a/README.md
+++ b/README.md
@ -102,9 +102,9 @@ Here we created a "website" index, that uses our "default" client.

 Our index is now available as a service: `fos_elastica.index.website`. It is an instance of `\Elastica\Index`.

-If you need to have different index name from the service name, for example, 
-in order to have different indexes for different environments then you can 
-use the ```index_name``` key to change the index name. The service name will 
+If you need to have different index name from the service name, for example,
+in order to have different indexes for different environments then you can
+use the ```index_name``` key to change the index name. The service name will
 remain the same across the environments:

    fos_elastica:
@ -114,8 +114,8 @@ remain the same across the environments:
            website:
                client: default
                index_name: website_qa
-                
-The service id will be `fos_elastica.index.website` but the underlying index name is website_qa.           
+
+The service id will be `fos_elastica.index.website` but the underlying index name is website_qa.

 #### Declare a type

@ -327,8 +327,9 @@ Its class must implement `FOS\ElasticaBundle\Provider\ProviderInterface`.
             * Insert the repository objects in the type index
             *
             * @param \Closure $loggerClosure
+             * @param array    $options
             */
-            public function populate(\Closure $loggerClosure = null)
+            public function populate(\Closure $loggerClosure = null, array $options = array())
            {
                if ($loggerClosure) {
                    $loggerClosure('Indexing users');
@ -591,7 +592,7 @@ Declare that you want to update the index in real time:
                        persistence:
                            driver: orm
                            model: Application\UserBundle\Entity\User
-                            listener: # by default, listens to "insert", "update" and "delete"
+                            listener: ~ # by default, listens to "insert", "update" and "delete"

 Now the index is automatically updated each time the state of the bound Doctrine repository changes.
 No need to repopulate the whole "user" index when a new `User` is created.
@ -629,6 +630,13 @@ In this case, the callback_class will be the `isIndexable()` method on the speci
 service and the object being considered for indexing will be passed as the only
 argument. This allows you to do more complex validation (e.g. ACL checks).

+If you have the [Symfony ExpressionLanguage](https://github.com/symfony/expression-language) component installed, you can use expressions
+to evaluate the callback:
+
+                        persistence:
+                            listener:
+                                is_indexable_callback: "user.isActive() && user.hasRole('ROLE_USER')"
+
 As you might expect, new entities will only be indexed if the callback_class returns
 `true`. Additionally, modified entities will be updated or removed from the
 index depending on whether the callback_class returns `true` or `false`, respectively.
@ -796,3 +804,51 @@ $term = new \Elastica\Filter\Term(array('active' => true));
 $filteredQuery = new \Elastica\Query\Filtered($query, $term);
 $results = $this->container->get('fos_elastica.finder.index.type')->find($filteredQuery);
 ```
+
+### Date format example
+
+If you want to specify a [date format](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-date-format.html):
+
+```yaml
+fos_elastica:
+    clients:
+        default: { host: localhost, port: 9200 }
+    indexes:
+        site:
+            types:
+                user:
+                    mappings:
+                        username: { type: string }
+                        lastlogin: { type: date, format: basic_date_time }
+                        birthday: { type: date, format: "yyyy-MM-dd" }
+```
+
+#### Dynamic templates
+
+Dynamic templates allow to define mapping templates that will be
+applied when dynamic introduction of fields / objects happens.
+
+[Documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-root-object-type.html#_dynamic_templates)
+
+```yaml
+fos_elastica:
+    clients:
+        default: { host: localhost, port: 9200 }
+    indexes:
+        site:
+            types:
+                user:
+                    dynamic_templates:
+                        my_template_1:
+                            match: apples_*
+                            mapping:
+                                type: float
+                        my_template_2:
+                            match: *
+                            match_mapping_type: string
+                            mapping:
+                                type: string
+                                index: not_analyzed
+                    mappings:
+                        username: { type: string }
+```
--- a/Resetter.php
+++ b/Resetter.php
@ -78,6 +78,10 @@ class Resetter
            $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type']));
        }

+        if (isset($indexConfig['dynamic_templates'])) {
+            $mapping->setParam('dynamic_templates', $indexConfig['dynamic_templates']);
+        }
+
        return $mapping;
    }

--- a/Resources/config/config.xml
+++ b/Resources/config/config.xml
@ -28,14 +28,6 @@
            <argument type="service" id="fos_elastica.logger" />
        </service>

-        <service id="fos_elastica.client" class="%fos_elastica.client.class%">
-            <argument /> <!-- config -->
-            <call method="setLogger">
-                <argument type="service" id="fos_elastica.logger" />
-            </call>
-            <tag name="fos_elastica.client" />
-        </service>
-
        <service id="fos_elastica.index_manager" class="FOS\ElasticaBundle\IndexManager">
            <argument /> <!-- indexes -->
            <argument /> <!-- default index -->
@ -45,26 +37,26 @@
            <argument /> <!-- index configs -->
        </service>

-        <service id="fos_elastica.object_persister.prototype" class="FOS\ElasticaBundle\Persister\ObjectPersister" abstract="true">
+        <service id="fos_elastica.object_persister" class="FOS\ElasticaBundle\Persister\ObjectPersister" abstract="true">
            <argument /> <!-- type -->
            <argument /> <!-- model to elastica transformer -->
            <argument /> <!-- model -->
            <argument /> <!-- properties mapping -->
        </service>

-        <service id="fos_elastica.finder.prototype" class="FOS\ElasticaBundle\Finder\TransformedFinder" public="true" abstract="true">
+        <service id="fos_elastica.finder" class="FOS\ElasticaBundle\Finder\TransformedFinder" public="true" abstract="true">
            <argument /> <!-- searchable -->
            <argument /> <!-- transformer -->
        </service>

-        <service id="fos_elastica.model_to_elastica_transformer.prototype.auto" class="FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer" public="false" abstract="true">
+        <service id="fos_elastica.model_to_elastica_transformer" class="FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer" public="false" abstract="true">
            <argument /> <!-- options -->
            <call method="setPropertyAccessor">
                <argument type="service" id="fos_elastica.property_accessor" />
            </call>
        </service>

-        <service id="fos_elastica.elastica_to_model_transformer.collection.prototype" class="%fos_elastica.elastica_to_model_transformer.collection.class%" public="true" abstract="true">
+        <service id="fos_elastica.elastica_to_model_transformer.collection" class="%fos_elastica.elastica_to_model_transformer.collection.class%" public="true" abstract="true">
            <argument type="collection" /> <!-- transformers -->
        </service>

--- a/Tests/DependencyInjection/ConfigurationTest.php
+++ b/Tests/DependencyInjection/ConfigurationTest.php
@ -0,0 +1,75 @@
+<?php
+
+namespace FOS\ElasticaBundle\Tests\Resetter\DependencyInjection;
+
+use FOS\ElasticaBundle\DependencyInjection\Configuration;
+
+/**
+ * ConfigurationTest
+ */
+class ConfigurationTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * @var Configuration
+     */
+    private $configuration;
+
+    public function setUp()
+    {
+        $this->configuration = new Configuration(array());
+    }
+
+    public function testEmptyConfigContainsFormatMappingOptionNode()
+    {
+        $tree = $this->configuration->getConfigTree();
+        $children = $tree->getChildren();
+        $children = $children['indexes']->getPrototype()->getChildren();
+        $typeNodes = $children['types']->getPrototype()->getChildren();
+        $mappings = $typeNodes['mappings']->getPrototype()->getChildren();
+
+        $this->assertArrayHasKey('format', $mappings);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mappings['format']);
+        $this->assertNull($mappings['format']->getDefaultValue());
+    }
+
+    public function testDynamicTemplateNodes()
+    {
+        $tree = $this->configuration->getConfigTree();
+        $children = $tree->getChildren();
+        $children = $children['indexes']->getPrototype()->getChildren();
+        $typeNodes = $children['types']->getPrototype()->getChildren();
+        $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren();
+
+        $this->assertArrayHasKey('match', $dynamicTemplates);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match']);
+        $this->assertNull($dynamicTemplates['match']->getDefaultValue());
+
+        $this->assertArrayHasKey('match_mapping_type', $dynamicTemplates);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match_mapping_type']);
+        $this->assertNull($dynamicTemplates['match_mapping_type']->getDefaultValue());
+
+        $this->assertArrayHasKey('mapping', $dynamicTemplates);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ArrayNode', $dynamicTemplates['mapping']);
+    }
+
+    public function testDynamicTemplateMappingNodes()
+    {
+        $tree = $this->configuration->getConfigTree();
+        $children = $tree->getChildren();
+        $children = $children['indexes']->getPrototype()->getChildren();
+        $typeNodes = $children['types']->getPrototype()->getChildren();
+        $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren();
+        $mapping = $dynamicTemplates['mapping']->getChildren();
+
+        $this->assertArrayHasKey('type', $mapping);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['type']);
+        $this->assertNull($mapping['type']->getDefaultValue());
+
+        $this->assertArrayHasKey('index', $mapping);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['index']);
+        $this->assertNull($mapping['index']->getDefaultValue());
+
+        $this->assertArrayHasKey('fields', $mapping);
+        $this->assertInstanceOf('Symfony\Component\Config\Definition\ArrayNode', $mapping['fields']);
+    }
+}
--- a/Tests/Doctrine/AbstractListenerTest.php
+++ b/Tests/Doctrine/AbstractListenerTest.php
@ -164,6 +164,7 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase
            array('nonexistentEntityMethod'),
            array(array(new Listener\IndexableDecider(), 'internalMethod')),
            array(42),
+            array('entity.getIsIndexable() && nonexistentEntityFunction()'),
        );
    }

@ -173,6 +174,7 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase
            array('getIsIndexable'),
            array(array(new Listener\IndexableDecider(), 'isIndexable')),
            array(function(Listener\Entity $entity) { return $entity->getIsIndexable(); }),
+            array('entity.getIsIndexable()')
        );
    }

--- a/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php
+++ b/Tests/Doctrine/ORM/ElasticaToModelTransformerTest.php
@ -0,0 +1,120 @@
+<?php
+
+namespace FOS\ElasticaBundle\Tests\Doctrine\ORM;
+
+use FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer;
+
+class ElasticaToModelTransformerTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * @var \Doctrine\Common\Persistence\ManagerRegistry|\PHPUnit_Framework_MockObject_MockObject
+     */
+    protected $registry;
+
+    /**
+     * @var \Doctrine\ORM\EntityManager|\PHPUnit_Framework_MockObject_MockObject
+     */
+    protected $manager;
+
+    /**
+     * @var \Doctrine\Common\Persistence\ObjectRepository|\PHPUnit_Framework_MockObject_MockObject
+     */
+    protected $repository;
+
+    /**
+     * @var string
+     */
+    protected $objectClass = 'stdClass';
+
+    /**
+     * Tests that the Transformer uses the query_builder_method configuration option
+     * allowing configuration of createQueryBuilder call.
+     */
+    public function testTransformUsesQueryBuilderMethodConfiguration()
+    {
+        $qb = $this->getMockBuilder('Doctrine\ORM\QueryBuilder')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->repository->expects($this->once())
+            ->method('customQueryBuilderCreator')
+            ->with($this->equalTo(ElasticaToModelTransformer::ENTITY_ALIAS))
+            ->will($this->returnValue($qb));
+        $this->repository->expects($this->never())
+            ->method('createQueryBuilder');
+
+        $transformer = new ElasticaToModelTransformer($this->registry, $this->objectClass, array(
+            'query_builder_method' => 'customQueryBuilderCreator',
+        ));
+
+        $class = new \ReflectionClass('FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer');
+        $method = $class->getMethod('getEntityQueryBuilder');
+        $method->setAccessible(true);
+
+        $method->invokeArgs($transformer, array());
+    }
+
+    /**
+     * Tests that the Transformer uses the query_builder_method configuration option
+     * allowing configuration of createQueryBuilder call.
+     */
+    public function testTransformUsesDefaultQueryBuilderMethodConfiguration()
+    {
+        $qb = $this->getMockBuilder('Doctrine\ORM\QueryBuilder')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->repository->expects($this->never())
+            ->method('customQueryBuilderCreator');
+        $this->repository->expects($this->once())
+            ->method('createQueryBuilder')
+            ->with($this->equalTo(ElasticaToModelTransformer::ENTITY_ALIAS))
+            ->will($this->returnValue($qb));
+
+        $transformer = new ElasticaToModelTransformer($this->registry, $this->objectClass);
+
+        $class = new \ReflectionClass('FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer');
+        $method = $class->getMethod('getEntityQueryBuilder');
+        $method->setAccessible(true);
+
+        $method->invokeArgs($transformer, array());
+    }
+
+    protected function setUp()
+    {
+        if (!interface_exists('Doctrine\Common\Persistence\ManagerRegistry')) {
+            $this->markTestSkipped('Doctrine Common is not present');
+        }
+        if (!class_exists('Doctrine\ORM\EntityManager')) {
+            $this->markTestSkipped('Doctrine Common is not present');
+        }
+
+        $this->registry = $this->getMockBuilder('Doctrine\Common\Persistence\ManagerRegistry')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->manager = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $this->registry->expects($this->any())
+            ->method('getManagerForClass')
+            ->with($this->objectClass)
+            ->will($this->returnValue($this->manager));
+
+        $this->repository = $this->getMock('Doctrine\Common\Persistence\ObjectRepository', array(
+            'customQueryBuilderCreator',
+            'createQueryBuilder',
+            'find',
+            'findAll',
+            'findBy',
+            'findOneBy',
+            'getClassName'
+        ));
+
+        $this->manager->expects($this->any())
+            ->method('getRepository')
+            ->with($this->objectClass)
+            ->will($this->returnValue($this->repository));
+    }
+}
--- a/Tests/ResetterTest.php
+++ b/Tests/ResetterTest.php
@ -16,7 +16,10 @@ class ResetterTest extends \PHPUnit_Framework_TestCase
                'index' => $this->getMockElasticaIndex(),
                'config' => array(
                    'mappings' => array(
-                        'a' => array('properties' => array()),
+                        'a' => array(
+                            'dynamic_templates' => array(),
+                            'properties' => array(),
+                        ),
                        'b' => array('properties' => array()),
                    ),
                ),
@ -100,6 +103,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase
            ->method('delete');

        $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']);
+        $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']);
        $type->expects($this->once())
            ->method('setMapping')
            ->with($mapping);
--- a/composer.json
+++ b/composer.json
@ -24,14 +24,16 @@
        "doctrine/mongodb-odm": "1.0.*@dev",
        "propel/propel1": "1.6.*",
        "pagerfanta/pagerfanta": "1.0.*@dev",
-        "knplabs/knp-components": "1.2.*"
+        "knplabs/knp-components": "1.2.*",
+        "symfony/expression-language" : "2.4.*@dev"
    },
    "suggest": {
        "doctrine/orm": ">=2.2,<2.5-dev",
        "doctrine/mongodb-odm": "1.0.*@dev",
        "propel/propel1": "1.6.*",
        "pagerfanta/pagerfanta": "1.0.*@dev",
-        "knplabs/knp-components": "1.2.*"
+        "knplabs/knp-components": "1.2.*",
+        "symfony/expression-language" : "2.4.*@dev"
    },
    "autoload": {
        "psr-0": { "FOS\\ElasticaBundle": "" }