Merge pull request #51 from willdurand/propel-support

Propel support
Conflicts:

	DependencyInjection/FOQElasticaExtension.php

DependencyInjection/Configuration.php

@@ -7,6 +7,8 @@ use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 
 class Configuration
 {
+    private $supportedDrivers = array('orm', 'mongodb', 'propel');
+
     /**
      * Generates the configuration tree.
      *
@@ -70,9 +72,20 @@ class Configuration
                             ->scalarNode('client')->end()
                             ->arrayNode('type_prototype')
                                 ->children()
-                                    ->arrayNode('doctrine')
+                                    ->arrayNode('persistence')
+                                        ->validate()
+                                            ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['listener']); })
+                                            ->thenInvalid('Propel doesn\'t support listeners')
+                                            ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['repository']); })
+                                            ->thenInvalid('Propel doesn\'t support the "repository" parameter')
+                                        ->end()
                                         ->children()
-                                            ->scalarNode('driver')->end()
+                                            ->scalarNode('driver')
+                                                ->validate()
+                                                    ->ifNotInArray($this->supportedDrivers)
+                                                    ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers))
+                                                ->end()
+                                            ->end()
                                             ->scalarNode('identifier')->defaultValue('id')->end()
                                             ->arrayNode('provider')
                                                 ->children()
@@ -134,9 +147,20 @@ class Configuration
             ->prototype('array')
                 ->treatNullLike(array())
                 ->children()
-                    ->arrayNode('doctrine')
+                    ->arrayNode('persistence')
+                        ->validate()
+                            ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['listener']); })
+                            ->thenInvalid('Propel doesn\'t support listeners')
+                            ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['repository']); })
+                            ->thenInvalid('Propel doesn\'t support the "repository" parameter')
+                        ->end()
                         ->children()
-                            ->scalarNode('driver')->end()
+                            ->scalarNode('driver')
+                                ->validate()
+                                    ->ifNotInArray($this->supportedDrivers)
+                                    ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers))
+                                ->end()
+                            ->end()
                             ->scalarNode('model')->end()
                             ->scalarNode('repository')->end()
                             ->scalarNode('identifier')->defaultValue('id')->end()

DependencyInjection/FOQElasticaExtension.php

@@ -14,16 +14,15 @@ use InvalidArgumentException;
 
 class FOQElasticaExtension extends Extension
 {
-    protected $supportedProviderDrivers = array('mongodb', 'orm');
-    protected $indexConfigs = array();
-    protected $typeFields = array();
-    protected $loadedDoctrineDrivers = array();
+    protected $indexConfigs     = array();
+    protected $typeFields       = array();
+    protected $loadedDrivers    = array();
 
     public function load(array $configs, ContainerBuilder $container)
     {
         $configuration = new Configuration();
-        $processor = new Processor();
-        $config = $processor->process($configuration->getConfigTree(), $configs);
+        $processor     = new Processor();
+        $config        = $processor->process($configuration->getConfigTree(), $configs);
 
         $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
         $loader->load('config.xml');
@@ -43,7 +42,7 @@ class FOQElasticaExtension extends Extension
         }
 
         $clientIdsByName = $this->loadClients($config['clients'], $container);
-        $indexIdsByName = $this->loadIndexes($config['indexes'], $container, $clientIdsByName, $config['default_client']);
+        $indexIdsByName  = $this->loadIndexes($config['indexes'], $container, $clientIdsByName, $config['default_client']);
         $indexDefsByName = array_map(function($id) use ($container) {
             return $container->getDefinition($id);
         }, $indexIdsByName);
@@ -96,6 +95,7 @@ class FOQElasticaExtension extends Extension
             } else {
                 $clientName = $defaultClientName;
             }
+
             $clientId = $clientIdsByName[$clientName];
             $indexId = sprintf('foq_elastica.index.%s', $name);
             $indexDefArgs = array($name);
@@ -141,8 +141,8 @@ class FOQElasticaExtension extends Extension
                 $typeName = sprintf('%s/%s', $indexName, $name);
                 $this->typeFields[$typeName] = array_keys($type['mappings']);
             }
-            if (isset($type['doctrine'])) {
-                $this->loadTypeDoctrineIntegration($type['doctrine'], $container, $typeDef, $indexName, $name);
+            if (isset($type['persistence'])) {
+                $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name);
             }
         }
     }
@@ -173,12 +173,9 @@ class FOQElasticaExtension extends Extension
      *
      * @return null
      **/
-    protected function loadTypeDoctrineIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName)
+    protected function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName)
     {
-        if (!in_array($typeConfig['driver'], $this->supportedProviderDrivers)) {
-            throw new InvalidArgumentException(sprintf('The provider driver "%s" is not supported', $typeConfig['driver']));
-        }
-        $this->loadDoctrineDriver($container, $typeConfig['driver']);
+        $this->loadDriver($container, $typeConfig['driver']);
 
         $elasticaToModelTransformerId = $this->loadElasticaToModelTransformer($typeConfig, $container, $indexName, $typeName);
         $modelToElasticaTransformerId = $this->loadModelToElasticaTransformer($typeConfig, $container, $indexName, $typeName);
@@ -204,10 +201,14 @@ class FOQElasticaExtension extends Extension
         $abstractId = sprintf('foq_elastica.elastica_to_model_transformer.prototype.%s', $typeConfig['driver']);
         $serviceId = sprintf('foq_elastica.elastica_to_model_transformer.%s.%s', $indexName, $typeName);
         $serviceDef = new DefinitionDecorator($abstractId);
-        $serviceDef->replaceArgument(1, $typeConfig['model']);
-        $serviceDef->replaceArgument(2, array(
-            'identifier' => $typeConfig['identifier'],
-            'hydrate' => $typeConfig['elastica_to_model_transformer']['hydrate']
+
+        // Doctrine has a mandatory service as first argument
+        $argPos = ('propel' === $typeConfig['driver']) ? 0 : 1;
+
+        $serviceDef->replaceArgument($argPos, $typeConfig['model']);
+        $serviceDef->replaceArgument($argPos + 1, array(
+            'identifier'    => $typeConfig['identifier'],
+            'hydrate'       => $typeConfig['elastica_to_model_transformer']['hydrate']
         ));
         $container->setDefinition($serviceId, $serviceDef);
 
@@ -253,13 +254,21 @@ class FOQElasticaExtension extends Extension
         $providerId = sprintf('foq_elastica.provider.%s.%s', $indexName, $typeName);
         $providerDef = new DefinitionDecorator($abstractProviderId);
         $providerDef->replaceArgument(0, $typeDef);
-        $providerDef->replaceArgument(2, new Reference($objectPersisterId));
-        $providerDef->replaceArgument(3, $typeConfig['model']);
-        $providerDef->replaceArgument(4, array(
-            'query_builder_method' => $typeConfig['provider']['query_builder_method'],
-            'batch_size'           => $typeConfig['provider']['batch_size'],
-            'clear_object_manager' => $typeConfig['provider']['clear_object_manager']
-        ));
+
+        // Doctrine has a mandatory service as second argument
+        $argPos = ('propel' === $typeConfig['driver']) ? 1 : 2;
+
+        $providerDef->replaceArgument($argPos, new Reference($objectPersisterId));
+        $providerDef->replaceArgument($argPos + 1, $typeConfig['model']);
+
+        $options = array('batch_size' => $typeConfig['provider']['batch_size']);
+
+        if ('propel' !== $typeConfig['driver']) {
+            $options['query_builder_method'] = $typeConfig['provider']['query_builder_method'];
+            $options['clear_object_manager'] = $typeConfig['provider']['clear_object_manager'];
+        }
+
+        $providerDef->replaceArgument($argPos + 2, $options);
         $container->setDefinition($providerId, $providerDef);
 
         return $providerId;
@@ -304,14 +313,16 @@ class FOQElasticaExtension extends Extension
         $finderDef->replaceArgument(1, new Reference($elasticaToModelId));
         $container->setDefinition($finderId, $finderDef);
 
-        $managerDef = $container->getDefinition('foq_elastica.manager');
-        $arguments = array( $typeConfig['model'], new Reference($finderId));
-        if (isset($typeConfig['repository'])) {
-            $arguments[] = $typeConfig['repository'];
-        }
+        if ('propel' !== $typeConfig['driver']) {
+            $managerDef = $container->getDefinition('foq_elastica.manager');
+            $arguments = array( $typeConfig['model'], new Reference($finderId));
+            if (isset($typeConfig['repository'])) {
+                $arguments[] = $typeConfig['repository'];
+            }
 
-        $managerDef->addMethodCall('addEntity', $arguments);
-        $container->setDefinition('foq_elastica.manager', $managerDef);
+            $managerDef->addMethodCall('addEntity', $arguments);
+            $container->setDefinition('foq_elastica.manager', $managerDef);
+        }
 
         return $finderId;
     }
@@ -339,13 +350,13 @@ class FOQElasticaExtension extends Extension
         $reseterDef->replaceArgument(0, $indexConfigs);
     }
 
-    protected function loadDoctrineDriver(ContainerBuilder $container, $driver)
+    protected function loadDriver(ContainerBuilder $container, $driver)
     {
-        if (in_array($driver, $this->loadedDoctrineDrivers)) {
+        if (in_array($driver, $this->loadedDrivers)) {
             return;
         }
         $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
         $loader->load($driver.'.xml');
-        $this->loadedDoctrineDrivers[] = $driver;
+        $this->loadedDrivers[] = $driver;
     }
 }

Propel/ElasticaToModelTransformer.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace FOQ\ElasticaBundle\Propel;
+
+use FOQ\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface;
+use Elastica_Document;
+
+/**
+ * Maps Elastica documents with Propel objects
+ * This mapper assumes an exact match between
+ * elastica documents ids and propel object ids
+ *
+ * @author William Durand <william.durand1@gmail.com>
+ */
+class ElasticaToModelTransformer implements ElasticaToModelTransformerInterface
+{
+    /**
+     * Class of the model to map to the elastica documents
+     *
+     * @var string
+     */
+    protected $objectClass = null;
+
+    /**
+     * Optional parameters
+     *
+     * @var array
+     */
+    protected $options = array(
+        'hydrate'    => true,
+        'identifier' => 'id'
+    );
+
+    /**
+     * Instantiates a new Mapper
+     *
+     * @param string $objectClass
+     * @param array $options
+     */
+    public function __construct($objectClass, array $options = array())
+    {
+        $this->objectClass = $objectClass;
+        $this->options     = array_merge($this->options, $options);
+    }
+
+    /**
+     * Transforms an array of elastica objects into an array of
+     * model objects fetched from the propel repository
+     *
+     * @param array of elastica objects
+     * @return array
+     */
+    public function transform(array $elasticaObjects)
+    {
+        $ids = array_map(function($elasticaObject) {
+            return $elasticaObject->getId();
+        }, $elasticaObjects);
+
+        $objects = $this->findByIdentifiers($this->objectClass, $this->options['identifier'], $ids, $this->options['hydrate']);
+
+        $identifierGetter = 'get'.ucfirst($this->options['identifier']);
+
+        // sort objects in the order of ids
+        $idPos = array_flip($ids);
+        $objects->uasort(function($a, $b) use ($idPos, $identifierGetter) {
+            return $idPos[$a->$identifierGetter()] > $idPos[$b->$identifierGetter()];
+        });
+
+        return $objects;
+    }
+
+
+    /**
+     * Fetch objects for theses identifier values
+     *
+     * @param string $class the model class
+     * @param string $identifierField like 'id'
+     * @param array $identifierValues ids values
+     * @param Boolean $hydrate whether or not to hydrate the objects, false returns arrays
+     * @return array of objects or arrays
+     */
+    protected function findByIdentifiers($class, $identifierField, array $identifierValues, $hydrate)
+    {
+        if (empty($identifierValues)) {
+            return array();
+        }
+
+        $queryClass   = $class.'Query';
+        $filterMethod = 'filterBy'.$this->camelize($identifierField);
+        $query = $queryClass::create()
+            ->$filterMethod($identifierValues)
+            ;
+
+        if (!$hydrate) {
+            return $query->toArray();
+        }
+
+        return $query->find();
+    }
+
+    /**
+     * @see https://github.com/doctrine/common/blob/master/lib/Doctrine/Common/Util/Inflector.php
+     */
+    private function camelize($str)
+    {
+        return ucfirst(str_replace(" ", "", ucwords(strtr($str, "_-", "  "))));
+    }
+}

Propel/Provider.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace FOQ\ElasticaBundle\Propel;
+
+use FOQ\ElasticaBundle\Provider\ProviderInterface;
+use FOQ\ElasticaBundle\Persister\ObjectPersisterInterface;
+use Elastica_Type;
+use Elastica_Document;
+use Closure;
+use InvalidArgumentException;
+
+/**
+ * Propel provider
+ *
+ * @author William Durand <william.durand1@gmail.com>
+ */
+class Provider implements ProviderInterface
+{
+    /**
+     * Elastica type
+     *
+     * @var Elastica_Type
+     */
+    protected $type;
+
+    /**
+     * Object persister
+     *
+     * @var ObjectPersisterInterface
+     */
+    protected $objectPersister;
+
+    /**
+     * Provider options
+     *
+     * @var array
+     */
+    protected $options = array(
+        'batch_size'           => 100,
+    );
+
+    public function __construct(Elastica_Type $type, ObjectPersisterInterface $objectPersister, $objectClass, array $options = array())
+    {
+        $this->type            = $type;
+        $this->objectClass     = $objectClass;
+        $this->objectPersister = $objectPersister;
+        $this->options         = array_merge($this->options, $options);
+    }
+
+    /**
+     * Insert the repository objects in the type index
+     *
+     * @param Closure $loggerClosure
+     */
+    public function populate(Closure $loggerClosure)
+    {
+        $queryClass = $this->objectClass . 'Query';
+        $nbObjects  = $queryClass::create()->count();
+
+        for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) {
+
+            $stepStartTime = microtime(true);
+            $objects = $queryClass::create()
+                ->limit($this->options['batch_size'])
+                ->offset($offset)
+                ->find();
+
+            $this->objectPersister->insertMany($objects->getArrayCopy());
+
+            $stepNbObjects = count($objects);
+            $stepCount = $stepNbObjects+$offset;
+            $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime);
+            $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', 100*$stepCount/$nbObjects, $stepCount, $nbObjects, $objectsPerSecond));
+        }
+    }
+}

README.md

@@ -95,7 +95,7 @@ $ php bin/vendors install
 
 #### Declare a client
 
-Elasticsearch client is comparable to doctrine connection.
+Elasticsearch client is comparable to a database connection.
 Most of the time, you will need only one.
 
     #app/config/config.yml
@@ -105,7 +105,7 @@ Most of the time, you will need only one.
 
 #### Declare an index
 
-Elasticsearch index is comparable to doctrine entity manager.
+Elasticsearch index is comparable to Doctrine entity manager.
 Most of the time, you will need only one.
 
     foq_elastica:
@@ -121,7 +121,7 @@ Our index is now available as a service: `foq_elastica.index.website`. It is an
 
 #### Declare a type
 
-Elasticsearch type is comparable to doctrine entity repository.
+Elasticsearch type is comparable to Doctrine entity repository.
 
     foq_elastica:
         clients:
@@ -148,12 +148,12 @@ It applies the configured mappings to the types.
 
 This command needs providers to insert new documents in the elasticsearch types.
 There are 2 ways to create providers.
-If your elasticsearch type matches a doctrine repository, go for the doctrine automatic provider.
+If your elasticsearch type matches a Doctrine repository or a Propel query, go for the persistence automatic provider.
 Or, for complete flexibility, go for manual provider.
 
-#### Doctrine automatic provider
+#### Persistence automatic provider
 
-If we want to index the entities from a doctrine repository,
+If we want to index the entities from a Doctrine repository or a Propel query,
 some configuration will let ElasticaBundle do it for us.
 
     foq_elastica:
@@ -168,31 +168,33 @@ some configuration will let ElasticaBundle do it for us.
                             username: { boost: 5 }
                             firstName: { boost: 3 }
                             # more mappings...
-                        doctrine:
-                            driver: orm
+                        persistence:
+                            driver: orm # orm, mongodb, propel are available
                             model: Application\UserBundle\Entity\User
                             provider:
 
-Two drivers are actually supported: orm and mongodb.
+Three drivers are actually supported: orm, mongodb, and propel.
 
-##### Use a custom doctrine query builder
+##### Use a custom Doctrine query builder
 
 You can control which entities will be indexed by specifying a custom query builder method.
 
-                        doctrine:
+                        persistence:
                             driver: orm
                             model: Application\UserBundle\Entity\User
                             provider:
                                 query_builder_method: createIsActiveQueryBuilder
 
-Your repository must implement this method and return a doctrine query builder.
+Your repository must implement this method and return a Doctrine query builder.
+
+> **Propel** doesn't support this feature yet.
 
 ##### Change the batch size
 
 By default, ElasticaBundle will index documents by paquets of 100.
 You can change this value in the provider configuration.
 
-                        doctrine:
+                        persistence:
                             driver: orm
                             model: Application\UserBundle\Entity\User
                             provider:
@@ -203,7 +205,7 @@ You can change this value in the provider configuration.
 By default, ElasticaBundle will use the `id` field of your entities as the elasticsearch document identifier.
 You can change this value in the provider configuration.
 
-                        doctrine:
+                        persistence:
                             driver: orm
                             model: Application\UserBundle\Entity\User
                             provider:
@@ -251,7 +253,7 @@ Its class must implement `FOQ\ElasticaBundle\Provider\ProviderInterface`.
             }
         }
 
-You will find a more complete implementation example in src/FOQ/ElasticaBundle/Provider/DoctrineProvider.php
+You will find a more complete implementation example in `src/FOQ/ElasticaBundle/Provider/Doctrine/ORM/Provider.php`.
 
 ### Search
 
@@ -265,9 +267,9 @@ You can just use the index and type Elastica objects, provided as services, to p
 
 #### Doctrine finder
 
-If your elasticsearch type is bound to a doctrine entity repository,
+If your elasticsearch type is bound to a Doctrine entity repository or a Propel query,
 you can get your entities instead of Elastica results when you perform a search.
-Declare that you want a doctrine finder in your configuration:
+Declare that you want a Doctrine/Propel finder in your configuration:
 
     foq_elastica:
         clients:
@@ -279,7 +281,7 @@ Declare that you want a doctrine finder in your configuration:
                     user:
                         mappings:
                             # your mappings
-                        doctrine:
+                        persistence:
                             driver: orm
                             model: Application\UserBundle\Entity\User
                             provider:
@@ -303,8 +305,8 @@ You can even get paginated results!
 
 ### Realtime, selective index update
 
-If you use the doctrine integration, you can let ElasticaBundle update the indexes automatically
-when an object is added, updated or removed. It uses doctrine lifecycle events.
+If you use the Doctrine integration, you can let ElasticaBundle update the indexes automatically
+when an object is added, updated or removed. It uses Doctrine lifecycle events.
 Declare that you want to update the index in real time:
 
     foq_elastica:
@@ -317,22 +319,24 @@ Declare that you want to update the index in real time:
                     user:
                         mappings:
                             # your mappings
-                        doctrine:
+                        persistence:
                             driver: orm
                             model: Application\UserBundle\Entity\User
                             listener: # by default, listens to "insert", "update" and "delete"
 
-Now the index is automatically updated each time the state of the bound doctrine repository changes.
+Now the index is automatically updated each time the state of the bound Doctrine repository changes.
 No need to repopulate the whole "user" index when a new `User` is created.
 
 You can also choose to only listen for some of the events:
 
-                        doctrine:
+                        persistence:
                             listener:
                                 insert: true
                                 update: false
                                 delete: true
 
+> **Propel** doesn't support this feature yet.
+
 ### Advanced elasticsearch configuration
 
 Any setting can be specified when declaring a type. For example, to enable a custom analyzer, you could write:

Resources/config/propel.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" ?>
+<container xmlns="http://symfony.com/schema/dic/services"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
+
+    <services>
+
+        <service id="foq_elastica.provider.prototype.propel" class="FOQ\ElasticaBundle\Propel\Provider" public="false" abstract="true">
+            <argument /> <!-- type -->
+            <argument /> <!-- object persister -->
+            <argument /> <!-- model -->
+            <argument type="collection" /> <!-- options -->
+        </service>
+
+        <service id="foq_elastica.elastica_to_model_transformer.prototype.propel" class="FOQ\ElasticaBundle\Propel\ElasticaToModelTransformer" public="false">
+            <argument /> <!-- model -->
+            <argument type="collection" /> <!-- options -->
+        </service>
+
+    </services>
+
+</container>

Tests/Persister/ObjectPersisterTest.php

@@ -24,7 +24,7 @@ class InvalidObjectPersister extends ObjectPersister
 {
     protected function transformToElasticaDocument($object)
     {
-        throw new \Exception('Invalid transformation');
+        throw new \BadMethodCallException('Invalid transformation');
     }
 }
 
@@ -57,7 +57,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase
     }
 
     /**
-     * @expectedException Exception
+     * @expectedException \BadMethodCallException
      */
     public function testThatErrorIsHandledWhenCannotReplaceObject()
     {
@@ -96,7 +96,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase
     }
 
     /**
-     * @expectedException Exception
+     * @expectedException \BadMethodCallException
      */
     public function testThatErrorIsHandledWhenCannotInsertObject()
     {
@@ -135,7 +135,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase
     }
 
     /**
-     * @expectedException Exception
+     * @expectedException \BadMethodCallException
      */
     public function testThatErrorIsHandledWhenCannotDeleteObject()
     {
@@ -176,7 +176,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase
     }
 
     /**
-     * @expectedException Exception
+     * @expectedException \BadMethodCallException
      */
     public function testThatErrorIsHandledWhenCannotInsertManyObject()
     {