diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 42f49b7..908cb2b 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -7,6 +7,8 @@ use Symfony\Component\Config\Definition\Builder\TreeBuilder; class Configuration { + private $supportedDrivers = array('orm', 'mongodb', 'propel'); + /** * Generates the configuration tree. * @@ -70,9 +72,20 @@ class Configuration ->scalarNode('client')->end() ->arrayNode('type_prototype') ->children() - ->arrayNode('doctrine') + ->arrayNode('persistence') + ->validate() + ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['listener']); }) + ->thenInvalid('Propel doesn\'t support listeners') + ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['repository']); }) + ->thenInvalid('Propel doesn\'t support the "repository" parameter') + ->end() ->children() - ->scalarNode('driver')->end() + ->scalarNode('driver') + ->validate() + ->ifNotInArray($this->supportedDrivers) + ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) + ->end() + ->end() ->scalarNode('identifier')->defaultValue('id')->end() ->arrayNode('provider') ->children() @@ -134,9 +147,20 @@ class Configuration ->prototype('array') ->treatNullLike(array()) ->children() - ->arrayNode('doctrine') + ->arrayNode('persistence') + ->validate() + ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['listener']); }) + ->thenInvalid('Propel doesn\'t support listeners') + ->ifTrue(function($v) { return 'propel' === $v['driver'] && isset($v['repository']); }) + ->thenInvalid('Propel doesn\'t support the "repository" parameter') + ->end() ->children() - ->scalarNode('driver')->end() + ->scalarNode('driver') + ->validate() + ->ifNotInArray($this->supportedDrivers) + ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) + ->end() + ->end() ->scalarNode('model')->end() ->scalarNode('repository')->end() ->scalarNode('identifier')->defaultValue('id')->end() diff --git a/DependencyInjection/FOQElasticaExtension.php b/DependencyInjection/FOQElasticaExtension.php index 88226bd..0b6f15f 100644 --- a/DependencyInjection/FOQElasticaExtension.php +++ b/DependencyInjection/FOQElasticaExtension.php @@ -14,16 +14,15 @@ use InvalidArgumentException; class FOQElasticaExtension extends Extension { - protected $supportedProviderDrivers = array('mongodb', 'orm'); - protected $indexConfigs = array(); - protected $typeFields = array(); - protected $loadedDoctrineDrivers = array(); + protected $indexConfigs = array(); + protected $typeFields = array(); + protected $loadedDrivers = array(); public function load(array $configs, ContainerBuilder $container) { $configuration = new Configuration(); - $processor = new Processor(); - $config = $processor->process($configuration->getConfigTree(), $configs); + $processor = new Processor(); + $config = $processor->process($configuration->getConfigTree(), $configs); $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load('config.xml'); @@ -43,7 +42,7 @@ class FOQElasticaExtension extends Extension } $clientIdsByName = $this->loadClients($config['clients'], $container); - $indexIdsByName = $this->loadIndexes($config['indexes'], $container, $clientIdsByName, $config['default_client']); + $indexIdsByName = $this->loadIndexes($config['indexes'], $container, $clientIdsByName, $config['default_client']); $indexDefsByName = array_map(function($id) use ($container) { return $container->getDefinition($id); }, $indexIdsByName); @@ -96,6 +95,7 @@ class FOQElasticaExtension extends Extension } else { $clientName = $defaultClientName; } + $clientId = $clientIdsByName[$clientName]; $indexId = sprintf('foq_elastica.index.%s', $name); $indexDefArgs = array($name); @@ -141,8 +141,8 @@ class FOQElasticaExtension extends Extension $typeName = sprintf('%s/%s', $indexName, $name); $this->typeFields[$typeName] = array_keys($type['mappings']); } - if (isset($type['doctrine'])) { - $this->loadTypeDoctrineIntegration($type['doctrine'], $container, $typeDef, $indexName, $name); + if (isset($type['persistence'])) { + $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); } } } @@ -173,12 +173,9 @@ class FOQElasticaExtension extends Extension * * @return null **/ - protected function loadTypeDoctrineIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName) + protected function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName) { - if (!in_array($typeConfig['driver'], $this->supportedProviderDrivers)) { - throw new InvalidArgumentException(sprintf('The provider driver "%s" is not supported', $typeConfig['driver'])); - } - $this->loadDoctrineDriver($container, $typeConfig['driver']); + $this->loadDriver($container, $typeConfig['driver']); $elasticaToModelTransformerId = $this->loadElasticaToModelTransformer($typeConfig, $container, $indexName, $typeName); $modelToElasticaTransformerId = $this->loadModelToElasticaTransformer($typeConfig, $container, $indexName, $typeName); @@ -204,10 +201,14 @@ class FOQElasticaExtension extends Extension $abstractId = sprintf('foq_elastica.elastica_to_model_transformer.prototype.%s', $typeConfig['driver']); $serviceId = sprintf('foq_elastica.elastica_to_model_transformer.%s.%s', $indexName, $typeName); $serviceDef = new DefinitionDecorator($abstractId); - $serviceDef->replaceArgument(1, $typeConfig['model']); - $serviceDef->replaceArgument(2, array( - 'identifier' => $typeConfig['identifier'], - 'hydrate' => $typeConfig['elastica_to_model_transformer']['hydrate'] + + // Doctrine has a mandatory service as first argument + $argPos = ('propel' === $typeConfig['driver']) ? 0 : 1; + + $serviceDef->replaceArgument($argPos, $typeConfig['model']); + $serviceDef->replaceArgument($argPos + 1, array( + 'identifier' => $typeConfig['identifier'], + 'hydrate' => $typeConfig['elastica_to_model_transformer']['hydrate'] )); $container->setDefinition($serviceId, $serviceDef); @@ -253,13 +254,21 @@ class FOQElasticaExtension extends Extension $providerId = sprintf('foq_elastica.provider.%s.%s', $indexName, $typeName); $providerDef = new DefinitionDecorator($abstractProviderId); $providerDef->replaceArgument(0, $typeDef); - $providerDef->replaceArgument(2, new Reference($objectPersisterId)); - $providerDef->replaceArgument(3, $typeConfig['model']); - $providerDef->replaceArgument(4, array( - 'query_builder_method' => $typeConfig['provider']['query_builder_method'], - 'batch_size' => $typeConfig['provider']['batch_size'], - 'clear_object_manager' => $typeConfig['provider']['clear_object_manager'] - )); + + // Doctrine has a mandatory service as second argument + $argPos = ('propel' === $typeConfig['driver']) ? 1 : 2; + + $providerDef->replaceArgument($argPos, new Reference($objectPersisterId)); + $providerDef->replaceArgument($argPos + 1, $typeConfig['model']); + + $options = array('batch_size' => $typeConfig['provider']['batch_size']); + + if ('propel' !== $typeConfig['driver']) { + $options['query_builder_method'] = $typeConfig['provider']['query_builder_method']; + $options['clear_object_manager'] = $typeConfig['provider']['clear_object_manager']; + } + + $providerDef->replaceArgument($argPos + 2, $options); $container->setDefinition($providerId, $providerDef); return $providerId; @@ -304,14 +313,16 @@ class FOQElasticaExtension extends Extension $finderDef->replaceArgument(1, new Reference($elasticaToModelId)); $container->setDefinition($finderId, $finderDef); - $managerDef = $container->getDefinition('foq_elastica.manager'); - $arguments = array( $typeConfig['model'], new Reference($finderId)); - if (isset($typeConfig['repository'])) { - $arguments[] = $typeConfig['repository']; - } + if ('propel' !== $typeConfig['driver']) { + $managerDef = $container->getDefinition('foq_elastica.manager'); + $arguments = array( $typeConfig['model'], new Reference($finderId)); + if (isset($typeConfig['repository'])) { + $arguments[] = $typeConfig['repository']; + } - $managerDef->addMethodCall('addEntity', $arguments); - $container->setDefinition('foq_elastica.manager', $managerDef); + $managerDef->addMethodCall('addEntity', $arguments); + $container->setDefinition('foq_elastica.manager', $managerDef); + } return $finderId; } @@ -339,13 +350,13 @@ class FOQElasticaExtension extends Extension $reseterDef->replaceArgument(0, $indexConfigs); } - protected function loadDoctrineDriver(ContainerBuilder $container, $driver) + protected function loadDriver(ContainerBuilder $container, $driver) { - if (in_array($driver, $this->loadedDoctrineDrivers)) { + if (in_array($driver, $this->loadedDrivers)) { return; } $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load($driver.'.xml'); - $this->loadedDoctrineDrivers[] = $driver; + $this->loadedDrivers[] = $driver; } } diff --git a/Propel/ElasticaToModelTransformer.php b/Propel/ElasticaToModelTransformer.php new file mode 100644 index 0000000..92f8142 --- /dev/null +++ b/Propel/ElasticaToModelTransformer.php @@ -0,0 +1,108 @@ + + */ +class ElasticaToModelTransformer implements ElasticaToModelTransformerInterface +{ + /** + * Class of the model to map to the elastica documents + * + * @var string + */ + protected $objectClass = null; + + /** + * Optional parameters + * + * @var array + */ + protected $options = array( + 'hydrate' => true, + 'identifier' => 'id' + ); + + /** + * Instantiates a new Mapper + * + * @param string $objectClass + * @param array $options + */ + public function __construct($objectClass, array $options = array()) + { + $this->objectClass = $objectClass; + $this->options = array_merge($this->options, $options); + } + + /** + * Transforms an array of elastica objects into an array of + * model objects fetched from the propel repository + * + * @param array of elastica objects + * @return array + */ + public function transform(array $elasticaObjects) + { + $ids = array_map(function($elasticaObject) { + return $elasticaObject->getId(); + }, $elasticaObjects); + + $objects = $this->findByIdentifiers($this->objectClass, $this->options['identifier'], $ids, $this->options['hydrate']); + + $identifierGetter = 'get'.ucfirst($this->options['identifier']); + + // sort objects in the order of ids + $idPos = array_flip($ids); + $objects->uasort(function($a, $b) use ($idPos, $identifierGetter) { + return $idPos[$a->$identifierGetter()] > $idPos[$b->$identifierGetter()]; + }); + + return $objects; + } + + + /** + * Fetch objects for theses identifier values + * + * @param string $class the model class + * @param string $identifierField like 'id' + * @param array $identifierValues ids values + * @param Boolean $hydrate whether or not to hydrate the objects, false returns arrays + * @return array of objects or arrays + */ + protected function findByIdentifiers($class, $identifierField, array $identifierValues, $hydrate) + { + if (empty($identifierValues)) { + return array(); + } + + $queryClass = $class.'Query'; + $filterMethod = 'filterBy'.$this->camelize($identifierField); + $query = $queryClass::create() + ->$filterMethod($identifierValues) + ; + + if (!$hydrate) { + return $query->toArray(); + } + + return $query->find(); + } + + /** + * @see https://github.com/doctrine/common/blob/master/lib/Doctrine/Common/Util/Inflector.php + */ + private function camelize($str) + { + return ucfirst(str_replace(" ", "", ucwords(strtr($str, "_-", " ")))); + } +} diff --git a/Propel/Provider.php b/Propel/Provider.php new file mode 100644 index 0000000..46d28a2 --- /dev/null +++ b/Propel/Provider.php @@ -0,0 +1,76 @@ + + */ +class Provider implements ProviderInterface +{ + /** + * Elastica type + * + * @var Elastica_Type + */ + protected $type; + + /** + * Object persister + * + * @var ObjectPersisterInterface + */ + protected $objectPersister; + + /** + * Provider options + * + * @var array + */ + protected $options = array( + 'batch_size' => 100, + ); + + public function __construct(Elastica_Type $type, ObjectPersisterInterface $objectPersister, $objectClass, array $options = array()) + { + $this->type = $type; + $this->objectClass = $objectClass; + $this->objectPersister = $objectPersister; + $this->options = array_merge($this->options, $options); + } + + /** + * Insert the repository objects in the type index + * + * @param Closure $loggerClosure + */ + public function populate(Closure $loggerClosure) + { + $queryClass = $this->objectClass . 'Query'; + $nbObjects = $queryClass::create()->count(); + + for ($offset = 0; $offset < $nbObjects; $offset += $this->options['batch_size']) { + + $stepStartTime = microtime(true); + $objects = $queryClass::create() + ->limit($this->options['batch_size']) + ->offset($offset) + ->find(); + + $this->objectPersister->insertMany($objects->getArrayCopy()); + + $stepNbObjects = count($objects); + $stepCount = $stepNbObjects+$offset; + $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); + $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', 100*$stepCount/$nbObjects, $stepCount, $nbObjects, $objectsPerSecond)); + } + } +} diff --git a/README.md b/README.md index 782a7e0..7ec16cd 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ $ php bin/vendors install #### Declare a client -Elasticsearch client is comparable to doctrine connection. +Elasticsearch client is comparable to a database connection. Most of the time, you will need only one. #app/config/config.yml @@ -105,7 +105,7 @@ Most of the time, you will need only one. #### Declare an index -Elasticsearch index is comparable to doctrine entity manager. +Elasticsearch index is comparable to Doctrine entity manager. Most of the time, you will need only one. foq_elastica: @@ -121,7 +121,7 @@ Our index is now available as a service: `foq_elastica.index.website`. It is an #### Declare a type -Elasticsearch type is comparable to doctrine entity repository. +Elasticsearch type is comparable to Doctrine entity repository. foq_elastica: clients: @@ -148,12 +148,12 @@ It applies the configured mappings to the types. This command needs providers to insert new documents in the elasticsearch types. There are 2 ways to create providers. -If your elasticsearch type matches a doctrine repository, go for the doctrine automatic provider. +If your elasticsearch type matches a Doctrine repository or a Propel query, go for the persistence automatic provider. Or, for complete flexibility, go for manual provider. -#### Doctrine automatic provider +#### Persistence automatic provider -If we want to index the entities from a doctrine repository, +If we want to index the entities from a Doctrine repository or a Propel query, some configuration will let ElasticaBundle do it for us. foq_elastica: @@ -168,31 +168,33 @@ some configuration will let ElasticaBundle do it for us. username: { boost: 5 } firstName: { boost: 3 } # more mappings... - doctrine: - driver: orm + persistence: + driver: orm # orm, mongodb, propel are available model: Application\UserBundle\Entity\User provider: -Two drivers are actually supported: orm and mongodb. +Three drivers are actually supported: orm, mongodb, and propel. -##### Use a custom doctrine query builder +##### Use a custom Doctrine query builder You can control which entities will be indexed by specifying a custom query builder method. - doctrine: + persistence: driver: orm model: Application\UserBundle\Entity\User provider: query_builder_method: createIsActiveQueryBuilder -Your repository must implement this method and return a doctrine query builder. +Your repository must implement this method and return a Doctrine query builder. + +> **Propel** doesn't support this feature yet. ##### Change the batch size By default, ElasticaBundle will index documents by paquets of 100. You can change this value in the provider configuration. - doctrine: + persistence: driver: orm model: Application\UserBundle\Entity\User provider: @@ -203,7 +205,7 @@ You can change this value in the provider configuration. By default, ElasticaBundle will use the `id` field of your entities as the elasticsearch document identifier. You can change this value in the provider configuration. - doctrine: + persistence: driver: orm model: Application\UserBundle\Entity\User provider: @@ -251,7 +253,7 @@ Its class must implement `FOQ\ElasticaBundle\Provider\ProviderInterface`. } } -You will find a more complete implementation example in src/FOQ/ElasticaBundle/Provider/DoctrineProvider.php +You will find a more complete implementation example in `src/FOQ/ElasticaBundle/Provider/Doctrine/ORM/Provider.php`. ### Search @@ -265,9 +267,9 @@ You can just use the index and type Elastica objects, provided as services, to p #### Doctrine finder -If your elasticsearch type is bound to a doctrine entity repository, +If your elasticsearch type is bound to a Doctrine entity repository or a Propel query, you can get your entities instead of Elastica results when you perform a search. -Declare that you want a doctrine finder in your configuration: +Declare that you want a Doctrine/Propel finder in your configuration: foq_elastica: clients: @@ -279,7 +281,7 @@ Declare that you want a doctrine finder in your configuration: user: mappings: # your mappings - doctrine: + persistence: driver: orm model: Application\UserBundle\Entity\User provider: @@ -303,8 +305,8 @@ You can even get paginated results! ### Realtime, selective index update -If you use the doctrine integration, you can let ElasticaBundle update the indexes automatically -when an object is added, updated or removed. It uses doctrine lifecycle events. +If you use the Doctrine integration, you can let ElasticaBundle update the indexes automatically +when an object is added, updated or removed. It uses Doctrine lifecycle events. Declare that you want to update the index in real time: foq_elastica: @@ -317,22 +319,24 @@ Declare that you want to update the index in real time: user: mappings: # your mappings - doctrine: + persistence: driver: orm model: Application\UserBundle\Entity\User listener: # by default, listens to "insert", "update" and "delete" -Now the index is automatically updated each time the state of the bound doctrine repository changes. +Now the index is automatically updated each time the state of the bound Doctrine repository changes. No need to repopulate the whole "user" index when a new `User` is created. You can also choose to only listen for some of the events: - doctrine: + persistence: listener: insert: true update: false delete: true +> **Propel** doesn't support this feature yet. + ### Advanced elasticsearch configuration Any setting can be specified when declaring a type. For example, to enable a custom analyzer, you could write: diff --git a/Resources/config/propel.xml b/Resources/config/propel.xml new file mode 100644 index 0000000..c489c97 --- /dev/null +++ b/Resources/config/propel.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/Tests/Persister/ObjectPersisterTest.php b/Tests/Persister/ObjectPersisterTest.php index f7b39b1..c449a9f 100644 --- a/Tests/Persister/ObjectPersisterTest.php +++ b/Tests/Persister/ObjectPersisterTest.php @@ -24,7 +24,7 @@ class InvalidObjectPersister extends ObjectPersister { protected function transformToElasticaDocument($object) { - throw new \Exception('Invalid transformation'); + throw new \BadMethodCallException('Invalid transformation'); } } @@ -57,7 +57,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase } /** - * @expectedException Exception + * @expectedException \BadMethodCallException */ public function testThatErrorIsHandledWhenCannotReplaceObject() { @@ -96,7 +96,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase } /** - * @expectedException Exception + * @expectedException \BadMethodCallException */ public function testThatErrorIsHandledWhenCannotInsertObject() { @@ -135,7 +135,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase } /** - * @expectedException Exception + * @expectedException \BadMethodCallException */ public function testThatErrorIsHandledWhenCannotDeleteObject() { @@ -176,7 +176,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase } /** - * @expectedException Exception + * @expectedException \BadMethodCallException */ public function testThatErrorIsHandledWhenCannotInsertManyObject() {