From e2e21b1e0ceb9745404f459b8a1b9a393233b100 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 15 Dec 2013 18:33:31 +1100 Subject: [PATCH 001/154] Allow bundle to be used without clients or indexes defined. --- DependencyInjection/FOSElasticaExtension.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 8b85629..cb2f706 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -24,12 +24,14 @@ class FOSElasticaExtension extends Extension $config = $this->processConfiguration($configuration, $configs); $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); - $loader->load('config.xml'); if (empty($config['clients']) || empty($config['indexes'])) { - throw new InvalidArgumentException('You must define at least one client and one index'); + // No Clients or indexes are defined + return; } + $loader->load('config.xml'); + if (empty($config['default_client'])) { $keys = array_keys($config['clients']); $config['default_client'] = reset($keys); From 90022b0d0a540e8bf5fc44cde6c8b996c072aefd Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 15 Dec 2013 18:33:45 +1100 Subject: [PATCH 002/154] Move type configuration into independent method --- DependencyInjection/Configuration.php | 228 +++++++++++--------------- 1 file changed, 95 insertions(+), 133 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e9088e8..11c1cf4 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -138,68 +138,9 @@ class Configuration implements ConfigurationInterface ->scalarNode('client')->end() ->scalarNode('finder') ->treatNullLike(true) - ->defaultFalse() - ->end() - ->arrayNode('type_prototype') - ->children() - ->scalarNode('index_analyzer')->end() - ->scalarNode('search_analyzer')->end() - ->arrayNode('persistence') - ->validate() - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) - ->thenInvalid('Propel doesn\'t support listeners') - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) - ->thenInvalid('Propel doesn\'t support the "repository" parameter') - ->end() - ->children() - ->scalarNode('driver') - ->validate() - ->ifNotInArray($this->supportedDrivers) - ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) - ->end() - ->end() - ->scalarNode('identifier')->defaultValue('id')->end() - ->arrayNode('provider') - ->children() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('batch_size')->defaultValue(100)->end() - ->scalarNode('clear_object_manager')->defaultTrue()->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('listener') - ->children() - ->scalarNode('insert')->defaultTrue()->end() - ->scalarNode('update')->defaultTrue()->end() - ->scalarNode('delete')->defaultTrue()->end() - ->scalarNode('service')->end() - ->variableNode('is_indexable_callback')->defaultNull()->end() - ->end() - ->end() - ->arrayNode('finder') - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('elastica_to_model_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('hydrate')->defaultTrue()->end() - ->scalarNode('ignore_missing')->defaultFalse()->end() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('model_to_elastica_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->end() - ->end() - ->end() + ->defaultTrue() ->end() + ->append($this->getTypePrototypeNode()) ->variableNode('settings')->defaultValue(array())->end() ->end() ->append($this->getTypesNode()) @@ -209,6 +150,19 @@ class Configuration implements ConfigurationInterface ; } + /** + * Builds a type prototype node for the index configuration. + */ + protected function getTypePrototypeNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('type_prototype'); + + $this->applyTypeConfiguration($node); + + return $node; + } + /** * Returns the array node used for "types". */ @@ -217,92 +171,100 @@ class Configuration implements ConfigurationInterface $builder = new TreeBuilder(); $node = $builder->root('types'); - $node + $childrenNode = $node ->useAttributeAsKey('name') ->prototype('array') - ->treatNullLike(array()) - ->children() - ->arrayNode('serializer') - ->addDefaultsIfNotSet() - ->children() - ->arrayNode('groups') - ->treatNullLike(array()) - ->prototype('scalar')->end() - ->end() - ->scalarNode('version')->end() + ->treatNullLike(array()); + + $this->applyTypeConfiguration($childrenNode); + + return $node; + } + + /** + * Applies all type configuration fields to a node. + */ + protected function applyTypeConfiguration($node) + { + $node + ->children() + ->arrayNode('serializer') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('groups') + ->treatNullLike(array()) + ->prototype('scalar')->end() ->end() + ->scalarNode('version')->end() ->end() - ->scalarNode('index_analyzer')->end() - ->scalarNode('search_analyzer')->end() - ->arrayNode('persistence') - ->validate() - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) - ->thenInvalid('Propel doesn\'t support listeners') - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) - ->thenInvalid('Propel doesn\'t support the "repository" parameter') + ->end() + ->scalarNode('index_analyzer')->end() + ->scalarNode('search_analyzer')->end() + ->arrayNode('persistence') + ->validate() + ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) + ->thenInvalid('Propel doesn\'t support listeners') + ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) + ->thenInvalid('Propel doesn\'t support the "repository" parameter') + ->end() + ->children() + ->scalarNode('driver') + ->validate() + ->ifNotInArray($this->supportedDrivers) + ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) + ->end() ->end() - ->children() - ->scalarNode('driver') - ->validate() - ->ifNotInArray($this->supportedDrivers) - ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) - ->end() + ->scalarNode('model')->end() + ->scalarNode('repository')->end() + ->scalarNode('identifier')->defaultValue('id')->end() + ->arrayNode('provider') + ->children() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() + ->scalarNode('batch_size')->defaultValue(100)->end() + ->scalarNode('clear_object_manager')->defaultTrue()->end() + ->scalarNode('service')->end() ->end() - ->scalarNode('model')->end() - ->scalarNode('repository')->end() - ->scalarNode('identifier')->defaultValue('id')->end() - ->arrayNode('provider') - ->children() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('batch_size')->defaultValue(100)->end() - ->scalarNode('clear_object_manager')->defaultTrue()->end() - ->scalarNode('service')->end() - ->end() + ->end() + ->arrayNode('listener') + ->children() + ->scalarNode('insert')->defaultTrue()->end() + ->scalarNode('update')->defaultTrue()->end() + ->scalarNode('delete')->defaultTrue()->end() + ->scalarNode('service')->end() + ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() - ->arrayNode('listener') - ->children() - ->scalarNode('insert')->defaultTrue()->end() - ->scalarNode('update')->defaultTrue()->end() - ->scalarNode('delete')->defaultTrue()->end() - ->scalarNode('service')->end() - ->variableNode('is_indexable_callback')->defaultNull()->end() - ->end() + ->end() + ->arrayNode('finder') + ->children() + ->scalarNode('service')->end() ->end() - ->arrayNode('finder') - ->children() - ->scalarNode('service')->end() - ->end() + ->end() + ->arrayNode('elastica_to_model_transformer') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('hydrate')->defaultTrue()->end() + ->scalarNode('ignore_missing')->defaultFalse()->end() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() + ->scalarNode('service')->end() ->end() - ->arrayNode('elastica_to_model_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('hydrate')->defaultTrue()->end() - ->scalarNode('ignore_missing')->defaultFalse()->end() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('model_to_elastica_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('service')->end() - ->end() + ->end() + ->arrayNode('model_to_elastica_transformer') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('service')->end() ->end() ->end() ->end() ->end() - ->append($this->getIdNode()) - ->append($this->getMappingsNode()) - ->append($this->getDynamicTemplateNode()) - ->append($this->getSourceNode()) - ->append($this->getBoostNode()) - ->append($this->getRoutingNode()) - ->append($this->getParentNode()) - ->append($this->getAllNode()) ->end() - ; - - return $node; + ->append($this->getIdNode()) + ->append($this->getMappingsNode()) + ->append($this->getDynamicTemplateNode()) + ->append($this->getSourceNode()) + ->append($this->getBoostNode()) + ->append($this->getRoutingNode()) + ->append($this->getParentNode()) + ->append($this->getAllNode()); } /** From 5f8b8003d1f10173094c8dec58db8fb486432a66 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Dec 2013 21:09:58 +1100 Subject: [PATCH 003/154] Refactor documentation --- DependencyInjection/Configuration.php | 22 +- README.md | 868 +----------------- Resources/doc/cookbook/custom-repositories.md | 72 ++ Resources/doc/cookbook/manual-provider.md | 56 ++ .../doc/cookbook/suppress-server-errors.md | 36 + Resources/doc/index.md | 14 + Resources/doc/serializer.md | 39 + Resources/doc/setup.md | 144 +++ Resources/doc/types.md | 264 ++++++ Resources/doc/usage.md | 177 ++++ 10 files changed, 838 insertions(+), 854 deletions(-) create mode 100644 Resources/doc/cookbook/custom-repositories.md create mode 100644 Resources/doc/cookbook/manual-provider.md create mode 100644 Resources/doc/cookbook/suppress-server-errors.md create mode 100644 Resources/doc/index.md create mode 100644 Resources/doc/serializer.md create mode 100644 Resources/doc/setup.md create mode 100644 Resources/doc/types.md create mode 100644 Resources/doc/usage.md diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 11c1cf4..b05bd54 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -8,6 +8,11 @@ use Symfony\Component\Config\Definition\ConfigurationInterface; class Configuration implements ConfigurationInterface { + /** + * Stores supported database drivers. + * + * @var array + */ private $supportedDrivers = array('orm', 'mongodb', 'propel'); private $configArray = array(); @@ -32,8 +37,12 @@ class Configuration implements ConfigurationInterface $rootNode ->children() - ->scalarNode('default_client')->end() - ->scalarNode('default_index')->end() + ->scalarNode('default_client') + ->info('Defaults to the first client defined') + ->end() + ->scalarNode('default_index') + ->info('Defaults to the first index defined') + ->end() ->scalarNode('default_manager')->defaultValue('orm')->end() ->arrayNode('serializer') ->treatNullLike(array()) @@ -105,6 +114,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('host')->end() ->scalarNode('port')->end() ->scalarNode('logger') + ->info('Set your own logger service for this client or disable the logger with false.') ->defaultValue('fos_elastica.logger') ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') @@ -134,11 +144,14 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->performNoDeepMerging() ->children() - ->scalarNode('index_name')->end() + ->scalarNode('index_name') + ->info('Defaults to the name of the index, but can be modified if the index name is different in ElasticSearch') + ->end() ->scalarNode('client')->end() ->scalarNode('finder') + ->info('Defines an index wide finder that will search all types.') ->treatNullLike(true) - ->defaultTrue() + ->defaultFalse() ->end() ->append($this->getTypePrototypeNode()) ->variableNode('settings')->defaultValue(array())->end() @@ -157,6 +170,7 @@ class Configuration implements ConfigurationInterface { $builder = new TreeBuilder(); $node = $builder->root('type_prototype'); + $node->info('Allows a prototype type definition that can be applied to all types.'); $this->applyTypeConfiguration($node); diff --git a/README.md b/README.md index 3006bd1..ce5afab 100644 --- a/README.md +++ b/README.md @@ -1,862 +1,30 @@ -[Elastica](https://github.com/ruflin/Elastica) integration in Symfony2 +FOSElasticaBundle +================= -### Installation +This bundle provides integration with [ElasticSearch](http://www.elasticsearch.org) and [Elastica](https://github.com/ruflin/Elastica) with +Symfony2. Features include: -#### Bundle and Dependencies +- Features... -For Symfony 2.0.x projects, you must use a 1.x release of this bundle. Please -check the bundle -[tags](https://github.com/FriendsOfSymfony/FOSElasticaBundle/tags) or the -[Packagist](https://packagist.org/packages/friendsofsymfony/elastica-bundle) -page for information on Symfony and Elastica compatibility. +[![Build Status](https://secure.travis-ci.org/FriendsOfSymfony/FOSElasticaBundle.png?branch=master)](http://travis-ci.org/FriendsOfSymfony/FOSElasticaBundle) [![Total Downloads](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/downloads.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Stable Version](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/v/stable.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) -Add FOSElasticaBundle to your application's `composer.json` file: +Documentation +------------- -```json -{ - "require": { - "friendsofsymfony/elastica-bundle": "3.0.*@dev" - } -} -``` +Documentation for FOSElasticaBundle is in `Resources/doc/index.md` -Install the bundle and its dependencies with the following command: +[Read the documentation for 3.0.x (master))](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/master/Resources/doc/index.md) -```bash -$ php composer.phar update friendsofsymfony/elastica-bundle -``` +[Read the documentation for 2.1.x](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/2.1.x/README.md) -You may rely on Composer to fetch the appropriate version of Elastica. Lastly, -enable the bundle in your application kernel: +Installation +------------ -```php -// app/AppKernel.php +Installation instructions can be found in the [documentation](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/master/Resources/doc/setup.md) -public function registerBundles() -{ - $bundles = array( - // ... - new FOS\ElasticaBundle\FOSElasticaBundle(), - ); -} -``` +License +------- -#### Elasticsearch +This bundle is under the MIT license. See the complete license in the bundle: -Instructions for installing and deploying Elasticsearch may be found -[here](http://www.elasticsearch.org/guide/reference/setup/installation/). - -### Basic configuration - -#### Declare a client - -Elasticsearch client is comparable to a database connection. -Most of the time, you will need only one. - - #app/config/config.yml - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - - -#### Declare a serializer - -Elastica can handle objects instead of data arrays if a serializer callable is configured - - #app/config/config.yml - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: callback_class - serializer: serializer - -``callback_class`` is the name of a class having a public method serialize($object) and should -extends from ``FOS\ElasticaBundle\Serializer\Callback``. - -``serializer`` is the service id for the actual serializer, e.g. ``serializer`` if you're using -JMSSerializerBundle. If this is configured you can use ``\Elastica\Type::addObject`` instead of -``\Elastica\Type::addDocument`` to add data to the index. The bundle provides a default implementation -with a serializer service id 'serializer' that can be turned on by adding the following line to your config. - - #app/config/config.yml - fos_elastica: - serializer: ~ - -#### Declare an index - -Elasticsearch index is comparable to Doctrine entity manager. -Most of the time, you will need only one. - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - -Here we created a "website" index, that uses our "default" client. - -Our index is now available as a service: `fos_elastica.index.website`. It is an instance of `\Elastica\Index`. - -If you need to have different index name from the service name, for example, -in order to have different indexes for different environments then you can -use the ```index_name``` key to change the index name. The service name will -remain the same across the environments: - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - indexes: - website: - client: default - index_name: website_qa - -The service id will be `fos_elastica.index.website` but the underlying index name is website_qa. - -#### Declare a type - -Elasticsearch type is comparable to Doctrine entity repository. - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - types: - user: - mappings: - username: { boost: 5 } - firstName: { boost: 3 } - lastName: { boost: 3 } - aboutMe: ~ - -Our type is now available as a service: `fos_elastica.index.website.user`. It is an instance of `\Elastica\Type`. - -### Declaring serializer groups - -If you are using the JMSSerializerBundle for serializing objects passed to elastica you can define serializer groups -per type. - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: %classname% - serializer: serializer - indexes: - website: - client: default - types: - user: - mappings: - username: { boost: 5 } - firstName: { boost: 3 } - lastName: { boost: 3 } - aboutMe: - serializer: - groups: [elastica, Default] - -### Declaring parent field - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - types: - comment: - mappings: - date: { boost: 5 } - content: ~ - _parent: { type: "post", property: "post", identifier: "id" } - -The parent field declaration has the following values: - - * `type`: The parent type. - * `property`: The property in the child entity where to look for the parent entity. It may be ignored if is equal to the parent type. - * `identifier`: The property in the parent entity which has the parent identifier. Defaults to `id`. - -Note that to create a document with a parent, you need to call `setParent` on the document rather than setting a _parent field. -If you do this wrong, you will see a `RoutingMissingException` as elasticsearch does not know where to store a document that should have a parent but does not specify it. - -### Declaring `nested` or `object` - -Note that object can autodetect properties - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - types: - post: - mappings: - date: { boost: 5 } - title: { boost: 3 } - content: ~ - comments: - type: "nested" - properties: - date: { boost: 5 } - content: ~ - user: - type: "object" - approver: - type: "object" - properties: - date: { boost: 5 } - -#### Doctrine ORM and `object` mappings - -Objects operate in the same way as the nested results but they need to have associations set up in Doctrine ORM so that they can be referenced correctly when indexing. - -If an "Entity was not found" error occurs while indexing, a null association has been discovered in the database. A custom Doctrine query must be used to utilize left joins instead of the default inner join. - -### Populate the types - - php app/console fos:elastica:populate - -This command deletes and creates the declared indexes and types. -It applies the configured mappings to the types. - -This command needs providers to insert new documents in the elasticsearch types. -There are 2 ways to create providers. -If your elasticsearch type matches a Doctrine repository or a Propel query, go for the persistence automatic provider. -Or, for complete flexibility, go for a manual provider. - -#### Persistence automatic provider - -If we want to index the entities from a Doctrine repository or a Propel query, -some configuration will let ElasticaBundle do it for us. - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - types: - user: - mappings: - username: { boost: 5 } - firstName: { boost: 3 } - # more mappings... - persistence: - driver: orm # orm, mongodb, propel are available - model: Application\UserBundle\Entity\User - provider: ~ - -Three drivers are actually supported: orm, mongodb, and propel. - -##### Use a custom Doctrine query builder - -You can control which entities will be indexed by specifying a custom query builder method. - - persistence: - driver: orm - model: Application\UserBundle\Entity\User - provider: - query_builder_method: createIsActiveQueryBuilder - -Your repository must implement this method and return a Doctrine query builder. - -> **Propel** doesn't support this feature yet. - -##### Change the batch size - -By default, ElasticaBundle will index documents by packets of 100. -You can change this value in the provider configuration. - - persistence: - driver: orm - model: Application\UserBundle\Entity\User - provider: - batch_size: 100 - -##### Change the document identifier field - -By default, ElasticaBundle will use the `id` field of your entities as the elasticsearch document identifier. -You can change this value in the persistence configuration. - - persistence: - driver: orm - model: Application\UserBundle\Entity\User - identifier: id - -#### Manual provider - -Create a service with the tag "fos_elastica.provider" and attributes for the -index and type for which the service will provide. - - - - - - -Its class must implement `FOS\ElasticaBundle\Provider\ProviderInterface`. - - userType = $userType; - } - - /** - * Insert the repository objects in the type index - * - * @param \Closure $loggerClosure - * @param array $options - */ - public function populate(\Closure $loggerClosure = null, array $options = array()) - { - if ($loggerClosure) { - $loggerClosure('Indexing users'); - } - - $document = new Document(); - $document->setData(array('username' => 'Bob')); - $this->userType->addDocuments(array($document)); - } - } - -You will find a more complete implementation example in `src/FOS/ElasticaBundle/Doctrine/AbstractProvider.php`. - -### Search - -You can just use the index and type Elastica objects, provided as services, to perform searches. - - /** var Elastica\Type */ - $userType = $this->container->get('fos_elastica.index.website.user'); - - /** var Elastica\ResultSet */ - $resultSet = $userType->search('bob'); - -#### Doctrine/Propel finder - -If your elasticsearch type is bound to a Doctrine entity repository or a Propel query, -you can get your entities instead of Elastica results when you perform a search. -Declare that you want a Doctrine/Propel finder in your configuration: - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - serializer: - callback_class: FOS\ElasticaBundle\Serializer\Callback - serializer: serializer - indexes: - website: - client: default - types: - user: - mappings: - # your mappings - persistence: - driver: orm - model: Application\UserBundle\Entity\User - provider: ~ - finder: ~ - -You can now use the `fos_elastica.finder.website.user` service: - - /** var FOS\ElasticaBundle\Finder\TransformedFinder */ - $finder = $container->get('fos_elastica.finder.website.user'); - - /** var array of Acme\UserBundle\Entity\User */ - $users = $finder->find('bob'); - - /** var array of Acme\UserBundle\Entity\User limited to 10 results */ - $users = $finder->find('bob', 10); - -You can even get paginated results! - -Pagerfanta: - - /** var Pagerfanta\Pagerfanta */ - $userPaginator = $finder->findPaginated('bob'); - - /** Number of results to be used for paging the results */ - $countOfResults = $userPaginator->getNbResults(); - -Knp paginator: - - $paginator = $this->get('knp_paginator'); - $userPaginator = $paginator->paginate($finder->createPaginatorAdapter('bob')); - -You can also get both the Elastica results and the entities together from the finder. -You can then access the score, highlights etc. from the Elastica\Result whilst -still also getting the entity. - - /** var array of FOS\ElasticaBundle\HybridResult */ - $hybridResults = $finder->findHybrid('bob'); - foreach ($hybridResults as $hybridResult) { - - /** var Acme\UserBundle\Entity\User */ - $user = $hybridResult->getTransformed(); - - /** var Elastica\Result */ - $result = $hybridResult->getResult(); - } - -If you would like to access facets while using Pagerfanta they can be accessed through -the Adapter seen in the example below. - -```php -$query = new \Elastica\Query(); -$facet = new \Elastica\Facet\Terms('tags'); -$facet->setField('companyGroup'); -$query->addFacet($facet); - -$companies = $finder->findPaginated($query); -$companies->setMaxPerPage($params['limit']); -$companies->setCurrentPage($params['page']); - -$facets = $companies->getAdapter()->getFacets()); -``` - -##### Index wide finder - -You can also define a finder that will work on the entire index. Adjust your index -configuration as per below: - - fos_elastica: - indexes: - website: - client: default - finder: ~ - -You can now use the index wide finder service `fos_elastica.finder.website`: - - /** var FOS\ElasticaBundle\Finder\MappedFinder */ - $finder = $container->get('fos_elastica.finder.website'); - - // Returns a mixed array of any objects mapped - $results = $finder->find('bob'); - -#### Repositories - -As well as using the finder service for a particular Doctrine/Propel entity you -can use a manager service for each driver and get a repository for an entity to search -against. This allows you to use the same service rather than the particular finder. For -example: - - /** var FOS\ElasticaBundle\Manager\RepositoryManager */ - $repositoryManager = $container->get('fos_elastica.manager.orm'); - - /** var FOS\ElasticaBundle\Repository */ - $repository = $repositoryManager->getRepository('UserBundle:User'); - - /** var array of Acme\UserBundle\Entity\User */ - $users = $repository->find('bob'); - -You can also specify the full name of the entity instead of the shortcut syntax: - - /** var FOS\ElasticaBundle\Repository */ - $repository = $repositoryManager->getRepository('Application\UserBundle\Entity\User'); - -> The **2.0** branch doesn't support using `UserBundle:User` style syntax and you must use the full name of the entity. . - -##### Default Manager - -If you are only using one driver then its manager service is automatically aliased -to `fos_elastica.manager`. So the above example could be simplified to: - - /** var FOS\ElasticaBundle\Manager\RepositoryManager */ - $repositoryManager = $container->get('fos_elastica.manager'); - - /** var FOS\ElasticaBundle\Repository */ - $repository = $repositoryManager->getRepository('UserBundle:User'); - - /** var array of Acme\UserBundle\Entity\User */ - $users = $repository->find('bob'); - -If you use multiple drivers then you can choose which one is aliased to `fos_elastica.manager` -using the `default_manager` parameter: - - fos_elastica: - default_manager: mongodb #defaults to orm - clients: - default: { host: localhost, port: 9200 } - #-- - -##### Custom Repositories - -As well as the default repository you can create a custom repository for an entity and add -methods for particular searches. These need to extend `FOS\ElasticaBundle\Repository` to have -access to the finder: - -``` -find($query); - } -} -``` - -To use the custom repository specify it in the mapping for the entity: - - fos_elastica: - clients: - default: { host: localhost, port: 9200 } - indexes: - website: - client: default - types: - user: - mappings: - # your mappings - persistence: - driver: orm - model: Application\UserBundle\Entity\User - provider: ~ - finder: ~ - repository: Acme\ElasticaBundle\SearchRepository\UserRepository - -Then the custom queries will be available when using the repository returned from the manager: - - /** var FOS\ElasticaBundle\Manager\RepositoryManager */ - $repositoryManager = $container->get('fos_elastica.manager'); - - /** var FOS\ElasticaBundle\Repository */ - $repository = $repositoryManager->getRepository('UserBundle:User'); - - /** var array of Acme\UserBundle\Entity\User */ - $users = $repository->findWithCustomQuery('bob'); - -Alternatively you can specify the custom repository using an annotation in the entity: - -``` - **Propel** doesn't support this feature yet. - -### Checking an entity method for listener - -If you use listeners to update your index, you may need to validate your -entities before you index them (e.g. only index "public" entities). Typically, -you'll want the listener to be consistent with the provider's query criteria. -This may be achieved by using the `is_indexable_callback` config parameter: - - persistence: - listener: - is_indexable_callback: "isPublic" - -If `is_indexable_callback` is a string and the entity has a method with the -specified name, the listener will only index entities for which the method -returns `true`. Additionally, you may provide a service and method name pair: - - persistence: - listener: - is_indexable_callback: [ "%custom_service_id%", "isIndexable" ] - -In this case, the callback_class will be the `isIndexable()` method on the specified -service and the object being considered for indexing will be passed as the only -argument. This allows you to do more complex validation (e.g. ACL checks). - -If you have the [Symfony ExpressionLanguage](https://github.com/symfony/expression-language) component installed, you can use expressions -to evaluate the callback: - - persistence: - listener: - is_indexable_callback: "user.isActive() && user.hasRole('ROLE_USER')" - -As you might expect, new entities will only be indexed if the callback_class returns -`true`. Additionally, modified entities will be updated or removed from the -index depending on whether the callback_class returns `true` or `false`, respectively. -The delete listener disregards the callback_class. - -> **Propel** doesn't support this feature yet. - -### Ignoring missing index results - -By default, FOSElasticaBundle will throw an exception if the results returned from -Elasticsearch are different from the results it finds from the chosen persistence -provider. This may pose problems for a large index where updates do not occur instantly -or another process has removed the results from your persistence provider without -updating Elasticsearch. - -The error you're likely to see is something like: -'Cannot find corresponding Doctrine objects for all Elastica results.' - -To solve this issue, each mapped object can be configured to ignore the missing results: - - persistence: - elastica_to_model_transformer: - ignore_missing: true - -### Advanced elasticsearch configuration - -Any setting can be specified when declaring a type. For example, to enable a custom analyzer, you could write: - - fos_elastica: - indexes: - doc: - settings: - index: - analysis: - analyzer: - my_analyzer: - type: custom - tokenizer: lowercase - filter : [my_ngram] - filter: - my_ngram: - type: "nGram" - min_gram: 3 - max_gram: 5 - types: - blog: - mappings: - title: { boost: 8, analyzer: my_analyzer } - -### Overriding the Client class to suppress exceptions - -By default, exceptions from the Elastica client library will propagate through -the bundle's Client class. For instance, if the elasticsearch server is offline, -issuing a request will result in an `Elastica\Exception\Connection` being thrown. -Depending on your needs, it may be desirable to suppress these exceptions and -allow searches to fail silently. - -One way to achieve this is to override the `fos_elastica.client.class` service -container parameter with a custom class. In the following example, we override -the `Client::request()` method and return the equivalent of an empty search -response if an exception occurred. - -``` -container->get('fos_elastica.finder.website.article'); -$boolQuery = new \Elastica\Query\Bool(); - -$fieldQuery = new \Elastica\Query\Text(); -$fieldQuery->setFieldQuery('title', 'I am a title string'); -$fieldQuery->setFieldParam('title', 'analyzer', 'my_analyzer'); -$boolQuery->addShould($fieldQuery); - -$tagsQuery = new \Elastica\Query\Terms(); -$tagsQuery->setTerms('tags', array('tag1', 'tag2')); -$boolQuery->addShould($tagsQuery); - -$categoryQuery = new \Elastica\Query\Terms(); -$categoryQuery->setTerms('categoryIds', array('1', '2', '3')); -$boolQuery->addMust($categoryQuery); - -$data = $finder->find($boolQuery); -``` - -Configuration: - -```yaml -fos_elastica: - clients: - default: { host: localhost, port: 9200 } - indexes: - site: - settings: - index: - analysis: - analyzer: - my_analyzer: - type: snowball - language: English - types: - article: - mappings: - title: { boost: 10, analyzer: my_analyzer } - tags: - categoryIds: - persistence: - driver: orm - model: Acme\DemoBundle\Entity\Article - provider: - finder: -``` - -### Filtering Results and Executing a Default Query - -If may want to omit certain results from a query, filtering can be more -performant than a basic query because the filter results can be cached. In turn, -the query is run against only a subset of the results. A common use case for -filtering would be if your data has fields that indicate whether records are -"active" or "inactive". The following example illustrates how to issue such a -query with Elastica: - -```php -$query = new \Elastica\Query\QueryString($queryString); -$term = new \Elastica\Filter\Term(array('active' => true)); - -$filteredQuery = new \Elastica\Query\Filtered($query, $term); -$results = $this->container->get('fos_elastica.finder.index.type')->find($filteredQuery); -``` - -### Date format example - -If you want to specify a [date format](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-date-format.html): - -```yaml -fos_elastica: - clients: - default: { host: localhost, port: 9200 } - indexes: - site: - types: - user: - mappings: - username: { type: string } - lastlogin: { type: date, format: basic_date_time } - birthday: { type: date, format: "yyyy-MM-dd" } -``` - -#### Dynamic templates - -Dynamic templates allow to define mapping templates that will be -applied when dynamic introduction of fields / objects happens. - -[Documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-root-object-type.html#_dynamic_templates) - -```yaml -fos_elastica: - clients: - default: { host: localhost, port: 9200 } - indexes: - site: - types: - user: - dynamic_templates: - my_template_1: - match: apples_* - mapping: - type: float - my_template_2: - match: * - match_mapping_type: string - mapping: - type: string - index: not_analyzed - mappings: - username: { type: string } -``` + Resources/meta/LICENSE diff --git a/Resources/doc/cookbook/custom-repositories.md b/Resources/doc/cookbook/custom-repositories.md new file mode 100644 index 0000000..47dc3fe --- /dev/null +++ b/Resources/doc/cookbook/custom-repositories.md @@ -0,0 +1,72 @@ +##### Custom Repositories + +As well as the default repository you can create a custom repository for an entity and add +methods for particular searches. These need to extend `FOS\ElasticaBundle\Repository` to have +access to the finder: + +``` +find($query); + } +} +``` + +To use the custom repository specify it in the mapping for the entity: + + fos_elastica: + clients: + default: { host: localhost, port: 9200 } + indexes: + website: + client: default + types: + user: + mappings: + # your mappings + persistence: + driver: orm + model: Application\UserBundle\Entity\User + provider: ~ + finder: ~ + repository: Acme\ElasticaBundle\SearchRepository\UserRepository + +Then the custom queries will be available when using the repository returned from the manager: + + /** var FOS\ElasticaBundle\Manager\RepositoryManager */ + $repositoryManager = $container->get('fos_elastica.manager'); + + /** var FOS\ElasticaBundle\Repository */ + $repository = $repositoryManager->getRepository('UserBundle:User'); + + /** var array of Acme\UserBundle\Entity\User */ + $users = $repository->findWithCustomQuery('bob'); + +Alternatively you can specify the custom repository using an annotation in the entity: + +``` +userType = $userType; + } + + /** + * Insert the repository objects in the type index + * + * @param \Closure $loggerClosure + * @param array $options + */ + public function populate(\Closure $loggerClosure = null, array $options = array()) + { + if ($loggerClosure) { + $loggerClosure('Indexing users'); + } + + $document = new Document(); + $document->setData(array('username' => 'Bob')); + $this->userType->addDocuments(array($document)); + } +} +``` + +You will find a more complete implementation example in `src/FOS/ElasticaBundle/Doctrine/AbstractProvider.php`. diff --git a/Resources/doc/cookbook/suppress-server-errors.md b/Resources/doc/cookbook/suppress-server-errors.md new file mode 100644 index 0000000..aa74276 --- /dev/null +++ b/Resources/doc/cookbook/suppress-server-errors.md @@ -0,0 +1,36 @@ +Suppressing Server Errors +======================== + +By default, exceptions from the Elastica client library will propagate through +the bundle's Client class. For instance, if the Elasticsearch server is offline, +issuing a request will result in an `Elastica\Exception\Connection` being thrown. +Depending on your needs, it may be desirable to suppress these exceptions and +allow searches to fail silently. + +One way to achieve this is to override the `fos_elastica.client.class` service +container parameter with a custom class. In the following example, we override +the `Client::request()` method and return the equivalent of an empty search +response if an exception occurred. + +``` + **Propel** doesn't support this feature yet. + +### Checking an entity method for listener + +If you use listeners to update your index, you may need to validate your +entities before you index them (e.g. only index "public" entities). Typically, +you'll want the listener to be consistent with the provider's query criteria. +This may be achieved by using the `is_indexable_callback` config parameter: + +```yaml + persistence: + listener: + is_indexable_callback: "isPublic" +``` + +If `is_indexable_callback` is a string and the entity has a method with the +specified name, the listener will only index entities for which the method +returns `true`. Additionally, you may provide a service and method name pair: + +```yaml + persistence: + listener: + is_indexable_callback: [ "%custom_service_id%", "isIndexable" ] +``` + +In this case, the callback_class will be the `isIndexable()` method on the specified +service and the object being considered for indexing will be passed as the only +argument. This allows you to do more complex validation (e.g. ACL checks). + +If you have the [Symfony ExpressionLanguage](https://github.com/symfony/expression-language) +component installed, you can use expressions to evaluate the callback: + +```yaml + persistence: + listener: + is_indexable_callback: "user.isActive() && user.hasRole('ROLE_USER')" +``` + +As you might expect, new entities will only be indexed if the callback_class returns +`true`. Additionally, modified entities will be updated or removed from the +index depending on whether the callback_class returns `true` or `false`, respectively. +The delete listener disregards the callback_class. + +> **Propel** doesn't support this feature yet. \ No newline at end of file diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md new file mode 100644 index 0000000..4c962e0 --- /dev/null +++ b/Resources/doc/usage.md @@ -0,0 +1,177 @@ +FOSElasticaBundle Usage +======================= + +Basic Searching with a Finder +----------------------------- + +The most useful searching method is to use a finder defined by the type configuration. +A finder will return results that have been hydrated by the configured persistence backend, +allowing you to use relationships of returned entities. For more information about +configuration options for this kind of searching, please see the [types](types.md) +documentation. + +```php +$finder = $this->container->get('fos_elastica.finder.search.user'); + +// Option 1. Returns all users who have example.net in any of their mapped fields +$results = $finder->find('example.net'); + +// Option 2. Returns a set of hybrid results that contain all Elasticsearch results +// and their transformed counterparts. Each result is an instance of a HybridResult +$results = $finder->findHybrid('example.net'); + +// Option 3a. Pagerfanta'd resultset +/** var Pagerfanta\Pagerfanta */ +$userPaginator = $finder->findPaginated('bob'); +$countOfResults = $userPaginator->getNbResults(); + +// Option 3b. KnpPaginator resultset + +``` + +Faceted Searching +----------------- + +When searching with facets, the facets can be retrieved when using the paginated +methods on the finder. + +```php +$query = new \Elastica\Query(); +$facet = new \Elastica\Facet\Terms('tags'); +$facet->setField('companyGroup'); +$query->addFacet($facet); + +$companies = $finder->findPaginated($query); +$companies->setMaxPerPage($params['limit']); +$companies->setCurrentPage($params['page']); + +$facets = $companies->getAdapter()->getFacets()); +``` + +Searching the entire index +-------------------------- + +You can also define a finder that will work on the entire index. Adjust your index +configuration as per below: + +```yaml +fos_elastica: + indexes: + website: + finder: ~ +``` + +You can now use the index wide finder service `fos_elastica.finder.website`: + +```php +/** var FOS\ElasticaBundle\Finder\MappedFinder */ +$finder = $container->get('fos_elastica.finder.website'); + +// Returns a mixed array of any objects mapped +$results = $finder->find('bob'); +``` + +Type Repositories +----------------- + +In the case where you need many different methods for different searching terms, it +may be better to separate methods for each type into their own dedicated repository +classes, just like Doctrine ORM's EntityRepository classes. + +The manager class that handles repositories has a service key of `fos_elastica.manager`. +The manager will default to handling ORM entities, and the configuration must be changed +for MongoDB users. + +```yaml +fos_elastica: + default_manager: mongodb +``` + +An example for using a repository: + +```php +/** var FOS\ElasticaBundle\Manager\RepositoryManager */ +$repositoryManager = $container->get('fos_elastica.manager'); + +/** var FOS\ElasticaBundle\Repository */ +$repository = $repositoryManager->getRepository('UserBundle:User'); + +/** var array of Acme\UserBundle\Entity\User */ +$users = $repository->find('bob'); +``` + +For more information about customising repositories, see the cookbook entry +[Custom Repositories](custom-repositories.md). + +Using a custom query builder method for transforming results +------------------------------------------------------------ + +When returning results from Elasticsearch to be transformed by the bundle, the default +`createQueryBuilder` method on each objects Repository class will be called. In many +circumstances this is not ideal and you'd prefer to use a different method to join in +any entity relations that are required on the page that will be displaying the results. + +```yaml + user: + elastica_to_model_transformer: + query_builder_method: createSearchQueryBuilder +``` + +Advanced Searching Example +-------------------------- + +If you would like to perform more advanced queries, here is one example using +the snowball stemming algorithm. + +It searches for Article entities using `title`, `tags`, and `categoryIds`. +Results must match at least one specified `categoryIds`, and should match the +`title` or `tags` criteria. Additionally, we define a snowball analyzer to +apply to queries against the `title` field. + +Assuming a type is configured as follows: + +```yaml +fos_elastica: + indexes: + site: + settings: + index: + analysis: + analyzer: + my_analyzer: + type: snowball + language: English + types: + article: + mappings: + title: { boost: 10, analyzer: my_analyzer } + tags: + categoryIds: + persistence: + driver: orm + model: Acme\DemoBundle\Entity\Article + provider: ~ + finder: ~ +``` + +The following code will execute a search against the Elasticsearch server: + +```php +$finder = $this->container->get('fos_elastica.finder.site.article'); +$boolQuery = new \Elastica\Query\Bool(); + +$fieldQuery = new \Elastica\Query\Text(); +$fieldQuery->setFieldQuery('title', 'I am a title string'); +$fieldQuery->setFieldParam('title', 'analyzer', 'my_analyzer'); +$boolQuery->addShould($fieldQuery); + +$tagsQuery = new \Elastica\Query\Terms(); +$tagsQuery->setTerms('tags', array('tag1', 'tag2')); +$boolQuery->addShould($tagsQuery); + +$categoryQuery = new \Elastica\Query\Terms(); +$categoryQuery->setTerms('categoryIds', array('1', '2', '3')); +$boolQuery->addMust($categoryQuery); + +$data = $finder->find($boolQuery); +``` From 2009f8810911e1999aa8d2966b0c076854258daa Mon Sep 17 00:00:00 2001 From: Peter Mitchell Date: Wed, 19 Mar 2014 08:40:52 -0400 Subject: [PATCH 004/154] Make fetchSlice compatible with custom repo method If a user configures query_builder_method for the doctrine provider, and does not alias the root entity to equal Provider::ENTITY_ALIAS then a DQL error will occur during index population. > [Semantical Error] [...] near 'a.id ASC': Error: 'a' is not defined. Provider::countObjects already handles this by dynamically fetching the alias with QueryBuilder::getRootAliases, and Provider::fetchSlice should be doing the same. nb. "Provider" refers to FOS\ElasticaBundle\Doctrine\ORM\Provider --- Doctrine/ORM/Provider.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doctrine/ORM/Provider.php b/Doctrine/ORM/Provider.php index 3549550..5d0164f 100644 --- a/Doctrine/ORM/Provider.php +++ b/Doctrine/ORM/Provider.php @@ -9,7 +9,7 @@ use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException; class Provider extends AbstractProvider { const ENTITY_ALIAS = 'a'; - + /** * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::countObjects() */ @@ -50,12 +50,13 @@ class Provider extends AbstractProvider */ $orderBy = $queryBuilder->getDQLPart('orderBy'); if (empty($orderBy)) { + $rootAliases = $queryBuilder->getRootAliases(); $identifierFieldNames = $this->managerRegistry ->getManagerForClass($this->objectClass) ->getClassMetadata($this->objectClass) ->getIdentifierFieldNames(); foreach ($identifierFieldNames as $fieldName) { - $queryBuilder->addOrderBy(static::ENTITY_ALIAS.'.'.$fieldName); + $queryBuilder->addOrderBy($rootAliases[0].'.'.$fieldName); } } From 720917f609381921db52622a9796b2c13a84647a Mon Sep 17 00:00:00 2001 From: nurikabe Date: Mon, 24 Mar 2014 11:16:31 -0400 Subject: [PATCH 005/154] Clone entities on delete to preserve ids --- Doctrine/Listener.php | 7 +++++-- Tests/Doctrine/AbstractListenerTest.php | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index c254513..43ac744 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -195,17 +195,20 @@ class Listener implements EventSubscriber $this->scheduledForUpdate[] = $entity; } else { // Delete if no longer indexable - $this->scheduledForDeletion[] = $entity; + $this->scheduledForDeletion[] = clone $entity; } } } + /** + * Delete objects preRemove instead of postRemove so that we have access to the id + */ public function preRemove(EventArgs $eventArgs) { $entity = $eventArgs->getEntity(); if ($entity instanceof $this->objectClass) { - $this->scheduledForDeletion[] = $entity; + $this->scheduledForDeletion[] = clone $entity; } } diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index a9eff66..8e49f24 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -133,7 +133,9 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener = $this->createListener($persister, get_class($entity), array()); $listener->preRemove($eventArgs); - $this->assertEquals($entity, current($listener->scheduledForDeletion)); + $scheduledClone = current($listener->scheduledForDeletion); + $this->assertEquals($entity, $scheduledClone); + $this->assertNotSame($entity, $scheduledClone); $persister->expects($this->once()) ->method('deleteMany') @@ -164,7 +166,9 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener = $this->createListener($persister, get_class($entity), array(), 'identifier'); $listener->preRemove($eventArgs); - $this->assertEquals($entity, current($listener->scheduledForDeletion)); + $scheduledClone = current($listener->scheduledForDeletion); + $this->assertEquals($entity, $scheduledClone); + $this->assertNotSame($entity, $scheduledClone); $persister->expects($this->once()) ->method('deleteMany') From 361d80a720e27ef63bc5bd6a188435a7a6b6102c Mon Sep 17 00:00:00 2001 From: nurikabe Date: Mon, 24 Mar 2014 11:18:55 -0400 Subject: [PATCH 006/154] Prevent division by zero error --- Doctrine/AbstractProvider.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 9d1575c..3fafe50 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -70,7 +70,8 @@ abstract class AbstractProvider extends BaseAbstractProvider $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; - $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); + $timeDifference = microtime(true) - $stepStartTime; + $objectsPerSecond = $timeDifference ? ($stepNbObjects / $timeDifference) : $stepNbObjects; $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); } } From eb7758605e9255b4d8e8a90567c52ae50f15f617 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 26 Mar 2014 10:07:50 +1100 Subject: [PATCH 007/154] Documentation updates --- README.md | 6 +++- .../cookbook/elastica-client-http-headers.md | 18 ++++++++++ Resources/doc/cookbook/logging.md | 36 +++++++++++++++++++ Resources/doc/index.md | 8 +++-- Resources/doc/setup.md | 15 ++++---- Resources/doc/types.md | 16 ++++++++- 6 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 Resources/doc/cookbook/elastica-client-http-headers.md create mode 100644 Resources/doc/cookbook/logging.md diff --git a/README.md b/README.md index ce5afab..8267e57 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,11 @@ FOSElasticaBundle This bundle provides integration with [ElasticSearch](http://www.elasticsearch.org) and [Elastica](https://github.com/ruflin/Elastica) with Symfony2. Features include: -- Features... +- Integrates the Elastica library into a Symfony2 environment +- Automatically generate mappings using a serializer +- Listeners for Doctrine events for automatic indexing + +> **Note** Propel support is limited and contributions fixing issues are welcome! [![Build Status](https://secure.travis-ci.org/FriendsOfSymfony/FOSElasticaBundle.png?branch=master)](http://travis-ci.org/FriendsOfSymfony/FOSElasticaBundle) [![Total Downloads](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/downloads.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Stable Version](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/v/stable.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) diff --git a/Resources/doc/cookbook/elastica-client-http-headers.md b/Resources/doc/cookbook/elastica-client-http-headers.md new file mode 100644 index 0000000..781d639 --- /dev/null +++ b/Resources/doc/cookbook/elastica-client-http-headers.md @@ -0,0 +1,18 @@ +Setting HTTP Headers on the Elastica Client +=========================================== + +It may be necessary to set HTTP headers on the Elastica client, for example an +Authorization header. + +They can be set using the headers configuration key: + +```yaml +# app/config/config.yml +fos_elastica: + clients: + default: + host: example.com + port: 80 + headers: + Authorization: "Basic jdumrGK7rY9TMuQOPng7GZycmxyMHNoir==" +``` diff --git a/Resources/doc/cookbook/logging.md b/Resources/doc/cookbook/logging.md new file mode 100644 index 0000000..b2d2e6c --- /dev/null +++ b/Resources/doc/cookbook/logging.md @@ -0,0 +1,36 @@ +Logging and its performance considerations +========================================== + +By default, FOSElasticaBundle sets a logger against each Elastica client configured and +logs all information sent to and received from Elasticsearch. This can lead to large +memory usage during population or reindexing of an index. + +FOSElasticaBundle provides an option to disable a logger by setting the property on the +client configuration to false: + +```yaml +# app/config/config.yml +fos_elastica: + clients: + default: + host: example.com + logger: false +``` + +It may be desirable to set this configuration property to `%kernel.debug%`, which would +only switch the logging capabilities of FOSElasticaBundle on when debugging is enabled. + +Custom Logger Service +--------------------- + +It is also possible to specify a custom logger instance to be injected into each client by +specifying the service id of the logger you wish to use. + +```yaml +# app/config/config.yml +fos_elastica: + clients: + default: + host: example.com + logger: 'acme.custom.logger' +``` diff --git a/Resources/doc/index.md b/Resources/doc/index.md index d9b6c65..1bc093e 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -1,7 +1,8 @@ FOSElasticaBundle Documentation =============================== -Available documentation for FOSElasticaBundle: +Available documentation for FOSElasticaBundle +--------------------------------------------- * [Setup](setup.md) * [Usage](usage.md) @@ -9,6 +10,9 @@ Available documentation for FOSElasticaBundle: * [Types](types.md) Cookbook Entries +---------------- * [Custom Repositories](cookbook/custom-repositories.md) -* [Suppressing server errors](cookbook/suppress-server-errors.md) \ No newline at end of file +* [HTTP Headers for Elastica](cookbook/elastica-client-http-headers.md) +* Performance - [Logging](cookbook/logging.md) +* [Suppressing server errors](cookbook/suppress-server-errors.md) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 7ae4018..e60c372 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -38,7 +38,7 @@ C) Basic Bundle Configuration ----------------------------- The basic minimal configuration for FOSElasticaBundle is one client with one Elasticsearch -index. In almost all cases, an application will only need a single index. An index can +index. In almost all cases, an application will only need a single index. An index can be considered comparable to a Doctrine Entity Manager, where the index will hold multiple type definitions. @@ -51,7 +51,7 @@ fos_elastica: search: ~ ``` -In this example, an Elastica index (an instance of `Elastica\Index`) is available as a +In this example, an Elastica index (an instance of `Elastica\Index`) is available as a service with the key `fos_elastica.index.search`. If the Elasticsearch index name needs to be different to the service name in your @@ -71,7 +71,7 @@ index of search_dev. D) Defining index types ----------------------- -By default, FOSElasticaBundle requires each type that is to be indexed to be mapped. +By default, FOSElasticaBundle requires each type that is to be indexed to be mapped. It is possible to use a serializer to avoid this requirement. To use a serializer, see the [serializer documentation](serializer.md) @@ -95,12 +95,10 @@ Each defined type is made available as a service, and in this case the service k `fos_elastica.index.search.user` and is an instance of `Elastica\Type`. FOSElasticaBundle requires a provider for each type that will notify when an object -that maps to a type has been modified. The bundle ships with support for Doctrine and +that maps to a type has been modified. The bundle ships with support for Doctrine and Propel objects. -Below is an example for the Doctrine ORM. For additional information regarding -integration with Doctrine or Propel or how to create a custom provider, see -[type-providers.md]. +Below is an example for the Doctrine ORM. ```yaml user: @@ -118,6 +116,7 @@ integration with Doctrine or Propel or how to create a custom provider, see provider: ~ listener: ~ finder: ~ + immediate: ~ ``` There are a significant number of options available for types, that can be @@ -127,7 +126,7 @@ E) Populating the Elasticsearch index ------------------------------------- When using the providers and listeners that come with the bundle, any new or modified -object will be indexed automatically. In some cases, where the database is modified +object will be indexed automatically. In some cases, where the database is modified externally, the Elasticsearch index must be updated manually. This can be achieved by running the console command: diff --git a/Resources/doc/types.md b/Resources/doc/types.md index a66971f..38fa045 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -261,4 +261,18 @@ As you might expect, new entities will only be indexed if the callback_class ret index depending on whether the callback_class returns `true` or `false`, respectively. The delete listener disregards the callback_class. -> **Propel** doesn't support this feature yet. \ No newline at end of file +> **Propel** doesn't support this feature yet. + +Flushing Method +--------------- + +FOSElasticaBundle, since 3.0.0 performs its indexing in the postFlush Doctrine event +instead of prePersist and preUpdate which means that indexing will only occur when there +has been a successful flush. This new default makes more sense but in the instance where +you want to perform indexing before the flush is confirmed you may set the `immediate` +option on a type persistence configuration to false. + +```yaml + persistence: + immediate: false +``` From e25a5420a59f44ae88ea77d94ca91e02f3a29a9c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 26 Mar 2014 12:36:02 +1100 Subject: [PATCH 008/154] Logger enabled with debugging --- DependencyInjection/Configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 5049c7d..348fd74 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -121,7 +121,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('host')->end() ->scalarNode('port')->end() ->scalarNode('logger') - ->defaultValue('fos_elastica.logger') + ->defaultValue('%kernel.debug%') ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() From 9f27c1dade8b6ceb2d04763cdf5afa0eb708a257 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 26 Mar 2014 12:36:36 +1100 Subject: [PATCH 009/154] Update changelog --- CHANGELOG-3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index fd2ecd5..abd05cd 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -14,6 +14,7 @@ To generate a changelog summary since the last version, run * 3.0.0-ALPHA3 (xxxx-xx-xx) + * a9c4c93: Logger is now only enabled in debug mode by default * #463: allowing hot swappable reindexing * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. * 7d13823: Dropped (broken) support for Symfony <2.3 From 38b4074745ad5c1a58882523857b0aa8ffd46ad0 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 26 Mar 2014 12:40:27 +1100 Subject: [PATCH 010/154] Update logger documentation based on configuration change --- Resources/doc/cookbook/logging.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Resources/doc/cookbook/logging.md b/Resources/doc/cookbook/logging.md index b2d2e6c..726e77e 100644 --- a/Resources/doc/cookbook/logging.md +++ b/Resources/doc/cookbook/logging.md @@ -5,8 +5,10 @@ By default, FOSElasticaBundle sets a logger against each Elastica client configu logs all information sent to and received from Elasticsearch. This can lead to large memory usage during population or reindexing of an index. -FOSElasticaBundle provides an option to disable a logger by setting the property on the -client configuration to false: +By default FOSElasticaBundle will only enable a logger when debug mode is enabled, meaning +in a production environment there wont be a logger enabled. To enable a logger anyway, you +can set the logger property of a client configuration to true or a service id of a logging +service you wish to use. ```yaml # app/config/config.yml @@ -14,12 +16,9 @@ fos_elastica: clients: default: host: example.com - logger: false + logger: true ``` -It may be desirable to set this configuration property to `%kernel.debug%`, which would -only switch the logging capabilities of FOSElasticaBundle on when debugging is enabled. - Custom Logger Service --------------------- From a81a6305209df01d43bf32159f6ad4d7334acfea Mon Sep 17 00:00:00 2001 From: Joakim Friberg Date: Thu, 27 Mar 2014 16:33:20 +0100 Subject: [PATCH 011/154] Corrected bundle class name in install instructions --- Resources/doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index e60c372..4ae40af 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -29,7 +29,7 @@ public function registerBundles() { $bundles = array( // ... - new FOS\RestBundle\FOSRestBundle(), + new FOS\ElasticaBundle\FOSElasticaBundle(), ); } ``` From 0de48d2190b03583703c3084a1a8296e4d8cb17b Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 29 Mar 2014 18:36:06 -0400 Subject: [PATCH 012/154] Use identifiers for bulk delete rather than cloning objects --- Doctrine/Listener.php | 41 ++++++++++++++++++++++--- Persister/ObjectPersister.php | 10 ++++++ Persister/ObjectPersisterInterface.php | 7 +++++ Tests/Doctrine/AbstractListenerTest.php | 23 ++++++-------- 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 43ac744..b6217a6 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -9,7 +9,12 @@ use FOS\ElasticaBundle\Persister\ObjectPersister; use Symfony\Component\ExpressionLanguage\Expression; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; use Symfony\Component\ExpressionLanguage\SyntaxError; +use Symfony\Component\PropertyAccess\PropertyAccess; +/** + * Automatically update ElasticSearch based on changes to the Doctrine source + * data. One listener is generated for each Doctrine entity / ElasticSearch type. + */ class Listener implements EventSubscriber { /** @@ -48,10 +53,14 @@ class Listener implements EventSubscriber protected $isIndexableCallback; /** - * Objects scheduled for insertion, replacement, or removal + * Objects scheduled for insertion and replacement */ public $scheduledForInsertion = array(); public $scheduledForUpdate = array(); + + /** + * IDs of objects scheduled for removal + */ public $scheduledForDeletion = array(); /** @@ -61,6 +70,13 @@ class Listener implements EventSubscriber */ protected $expressionLanguage; + /** + * PropertyAccessor instance + * + * @var PropertyAccessorInterface + */ + protected $propertyAccessor; + /** * Constructor. * @@ -75,6 +91,8 @@ class Listener implements EventSubscriber $this->objectClass = $objectClass; $this->events = $events; $this->esIdentifierField = $esIdentifierField; + + $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); } /** @@ -195,20 +213,21 @@ class Listener implements EventSubscriber $this->scheduledForUpdate[] = $entity; } else { // Delete if no longer indexable - $this->scheduledForDeletion[] = clone $entity; + $this->scheduleForDeletion($entity); } } } /** - * Delete objects preRemove instead of postRemove so that we have access to the id + * Delete objects preRemove instead of postRemove so that we have access to the id. Because this is called + * preRemove, first check that the entity is managed by Doctrine */ public function preRemove(EventArgs $eventArgs) { $entity = $eventArgs->getEntity(); if ($entity instanceof $this->objectClass) { - $this->scheduledForDeletion[] = clone $entity; + $this->scheduleForDeletion($entity); } } @@ -224,7 +243,7 @@ class Listener implements EventSubscriber $this->objectPersister->replaceMany($this->scheduledForUpdate); } if (count($this->scheduledForDeletion)) { - $this->objectPersister->deleteMany($this->scheduledForDeletion); + $this->objectPersister->deleteManyByIdentifiers($this->scheduledForDeletion); } } @@ -245,4 +264,16 @@ class Listener implements EventSubscriber { $this->persistScheduled(); } + + /** + * Record the specified identifier to delete. Do not need to entire object. + * @param mixed $object + * @return mixed + */ + protected function scheduleForDeletion($object) + { + if ($identifierValue = $this->propertyAccessor->getValue($object, $this->esIdentifierField)) { + $this->scheduledForDeletion[] = $identifierValue; + } + } } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 3592a78..64cf5db 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -126,6 +126,16 @@ class ObjectPersister implements ObjectPersisterInterface $this->type->deleteDocuments($documents); } + /** + * Bulk deletes records from an array of identifiers + * + * @param array $identifiers array of domain model object identifiers + */ + public function deleteManyByIdentifiers(array $identifiers) + { + $this->type->getIndex()->getClient()->deleteIds($identifiers, $this->type->getIndex(), $this->type); + } + /** * Transforms an object to an elastica document * diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index a25aafc..2b4c8ee 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -61,4 +61,11 @@ interface ObjectPersisterInterface * @param array $objects array of domain model objects */ function deleteMany(array $objects); + + /** + * Bulk deletes records from an array of identifiers + * + * @param array $identifiers array of domain model object identifiers + */ + public function deleteManyByIdentifiers(array $identifiers); } diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index 8e49f24..ee657f1 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -100,13 +100,13 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener->postUpdate($eventArgs); $this->assertEmpty($listener->scheduledForUpdate); - $this->assertEquals($entity, current($listener->scheduledForDeletion)); + $this->assertEquals($entity->getId(), current($listener->scheduledForDeletion)); $persister->expects($this->never()) ->method('replaceOne'); $persister->expects($this->once()) - ->method('deleteMany') - ->with(array($entity)); + ->method('deleteManyByIdentifiers') + ->with(array($entity->getId())); $listener->postFlush($eventArgs); } @@ -133,13 +133,11 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener = $this->createListener($persister, get_class($entity), array()); $listener->preRemove($eventArgs); - $scheduledClone = current($listener->scheduledForDeletion); - $this->assertEquals($entity, $scheduledClone); - $this->assertNotSame($entity, $scheduledClone); + $this->assertEquals($entity->getId(), current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteMany') - ->with(array($entity)); + ->method('deleteManyByIdentifiers') + ->with(array($entity->getId())); $listener->postFlush($eventArgs); } @@ -151,6 +149,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $persister = $this->getMockPersister(); $entity = new Listener\Entity(1); + $entity->identifier = 'foo'; $eventArgs = $this->createLifecycleEventArgs($entity, $objectManager); $objectManager->expects($this->any()) @@ -166,13 +165,11 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener = $this->createListener($persister, get_class($entity), array(), 'identifier'); $listener->preRemove($eventArgs); - $scheduledClone = current($listener->scheduledForDeletion); - $this->assertEquals($entity, $scheduledClone); - $this->assertNotSame($entity, $scheduledClone); + $this->assertEquals($entity->identifier, current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteMany') - ->with(array($entity)); + ->method('deleteManyByIdentifiers') + ->with(array($entity->identifier)); $listener->postFlush($eventArgs); } From f15ca028596a9226fcff636138533a1a39f39ce5 Mon Sep 17 00:00:00 2001 From: Joseph Bielawski Date: Mon, 31 Mar 2014 11:59:37 +0200 Subject: [PATCH 013/154] Fix documentation for client overwriting. --- .../doc/cookbook/suppress-server-errors.md | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/Resources/doc/cookbook/suppress-server-errors.md b/Resources/doc/cookbook/suppress-server-errors.md index aa74276..d89ffd6 100644 --- a/Resources/doc/cookbook/suppress-server-errors.md +++ b/Resources/doc/cookbook/suppress-server-errors.md @@ -1,5 +1,5 @@ Suppressing Server Errors -======================== +========================= By default, exceptions from the Elastica client library will propagate through the bundle's Client class. For instance, if the Elasticsearch server is offline, @@ -12,25 +12,44 @@ container parameter with a custom class. In the following example, we override the `Client::request()` method and return the equivalent of an empty search response if an exception occurred. -``` +Sample client code: +------------------- + +```php + + + + + Acme\ElasticaBundle\Client + + + From 588c4e2d025bea489547dcc2e0bc068f4d97919e Mon Sep 17 00:00:00 2001 From: Joris van de Sande Date: Mon, 31 Mar 2014 12:52:29 +0200 Subject: [PATCH 014/154] Unset nested "fields" for deeper nested configs too --- DependencyInjection/Configuration.php | 4 ++++ Tests/DependencyInjection/ConfigurationTest.php | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 348fd74..e77919c 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -458,6 +458,10 @@ class Configuration implements ConfigurationInterface ->arrayNode($property) ->useAttributeAsKey('name') ->prototype('array') + ->validate() + ->ifTrue(function($v) { return isset($v['fields']) && empty($v['fields']); }) + ->then(function($v) { unset($v['fields']); return $v; }) + ->end() ->treatNullLike(array()) ->addDefaultsIfNotSet() ->children(); diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 93143ec..5919ea7 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -118,7 +118,19 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase 'autocomplete' => null ) ), - 'content' => null + 'content' => null, + 'children' => array( + 'type' => 'nested', + 'properties' => array( + 'title' => array( + 'type' => 'string', + 'fields' => array( + 'autocomplete' => null + ) + ), + 'content' => null + ) + ) ) ) ) @@ -132,5 +144,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['content']); $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['title']); + $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['content']); + $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['title']); } } From 20810fa4150e1c739ad2c848d81af61b738e76e0 Mon Sep 17 00:00:00 2001 From: baggachipz Date: Mon, 31 Mar 2014 15:16:32 -0400 Subject: [PATCH 015/154] Upsert-type functionality for existing ORM Entities This is an attempt to fix the issue: https://github.com/FriendsOfSymfony/FOSElasticaBundle/issues/526 It will cause a significant slowdown in a large batch, but it appears to be the only way to prevent an exception from bubbling up during a normal use case. --- Persister/ObjectPersister.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 3592a78..aa20561 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -107,7 +107,15 @@ class ObjectPersister implements ObjectPersisterInterface { $documents = array(); foreach ($objects as $object) { - $documents[] = $this->transformToElasticaDocument($object); + $document = $this->transformToElasticaDocument($object); + + try { + $this->type->getDocument($document->getId()); + } catch (NotFoundException $e) { + $this->type->addDocument($document); + } + + $documents[] = $document; } $this->type->updateDocuments($documents); } @@ -136,4 +144,4 @@ class ObjectPersister implements ObjectPersisterInterface { return $this->transformer->transform($object, $this->fields); } -} \ No newline at end of file +} From a89856be504b0d87808b188f6cddcc6b0a427887 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Wed, 2 Apr 2014 07:18:46 -0400 Subject: [PATCH 016/154] Reset / create new index even if the index doesn't exist --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 98834c7..af5fd5d 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -109,7 +109,7 @@ class PopulateCommand extends ContainerAwareCommand */ private function populateIndex(OutputInterface $output, $index, $reset, $options) { - if ($reset && $this->indexManager->getIndex($index)->exists()) { + if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index); } From 1628413e65bcfc4457f8f627de8b836964d1512c Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Wed, 2 Apr 2014 10:06:24 -0500 Subject: [PATCH 017/154] Added support for getting documents in unified Doctrine Listener --- Doctrine/Listener.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index b6217a6..47c3c9a 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -197,7 +197,7 @@ class Listener implements EventSubscriber public function postPersist(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); + $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { $this->scheduledForInsertion[] = $entity; @@ -206,7 +206,7 @@ class Listener implements EventSubscriber public function postUpdate(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); + $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); if ($entity instanceof $this->objectClass) { if ($this->isObjectIndexable($entity)) { @@ -224,7 +224,7 @@ class Listener implements EventSubscriber */ public function preRemove(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); + $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); if ($entity instanceof $this->objectClass) { $this->scheduleForDeletion($entity); From e74acb1e4fd577a4aa51e166e0f2b9e2e9f06db4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 3 Apr 2014 08:00:52 +1100 Subject: [PATCH 018/154] Revert "Avoid index reset error in case of unexistant index" This reverts commit a121a777743525f46c65dc29d983c8eb92720664. --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 98834c7..af5fd5d 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -109,7 +109,7 @@ class PopulateCommand extends ContainerAwareCommand */ private function populateIndex(OutputInterface $output, $index, $reset, $options) { - if ($reset && $this->indexManager->getIndex($index)->exists()) { + if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index); } From 13c2d10e399317af9b9aa24d55b382befb3830cf Mon Sep 17 00:00:00 2001 From: Patrick Zahnd Date: Fri, 13 Sep 2013 15:34:59 +0200 Subject: [PATCH 019/154] Added knp paginator sort functionality to PaginateElasticaQuerySubscriber --- Paginator/RawPaginatorAdapter.php | 10 +++++ Resources/config/config.xml | 3 ++ .../PaginateElasticaQuerySubscriber.php | 43 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index 8bd4ee2..9136bc0 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -126,4 +126,14 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface return $this->facets; } + + /** + * Returns the Query + * + * @return Query the search query + */ + public function getQuery() + { + return $this->query; + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 4419b4a..b9da716 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -86,6 +86,9 @@ + + + diff --git a/Subscriber/PaginateElasticaQuerySubscriber.php b/Subscriber/PaginateElasticaQuerySubscriber.php index cbe508d..fc775c7 100644 --- a/Subscriber/PaginateElasticaQuerySubscriber.php +++ b/Subscriber/PaginateElasticaQuerySubscriber.php @@ -9,9 +9,20 @@ use FOS\ElasticaBundle\Paginator\PartialResultsInterface; class PaginateElasticaQuerySubscriber implements EventSubscriberInterface { + private $container; + + public function setContainer($container) + { + $this->container = $container; + } + public function items(ItemsEvent $event) { if ($event->target instanceof PaginatorAdapterInterface) { + + // Add sort to query + $this->addPagingSort($event); + /** @var $results PartialResultsInterface */ $results = $event->target->getResults($event->getOffset(), $event->getLimit()); @@ -26,6 +37,38 @@ class PaginateElasticaQuerySubscriber implements EventSubscriberInterface } } + /** + * Adds knp paging sort to query + * + * @param ItemsEvent $event + * @return void + */ + protected function addPagingSort(ItemsEvent $event) + { + $request = $this->container->get('request'); + $options = $event->options; + $sortField = $request->get($options['sortFieldParameterName']); + + if (!empty($sortField)) { + // determine sort direction + $dir = 'asc'; + $sortDirection = $request->get($options['sortDirectionParameterName']); + if ('desc' === strtolower($sortDirection)) { + $dir = 'desc'; + } + + // check if the requested sort field is in the sort whitelist + if (isset($options['sortFieldWhitelist']) && !in_array($sortField, $options['sortFieldWhitelist'])) { + throw new \UnexpectedValueException(sprintf('Cannot sort by: [%s] this field is not in whitelist', $sortField)); + } + + // set sort on active query + $event->target->getQuery()->setSort(array( + $sortField => array('order' => $dir), + )); + } + } + public static function getSubscribedEvents() { return array( From 1bc148569baa3b7ad8ac31211e93c629d5fe7cda Mon Sep 17 00:00:00 2001 From: Piotr Antosik Date: Mon, 24 Feb 2014 13:47:26 +0100 Subject: [PATCH 020/154] Added knp paginator sort functionality to PaginateElasticaQuerySubscriber --- Resources/config/config.xml | 6 +++--- .../PaginateElasticaQuerySubscriber.php | 20 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index b9da716..1e49590 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -85,10 +85,10 @@ - - - + + + diff --git a/Subscriber/PaginateElasticaQuerySubscriber.php b/Subscriber/PaginateElasticaQuerySubscriber.php index fc775c7..0b7cfd6 100644 --- a/Subscriber/PaginateElasticaQuerySubscriber.php +++ b/Subscriber/PaginateElasticaQuerySubscriber.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle\Subscriber; +use Symfony\Component\HttpFoundation\Request; use Symfony\Component\EventDispatcher\EventSubscriberInterface; use Knp\Component\Pager\Event\ItemsEvent; use FOS\ElasticaBundle\Paginator\PaginatorAdapterInterface; @@ -9,19 +10,18 @@ use FOS\ElasticaBundle\Paginator\PartialResultsInterface; class PaginateElasticaQuerySubscriber implements EventSubscriberInterface { - private $container; + private $request; - public function setContainer($container) + public function setRequest(Request $request = null) { - $this->container = $container; + $this->request = $request; } public function items(ItemsEvent $event) { if ($event->target instanceof PaginatorAdapterInterface) { - // Add sort to query - $this->addPagingSort($event); + $this->setSorting($event); /** @var $results PartialResultsInterface */ $results = $event->target->getResults($event->getOffset(), $event->getLimit()); @@ -40,19 +40,17 @@ class PaginateElasticaQuerySubscriber implements EventSubscriberInterface /** * Adds knp paging sort to query * - * @param ItemsEvent $event - * @return void + * @param ItemsEvent $event */ - protected function addPagingSort(ItemsEvent $event) + protected function setSorting(ItemsEvent $event) { - $request = $this->container->get('request'); $options = $event->options; - $sortField = $request->get($options['sortFieldParameterName']); + $sortField = $this->request->get($options['sortFieldParameterName']); if (!empty($sortField)) { // determine sort direction $dir = 'asc'; - $sortDirection = $request->get($options['sortDirectionParameterName']); + $sortDirection = $this->request->get($options['sortDirectionParameterName']); if ('desc' === strtolower($sortDirection)) { $dir = 'desc'; } From b6e2583455e9397b45e25167f527351458e6c410 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Fri, 4 Apr 2014 10:47:50 -0500 Subject: [PATCH 021/154] Added getDoctrineObject method to retrieve an entity or document from a LifecycleEventArgs instance in the unified Listener class --- Doctrine/Listener.php | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 47c3c9a..ad5641e 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -179,6 +179,34 @@ class Listener implements EventSubscriber return strtolower($ref->getShortName()); } + /** + * Provides unified method for retrieving a doctrine object from an EventArgs instance + * + * @param EventArgs $eventArgs + * @return object Entity | Document + * @throws \RuntimeException if no valid getter is found. + */ + private function getDoctrineObject(EventArgs $eventArgs) + { + + if (method_exists($eventArgs, 'getObject')) { + + return $eventArgs->getObject(); + + } elseif (method_exists($eventArgs, 'getEntity')) { + + return $eventArgs->getEntity(); + + } elseif (method_exists($eventArgs, 'getDocument')) { + + return $eventArgs->getDocument(); + + } + + throw new \RuntimeException('Unable to retrieve object from EventArgs.'); + + } + /** * @return bool|ExpressionLanguage */ @@ -197,7 +225,7 @@ class Listener implements EventSubscriber public function postPersist(EventArgs $eventArgs) { - $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); + $entity = $this->getDoctrineObject($eventArgs); if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { $this->scheduledForInsertion[] = $entity; @@ -206,7 +234,7 @@ class Listener implements EventSubscriber public function postUpdate(EventArgs $eventArgs) { - $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); + $entity = $this->getDoctrineObject($eventArgs); if ($entity instanceof $this->objectClass) { if ($this->isObjectIndexable($entity)) { @@ -224,7 +252,7 @@ class Listener implements EventSubscriber */ public function preRemove(EventArgs $eventArgs) { - $entity = (method_exists($eventArgs, 'getEntity')) ? $eventArgs->getEntity() : $eventArgs->getDocument(); + $entity = $this->getDoctrineObject($eventArgs); if ($entity instanceof $this->objectClass) { $this->scheduleForDeletion($entity); From 6f444f1ce84cf85f51e801ffd70387211238aeba Mon Sep 17 00:00:00 2001 From: Evan Owens Date: Fri, 4 Apr 2014 15:51:55 -0400 Subject: [PATCH 022/154] Bulk upsert --- Persister/ObjectPersister.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index e1c086d..38541c8 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -99,7 +99,7 @@ class ObjectPersister implements ObjectPersisterInterface } /** - * Bulk updates an array of objects in the type + * Bulk update an array of objects in the type. Create document if it does not already exist. * * @param array $objects array of domain model objects */ @@ -108,13 +108,7 @@ class ObjectPersister implements ObjectPersisterInterface $documents = array(); foreach ($objects as $object) { $document = $this->transformToElasticaDocument($object); - - try { - $this->type->getDocument($document->getId()); - } catch (NotFoundException $e) { - $this->type->addDocument($document); - } - + $document->setDocAsUpsert(true); $documents[] = $document; } $this->type->updateDocuments($documents); From 10f6149f8cff0e1162e2eb3c6c5181176fcd604d Mon Sep 17 00:00:00 2001 From: Evan Owens Date: Sat, 5 Apr 2014 07:04:52 +1000 Subject: [PATCH 023/154] Remove rogue paren --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8267e57..7909d2b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Documentation Documentation for FOSElasticaBundle is in `Resources/doc/index.md` -[Read the documentation for 3.0.x (master))](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/master/Resources/doc/index.md) +[Read the documentation for 3.0.x (master)](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/master/Resources/doc/index.md) [Read the documentation for 2.1.x](https://github.com/FriendsOfSymfony/FOSElasticaBundle/blob/2.1.x/README.md) From 377b2843baf6f80f4c10f6aaa0f4b3a8b7f7c79c Mon Sep 17 00:00:00 2001 From: Evan Owens Date: Sat, 5 Apr 2014 07:12:16 +1000 Subject: [PATCH 024/154] Fix documentation about `immediate` Unless I misimplemented this, "immediate" means persist to ElasticSearch "immediately"; before flushing. Default behavior is false; persist to ElasticSearch postFlush. --- Resources/doc/types.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/doc/types.md b/Resources/doc/types.md index 38fa045..e01b5ae 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -270,9 +270,9 @@ FOSElasticaBundle, since 3.0.0 performs its indexing in the postFlush Doctrine e instead of prePersist and preUpdate which means that indexing will only occur when there has been a successful flush. This new default makes more sense but in the instance where you want to perform indexing before the flush is confirmed you may set the `immediate` -option on a type persistence configuration to false. +option on a type persistence configuration to `true`. ```yaml persistence: - immediate: false + immediate: true ``` From 53332eb057b79ec75b94bf56c6281969cfff203b Mon Sep 17 00:00:00 2001 From: Evan Owens Date: Fri, 4 Apr 2014 18:32:48 -0400 Subject: [PATCH 025/154] Allow for catching/logging persistance errors per listener --- DependencyInjection/Configuration.php | 3 ++ DependencyInjection/FOSElasticaExtension.php | 6 ++- Doctrine/Listener.php | 7 +++- Persister/ObjectPersister.php | 42 +++++++++++++++++--- Resources/doc/types.md | 18 ++++++++- 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e77919c..002c78e 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -289,6 +289,9 @@ class Configuration implements ConfigurationInterface ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() ->booleanNode('immediate')->defaultFalse()->end() + ->scalarNode('logger') + ->treatTrueLike('fos_elastica.logger') + ->end() ->scalarNode('service')->end() ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 1d1540f..7511e2b 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -432,8 +432,12 @@ class FOSElasticaExtension extends Extension $listenerDef = new DefinitionDecorator($abstractListenerId); $listenerDef->replaceArgument(0, new Reference($objectPersisterId)); $listenerDef->replaceArgument(1, $typeConfig['model']); - $listenerDef->replaceArgument(3, $typeConfig['identifier']); $listenerDef->replaceArgument(2, $this->getDoctrineEvents($typeConfig)); + $listenerDef->replaceArgument(3, $typeConfig['identifier']); + if (isset($typeConfig['listener']['logger'])) { + $listenerDef->replaceArgument(4, new Reference($typeConfig['listener']['logger'])); + } + switch ($typeConfig['driver']) { case 'orm': $listenerDef->addTag('doctrine.event_subscriber'); break; case 'mongodb': $listenerDef->addTag('doctrine_mongodb.odm.event_subscriber'); break; diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index b6217a6..ded3998 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle\Doctrine; +use Psr\Log\LoggerInterface; use Doctrine\Common\EventArgs; use Doctrine\Common\EventSubscriber; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; @@ -85,13 +86,17 @@ class Listener implements EventSubscriber * @param array $events * @param string $esIdentifierField */ - public function __construct(ObjectPersisterInterface $objectPersister, $objectClass, array $events, $esIdentifierField = 'id') + public function __construct(ObjectPersisterInterface $objectPersister, $objectClass, array $events, $esIdentifierField = 'id', $logger = null) { $this->objectPersister = $objectPersister; $this->objectClass = $objectClass; $this->events = $events; $this->esIdentifierField = $esIdentifierField; + if ($logger) { + $this->objectPersister->setLogger($logger); + } + $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 38541c8..cea6167 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -2,6 +2,8 @@ namespace FOS\ElasticaBundle\Persister; +use Psr\Log\LoggerInterface; +use Elastica\Exception\BulkException; use Elastica\Exception\NotFoundException; use FOS\ElasticaBundle\Transformer\ModelToElasticaTransformerInterface; use Elastica\Type; @@ -19,6 +21,7 @@ class ObjectPersister implements ObjectPersisterInterface protected $transformer; protected $objectClass; protected $fields; + protected $logger; public function __construct(Type $type, ModelToElasticaTransformerInterface $transformer, $objectClass, array $fields) { @@ -28,6 +31,18 @@ class ObjectPersister implements ObjectPersisterInterface $this->fields = $fields; } + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + private function log(BulkException $e) + { + if ($this->logger) { + $this->logger->error($e); + } + } + /** * Insert one object into the type * The object will be transformed to an elastica document @@ -95,7 +110,11 @@ class ObjectPersister implements ObjectPersisterInterface foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } - $this->type->addDocuments($documents); + try { + $this->type->addDocuments($documents); + } catch (BulkException $e) { + $this->log($e); + } } /** @@ -111,7 +130,12 @@ class ObjectPersister implements ObjectPersisterInterface $document->setDocAsUpsert(true); $documents[] = $document; } - $this->type->updateDocuments($documents); + + try { + $this->type->updateDocuments($documents); + } catch (BulkException $e) { + $this->log($e); + } } /** @@ -125,7 +149,11 @@ class ObjectPersister implements ObjectPersisterInterface foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } - $this->type->deleteDocuments($documents); + try { + $this->type->deleteDocuments($documents); + } catch (BulkException $e) { + $this->log($e); + } } /** @@ -135,7 +163,11 @@ class ObjectPersister implements ObjectPersisterInterface */ public function deleteManyByIdentifiers(array $identifiers) { - $this->type->getIndex()->getClient()->deleteIds($identifiers, $this->type->getIndex(), $this->type); + try { + $this->type->getIndex()->getClient()->deleteIds($identifiers, $this->type->getIndex(), $this->type); + } catch (BulkException $e) { + $this->log($e); + } } /** @@ -148,4 +180,4 @@ class ObjectPersister implements ObjectPersisterInterface { return $this->transformer->transform($object, $this->fields); } -} +} \ No newline at end of file diff --git a/Resources/doc/types.md b/Resources/doc/types.md index 38fa045..7e0fe05 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -274,5 +274,21 @@ option on a type persistence configuration to false. ```yaml persistence: - immediate: false + listener: + is_indexable_callback: "user.isActive() && user.hasRole('ROLE_USER')" ``` + +Logging Errors +-------------- + +By default FOSElasticaBundle will not catch errors thrown by Elastica/ElasticSearch. +Configure a logger per listener if you would rather catch and log these. + +```yaml + persistence: + listener: + logger: true +``` + +Specifying `true` will use the default Elastica logger. Alternatively define your own +logger service id. From a4834716945cf6be391c9a8f24626fc42fe67c9e Mon Sep 17 00:00:00 2001 From: nurikabe Date: Fri, 4 Apr 2014 22:56:47 -0400 Subject: [PATCH 026/154] Set listener loggers to fos_elastica.logger if null --- DependencyInjection/Configuration.php | 1 + Persister/ObjectPersister.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 002c78e..1bdcdcd 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -290,6 +290,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('delete')->defaultTrue()->end() ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('logger') + ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() ->scalarNode('service')->end() diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index cea6167..98165ef 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -180,4 +180,4 @@ class ObjectPersister implements ObjectPersisterInterface { return $this->transformer->transform($object, $this->fields); } -} \ No newline at end of file +} From 998c69bfc38328bfe66e9cd263262dc7a34d7274 Mon Sep 17 00:00:00 2001 From: Hugo Henrique Date: Sun, 6 Apr 2014 15:17:02 -0300 Subject: [PATCH 027/154] Update usage.md This link to documentation custom repositories is broken --- Resources/doc/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index 4c962e0..fd6a878 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -101,7 +101,7 @@ $users = $repository->find('bob'); ``` For more information about customising repositories, see the cookbook entry -[Custom Repositories](custom-repositories.md). +[Custom Repositories](cookbook/custom-repositories.md). Using a custom query builder method for transforming results ------------------------------------------------------------ From 22e5a1d4ab1ca8efb3da6ad803268c68cceba491 Mon Sep 17 00:00:00 2001 From: Joshua Worden Date: Mon, 7 Apr 2014 10:58:11 -0500 Subject: [PATCH 028/154] CS fix --- Doctrine/Listener.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index ad5641e..5d89e63 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -190,17 +190,11 @@ class Listener implements EventSubscriber { if (method_exists($eventArgs, 'getObject')) { - return $eventArgs->getObject(); - } elseif (method_exists($eventArgs, 'getEntity')) { - return $eventArgs->getEntity(); - } elseif (method_exists($eventArgs, 'getDocument')) { - return $eventArgs->getDocument(); - } throw new \RuntimeException('Unable to retrieve object from EventArgs.'); From f07e55417d4fb679b616fa5c9208ac36278c0629 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Mon, 7 Apr 2014 16:16:37 -0400 Subject: [PATCH 029/154] Re-throw exception if no logger defined --- Persister/ObjectPersister.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 98165ef..c279ec7 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -36,11 +36,20 @@ class ObjectPersister implements ObjectPersisterInterface $this->logger = $logger; } + /** + * Log exception if logger defined for persister belonging to the current listener, otherwise re-throw + * + * @param BulkException $e + * @throws BulkException + * @return null + */ private function log(BulkException $e) { - if ($this->logger) { - $this->logger->error($e); + if (! $this->logger) { + throw $e; } + + $this->logger->error($e); } /** From b3fdf7b256bc66f27fef1d6e851411a8755f199c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 10 Apr 2014 13:05:23 +1000 Subject: [PATCH 030/154] Logger for a listener is false by default --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 1bdcdcd..b0a7b59 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -290,6 +290,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('delete')->defaultTrue()->end() ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('logger') + ->defaultFalse() ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() From 1bc085141b5a5e368b09a57245a40883815351e7 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 10 Apr 2014 13:14:03 +1000 Subject: [PATCH 031/154] Fix logger option for listeners --- DependencyInjection/FOSElasticaExtension.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 7511e2b..51cb94f 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -208,7 +208,7 @@ class FOSElasticaExtension extends Extension } $callbackClassImplementedInterfaces = class_implements($this->serializerConfig['callback_class']); // PHP < 5.4 friendly if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) { - $callbackDef->addMethodCall('setContainer', array(new Reference('service_container'))); + $callbackDef->addMethodCall('setContainer', array(new Reference('service_container'))); } $container->setDefinition($callbackId, $callbackDef); @@ -434,7 +434,7 @@ class FOSElasticaExtension extends Extension $listenerDef->replaceArgument(1, $typeConfig['model']); $listenerDef->replaceArgument(2, $this->getDoctrineEvents($typeConfig)); $listenerDef->replaceArgument(3, $typeConfig['identifier']); - if (isset($typeConfig['listener']['logger'])) { + if ($typeConfig['listener']['logger']) { $listenerDef->replaceArgument(4, new Reference($typeConfig['listener']['logger'])); } From 088452cf8895f1ee2776f03abff470d434bac4e3 Mon Sep 17 00:00:00 2001 From: Dylan Schoenmakers Date: Thu, 3 Apr 2014 18:50:03 +0900 Subject: [PATCH 032/154] Added example to usage.md Added example for using a custom query builder method to usage.md. --- Resources/doc/usage.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index 4c962e0..89120e2 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -112,11 +112,35 @@ circumstances this is not ideal and you'd prefer to use a different method to jo any entity relations that are required on the page that will be displaying the results. ```yaml - user: + user: + persistence: elastica_to_model_transformer: query_builder_method: createSearchQueryBuilder ``` +An example for using a custom query builder method: + +```php +class UserRepository extends EntityRepository +{ + /** + * Used by Elastica to transform results to model + * + * @param string $entityAlias + * @return Doctrine\ORM\QueryBuilder + */ + public function createSearchQueryBuilder($entityAlias) + { + $qb = $this->createQueryBuilder($entityAlias); + + $qb->select($entityAlias, 'g') + ->innerJoin($entityAlias.'.groups', 'g'); + + return $qb; + } +} +``` + Advanced Searching Example -------------------------- From 4cfe24ae028e93ad7c5324e7808b577cf766ca5d Mon Sep 17 00:00:00 2001 From: Joshua Worden Date: Thu, 10 Apr 2014 07:38:01 -0500 Subject: [PATCH 033/154] Update CS --- Doctrine/Listener.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 5d89e63..2b6df94 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -188,7 +188,6 @@ class Listener implements EventSubscriber */ private function getDoctrineObject(EventArgs $eventArgs) { - if (method_exists($eventArgs, 'getObject')) { return $eventArgs->getObject(); } elseif (method_exists($eventArgs, 'getEntity')) { @@ -198,7 +197,6 @@ class Listener implements EventSubscriber } throw new \RuntimeException('Unable to retrieve object from EventArgs.'); - } /** From 449c33aea3fbb50966157c442908646a69d17051 Mon Sep 17 00:00:00 2001 From: Bob van de Vijver Date: Sat, 12 Apr 2014 13:30:52 +0200 Subject: [PATCH 034/154] Fixes multiple updates on multiple flush executions --- Doctrine/Listener.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 763ddec..ff9fc60 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -258,17 +258,21 @@ class Listener implements EventSubscriber /** * Persist scheduled objects to ElasticSearch + * After persisting, clear the scheduled queue to prevent multiple data updates when using multiple flush calls */ private function persistScheduled() { if (count($this->scheduledForInsertion)) { $this->objectPersister->insertMany($this->scheduledForInsertion); + $this->scheduledForInsertion = array(); } if (count($this->scheduledForUpdate)) { $this->objectPersister->replaceMany($this->scheduledForUpdate); + $this->scheduledForUpdate = array(); } if (count($this->scheduledForDeletion)) { $this->objectPersister->deleteManyByIdentifiers($this->scheduledForDeletion); + $this->scheduledForDeletion = array(); } } From 165696a5a0372db7634772ca8e1ee9847a4022ea Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 15 Apr 2014 08:43:11 +1000 Subject: [PATCH 035/154] Bump Elastica requirement --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 8f22393..8dd19b6 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.20, <1.1-dev", + "ruflin/elastica": ">=0.90.2.0, <1.1-dev", "psr/log": "~1.0" }, "require-dev":{ From 4e638a0492896223808c36350ac60f4b35549b48 Mon Sep 17 00:00:00 2001 From: Ahmed Mohamed Date: Wed, 16 Apr 2014 05:27:23 +0200 Subject: [PATCH 036/154] Ensure non-empty properties under fields mappings --- Transformer/ModelToElasticaAutoTransformer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Transformer/ModelToElasticaAutoTransformer.php b/Transformer/ModelToElasticaAutoTransformer.php index 3107d0a..9694a3b 100644 --- a/Transformer/ModelToElasticaAutoTransformer.php +++ b/Transformer/ModelToElasticaAutoTransformer.php @@ -71,7 +71,7 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf $value = $this->propertyAccessor->getValue($object, $key); - if (isset($mapping['type']) && in_array($mapping['type'], array('nested', 'object')) && isset($mapping['properties'])) { + if (isset($mapping['type']) && in_array($mapping['type'], array('nested', 'object')) && isset($mapping['properties']) && !empty($mapping['properties'])) { /* $value is a nested document or object. Transform $value into * an array of documents, respective the mapped properties. */ From d33f2e547f8a35277f9835a18e137b82892d9627 Mon Sep 17 00:00:00 2001 From: Piotr Antosik Date: Thu, 17 Apr 2014 10:14:53 +0200 Subject: [PATCH 037/154] Fix immediate doc --- Resources/doc/types.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Resources/doc/types.md b/Resources/doc/types.md index 99c3668..e55df5a 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -274,7 +274,8 @@ option on a type persistence configuration to `true`. ```yaml persistence: - immediate: true + listener: + immediate: true ``` Logging Errors From 2bd6aba7ef0a722e690a550c21445d2faf81f09f Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 18 Apr 2014 13:57:08 +0200 Subject: [PATCH 038/154] Added GeoShape mapping options --- DependencyInjection/Configuration.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b0a7b59..bb67e42 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -423,6 +423,9 @@ class Configuration implements ConfigurationInterface ->booleanNode('include_in_all')->defaultValue(true)->end() ->booleanNode('enabled')->defaultValue(true)->end() ->scalarNode('lat_lon')->end() + ->scalarNode('tree')->end() + ->scalarNode('precision')->end() + ->scalarNode('tree_levels')->end() ->scalarNode('index_name')->end() ->booleanNode('omit_norms')->end() ->scalarNode('index_options')->end() From 0dcb77d7491ef2f2bdd87db357b393c8b790f28b Mon Sep 17 00:00:00 2001 From: Ruslan Zavacky Date: Sat, 19 Apr 2014 18:31:56 +0300 Subject: [PATCH 039/154] More classes exposed as parameters. --- Resources/config/config.xml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 1e49590..c10568d 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -8,6 +8,10 @@ FOS\ElasticaBundle\Client FOS\ElasticaBundle\DynamicIndex Elastica\Type + FOS\ElasticaBundle\IndexManager + FOS\ElasticaBundle\Resetter + FOS\ElasticaBundle\Finder\TransformedFinder + FOS\ElasticaBundle\Subscriber\PaginateElasticaQuerySubscriber FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector FOS\ElasticaBundle\Manager\RepositoryManager @@ -32,12 +36,12 @@ - + - + @@ -48,7 +52,7 @@ - + @@ -84,7 +88,7 @@ - + From 806452813a33b19641fa9172b7acf96d58f0b587 Mon Sep 17 00:00:00 2001 From: Ahmed Mohamed Date: Sun, 20 Apr 2014 03:41:41 +0200 Subject: [PATCH 040/154] Add test multiple objects mapping of at least one auto-mapped and at least one manually mapped --- .../ModelToElasticaAutoTransformerTest.php | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php index 646d1a4..1fa6a8e 100644 --- a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php +++ b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php @@ -107,6 +107,11 @@ class POPO return array('foo' => 'foo', 'bar' => 'foo', 'id' => 1); } + public function getNestedObject() + { + return array('key1' => (object)array('id' => 1, 'key1sub1' => 'value1sub1', 'key1sub2' => 'value1sub2')); + } + public function getUpper() { return (object) array('id' => 'parent', 'name' => 'a random name'); @@ -296,6 +301,51 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase ), $data['obj']); } + public function testObjectsMappingOfAtLeastOneAutoMappedObjectAndAtLeastOneManuallyMappedObject() + { + $transformer = $this->getTransformer(); + $document = $transformer->transform( + new POPO(), + array( + 'obj' => array('type' => 'object', 'properties' => array()), + 'nestedObject' => array( + 'type' => 'object', + 'properties' => array( + 'key1sub1' => array( + 'type' => 'string', + 'properties' => array() + ), + 'key1sub2' => array( + 'type' => 'string', + 'properties' => array() + ) + ) + ) + ) + ); + $data = $document->getData(); + + $this->assertTrue(array_key_exists('obj', $data)); + $this->assertTrue(array_key_exists('nestedObject', $data)); + $this->assertInternalType('array', $data['obj']); + $this->assertInternalType('array', $data['nestedObject']); + $this->assertEquals( + array( + 'foo' => 'foo', + 'bar' => 'foo', + 'id' => 1 + ), + $data['obj'] + ); + $this->assertEquals( + array( + 'key1sub1' => 'value1sub1', + 'key1sub2' => 'value1sub2' + ), + $data['nestedObject'][0] + ); + } + public function testParentMapping() { $transformer = $this->getTransformer(); From c9fd1cc5d99ba7d46050802b5dc9a78b44eb6dbc Mon Sep 17 00:00:00 2001 From: Ahmed Mohamed Date: Sun, 20 Apr 2014 06:11:20 +0200 Subject: [PATCH 041/154] Revert declaring PaginateElasticaQuerySubscriber as a paremeter, so not to break Knp compiler pass --- Resources/config/config.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index c10568d..7687250 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -11,7 +11,6 @@ FOS\ElasticaBundle\IndexManager FOS\ElasticaBundle\Resetter FOS\ElasticaBundle\Finder\TransformedFinder - FOS\ElasticaBundle\Subscriber\PaginateElasticaQuerySubscriber FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector FOS\ElasticaBundle\Manager\RepositoryManager @@ -88,7 +87,7 @@ - + From c93bbb9081c3915e457713990c02afc700c0a933 Mon Sep 17 00:00:00 2001 From: Evan Villemez Date: Thu, 1 May 2014 11:05:16 -0400 Subject: [PATCH 042/154] stop config from adding empty arrays into type mappings --- DependencyInjection/Configuration.php | 24 ++++++-- .../DependencyInjection/ConfigurationTest.php | 56 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b0a7b59..e4fd323 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -351,8 +351,16 @@ class Configuration implements ConfigurationInterface ->useAttributeAsKey('name') ->prototype('array') ->validate() - ->ifTrue(function($v) { return isset($v['fields']) && empty($v['fields']); }) - ->then(function($v) { unset($v['fields']); return $v; }) + ->always() + ->then(function($v) { + foreach (array('fields','properties') as $prop) { + if (isset($v[$prop]) && empty($v[$prop])) { + unset($v[$prop]); + } + } + + return $v; + }) ->end() ->treatNullLike(array()) ->addDefaultsIfNotSet() @@ -464,8 +472,16 @@ class Configuration implements ConfigurationInterface ->useAttributeAsKey('name') ->prototype('array') ->validate() - ->ifTrue(function($v) { return isset($v['fields']) && empty($v['fields']); }) - ->then(function($v) { unset($v['fields']); return $v; }) + ->always() + ->then(function($v) { + foreach (array('fields','properties') as $prop) { + if (isset($v[$prop]) && empty($v[$prop])) { + unset($v[$prop]); + } + } + + return $v; + }) ->end() ->treatNullLike(array()) ->addDefaultsIfNotSet() diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 5919ea7..08ada08 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -147,4 +147,60 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['content']); $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['title']); } + + public function testEmptyPropertiesIndexIsUnset() + { + $config = array( + 'indexes' => array( + 'test' => array( + 'types' => array( + 'test' => array( + 'mappings' => array( + 'title' => array( + 'type' => 'string', + 'fields' => array( + 'autocomplete' => null + ) + ), + 'content' => null, + 'children' => array( + 'type' => 'object', + 'properties' => array( + 'title' => array( + 'type' => 'string', + 'fields' => array( + 'autocomplete' => null + ) + ), + 'content' => null, + 'tags' => array( + 'properties' => array( + 'tag' => array( + 'type' => 'string', + 'index' => 'not_analyzed' + ) + ) + ) + ) + ), + ) + ) + ) + ) + ) + ); + + $processor = new Processor(); + + $configuration = $processor->processConfiguration(new Configuration(array($config)), array($config)); + + $mapping = $configuration['indexes']['test']['types']['test']['mappings']; + $this->assertArrayNotHasKey('properties', $mapping['content']); + $this->assertArrayNotHasKey('properties', $mapping['title']); + $this->assertArrayHasKey('properties', $mapping['children']); + $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['title']); + $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['content']); + $this->assertArrayHasKey('properties', $mapping['children']['properties']['tags']); + $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['tags']['properties']['tag']); + } } From d9f3fa1a595d82b4305b2706e44a65c6add05d16 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Fri, 2 May 2014 15:07:15 +0100 Subject: [PATCH 043/154] commit 1dcaadbe6f99b42b346e04239c0c8fd7812bbf51 Persister/ObjectPersister.php line 112: $this->type->updateDocuments($documents); introduces a dependency on Elastica 0.9.10.0 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 8dd19b6..24801e7 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.2.0, <1.1-dev", + "ruflin/elastica": ">=0.90.10.0, <1.1-dev", "psr/log": "~1.0" }, "require-dev":{ From ae02364e7c66fae05176e0462090a85fd0b16bf9 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 5 May 2014 08:53:33 +0200 Subject: [PATCH 044/154] use elastica library 1.1 and higher --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 24801e7..580785d 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.10.0, <1.1-dev", + "ruflin/elastica": ">=1.1-dev", "psr/log": "~1.0" }, "require-dev":{ From eaa9f8399744fb1966848e3bc2b83d1b2f5c9c54 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 5 May 2014 11:55:48 +0200 Subject: [PATCH 045/154] remove empty fields arrays from mapping, this is not ignored anymore by elasticsearch 1.* --- DependencyInjection/FOSElasticaExtension.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 51cb94f..2019d5e 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -234,6 +234,7 @@ class FOSElasticaExtension extends Extension $this->indexConfigs[$indexName]['config']['mappings'][$name]['_routing'] = $type['_routing']; } if (isset($type['mappings']) && !empty($type['mappings'])) { + $this->cleanUpMapping($type['mappings']); $this->indexConfigs[$indexName]['config']['mappings'][$name]['properties'] = $type['mappings']; $typeName = sprintf('%s/%s', $indexName, $name); $this->typeFields[$typeName] = $type['mappings']; @@ -570,4 +571,14 @@ class FOSElasticaExtension extends Extension $container->setAlias('fos_elastica.manager', sprintf('fos_elastica.manager.%s', $defaultManagerService)); } + + protected function cleanUpMapping(&$mappings) + { + foreach ($mappings as &$fieldProperties) + if (empty($fieldProperties['fields'])) { + unset($fieldProperties['fields']); + } else { + $this->cleanUpMapping($fieldProperties['fields']); + } + } } From b1d64e358d2e4070668c4af158c9573daf33bda5 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 5 May 2014 12:01:05 +0200 Subject: [PATCH 046/154] Also cleanup fields in properties of objects --- DependencyInjection/FOSElasticaExtension.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 2019d5e..ede7c21 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -574,11 +574,16 @@ class FOSElasticaExtension extends Extension protected function cleanUpMapping(&$mappings) { - foreach ($mappings as &$fieldProperties) - if (empty($fieldProperties['fields'])) { - unset($fieldProperties['fields']); - } else { - $this->cleanUpMapping($fieldProperties['fields']); + foreach ($mappings as &$fieldProperties) { + if (empty($fieldProperties['fields'])) { + unset($fieldProperties['fields']); + } else { + $this->cleanUpMapping($fieldProperties['fields']); + } + + if (!empty($fieldProperties['properties'])) { + $this->cleanUpMapping($fieldProperties['properties']); + } } } } From 5da8ee1a1655e8bd44c2db4e354b529dcf6c98f5 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Tue, 6 May 2014 08:18:37 +0200 Subject: [PATCH 047/154] still supporting 0.9 versions of elastica --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 580785d..a991d38 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=1.1-dev", + "ruflin/elastica": ">=0.90.10.0, <1.2-dev", "psr/log": "~1.0" }, "require-dev":{ From b0841c18ecb89a3ce3e2b8192411e9e9fdec3c00 Mon Sep 17 00:00:00 2001 From: caponica Date: Thu, 8 May 2014 00:26:30 +0100 Subject: [PATCH 048/154] Changed QueryBuilder method from ->where() to ->andWhere() so it works with customised QueryBuilders which have an existing where clause (instead of over-writing any existing DQL 'where' part) --- Doctrine/ORM/ElasticaToModelTransformer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doctrine/ORM/ElasticaToModelTransformer.php b/Doctrine/ORM/ElasticaToModelTransformer.php index 20ec6e8..a57a84c 100644 --- a/Doctrine/ORM/ElasticaToModelTransformer.php +++ b/Doctrine/ORM/ElasticaToModelTransformer.php @@ -29,7 +29,7 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer $hydrationMode = $hydrate ? Query::HYDRATE_OBJECT : Query::HYDRATE_ARRAY; $qb = $this->getEntityQueryBuilder(); - $qb->where($qb->expr()->in(static::ENTITY_ALIAS.'.'.$this->options['identifier'], ':values')) + $qb->andWhere($qb->expr()->in(static::ENTITY_ALIAS.'.'.$this->options['identifier'], ':values')) ->setParameter('values', $identifierValues); return $qb->getQuery()->setHydrationMode($hydrationMode)->execute(); From e5754ef5fcc67bc3bc0815696a405e7e5770e814 Mon Sep 17 00:00:00 2001 From: Thomas Ploch Date: Tue, 13 May 2014 13:13:06 +0200 Subject: [PATCH 049/154] Make it possible to disable flush event through configuration --- DependencyInjection/Configuration.php | 1 + DependencyInjection/FOSElasticaExtension.php | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b0a7b59..cb813a5 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -288,6 +288,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->scalarNode('flush')->defaultTrue()->end() ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('logger') ->defaultFalse() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index ede7c21..7d754f1 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -465,9 +465,6 @@ class FOSElasticaExtension extends Extension */ private function getDoctrineEvents(array $typeConfig) { - // Flush always calls depending on actions scheduled in lifecycle listeners - $typeConfig['listener']['flush'] = true; - switch ($typeConfig['driver']) { case 'orm': $eventsClass = '\Doctrine\ORM\Events'; From e1bbb87cfe011c0277a03e9ad66cdde2e521d86b Mon Sep 17 00:00:00 2001 From: Milan Magudia Date: Fri, 16 May 2014 16:24:37 +0100 Subject: [PATCH 050/154] Fix for Issue #543 Client has a dependency on a non-existent service "%kernel.debug%" --- DependencyInjection/Configuration.php | 7 +++++-- DependencyInjection/FOSElasticaExtension.php | 2 +- Tests/DependencyInjection/ConfigurationTest.php | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 3c57558..fbdba94 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -16,10 +16,13 @@ class Configuration implements ConfigurationInterface private $supportedDrivers = array('orm', 'mongodb', 'propel'); private $configArray = array(); + private $debug; - public function __construct($configArray) + public function __construct($configArray, $debug) { + $this->configArray = $configArray; + $this->debug = $debug; } /** @@ -121,7 +124,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('host')->end() ->scalarNode('port')->end() ->scalarNode('logger') - ->defaultValue('%kernel.debug%') + ->defaultValue(($this->debug) ? 'fos_elastica.logger' : false) ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 7d754f1..f58cd5b 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -61,7 +61,7 @@ class FOSElasticaExtension extends Extension public function getConfiguration(array $config, ContainerBuilder $container) { - return new Configuration($config); + return new Configuration($config, $container->getParameter('kernel.debug')); } /** diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 08ada08..38ffc58 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -17,7 +17,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase public function setUp() { - $this->configuration = new Configuration(array()); + $this->configuration = new Configuration(array(), false); } public function testEmptyConfigContainsFormatMappingOptionNode() @@ -140,7 +140,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $processor = new Processor(); - $configuration = $processor->processConfiguration(new Configuration(array($config)), array($config)); + $configuration = $processor->processConfiguration(new Configuration(array($config), false), array($config)); $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['content']); $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['title']); @@ -192,7 +192,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $processor = new Processor(); - $configuration = $processor->processConfiguration(new Configuration(array($config)), array($config)); + $configuration = $processor->processConfiguration(new Configuration(array($config), false), array($config)); $mapping = $configuration['indexes']['test']['types']['test']['mappings']; $this->assertArrayNotHasKey('properties', $mapping['content']); From 2c208a4f103446bd2e4bca15846d43d43ebd11ae Mon Sep 17 00:00:00 2001 From: Milan Magudia Date: Thu, 22 May 2014 16:18:08 +0100 Subject: [PATCH 051/154] Allow other transport options to be used i.e. Http, Https, Guzzle etc... --- DependencyInjection/Configuration.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index fbdba94..3dfe5ae 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -92,6 +92,7 @@ class Configuration implements ConfigurationInterface 'port' => $v['port'], 'logger' => isset($v['logger']) ? $v['logger'] : null, 'headers' => isset($v['headers']) ? $v['headers'] : null, + 'transport' => isset($v['transport']) ? $v['transport'] : null, ) ) ); @@ -132,6 +133,7 @@ class Configuration implements ConfigurationInterface ->useAttributeAsKey('name') ->prototype('scalar')->end() ->end() + ->scalarNode('transport')->end() ->scalarNode('timeout')->end() ->end() ->end() From 28d0ee925d1eb8369944985ae246d40bc3025034 Mon Sep 17 00:00:00 2001 From: Darius Staisiunas Date: Fri, 23 May 2014 12:55:33 +0300 Subject: [PATCH 052/154] added support for geohash --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index fbdba94..215f4a2 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -435,6 +435,7 @@ class Configuration implements ConfigurationInterface ->booleanNode('include_in_all')->defaultValue(true)->end() ->booleanNode('enabled')->defaultValue(true)->end() ->scalarNode('lat_lon')->end() + ->scalarNode('geohash')->end() ->scalarNode('index_name')->end() ->booleanNode('omit_norms')->end() ->scalarNode('index_options')->end() From 2029aba76a92d5eaffddb7998c1ef98aa264b318 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 31 Mar 2014 16:45:31 +1100 Subject: [PATCH 053/154] Ability for FOSElasticaBundle to disable persistence backend logging for population Update documentation and changelog --- CHANGELOG-3.0.md | 1 + DependencyInjection/Configuration.php | 3 ++- Doctrine/AbstractProvider.php | 27 ++++++++++++++++++++- Doctrine/MongoDB/Provider.php | 34 ++++++++++++++++++++++++++ Doctrine/ORM/Provider.php | 35 +++++++++++++++++++++++++++ Resources/doc/types.md | 15 ++++++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index abd05cd..9d74640 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -19,6 +19,7 @@ To generate a changelog summary since the last version, run * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. * 7d13823: Dropped (broken) support for Symfony <2.3 * #496: Added support for HTTP headers + * #528: FOSElasticaBundle will disable Doctrine logging when populating for a large increase in speed * 3.0.0-ALPHA2 (2014-03-17) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index d26a516..dff773e 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -187,9 +187,10 @@ class Configuration implements ConfigurationInterface ->scalarNode('identifier')->defaultValue('id')->end() ->arrayNode('provider') ->children() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() ->scalarNode('batch_size')->defaultValue(100)->end() ->scalarNode('clear_object_manager')->defaultTrue()->end() + ->scalarNode('disable_logger')->defaultValue('%kernel.debug%')->end() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() ->scalarNode('service')->end() ->end() ->end() diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 3fafe50..1c3f853 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -23,6 +23,7 @@ abstract class AbstractProvider extends BaseAbstractProvider { parent::__construct($objectPersister, $objectClass, array_merge(array( 'clear_object_manager' => true, + 'disable_logging' => false, 'ignore_errors' => false, 'query_builder_method' => 'createQueryBuilder', ), $options)); @@ -35,12 +36,17 @@ abstract class AbstractProvider extends BaseAbstractProvider */ public function populate(\Closure $loggerClosure = null, array $options = array()) { + if (!$this->options['disable_logging']) { + $logger = $this->disableLogging(); + } + $queryBuilder = $this->createQueryBuilder(); $nbObjects = $this->countObjects($queryBuilder); $offset = isset($options['offset']) ? intval($options['offset']) : 0; $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors']; + $manager = $this->managerRegistry->getManagerForClass($this->objectClass); for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { @@ -61,7 +67,7 @@ abstract class AbstractProvider extends BaseAbstractProvider } if ($this->options['clear_object_manager']) { - $this->managerRegistry->getManagerForClass($this->objectClass)->clear(); + $manager->clear(); } usleep($sleep); @@ -75,6 +81,10 @@ abstract class AbstractProvider extends BaseAbstractProvider $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); } } + + if (!$this->options['disable_logging']) { + $this->enableLogging($logger); + } } /** @@ -85,6 +95,21 @@ abstract class AbstractProvider extends BaseAbstractProvider */ protected abstract function countObjects($queryBuilder); + /** + * Disables logging and returns the logger that was previously set. + * + * @return mixed + */ + protected abstract function disableLogging(); + + /** + * Reenables the logger with the previously returned logger from disableLogging(); + * + * @param mixed $logger + * @return mixed + */ + protected abstract function enableLogging($logger); + /** * Fetches a slice of objects using the query builder. * diff --git a/Doctrine/MongoDB/Provider.php b/Doctrine/MongoDB/Provider.php index 16d9f76..9e1c5dd 100644 --- a/Doctrine/MongoDB/Provider.php +++ b/Doctrine/MongoDB/Provider.php @@ -8,6 +8,40 @@ use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException; class Provider extends AbstractProvider { + /** + * Disables logging and returns the logger that was previously set. + * + * @return mixed + */ + protected function disableLogging() + { + $configuration = $this->managerRegistry + ->getManagerForClass($this->objectClass) + ->getConnection() + ->getConfiguration(); + + $logger = $configuration->getLoggerCallable(); + $configuration->setLoggerCallable(null); + + return $logger; + } + + /** + * Reenables the logger with the previously returned logger from disableLogging(); + * + * @param mixed $logger + * @return mixed + */ + protected function enableLogging($logger) + { + $configuration = $this->managerRegistry + ->getManagerForClass($this->objectClass) + ->getConnection() + ->getConfiguration(); + + $configuration->setLoggerCallable($logger); + } + /** * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::countObjects() */ diff --git a/Doctrine/ORM/Provider.php b/Doctrine/ORM/Provider.php index 5d0164f..dfd6700 100644 --- a/Doctrine/ORM/Provider.php +++ b/Doctrine/ORM/Provider.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Doctrine\ORM; use Doctrine\ORM\QueryBuilder; +use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Doctrine\AbstractProvider; use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException; @@ -10,6 +11,40 @@ class Provider extends AbstractProvider { const ENTITY_ALIAS = 'a'; + /** + * Disables logging and returns the logger that was previously set. + * + * @return mixed + */ + protected function disableLogging() + { + $configuration = $this->managerRegistry + ->getManagerForClass($this->objectClass) + ->getConnection() + ->getConfiguration(); + + $logger = $configuration->getSQLLogger(); + $configuration->setSQLLogger(null); + + return $logger; + } + + /** + * Reenables the logger with the previously returned logger from disableLogging(); + * + * @param mixed $logger + * @return mixed + */ + protected function enableLogging($logger) + { + $configuration = $this->managerRegistry + ->getManagerForClass($this->objectClass) + ->getConnection() + ->getConfiguration(); + + $configuration->setSQLLogger($logger); + } + /** * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::countObjects() */ diff --git a/Resources/doc/types.md b/Resources/doc/types.md index e55df5a..aa76a43 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -188,6 +188,21 @@ persistence configuration. identifier: searchId ``` +### Turning on the persistence backend logger in production + +FOSElasticaBundle will turn of your persistence backend's logging configuration by default +when Symfony2 is not in debug mode. + +To enable the logger (turn off this behaviour) set disable_logger to false for the +provider + +```yaml + user: + persistence: + provider: + disable_logger: false +``` + Listener Configuration ---------------------- From 6d2b7a836790dcb9ebe51e8f7ed606d560ef78bf Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 20:36:19 +1000 Subject: [PATCH 054/154] Combine client normalisation into a single method --- DependencyInjection/Configuration.php | 31 +++++---------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index dff773e..b420143 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -60,16 +60,6 @@ class Configuration implements ConfigurationInterface return $treeBuilder; } - /** - * Generates the configuration tree. - * - * @return \Symfony\Component\DependencyInjection\Configuration\NodeInterface - */ - public function getConfigTree() - { - return $this->getConfigTreeBuilder()->buildTree(); - } - /** * Adds the configuration for the "clients" key */ @@ -83,28 +73,17 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->performNoDeepMerging() ->beforeNormalization() - ->ifTrue(function($v) { return isset($v['host']) && isset($v['port']); }) + ->ifTrue(function($v) { return (isset($v['host']) && isset($v['port'])) || isset($v['url']); }) ->then(function($v) { return array( 'servers' => array( array( - 'host' => $v['host'], - 'port' => $v['port'], + 'host' => isset($v['host']) ? $v['host'] : null, + 'port' => isset($v['port']) ? $v['port'] : null, + 'url' => isset($v['url']) ? $v['url'] : null, 'logger' => isset($v['logger']) ? $v['logger'] : null, 'headers' => isset($v['headers']) ? $v['headers'] : null, - ) - ) - ); - }) - ->end() - ->beforeNormalization() - ->ifTrue(function($v) { return isset($v['url']); }) - ->then(function($v) { - return array( - 'servers' => array( - array( - 'url' => $v['url'], - 'logger' => isset($v['logger']) ? $v['logger'] : null + 'timeout' => isset($v['timeout']) ? $v['timeout'] : null, ) ) ); From 843c76b6cabd0fb71ef03cd95b9702e9dd41b2fc Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 20:39:37 +1000 Subject: [PATCH 055/154] Move persistence node to its own method --- DependencyInjection/Configuration.php | 194 ++++++++++---------------- 1 file changed, 75 insertions(+), 119 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b420143..c736b9a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -149,62 +149,7 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() - ->arrayNode('persistence') - ->validate() - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) - ->thenInvalid('Propel doesn\'t support listeners') - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) - ->thenInvalid('Propel doesn\'t support the "repository" parameter') - ->end() - ->children() - ->scalarNode('driver') - ->validate() - ->ifNotInArray($this->supportedDrivers) - ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) - ->end() - ->end() - ->scalarNode('identifier')->defaultValue('id')->end() - ->arrayNode('provider') - ->children() - ->scalarNode('batch_size')->defaultValue(100)->end() - ->scalarNode('clear_object_manager')->defaultTrue()->end() - ->scalarNode('disable_logger')->defaultValue('%kernel.debug%')->end() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('listener') - ->children() - ->scalarNode('insert')->defaultTrue()->end() - ->scalarNode('update')->defaultTrue()->end() - ->scalarNode('delete')->defaultTrue()->end() - ->scalarNode('persist')->defaultValue('postFlush')->end() - ->scalarNode('service')->end() - ->variableNode('is_indexable_callback')->defaultNull()->end() - ->end() - ->end() - ->arrayNode('finder') - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('elastica_to_model_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('hydrate')->defaultTrue()->end() - ->scalarNode('ignore_missing')->defaultFalse()->end() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('model_to_elastica_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->end() - ->end() + ->append($this->getPersistenceNode()) ->end() ->end() ->variableNode('settings')->defaultValue(array())->end() @@ -241,69 +186,7 @@ class Configuration implements ConfigurationInterface ->end() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() - ->arrayNode('persistence') - ->validate() - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) - ->thenInvalid('Propel doesn\'t support listeners') - ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) - ->thenInvalid('Propel doesn\'t support the "repository" parameter') - ->end() - ->children() - ->scalarNode('driver') - ->validate() - ->ifNotInArray($this->supportedDrivers) - ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) - ->end() - ->end() - ->scalarNode('model')->end() - ->scalarNode('repository')->end() - ->scalarNode('identifier')->defaultValue('id')->end() - ->arrayNode('provider') - ->children() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('batch_size')->defaultValue(100)->end() - ->scalarNode('clear_object_manager')->defaultTrue()->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('listener') - ->children() - ->scalarNode('insert')->defaultTrue()->end() - ->scalarNode('update')->defaultTrue()->end() - ->scalarNode('delete')->defaultTrue()->end() - ->scalarNode('flush')->defaultTrue()->end() - ->booleanNode('immediate')->defaultFalse()->end() - ->scalarNode('logger') - ->defaultFalse() - ->treatNullLike('fos_elastica.logger') - ->treatTrueLike('fos_elastica.logger') - ->end() - ->scalarNode('service')->end() - ->variableNode('is_indexable_callback')->defaultNull()->end() - ->end() - ->end() - ->arrayNode('finder') - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('elastica_to_model_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('hydrate')->defaultTrue()->end() - ->scalarNode('ignore_missing')->defaultFalse()->end() - ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() - ->scalarNode('service')->end() - ->end() - ->end() - ->arrayNode('model_to_elastica_transformer') - ->addDefaultsIfNotSet() - ->children() - ->scalarNode('service')->end() - ->end() - ->end() - ->end() - ->end() + ->append($this->getPersistenceNode()) ->end() ->append($this->getIdNode()) ->append($this->getMappingsNode()) @@ -738,4 +621,77 @@ class Configuration implements ConfigurationInterface return $node; } + + /** + * @return ArrayNodeDefinition|\Symfony\Component\Config\Definition\Builder\NodeDefinition + */ + protected function getPersistenceNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('persistence'); + + $node + ->validate() + ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['listener']); }) + ->thenInvalid('Propel doesn\'t support listeners') + ->ifTrue(function($v) { return isset($v['driver']) && 'propel' === $v['driver'] && isset($v['repository']); }) + ->thenInvalid('Propel doesn\'t support the "repository" parameter') + ->end() + ->children() + ->scalarNode('driver') + ->validate() + ->ifNotInArray($this->supportedDrivers) + ->thenInvalid('The driver %s is not supported. Please choose one of '.json_encode($this->supportedDrivers)) + ->end() + ->end() + ->scalarNode('model')->end() + ->scalarNode('repository')->end() + ->scalarNode('identifier')->defaultValue('id')->end() + ->arrayNode('provider') + ->children() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() + ->scalarNode('batch_size')->defaultValue(100)->end() + ->scalarNode('clear_object_manager')->defaultTrue()->end() + ->scalarNode('service')->end() + ->end() + ->end() + ->arrayNode('listener') + ->children() + ->scalarNode('insert')->defaultTrue()->end() + ->scalarNode('update')->defaultTrue()->end() + ->scalarNode('delete')->defaultTrue()->end() + ->booleanNode('immediate')->defaultFalse()->end() + ->scalarNode('logger') + ->defaultFalse() + ->treatNullLike('fos_elastica.logger') + ->treatTrueLike('fos_elastica.logger') + ->end() + ->scalarNode('service')->end() + ->variableNode('is_indexable_callback')->defaultNull()->end() + ->end() + ->end() + ->arrayNode('finder') + ->children() + ->scalarNode('service')->end() + ->end() + ->end() + ->arrayNode('elastica_to_model_transformer') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('hydrate')->defaultTrue()->end() + ->scalarNode('ignore_missing')->defaultFalse()->end() + ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() + ->scalarNode('service')->end() + ->end() + ->end() + ->arrayNode('model_to_elastica_transformer') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('service')->end() + ->end() + ->end() + ->end(); + + return $node; + } } From 41c4d77b20b0876561eeab1a381a0f11ca9c8941 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 20:41:09 +1000 Subject: [PATCH 056/154] Move serializer node to its own method, add serializer to type_prototype --- DependencyInjection/Configuration.php | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index c736b9a..a8b1d2f 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -150,6 +150,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() ->append($this->getPersistenceNode()) + ->append($this->getSerializerNode()) ->end() ->end() ->variableNode('settings')->defaultValue(array())->end() @@ -174,19 +175,10 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->treatNullLike(array()) ->children() - ->arrayNode('serializer') - ->addDefaultsIfNotSet() - ->children() - ->arrayNode('groups') - ->treatNullLike(array()) - ->prototype('scalar')->end() - ->end() - ->scalarNode('version')->end() - ->end() - ->end() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() ->append($this->getPersistenceNode()) + ->append($this->getSerializerNode()) ->end() ->append($this->getIdNode()) ->append($this->getMappingsNode()) @@ -694,4 +686,25 @@ class Configuration implements ConfigurationInterface return $node; } + + /** + * @return ArrayNodeDefinition|\Symfony\Component\Config\Definition\Builder\NodeDefinition + */ + protected function getSerializerNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('serializer'); + + $node + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('groups') + ->treatNullLike(array()) + ->prototype('scalar')->end() + ->end() + ->scalarNode('version')->end() + ->end(); + + return $node; + } } From 352e3b68ac26da2faf73125613619c30fa0b6590 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 22:24:11 +1000 Subject: [PATCH 057/154] Add configuration tests --- .../DependencyInjection/ConfigurationTest.php | 200 ++++++++++-------- 1 file changed, 108 insertions(+), 92 deletions(-) diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 38ffc58..e2fba93 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -11,141 +11,157 @@ use Symfony\Component\Config\Definition\Processor; class ConfigurationTest extends \PHPUnit_Framework_TestCase { /** - * @var Configuration + * @var Processor */ - private $configuration; + private $processor; public function setUp() { - $this->configuration = new Configuration(array(), false); + $this->processor = new Processor(); } - public function testEmptyConfigContainsFormatMappingOptionNode() + private function getConfigs(array $configArray) { - $tree = $this->configuration->getConfigTree(); - $children = $tree->getChildren(); - $children = $children['indexes']->getPrototype()->getChildren(); - $typeNodes = $children['types']->getPrototype()->getChildren(); - $mappings = $typeNodes['mappings']->getPrototype()->getChildren(); + $configuration = new Configuration($configArray, true); - $this->assertArrayHasKey('format', $mappings); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mappings['format']); - $this->assertNull($mappings['format']->getDefaultValue()); + return $this->processor->processConfiguration($configuration, array($configArray)); } - public function testDynamicTemplateNodes() + public function testUnconfiguredConfiguration() { - $tree = $this->configuration->getConfigTree(); - $children = $tree->getChildren(); - $children = $children['indexes']->getPrototype()->getChildren(); - $typeNodes = $children['types']->getPrototype()->getChildren(); - $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren(); + $configuration = $this->getConfigs(array()); - $this->assertArrayHasKey('match', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match']); - $this->assertNull($dynamicTemplates['match']->getDefaultValue()); - - $this->assertArrayHasKey('match_mapping_type', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match_mapping_type']); - $this->assertNull($dynamicTemplates['match_mapping_type']->getDefaultValue()); - - $this->assertArrayHasKey('unmatch', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['unmatch']); - $this->assertNull($dynamicTemplates['unmatch']->getDefaultValue()); - - $this->assertArrayHasKey('path_match', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['path_match']); - $this->assertNull($dynamicTemplates['path_match']->getDefaultValue()); - - $this->assertArrayHasKey('path_unmatch', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['path_unmatch']); - $this->assertNull($dynamicTemplates['path_unmatch']->getDefaultValue()); - - $this->assertArrayHasKey('match_pattern', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $dynamicTemplates['match_pattern']); - $this->assertNull($dynamicTemplates['match_pattern']->getDefaultValue()); - - $this->assertArrayHasKey('mapping', $dynamicTemplates); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ArrayNode', $dynamicTemplates['mapping']); + $this->assertSame(array( + 'clients' => array(), + 'indexes' => array(), + 'default_manager' => 'orm' + ), $configuration); } - public function testDynamicTemplateMappingNodes() + public function testClientConfiguration() { - $tree = $this->configuration->getConfigTree(); - $children = $tree->getChildren(); - $children = $children['indexes']->getPrototype()->getChildren(); - $typeNodes = $children['types']->getPrototype()->getChildren(); - $dynamicTemplates = $typeNodes['dynamic_templates']->getPrototype()->getChildren(); - $mapping = $dynamicTemplates['mapping']->getChildren(); + $configuration = $this->getConfigs(array( + 'clients' => array( + 'default' => array( + 'url' => 'http://localhost:9200', + ), + 'clustered' => array( + 'servers' => array( + array( + 'url' => 'http://es1:9200', + 'headers' => array( + 'Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==' + ) + ), + array( + 'url' => 'http://es2:9200', + 'headers' => array( + 'Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==' + ) + ), + ) + ) + ) + )); - $this->assertArrayHasKey('type', $mapping); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['type']); - $this->assertSame('string', $mapping['type']->getDefaultValue()); + $this->assertCount(2, $configuration['clients']); + $this->assertCount(1, $configuration['clients']['default']['servers']); + $this->assertCount(0, $configuration['clients']['default']['servers'][0]['headers']); - $this->assertArrayHasKey('index', $mapping); - $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['index']); - $this->assertNull($mapping['index']->getDefaultValue()); + $this->assertCount(2, $configuration['clients']['clustered']['servers']); + $this->assertEquals('http://es2:9200/', $configuration['clients']['clustered']['servers'][1]['url']); + $this->assertCount(1, $configuration['clients']['clustered']['servers'][1]['headers']); + $this->assertEquals('Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==', $configuration['clients']['clustered']['servers'][0]['headers'][0]); + } + + public function testLogging() + { + $configuration = $this->getConfigs(array( + 'clients' => array( + 'logging_enabled' => array( + 'url' => 'http://localhost:9200', + 'logger' => true, + ), + 'logging_disabled' => array( + 'url' => 'http://localhost:9200', + 'logger' => false, + ), + 'logging_not_mentioned' => array( + 'url' => 'http://localhost:9200', + ), + 'logging_custom' => array( + 'url' => 'http://localhost:9200', + 'logger' => 'custom.service' + ), + ) + )); + + $this->assertCount(4, $configuration['clients']); + + $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_enabled']['servers'][0]['logger']); + $this->assertFalse($configuration['clients']['logging_disabled']['servers'][0]['logger']); + $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_not_mentioned']['servers'][0]['logger']); + $this->assertEquals('custom.service', $configuration['clients']['logging_custom']['servers'][0]['logger']); } public function testSlashIsAddedAtTheEndOfServerUrl() { $config = array( 'clients' => array( - 'default' => array( - 'url' => 'http://www.github.com', - ), + 'default' => array('url' => 'http://www.github.com'), ), - ); - - $processor = new Processor(); - - $configuration = $processor->processConfiguration($this->configuration, array($config)); + ); + $configuration = $this->getConfigs($config); $this->assertEquals('http://www.github.com/', $configuration['clients']['default']['servers'][0]['url']); } - public function testEmptyFieldsIndexIsUnset() + public function testTypeConfig() { - $config = array( + $configuration = $this->getConfigs(array( + 'clients' => array( + 'default' => array('url' => 'http://localhost:9200'), + ), 'indexes' => array( 'test' => array( + 'type_prototype' => array( + 'index_analyzer' => 'custom_analyzer', + 'persistence' => array( + 'identifier' => 'ID', + ), + 'serializer' => array( + 'groups' => array('Search'), + 'version' => 1 + ) + ), 'types' => array( 'test' => array( 'mappings' => array( - 'title' => array( - 'type' => 'string', - 'fields' => array( - 'autocomplete' => null - ) - ), - 'content' => null, + 'title' => array(), + 'published' => array('type' => 'datetime'), + 'body' => null, + ), + 'persistence' => array( + 'listener' => array( + 'logger' => true, + ) + ) + ), + 'test2' => array( + 'mappings' => array( + 'title' => null, 'children' => array( 'type' => 'nested', - 'properties' => array( - 'title' => array( - 'type' => 'string', - 'fields' => array( - 'autocomplete' => null - ) - ), - 'content' => null - ) ) ) ) ) ) ) - ); + )); - $processor = new Processor(); - - $configuration = $processor->processConfiguration(new Configuration(array($config), false), array($config)); - - $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['content']); - $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['title']); - $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['content']); - $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['children']['properties']['title']); + $this->assertEquals('string', $configuration['indexes']['test']['types']['test']['mappings']['title']['type']); + $this->assertTrue($configuration['indexes']['test']['types']['test']['mappings']['title']['include_in_all']); } public function testEmptyPropertiesIndexIsUnset() From 3addfffc916f4ed09d774537a4f77728e6fd1ddb Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 22:57:57 +1000 Subject: [PATCH 058/154] Added logging of server errors to example --- .../doc/cookbook/suppress-server-errors.md | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Resources/doc/cookbook/suppress-server-errors.md b/Resources/doc/cookbook/suppress-server-errors.md index d89ffd6..e4e371e 100644 --- a/Resources/doc/cookbook/suppress-server-errors.md +++ b/Resources/doc/cookbook/suppress-server-errors.md @@ -32,6 +32,16 @@ class Client extends BaseClient try { return parent::request($path, $method, $data, $query); } catch (ExceptionInterface $e) { + if ($this->_logger) { + $this->_logger->warning('Failed to send a request to ElasticSearch', array( + 'exception' => $e->getMessage(), + 'path' => $path, + 'method' => $method, + 'data' => $data, + 'query' => $query + )); + } + return new Response('{"took":0,"timed_out":false,"hits":{"total":0,"max_score":0,"hits":[]}}'); } } @@ -41,15 +51,9 @@ class Client extends BaseClient Configuration change: --------------------- -```xml - +You must update a parameter in your `app/config/config.yml` file to point to your overridden client: - - - - Acme\ElasticaBundle\Client - - - +```yaml +parameters: + fos_elastica.client.class: Acme\ElasticaBundle\Client +``` From f8a445b46c939f04bc06a7f1d7631aca12ac618b Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 23:11:45 +1000 Subject: [PATCH 059/154] Fix disabling of logger in DoctrineProvider --- DependencyInjection/Configuration.php | 5 ++++- Doctrine/AbstractProvider.php | 6 +++--- Resources/doc/types.md | 9 ++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 154abcc..d84cd3c 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -189,7 +189,10 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('batch_size')->defaultValue(100)->end() ->scalarNode('clear_object_manager')->defaultTrue()->end() - ->scalarNode('disable_logger')->defaultValue('%kernel.debug%')->end() + ->booleanNode('debug_logging') + ->defaultValue($this->debug) + ->treatNullLike($this->debug) + ->end() ->scalarNode('query_builder_method')->defaultValue('createQueryBuilder')->end() ->scalarNode('service')->end() ->end() diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 1c3f853..b9ffda5 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -23,7 +23,7 @@ abstract class AbstractProvider extends BaseAbstractProvider { parent::__construct($objectPersister, $objectClass, array_merge(array( 'clear_object_manager' => true, - 'disable_logging' => false, + 'debug_logging' => false, 'ignore_errors' => false, 'query_builder_method' => 'createQueryBuilder', ), $options)); @@ -36,7 +36,7 @@ abstract class AbstractProvider extends BaseAbstractProvider */ public function populate(\Closure $loggerClosure = null, array $options = array()) { - if (!$this->options['disable_logging']) { + if (!$this->options['debug_logging']) { $logger = $this->disableLogging(); } @@ -82,7 +82,7 @@ abstract class AbstractProvider extends BaseAbstractProvider } } - if (!$this->options['disable_logging']) { + if (!$this->options['debug_logging']) { $this->enableLogging($logger); } } diff --git a/Resources/doc/types.md b/Resources/doc/types.md index aa76a43..aacb09e 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -191,16 +191,15 @@ persistence configuration. ### Turning on the persistence backend logger in production FOSElasticaBundle will turn of your persistence backend's logging configuration by default -when Symfony2 is not in debug mode. - -To enable the logger (turn off this behaviour) set disable_logger to false for the -provider +when Symfony2 is not in debug mode. You can force FOSElasticaBundle to always disable +logging by setting debug_logging to false, to leave logging alone by setting it to true, +or leave it set to its default value which will mirror %kernel.debug%. ```yaml user: persistence: provider: - disable_logger: false + debug_logging: false ``` Listener Configuration From f20392d78b912e4a50be9b9e1c87c95e9e3560dc Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 23:19:55 +1000 Subject: [PATCH 060/154] Fix test failures for DoctrineProvider --- Tests/Doctrine/AbstractProviderTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index 2492eed..dcceccf 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -17,7 +17,7 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase } $this->objectClass = 'objectClass'; - $this->options = array(); + $this->options = array('debug_logging' => true); $this->objectPersister = $this->getMockObjectPersister(); $this->managerRegistry = $this->getMockManagerRegistry(); From e77aa0c180d662ecf99fafed8bca23ccee6400b2 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 23 May 2014 23:20:52 +1000 Subject: [PATCH 061/154] Test on php 5.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8f6a9d8..72632de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: php php: - 5.3 - 5.4 + - 5.5 before_script: - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini From a9ea78443fbed12b78cd5767829eb1f6f766f523 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sat, 24 May 2014 00:17:59 +1000 Subject: [PATCH 062/154] Support Elastica proxy option --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index d0ec09f..9b2bc89 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -102,6 +102,7 @@ class Configuration implements ConfigurationInterface ->end() ->scalarNode('host')->end() ->scalarNode('port')->end() + ->scalarNode('proxy')->end() ->scalarNode('logger') ->defaultValue(($this->debug) ? 'fos_elastica.logger' : false) ->treatNullLike('fos_elastica.logger') From f9745c8d21895ecac980a1b4160fc6780cea2716 Mon Sep 17 00:00:00 2001 From: Delf Tonder Date: Sat, 24 May 2014 12:49:12 +0200 Subject: [PATCH 063/154] update setup.md - immediate is an listener option fixed yml config example (having immediate as an persistence option will result in parsing error) --- Resources/doc/setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 4ae40af..9a39b95 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -114,9 +114,9 @@ Below is an example for the Doctrine ORM. driver: orm model: Acme\ApplicationBundle\Entity\User provider: ~ - listener: ~ + listener: + immediate: ~ finder: ~ - immediate: ~ ``` There are a significant number of options available for types, that can be From f97e66712a861067263567021f402c2ff04100fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Sj=C3=B6sten?= Date: Sun, 25 May 2014 00:31:40 +0100 Subject: [PATCH 064/154] Don't default url --- DependencyInjection/Configuration.php | 2 +- Tests/DependencyInjection/ConfigurationTest.php | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 9b2bc89..275b23d 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -96,7 +96,7 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('url') ->validate() - ->ifTrue(function($url) { return substr($url, -1) !== '/'; }) + ->ifTrue(function($url) { return $url && substr($url, -1) !== '/'; }) ->then(function($url) { return $url.'/'; }) ->end() ->end() diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index e2fba93..bddd62e 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -219,4 +219,18 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertArrayHasKey('properties', $mapping['children']['properties']['tags']); $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['tags']['properties']['tag']); } + + public function testClientConfigurationNoUrl() + { + $configuration = $this->getConfigs(array( + 'clients' => array( + 'default' => array( + 'host' => 'localhost', + 'port' => 9200, + ), + ) + )); + + $this->assertTrue(empty($configuration['clients']['default']['servers'][0]['url'])); + } } From c38dc107e766ac100e2145db64d4fa7277ef517d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 25 May 2014 17:34:44 +1000 Subject: [PATCH 065/154] Rename mappings to properties maintaining BC Fixes #407 --- DependencyInjection/Configuration.php | 18 +++++-- DependencyInjection/FOSElasticaExtension.php | 47 +++++++++---------- Resetter.php | 4 +- .../DependencyInjection/ConfigurationTest.php | 30 ++++++++++-- Tests/ResetterTest.php | 16 +++---- 5 files changed, 74 insertions(+), 41 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 275b23d..ac7d3a2 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -175,6 +175,16 @@ class Configuration implements ConfigurationInterface ->useAttributeAsKey('name') ->prototype('array') ->treatNullLike(array()) + // BC - Renaming 'mappings' node to 'properties' + ->beforeNormalization() + ->ifTrue(function($v) { return isset($v['mappings']); }) + ->then(function($v) { + $v['properties'] = $v['mappings']; + unset($v['mappings']); + + return $v; + }) + ->end() ->children() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() @@ -182,7 +192,7 @@ class Configuration implements ConfigurationInterface ->append($this->getSerializerNode()) ->end() ->append($this->getIdNode()) - ->append($this->getMappingsNode()) + ->append($this->getPropertiesNode()) ->append($this->getDynamicTemplateNode()) ->append($this->getSourceNode()) ->append($this->getBoostNode()) @@ -198,12 +208,12 @@ class Configuration implements ConfigurationInterface } /** - * Returns the array node used for "mappings". + * Returns the array node used for "properties". */ - protected function getMappingsNode() + protected function getPropertiesNode() { $builder = new TreeBuilder(); - $node = $builder->root('mappings'); + $node = $builder->root('properties'); $nestings = $this->getNestings(); diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index f58cd5b..1529544 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -4,7 +4,6 @@ namespace FOS\ElasticaBundle\DependencyInjection; use Symfony\Component\HttpKernel\DependencyInjection\Extension; use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; -use Symfony\Component\DependencyInjection\ContainerAwareInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\DefinitionDecorator; @@ -129,7 +128,7 @@ class FOSElasticaExtension extends Extension 'index' => new Reference($indexId), 'name_or_alias' => $indexName, 'config' => array( - 'mappings' => array() + 'properties' => array() ) ); if ($index['finder']) { @@ -217,30 +216,30 @@ class FOSElasticaExtension extends Extension } $container->setDefinition($typeId, $typeDef); - $this->indexConfigs[$indexName]['config']['mappings'][$name] = array( + $this->indexConfigs[$indexName]['config']['properties'][$name] = array( "_source" => array("enabled" => true), // Add a default setting for empty mapping settings ); if (isset($type['_id'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_id'] = $type['_id']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_id'] = $type['_id']; } if (isset($type['_source'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_source'] = $type['_source']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_source'] = $type['_source']; } if (isset($type['_boost'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_boost'] = $type['_boost']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_boost'] = $type['_boost']; } if (isset($type['_routing'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_routing'] = $type['_routing']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_routing'] = $type['_routing']; } - if (isset($type['mappings']) && !empty($type['mappings'])) { - $this->cleanUpMapping($type['mappings']); - $this->indexConfigs[$indexName]['config']['mappings'][$name]['properties'] = $type['mappings']; + if (isset($type['properties']) && !empty($type['properties'])) { + $this->cleanUpProperties($type['properties']); + $this->indexConfigs[$indexName]['config']['properties'][$name]['properties'] = $type['properties']; $typeName = sprintf('%s/%s', $indexName, $name); - $this->typeFields[$typeName] = $type['mappings']; + $this->typeFields[$typeName] = $type['properties']; } if (isset($type['_parent'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_parent'] = array('type' => $type['_parent']['type']); + $this->indexConfigs[$indexName]['config']['properties'][$name]['_parent'] = array('type' => $type['_parent']['type']); $typeName = sprintf('%s/%s', $indexName, $name); $this->typeFields[$typeName]['_parent'] = $type['_parent']; } @@ -248,27 +247,27 @@ class FOSElasticaExtension extends Extension $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); } if (isset($type['index_analyzer'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['index_analyzer'] = $type['index_analyzer']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['index_analyzer'] = $type['index_analyzer']; } if (isset($type['search_analyzer'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['search_analyzer'] = $type['search_analyzer']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['search_analyzer'] = $type['search_analyzer']; } if (isset($type['index'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['index'] = $type['index']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['index'] = $type['index']; } if (isset($type['_all'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_all'] = $type['_all']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_all'] = $type['_all']; } if (isset($type['_timestamp'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_timestamp'] = $type['_timestamp']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_timestamp'] = $type['_timestamp']; } if (isset($type['_ttl'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['_ttl'] = $type['_ttl']; + $this->indexConfigs[$indexName]['config']['properties'][$name]['_ttl'] = $type['_ttl']; } if (!empty($type['dynamic_templates'])) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'] = array(); + $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'] = array(); foreach ($type['dynamic_templates'] as $templateName => $templateData) { - $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'][] = array($templateName => $templateData); + $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'][] = array($templateName => $templateData); } } } @@ -569,17 +568,17 @@ class FOSElasticaExtension extends Extension $container->setAlias('fos_elastica.manager', sprintf('fos_elastica.manager.%s', $defaultManagerService)); } - protected function cleanUpMapping(&$mappings) + protected function cleanUpProperties(&$properties) { - foreach ($mappings as &$fieldProperties) { + foreach ($properties as &$fieldProperties) { if (empty($fieldProperties['fields'])) { unset($fieldProperties['fields']); } else { - $this->cleanUpMapping($fieldProperties['fields']); + $this->cleanUpProperties($fieldProperties['fields']); } if (!empty($fieldProperties['properties'])) { - $this->cleanUpMapping($fieldProperties['properties']); + $this->cleanUpProperties($fieldProperties['properties']); } } } diff --git a/Resetter.php b/Resetter.php index fe963d0..d614f1b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -68,7 +68,7 @@ class Resetter { $indexConfig = $this->getIndexConfig($indexName); - if (!isset($indexConfig['config']['mappings'][$typeName]['properties'])) { + if (!isset($indexConfig['config']['properties'][$typeName]['properties'])) { throw new \InvalidArgumentException(sprintf('The mapping for index "%s" and type "%s" does not exist.', $indexName, $typeName)); } @@ -80,7 +80,7 @@ class Resetter throw $e; } } - $mapping = $this->createMapping($indexConfig['config']['mappings'][$typeName]); + $mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]); $type->setMapping($mapping); } diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index bddd62e..efaaa52 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -160,8 +160,8 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase ) )); - $this->assertEquals('string', $configuration['indexes']['test']['types']['test']['mappings']['title']['type']); - $this->assertTrue($configuration['indexes']['test']['types']['test']['mappings']['title']['include_in_all']); + $this->assertEquals('string', $configuration['indexes']['test']['types']['test']['properties']['title']['type']); + $this->assertTrue($configuration['indexes']['test']['types']['test']['properties']['title']['include_in_all']); } public function testEmptyPropertiesIndexIsUnset() @@ -210,7 +210,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $configuration = $processor->processConfiguration(new Configuration(array($config), false), array($config)); - $mapping = $configuration['indexes']['test']['types']['test']['mappings']; + $mapping = $configuration['indexes']['test']['types']['test']['properties']; $this->assertArrayNotHasKey('properties', $mapping['content']); $this->assertArrayNotHasKey('properties', $mapping['title']); $this->assertArrayHasKey('properties', $mapping['children']); @@ -233,4 +233,28 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertTrue(empty($configuration['clients']['default']['servers'][0]['url'])); } + + public function testMappingsRenamedToProperties() + { + $configuration = $this->getConfigs(array( + 'clients' => array( + 'default' => array('url' => 'http://localhost:9200'), + ), + 'indexes' => array( + 'test' => array( + 'types' => array( + 'test' => array( + 'mappings' => array( + 'title' => array(), + 'published' => array('type' => 'datetime'), + 'body' => null, + ) + ) + ) + ) + ) + )); + + $this->assertCount(3, $configuration['indexes']['test']['types']['test']['properties']); + } } diff --git a/Tests/ResetterTest.php b/Tests/ResetterTest.php index b4e5649..e27770f 100644 --- a/Tests/ResetterTest.php +++ b/Tests/ResetterTest.php @@ -18,7 +18,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase 'foo' => array( 'index' => $this->getMockElasticaIndex(), 'config' => array( - 'mappings' => array( + 'properties' => array( 'a' => array( 'dynamic_templates' => array(), 'properties' => array(), @@ -30,7 +30,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase 'bar' => array( 'index' => $this->getMockElasticaIndex(), 'config' => array( - 'mappings' => array( + 'properties' => array( 'a' => array('properties' => array()), 'b' => array('properties' => array()), ), @@ -39,7 +39,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase 'parent' => array( 'index' => $this->getMockElasticaIndex(), 'config' => array( - 'mappings' => array( + 'properties' => array( 'a' => array( 'properties' => array( 'field_2' => array() @@ -105,8 +105,8 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $type->expects($this->once()) ->method('delete'); - $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); - $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']); + $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['properties']['a']['properties']); + $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['properties']['a']['dynamic_templates']); $type->expects($this->once()) ->method('setMapping') ->with($mapping); @@ -149,8 +149,8 @@ class ResetterTest extends \PHPUnit_Framework_TestCase new Response(array('error' => 'TypeMissingException[[de_20131022] type[bla] missing]', 'status' => 404))) )); - $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); - $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']); + $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['properties']['a']['properties']); + $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['properties']['a']['dynamic_templates']); $type->expects($this->once()) ->method('setMapping') ->with($mapping); @@ -171,7 +171,7 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $type->expects($this->once()) ->method('delete'); - $mapping = Mapping::create($this->indexConfigsByName['parent']['config']['mappings']['a']['properties']); + $mapping = Mapping::create($this->indexConfigsByName['parent']['config']['properties']['a']['properties']); $mapping->setParam('_parent', array('type' => 'b')); $type->expects($this->once()) ->method('setMapping') From a79fa0242e834449ff075285764f878c3e2f5321 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 25 May 2014 20:08:01 +1000 Subject: [PATCH 066/154] Simplified Configuration.php --- DependencyInjection/Configuration.php | 238 +----------------- DependencyInjection/FOSElasticaExtension.php | 4 +- .../DependencyInjection/ConfigurationTest.php | 63 +---- 3 files changed, 15 insertions(+), 290 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 275b23d..0d0d238 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -15,13 +15,15 @@ class Configuration implements ConfigurationInterface */ private $supportedDrivers = array('orm', 'mongodb', 'propel'); - private $configArray = array(); + /** + * If the kernel is running in debug mode. + * + * @var bool + */ private $debug; - public function __construct($configArray, $debug) + public function __construct($debug) { - - $this->configArray = $configArray; $this->debug = $debug; } @@ -104,7 +106,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('port')->end() ->scalarNode('proxy')->end() ->scalarNode('logger') - ->defaultValue(($this->debug) ? 'fos_elastica.logger' : false) + ->defaultValue($this->debug ? 'fos_elastica.logger' : false) ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() @@ -205,28 +207,10 @@ class Configuration implements ConfigurationInterface $builder = new TreeBuilder(); $node = $builder->root('mappings'); - $nestings = $this->getNestings(); - - $childrenNode = $node + $node ->useAttributeAsKey('name') - ->prototype('array') - ->validate() - ->always() - ->then(function($v) { - foreach (array('fields','properties') as $prop) { - if (isset($v[$prop]) && empty($v[$prop])) { - unset($v[$prop]); - } - } - - return $v; - }) - ->end() - ->treatNullLike(array()) - ->addDefaultsIfNotSet() - ->children(); - - $this->addFieldConfig($childrenNode, $nestings); + ->prototype('variable') + ->treatNullLike(array()); return $node; } @@ -249,7 +233,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('path_match')->end() ->scalarNode('path_unmatch')->end() ->scalarNode('match_pattern')->end() - ->append($this->getDynamicTemplateMapping()) + ->append($this->getMappingsNode()) ->end() ->end() ; @@ -257,206 +241,6 @@ class Configuration implements ConfigurationInterface return $node; } - /** - * @return the array node used for mapping in dynamic templates - */ - protected function getDynamicTemplateMapping() - { - $builder = new TreeBuilder(); - $node = $builder->root('mapping'); - - $nestings = $this->getNestingsForDynamicTemplates(); - - $this->addFieldConfig($node->children(), $nestings); - - return $node; - } - - /** - * @param \Symfony\Component\Config\Definition\Builder\NodeBuilder $node The node to which to attach the field config to - * @param array $nestings the nested mappings for the current field level - */ - protected function addFieldConfig($node, $nestings) - { - $node - ->scalarNode('type')->defaultValue('string')->end() - ->scalarNode('boost')->end() - ->scalarNode('store')->end() - ->scalarNode('index')->end() - ->scalarNode('index_analyzer')->end() - ->scalarNode('search_analyzer')->end() - ->scalarNode('analyzer')->end() - ->scalarNode('term_vector')->end() - ->scalarNode('null_value')->end() - ->booleanNode('include_in_all')->defaultValue(true)->end() - ->booleanNode('enabled')->defaultValue(true)->end() - ->scalarNode('lat_lon')->end() - ->scalarNode('tree')->end() - ->scalarNode('precision')->end() - ->scalarNode('tree_levels')->end() - ->scalarNode('geohash')->end() - ->scalarNode('index_name')->end() - ->booleanNode('omit_norms')->end() - ->scalarNode('index_options')->end() - ->scalarNode('ignore_above')->end() - ->scalarNode('position_offset_gap')->end() - ->arrayNode('_parent') - ->treatNullLike(array()) - ->children() - ->scalarNode('type')->end() - ->scalarNode('identifier')->defaultValue('id')->end() - ->end() - ->end() - ->scalarNode('format')->end() - ->scalarNode('similarity')->end(); - ; - - if (isset($nestings['fields'])) { - $this->addNestedFieldConfig($node, $nestings, 'fields'); - } - - if (isset($nestings['properties'])) { - $node - ->booleanNode('include_in_parent')->end() - ->booleanNode('include_in_root')->end() - ; - $this->addNestedFieldConfig($node, $nestings, 'properties'); - } - } - - /** - * @param \Symfony\Component\Config\Definition\Builder\NodeBuilder $node The node to which to attach the nested config to - * @param array $nestings The nestings for the current field level - * @param string $property the name of the nested property ('fields' or 'properties') - */ - protected function addNestedFieldConfig($node, $nestings, $property) - { - $childrenNode = $node - ->arrayNode($property) - ->useAttributeAsKey('name') - ->prototype('array') - ->validate() - ->always() - ->then(function($v) { - foreach (array('fields','properties') as $prop) { - if (isset($v[$prop]) && empty($v[$prop])) { - unset($v[$prop]); - } - } - - return $v; - }) - ->end() - ->treatNullLike(array()) - ->addDefaultsIfNotSet() - ->children(); - - $this->addFieldConfig($childrenNode, $nestings[$property]); - - $childrenNode - ->end() - ->end() - ->end() - ; - } - - /** - * @return array The unique nested mappings for all types - */ - protected function getNestings() - { - if (!isset($this->configArray[0]['indexes'])) { - return array(); - } - - $nestings = array(); - foreach ($this->configArray[0]['indexes'] as $index) { - if (empty($index['types'])) { - continue; - } - - foreach ($index['types'] as $type) { - if (empty($type['mappings'])) { - continue; - } - - $nestings = array_merge_recursive($nestings, $this->getNestingsForType($type['mappings'], $nestings)); - } - } - return $nestings; - } - - /** - * @return array The unique nested mappings for all dynamic templates - */ - protected function getNestingsForDynamicTemplates() - { - if (!isset($this->configArray[0]['indexes'])) { - return array(); - } - - $nestings = array(); - foreach ($this->configArray[0]['indexes'] as $index) { - if (empty($index['types'])) { - continue; - } - - foreach ($index['types'] as $type) { - if (empty($type['dynamic_templates'])) { - continue; - } - - foreach ($type['dynamic_templates'] as $definition) { - $field = $definition['mapping']; - - if (isset($field['fields'])) { - $this->addPropertyNesting($field, $nestings, 'fields'); - } else if (isset($field['properties'])) { - $this->addPropertyNesting($field, $nestings, 'properties'); - } - } - - } - } - return $nestings; - } - - /** - * @param array $mappings The mappings for the current type - * @return array The nested mappings defined for this type - */ - protected function getNestingsForType(array $mappings = null) - { - if ($mappings === null) { - return array(); - } - - $nestings = array(); - - foreach ($mappings as $field) { - if (isset($field['fields'])) { - $this->addPropertyNesting($field, $nestings, 'fields'); - } else if (isset($field['properties'])) { - $this->addPropertyNesting($field, $nestings, 'properties'); - } - } - - return $nestings; - } - - /** - * @param array $field The field mapping definition - * @param array $nestings The nestings array - * @param string $property The nested property name ('fields' or 'properties') - */ - protected function addPropertyNesting($field, &$nestings, $property) - { - if (!isset($nestings[$property])) { - $nestings[$property] = array(); - } - $nestings[$property] = array_merge_recursive($nestings[$property], $this->getNestingsForType($field[$property])); - } - /** * Returns the array node used for "_id". */ diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index f58cd5b..806c273 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -22,7 +22,7 @@ class FOSElasticaExtension extends Extension public function load(array $configs, ContainerBuilder $container) { $configuration = $this->getConfiguration($configs, $container); - $config = $this->processConfiguration($configuration, $configs); + $config = $this->processConfiguration($configuration, $configs); $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); @@ -61,7 +61,7 @@ class FOSElasticaExtension extends Extension public function getConfiguration(array $config, ContainerBuilder $container) { - return new Configuration($config, $container->getParameter('kernel.debug')); + return new Configuration($container->getParameter('kernel.debug')); } /** diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index bddd62e..4a4da67 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -22,7 +22,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase private function getConfigs(array $configArray) { - $configuration = new Configuration($configArray, true); + $configuration = new Configuration(true); return $this->processor->processConfiguration($configuration, array($configArray)); } @@ -118,7 +118,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase public function testTypeConfig() { - $configuration = $this->getConfigs(array( + $this->getConfigs(array( 'clients' => array( 'default' => array('url' => 'http://localhost:9200'), ), @@ -159,65 +159,6 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase ) ) )); - - $this->assertEquals('string', $configuration['indexes']['test']['types']['test']['mappings']['title']['type']); - $this->assertTrue($configuration['indexes']['test']['types']['test']['mappings']['title']['include_in_all']); - } - - public function testEmptyPropertiesIndexIsUnset() - { - $config = array( - 'indexes' => array( - 'test' => array( - 'types' => array( - 'test' => array( - 'mappings' => array( - 'title' => array( - 'type' => 'string', - 'fields' => array( - 'autocomplete' => null - ) - ), - 'content' => null, - 'children' => array( - 'type' => 'object', - 'properties' => array( - 'title' => array( - 'type' => 'string', - 'fields' => array( - 'autocomplete' => null - ) - ), - 'content' => null, - 'tags' => array( - 'properties' => array( - 'tag' => array( - 'type' => 'string', - 'index' => 'not_analyzed' - ) - ) - ) - ) - ), - ) - ) - ) - ) - ) - ); - - $processor = new Processor(); - - $configuration = $processor->processConfiguration(new Configuration(array($config), false), array($config)); - - $mapping = $configuration['indexes']['test']['types']['test']['mappings']; - $this->assertArrayNotHasKey('properties', $mapping['content']); - $this->assertArrayNotHasKey('properties', $mapping['title']); - $this->assertArrayHasKey('properties', $mapping['children']); - $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['title']); - $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['content']); - $this->assertArrayHasKey('properties', $mapping['children']['properties']['tags']); - $this->assertArrayNotHasKey('properties', $mapping['children']['properties']['tags']['properties']['tag']); } public function testClientConfigurationNoUrl() From 53180e281041f7168943f54b6480582b536221ce Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 25 May 2014 20:14:51 +1000 Subject: [PATCH 067/154] Bring tidy in line with property renaming --- DependencyInjection/Configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 208026b..60b0684 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -243,7 +243,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('path_match')->end() ->scalarNode('path_unmatch')->end() ->scalarNode('match_pattern')->end() - ->append($this->getMappingsNode()) + ->append($this->getPropertiesNode()) ->end() ->end() ; From dad15d0b387cab71ca1fa68dc70f85ad4310ad38 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 21 Apr 2014 09:21:50 +1000 Subject: [PATCH 068/154] Deprecate top level classes --- CHANGELOG-3.0.md | 20 +- Client.php | 38 +-- DynamicIndex.php | 20 +- Elastica/Client.php | 47 ++++ Elastica/Index.php | 28 ++ Index/IndexManager.php | 63 +++++ Index/Resetter.php | 246 ++++++++++++++++++ IndexManager.php | 61 +---- Resetter.php | 240 +---------------- Resources/config/config.xml | 4 +- .../LoggingClientTest.php} | 6 +- Tests/{ => Index}/IndexManagerTest.php | 4 +- Tests/{ => Index}/ResetterTest.php | 4 +- 13 files changed, 425 insertions(+), 356 deletions(-) create mode 100644 Elastica/Client.php create mode 100644 Elastica/Index.php create mode 100644 Index/IndexManager.php create mode 100644 Index/Resetter.php rename Tests/{ClientTest.php => Elastica/LoggingClientTest.php} (86%) rename Tests/{ => Index}/IndexManagerTest.php (95%) rename Tests/{ => Index}/ResetterTest.php (98%) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 9d74640..095c268 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,7 +12,25 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.0-ALPHA3 (xxxx-xx-xx) +* 3.0.0-ALPHA6 + + * Deprecated FOS\ElasticaBundle\Client in favour of FOS\ElasticaBundle\Elastica\Client + * Deprecated FOS\ElasticaBundle\DynamicIndex in favour of FOS\ElasticaBundle\Elastica\Index + * Deprecated FOS\ElasticaBundle\IndexManager in favour of FOS\ElasticaBundle\Index\IndexManager + * Deprecated FOS\ElasticaBundle\Resetter in favour of FOS\ElasticaBundle\Index\Resetter + +* 3.0.0-ALPHA5 (2014-05-23) + +* Doctrine Provider speed up by disabling persistence logging while populating documents + +* 3.0.0-ALPHA4 (2014-04-10) + + * Indexes are now capable of logging errors with Elastica + * Fixed deferred indexing of deleted documents + * Resetting an index will now create it even if it doesn't exist + * Bulk upserting of documents is now supported when populating + +* 3.0.0-ALPHA3 (2014-04-01) * a9c4c93: Logger is now only enabled in debug mode by default * #463: allowing hot swappable reindexing diff --git a/Client.php b/Client.php index 3eb98fe..f85756d 100644 --- a/Client.php +++ b/Client.php @@ -2,43 +2,11 @@ namespace FOS\ElasticaBundle; -use Elastica\Client as ElasticaClient; -use Elastica\Request; -use FOS\ElasticaBundle\Logger\ElasticaLogger; +use FOS\ElasticaBundle\Elastica\LoggingClient; /** - * @author Gordon Franke + * @deprecated Use \FOS\ElasticaBundle\Elastica\LoggingClient */ -class Client extends ElasticaClient +class Client extends LoggingClient { - /** - * {@inheritdoc} - */ - public function request($path, $method = Request::GET, $data = array(), array $query = array()) - { - $start = microtime(true); - $response = parent::request($path, $method, $data, $query); - - if (null !== $this->_logger and $this->_logger instanceof ElasticaLogger) { - $time = microtime(true) - $start; - - $connection = $this->getLastRequest()->getConnection(); - - $connection_array = array( - 'host' => $connection->getHost(), - 'port' => $connection->getPort(), - 'transport' => $connection->getTransport(), - 'headers' => $connection->getConfig('headers'), - ); - - $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); - } - - return $response; - } - - public function getIndex($name) - { - return new DynamicIndex($this, $name); - } } diff --git a/DynamicIndex.php b/DynamicIndex.php index cbec8e9..484a0d6 100644 --- a/DynamicIndex.php +++ b/DynamicIndex.php @@ -2,27 +2,11 @@ namespace FOS\ElasticaBundle; -use Elastica\Index; +use FOS\ElasticaBundle\Elastica\Index; /** - * Elastica index capable of reassigning name dynamically - * - * @author Konstantin Tjuterev + * @deprecated Use \FOS\ElasticaBundle\Elastica\TransformingIndex */ class DynamicIndex extends Index { - /** - * Reassign index name - * - * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName - * since it's used for a very specific case and normally should not be used - * - * @param string $name Index name - * - * @return void - */ - public function overrideName($name) - { - $this->_name = $name; - } } diff --git a/Elastica/Client.php b/Elastica/Client.php new file mode 100644 index 0000000..64a7d3d --- /dev/null +++ b/Elastica/Client.php @@ -0,0 +1,47 @@ + + */ +class Client extends BaseClient +{ + /** + * {@inheritdoc} + */ + public function request($path, $method = Request::GET, $data = array(), array $query = array()) + { + $start = microtime(true); + $response = parent::request($path, $method, $data, $query); + + if ($this->_logger and $this->_logger instanceof ElasticaLogger) { + $time = microtime(true) - $start; + + $connection = $this->getLastRequest()->getConnection(); + + $connection_array = array( + 'host' => $connection->getHost(), + 'port' => $connection->getPort(), + 'transport' => $connection->getTransport(), + 'headers' => $connection->getConfig('headers'), + ); + + $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); + } + + return $response; + } + + public function getIndex($name) + { + return new Index($this, $name); + } +} diff --git a/Elastica/Index.php b/Elastica/Index.php new file mode 100644 index 0000000..35d49f9 --- /dev/null +++ b/Elastica/Index.php @@ -0,0 +1,28 @@ + + */ +class Index extends BaseIndex +{ + /** + * Reassign index name + * + * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName + * since it's used for a very specific case and normally should not be used + * + * @param string $name Index name + * + * @return void + */ + public function overrideName($name) + { + $this->_name = $name; + } +} diff --git a/Index/IndexManager.php b/Index/IndexManager.php new file mode 100644 index 0000000..543ccae --- /dev/null +++ b/Index/IndexManager.php @@ -0,0 +1,63 @@ +indexesByName = $indexesByName; + $this->defaultIndexName = $defaultIndex->getName(); + } + + /** + * Gets all registered indexes + * + * @return array + */ + public function getAllIndexes() + { + return $this->indexesByName; + } + + /** + * Gets an index by its name + * + * @param string $name Index to return, or the default index if null + * @return Index + * @throws \InvalidArgumentException if no index exists for the given name + */ + public function getIndex($name = null) + { + if (null === $name) { + $name = $this->defaultIndexName; + } + + if (!isset($this->indexesByName[$name])) { + throw new \InvalidArgumentException(sprintf('The index "%s" does not exist', $name)); + } + + return $this->indexesByName[$name]; + } + + /** + * Gets the default index + * + * @return Index + */ + public function getDefaultIndex() + { + return $this->getIndex($this->defaultIndexName); + } +} diff --git a/Index/Resetter.php b/Index/Resetter.php new file mode 100644 index 0000000..99ae75e --- /dev/null +++ b/Index/Resetter.php @@ -0,0 +1,246 @@ +indexConfigsByName = $indexConfigsByName; + } + + /** + * Deletes and recreates all indexes + */ + public function resetAllIndexes() + { + foreach (array_keys($this->indexConfigsByName) as $name) { + $this->resetIndex($name); + } + } + + /** + * Deletes and recreates the named index + * + * @param string $indexName + * @throws \InvalidArgumentException if no index exists for the given name + */ + public function resetIndex($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + $esIndex = $indexConfig['index']; + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $name = $indexConfig['name_or_alias']; + $name .= uniqid(); + $esIndex->overrideName($name); + $esIndex->create($indexConfig['config']); + + return; + } + + $esIndex->create($indexConfig['config'], true); + } + + /** + * Deletes and recreates a mapping type for the named index + * + * @param string $indexName + * @param string $typeName + * @throws \InvalidArgumentException if no index or type mapping exists for the given names + * @throws ResponseException + */ + public function resetIndexType($indexName, $typeName) + { + $indexConfig = $this->getIndexConfig($indexName); + + if (!isset($indexConfig['config']['properties'][$typeName]['properties'])) { + throw new \InvalidArgumentException(sprintf('The mapping for index "%s" and type "%s" does not exist.', $indexName, $typeName)); + } + + $type = $indexConfig['index']->getType($typeName); + try { + $type->delete(); + } catch (ResponseException $e) { + if (strpos($e->getMessage(), 'TypeMissingException') === false) { + throw $e; + } + } + $mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]); + $type->setMapping($mapping); + } + + /** + * create type mapping object + * + * @param array $indexConfig + * @return Mapping + */ + protected function createMapping($indexConfig) + { + $mapping = Mapping::create($indexConfig['properties']); + + $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); + foreach ($mappingSpecialFields as $specialField) { + if (isset($indexConfig[$specialField])) { + $mapping->setParam($specialField, $indexConfig[$specialField]); + } + } + + if (isset($indexConfig['_parent'])) { + $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); + } + + return $mapping; + } + + /** + * Gets an index config by its name + * + * @param string $indexName Index name + * + * @param $indexName + * @return array + * @throws \InvalidArgumentException if no index config exists for the given name + */ + protected function getIndexConfig($indexName) + { + if (!isset($this->indexConfigsByName[$indexName])) { + throw new \InvalidArgumentException(sprintf('The configuration for index "%s" does not exist.', $indexName)); + } + + return $this->indexConfigsByName[$indexName]; + } + + public function postPopulate($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $this->switchIndexAlias($indexName); + } + } + + /** + * Switches the alias for given index (by key) to the newly populated index + * and deletes the old index + * + * @param string $indexName Index name + * + * @throws \RuntimeException + */ + private function switchIndexAlias($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + $esIndex = $indexConfig['index']; + $aliasName = $indexConfig['name_or_alias']; + $oldIndexName = false; + $newIndexName = $esIndex->getName(); + + $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); + + if (count($aliasedIndexes) > 1) { + throw new \RuntimeException( + sprintf( + 'Alias %s is used for multiple indexes: [%s]. + Make sure it\'s either not used or is assigned to one index only', + $aliasName, + join(', ', $aliasedIndexes) + ) + ); + } + + // Change the alias to point to the new index + // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic + // In 0.20.x it's atomic, but it doesn't return the old index name + $aliasUpdateRequest = array('actions' => array()); + if (count($aliasedIndexes) == 1) { + // if the alias is set - add an action to remove it + $oldIndexName = $aliasedIndexes[0]; + $aliasUpdateRequest['actions'][] = array( + 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) + ); + } + + // add an action to point the alias to the new index + $aliasUpdateRequest['actions'][] = array( + 'add' => array('index' => $newIndexName, 'alias' => $aliasName) + ); + + try { + $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); + } catch (ExceptionInterface $renameAliasException) { + $additionalError = ''; + // if we failed to move the alias, delete the newly built index + try { + $esIndex->delete(); + } catch (ExceptionInterface $deleteNewIndexException) { + $additionalError = sprintf( + 'Tried to delete newly built index %s, but also failed: %s', + $newIndexName, + $deleteNewIndexException->getError() + ); + } + + throw new \RuntimeException( + sprintf( + 'Failed to updated index alias: %s. %s', + $renameAliasException->getMessage(), + $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) + ) + ); + } + + // Delete the old index after the alias has been switched + if ($oldIndexName) { + $oldIndex = new Index($esIndex->getClient(), $oldIndexName); + try { + $oldIndex->delete(); + } catch (ExceptionInterface $deleteOldIndexException) { + throw new \RuntimeException( + sprintf( + 'Failed to delete old index %s with message: %s', + $oldIndexName, + $deleteOldIndexException->getMessage() + ) + ); + } + } + } + + /** + * Returns array of indexes which are mapped to given alias + * + * @param Index $esIndex ES Index + * @param string $aliasName Alias name + * + * @return array + */ + private function getAliasedIndexes(Index $esIndex, $aliasName) + { + $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); + $aliasedIndexes = array(); + + foreach ($aliasesInfo as $indexName => $indexInfo) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } + } + + return $aliasedIndexes; + } +} diff --git a/IndexManager.php b/IndexManager.php index e20a791..e7c74c8 100644 --- a/IndexManager.php +++ b/IndexManager.php @@ -2,62 +2,11 @@ namespace FOS\ElasticaBundle; -use Elastica\Index; +use FOS\ElasticaBundle\Index\IndexManager as BaseIndexManager; -class IndexManager +/** + * @deprecated Use \FOS\ElasticaBundle\Index\IndexManager + */ +class IndexManager extends BaseIndexManager { - protected $indexesByName; - protected $defaultIndexName; - - /** - * Constructor. - * - * @param array $indexesByName - * @param Index $defaultIndex - */ - public function __construct(array $indexesByName, Index $defaultIndex) - { - $this->indexesByName = $indexesByName; - $this->defaultIndexName = $defaultIndex->getName(); - } - - /** - * Gets all registered indexes - * - * @return array - */ - public function getAllIndexes() - { - return $this->indexesByName; - } - - /** - * Gets an index by its name - * - * @param string $name Index to return, or the default index if null - * @return Index - * @throws \InvalidArgumentException if no index exists for the given name - */ - public function getIndex($name = null) - { - if (null === $name) { - $name = $this->defaultIndexName; - } - - if (!isset($this->indexesByName[$name])) { - throw new \InvalidArgumentException(sprintf('The index "%s" does not exist', $name)); - } - - return $this->indexesByName[$name]; - } - - /** - * Gets the default index - * - * @return Index - */ - public function getDefaultIndex() - { - return $this->getIndex($this->defaultIndexName); - } } diff --git a/Resetter.php b/Resetter.php index d614f1b..2067579 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,245 +2,11 @@ namespace FOS\ElasticaBundle; -use Elastica\Exception\ExceptionInterface; -use Elastica\Index; -use Elastica\Exception\ResponseException; -use Elastica\Type\Mapping; +use FOS\ElasticaBundle\Index\Resetter as BaseResetter; /** - * Deletes and recreates indexes + * @deprecated Use \FOS\ElasticaBundle\Index\Resetter */ -class Resetter +class Resetter extends BaseResetter { - protected $indexConfigsByName; - - /** - * Constructor. - * - * @param array $indexConfigsByName - */ - public function __construct(array $indexConfigsByName) - { - $this->indexConfigsByName = $indexConfigsByName; - } - - /** - * Deletes and recreates all indexes - */ - public function resetAllIndexes() - { - foreach (array_keys($this->indexConfigsByName) as $name) { - $this->resetIndex($name); - } - } - - /** - * Deletes and recreates the named index - * - * @param string $indexName - * @throws \InvalidArgumentException if no index exists for the given name - */ - public function resetIndex($indexName) - { - $indexConfig = $this->getIndexConfig($indexName); - $esIndex = $indexConfig['index']; - if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { - $name = $indexConfig['name_or_alias']; - $name .= uniqid(); - $esIndex->overrideName($name); - $esIndex->create($indexConfig['config']); - - return; - } - - $esIndex->create($indexConfig['config'], true); - } - - /** - * Deletes and recreates a mapping type for the named index - * - * @param string $indexName - * @param string $typeName - * @throws \InvalidArgumentException if no index or type mapping exists for the given names - * @throws ResponseException - */ - public function resetIndexType($indexName, $typeName) - { - $indexConfig = $this->getIndexConfig($indexName); - - if (!isset($indexConfig['config']['properties'][$typeName]['properties'])) { - throw new \InvalidArgumentException(sprintf('The mapping for index "%s" and type "%s" does not exist.', $indexName, $typeName)); - } - - $type = $indexConfig['index']->getType($typeName); - try { - $type->delete(); - } catch (ResponseException $e) { - if (strpos($e->getMessage(), 'TypeMissingException') === false) { - throw $e; - } - } - $mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]); - $type->setMapping($mapping); - } - - /** - * create type mapping object - * - * @param array $indexConfig - * @return Mapping - */ - protected function createMapping($indexConfig) - { - $mapping = Mapping::create($indexConfig['properties']); - - $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); - foreach ($mappingSpecialFields as $specialField) { - if (isset($indexConfig[$specialField])) { - $mapping->setParam($specialField, $indexConfig[$specialField]); - } - } - - if (isset($indexConfig['_parent'])) { - $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); - } - - return $mapping; - } - - /** - * Gets an index config by its name - * - * @param string $indexName Index name - * - * @param $indexName - * @return array - * @throws \InvalidArgumentException if no index config exists for the given name - */ - protected function getIndexConfig($indexName) - { - if (!isset($this->indexConfigsByName[$indexName])) { - throw new \InvalidArgumentException(sprintf('The configuration for index "%s" does not exist.', $indexName)); - } - - return $this->indexConfigsByName[$indexName]; - } - - public function postPopulate($indexName) - { - $indexConfig = $this->getIndexConfig($indexName); - if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { - $this->switchIndexAlias($indexName); - } - } - - /** - * Switches the alias for given index (by key) to the newly populated index - * and deletes the old index - * - * @param string $indexName Index name - * - * @throws \RuntimeException - */ - private function switchIndexAlias($indexName) - { - $indexConfig = $this->getIndexConfig($indexName); - $esIndex = $indexConfig['index']; - $aliasName = $indexConfig['name_or_alias']; - $oldIndexName = false; - $newIndexName = $esIndex->getName(); - - $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); - - if (count($aliasedIndexes) > 1) { - throw new \RuntimeException( - sprintf( - 'Alias %s is used for multiple indexes: [%s]. - Make sure it\'s either not used or is assigned to one index only', - $aliasName, - join(', ', $aliasedIndexes) - ) - ); - } - - // Change the alias to point to the new index - // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic - // In 0.20.x it's atomic, but it doesn't return the old index name - $aliasUpdateRequest = array('actions' => array()); - if (count($aliasedIndexes) == 1) { - // if the alias is set - add an action to remove it - $oldIndexName = $aliasedIndexes[0]; - $aliasUpdateRequest['actions'][] = array( - 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) - ); - } - - // add an action to point the alias to the new index - $aliasUpdateRequest['actions'][] = array( - 'add' => array('index' => $newIndexName, 'alias' => $aliasName) - ); - - try { - $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); - } catch (ExceptionInterface $renameAliasException) { - $additionalError = ''; - // if we failed to move the alias, delete the newly built index - try { - $esIndex->delete(); - } catch (ExceptionInterface $deleteNewIndexException) { - $additionalError = sprintf( - 'Tried to delete newly built index %s, but also failed: %s', - $newIndexName, - $deleteNewIndexException->getError() - ); - } - - throw new \RuntimeException( - sprintf( - 'Failed to updated index alias: %s. %s', - $renameAliasException->getMessage(), - $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) - ) - ); - } - - // Delete the old index after the alias has been switched - if ($oldIndexName) { - $oldIndex = new Index($esIndex->getClient(), $oldIndexName); - try { - $oldIndex->delete(); - } catch (ExceptionInterface $deleteOldIndexException) { - throw new \RuntimeException( - sprintf( - 'Failed to delete old index %s with message: %s', - $oldIndexName, - $deleteOldIndexException->getMessage() - ) - ); - } - } - } - - /** - * Returns array of indexes which are mapped to given alias - * - * @param Index $esIndex ES Index - * @param string $aliasName Alias name - * - * @return array - */ - private function getAliasedIndexes(Index $esIndex, $aliasName) - { - $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); - $aliasedIndexes = array(); - - foreach ($aliasesInfo as $indexName => $indexInfo) { - $aliases = array_keys($indexInfo['aliases']); - if (in_array($aliasName, $aliases)) { - $aliasedIndexes[] = $indexName; - } - } - - return $aliasedIndexes; - } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 7687250..cff2b49 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -5,8 +5,8 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - FOS\ElasticaBundle\Client - FOS\ElasticaBundle\DynamicIndex + FOS\ElasticaBundle\Elastica\Client + FOS\ElasticaBundle\Elastica\Index Elastica\Type FOS\ElasticaBundle\IndexManager FOS\ElasticaBundle\Resetter diff --git a/Tests/ClientTest.php b/Tests/Elastica/LoggingClientTest.php similarity index 86% rename from Tests/ClientTest.php rename to Tests/Elastica/LoggingClientTest.php index 8a9d91a..b08a2cf 100644 --- a/Tests/ClientTest.php +++ b/Tests/Elastica/LoggingClientTest.php @@ -1,11 +1,11 @@ isType('array') ); - $client = $this->getMockBuilder('FOS\ElasticaBundle\Client') + $client = $this->getMockBuilder('FOS\ElasticaBundle\Elastica\LoggingClient') ->setMethods(array('getConnection')) ->getMock(); diff --git a/Tests/IndexManagerTest.php b/Tests/Index/IndexManagerTest.php similarity index 95% rename from Tests/IndexManagerTest.php rename to Tests/Index/IndexManagerTest.php index 0a8ea37..624f64e 100644 --- a/Tests/IndexManagerTest.php +++ b/Tests/Index/IndexManagerTest.php @@ -1,8 +1,8 @@ Date: Sun, 25 May 2014 18:51:14 +0200 Subject: [PATCH 069/154] fixing missing flush event handler In [commit](https://github.com/FriendsOfSymfony/FOSElasticaBundle/commit/843c76b6cabd0fb71ef03cd95b9702e9dd41b2fc#diff-850942b3ba24ab03a40aaa81b6152852) the configuration-definition for the flush listener was accidentally removed. As the flush listener is no longer set to be enabled in the extensions getDoctrineEvents method, the flush listener is not set. This results in a situation were we are only able to have the modified objects on the list for index-update, but never actually sending the update to the ES host. --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 275b23d..35d399f 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -654,6 +654,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->scalarNode('flush')->defaultTrue()->end() ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('logger') ->defaultFalse() From 2e0aa064a2b1d71a6315e21d5d7fedabeb58a0ab Mon Sep 17 00:00:00 2001 From: Tornaldo Date: Sun, 1 Jun 2014 01:27:20 +0200 Subject: [PATCH 070/154] Update usage.md --- Resources/doc/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index c1d5982..91f13ae 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -184,7 +184,7 @@ The following code will execute a search against the Elasticsearch server: $finder = $this->container->get('fos_elastica.finder.site.article'); $boolQuery = new \Elastica\Query\Bool(); -$fieldQuery = new \Elastica\Query\Text(); +$fieldQuery = new \Elastica\Query\Match(); $fieldQuery->setFieldQuery('title', 'I am a title string'); $fieldQuery->setFieldParam('title', 'analyzer', 'my_analyzer'); $boolQuery->addShould($fieldQuery); From 6a822504bc0cad280c4c7a713d089d5bcb919c76 Mon Sep 17 00:00:00 2001 From: Oskar Stark Date: Sun, 1 Jun 2014 15:57:29 +0200 Subject: [PATCH 071/154] use $this->container instead of $container --- Resources/doc/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index 91f13ae..55d90ab 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -65,7 +65,7 @@ You can now use the index wide finder service `fos_elastica.finder.website`: ```php /** var FOS\ElasticaBundle\Finder\MappedFinder */ -$finder = $container->get('fos_elastica.finder.website'); +$finder = $this->container->get('fos_elastica.finder.website'); // Returns a mixed array of any objects mapped $results = $finder->find('bob'); @@ -91,7 +91,7 @@ An example for using a repository: ```php /** var FOS\ElasticaBundle\Manager\RepositoryManager */ -$repositoryManager = $container->get('fos_elastica.manager'); +$repositoryManager = $this->container->get('fos_elastica.manager'); /** var FOS\ElasticaBundle\Repository */ $repository = $repositoryManager->getRepository('UserBundle:User'); From 1dc6856ef9c28f253a2e47f3dfb3396a4b24b5c5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 2 Jun 2014 00:40:03 +1000 Subject: [PATCH 072/154] Configuration rework --- DependencyInjection/Configuration.php | 2 +- DependencyInjection/FOSElasticaExtension.php | 406 ++++++++++--------- Resources/config/config.xml | 84 +--- Resources/config/index.xml | 39 ++ Resources/config/mongodb.xml | 4 +- Resources/config/orm.xml | 12 +- Resources/config/persister.xml | 27 ++ Resources/config/propel.xml | 3 - Resources/config/provider.xml | 18 + Resources/config/transformer.xml | 32 ++ Tests/Integration/MappingTest.php | 17 + 11 files changed, 372 insertions(+), 272 deletions(-) create mode 100644 Resources/config/index.xml create mode 100644 Resources/config/persister.xml create mode 100644 Resources/config/provider.xml create mode 100644 Resources/config/transformer.xml create mode 100644 Tests/Integration/MappingTest.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 60b0684..2acc9e5 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -288,7 +288,7 @@ class Configuration implements ConfigurationInterface ->end() ->scalarNode('compress')->end() ->scalarNode('compress_threshold')->end() - ->scalarNode('enabled')->end() + ->scalarNode('enabled')->defaultTrue()->end() ->end() ; diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index e896af2..c844f3a 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -13,9 +13,28 @@ use InvalidArgumentException; class FOSElasticaExtension extends Extension { - protected $indexConfigs = array(); - protected $typeFields = array(); - protected $loadedDrivers = array(); + /** + * Definition of elastica clients as configured by this extension. + * + * @var array + */ + private $clients = array(); + + /** + * An array of indexes as configured by the extension. + * + * @var array + */ + private $indexConfigs = array(); + + /** + * If we've encountered a type mapped to a specific persistence driver, it will be loaded + * here. + * + * @var array + */ + private $loadedDrivers = array(); + protected $serializerConfig = array(); public function load(array $configs, ContainerBuilder $container) @@ -30,7 +49,9 @@ class FOSElasticaExtension extends Extension return; } - $loader->load('config.xml'); + foreach (array('config', 'index', 'persister', 'provider', 'transformer') as $basename) { + $loader->load(sprintf('%s.xml', $basename)); + } if (empty($config['default_client'])) { $keys = array_keys($config['clients']); @@ -42,22 +63,24 @@ class FOSElasticaExtension extends Extension $config['default_index'] = reset($keys); } - $clientIdsByName = $this->loadClients($config['clients'], $container); $this->serializerConfig = isset($config['serializer']) ? $config['serializer'] : null; - $indexIdsByName = $this->loadIndexes($config['indexes'], $container, $clientIdsByName, $config['default_client']); - $indexRefsByName = array_map(function($id) { - return new Reference($id); - }, $indexIdsByName); - - $this->loadIndexManager($indexRefsByName, $container); - $this->loadResetter($this->indexConfigs, $container); + $this->loadClients($config['clients'], $container); $container->setAlias('fos_elastica.client', sprintf('fos_elastica.client.%s', $config['default_client'])); + + $this->loadIndexes($config['indexes'], $container); $container->setAlias('fos_elastica.index', sprintf('fos_elastica.index.%s', $config['default_index'])); + $this->loadIndexManager($container); + $this->createDefaultManagerAlias($config['default_manager'], $container); } + /** + * @param array $config + * @param ContainerBuilder $container + * @return Configuration|null|\Symfony\Component\Config\Definition\ConfigurationInterface + */ public function getConfiguration(array $config, ContainerBuilder $container) { return new Configuration($container->getParameter('kernel.debug')); @@ -70,24 +93,26 @@ class FOSElasticaExtension extends Extension * @param ContainerBuilder $container A ContainerBuilder instance * @return array */ - protected function loadClients(array $clients, ContainerBuilder $container) + private function loadClients(array $clients, ContainerBuilder $container) { - $clientIds = array(); foreach ($clients as $name => $clientConfig) { $clientId = sprintf('fos_elastica.client.%s', $name); - $clientDef = new Definition('%fos_elastica.client.class%', array($clientConfig)); + + $clientDef = new DefinitionDecorator('fos_elastica.client_prototype'); + $clientDef->replaceArgument(0, $clientConfig); + $logger = $clientConfig['servers'][0]['logger']; if (false !== $logger) { $clientDef->addMethodCall('setLogger', array(new Reference($logger))); } - $clientDef->addTag('fos_elastica.client'); $container->setDefinition($clientId, $clientDef); - $clientIds[$name] = $clientId; + $this->clients[$name] = array( + 'id' => $clientId, + 'reference' => new Reference($clientId) + ); } - - return $clientIds; } /** @@ -95,56 +120,44 @@ class FOSElasticaExtension extends Extension * * @param array $indexes An array of indexes configurations * @param ContainerBuilder $container A ContainerBuilder instance - * @param array $clientIdsByName - * @param $defaultClientName - * @param $serializerConfig * @throws \InvalidArgumentException * @return array */ - protected function loadIndexes(array $indexes, ContainerBuilder $container, array $clientIdsByName, $defaultClientName) + private function loadIndexes(array $indexes, ContainerBuilder $container) { - $indexIds = array(); foreach ($indexes as $name => $index) { - if (isset($index['client'])) { - $clientName = $index['client']; - if (!isset($clientIdsByName[$clientName])) { - throw new InvalidArgumentException(sprintf('The elastica client with name "%s" is not defined', $clientName)); - } - } else { - $clientName = $defaultClientName; - } - - $clientId = $clientIdsByName[$clientName]; $indexId = sprintf('fos_elastica.index.%s', $name); - $indexName = isset($index['index_name']) ? $index['index_name'] : $name; - $indexDefArgs = array($indexName); - $indexDef = new Definition('%fos_elastica.index.class%', $indexDefArgs); - $indexDef->setFactoryService($clientId); - $indexDef->setFactoryMethod('getIndex'); + $indexName = $index['index_name'] ?: $name; + + $indexDef = new DefinitionDecorator('fos_elastica.index_prototype'); + $indexDef->replaceArgument(0, $indexName); + + if ($index['client']) { + $client = $this->getClient($index['client']); + $indexDef->setFactoryService($client); + } + $container->setDefinition($indexId, $indexDef); - $typePrototypeConfig = isset($index['type_prototype']) ? $index['type_prototype'] : array(); - $indexIds[$name] = $indexId; + $reference = new Reference($indexId); + $this->indexConfigs[$name] = array( - 'index' => new Reference($indexId), - 'name_or_alias' => $indexName, 'config' => array( - 'properties' => array() - ) + 'properties' => array(), + 'settings' => $index['settings'] + ), + 'elasticsearch_name' => $indexName, + 'index' => $reference, + 'name' => $name, + 'type_prototype' => isset($index['type_prototype']) ? $index['type_prototype'] : array(), + 'use_alias' => $index['use_alias'], ); + if ($index['finder']) { - $this->loadIndexFinder($container, $name, $indexId); - } - if (!empty($index['settings'])) { - $this->indexConfigs[$name]['config']['settings'] = $index['settings']; - } - if ($index['use_alias']) { - $this->indexConfigs[$name]['use_alias'] = true; + $this->loadIndexFinder($container, $name, $reference); } - $this->loadTypes(isset($index['types']) ? $index['types'] : array(), $container, $name, $indexId, $typePrototypeConfig); + $this->loadTypes((array) $index['types'], $container, $this->indexConfigs[$name]); } - - return $indexIds; } /** @@ -152,10 +165,10 @@ class FOSElasticaExtension extends Extension * * @param \Symfony\Component\DependencyInjection\ContainerBuilder $container * @param string $name The index name - * @param string $indexId The index service identifier + * @param Reference $index Reference to the related index * @return string */ - protected function loadIndexFinder(ContainerBuilder $container, $name, $indexId) + private function loadIndexFinder(ContainerBuilder $container, $name, Reference $index) { /* Note: transformer services may conflict with "collection.index", if * an index and type names were "collection" and an index, respectively. @@ -166,34 +179,60 @@ class FOSElasticaExtension extends Extension $finderId = sprintf('fos_elastica.finder.%s', $name); $finderDef = new DefinitionDecorator('fos_elastica.finder'); - $finderDef->replaceArgument(0, new Reference($indexId)); + $finderDef->replaceArgument(0, $index); $finderDef->replaceArgument(1, new Reference($transformerId)); $container->setDefinition($finderId, $finderDef); - - return $finderId; } /** * Loads the configured types. * - * @param array $types An array of types configurations - * @param ContainerBuilder $container A ContainerBuilder instance - * @param $indexName - * @param $indexId - * @param array $typePrototypeConfig - * @param $serializerConfig + * @param array $types + * @param ContainerBuilder $container + * @param array $indexConfig */ - protected function loadTypes(array $types, ContainerBuilder $container, $indexName, $indexId, array $typePrototypeConfig) + private function loadTypes(array $types, ContainerBuilder $container, array $indexConfig) { foreach ($types as $name => $type) { - $type = self::deepArrayUnion($typePrototypeConfig, $type); - $typeId = sprintf('%s.%s', $indexId, $name); - $typeDefArgs = array($name); - $typeDef = new Definition('%fos_elastica.type.class%', $typeDefArgs); - $typeDef->setFactoryService($indexId); - $typeDef->setFactoryMethod('getType'); - if ($this->serializerConfig) { + $indexName = $indexConfig['name']; + $type = self::deepArrayUnion($indexConfig['type_prototype'], $type); + + $typeId = sprintf('%s.%s', $indexName, $name); + $typeDef = new DefinitionDecorator('fos_elastica.type_prototype'); + $typeDef->replaceArgument(0, $name); + $typeDef->setFactoryService($indexConfig['reference']); + + if (isset($type['persistence'])) { + $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); + } + + foreach (array( + 'index_analyzer', + 'properties', + 'search_analyzer', + '_all', + '_boost', + '_id', + '_parent', + '_routing', + '_source', + '_timestamp', + '_ttl', + ) as $field) { + $this->indexConfigs[$indexName]['config']['properties'][$name][$field] = $type[$field]; + } + + if (!empty($type['dynamic_templates'])) { + $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'] = array(); + foreach ($type['dynamic_templates'] as $templateName => $templateData) { + $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'][] = array($templateName => $templateData); + } + } + + $container->setDefinition($typeId, $typeDef); + + /*if ($this->serializerConfig) { $callbackDef = new Definition($this->serializerConfig['callback_class']); $callbackId = sprintf('%s.%s.serializer.callback', $indexId, $name); @@ -213,63 +252,7 @@ class FOSElasticaExtension extends Extension $container->setDefinition($callbackId, $callbackDef); $typeDef->addMethodCall('setSerializer', array(array(new Reference($callbackId), 'serialize'))); - } - $container->setDefinition($typeId, $typeDef); - - $this->indexConfigs[$indexName]['config']['properties'][$name] = array( - "_source" => array("enabled" => true), // Add a default setting for empty mapping settings - ); - - if (isset($type['_id'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_id'] = $type['_id']; - } - if (isset($type['_source'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_source'] = $type['_source']; - } - if (isset($type['_boost'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_boost'] = $type['_boost']; - } - if (isset($type['_routing'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_routing'] = $type['_routing']; - } - if (isset($type['properties']) && !empty($type['properties'])) { - $this->cleanUpProperties($type['properties']); - $this->indexConfigs[$indexName]['config']['properties'][$name]['properties'] = $type['properties']; - $typeName = sprintf('%s/%s', $indexName, $name); - $this->typeFields[$typeName] = $type['properties']; - } - if (isset($type['_parent'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_parent'] = array('type' => $type['_parent']['type']); - $typeName = sprintf('%s/%s', $indexName, $name); - $this->typeFields[$typeName]['_parent'] = $type['_parent']; - } - if (isset($type['persistence'])) { - $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); - } - if (isset($type['index_analyzer'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['index_analyzer'] = $type['index_analyzer']; - } - if (isset($type['search_analyzer'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['search_analyzer'] = $type['search_analyzer']; - } - if (isset($type['index'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['index'] = $type['index']; - } - if (isset($type['_all'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_all'] = $type['_all']; - } - if (isset($type['_timestamp'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_timestamp'] = $type['_timestamp']; - } - if (isset($type['_ttl'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['_ttl'] = $type['_ttl']; - } - if (!empty($type['dynamic_templates'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'] = array(); - foreach ($type['dynamic_templates'] as $templateName => $templateData) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'][] = array($templateName => $templateData); - } - } + }*/ } } @@ -281,7 +264,7 @@ class FOSElasticaExtension extends Extension * * @return array The merged array */ - static protected function deepArrayUnion($array1, $array2) + private static function deepArrayUnion($array1, $array2) { foreach ($array2 as $key => $value) { if (is_array($value) && isset($array1[$key]) && is_array($array1[$key])) { @@ -303,30 +286,40 @@ class FOSElasticaExtension extends Extension * @param $indexName * @param $typeName */ - protected function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName) + private function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName) { $this->loadDriver($container, $typeConfig['driver']); $elasticaToModelTransformerId = $this->loadElasticaToModelTransformer($typeConfig, $container, $indexName, $typeName); $modelToElasticaTransformerId = $this->loadModelToElasticaTransformer($typeConfig, $container, $indexName, $typeName); - $objectPersisterId = $this->loadObjectPersister($typeConfig, $typeDef, $container, $indexName, $typeName, $modelToElasticaTransformerId); + $objectPersisterId = $this->loadObjectPersister($typeConfig, $typeDef, $container, $indexName, $typeName, $modelToElasticaTransformerId); if (isset($typeConfig['provider'])) { - $this->loadTypeProvider($typeConfig, $container, $objectPersisterId, $typeDef, $indexName, $typeName); + $this->loadTypeProvider($typeConfig, $container, $objectPersisterId, $indexName, $typeName); } if (isset($typeConfig['finder'])) { $this->loadTypeFinder($typeConfig, $container, $elasticaToModelTransformerId, $typeDef, $indexName, $typeName); } if (isset($typeConfig['listener'])) { - $this->loadTypeListener($typeConfig, $container, $objectPersisterId, $typeDef, $indexName, $typeName); + $this->loadTypeListener($typeConfig, $container, $objectPersisterId, $indexName, $typeName); } } - protected function loadElasticaToModelTransformer(array $typeConfig, ContainerBuilder $container, $indexName, $typeName) + /** + * Creates and loads an ElasticaToModelTransformer. + * + * @param array $typeConfig + * @param ContainerBuilder $container + * @param string $indexName + * @param string $typeName + * @return string + */ + private function loadElasticaToModelTransformer(array $typeConfig, ContainerBuilder $container, $indexName, $typeName) { if (isset($typeConfig['elastica_to_model_transformer']['service'])) { return $typeConfig['elastica_to_model_transformer']['service']; } + /* Note: transformer services may conflict with "prototype.driver", if * the index and type names were "prototype" and a driver, respectively. */ @@ -339,28 +332,32 @@ class FOSElasticaExtension extends Extension $argPos = ('propel' === $typeConfig['driver']) ? 0 : 1; $serviceDef->replaceArgument($argPos, $typeConfig['model']); - $serviceDef->replaceArgument($argPos + 1, array( - 'hydrate' => $typeConfig['elastica_to_model_transformer']['hydrate'], - 'identifier' => $typeConfig['identifier'], - 'ignore_missing' => $typeConfig['elastica_to_model_transformer']['ignore_missing'], - 'query_builder_method' => $typeConfig['elastica_to_model_transformer']['query_builder_method'] - )); + $serviceDef->replaceArgument($argPos + 1, array_merge($typeConfig['elastica_to_model_transformer'], array( + 'identifier' => $typeConfig['identifier'], + ))); $container->setDefinition($serviceId, $serviceDef); return $serviceId; } - protected function loadModelToElasticaTransformer(array $typeConfig, ContainerBuilder $container, $indexName, $typeName) + /** + * Creates and loads a ModelToElasticaTransformer for an index/type. + * + * @param array $typeConfig + * @param ContainerBuilder $container + * @param string $indexName + * @param string $typeName + * @return string + */ + private function loadModelToElasticaTransformer(array $typeConfig, ContainerBuilder $container, $indexName, $typeName) { if (isset($typeConfig['model_to_elastica_transformer']['service'])) { return $typeConfig['model_to_elastica_transformer']['service']; } - if ($this->serializerConfig) { - $abstractId = sprintf('fos_elastica.model_to_elastica_identifier_transformer'); - } else { - $abstractId = sprintf('fos_elastica.model_to_elastica_transformer'); - } + $abstractId = $this->serializerConfig ? + 'fos_elastica.model_to_elastica_identifier_transformer' : + 'fos_elastica.model_to_elastica_transformer'; $serviceId = sprintf('fos_elastica.model_to_elastica_transformer.%s.%s', $indexName, $typeName); $serviceDef = new DefinitionDecorator($abstractId); @@ -372,7 +369,18 @@ class FOSElasticaExtension extends Extension return $serviceId; } - protected function loadObjectPersister(array $typeConfig, Definition $typeDef, ContainerBuilder $container, $indexName, $typeName, $transformerId) + /** + * Creates and loads an object persister for a type. + * + * @param array $typeConfig + * @param Definition $typeDef + * @param ContainerBuilder $container + * @param string $indexName + * @param string $typeName + * @param string $transformerId + * @return string + */ + private function loadObjectPersister(array $typeConfig, Definition $typeDef, ContainerBuilder $container, $indexName, $typeName, $transformerId) { $arguments = array( $typeDef, @@ -386,8 +394,9 @@ class FOSElasticaExtension extends Extension $arguments[] = array(new Reference($callbackId), 'serialize'); } else { $abstractId = 'fos_elastica.object_persister'; - $arguments[] = $this->typeFields[sprintf('%s/%s', $indexName, $typeName)]; + $arguments[] = $this->indexConfigs[$indexName]['config']['properties'][$typeName]['properties']; } + $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName); $serviceDef = new DefinitionDecorator($abstractId); foreach ($arguments as $i => $argument) { @@ -399,11 +408,22 @@ class FOSElasticaExtension extends Extension return $serviceId; } - protected function loadTypeProvider(array $typeConfig, ContainerBuilder $container, $objectPersisterId, $typeDef, $indexName, $typeName) + /** + * Loads a provider for a type. + * + * @param array $typeConfig + * @param ContainerBuilder $container + * @param string $objectPersisterId + * @param string $indexName + * @param string $typeName + * @return string + */ + private function loadTypeProvider(array $typeConfig, ContainerBuilder $container, $objectPersisterId, $indexName, $typeName) { if (isset($typeConfig['provider']['service'])) { return $typeConfig['provider']['service']; } + /* Note: provider services may conflict with "prototype.driver", if the * index and type names were "prototype" and a driver, respectively. */ @@ -414,16 +434,28 @@ class FOSElasticaExtension extends Extension $providerDef->replaceArgument(1, $typeConfig['model']); // Propel provider can simply ignore Doctrine-specific options $providerDef->replaceArgument(2, array_diff_key($typeConfig['provider'], array('service' => 1))); + $container->setDefinition($providerId, $providerDef); return $providerId; } - protected function loadTypeListener(array $typeConfig, ContainerBuilder $container, $objectPersisterId, $typeDef, $indexName, $typeName) + /** + * Loads doctrine listeners to handle indexing of new or updated objects. + * + * @param array $typeConfig + * @param ContainerBuilder $container + * @param string $objectPersisterId + * @param string $indexName + * @param string $typeName + * @return string + */ + private function loadTypeListener(array $typeConfig, ContainerBuilder $container, $objectPersisterId, $indexName, $typeName) { if (isset($typeConfig['listener']['service'])) { return $typeConfig['listener']['service']; } + /* Note: listener services may conflict with "prototype.driver", if the * index and type names were "prototype" and a driver, respectively. */ @@ -438,10 +470,6 @@ class FOSElasticaExtension extends Extension $listenerDef->replaceArgument(4, new Reference($typeConfig['listener']['logger'])); } - switch ($typeConfig['driver']) { - case 'orm': $listenerDef->addTag('doctrine.event_subscriber'); break; - case 'mongodb': $listenerDef->addTag('doctrine_mongodb.odm.event_subscriber'); break; - } if (isset($typeConfig['listener']['is_indexable_callback'])) { $callback = $typeConfig['listener']['is_indexable_callback']; @@ -493,7 +521,18 @@ class FOSElasticaExtension extends Extension return $events; } - protected function loadTypeFinder(array $typeConfig, ContainerBuilder $container, $elasticaToModelId, $typeDef, $indexName, $typeName) + /** + * Loads a Type specific Finder. + * + * @param array $typeConfig + * @param ContainerBuilder $container + * @param string $elasticaToModelId + * @param Definition $typeDef + * @param string $indexName + * @param string $typeName + * @return string + */ + private function loadTypeFinder(array $typeConfig, ContainerBuilder $container, $elasticaToModelId, Definition $typeDef, $indexName, $typeName) { if (isset($typeConfig['finder']['service'])) { $finderId = $typeConfig['finder']['service']; @@ -519,39 +558,39 @@ class FOSElasticaExtension extends Extension /** * Loads the index manager * - * @param array $indexRefsByName * @param ContainerBuilder $container **/ - protected function loadIndexManager(array $indexRefsByName, ContainerBuilder $container) + private function loadIndexManager(ContainerBuilder $container) { $managerDef = $container->getDefinition('fos_elastica.index_manager'); - $managerDef->replaceArgument(0, $indexRefsByName); + $managerDef->replaceArgument(0, array_keys($this->clients)); $managerDef->replaceArgument(1, new Reference('fos_elastica.index')); } /** - * Loads the resetter + * Makes sure a specific driver has been loaded. * - * @param array $indexConfigs - * @param \Symfony\Component\DependencyInjection\ContainerBuilder $container + * @param ContainerBuilder $container + * @param string $driver */ - protected function loadResetter(array $indexConfigs, ContainerBuilder $container) - { - $resetterDef = $container->getDefinition('fos_elastica.resetter'); - $resetterDef->replaceArgument(0, $indexConfigs); - } - - protected function loadDriver(ContainerBuilder $container, $driver) + private function loadDriver(ContainerBuilder $container, $driver) { if (in_array($driver, $this->loadedDrivers)) { return; } + $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load($driver.'.xml'); $this->loadedDrivers[] = $driver; } - protected function createDefaultManagerAlias($defaultManager, ContainerBuilder $container) + /** + * Creates a default manager alias for defined default manager or the first loaded driver. + * + * @param string $defaultManager + * @param ContainerBuilder $container + */ + private function createDefaultManagerAlias($defaultManager, ContainerBuilder $container) { if (0 == count($this->loadedDrivers)) { return; @@ -568,18 +607,19 @@ class FOSElasticaExtension extends Extension $container->setAlias('fos_elastica.manager', sprintf('fos_elastica.manager.%s', $defaultManagerService)); } - protected function cleanUpProperties(&$properties) + /** + * Returns a reference to a client given its configured name. + * + * @param string $clientName + * @return Reference + * @throws \InvalidArgumentException + */ + private function getClient($clientName) { - foreach ($properties as &$fieldProperties) { - if (empty($fieldProperties['fields'])) { - unset($fieldProperties['fields']); - } else { - $this->cleanUpProperties($fieldProperties['fields']); - } - - if (!empty($fieldProperties['properties'])) { - $this->cleanUpProperties($fieldProperties['properties']); - } + if (!array_key_exists($clientName, $this->clients)) { + throw new InvalidArgumentException(sprintf('The elastica client with name "%s" is not defined', $clientName)); } + + return $this->clients[$clientName]['reference']; } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index cff2b49..c1ac0f9 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -1,100 +1,34 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - FOS\ElasticaBundle\Elastica\Client - FOS\ElasticaBundle\Elastica\Index - Elastica\Type - FOS\ElasticaBundle\IndexManager - FOS\ElasticaBundle\Resetter - FOS\ElasticaBundle\Finder\TransformedFinder + FOS\ElasticaBundle\Elastica\LoggingClient FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector - FOS\ElasticaBundle\Manager\RepositoryManager - FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerCollection - FOS\ElasticaBundle\Provider\ProviderRegistry Symfony\Component\PropertyAccess\PropertyAccessor - FOS\ElasticaBundle\Persister\ObjectPersister - FOS\ElasticaBundle\Persister\ObjectSerializerPersister - FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer - FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer + + + + + %kernel.debug% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Resources/config/index.xml b/Resources/config/index.xml new file mode 100644 index 0000000..85f5744 --- /dev/null +++ b/Resources/config/index.xml @@ -0,0 +1,39 @@ + + + + + + FOS\ElasticaBundle\Elastica\Index + Elastica\Type + FOS\ElasticaBundle\Index\IndexManager + FOS\ElasticaBundle\Index\Resetter + FOS\ElasticaBundle\Finder\TransformedFinder + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 0af7aa1..0c6b2af 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -5,7 +5,6 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - @@ -18,6 +17,7 @@ + @@ -33,7 +33,5 @@ - - diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 5bd16e5..bf67688 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -1,11 +1,10 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - @@ -19,6 +18,7 @@ + @@ -31,10 +31,8 @@ - - + + - - diff --git a/Resources/config/persister.xml b/Resources/config/persister.xml new file mode 100644 index 0000000..8bd4dca --- /dev/null +++ b/Resources/config/persister.xml @@ -0,0 +1,27 @@ + + + + + + FOS\ElasticaBundle\Persister\ObjectPersister + FOS\ElasticaBundle\Persister\ObjectSerializerPersister + + + + + + + + + + + + + + + + + + diff --git a/Resources/config/propel.xml b/Resources/config/propel.xml index 7a7d93e..4ccc867 100644 --- a/Resources/config/propel.xml +++ b/Resources/config/propel.xml @@ -4,7 +4,6 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - @@ -22,7 +21,5 @@ - - diff --git a/Resources/config/provider.xml b/Resources/config/provider.xml new file mode 100644 index 0000000..0732d54 --- /dev/null +++ b/Resources/config/provider.xml @@ -0,0 +1,18 @@ + + + + + + FOS\ElasticaBundle\Provider\ProviderRegistry + + + + + + + + + + diff --git a/Resources/config/transformer.xml b/Resources/config/transformer.xml new file mode 100644 index 0000000..e3abbbc --- /dev/null +++ b/Resources/config/transformer.xml @@ -0,0 +1,32 @@ + + + + + + FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerCollection + FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer + FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer + + + + + + + + + + + + + + + + + + + + + + diff --git a/Tests/Integration/MappingTest.php b/Tests/Integration/MappingTest.php new file mode 100644 index 0000000..be134ed --- /dev/null +++ b/Tests/Integration/MappingTest.php @@ -0,0 +1,17 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + + +namespace FOS\ElasticaBundle\Tests\Integration; + + +class MappingTest { + +} \ No newline at end of file From 8540f13bbf606ae409ba3c02f8d9243577d249d5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 2 Jun 2014 21:35:51 +1000 Subject: [PATCH 073/154] Fix nested property configuration Fixes #592 --- DependencyInjection/Configuration.php | 10 ++-- .../DependencyInjection/ConfigurationTest.php | 48 ++++++++++++++++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 7a12e95..db8ae6a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -20,7 +20,6 @@ class Configuration implements ConfigurationInterface public function __construct($configArray, $debug) { - $this->configArray = $configArray; $this->debug = $debug; } @@ -386,13 +385,16 @@ class Configuration implements ConfigurationInterface } foreach ($index['types'] as $type) { - if (empty($type['mappings'])) { - continue; + if (array_key_exists('mappings', $type) and !empty($type['mappings'])) { + $nestings = array_merge_recursive($nestings, $this->getNestingsForType($type['mappings'], $nestings)); } - $nestings = array_merge_recursive($nestings, $this->getNestingsForType($type['mappings'], $nestings)); + if (array_key_exists('properties', $type) and !empty($type['properties'])) { + $nestings = array_merge_recursive($nestings, $this->getNestingsForType($type['properties'], $nestings)); + } } } + return $nestings; } diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index efaaa52..8f0e1b9 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -22,7 +22,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase private function getConfigs(array $configArray) { - $configuration = new Configuration($configArray, true); + $configuration = new Configuration(array($configArray), true); return $this->processor->processConfiguration($configuration, array($configArray)); } @@ -257,4 +257,50 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertCount(3, $configuration['indexes']['test']['types']['test']['properties']); } + + public function testNestedProperties() + { + $configuration = $this->getConfigs(array( + 'clients' => array( + 'default' => array('url' => 'http://localhost:9200'), + ), + 'indexes' => array( + 'test' => array( + 'types' => array( + 'user' => array( + 'properties' => array( + 'field1' => array(), + ), + 'persistence' => array(), + ), + 'user_profile' => array( + '_parent' => array( + 'type' => 'user', + 'property' => 'owner', + ), + 'properties' => array( + 'field1' => array(), + 'field2' => array( + 'type' => 'nested', + 'properties' => array( + 'nested_field1' => array( + 'type' => 'integer' + ), + 'nested_field2' => array( + 'type' => 'object', + 'properties' => array( + 'id' => array( + 'type' => 'integer' + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + )); + } } From 366fb3960653392becd4cb63be334a459fe3fd18 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Wed, 4 Jun 2014 17:26:25 +0100 Subject: [PATCH 074/154] fix method call --- Resetter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index d614f1b..9fdcbca 100644 --- a/Resetter.php +++ b/Resetter.php @@ -191,7 +191,7 @@ class Resetter $additionalError = sprintf( 'Tried to delete newly built index %s, but also failed: %s', $newIndexName, - $deleteNewIndexException->getError() + $deleteNewIndexException->getMessage() ); } From 5009673b6a7b3bff3266b4766fe4e892e7a8b1a4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 8 Jun 2014 22:35:38 +1000 Subject: [PATCH 075/154] Functional Tests v1 --- .travis.yml | 3 + Tests/Functional/MappingToElasticaTest.php | 52 +++++++++ Tests/Functional/WebTestCase.php | 40 +++++++ Tests/Functional/app/AppKernel.php | 118 +++++++++++++++++++++ Tests/Functional/app/Basic/bundles.php | 10 ++ Tests/Functional/app/Basic/config.yml | 53 +++++++++ Tests/Functional/app/config/config.yml | 8 ++ composer.json | 13 +-- 8 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 Tests/Functional/MappingToElasticaTest.php create mode 100644 Tests/Functional/WebTestCase.php create mode 100644 Tests/Functional/app/AppKernel.php create mode 100644 Tests/Functional/app/Basic/bundles.php create mode 100644 Tests/Functional/app/Basic/config.yml create mode 100644 Tests/Functional/app/config/config.yml diff --git a/.travis.yml b/.travis.yml index 72632de..4fdf2b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,6 @@ php: before_script: - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - composer install --dev --prefer-source + +services: + - elasticsearch \ No newline at end of file diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php new file mode 100644 index 0000000..3c32e10 --- /dev/null +++ b/Tests/Functional/MappingToElasticaTest.php @@ -0,0 +1,52 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +use Symfony\Bundle\FrameworkBundle\Client; + +/** + * @group functional + */ +class MappingToElasticaTest extends WebTestCase +{ + public function testReset() + { + $client = $this->createClient(array('test_case' => 'Basic')); + $resetter = $this->getResetter($client); + + $resetter->resetIndex('index'); + $resetter->resetIndexType('index', 'type'); + } + + /** + * @param Client $client + * @return \FOS\ElasticaBundle\Resetter $resetter + */ + private function getResetter(Client $client) + { + return $client->getContainer()->get('fos_elastica.resetter'); + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('Basic'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('Basic'); + } +} diff --git a/Tests/Functional/WebTestCase.php b/Tests/Functional/WebTestCase.php new file mode 100644 index 0000000..38f5489 --- /dev/null +++ b/Tests/Functional/WebTestCase.php @@ -0,0 +1,40 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +use Symfony\Bundle\FrameworkBundle\Tests\Functional\WebTestCase as BaseWebTestCase; + +class WebTestCase extends BaseWebTestCase +{ + protected static function getKernelClass() + { + require_once __DIR__.'/app/AppKernel.php'; + + return 'FOS\ElasticaBundle\Tests\Functional\app\AppKernel'; + } + + protected static function createKernel(array $options = array()) + { + $class = self::getKernelClass(); + + if (!isset($options['test_case'])) { + throw new \InvalidArgumentException('The option "test_case" must be set.'); + } + + return new $class( + $options['test_case'], + isset($options['root_config']) ? $options['root_config'] : 'config.yml', + isset($options['environment']) ? $options['environment'] : 'foselasticabundle'.strtolower($options['test_case']), + isset($options['debug']) ? $options['debug'] : true + ); + } +} diff --git a/Tests/Functional/app/AppKernel.php b/Tests/Functional/app/AppKernel.php new file mode 100644 index 0000000..f47a5b3 --- /dev/null +++ b/Tests/Functional/app/AppKernel.php @@ -0,0 +1,118 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Tests\Functional\app; + +// get the autoload file +$dir = __DIR__; +$lastDir = null; +while ($dir !== $lastDir) { + $lastDir = $dir; + + if (file_exists($dir.'/autoload.php')) { + require_once $dir.'/autoload.php'; + break; + } + + if (file_exists($dir.'/autoload.php.dist')) { + require_once $dir.'/autoload.php.dist'; + break; + } + + if (file_exists($dir.'/vendor/autoload.php')) { + require_once $dir.'/vendor/autoload.php'; + break; + } + + $dir = dirname($dir); +} + +use Symfony\Component\Config\Loader\LoaderInterface; +use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\HttpKernel\Kernel; + +/** + * App Test Kernel for functional tests. + * + * @author Johannes M. Schmitt + */ +class AppKernel extends Kernel +{ + private $testCase; + private $rootConfig; + + public function __construct($testCase, $rootConfig, $environment, $debug) + { + if (!is_dir(__DIR__.'/'.$testCase)) { + throw new \InvalidArgumentException(sprintf('The test case "%s" does not exist.', $testCase)); + } + $this->testCase = $testCase; + + $fs = new Filesystem(); + if (!$fs->isAbsolutePath($rootConfig) && !file_exists($rootConfig = __DIR__.'/'.$testCase.'/'.$rootConfig)) { + throw new \InvalidArgumentException(sprintf('The root config "%s" does not exist.', $rootConfig)); + } + $this->rootConfig = $rootConfig; + + parent::__construct($environment, $debug); + } + + public function registerBundles() + { + if (!file_exists($filename = $this->getRootDir().'/'.$this->testCase.'/bundles.php')) { + throw new \RuntimeException(sprintf('The bundles file "%s" does not exist.', $filename)); + } + + return include $filename; + } + + public function init() + { + } + + public function getRootDir() + { + return __DIR__; + } + + public function getCacheDir() + { + return sys_get_temp_dir().'/'.Kernel::VERSION.'/'.$this->testCase.'/cache/'.$this->environment; + } + + public function getLogDir() + { + return sys_get_temp_dir().'/'.Kernel::VERSION.'/'.$this->testCase.'/logs'; + } + + public function registerContainerConfiguration(LoaderInterface $loader) + { + $loader->load($this->rootConfig); + } + + public function serialize() + { + return serialize(array($this->testCase, $this->rootConfig, $this->getEnvironment(), $this->isDebug())); + } + + public function unserialize($str) + { + call_user_func_array(array($this, '__construct'), unserialize($str)); + } + + protected function getKernelParameters() + { + $parameters = parent::getKernelParameters(); + $parameters['kernel.test_case'] = $this->testCase; + + return $parameters; + } +} \ No newline at end of file diff --git a/Tests/Functional/app/Basic/bundles.php b/Tests/Functional/app/Basic/bundles.php new file mode 100644 index 0000000..9f23bdf --- /dev/null +++ b/Tests/Functional/app/Basic/bundles.php @@ -0,0 +1,10 @@ +=2.2,<2.5-dev", + "doctrine/orm": "~2.2", "doctrine/mongodb-odm": "1.0.*@dev", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", - "knplabs/knp-components": "1.2.*", - "symfony/expression-language" : "2.4.*@dev" + "knplabs/knp-components": "~1.2", + "symfony/browser-kit" : "~2.3", + "symfony/expression-language" : "~2.4" }, "suggest": { - "doctrine/orm": ">=2.2,<2.5-dev", + "doctrine/orm": "~2.2", "doctrine/mongodb-odm": "1.0.*@dev", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", - "knplabs/knp-components": "1.2.*", - "symfony/expression-language" : "2.4.*@dev" + "knplabs/knp-components": "~1.2", + "symfony/expression-language" : "~2.4" }, "autoload": { "psr-0": { "FOS\\ElasticaBundle": "" } From fe19df365a74373c19161449526d35beb708e49b Mon Sep 17 00:00:00 2001 From: Gilles Doge Date: Tue, 10 Jun 2014 18:18:30 +0200 Subject: [PATCH 076/154] Make the class of fos_elastica.paginator.subscriber service configurable --- Resources/config/config.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 7687250..2b48e87 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -21,6 +21,7 @@ FOS\ElasticaBundle\Persister\ObjectSerializerPersister FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer + FOS\ElasticaBundle\Subscriber\PaginateElasticaQuerySubscriber @@ -87,7 +88,7 @@ - + From 66d241099984810e26ae809629b23e7dcade116f Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 15:57:27 +1000 Subject: [PATCH 077/154] Move Indexable callback calculations to a new service --- DependencyInjection/Configuration.php | 17 +- DependencyInjection/FOSElasticaExtension.php | 24 ++- Doctrine/Listener.php | 172 +++++------------- Persister/ObjectPersister.php | 21 +++ Provider/Indexable.php | 174 +++++++++++++++++++ Provider/IndexableInterface.php | 25 +++ Resources/config/config.xml | 5 + Resources/config/mongodb.xml | 3 +- Resources/config/orm.xml | 6 +- Tests/Doctrine/AbstractListenerTest.php | 142 +++++++-------- Tests/Functional/IndexableCallbackTest.php | 51 ++++++ Tests/Functional/TypeObj.php | 25 +++ Tests/Functional/app/Basic/config.yml | 2 +- Tests/Functional/app/ORM/bundles.php | 11 ++ Tests/Functional/app/ORM/config.yml | 44 +++++ Tests/Provider/IndexableTest.php | 91 ++++++++++ composer.json | 1 + 17 files changed, 584 insertions(+), 230 deletions(-) create mode 100644 Provider/Indexable.php create mode 100644 Provider/IndexableInterface.php create mode 100644 Tests/Functional/IndexableCallbackTest.php create mode 100644 Tests/Functional/TypeObj.php create mode 100644 Tests/Functional/app/ORM/bundles.php create mode 100644 Tests/Functional/app/ORM/config.yml create mode 100644 Tests/Provider/IndexableTest.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index ce1c982..f32f295 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -187,9 +187,23 @@ class Configuration implements ConfigurationInterface return $v; }) ->end() + ->beforeNormalization() + ->ifTrue(function ($v) { + return isset($v['persistence']) && + isset($v['persistence']['listener']) && + isset($v['persistence']['listener']['is_indexable_callback']); + }) + ->then(function ($v) { + $v['indexable_callback'] = $v['persistence']['listener']['is_indexable_callback']; + unset($v['persistence']['listener']['is_indexable_callback']); + + return $v; + }) + ->end() ->children() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() + ->scalarNode('indexable_callback')->end() ->append($this->getPersistenceNode()) ->append($this->getSerializerNode()) ->end() @@ -230,7 +244,7 @@ class Configuration implements ConfigurationInterface unset($v[$prop]); } } - + return $v; }) ->end() @@ -674,7 +688,6 @@ class Configuration implements ConfigurationInterface ->treatTrueLike('fos_elastica.logger') ->end() ->scalarNode('service')->end() - ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() ->end() ->arrayNode('finder') diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 1529544..9e50c0b 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -186,8 +186,11 @@ class FOSElasticaExtension extends Extension */ protected function loadTypes(array $types, ContainerBuilder $container, $indexName, $indexId, array $typePrototypeConfig) { + $indexableCallbacks = array(); + foreach ($types as $name => $type) { $type = self::deepArrayUnion($typePrototypeConfig, $type); + $typeName = sprintf('%s/%s', $indexName, $name); $typeId = sprintf('%s.%s', $indexId, $name); $typeDefArgs = array($name); $typeDef = new Definition('%fos_elastica.type.class%', $typeDefArgs); @@ -240,7 +243,6 @@ class FOSElasticaExtension extends Extension } if (isset($type['_parent'])) { $this->indexConfigs[$indexName]['config']['properties'][$name]['_parent'] = array('type' => $type['_parent']['type']); - $typeName = sprintf('%s/%s', $indexName, $name); $this->typeFields[$typeName]['_parent'] = $type['_parent']; } if (isset($type['persistence'])) { @@ -252,6 +254,9 @@ class FOSElasticaExtension extends Extension if (isset($type['search_analyzer'])) { $this->indexConfigs[$indexName]['config']['properties'][$name]['search_analyzer'] = $type['search_analyzer']; } + if (isset($type['indexable_callback'])) { + $indexableCallbacks[$typeName] = $type['indexable_callback']; + } if (isset($type['index'])) { $this->indexConfigs[$indexName]['config']['properties'][$name]['index'] = $type['index']; } @@ -271,6 +276,9 @@ class FOSElasticaExtension extends Extension } } } + + $indexable = $container->getDefinition('fos_elastica.indexable'); + $indexable->replaceArgument(0, $indexableCallbacks); } /** @@ -431,8 +439,7 @@ class FOSElasticaExtension extends Extension $listenerId = sprintf('fos_elastica.listener.%s.%s', $indexName, $typeName); $listenerDef = new DefinitionDecorator($abstractListenerId); $listenerDef->replaceArgument(0, new Reference($objectPersisterId)); - $listenerDef->replaceArgument(1, $typeConfig['model']); - $listenerDef->replaceArgument(2, $this->getDoctrineEvents($typeConfig)); + $listenerDef->replaceArgument(1, $this->getDoctrineEvents($typeConfig)); $listenerDef->replaceArgument(3, $typeConfig['identifier']); if ($typeConfig['listener']['logger']) { $listenerDef->replaceArgument(4, new Reference($typeConfig['listener']['logger'])); @@ -442,18 +449,7 @@ class FOSElasticaExtension extends Extension case 'orm': $listenerDef->addTag('doctrine.event_subscriber'); break; case 'mongodb': $listenerDef->addTag('doctrine_mongodb.odm.event_subscriber'); break; } - if (isset($typeConfig['listener']['is_indexable_callback'])) { - $callback = $typeConfig['listener']['is_indexable_callback']; - if (is_array($callback)) { - list($class) = $callback + array(null); - if (is_string($class) && !class_exists($class)) { - $callback[0] = new Reference($class); - } - } - - $listenerDef->addMethodCall('setIsIndexableCallback', array($callback)); - } $container->setDefinition($listenerId, $listenerDef); return $listenerId; diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index ff9fc60..4a01aa1 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -2,15 +2,13 @@ namespace FOS\ElasticaBundle\Doctrine; -use Psr\Log\LoggerInterface; use Doctrine\Common\EventArgs; use Doctrine\Common\EventSubscriber; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Persister\ObjectPersister; -use Symfony\Component\ExpressionLanguage\Expression; -use Symfony\Component\ExpressionLanguage\ExpressionLanguage; -use Symfony\Component\ExpressionLanguage\SyntaxError; +use FOS\ElasticaBundle\Provider\IndexableInterface; use Symfony\Component\PropertyAccess\PropertyAccess; +use Symfony\Component\PropertyAccess\PropertyAccessorInterface; /** * Automatically update ElasticSearch based on changes to the Doctrine source @@ -25,13 +23,6 @@ class Listener implements EventSubscriber */ protected $objectPersister; - /** - * Class of the domain model - * - * @var string - */ - protected $objectClass; - /** * List of subscribed events * @@ -46,13 +37,6 @@ class Listener implements EventSubscriber */ protected $esIdentifierField; - /** - * Callback for determining if an object should be indexed - * - * @var mixed - */ - protected $isIndexableCallback; - /** * Objects scheduled for insertion and replacement */ @@ -64,13 +48,6 @@ class Listener implements EventSubscriber */ public $scheduledForDeletion = array(); - /** - * An instance of ExpressionLanguage - * - * @var ExpressionLanguage - */ - protected $expressionLanguage; - /** * PropertyAccessor instance * @@ -78,26 +55,36 @@ class Listener implements EventSubscriber */ protected $propertyAccessor; + /** + * @var \FOS\ElasticaBundle\Provider\IndexableInterface + */ + private $indexable; + /** * Constructor. * * @param ObjectPersisterInterface $objectPersister - * @param string $objectClass - * @param array $events - * @param string $esIdentifierField + * @param array $events + * @param IndexableInterface $indexable + * @param string $esIdentifierField + * @param null $logger */ - public function __construct(ObjectPersisterInterface $objectPersister, $objectClass, array $events, $esIdentifierField = 'id', $logger = null) - { - $this->objectPersister = $objectPersister; - $this->objectClass = $objectClass; - $this->events = $events; - $this->esIdentifierField = $esIdentifierField; + public function __construct( + ObjectPersisterInterface $objectPersister, + array $events, + IndexableInterface $indexable, + $esIdentifierField = 'id', + $logger = null + ) { + $this->esIdentifierField = $esIdentifierField; + $this->events = $events; + $this->indexable = $indexable; + $this->objectPersister = $objectPersister; + $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); if ($logger) { $this->objectPersister->setLogger($logger); } - - $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); } /** @@ -108,82 +95,6 @@ class Listener implements EventSubscriber return $this->events; } - /** - * Set the callback for determining object index eligibility. - * - * If callback is a string, it must be public method on the object class - * that expects no arguments and returns a boolean. Otherwise, the callback - * should expect the object for consideration as its only argument and - * return a boolean. - * - * @param callback $callback - * @throws \RuntimeException if the callback is not callable - */ - public function setIsIndexableCallback($callback) - { - if (is_string($callback)) { - if (!is_callable(array($this->objectClass, $callback))) { - if (false !== ($expression = $this->getExpressionLanguage())) { - $callback = new Expression($callback); - try { - $expression->compile($callback, array($this->getExpressionVar())); - } catch (SyntaxError $e) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e); - } - } else { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback)); - } - } - } elseif (!is_callable($callback)) { - if (is_array($callback)) { - list($class, $method) = $callback + array(null, null); - if (is_object($class)) { - $class = get_class($class); - } - - if ($class && $method) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method)); - } - } - throw new \RuntimeException('Indexable callback is not callable.'); - } - - $this->isIndexableCallback = $callback; - } - - /** - * Return whether the object is indexable with respect to the callback. - * - * @param object $object - * @return boolean - */ - protected function isObjectIndexable($object) - { - if (!$this->isIndexableCallback) { - return true; - } - - if ($this->isIndexableCallback instanceof Expression) { - return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object)); - } - - return is_string($this->isIndexableCallback) - ? call_user_func(array($object, $this->isIndexableCallback)) - : call_user_func($this->isIndexableCallback, $object); - } - - /** - * @param mixed $object - * @return string - */ - private function getExpressionVar($object = null) - { - $class = $object ?: $this->objectClass; - $ref = new \ReflectionClass($class); - - return strtolower($ref->getShortName()); - } - /** * Provides unified method for retrieving a doctrine object from an EventArgs instance * @@ -204,27 +115,11 @@ class Listener implements EventSubscriber throw new \RuntimeException('Unable to retrieve object from EventArgs.'); } - /** - * @return bool|ExpressionLanguage - */ - private function getExpressionLanguage() - { - if (null === $this->expressionLanguage) { - if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { - return false; - } - - $this->expressionLanguage = new ExpressionLanguage(); - } - - return $this->expressionLanguage; - } - public function postPersist(EventArgs $eventArgs) { $entity = $this->getDoctrineObject($eventArgs); - if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { + if ($this->objectPersister->handlesObject($entity) && $this->isObjectIndexable($entity)) { $this->scheduledForInsertion[] = $entity; } } @@ -233,7 +128,7 @@ class Listener implements EventSubscriber { $entity = $this->getDoctrineObject($eventArgs); - if ($entity instanceof $this->objectClass) { + if ($this->objectPersister->handlesObject($entity)) { if ($this->isObjectIndexable($entity)) { $this->scheduledForUpdate[] = $entity; } else { @@ -251,7 +146,7 @@ class Listener implements EventSubscriber { $entity = $this->getDoctrineObject($eventArgs); - if ($entity instanceof $this->objectClass) { + if ($this->objectPersister->handlesObject($entity)) { $this->scheduleForDeletion($entity); } } @@ -305,4 +200,19 @@ class Listener implements EventSubscriber $this->scheduledForDeletion[] = $identifierValue; } } + + /** + * Checks if the object is indexable or not. + * + * @param object $object + * @return bool + */ + private function isObjectIndexable($object) + { + return $this->indexable->isObjectIndexable( + $this->objectPersister->getType()->getIndex()->getName(), + $this->objectPersister->getType()->getName(), + $object + ); + } } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index c279ec7..2b6a8af 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -31,6 +31,27 @@ class ObjectPersister implements ObjectPersisterInterface $this->fields = $fields; } + /** + * @internal Temporary method that will be removed. + * + * @return Type + */ + public function getType() + { + return $this->type; + } + + /** + * If the ObjectPersister handles a given object. + * + * @param object $object + * @return bool + */ + public function handlesObject($object) + { + return $object instanceof $this->objectClass; + } + public function setLogger(LoggerInterface $logger) { $this->logger = $logger; diff --git a/Provider/Indexable.php b/Provider/Indexable.php new file mode 100644 index 0000000..b388c58 --- /dev/null +++ b/Provider/Indexable.php @@ -0,0 +1,174 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Provider; + +use Symfony\Component\ExpressionLanguage\Expression; +use Symfony\Component\ExpressionLanguage\ExpressionLanguage; +use Symfony\Component\ExpressionLanguage\SyntaxError; +use Symfony\Component\PropertyAccess\PropertyAccess; +use Symfony\Component\PropertyAccess\PropertyAccessorInterface; + +class Indexable implements IndexableInterface +{ + /** + * An array of raw configured callbacks for all types. + * + * @var array + */ + private $callbacks = array(); + + /** + * An instance of ExpressionLanguage + * + * @var ExpressionLanguage + */ + private $expressionLanguage; + + /** + * An array of initialised callbacks. + * + * @var array + */ + private $initialisedCallbacks = array(); + + /** + * PropertyAccessor instance + * + * @var PropertyAccessorInterface + */ + private $propertyAccessor; + + /** + * @param array $callbacks + */ + public function __construct(array $callbacks) + { + $this->callbacks = $callbacks; + $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); + } + + /** + * Return whether the object is indexable with respect to the callback. + * + * @param string $indexName + * @param string $typeName + * @param mixed $object + * @return bool + */ + public function isObjectIndexable($indexName, $typeName, $object) + { + $type = sprintf('%s/%s', $indexName, $typeName); + $callback = $this->getCallback($type, $object); + if (!$callback) { + return true; + } + + if ($callback instanceof Expression) { + return $this->getExpressionLanguage()->evaluate($callback, array( + 'object' => $object, + $this->getExpressionVar($object) => $object + )); + } + + return is_string($callback) + ? call_user_func(array($object, $callback)) + : call_user_func($callback, $object); + } + + /** + * Builds and initialises a callback. + * + * @param string $type + * @param object $object + * @return mixed + */ + private function buildCallback($type, $object) + { + if (!array_key_exists($type, $this->callbacks)) { + throw new \InvalidArgumentException(sprintf('Callback for type "%s" is not configured', $type)); + } + + $callback = $this->callbacks[$type]; + + if (is_callable($callback) or is_callable(array($object, $callback))) { + return $callback; + } + + if (is_array($callback)) { + list($class, $method) = $callback + array(null, null); + if (is_object($class)) { + $class = get_class($class); + } + + if ($class && $method) { + throw new \InvalidArgumentException(sprintf('Callback for type "%s", "%s::%s()", is not callable.', $type, $class, $method)); + } + } + + if (is_string($callback) && $expression = $this->getExpressionLanguage()) { + $callback = new Expression($callback); + + try { + $expression->compile($callback, array('object', $this->getExpressionVar($object))); + + return $callback; + } catch (SyntaxError $e) { + throw new \InvalidArgumentException(sprintf('Callback for type "%s" is an invalid expression', $type), $e->getCode(), $e); + } + } + + throw new \InvalidArgumentException(sprintf('Callback for type "%s" is not a valid callback.', $type)); + } + + /** + * Retreives a cached callback, or creates a new callback if one is not found. + * + * @param string $type + * @param object $object + * @return mixed + */ + private function getCallback($type, $object) + { + if (!array_key_exists($type, $this->initialisedCallbacks)) { + $this->initialisedCallbacks[$type] = $this->buildCallback($type, $object); + } + + return $this->initialisedCallbacks[$type]; + } + + /** + * @return bool|ExpressionLanguage + */ + private function getExpressionLanguage() + { + if (null === $this->expressionLanguage) { + if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { + return false; + } + + $this->expressionLanguage = new ExpressionLanguage(); + } + + return $this->expressionLanguage; + } + + /** + * @param mixed $object + * @return string + */ + private function getExpressionVar($object = null) + { + $ref = new \ReflectionClass($object); + + return strtolower($ref->getShortName()); + } +} diff --git a/Provider/IndexableInterface.php b/Provider/IndexableInterface.php new file mode 100644 index 0000000..4871b58 --- /dev/null +++ b/Provider/IndexableInterface.php @@ -0,0 +1,25 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Provider; + +interface IndexableInterface +{ + /** + * Checks if an object passed should be indexable or not. + * + * @param string $indexName + * @param string $typeName + * @param mixed $object + * @return bool + */ + public function isObjectIndexable($indexName, $typeName, $object); +} diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 7687250..f4b2606 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -9,6 +9,7 @@ FOS\ElasticaBundle\DynamicIndex Elastica\Type FOS\ElasticaBundle\IndexManager + FOS\ElasticaBundle\Provider\Indexable FOS\ElasticaBundle\Resetter FOS\ElasticaBundle\Finder\TransformedFinder FOS\ElasticaBundle\Logger\ElasticaLogger @@ -44,6 +45,10 @@ + + + + diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 0af7aa1..e575e5d 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -15,9 +15,10 @@ - + + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 5bd16e5..43d1670 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -15,10 +15,10 @@ - - - + + + diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index ee657f1..de5ba0c 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -11,12 +11,12 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase { public function testObjectInsertedOnPersist() { - $persister = $this->getMockPersister(); - $entity = new Listener\Entity(1); + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $indexable = $this->getMockIndexable('index', 'type', $entity, true); - $listener = $this->createListener($persister, get_class($entity), array()); + $listener = $this->createListener($persister, array(), $indexable); $listener->postPersist($eventArgs); $this->assertEquals($entity, current($listener->scheduledForInsertion)); @@ -28,18 +28,14 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener->postFlush($eventArgs); } - /** - * @dataProvider provideIsIndexableCallbacks - */ - public function testNonIndexableObjectNotInsertedOnPersist($isIndexableCallback) + public function testNonIndexableObjectNotInsertedOnPersist() { - $persister = $this->getMockPersister(); - - $entity = new Listener\Entity(1, false); + $entity = new Listener\Entity(1); + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $indexable = $this->getMockIndexable('index', 'type', $entity, false); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->setIsIndexableCallback($isIndexableCallback); + $listener = $this->createListener($persister, array(), $indexable); $listener->postPersist($eventArgs); $this->assertEmpty($listener->scheduledForInsertion); @@ -54,12 +50,12 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase public function testObjectReplacedOnUpdate() { - $persister = $this->getMockPersister(); - $entity = new Listener\Entity(1); + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $indexable = $this->getMockIndexable('index', 'type', $entity, true); - $listener = $this->createListener($persister, get_class($entity), array()); + $listener = $this->createListener($persister, array(), $indexable); $listener->postUpdate($eventArgs); $this->assertEquals($entity, current($listener->scheduledForUpdate)); @@ -73,17 +69,15 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener->postFlush($eventArgs); } - /** - * @dataProvider provideIsIndexableCallbacks - */ - public function testNonIndexableObjectRemovedOnUpdate($isIndexableCallback) + public function testNonIndexableObjectRemovedOnUpdate() { $classMetadata = $this->getMockClassMetadata(); $objectManager = $this->getMockObjectManager(); - $persister = $this->getMockPersister(); - $entity = new Listener\Entity(1, false); + $entity = new Listener\Entity(1); + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $objectManager); + $indexable = $this->getMockIndexable('index', 'type', $entity, false); $objectManager->expects($this->any()) ->method('getClassMetadata') @@ -95,8 +89,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->setIsIndexableCallback($isIndexableCallback); + $listener = $this->createListener($persister, array(), $indexable); $listener->postUpdate($eventArgs); $this->assertEmpty($listener->scheduledForUpdate); @@ -115,10 +108,11 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase { $classMetadata = $this->getMockClassMetadata(); $objectManager = $this->getMockObjectManager(); - $persister = $this->getMockPersister(); $entity = new Listener\Entity(1); + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $objectManager); + $indexable = $this->getMockIndexable('index', 'type', $entity); $objectManager->expects($this->any()) ->method('getClassMetadata') @@ -130,7 +124,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, get_class($entity), array()); + $listener = $this->createListener($persister, array(), $indexable); $listener->preRemove($eventArgs); $this->assertEquals($entity->getId(), current($listener->scheduledForDeletion)); @@ -146,11 +140,12 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase { $classMetadata = $this->getMockClassMetadata(); $objectManager = $this->getMockObjectManager(); - $persister = $this->getMockPersister(); $entity = new Listener\Entity(1); $entity->identifier = 'foo'; + $persister = $this->getMockPersister($entity, 'index', 'type'); $eventArgs = $this->createLifecycleEventArgs($entity, $objectManager); + $indexable = $this->getMockIndexable('index', 'type', $entity); $objectManager->expects($this->any()) ->method('getClassMetadata') @@ -162,7 +157,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'identifier') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, get_class($entity), array(), 'identifier'); + $listener = $this->createListener($persister, array(), $indexable, 'identifier'); $listener->preRemove($eventArgs); $this->assertEquals($entity->identifier, current($listener->scheduledForDeletion)); @@ -174,36 +169,6 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $listener->postFlush($eventArgs); } - /** - * @dataProvider provideInvalidIsIndexableCallbacks - * @expectedException \RuntimeException - */ - public function testInvalidIsIndexableCallbacks($isIndexableCallback) - { - $listener = $this->createListener($this->getMockPersister(), 'FOS\ElasticaBundle\Tests\Doctrine\Listener\Entity', array()); - $listener->setIsIndexableCallback($isIndexableCallback); - } - - public function provideInvalidIsIndexableCallbacks() - { - return array( - array('nonexistentEntityMethod'), - array(array(new Listener\IndexableDecider(), 'internalMethod')), - array(42), - array('entity.getIsIndexable() && nonexistentEntityFunction()'), - ); - } - - public function provideIsIndexableCallbacks() - { - return array( - array('getIsIndexable'), - array(array(new Listener\IndexableDecider(), 'isIndexable')), - array(function(Listener\Entity $entity) { return $entity->getIsIndexable(); }), - array('entity.getIsIndexable()') - ); - } - abstract protected function getLifecycleEventArgsClass(); abstract protected function getListenerClass(); @@ -240,9 +205,48 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->getMock(); } - private function getMockPersister() + private function getMockPersister($object, $indexName, $typeName) { - return $this->getMock('FOS\ElasticaBundle\Persister\ObjectPersisterInterface'); + $mock = $this->getMockBuilder('FOS\ElasticaBundle\Persister\ObjectPersister') + ->disableOriginalConstructor() + ->getMock(); + + $mock->expects($this->any()) + ->method('handlesObject') + ->with($object) + ->will($this->returnValue(true)); + + $index = $this->getMockBuilder('Elastica\Index')->disableOriginalConstructor()->getMock(); + $index->expects($this->any()) + ->method('getName') + ->will($this->returnValue($indexName)); + $type = $this->getMockBuilder('Elastica\Type')->disableOriginalConstructor()->getMock(); + $type->expects($this->any()) + ->method('getName') + ->will($this->returnValue($typeName)); + $type->expects($this->any()) + ->method('getIndex') + ->will($this->returnValue($index)); + + $mock->expects($this->any()) + ->method('getType') + ->will($this->returnValue($type)); + + return $mock; + } + + private function getMockIndexable($indexName, $typeName, $object, $return = null) + { + $mock = $this->getMock('FOS\ElasticaBundle\Provider\IndexableInterface'); + + if (null !== $return) { + $mock->expects($this->once()) + ->method('isObjectIndexable') + ->with($indexName, $typeName, $object) + ->will($this->returnValue($return)); + } + + return $mock; } } @@ -251,33 +255,15 @@ namespace FOS\ElasticaBundle\Tests\Doctrine\Listener; class Entity { private $id; - private $isIndexable; - public function __construct($id, $isIndexable = true) + public function __construct($id) { $this->id = $id; - $this->isIndexable = $isIndexable; } public function getId() { return $this->id; } - - public function getIsIndexable() - { - return $this->isIndexable; - } } -class IndexableDecider -{ - public function isIndexable(Entity $entity) - { - return $entity->getIsIndexable(); - } - - protected function internalMethod() - { - } -} diff --git a/Tests/Functional/IndexableCallbackTest.php b/Tests/Functional/IndexableCallbackTest.php new file mode 100644 index 0000000..89fca1d --- /dev/null +++ b/Tests/Functional/IndexableCallbackTest.php @@ -0,0 +1,51 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +/** + * @group functional + */ +class IndexableCallbackTest extends WebTestCase +{ + /** + * 2 reasons for this test: + * + * 1) To test that the configuration rename from is_indexable_callback under the listener + * key is respected, and + * 2) To test the Extension's set up of the Indexable service. + */ + public function testIndexableCallback() + { + $client = $this->createClient(array('test_case' => 'ORM')); + + /** @var \FOS\ElasticaBundle\Provider\Indexable $in */ + $in = $client->getContainer()->get('fos_elastica.indexable'); + + $this->assertTrue($in->isObjectIndexable('index', 'type', new TypeObj())); + $this->assertFalse($in->isObjectIndexable('index', 'type2', new TypeObj())); + $this->assertFalse($in->isObjectIndexable('index', 'type3', new TypeObj())); + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('ORM'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('ORM'); + } +} diff --git a/Tests/Functional/TypeObj.php b/Tests/Functional/TypeObj.php new file mode 100644 index 0000000..c264e7b --- /dev/null +++ b/Tests/Functional/TypeObj.php @@ -0,0 +1,25 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +class TypeObj +{ + public function isIndexable() + { + return true; + } + + public function isntIndexable() + { + return false; + } +} diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 7025532..09e5aec 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -50,4 +50,4 @@ fos_elastica: _parent: type: "parent" property: "parent" - identifier: "id" \ No newline at end of file + identifier: "id" diff --git a/Tests/Functional/app/ORM/bundles.php b/Tests/Functional/app/ORM/bundles.php new file mode 100644 index 0000000..d0b6efb --- /dev/null +++ b/Tests/Functional/app/ORM/bundles.php @@ -0,0 +1,11 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Tests\Provider; + +use FOS\ElasticaBundle\Provider\Indexable; + +class IndexableTest extends \PHPUnit_Framework_TestCase +{ + /** + * @dataProvider provideIsIndexableCallbacks + */ + public function testValidIndexableCallbacks($callback, $return) + { + $indexable = new Indexable(array( + 'index/type' => $callback + )); + $index = $indexable->isObjectIndexable('index', 'type', new Entity); + + $this->assertEquals($return, $index); + } + + /** + * @dataProvider provideInvalidIsIndexableCallbacks + * @expectedException \InvalidArgumentException + */ + public function testInvalidIsIndexableCallbacks($callback) + { + $indexable = new Indexable(array( + 'index/type' => $callback + )); + $indexable->isObjectIndexable('index', 'type', new Entity); + } + + public function provideInvalidIsIndexableCallbacks() + { + return array( + array('nonexistentEntityMethod'), + array(array(new IndexableDecider(), 'internalMethod')), + array(42), + array('entity.getIsIndexable() && nonexistentEntityFunction()'), + ); + } + + public function provideIsIndexableCallbacks() + { + return array( + array('isIndexable', false), + array(array(new IndexableDecider(), 'isIndexable'), true), + array(function(Entity $entity) { return $entity->maybeIndex(); }, true), + array('entity.maybeIndex()', true), + array('!object.isIndexable() && entity.property == "abc"', true), + array('entity.property != "abc"', false), + ); + } +} + +class Entity +{ + public $property = 'abc'; + + public function isIndexable() + { + return false; + } + + public function maybeIndex() + { + return true; + } +} + +class IndexableDecider +{ + public function isIndexable(Entity $entity) + { + return !$entity->isIndexable(); + } + + protected function internalMethod() + { + } +} diff --git a/composer.json b/composer.json index 8783822..d67e329 100644 --- a/composer.json +++ b/composer.json @@ -21,6 +21,7 @@ }, "require-dev":{ "doctrine/orm": "~2.2", + "doctrine/doctrine-bundle": "~1.2@beta", "doctrine/mongodb-odm": "1.0.*@dev", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", From e54cc3c2434cebc341f06d96596f11f14052094c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 16:17:04 +1000 Subject: [PATCH 078/154] Implement callback checking in the provider --- Doctrine/AbstractProvider.php | 24 +++++-- Persister/ObjectPersisterInterface.php | 7 ++ Provider/AbstractProvider.php | 35 ++++++++-- Tests/Doctrine/AbstractProviderTest.php | 88 ++++++++++++++++++++----- 4 files changed, 126 insertions(+), 28 deletions(-) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index b9ffda5..0c43b2d 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -6,6 +6,7 @@ use Doctrine\Common\Persistence\ManagerRegistry; use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Provider\AbstractProvider as BaseAbstractProvider; +use FOS\ElasticaBundle\Provider\IndexableInterface; abstract class AbstractProvider extends BaseAbstractProvider { @@ -15,13 +16,19 @@ abstract class AbstractProvider extends BaseAbstractProvider * Constructor. * * @param ObjectPersisterInterface $objectPersister - * @param string $objectClass - * @param array $options - * @param ManagerRegistry $managerRegistry + * @param IndexableInterface $indexable + * @param string $objectClass + * @param array $options + * @param ManagerRegistry $managerRegistry */ - public function __construct(ObjectPersisterInterface $objectPersister, $objectClass, array $options, $managerRegistry) - { - parent::__construct($objectPersister, $objectClass, array_merge(array( + public function __construct( + ObjectPersisterInterface $objectPersister, + IndexableInterface $indexable, + $objectClass, + array $options, + ManagerRegistry $managerRegistry + ) { + parent::__construct($objectPersister, $indexable, $objectClass, array_merge(array( 'clear_object_manager' => true, 'debug_logging' => false, 'ignore_errors' => false, @@ -53,6 +60,10 @@ abstract class AbstractProvider extends BaseAbstractProvider $stepStartTime = microtime(true); } $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); + if ($loggerClosure) { + $stepNbObjects = count($objects); + } + $objects = array_filter($objects, array($this, 'isObjectIndexable')); if (!$ignoreErrors) { $this->objectPersister->insertMany($objects); @@ -73,7 +84,6 @@ abstract class AbstractProvider extends BaseAbstractProvider usleep($sleep); if ($loggerClosure) { - $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $timeDifference = microtime(true) - $stepStartTime; diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index 2b4c8ee..5953d5a 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -68,4 +68,11 @@ interface ObjectPersisterInterface * @param array $identifiers array of domain model object identifiers */ public function deleteManyByIdentifiers(array $identifiers); + + /** + * Returns the elastica type used by this persister + * + * @return \Elastica\Type + */ + public function getType(); } diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 2761a25..8642be8 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -24,23 +24,48 @@ abstract class AbstractProvider implements ProviderInterface */ protected $options; + /** + * @var Indexable + */ + private $indexable; + /** * Constructor. * * @param ObjectPersisterInterface $objectPersister - * @param string $objectClass - * @param array $options + * @param IndexableInterface $indexable + * @param string $objectClass + * @param array $options */ - public function __construct(ObjectPersisterInterface $objectPersister, $objectClass, array $options = array()) - { - $this->objectPersister = $objectPersister; + public function __construct( + ObjectPersisterInterface $objectPersister, + IndexableInterface $indexable, + $objectClass, + array $options = array() + ) { + $this->indexable = $indexable; $this->objectClass = $objectClass; + $this->objectPersister = $objectPersister; $this->options = array_merge(array( 'batch_size' => 100, ), $options); } + /** + * Checks if a given object should be indexed or not. + * + * @param object $object + * @return bool + */ + protected function isObjectIndexable($object) + { + $typeName = $this->objectPersister->getType()->getName(); + $indexName = $this->objectPersister->getType()->getIndex()->getName(); + + return $this->indexable->isObjectIndexable($indexName, $typeName, $object); + } + /** * Get string with RAM usage information (current and peak) * diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index dcceccf..3640d16 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -9,24 +9,42 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase private $objectPersister; private $options; private $managerRegistry; + private $indexable; public function setUp() { - if (!interface_exists('Doctrine\Common\Persistence\ManagerRegistry')) { - $this->markTestSkipped('Doctrine Common is not available.'); - } + if (!interface_exists('Doctrine\Common\Persistence\ManagerRegistry')) { + $this->markTestSkipped('Doctrine Common is not available.'); + } - $this->objectClass = 'objectClass'; - $this->options = array('debug_logging' => true); + $this->objectClass = 'objectClass'; + $this->options = array('debug_logging' => true); - $this->objectPersister = $this->getMockObjectPersister(); - $this->managerRegistry = $this->getMockManagerRegistry(); - $this->objectManager = $this->getMockObjectManager(); + $this->objectPersister = $this->getMockObjectPersister(); + $this->managerRegistry = $this->getMockManagerRegistry(); + $this->objectManager = $this->getMockObjectManager(); + $this->indexable = $this->getMockIndexable(); - $this->managerRegistry->expects($this->any()) - ->method('getManagerForClass') - ->with($this->objectClass) - ->will($this->returnValue($this->objectManager)); + $index = $this->getMockBuilder('Elastica\Index')->disableOriginalConstructor()->getMock(); + $index->expects($this->any()) + ->method('getName') + ->will($this->returnValue('index')); + $type = $this->getMockBuilder('Elastica\Type')->disableOriginalConstructor()->getMock(); + $type->expects($this->any()) + ->method('getName') + ->will($this->returnValue('type')); + $type->expects($this->any()) + ->method('getIndex') + ->will($this->returnValue($index)); + + $this->objectPersister->expects($this->any()) + ->method('getType') + ->will($this->returnValue($type)); + + $this->managerRegistry->expects($this->any()) + ->method('getManagerForClass') + ->with($this->objectClass) + ->will($this->returnValue($this->objectManager)); } /** @@ -59,14 +77,13 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->with($queryBuilder, $batchSize, $offset) ->will($this->returnValue($objects)); - $this->objectPersister->expects($this->at($i)) - ->method('insertMany') - ->with($objects); - $this->objectManager->expects($this->at($i)) ->method('clear'); } + $this->objectPersister->expects($this->exactly(count($objectsByIteration))) + ->method('insertMany'); + $provider->populate(); } @@ -159,6 +176,36 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase $provider->populate(null, array('ignore-errors' => false)); } + public function testPopulateRunsIndexCallable() + { + $nbObjects = 2; + $objects = array(1, 2); + + $provider = $this->getMockAbstractProvider(); + $provider->expects($this->any()) + ->method('countObjects') + ->will($this->returnValue($nbObjects)); + $provider->expects($this->any()) + ->method('fetchSlice') + ->will($this->returnValue($objects)); + + $this->indexable->expects($this->at(0)) + ->method('isObjectIndexable') + ->with('index', 'type', 1) + ->will($this->returnValue(false)); + $this->indexable->expects($this->at(1)) + ->method('isObjectIndexable') + ->with('index', 'type', 2) + ->will($this->returnValue(true)); + + + $this->objectPersister->expects($this->once()) + ->method('insertMany') + ->with(array(1 => 2)); + + $provider->populate(); + } + /** * @return \FOS\ElasticaBundle\Doctrine\AbstractProvider|\PHPUnit_Framework_MockObject_MockObject */ @@ -166,6 +213,7 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase { return $this->getMockForAbstractClass('FOS\ElasticaBundle\Doctrine\AbstractProvider', array( $this->objectPersister, + $this->indexable, $this->objectClass, $this->options, $this->managerRegistry, @@ -205,6 +253,14 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase { return $this->getMock('FOS\ElasticaBundle\Persister\ObjectPersisterInterface'); } + + /** + * @return \FOS\ElasticaBundle\Provider\IndexableInterface|\PHPUnit_Framework_MockObject_MockObject + */ + private function getMockIndexable() + { + return $this->getMock('FOS\ElasticaBundle\Provider\IndexableInterface'); + } } /** From 391e18dcbf9e165a4881119b3e321806ed1b0b0f Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 16:20:17 +1000 Subject: [PATCH 079/154] Update changelog --- CHANGELOG-3.0.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 9d74640..f93832d 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,7 +12,17 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.0-ALPHA3 (xxxx-xx-xx) +* 3.0.0-ALPHA6 (xxxx-xx-xx) + + * Moved `is_indexable_callback` from the listener properties to a type property called + `indexable_callback` which is run when both populating and listening for object + changes. + * BC BREAK `ObjectPersisterInterface` added method getType() (To be removed during + ObjectPersister refactoring before 3.0.0 stable when ObjectPersister will change) + * AbstractProvider constructor change: Second argument is now an `IndexableInterface` + instance. + +* 3.0.0-ALPHA3 (2014-04-01) * a9c4c93: Logger is now only enabled in debug mode by default * #463: allowing hot swappable reindexing From 629ca0df2eb0784fc90b812bb17c32d586ab3ade Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Mon, 16 Jun 2014 12:59:09 +0100 Subject: [PATCH 080/154] make sure headers is set prior to accessing --- Client.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Client.php b/Client.php index 3eb98fe..cfcfea6 100644 --- a/Client.php +++ b/Client.php @@ -28,7 +28,7 @@ class Client extends ElasticaClient 'host' => $connection->getHost(), 'port' => $connection->getPort(), 'transport' => $connection->getTransport(), - 'headers' => $connection->getConfig('headers'), + 'headers' => $connection->hasConfig('headers')?$connection->getConfig('headers'):array(), ); $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); From 64be10447de535ee85f1cc2511ea55e943642df9 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 22:33:04 +1000 Subject: [PATCH 081/154] Move Search annotation --- Annotation/Search.php | 16 ++++++++++++++++ CHANGELOG-3.0.md | 3 ++- Configuration/Search.php | 19 ++++++++++++++----- 3 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 Annotation/Search.php diff --git a/Annotation/Search.php b/Annotation/Search.php new file mode 100644 index 0000000..26e1dbf --- /dev/null +++ b/Annotation/Search.php @@ -0,0 +1,16 @@ + + * @Annotation + * @Target("CLASS") + */ +class Search +{ + /** @var string */ + public $repositoryClass; +} diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 095c268..62e347a 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -14,6 +14,7 @@ To generate a changelog summary since the last version, run * 3.0.0-ALPHA6 + * Annotation @Search moved to FOS\ElasticaBundle\Annotation\Search with FOS\ElasticaBundle\Configuration\Search deprecated * Deprecated FOS\ElasticaBundle\Client in favour of FOS\ElasticaBundle\Elastica\Client * Deprecated FOS\ElasticaBundle\DynamicIndex in favour of FOS\ElasticaBundle\Elastica\Index * Deprecated FOS\ElasticaBundle\IndexManager in favour of FOS\ElasticaBundle\Index\IndexManager @@ -21,7 +22,7 @@ To generate a changelog summary since the last version, run * 3.0.0-ALPHA5 (2014-05-23) -* Doctrine Provider speed up by disabling persistence logging while populating documents + * Doctrine Provider speed up by disabling persistence logging while populating documents * 3.0.0-ALPHA4 (2014-04-10) diff --git a/Configuration/Search.php b/Configuration/Search.php index cee10ab..1306f92 100644 --- a/Configuration/Search.php +++ b/Configuration/Search.php @@ -1,16 +1,25 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + namespace FOS\ElasticaBundle\Configuration; +use FOS\ElasticaBundle\Annotation\Search as BaseSearch; + /** * Annotation class for setting search repository. * - * @author Richard Miller * @Annotation + * @deprecated Use FOS\ElasticaBundle\Annotation\Search instead * @Target("CLASS") */ -class Search +class Search extends BaseSearch { - /** @var string */ - public $repositoryClass; -} +} From 7682d5a80a56cf7de2538c5370e1494fb418bbce Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 22:33:31 +1000 Subject: [PATCH 082/154] Update composer.json --- composer.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index 8783822..806d0b0 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,8 @@ "homepage": "https://github.com/FriendsOfSymfony/FOSElasticaBundle", "license": "MIT", "authors": [ - { "name": "Thibault Duplessis", "email": "thibault.duplessis@gmail.com" }, + { "name": "FriendsOfSymfony Community", "homepage": "https://github.com/FriendsOfSymfony/FOSElasticaBundle/contributors" }, + { "name": "Tim Nagel", "email": "tim@nagel.com.au" }, { "name": "Richard Miller", "email": "richard.miller@limethinking.co.uk" }, { "name": "Jeremy Mikola", "email": "jmikola@gmail.com" } ], @@ -16,12 +17,13 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.10.0, <1.2-dev", + "ruflin/elastica": ">=0.90.10.0, <1.3-dev", "psr/log": "~1.0" }, "require-dev":{ "doctrine/orm": "~2.2", - "doctrine/mongodb-odm": "1.0.*@dev", + "doctrine/mongodb-odm": "1.0.*@beta", + "phpunit/phpunit": "~4.1", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", "knplabs/knp-components": "~1.2", @@ -37,9 +39,8 @@ "symfony/expression-language" : "~2.4" }, "autoload": { - "psr-0": { "FOS\\ElasticaBundle": "" } + "psr-4": { "FOS\\ElasticaBundle\\": "" } }, - "target-dir": "FOS/ElasticaBundle", "extra": { "branch-alias": { "dev-master": "3.0.x-dev" From 813a4a5d264644b8bdd04570724dff3c39b75b1d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 22:43:16 +1000 Subject: [PATCH 083/154] Fix previous commits --- Client.php | 4 ++-- DependencyInjection/Configuration.php | 8 +++---- DependencyInjection/FOSElasticaExtension.php | 22 +++++++++++++++---- Resources/config/config.xml | 13 +++++------ .../{LoggingClientTest.php => ClientTest.php} | 2 +- 5 files changed, 31 insertions(+), 18 deletions(-) rename Tests/Elastica/{LoggingClientTest.php => ClientTest.php} (98%) diff --git a/Client.php b/Client.php index f85756d..d0cee46 100644 --- a/Client.php +++ b/Client.php @@ -2,11 +2,11 @@ namespace FOS\ElasticaBundle; -use FOS\ElasticaBundle\Elastica\LoggingClient; +use FOS\ElasticaBundle\Elastica\Client as BaseClient; /** * @deprecated Use \FOS\ElasticaBundle\Elastica\LoggingClient */ -class Client extends LoggingClient +class Client extends BaseClient { } diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 754303b..7e8bca4 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -183,11 +183,11 @@ class Configuration implements ConfigurationInterface ->beforeNormalization() ->ifTrue(function($v) { return isset($v['mappings']); }) ->then(function($v) { - $v['properties'] = $v['mappings']; - unset($v['mappings']); + $v['properties'] = $v['mappings']; + unset($v['mappings']); - return $v; - }) + return $v; + }) ->end() ->children() ->scalarNode('index_analyzer')->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index c844f3a..0b994c5 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -72,6 +72,7 @@ class FOSElasticaExtension extends Extension $container->setAlias('fos_elastica.index', sprintf('fos_elastica.index.%s', $config['default_index'])); $this->loadIndexManager($container); + $this->loadResetter($container); $this->createDefaultManagerAlias($config['default_manager'], $container); } @@ -132,7 +133,7 @@ class FOSElasticaExtension extends Extension $indexDef = new DefinitionDecorator('fos_elastica.index_prototype'); $indexDef->replaceArgument(0, $indexName); - if ($index['client']) { + if (isset($index['client'])) { $client = $this->getClient($index['client']); $indexDef->setFactoryService($client); } @@ -146,7 +147,7 @@ class FOSElasticaExtension extends Extension 'settings' => $index['settings'] ), 'elasticsearch_name' => $indexName, - 'index' => $reference, + 'reference' => $reference, 'name' => $name, 'type_prototype' => isset($index['type_prototype']) ? $index['type_prototype'] : array(), 'use_alias' => $index['use_alias'], @@ -220,7 +221,9 @@ class FOSElasticaExtension extends Extension '_timestamp', '_ttl', ) as $field) { - $this->indexConfigs[$indexName]['config']['properties'][$name][$field] = $type[$field]; + if (array_key_exists($field, $type)) { + $this->indexConfigs[$indexName]['config']['properties'][$name][$field] = $type[$field]; + } } if (!empty($type['dynamic_templates'])) { @@ -390,7 +393,7 @@ class FOSElasticaExtension extends Extension if ($this->serializerConfig) { $abstractId = 'fos_elastica.object_serializer_persister'; - $callbackId = sprintf('%s.%s.serializer.callback', $this->indexConfigs[$indexName]['index'], $typeName); + $callbackId = sprintf('%s.%s.serializer.callback', $this->indexConfigs[$indexName]['reference'], $typeName); $arguments[] = array(new Reference($callbackId), 'serialize'); } else { $abstractId = 'fos_elastica.object_persister'; @@ -622,4 +625,15 @@ class FOSElasticaExtension extends Extension return $this->clients[$clientName]['reference']; } + + /** + * Loads the resetter + * + * @param \Symfony\Component\DependencyInjection\ContainerBuilder $container + */ + private function loadResetter(ContainerBuilder $container) + { + $resetterDef = $container->getDefinition('fos_elastica.resetter'); + $resetterDef->replaceArgument(0, $this->indexConfigs); + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index c1ac0f9..3cb6280 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -5,7 +5,7 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> - FOS\ElasticaBundle\Elastica\LoggingClient + FOS\ElasticaBundle\Elastica\Client FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector Symfony\Component\PropertyAccess\PropertyAccessor @@ -15,7 +15,11 @@ - + + + + + @@ -24,11 +28,6 @@ - - - - - diff --git a/Tests/Elastica/LoggingClientTest.php b/Tests/Elastica/ClientTest.php similarity index 98% rename from Tests/Elastica/LoggingClientTest.php rename to Tests/Elastica/ClientTest.php index b08a2cf..43ac7a2 100644 --- a/Tests/Elastica/LoggingClientTest.php +++ b/Tests/Elastica/ClientTest.php @@ -28,7 +28,7 @@ class LoggingClientTest extends \PHPUnit_Framework_TestCase $this->isType('array') ); - $client = $this->getMockBuilder('FOS\ElasticaBundle\Elastica\LoggingClient') + $client = $this->getMockBuilder('FOS\ElasticaBundle\Elastica\Client') ->setMethods(array('getConnection')) ->getMock(); From ada3942576a920c56c020fb9bde6b18a0bffacbf Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 23:23:49 +1000 Subject: [PATCH 084/154] Config Source providers --- Configuration/IndexConfig.php | 122 ++++++++++++++++++ Configuration/Manager.php | 59 +++++++++ Configuration/ManagerInterface.php | 35 +++++ Configuration/Source/ContainerSource.php | 62 +++++++++ Configuration/Source/SourceInterface.php | 26 ++++ Configuration/TypeConfig.php | 31 +++++ .../Compiler/ConfigSourcePass.php | 36 ++++++ DependencyInjection/FOSElasticaExtension.php | 49 ++----- FOSElasticaBundle.php | 2 + Resources/config/config.xml | 4 + Resources/config/source.xml | 13 ++ Tests/Functional/ConfigurationManagerTest.php | 54 ++++++++ 12 files changed, 457 insertions(+), 36 deletions(-) create mode 100644 Configuration/IndexConfig.php create mode 100644 Configuration/Manager.php create mode 100644 Configuration/ManagerInterface.php create mode 100644 Configuration/Source/ContainerSource.php create mode 100644 Configuration/Source/SourceInterface.php create mode 100644 Configuration/TypeConfig.php create mode 100644 DependencyInjection/Compiler/ConfigSourcePass.php create mode 100644 Resources/config/source.xml create mode 100644 Tests/Functional/ConfigurationManagerTest.php diff --git a/Configuration/IndexConfig.php b/Configuration/IndexConfig.php new file mode 100644 index 0000000..684713e --- /dev/null +++ b/Configuration/IndexConfig.php @@ -0,0 +1,122 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration; + +class IndexConfig +{ + /** + * The name of the index for ElasticSearch. + * + * @var string + */ + private $elasticSearchName; + + /** + * The internal name of the index. May not be the same as the name used in ElasticSearch, + * especially if aliases are enabled. + * + * @var string + */ + private $name; + + /** + * An array of settings sent to ElasticSearch when creating the index. + * + * @var array + */ + private $settings; + + /** + * All types that belong to this index. + * + * @var TypeConfig[] + */ + private $types; + + /** + * Indicates if the index should use an alias, allowing an index repopulation to occur + * without overwriting the current index. + * + * @var bool + */ + private $useAlias = false; + + /** + * Constructor expects an array as generated by the Container Configuration builder. + * + * @param string $name + * @param TypeConfig[] $types + * @param array $config + */ + public function __construct($name, array $types, array $config) + { + $this->elasticSearchName = $config['elasticSearchName']; + $this->name = $name; + $this->settings = $config['settings']; + $this->types = $types; + $this->useAlias = $config['useAlias']; + } + + /** + * @return string + */ + public function getElasticSearchName() + { + return $this->elasticSearchName; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + /** + * @return array + */ + public function getSettings() + { + return $this->settings; + } + + /** + * @param string $typeName + * @return TypeConfig + * @throws \InvalidArgumentException + */ + public function getType($typeName) + { + if (!array_key_exists($typeName, $this->types)) { + throw new \InvalidArgumentException(sprintf('Type "%s" does not exist on index "%s"', $typeName, $this->name)); + } + + return $this->types[$typeName]; + } + + /** + * @return \FOS\ElasticaBundle\Configuration\TypeConfig[] + */ + public function getTypes() + { + return $this->types; + } + + /** + * @return boolean + */ + public function getUseAlias() + { + return $this->useAlias; + } +} diff --git a/Configuration/Manager.php b/Configuration/Manager.php new file mode 100644 index 0000000..85afb99 --- /dev/null +++ b/Configuration/Manager.php @@ -0,0 +1,59 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration; + +/** + * Central manager for index and type configuration. + */ +class Manager implements ManagerInterface +{ + /** + * @var IndexConfig[] + */ + private $indexes = array(); + + /** + * @param Source\SourceInterface[] $sources + */ + public function __construct(array $sources) + { + foreach ($sources as $source) { + $this->indexes = array_merge($source->getConfiguration(), $this->indexes); + } + } + + public function getTypeConfiguration($indexName, $typeName) + { + $index = $this->getIndexConfiguration($indexName); + $type = $index->getType($typeName); + + if (!$type) { + throw new \InvalidArgumentException(sprintf('Type with name "%s" on index "%s" is not configured', $typeName, $indexName)); + } + + return $type; + } + + public function getIndexConfiguration($indexName) + { + if (!$this->hasIndexConfiguration($indexName)) { + throw new \InvalidArgumentException(sprintf('Index with name "%s" is not configured.', $indexName)); + } + + return $this->indexes[$indexName]; + } + + public function hasIndexConfiguration($indexName) + { + return isset($this->indexes[$indexName]); + } +} diff --git a/Configuration/ManagerInterface.php b/Configuration/ManagerInterface.php new file mode 100644 index 0000000..7852828 --- /dev/null +++ b/Configuration/ManagerInterface.php @@ -0,0 +1,35 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration; + +/** + * Central manager for index and type configuration. + */ +interface ManagerInterface +{ + /** + * Returns configuration for an index. + * + * @param $index + * @return IndexConfig + */ + public function getIndexConfiguration($index); + + /** + * Returns a type configuration. + * + * @param string $index + * @param string $type + * @return TypeConfig + */ + public function getTypeConfiguration($index, $type); +} \ No newline at end of file diff --git a/Configuration/Source/ContainerSource.php b/Configuration/Source/ContainerSource.php new file mode 100644 index 0000000..5fba57a --- /dev/null +++ b/Configuration/Source/ContainerSource.php @@ -0,0 +1,62 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration\Source; + +use FOS\ElasticaBundle\Configuration\IndexConfig; +use FOS\ElasticaBundle\Configuration\TypeConfig; + +/** + * Returns index and type configuration from the container. + */ +class ContainerSource implements SourceInterface +{ + /** + * The internal container representation of information. + * + * @var array + */ + private $configArray; + + public function __construct(array $configArray) + { + $this->configArray = $configArray; + } + + /** + * Should return all configuration available from the data source. + * + * @return IndexConfig[] + */ + public function getConfiguration() + { + $indexes = array(); + foreach ($this->configArray as $config) { + $types = array(); + foreach ($config['types'] as $typeConfig) { + $types[$typeConfig['name']] = new TypeConfig($typeConfig['name'], array( + 'dynamicTemplates' => $typeConfig['dynamic_templates'], + 'properties' => $typeConfig['properties'], + ), $config['type_prototype']); + } + + $index = new IndexConfig($config['name'], $types, array( + 'elasticSearchName' => $config['elasticsearch_name'], + 'settings' => $config['settings'], + 'useAlias' => $config['use_alias'], + )); + + $indexes[$config['name']] = $index; + } + + return $indexes; + } +} diff --git a/Configuration/Source/SourceInterface.php b/Configuration/Source/SourceInterface.php new file mode 100644 index 0000000..34e9901 --- /dev/null +++ b/Configuration/Source/SourceInterface.php @@ -0,0 +1,26 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration\Source; + +/** + * Represents a source of index and type information (ie, the Container configuration or + * annotations). + */ +interface SourceInterface +{ + /** + * Should return all configuration available from the data source. + * + * @return \FOS\ElasticaBundle\Configuration\IndexConfig[] + */ + public function getConfiguration(); +} diff --git a/Configuration/TypeConfig.php b/Configuration/TypeConfig.php new file mode 100644 index 0000000..c78a32a --- /dev/null +++ b/Configuration/TypeConfig.php @@ -0,0 +1,31 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Configuration; + +class TypeConfig +{ + /** + * @var array + */ + private $config; + + /** + * @var string + */ + private $name; + + public function __construct($name, array $config, array $prototype) + { + $this->config = $config; + $this->name = $name; + } +} diff --git a/DependencyInjection/Compiler/ConfigSourcePass.php b/DependencyInjection/Compiler/ConfigSourcePass.php new file mode 100644 index 0000000..b35a665 --- /dev/null +++ b/DependencyInjection/Compiler/ConfigSourcePass.php @@ -0,0 +1,36 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\DependencyInjection\Compiler; + +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Reference; + +class ConfigSourcePass implements CompilerPassInterface +{ + /** + * {@inheritDoc} + */ + public function process(ContainerBuilder $container) + { + if (!$container->hasDefinition('fos_elastica.config_manager')) { + return; + } + + $sources = array(); + foreach (array_keys($container->findTaggedServiceIds('fos_elastica.config_source')) as $id) { + $sources[] = new Reference($id); + } + + $container->getDefinition('fos_elastica.config_manager')->replaceArgument(0, $sources); + } +} diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 0b994c5..7699fd7 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -49,7 +49,7 @@ class FOSElasticaExtension extends Extension return; } - foreach (array('config', 'index', 'persister', 'provider', 'transformer') as $basename) { + foreach (array('config', 'index', 'persister', 'provider', 'source', 'transformer') as $basename) { $loader->load(sprintf('%s.xml', $basename)); } @@ -71,6 +71,8 @@ class FOSElasticaExtension extends Extension $this->loadIndexes($config['indexes'], $container); $container->setAlias('fos_elastica.index', sprintf('fos_elastica.index.%s', $config['default_index'])); + $container->getDefinition('fos_elastica.config_source.container')->replaceArgument(0, $this->indexConfigs); + $this->loadIndexManager($container); $this->loadResetter($container); @@ -142,13 +144,10 @@ class FOSElasticaExtension extends Extension $reference = new Reference($indexId); $this->indexConfigs[$name] = array( - 'config' => array( - 'properties' => array(), - 'settings' => $index['settings'] - ), 'elasticsearch_name' => $indexName, 'reference' => $reference, 'name' => $name, + 'settings' => $index['settings'], 'type_prototype' => isset($index['type_prototype']) ? $index['type_prototype'] : array(), 'use_alias' => $index['use_alias'], ); @@ -197,7 +196,6 @@ class FOSElasticaExtension extends Extension { foreach ($types as $name => $type) { $indexName = $indexConfig['name']; - $type = self::deepArrayUnion($indexConfig['type_prototype'], $type); $typeId = sprintf('%s.%s', $indexName, $name); $typeDef = new DefinitionDecorator('fos_elastica.type_prototype'); @@ -208,7 +206,14 @@ class FOSElasticaExtension extends Extension $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); } + $this->indexConfigs[$indexName]['types'][$name] = array( + 'dynamic_templates' => array(), + 'name' => $name, + 'properties' => array() + ); + foreach (array( + 'dynamic_templates', 'index_analyzer', 'properties', 'search_analyzer', @@ -222,14 +227,7 @@ class FOSElasticaExtension extends Extension '_ttl', ) as $field) { if (array_key_exists($field, $type)) { - $this->indexConfigs[$indexName]['config']['properties'][$name][$field] = $type[$field]; - } - } - - if (!empty($type['dynamic_templates'])) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'] = array(); - foreach ($type['dynamic_templates'] as $templateName => $templateData) { - $this->indexConfigs[$indexName]['config']['properties'][$name]['dynamic_templates'][] = array($templateName => $templateData); + $this->indexConfigs[$indexName]['types'][$name]['properties'][$field] = $type[$field]; } } @@ -259,27 +257,6 @@ class FOSElasticaExtension extends Extension } } - /** - * Merges two arrays without reindexing numeric keys. - * - * @param array $array1 An array to merge - * @param array $array2 An array to merge - * - * @return array The merged array - */ - private static function deepArrayUnion($array1, $array2) - { - foreach ($array2 as $key => $value) { - if (is_array($value) && isset($array1[$key]) && is_array($array1[$key])) { - $array1[$key] = self::deepArrayUnion($array1[$key], $value); - } else { - $array1[$key] = $value; - } - } - - return $array1; - } - /** * Loads the optional provider and finder for a type * @@ -397,7 +374,7 @@ class FOSElasticaExtension extends Extension $arguments[] = array(new Reference($callbackId), 'serialize'); } else { $abstractId = 'fos_elastica.object_persister'; - $arguments[] = $this->indexConfigs[$indexName]['config']['properties'][$typeName]['properties']; + $arguments[] = $this->indexConfigs[$indexName]['types'][$typeName]['properties']; } $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName); diff --git a/FOSElasticaBundle.php b/FOSElasticaBundle.php index 44424d3..ac4cc3d 100644 --- a/FOSElasticaBundle.php +++ b/FOSElasticaBundle.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle; +use FOS\ElasticaBundle\DependencyInjection\Compiler\ConfigSourcePass; use FOS\ElasticaBundle\DependencyInjection\Compiler\RegisterProvidersPass; use FOS\ElasticaBundle\DependencyInjection\Compiler\TransformerPass; use Symfony\Component\DependencyInjection\ContainerBuilder; @@ -23,5 +24,6 @@ class FOSElasticaBundle extends Bundle $container->addCompilerPass(new RegisterProvidersPass(), PassConfig::TYPE_BEFORE_REMOVING); $container->addCompilerPass(new TransformerPass()); + $container->addCompilerPass(new ConfigSourcePass()); } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 3cb6280..4ee733f 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -17,6 +17,10 @@ + + + + diff --git a/Resources/config/source.xml b/Resources/config/source.xml new file mode 100644 index 0000000..c0f085c --- /dev/null +++ b/Resources/config/source.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + diff --git a/Tests/Functional/ConfigurationManagerTest.php b/Tests/Functional/ConfigurationManagerTest.php new file mode 100644 index 0000000..0574481 --- /dev/null +++ b/Tests/Functional/ConfigurationManagerTest.php @@ -0,0 +1,54 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +use Symfony\Bundle\FrameworkBundle\Client; + +/** + * @group functional + */ +class ConfigurationManagerTest extends WebTestCase +{ + public function testContainerSource() + { + $client = $this->createClient(array('test_case' => 'Basic')); + $manager = $this->getManager($client); + + $index = $manager->getIndexConfiguration('index'); + var_dump($index); die; + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('Basic'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('Basic'); + } + + /** + * @param Client $client + * @return \FOS\ElasticaBundle\Configuration\Manager + */ + private function getManager(Client $client) + { + $manager = $client->getContainer()->get('fos_elastica.config_manager'); + + return $manager; + } +} From 0383811834eb9ac85e001f4c83964a07573cc93f Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 23:28:53 +1000 Subject: [PATCH 085/154] Fix test, add test for failing mappings --- Tests/Doctrine/AbstractProviderTest.php | 9 ++++--- Tests/Functional/MappingToElasticaTest.php | 28 ++++++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index dcceccf..c53940c 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -2,6 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Doctrine; +use Elastica\Bulk\ResponseSet; +use Elastica\Response; + class AbstractProviderTest extends \PHPUnit_Framework_TestCase { private $objectClass; @@ -177,9 +180,9 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase */ private function getMockBulkResponseException() { - return $this->getMockBuilder('Elastica\Exception\Bulk\ResponseException') - ->disableOriginalConstructor() - ->getMock(); + return $this->getMock('Elastica\Exception\Bulk\ResponseException', null, array( + new ResponseSet(new Response(array()), array()) + )); } /** diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index 3c32e10..bffcf76 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -18,13 +18,28 @@ use Symfony\Bundle\FrameworkBundle\Client; */ class MappingToElasticaTest extends WebTestCase { - public function testReset() + public function testResetIndexAddsMappings() { $client = $this->createClient(array('test_case' => 'Basic')); $resetter = $this->getResetter($client); - $resetter->resetIndex('index'); + + $type = $this->getType($client); + $mapping = $type->getMapping(); + + $this->assertNotEmpty($mapping, 'Mapping was populated'); + } + + public function testResetType() + { + $client = $this->createClient(array('test_case' => 'Basic')); + $resetter = $this->getResetter($client); $resetter->resetIndexType('index', 'type'); + + $type = $this->getType($client); + $mapping = $type->getMapping(); + + $this->assertNotEmpty($mapping, 'Mapping was populated'); } /** @@ -36,6 +51,15 @@ class MappingToElasticaTest extends WebTestCase return $client->getContainer()->get('fos_elastica.resetter'); } + /** + * @param Client $client + * @return \Elastica\Type + */ + private function getType(Client $client) + { + return $client->getContainer()->get('fos_elastica.index.index.type'); + } + protected function setUp() { parent::setUp(); From 934c6af8b8254c9f8be328ec842922a330e8c681 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Mon, 16 Jun 2014 14:30:03 +0100 Subject: [PATCH 086/154] make PSR2 compliant --- Client.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Client.php b/Client.php index cfcfea6..265773a 100644 --- a/Client.php +++ b/Client.php @@ -28,7 +28,7 @@ class Client extends ElasticaClient 'host' => $connection->getHost(), 'port' => $connection->getPort(), 'transport' => $connection->getTransport(), - 'headers' => $connection->hasConfig('headers')?$connection->getConfig('headers'):array(), + 'headers' => $connection->hasConfig('headers') ? $connection->getConfig('headers') : array(), ); $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); From 2e23f4a1bf31eb4e2775880c2ecd319717fac984 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 10:07:10 +1000 Subject: [PATCH 087/154] Added documentation changes for indexable_callback --- Resources/doc/setup.md | 2 +- Resources/doc/types.md | 81 ++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 9a39b95..485f290 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -7,7 +7,7 @@ A) Install FOSElasticaBundle FOSElasticaBundle is installed using [Composer](https://getcomposer.org). ```bash -$ php composer.phar require friendsofsymfony/elastica-bundle "3.0.*" +$ php composer.phar require friendsofsymfony/elastica-bundle "3.0.*@alpha" ``` ### Elasticsearch diff --git a/Resources/doc/types.md b/Resources/doc/types.md index aacb09e..be687d7 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -149,6 +149,44 @@ analyzer, you could write: title: { boost: 8, analyzer: my_analyzer } ``` +Testing if an object should be indexed +-------------------------------------- + +FOSElasticaBundle can be configured to automatically index changes made for +different kinds of objects if your persistence backend supports these methods, +but in some cases you might want to run an external service or call a property +on the object to see if it should be indexed. + +A property, `indexable_callback` is provided under the type configuration that +lets you configure this behaviour which will apply for any automated watching +for changes and for a repopulation of an index. + +In the example below, we're checking the enabled property on the user to only +index enabled users. + +```yaml + types: + users: + indexable_callback: 'enabled' +``` + +The callback option supports multiple approaches: + +* A method on the object itself provided as a string. `enabled` will call + `Object->enabled()` +* An array of a service id and a method which will be called with the object as the first + and only argument. `[ @my_custom_service, 'userIndexable' ]` will call the userIndexable + method on a service defined as my_custom_service. +* If you have the ExpressionLanguage component installed, A valid ExpressionLanguage + expression provided as a string. The object being indexed will be supplied as `object` + in the expression. `object.isEnabled() or object.shouldBeIndexedAnyway()`. For more + information on the ExpressionLanguage component and its capabilities see its + [documentation](http://symfony.com/doc/current/components/expression_language/index.html) + +In all cases, the callback should return a true or false, with true indicating it will be +indexed, and a false indicating the object should not be indexed, or should be removed +from the index if we are running an update. + Provider Configuration ---------------------- @@ -234,49 +272,6 @@ You can also choose to only listen for some of the events: > **Propel** doesn't support this feature yet. -### Checking an entity method for listener - -If you use listeners to update your index, you may need to validate your -entities before you index them (e.g. only index "public" entities). Typically, -you'll want the listener to be consistent with the provider's query criteria. -This may be achieved by using the `is_indexable_callback` config parameter: - -```yaml - persistence: - listener: - is_indexable_callback: "isPublic" -``` - -If `is_indexable_callback` is a string and the entity has a method with the -specified name, the listener will only index entities for which the method -returns `true`. Additionally, you may provide a service and method name pair: - -```yaml - persistence: - listener: - is_indexable_callback: [ "%custom_service_id%", "isIndexable" ] -``` - -In this case, the callback_class will be the `isIndexable()` method on the specified -service and the object being considered for indexing will be passed as the only -argument. This allows you to do more complex validation (e.g. ACL checks). - -If you have the [Symfony ExpressionLanguage](https://github.com/symfony/expression-language) -component installed, you can use expressions to evaluate the callback: - -```yaml - persistence: - listener: - is_indexable_callback: "user.isActive() && user.hasRole('ROLE_USER')" -``` - -As you might expect, new entities will only be indexed if the callback_class returns -`true`. Additionally, modified entities will be updated or removed from the -index depending on whether the callback_class returns `true` or `false`, respectively. -The delete listener disregards the callback_class. - -> **Propel** doesn't support this feature yet. - Flushing Method --------------- From b49437529c54f2ad9ffaef87019e5539a8b9372d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 10:41:11 +1000 Subject: [PATCH 088/154] Fix incorrect provider configuration --- Propel/Provider.php | 10 +++++++--- Resources/config/mongodb.xml | 2 +- Resources/config/orm.xml | 2 +- Resources/config/propel.xml | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Propel/Provider.php b/Propel/Provider.php index 393beba..c242d47 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -30,14 +30,18 @@ class Provider extends AbstractProvider $objects = $queryClass::create() ->limit($batchSize) ->offset($offset) - ->find(); + ->find() + ->getArrayCopy(); + if ($loggerClosure) { + $stepNbObjects = count($objects); + } + $objects = array_filter($objects, array($this, 'isObjectIndexable')); - $this->objectPersister->insertMany($objects->getArrayCopy()); + $this->objectPersister->insertMany($objects); usleep($sleep); if ($loggerClosure) { - $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index e575e5d..ffa4f22 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -23,7 +23,7 @@ - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 43d1670..351992a 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -23,7 +23,7 @@ - + diff --git a/Resources/config/propel.xml b/Resources/config/propel.xml index 7a7d93e..cbc8cd7 100644 --- a/Resources/config/propel.xml +++ b/Resources/config/propel.xml @@ -7,7 +7,7 @@ - + From e225d841edbd6633a0e864838af5b13f777b740b Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 10:54:09 +1000 Subject: [PATCH 089/154] Wrong service. Whoops. --- Resources/config/mongodb.xml | 4 ++-- Resources/config/orm.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index ffa4f22..73b70c5 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -8,7 +8,7 @@ - + @@ -23,7 +23,7 @@ - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 351992a..807a5bf 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -8,7 +8,7 @@ - + @@ -23,7 +23,7 @@ - + From aafb8c8e89d23fd455263bddc817874092c787b6 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 12:03:38 +1000 Subject: [PATCH 090/154] Fix indexable callbacks with alias based indexes --- DependencyInjection/FOSElasticaExtension.php | 13 +++++++++--- Doctrine/Listener.php | 18 +++++++++-------- DynamicIndex.php | 21 ++++++++++++++++---- Persister/ObjectPersister.php | 10 ---------- Persister/ObjectPersisterInterface.php | 7 ------- Provider/AbstractProvider.php | 9 +++++---- Provider/Indexable.php | 2 +- Resources/config/mongodb.xml | 1 + Resources/config/orm.xml | 1 + Resources/config/propel.xml | 1 + Tests/Doctrine/AbstractListenerTest.php | 12 +++++------ Tests/Doctrine/AbstractProviderTest.php | 18 +---------------- Tests/Provider/IndexableTest.php | 8 ++++++++ 13 files changed, 61 insertions(+), 60 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 9e50c0b..66a2cef 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -419,9 +419,12 @@ class FOSElasticaExtension extends Extension $providerDef = new DefinitionDecorator('fos_elastica.provider.prototype.' . $typeConfig['driver']); $providerDef->addTag('fos_elastica.provider', array('index' => $indexName, 'type' => $typeName)); $providerDef->replaceArgument(0, new Reference($objectPersisterId)); - $providerDef->replaceArgument(1, $typeConfig['model']); + $providerDef->replaceArgument(2, $typeConfig['model']); // Propel provider can simply ignore Doctrine-specific options - $providerDef->replaceArgument(2, array_diff_key($typeConfig['provider'], array('service' => 1))); + $providerDef->replaceArgument(3, array_merge(array_diff_key($typeConfig['provider'], array('service' => 1)), array( + 'indexName' => $indexName, + 'typeName' => $typeName, + ))); $container->setDefinition($providerId, $providerDef); return $providerId; @@ -440,7 +443,11 @@ class FOSElasticaExtension extends Extension $listenerDef = new DefinitionDecorator($abstractListenerId); $listenerDef->replaceArgument(0, new Reference($objectPersisterId)); $listenerDef->replaceArgument(1, $this->getDoctrineEvents($typeConfig)); - $listenerDef->replaceArgument(3, $typeConfig['identifier']); + $listenerDef->replaceArgument(3, array( + 'identifier' => $typeConfig['identifier'], + 'indexName' => $indexName, + 'typeName' => $typeName, + )); if ($typeConfig['listener']['logger']) { $listenerDef->replaceArgument(4, new Reference($typeConfig['listener']['logger'])); } diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 4a01aa1..73a271d 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -31,11 +31,11 @@ class Listener implements EventSubscriber protected $events; /** - * Name of domain model field used as the ES identifier + * Configuration for the listener * * @var string */ - protected $esIdentifierField; + private $config; /** * Objects scheduled for insertion and replacement @@ -66,17 +66,19 @@ class Listener implements EventSubscriber * @param ObjectPersisterInterface $objectPersister * @param array $events * @param IndexableInterface $indexable - * @param string $esIdentifierField + * @param array $config * @param null $logger */ public function __construct( ObjectPersisterInterface $objectPersister, array $events, IndexableInterface $indexable, - $esIdentifierField = 'id', + array $config = array(), $logger = null ) { - $this->esIdentifierField = $esIdentifierField; + $this->config = array_merge(array( + 'identifier' => 'id', + ), $config); $this->events = $events; $this->indexable = $indexable; $this->objectPersister = $objectPersister; @@ -196,7 +198,7 @@ class Listener implements EventSubscriber */ protected function scheduleForDeletion($object) { - if ($identifierValue = $this->propertyAccessor->getValue($object, $this->esIdentifierField)) { + if ($identifierValue = $this->propertyAccessor->getValue($object, $this->config['identifier'])) { $this->scheduledForDeletion[] = $identifierValue; } } @@ -210,8 +212,8 @@ class Listener implements EventSubscriber private function isObjectIndexable($object) { return $this->indexable->isObjectIndexable( - $this->objectPersister->getType()->getIndex()->getName(), - $this->objectPersister->getType()->getName(), + $this->config['indexName'], + $this->config['typeName'], $object ); } diff --git a/DynamicIndex.php b/DynamicIndex.php index cbec8e9..f890234 100644 --- a/DynamicIndex.php +++ b/DynamicIndex.php @@ -11,18 +11,31 @@ use Elastica\Index; */ class DynamicIndex extends Index { + private $originalName; + /** * Reassign index name * - * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName - * since it's used for a very specific case and normally should not be used + * While it's technically a regular setter for name property, it's specifically named + * overrideName, but not setName since it's used for a very specific case and normally + * should not be used. * * @param string $name Index name - * - * @return void */ public function overrideName($name) { + $this->originalName = $this->_name; $this->_name = $name; } + + /** + * Returns the original name of the index if the index has been renamed for reindexing + * or realiasing purposes. + * + * @return string + */ + public function getOriginalName() + { + return $this->originalName ?: $this->_name; + } } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 2b6a8af..54d5fd1 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -31,16 +31,6 @@ class ObjectPersister implements ObjectPersisterInterface $this->fields = $fields; } - /** - * @internal Temporary method that will be removed. - * - * @return Type - */ - public function getType() - { - return $this->type; - } - /** * If the ObjectPersister handles a given object. * diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index 5953d5a..2b4c8ee 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -68,11 +68,4 @@ interface ObjectPersisterInterface * @param array $identifiers array of domain model object identifiers */ public function deleteManyByIdentifiers(array $identifiers); - - /** - * Returns the elastica type used by this persister - * - * @return \Elastica\Type - */ - public function getType(); } diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 8642be8..82ea914 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -60,10 +60,11 @@ abstract class AbstractProvider implements ProviderInterface */ protected function isObjectIndexable($object) { - $typeName = $this->objectPersister->getType()->getName(); - $indexName = $this->objectPersister->getType()->getIndex()->getName(); - - return $this->indexable->isObjectIndexable($indexName, $typeName, $object); + return $this->indexable->isObjectIndexable( + $this->options['indexName'], + $this->options['typeName'], + $object + ); } /** diff --git a/Provider/Indexable.php b/Provider/Indexable.php index b388c58..09168a1 100644 --- a/Provider/Indexable.php +++ b/Provider/Indexable.php @@ -94,7 +94,7 @@ class Indexable implements IndexableInterface private function buildCallback($type, $object) { if (!array_key_exists($type, $this->callbacks)) { - throw new \InvalidArgumentException(sprintf('Callback for type "%s" is not configured', $type)); + return null; } $callback = $this->callbacks[$type]; diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 73b70c5..df2d2cc 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -9,6 +9,7 @@ + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 807a5bf..0cd6c42 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -9,6 +9,7 @@ + diff --git a/Resources/config/propel.xml b/Resources/config/propel.xml index cbc8cd7..962cb09 100644 --- a/Resources/config/propel.xml +++ b/Resources/config/propel.xml @@ -8,6 +8,7 @@ + diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index de5ba0c..1f238d6 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -16,7 +16,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); $indexable = $this->getMockIndexable('index', 'type', $entity, true); - $listener = $this->createListener($persister, array(), $indexable); + $listener = $this->createListener($persister, array(), $indexable, array('indexName' => 'index', 'typeName' => 'type')); $listener->postPersist($eventArgs); $this->assertEquals($entity, current($listener->scheduledForInsertion)); @@ -35,7 +35,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); $indexable = $this->getMockIndexable('index', 'type', $entity, false); - $listener = $this->createListener($persister, array(), $indexable); + $listener = $this->createListener($persister, array(), $indexable, array('indexName' => 'index', 'typeName' => 'type')); $listener->postPersist($eventArgs); $this->assertEmpty($listener->scheduledForInsertion); @@ -55,7 +55,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); $indexable = $this->getMockIndexable('index', 'type', $entity, true); - $listener = $this->createListener($persister, array(), $indexable); + $listener = $this->createListener($persister, array(), $indexable, array('indexName' => 'index', 'typeName' => 'type')); $listener->postUpdate($eventArgs); $this->assertEquals($entity, current($listener->scheduledForUpdate)); @@ -89,7 +89,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, array(), $indexable); + $listener = $this->createListener($persister, array(), $indexable, array('indexName' => 'index', 'typeName' => 'type')); $listener->postUpdate($eventArgs); $this->assertEmpty($listener->scheduledForUpdate); @@ -124,7 +124,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, array(), $indexable); + $listener = $this->createListener($persister, array(), $indexable, array('indexName' => 'index', 'typeName' => 'type')); $listener->preRemove($eventArgs); $this->assertEquals($entity->getId(), current($listener->scheduledForDeletion)); @@ -157,7 +157,7 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'identifier') ->will($this->returnValue($entity->getId())); - $listener = $this->createListener($persister, array(), $indexable, 'identifier'); + $listener = $this->createListener($persister, array(), $indexable, array('identifier' => 'identifier', 'indexName' => 'index', 'typeName' => 'type')); $listener->preRemove($eventArgs); $this->assertEquals($entity->identifier, current($listener->scheduledForDeletion)); diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index e5f9735..cd0a58c 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -21,29 +21,13 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase } $this->objectClass = 'objectClass'; - $this->options = array('debug_logging' => true); + $this->options = array('debug_logging' => true, 'indexName' => 'index', 'typeName' => 'type'); $this->objectPersister = $this->getMockObjectPersister(); $this->managerRegistry = $this->getMockManagerRegistry(); $this->objectManager = $this->getMockObjectManager(); $this->indexable = $this->getMockIndexable(); - $index = $this->getMockBuilder('Elastica\Index')->disableOriginalConstructor()->getMock(); - $index->expects($this->any()) - ->method('getName') - ->will($this->returnValue('index')); - $type = $this->getMockBuilder('Elastica\Type')->disableOriginalConstructor()->getMock(); - $type->expects($this->any()) - ->method('getName') - ->will($this->returnValue('type')); - $type->expects($this->any()) - ->method('getIndex') - ->will($this->returnValue($index)); - - $this->objectPersister->expects($this->any()) - ->method('getType') - ->will($this->returnValue($type)); - $this->managerRegistry->expects($this->any()) ->method('getManagerForClass') ->with($this->objectClass) diff --git a/Tests/Provider/IndexableTest.php b/Tests/Provider/IndexableTest.php index 61d7f87..aae6484 100644 --- a/Tests/Provider/IndexableTest.php +++ b/Tests/Provider/IndexableTest.php @@ -15,6 +15,14 @@ use FOS\ElasticaBundle\Provider\Indexable; class IndexableTest extends \PHPUnit_Framework_TestCase { + public function testIndexableUnknown() + { + $indexable = new Indexable(array()); + $index = $indexable->isObjectIndexable('index', 'type', new Entity); + + $this->assertTrue($index); + } + /** * @dataProvider provideIsIndexableCallbacks */ From 089e7f0d2ee45810fa989eabf92a4919ec661a7e Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 12:12:54 +1000 Subject: [PATCH 091/154] Add unstable badge [skip ci] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7909d2b..631951a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Symfony2. Features include: > **Note** Propel support is limited and contributions fixing issues are welcome! -[![Build Status](https://secure.travis-ci.org/FriendsOfSymfony/FOSElasticaBundle.png?branch=master)](http://travis-ci.org/FriendsOfSymfony/FOSElasticaBundle) [![Total Downloads](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/downloads.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Stable Version](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/v/stable.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) +[![Build Status](https://secure.travis-ci.org/FriendsOfSymfony/FOSElasticaBundle.png?branch=master)](http://travis-ci.org/FriendsOfSymfony/FOSElasticaBundle) [![Total Downloads](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/downloads.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Stable Version](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/v/stable.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Unstable Version](https://poser.pugx.org/friendsofsymfony/elastica-bundle/v/unstable.svg)](https://packagist.org/packages/friendsofsymfony/elastica-bundle) Documentation ------------- From 1e07d3c7fe5993a22a7ef75c6f29564cfc34d6fc Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 16 Jun 2014 22:33:31 +1000 Subject: [PATCH 092/154] Update composer.json --- .travis.yml | 4 +++- composer.json | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fdf2b9..d6ba18a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,7 @@ before_script: - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - composer install --dev --prefer-source +script: vendor/bin/phpunit + services: - - elasticsearch \ No newline at end of file + - elasticsearch diff --git a/composer.json b/composer.json index d67e329..bde8b3e 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,8 @@ "homepage": "https://github.com/FriendsOfSymfony/FOSElasticaBundle", "license": "MIT", "authors": [ - { "name": "Thibault Duplessis", "email": "thibault.duplessis@gmail.com" }, + { "name": "FriendsOfSymfony Community", "homepage": "https://github.com/FriendsOfSymfony/FOSElasticaBundle/contributors" }, + { "name": "Tim Nagel", "email": "tim@nagel.com.au" }, { "name": "Richard Miller", "email": "richard.miller@limethinking.co.uk" }, { "name": "Jeremy Mikola", "email": "jmikola@gmail.com" } ], @@ -16,13 +17,14 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.10.0, <1.2-dev", + "ruflin/elastica": ">=0.90.10.0, <1.3-dev", "psr/log": "~1.0" }, "require-dev":{ "doctrine/orm": "~2.2", "doctrine/doctrine-bundle": "~1.2@beta", - "doctrine/mongodb-odm": "1.0.*@dev", + "doctrine/mongodb-odm": "1.0.*@beta", + "phpunit/phpunit": "~4.1", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", "knplabs/knp-components": "~1.2", @@ -38,9 +40,8 @@ "symfony/expression-language" : "~2.4" }, "autoload": { - "psr-0": { "FOS\\ElasticaBundle": "" } + "psr-4": { "FOS\\ElasticaBundle\\": "" } }, - "target-dir": "FOS/ElasticaBundle", "extra": { "branch-alias": { "dev-master": "3.0.x-dev" From ec5c05e8becd6342b20dc0499759e290a6a6f0df Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Jun 2014 23:22:58 +1000 Subject: [PATCH 093/154] dev --- Configuration/IndexConfig.php | 2 +- Configuration/Manager.php | 23 ++++---- Configuration/ManagerInterface.php | 9 +++- DependencyInjection/FOSElasticaExtension.php | 19 ++----- Index/IndexManager.php | 6 +-- Index/Resetter.php | 54 +++++++------------ Resources/config/index.xml | 5 +- Tests/Functional/ConfigurationManagerTest.php | 6 ++- 8 files changed, 59 insertions(+), 65 deletions(-) diff --git a/Configuration/IndexConfig.php b/Configuration/IndexConfig.php index 684713e..9add5e3 100644 --- a/Configuration/IndexConfig.php +++ b/Configuration/IndexConfig.php @@ -115,7 +115,7 @@ class IndexConfig /** * @return boolean */ - public function getUseAlias() + public function isUseAlias() { return $this->useAlias; } diff --git a/Configuration/Manager.php b/Configuration/Manager.php index 85afb99..4436986 100644 --- a/Configuration/Manager.php +++ b/Configuration/Manager.php @@ -31,6 +31,20 @@ class Manager implements ManagerInterface } } + public function getIndexConfiguration($indexName) + { + if (!$this->hasIndexConfiguration($indexName)) { + throw new \InvalidArgumentException(sprintf('Index with name "%s" is not configured.', $indexName)); + } + + return $this->indexes[$indexName]; + } + + public function getIndexNames() + { + return array_keys($this->indexes); + } + public function getTypeConfiguration($indexName, $typeName) { $index = $this->getIndexConfiguration($indexName); @@ -43,15 +57,6 @@ class Manager implements ManagerInterface return $type; } - public function getIndexConfiguration($indexName) - { - if (!$this->hasIndexConfiguration($indexName)) { - throw new \InvalidArgumentException(sprintf('Index with name "%s" is not configured.', $indexName)); - } - - return $this->indexes[$indexName]; - } - public function hasIndexConfiguration($indexName) { return isset($this->indexes[$indexName]); diff --git a/Configuration/ManagerInterface.php b/Configuration/ManagerInterface.php index 7852828..96d510f 100644 --- a/Configuration/ManagerInterface.php +++ b/Configuration/ManagerInterface.php @@ -24,6 +24,13 @@ interface ManagerInterface */ public function getIndexConfiguration($index); + /** + * Returns an array of known index names. + * + * @return array + */ + public function getIndexNames(); + /** * Returns a type configuration. * @@ -32,4 +39,4 @@ interface ManagerInterface * @return TypeConfig */ public function getTypeConfiguration($index, $type); -} \ No newline at end of file +} diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index d122218..4d76baf 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -74,7 +74,6 @@ class FOSElasticaExtension extends Extension $container->getDefinition('fos_elastica.config_source.container')->replaceArgument(0, $this->indexConfigs); $this->loadIndexManager($container); - $this->loadResetter($container); $this->createDefaultManagerAlias($config['default_manager'], $container); } @@ -549,9 +548,12 @@ class FOSElasticaExtension extends Extension **/ private function loadIndexManager(ContainerBuilder $container) { + $indexRefs = array_map(function ($index) { + return $index['reference']; + }, $this->indexConfigs); + $managerDef = $container->getDefinition('fos_elastica.index_manager'); - $managerDef->replaceArgument(0, array_keys($this->clients)); - $managerDef->replaceArgument(1, new Reference('fos_elastica.index')); + $managerDef->replaceArgument(0, $indexRefs); } /** @@ -609,15 +611,4 @@ class FOSElasticaExtension extends Extension return $this->clients[$clientName]['reference']; } - - /** - * Loads the resetter - * - * @param \Symfony\Component\DependencyInjection\ContainerBuilder $container - */ - private function loadResetter(ContainerBuilder $container) - { - $resetterDef = $container->getDefinition('fos_elastica.resetter'); - $resetterDef->replaceArgument(0, $this->indexConfigs); - } } diff --git a/Index/IndexManager.php b/Index/IndexManager.php index 543ccae..9dd7bfe 100644 --- a/Index/IndexManager.php +++ b/Index/IndexManager.php @@ -2,12 +2,12 @@ namespace FOS\ElasticaBundle\Index; -use Elastica\Index; +use FOS\ElasticaBundle\Elastica\Index; class IndexManager { - protected $indexesByName; - protected $defaultIndexName; + private $indexesByName; + private $defaultIndexName; /** * Constructor. diff --git a/Index/Resetter.php b/Index/Resetter.php index 3356c27..aabad87 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -6,22 +6,27 @@ use Elastica\Exception\ExceptionInterface; use Elastica\Index; use Elastica\Exception\ResponseException; use Elastica\Type\Mapping; +use FOS\ElasticaBundle\Configuration\Manager; /** * Deletes and recreates indexes */ class Resetter { - protected $indexConfigsByName; + /*** + * @var \FOS\ElasticaBundle\Configuration\Manager + */ + private $configManager; /** - * Constructor. - * - * @param array $indexConfigsByName + * @var IndexManager */ - public function __construct(array $indexConfigsByName) + private $indexManager; + + public function __construct(Manager $configManager, IndexManager $indexManager) { - $this->indexConfigsByName = $indexConfigsByName; + $this->configManager = $configManager; + $this->indexManager = $indexManager; } /** @@ -29,7 +34,7 @@ class Resetter */ public function resetAllIndexes() { - foreach (array_keys($this->indexConfigsByName) as $name) { + foreach ($this->configManager->getIndexNames() as $name) { $this->resetIndex($name); } } @@ -42,18 +47,17 @@ class Resetter */ public function resetIndex($indexName) { - $indexConfig = $this->getIndexConfig($indexName); - $esIndex = $indexConfig['index']; - if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { - $name = $indexConfig['name_or_alias']; - $name .= uniqid(); - $esIndex->overrideName($name); - $esIndex->create($indexConfig['config']); + $indexConfig = $this->configManager->getIndexConfiguration($indexName); + $index = $this->indexManager->getIndex($indexName); - return; + if ($indexConfig->isUseAlias()) { + $name = sprintf('%s_%s', $indexConfig->getElasticSearchName(), uniqid()); + + $index->overrideName($name); } - $esIndex->create($indexConfig['config'], true); + + $index->create($indexConfig['config'], true); } /** @@ -108,24 +112,6 @@ class Resetter return $mapping; } - /** - * Gets an index config by its name - * - * @param string $indexName Index name - * - * @param $indexName - * @return array - * @throws \InvalidArgumentException if no index config exists for the given name - */ - protected function getIndexConfig($indexName) - { - if (!isset($this->indexConfigsByName[$indexName])) { - throw new \InvalidArgumentException(sprintf('The configuration for index "%s" does not exist.', $indexName)); - } - - return $this->indexConfigsByName[$indexName]; - } - public function postPopulate($indexName) { $indexConfig = $this->getIndexConfig($indexName); diff --git a/Resources/config/index.xml b/Resources/config/index.xml index de1a52e..331bfdd 100644 --- a/Resources/config/index.xml +++ b/Resources/config/index.xml @@ -28,11 +28,12 @@ - + - + + diff --git a/Tests/Functional/ConfigurationManagerTest.php b/Tests/Functional/ConfigurationManagerTest.php index 0574481..88bfa8b 100644 --- a/Tests/Functional/ConfigurationManagerTest.php +++ b/Tests/Functional/ConfigurationManagerTest.php @@ -24,7 +24,11 @@ class ConfigurationManagerTest extends WebTestCase $manager = $this->getManager($client); $index = $manager->getIndexConfiguration('index'); - var_dump($index); die; + + $this->assertEquals('index', $index->getName()); + $this->assertCount(2, $index->getTypes()); + $this->assertInstanceOf('FOS\\ElasticaBundle\\Configuration\\TypeConfig', $index->getType('type')); + $this->assertInstanceOf('FOS\\ElasticaBundle\\Configuration\\TypeConfig', $index->getType('parent')); } protected function setUp() From 8905b4248c07e34c5b5e0c055e7a385e2775978e Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 16:47:01 +1000 Subject: [PATCH 094/154] Rename Manager to ConfigManager --- Configuration/{Manager.php => ConfigManager.php} | 4 ++-- Resources/config/config.xml | 2 +- Resources/config/index.xml | 4 ++-- Tests/Functional/ConfigurationManagerTest.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename Configuration/{Manager.php => ConfigManager.php} (96%) diff --git a/Configuration/Manager.php b/Configuration/ConfigManager.php similarity index 96% rename from Configuration/Manager.php rename to Configuration/ConfigManager.php index 4436986..bef061b 100644 --- a/Configuration/Manager.php +++ b/Configuration/ConfigManager.php @@ -14,7 +14,7 @@ namespace FOS\ElasticaBundle\Configuration; /** * Central manager for index and type configuration. */ -class Manager implements ManagerInterface +class ConfigManager implements ManagerInterface { /** * @var IndexConfig[] @@ -61,4 +61,4 @@ class Manager implements ManagerInterface { return isset($this->indexes[$indexName]); } -} +} diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 4ee733f..75531a5 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -17,7 +17,7 @@ - + diff --git a/Resources/config/index.xml b/Resources/config/index.xml index 331bfdd..ef9f4e7 100644 --- a/Resources/config/index.xml +++ b/Resources/config/index.xml @@ -5,12 +5,12 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> + FOS\ElasticaBundle\Finder\TransformedFinder FOS\ElasticaBundle\Elastica\Index - Elastica\Type FOS\ElasticaBundle\Provider\Indexable FOS\ElasticaBundle\Index\IndexManager FOS\ElasticaBundle\Index\Resetter - FOS\ElasticaBundle\Finder\TransformedFinder + Elastica\Type diff --git a/Tests/Functional/ConfigurationManagerTest.php b/Tests/Functional/ConfigurationManagerTest.php index 88bfa8b..6fdc1d7 100644 --- a/Tests/Functional/ConfigurationManagerTest.php +++ b/Tests/Functional/ConfigurationManagerTest.php @@ -47,7 +47,7 @@ class ConfigurationManagerTest extends WebTestCase /** * @param Client $client - * @return \FOS\ElasticaBundle\Configuration\Manager + * @return \FOS\ElasticaBundle\Configuration\ConfigManager */ private function getManager(Client $client) { From afbaf875b990f326d8b284c39ddef082ae6e0b3e Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 16:48:00 +1000 Subject: [PATCH 095/154] Cache creation of indexes and types in Elastica to avoid recreation --- Elastica/Client.php | 13 ++++++++++++- Elastica/Index.php | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Elastica/Client.php b/Elastica/Client.php index 0db15b3..e921965 100644 --- a/Elastica/Client.php +++ b/Elastica/Client.php @@ -14,6 +14,13 @@ use FOS\ElasticaBundle\Logger\ElasticaLogger; */ class Client extends BaseClient { + /** + * Stores created indexes to avoid recreation. + * + * @var array + */ + private $indexCache = array(); + /** * {@inheritdoc} */ @@ -42,6 +49,10 @@ class Client extends BaseClient public function getIndex($name) { - return new Index($this, $name); + if (isset($this->indexCache[$name])) { + return $this->indexCache[$name]; + } + + return $this->indexCache[$name] = new Index($this, $name); } } diff --git a/Elastica/Index.php b/Elastica/Index.php index ec32c62..bf37c51 100644 --- a/Elastica/Index.php +++ b/Elastica/Index.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Elastica; use Elastica\Index as BaseIndex; +use Elastica\Type; /** * Overridden Elastica Index class that provides dynamic index name changes. @@ -13,6 +14,13 @@ class Index extends BaseIndex { private $originalName; + /** + * Stores created types to avoid recreation. + * + * @var array + */ + private $typeCache = array(); + /** * Returns the original name of the index if the index has been renamed for reindexing * or realiasing purposes. @@ -24,6 +32,15 @@ class Index extends BaseIndex return $this->originalName ?: $this->_name; } + public function getType($type) + { + if (isset($this->typeCache[$type])) { + return $this->typeCache[$type]; + } + + return $this->typeCache[$type] = parent::getType($type); + } + /** * Reassign index name * From b155f304e434b0cc0e0b6ee9310bd9b1e40e3c20 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 16:49:38 +1000 Subject: [PATCH 096/154] Move IndexManager's resolution to tagged index services --- DependencyInjection/Compiler/IndexPass.php | 38 ++++++++++++++++++++ DependencyInjection/FOSElasticaExtension.php | 3 ++ FOSElasticaBundle.php | 4 ++- Index/IndexManager.php | 26 +++++++------- Resources/config/index.xml | 1 + 5 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 DependencyInjection/Compiler/IndexPass.php diff --git a/DependencyInjection/Compiler/IndexPass.php b/DependencyInjection/Compiler/IndexPass.php new file mode 100644 index 0000000..e131214 --- /dev/null +++ b/DependencyInjection/Compiler/IndexPass.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\DependencyInjection\Compiler; + +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Reference; + +class IndexPass implements CompilerPassInterface +{ + /** + * {@inheritDoc} + */ + public function process(ContainerBuilder $container) + { + if (!$container->hasDefinition('fos_elastica.index_manager')) { + return; + } + + $indexes = array(); + foreach ($container->findTaggedServiceIds('fos_elastica.index') as $id => $tags) { + foreach ($tags as $tag) { + $indexes[$tag['name']] = new Reference($id); + } + } + + $container->getDefinition('fos_elastica.index_manager')->replaceArgument(0, $indexes); + } +} diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 4d76baf..4730c0b 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -133,6 +133,9 @@ class FOSElasticaExtension extends Extension $indexDef = new DefinitionDecorator('fos_elastica.index_prototype'); $indexDef->replaceArgument(0, $indexName); + $indexDef->addTag('fos_elastica.index', array( + 'name' => $name, + )); if (isset($index['client'])) { $client = $this->getClient($index['client']); diff --git a/FOSElasticaBundle.php b/FOSElasticaBundle.php index ac4cc3d..3dec2a0 100644 --- a/FOSElasticaBundle.php +++ b/FOSElasticaBundle.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle; use FOS\ElasticaBundle\DependencyInjection\Compiler\ConfigSourcePass; +use FOS\ElasticaBundle\DependencyInjection\Compiler\IndexPass; use FOS\ElasticaBundle\DependencyInjection\Compiler\RegisterProvidersPass; use FOS\ElasticaBundle\DependencyInjection\Compiler\TransformerPass; use Symfony\Component\DependencyInjection\ContainerBuilder; @@ -22,8 +23,9 @@ class FOSElasticaBundle extends Bundle { parent::build($container); + $container->addCompilerPass(new ConfigSourcePass()); + $container->addCompilerPass(new IndexPass()); $container->addCompilerPass(new RegisterProvidersPass(), PassConfig::TYPE_BEFORE_REMOVING); $container->addCompilerPass(new TransformerPass()); - $container->addCompilerPass(new ConfigSourcePass()); } } diff --git a/Index/IndexManager.php b/Index/IndexManager.php index 9dd7bfe..38249a7 100644 --- a/Index/IndexManager.php +++ b/Index/IndexManager.php @@ -6,19 +6,19 @@ use FOS\ElasticaBundle\Elastica\Index; class IndexManager { - private $indexesByName; - private $defaultIndexName; + /** + * @var array + */ + private $indexes; /** - * Constructor. - * - * @param array $indexesByName + * @param array $indexes * @param Index $defaultIndex */ - public function __construct(array $indexesByName, Index $defaultIndex) + public function __construct(array $indexes, Index $defaultIndex) { - $this->indexesByName = $indexesByName; - $this->defaultIndexName = $defaultIndex->getName(); + $this->defaultIndex = $defaultIndex; + $this->indexes = $indexes; } /** @@ -28,7 +28,7 @@ class IndexManager */ public function getAllIndexes() { - return $this->indexesByName; + return $this->indexes; } /** @@ -41,14 +41,14 @@ class IndexManager public function getIndex($name = null) { if (null === $name) { - $name = $this->defaultIndexName; + return $this->defaultIndex; } - if (!isset($this->indexesByName[$name])) { + if (!isset($this->indexes[$name])) { throw new \InvalidArgumentException(sprintf('The index "%s" does not exist', $name)); } - return $this->indexesByName[$name]; + return $this->indexes[$name]; } /** @@ -58,6 +58,6 @@ class IndexManager */ public function getDefaultIndex() { - return $this->getIndex($this->defaultIndexName); + return $this->defaultIndex; } } diff --git a/Resources/config/index.xml b/Resources/config/index.xml index ef9f4e7..60b75a9 100644 --- a/Resources/config/index.xml +++ b/Resources/config/index.xml @@ -20,6 +20,7 @@ + From 949ea6963f6c642d4fb20550b4f78eb5cf6874e3 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 19:55:33 +1000 Subject: [PATCH 097/154] Revert "Make the class of fos_elastica.paginator.subscriber service configurable" This reverts commit fe19df365a74373c19161449526d35beb708e49b. --- Resources/config/config.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 5058626..f4b2606 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -22,7 +22,6 @@ FOS\ElasticaBundle\Persister\ObjectSerializerPersister FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer - FOS\ElasticaBundle\Subscriber\PaginateElasticaQuerySubscriber @@ -93,7 +92,7 @@ - + From 3ae382c9331462ca5543b3686444877e4a177fe5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 20:02:05 +1000 Subject: [PATCH 098/154] Add tests to make sure KnpPaginator plays nice --- Tests/Functional/app/Basic/bundles.php | 7 +++++-- Tests/Functional/app/Basic/config.yml | 7 +++++++ Tests/Functional/app/config/config.yml | 2 +- composer.json | 4 +++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Tests/Functional/app/Basic/bundles.php b/Tests/Functional/app/Basic/bundles.php index 9f23bdf..7bcaae8 100644 --- a/Tests/Functional/app/Basic/bundles.php +++ b/Tests/Functional/app/Basic/bundles.php @@ -1,10 +1,13 @@ Date: Wed, 18 Jun 2014 16:12:16 +0300 Subject: [PATCH 099/154] Added elapsed item to toolbar and menu Kind similar to doctrine toolbar item --- Resources/views/Collector/elastica.html.twig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Resources/views/Collector/elastica.html.twig b/Resources/views/Collector/elastica.html.twig index 637dae7..e6d7072 100644 --- a/Resources/views/Collector/elastica.html.twig +++ b/Resources/views/Collector/elastica.html.twig @@ -4,6 +4,9 @@ {% set icon %} elastica {{ collector.querycount }} + {% if collector.querycount > 0 %} + in {{ '%0.2f'|format(collector.time * 1000) }} ms + {% endif %} {% endset %} {% set text %}
@@ -24,6 +27,7 @@ Elastica {{ collector.querycount }} + {{ '%0.0f'|format(collector.time * 1000) }} ms {% endblock %} From 4e990e0cee2371b083bf2875579018791bcda2ed Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 18 Jun 2014 17:13:29 +1000 Subject: [PATCH 100/154] Fixed mapping issues --- Configuration/IndexConfig.php | 6 +- Configuration/Source/ContainerSource.php | 10 +- Configuration/TypeConfig.php | 31 ++- DependencyInjection/FOSElasticaExtension.php | 66 ++++--- Index/AliasProcessor.php | 134 +++++++++++++ Index/MappingBuilder.php | 114 +++++++++++ Index/Resetter.php | 191 ++++--------------- Resources/config/config.xml | 3 + Resources/config/index.xml | 5 + Resources/config/mongodb.xml | 7 +- Resources/config/orm.xml | 8 +- Tests/FOSElasticaBundleTest.php | 19 +- Tests/Index/IndexManagerTest.php | 45 ++--- Tests/Index/ResetterTest.php | 51 ++++- 14 files changed, 453 insertions(+), 237 deletions(-) create mode 100644 Index/AliasProcessor.php create mode 100644 Index/MappingBuilder.php diff --git a/Configuration/IndexConfig.php b/Configuration/IndexConfig.php index 9add5e3..7416424 100644 --- a/Configuration/IndexConfig.php +++ b/Configuration/IndexConfig.php @@ -59,11 +59,11 @@ class IndexConfig */ public function __construct($name, array $types, array $config) { - $this->elasticSearchName = $config['elasticSearchName']; + $this->elasticSearchName = isset($config['elasticSearchName']) ? $config['elasticSearchName'] : $name; $this->name = $name; - $this->settings = $config['settings']; + $this->settings = isset($config['settings']) ? $config['settings'] : array(); $this->types = $types; - $this->useAlias = $config['useAlias']; + $this->useAlias = isset($config['useAlias']) ? $config['useAlias'] : false; } /** diff --git a/Configuration/Source/ContainerSource.php b/Configuration/Source/ContainerSource.php index 5fba57a..8d094c7 100644 --- a/Configuration/Source/ContainerSource.php +++ b/Configuration/Source/ContainerSource.php @@ -42,10 +42,12 @@ class ContainerSource implements SourceInterface foreach ($this->configArray as $config) { $types = array(); foreach ($config['types'] as $typeConfig) { - $types[$typeConfig['name']] = new TypeConfig($typeConfig['name'], array( - 'dynamicTemplates' => $typeConfig['dynamic_templates'], - 'properties' => $typeConfig['properties'], - ), $config['type_prototype']); + $types[$typeConfig['name']] = new TypeConfig( + $typeConfig['name'], + $typeConfig['mapping'], + $typeConfig['config'] + ); + // TODO: handle prototypes.. } $index = new IndexConfig($config['name'], $types, array( diff --git a/Configuration/TypeConfig.php b/Configuration/TypeConfig.php index c78a32a..5d3f084 100644 --- a/Configuration/TypeConfig.php +++ b/Configuration/TypeConfig.php @@ -18,14 +18,43 @@ class TypeConfig */ private $config; + /** + * @var array + */ + private $mapping; + /** * @var string */ private $name; - public function __construct($name, array $config, array $prototype) + public function __construct($name, array $mapping, array $config = array()) { $this->config = $config; + $this->mapping = $mapping; $this->name = $name; } + + /** + * @return array + */ + public function getMapping() + { + return $this->mapping; + } + + public function getModel() + { + return isset($this->config['persistence']['model']) ? + $this->config['persistence']['model'] : + null; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 4730c0b..158bea5 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -201,19 +201,16 @@ class FOSElasticaExtension extends Extension foreach ($types as $name => $type) { $indexName = $indexConfig['name']; - $typeId = sprintf('%s.%s', $indexName, $name); + $typeId = sprintf('%s.%s', $indexConfig['reference'], $name); $typeDef = new DefinitionDecorator('fos_elastica.type_prototype'); $typeDef->replaceArgument(0, $name); $typeDef->setFactoryService($indexConfig['reference']); + $container->setDefinition($typeId, $typeDef); - if (isset($type['persistence'])) { - $this->loadTypePersistenceIntegration($type['persistence'], $container, $typeDef, $indexName, $name); - } - - $this->indexConfigs[$indexName]['types'][$name] = array( - 'dynamic_templates' => array(), + $typeConfig = array( 'name' => $name, - 'properties' => array() + 'mapping' => array(), // An array containing anything that gets sent directly to ElasticSearch + 'config' => array(), ); foreach (array( @@ -230,17 +227,32 @@ class FOSElasticaExtension extends Extension '_timestamp', '_ttl', ) as $field) { - if (array_key_exists($field, $type)) { - $this->indexConfigs[$indexName]['types'][$name]['properties'][$field] = $type[$field]; + if (isset($type[$field])) { + $typeConfig['mapping'][$field] = $type[$field]; } } + foreach (array( + 'persistence', + 'serializer' + ) as $field) { + $typeConfig['config'][$field] = array_key_exists($field, $type) ? + $type[$field] : + null; + } + + $this->indexConfigs[$indexName]['types'][$name] = $typeConfig; + + if (isset($type['persistence'])) { + $this->loadTypePersistenceIntegration($type['persistence'], $container, new Reference($typeId), $indexName, $name); + + $typeConfig['persistence'] = $type['persistence']; + } + if (isset($type['indexable_callback'])) { $indexableCallbacks[sprintf('%s/%s', $indexName, $name)] = $type['indexable_callback']; } - $container->setDefinition($typeId, $typeDef); - /*if ($this->serializerConfig) { $callbackDef = new Definition($this->serializerConfig['callback_class']); $callbackId = sprintf('%s.%s.serializer.callback', $indexId, $name); @@ -272,24 +284,24 @@ class FOSElasticaExtension extends Extension * Loads the optional provider and finder for a type * * @param array $typeConfig - * @param \Symfony\Component\DependencyInjection\ContainerBuilder $container - * @param \Symfony\Component\DependencyInjection\Definition $typeDef - * @param $indexName - * @param $typeName + * @param ContainerBuilder $container + * @param Reference $typeRef + * @param string $indexName + * @param string $typeName */ - private function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Definition $typeDef, $indexName, $typeName) + private function loadTypePersistenceIntegration(array $typeConfig, ContainerBuilder $container, Reference $typeRef, $indexName, $typeName) { $this->loadDriver($container, $typeConfig['driver']); $elasticaToModelTransformerId = $this->loadElasticaToModelTransformer($typeConfig, $container, $indexName, $typeName); $modelToElasticaTransformerId = $this->loadModelToElasticaTransformer($typeConfig, $container, $indexName, $typeName); - $objectPersisterId = $this->loadObjectPersister($typeConfig, $typeDef, $container, $indexName, $typeName, $modelToElasticaTransformerId); + $objectPersisterId = $this->loadObjectPersister($typeConfig, $typeRef, $container, $indexName, $typeName, $modelToElasticaTransformerId); if (isset($typeConfig['provider'])) { $this->loadTypeProvider($typeConfig, $container, $objectPersisterId, $indexName, $typeName); } if (isset($typeConfig['finder'])) { - $this->loadTypeFinder($typeConfig, $container, $elasticaToModelTransformerId, $typeDef, $indexName, $typeName); + $this->loadTypeFinder($typeConfig, $container, $elasticaToModelTransformerId, $typeRef, $indexName, $typeName); } if (isset($typeConfig['listener'])) { $this->loadTypeListener($typeConfig, $container, $objectPersisterId, $indexName, $typeName); @@ -364,17 +376,17 @@ class FOSElasticaExtension extends Extension * Creates and loads an object persister for a type. * * @param array $typeConfig - * @param Definition $typeDef + * @param Reference $typeRef * @param ContainerBuilder $container * @param string $indexName * @param string $typeName * @param string $transformerId * @return string */ - private function loadObjectPersister(array $typeConfig, Definition $typeDef, ContainerBuilder $container, $indexName, $typeName, $transformerId) + private function loadObjectPersister(array $typeConfig, Reference $typeRef, ContainerBuilder $container, $indexName, $typeName, $transformerId) { $arguments = array( - $typeDef, + $typeRef, new Reference($transformerId), $typeConfig['model'], ); @@ -385,7 +397,7 @@ class FOSElasticaExtension extends Extension $arguments[] = array(new Reference($callbackId), 'serialize'); } else { $abstractId = 'fos_elastica.object_persister'; - $arguments[] = $this->indexConfigs[$indexName]['types'][$typeName]['properties']; + $arguments[] = $this->indexConfigs[$indexName]['types'][$typeName]['mapping']['properties']; } $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName); @@ -515,20 +527,20 @@ class FOSElasticaExtension extends Extension * * @param array $typeConfig * @param ContainerBuilder $container - * @param string $elasticaToModelId - * @param Definition $typeDef + * @param $elasticaToModelId + * @param Reference $typeRef * @param string $indexName * @param string $typeName * @return string */ - private function loadTypeFinder(array $typeConfig, ContainerBuilder $container, $elasticaToModelId, Definition $typeDef, $indexName, $typeName) + private function loadTypeFinder(array $typeConfig, ContainerBuilder $container, $elasticaToModelId, Reference $typeRef, $indexName, $typeName) { if (isset($typeConfig['finder']['service'])) { $finderId = $typeConfig['finder']['service']; } else { $finderId = sprintf('fos_elastica.finder.%s.%s', $indexName, $typeName); $finderDef = new DefinitionDecorator('fos_elastica.finder'); - $finderDef->replaceArgument(0, $typeDef); + $finderDef->replaceArgument(0, $typeRef); $finderDef->replaceArgument(1, new Reference($elasticaToModelId)); $container->setDefinition($finderId, $finderDef); } diff --git a/Index/AliasProcessor.php b/Index/AliasProcessor.php new file mode 100644 index 0000000..5c4592d --- /dev/null +++ b/Index/AliasProcessor.php @@ -0,0 +1,134 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Index; + +use Elastica\Exception\ExceptionInterface; +use FOS\ElasticaBundle\Configuration\IndexConfig; +use FOS\ElasticaBundle\Elastica\Client; +use FOS\ElasticaBundle\Elastica\Index; + +class AliasProcessor +{ + /** + * Sets the randomised root name for an index. + * + * @param IndexConfig $indexConfig + * @param Index $index + */ + public function setRootName(IndexConfig $indexConfig, Index $index) + { + $index->overrideName(sprintf('%s_%s', $indexConfig->getElasticSearchName(), uniqid())); + } + + /** + * Switches an index to become the new target for an alias. Only applies for + * indexes that are set to use aliases. + * + * @param IndexConfig $indexConfig + * @param Index $index + * @throws \RuntimeException + */ + public function switchIndexAlias(IndexConfig $indexConfig, Index $index) + { + $aliasName = $indexConfig->getElasticSearchName(); + $oldIndexName = false; + $newIndexName = $index->getName(); + + $aliasedIndexes = $this->getAliasedIndexes($aliasName); + + if (count($aliasedIndexes) > 1) { + throw new \RuntimeException( + sprintf( + 'Alias %s is used for multiple indexes: [%s]. + Make sure it\'s either not used or is assigned to one index only', + $aliasName, + join(', ', $aliasedIndexes) + ) + ); + } + + $aliasUpdateRequest = array('actions' => array()); + if (count($aliasedIndexes) == 1) { + // if the alias is set - add an action to remove it + $oldIndexName = $aliasedIndexes[0]; + $aliasUpdateRequest['actions'][] = array( + 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) + ); + } + + // add an action to point the alias to the new index + $aliasUpdateRequest['actions'][] = array( + 'add' => array('index' => $newIndexName, 'alias' => $aliasName) + ); + + try { + $this->client->request('_aliases', 'POST', $aliasUpdateRequest); + } catch (ExceptionInterface $renameAliasException) { + $additionalError = ''; + // if we failed to move the alias, delete the newly built index + try { + $index->delete(); + } catch (ExceptionInterface $deleteNewIndexException) { + $additionalError = sprintf( + 'Tried to delete newly built index %s, but also failed: %s', + $newIndexName, + $deleteNewIndexException->getMessage() + ); + } + + throw new \RuntimeException( + sprintf( + 'Failed to updated index alias: %s. %s', + $renameAliasException->getMessage(), + $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) + ), 0, $renameAliasException + ); + } + + // Delete the old index after the alias has been switched + if ($oldIndexName) { + $oldIndex = new Index($this->client, $oldIndexName); + try { + $oldIndex->delete(); + } catch (ExceptionInterface $deleteOldIndexException) { + throw new \RuntimeException( + sprintf( + 'Failed to delete old index %s with message: %s', + $oldIndexName, + $deleteOldIndexException->getMessage() + ), 0, $deleteOldIndexException + ); + } + } + } + + /** + * Returns array of indexes which are mapped to given alias + * + * @param string $aliasName Alias name + * @return array + */ + private function getAliasedIndexes($aliasName) + { + $aliasesInfo = $this->client->request('_aliases', 'GET')->getData(); + $aliasedIndexes = array(); + + foreach ($aliasesInfo as $indexName => $indexInfo) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } + } + + return $aliasedIndexes; + } +} diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php new file mode 100644 index 0000000..53524ac --- /dev/null +++ b/Index/MappingBuilder.php @@ -0,0 +1,114 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Index; + +use FOS\ElasticaBundle\Configuration\IndexConfig; +use FOS\ElasticaBundle\Configuration\TypeConfig; + +class MappingBuilder +{ + /** + * Builds mappings for an entire index. + * + * @param IndexConfig $indexConfig + * @return array + */ + public function buildIndexMapping(IndexConfig $indexConfig) + { + $typeMappings = array(); + foreach ($indexConfig->getTypes() as $typeConfig) { + $typeMappings[$typeConfig->getName()] = $this->buildTypeMapping($typeConfig); + } + + $mapping = array( + 'mappings' => $typeMappings, + 'settings' => $indexConfig->getSettings(), + // 'warmers' => $indexConfig->getWarmers(), + ); + + return $mapping; + } + + /** + * Builds mappings for a single type. + * + * @param TypeConfig $typeConfig + * @return array + */ + public function buildTypeMapping(TypeConfig $typeConfig) + { + $mapping = array_merge($typeConfig->getMapping(), array( + // 'date_detection' => true, + // 'dynamic_date_formats' => array() + // 'dynamic_templates' => $typeConfig->getDynamicTemplates(), + // 'index_analyzer' => $typeConfig->getIndexAnalyzer(), + // 'numeric_detection' => false, + // 'properties' => array(), + // 'search_analyzer' => $typeConfig->getSearchAnalyzer(), + )); + + $this->fixProperties($mapping['properties']); + + if ($typeConfig->getModel()) { + $mapping['_meta']['model'] = $typeConfig->getModel(); + } + + return $mapping; + } + + + /** + * create type mapping object + * + * @param array $indexConfig + * @return Mapping + */ + protected function createMapping($indexConfig) + { + /*$mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]);*/ + $mapping = Mapping::create($indexConfig['properties']); + + $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); + foreach ($mappingSpecialFields as $specialField) { + if (isset($indexConfig[$specialField])) { + $mapping->setParam($specialField, $indexConfig[$specialField]); + } + } + + if (isset($indexConfig['_parent'])) { + $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); + } + + return $mapping; + } + + /** + * Fixes any properties and applies basic defaults for any field that does not have + * required options. + * + * @param $properties + */ + private function fixProperties(&$properties) + { + foreach ($properties as $name => &$property) { + if (!isset($property['type'])) { + $property['type'] = 'string'; + } + if (!isset($property['store'])) { + $property['store'] = true; + } + if (isset($property['properties'])) { + $this->fixProperties($property['properties']); + } + } + } +} diff --git a/Index/Resetter.php b/Index/Resetter.php index aabad87..174c410 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -2,17 +2,23 @@ namespace FOS\ElasticaBundle\Index; -use Elastica\Exception\ExceptionInterface; use Elastica\Index; use Elastica\Exception\ResponseException; use Elastica\Type\Mapping; -use FOS\ElasticaBundle\Configuration\Manager; +use FOS\ElasticaBundle\Configuration\IndexConfig; +use FOS\ElasticaBundle\Configuration\ConfigManager; +use FOS\ElasticaBundle\Elastica\Client; /** * Deletes and recreates indexes */ class Resetter { + /** + * @var AliasProcessor + */ + private $aliasProcessor; + /*** * @var \FOS\ElasticaBundle\Configuration\Manager */ @@ -23,10 +29,17 @@ class Resetter */ private $indexManager; - public function __construct(Manager $configManager, IndexManager $indexManager) + /** + * @var MappingBuilder + */ + private $mappingBuilder; + + public function __construct(ConfigManager $configManager, IndexManager $indexManager, AliasProcessor $aliasProcessor, MappingBuilder $mappingBuilder) { + $this->aliasProcessor = $aliasProcessor; $this->configManager = $configManager; $this->indexManager = $indexManager; + $this->mappingBuilder = $mappingBuilder; } /** @@ -40,24 +53,28 @@ class Resetter } /** - * Deletes and recreates the named index + * Deletes and recreates the named index. If populating, creates a new index + * with a randomised name for an alias to be set after population. * * @param string $indexName + * @param bool $populating * @throws \InvalidArgumentException if no index exists for the given name */ - public function resetIndex($indexName) + public function resetIndex($indexName, $populating = false) { $indexConfig = $this->configManager->getIndexConfiguration($indexName); $index = $this->indexManager->getIndex($indexName); if ($indexConfig->isUseAlias()) { - $name = sprintf('%s_%s', $indexConfig->getElasticSearchName(), uniqid()); - - $index->overrideName($name); + $this->aliasProcessor->setRootName($indexConfig, $index); } + $mapping = $this->mappingBuilder->buildIndexMapping($indexConfig); + $index->create($mapping, true); - $index->create($indexConfig['config'], true); + if (!$populating and $indexConfig->isUseAlias()) { + $this->aliasProcessor->switchIndexAlias($indexConfig, $index); + } } /** @@ -70,13 +87,9 @@ class Resetter */ public function resetIndexType($indexName, $typeName) { - $indexConfig = $this->getIndexConfig($indexName); + $typeConfig = $this->configManager->getTypeConfiguration($indexName, $typeName); + $type = $this->indexManager->getIndex($indexName)->getType($typeName); - if (!isset($indexConfig['config']['properties'][$typeName]['properties'])) { - throw new \InvalidArgumentException(sprintf('The mapping for index "%s" and type "%s" does not exist.', $indexName, $typeName)); - } - - $type = $indexConfig['index']->getType($typeName); try { $type->delete(); } catch (ResponseException $e) { @@ -84,149 +97,27 @@ class Resetter throw $e; } } - $mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]); + + $mapping = new Mapping; + foreach ($this->mappingBuilder->buildTypeMapping($typeConfig) as $name => $field) { + $mapping->setParam($name, $field); + } + $type->setMapping($mapping); } /** - * create type mapping object + * A command run when a population has finished. * - * @param array $indexConfig - * @return Mapping + * @param $indexName */ - protected function createMapping($indexConfig) - { - $mapping = Mapping::create($indexConfig['properties']); - - $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); - foreach ($mappingSpecialFields as $specialField) { - if (isset($indexConfig[$specialField])) { - $mapping->setParam($specialField, $indexConfig[$specialField]); - } - } - - if (isset($indexConfig['_parent'])) { - $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); - } - - return $mapping; - } - public function postPopulate($indexName) { - $indexConfig = $this->getIndexConfig($indexName); - if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { - $this->switchIndexAlias($indexName); + $indexConfig = $this->configManager->getIndexConfiguration($indexName); + + if ($indexConfig->isUseAlias()) { + $index = $this->indexManager->getIndex($indexName); + $this->aliasProcessor->switchIndexAlias($indexConfig, $index); } } - - /** - * Switches the alias for given index (by key) to the newly populated index - * and deletes the old index - * - * @param string $indexName Index name - * - * @throws \RuntimeException - */ - private function switchIndexAlias($indexName) - { - $indexConfig = $this->getIndexConfig($indexName); - $esIndex = $indexConfig['index']; - $aliasName = $indexConfig['name_or_alias']; - $oldIndexName = false; - $newIndexName = $esIndex->getName(); - - $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); - - if (count($aliasedIndexes) > 1) { - throw new \RuntimeException( - sprintf( - 'Alias %s is used for multiple indexes: [%s]. - Make sure it\'s either not used or is assigned to one index only', - $aliasName, - join(', ', $aliasedIndexes) - ) - ); - } - - // Change the alias to point to the new index - // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic - // In 0.20.x it's atomic, but it doesn't return the old index name - $aliasUpdateRequest = array('actions' => array()); - if (count($aliasedIndexes) == 1) { - // if the alias is set - add an action to remove it - $oldIndexName = $aliasedIndexes[0]; - $aliasUpdateRequest['actions'][] = array( - 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) - ); - } - - // add an action to point the alias to the new index - $aliasUpdateRequest['actions'][] = array( - 'add' => array('index' => $newIndexName, 'alias' => $aliasName) - ); - - try { - $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); - } catch (ExceptionInterface $renameAliasException) { - $additionalError = ''; - // if we failed to move the alias, delete the newly built index - try { - $esIndex->delete(); - } catch (ExceptionInterface $deleteNewIndexException) { - $additionalError = sprintf( - 'Tried to delete newly built index %s, but also failed: %s', - $newIndexName, - $deleteNewIndexException->getMessage() - ); - } - - throw new \RuntimeException( - sprintf( - 'Failed to updated index alias: %s. %s', - $renameAliasException->getMessage(), - $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) - ) - ); - } - - // Delete the old index after the alias has been switched - if ($oldIndexName) { - $oldIndex = new Index($esIndex->getClient(), $oldIndexName); - try { - $oldIndex->delete(); - } catch (ExceptionInterface $deleteOldIndexException) { - throw new \RuntimeException( - sprintf( - 'Failed to delete old index %s with message: %s', - $oldIndexName, - $deleteOldIndexException->getMessage() - ) - ); - } - } - } - - /** - * Returns array of indexes which are mapped to given alias - * - * @param Index $esIndex ES Index - * @param string $aliasName Alias name - * - * @return array - */ - private function getAliasedIndexes(Index $esIndex, $aliasName) - { - $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); - $aliasedIndexes = array(); - - foreach ($aliasesInfo as $indexName => $indexInfo) { - $aliases = array_keys($indexInfo['aliases']); - if (in_array($aliasName, $aliases)) { - $aliasedIndexes[] = $indexName; - } - } - - return $aliasedIndexes; - } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 75531a5..6d86963 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -8,6 +8,7 @@ FOS\ElasticaBundle\Elastica\Client FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector + FOS\ElasticaBundle\Index\MappingBuilder Symfony\Component\PropertyAccess\PropertyAccessor @@ -32,6 +33,8 @@ + + diff --git a/Resources/config/index.xml b/Resources/config/index.xml index 60b75a9..17a0ec9 100644 --- a/Resources/config/index.xml +++ b/Resources/config/index.xml @@ -5,6 +5,7 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> + FOS\ElasticaBundle\Index\AliasProcessor FOS\ElasticaBundle\Finder\TransformedFinder FOS\ElasticaBundle\Elastica\Index FOS\ElasticaBundle\Provider\Indexable @@ -14,6 +15,8 @@ + + @@ -35,6 +38,8 @@ + + diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index ed4787a..114ac5e 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -13,15 +13,16 @@ - + - + + - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index bbc1ec4..1f547b9 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -13,16 +13,16 @@ - + - - + + - + diff --git a/Tests/FOSElasticaBundleTest.php b/Tests/FOSElasticaBundleTest.php index 2bfc7f9..3828e8b 100644 --- a/Tests/FOSElasticaBundleTest.php +++ b/Tests/FOSElasticaBundleTest.php @@ -9,31 +9,16 @@ class FOSElasticaBundleTest extends \PHPUnit_Framework_TestCase { public function testCompilerPassesAreRegistered() { - $passes = array( - array ( - 'FOS\ElasticaBundle\DependencyInjection\Compiler\RegisterProvidersPass', - PassConfig::TYPE_BEFORE_REMOVING - ), - array ( - 'FOS\ElasticaBundle\DependencyInjection\Compiler\TransformerPass' - ), - ); - $container = $this ->getMock('Symfony\Component\DependencyInjection\ContainerBuilder'); $container - ->expects($this->at(0)) + ->expects($this->atLeastOnce()) ->method('addCompilerPass') - ->with($this->isInstanceOf($passes[0][0]), $passes[0][1]); + ->with($this->isInstanceOf('Symfony\\Component\\DependencyInjection\\Compiler\\CompilerPassInterface')); - $container - ->expects($this->at(1)) - ->method('addCompilerPass') - ->with($this->isInstanceOf($passes[1][0])); $bundle = new FOSElasticaBundle(); - $bundle->build($container); } } diff --git a/Tests/Index/IndexManagerTest.php b/Tests/Index/IndexManagerTest.php index 624f64e..98e4d8a 100644 --- a/Tests/Index/IndexManagerTest.php +++ b/Tests/Index/IndexManagerTest.php @@ -6,40 +6,41 @@ use FOS\ElasticaBundle\Index\IndexManager; class IndexManagerTest extends \PHPUnit_Framework_TestCase { - private $defaultIndexName; - private $indexesByName; - /** @var IndexManager */ + private $indexes = array(); + + /** + * @var IndexManager + */ private $indexManager; + public function setUp() { - $this->defaultIndexName = 'index2'; - $this->indexesByName = array( - 'index1' => 'test1', - 'index2' => 'test2', - ); + foreach (array('index1', 'index2', 'index3') as $indexName) { + $index = $this->getMockBuilder('FOS\\ElasticaBundle\\Elastica\\Index') + ->disableOriginalConstructor() + ->getMock(); - /** @var $defaultIndex \PHPUnit_Framework_MockObject_MockObject|\Elastica\Index */ - $defaultIndex = $this->getMockBuilder('Elastica\Index') - ->disableOriginalConstructor() - ->getMock(); + $index->expects($this->any()) + ->method('getName') + ->will($this->returnValue($indexName)); - $defaultIndex->expects($this->any()) - ->method('getName') - ->will($this->returnValue($this->defaultIndexName)); + $this->indexes[$indexName] = $index; + } - $this->indexManager = new IndexManager($this->indexesByName, $defaultIndex); + $this->indexManager = new IndexManager($this->indexes, $this->indexes['index2']); } public function testGetAllIndexes() { - $this->assertEquals($this->indexesByName, $this->indexManager->getAllIndexes()); + $this->assertEquals($this->indexes, $this->indexManager->getAllIndexes()); } public function testGetIndex() { - $this->assertEquals($this->indexesByName['index1'], $this->indexManager->getIndex('index1')); - $this->assertEquals($this->indexesByName['index2'], $this->indexManager->getIndex('index2')); + $this->assertEquals($this->indexes['index1'], $this->indexManager->getIndex('index1')); + $this->assertEquals($this->indexes['index2'], $this->indexManager->getIndex('index2')); + $this->assertEquals($this->indexes['index3'], $this->indexManager->getIndex('index3')); } /** @@ -47,12 +48,12 @@ class IndexManagerTest extends \PHPUnit_Framework_TestCase */ public function testGetIndexShouldThrowExceptionForInvalidName() { - $this->indexManager->getIndex('index3'); + $this->indexManager->getIndex('index4'); } public function testGetDefaultIndex() { - $this->assertEquals('test2', $this->indexManager->getIndex()); - $this->assertEquals('test2', $this->indexManager->getDefaultIndex()); + $this->assertEquals('index2', $this->indexManager->getIndex()->getName()); + $this->assertEquals('index2', $this->indexManager->getDefaultIndex()->getName()); } } diff --git a/Tests/Index/ResetterTest.php b/Tests/Index/ResetterTest.php index ae7fdf5..28f0a68 100644 --- a/Tests/Index/ResetterTest.php +++ b/Tests/Index/ResetterTest.php @@ -6,15 +6,40 @@ use Elastica\Exception\ResponseException; use Elastica\Request; use Elastica\Response; use Elastica\Type\Mapping; +use FOS\ElasticaBundle\Configuration\IndexConfig; use FOS\ElasticaBundle\Index\Resetter; class ResetterTest extends \PHPUnit_Framework_TestCase { - private $indexConfigsByName; + /** + * @var Resetter + */ + private $resetter; + + private $configManager; + private $indexManager; + private $aliasProcessor; + private $mappingBuilder; public function setUp() { - $this->indexConfigsByName = array( + $this->markTestIncomplete('To be rewritten'); + $this->configManager = $this->getMockBuilder('FOS\\ElasticaBundle\\Configuration\\ConfigManager') + ->disableOriginalConstructor() + ->getMock(); + $this->indexManager = $this->getMockBuilder('FOS\\ElasticaBundle\\Index\\IndexManager') + ->disableOriginalConstructor() + ->getMock(); + $this->aliasProcessor = $this->getMockBuilder('FOS\\ElasticaBundle\\Index\\AliasProcessor') + ->disableOriginalConstructor() + ->getMock(); + $this->mappingBuilder = $this->getMockBuilder('FOS\\ElasticaBundle\\Index\\MappingBuilder') + ->disableOriginalConstructor() + ->getMock(); + + $this->resetter = new Resetter($this->configManager, $this->indexManager, $this->aliasProcessor, $this->mappingBuilder); + + /*$this->indexConfigsByName = array( 'foo' => array( 'index' => $this->getMockElasticaIndex(), 'config' => array( @@ -54,12 +79,26 @@ class ResetterTest extends \PHPUnit_Framework_TestCase ), ), ), - ); + );*/ } public function testResetAllIndexes() { - $this->indexConfigsByName['foo']['index']->expects($this->once()) + $this->configManager->expects($this->once()) + ->method('getIndexNames') + ->will($this->returnValue(array('index1'))); + + $this->configManager->expects($this->once()) + ->method('getIndexConfiguration') + ->with('index1') + ->will($this->returnValue(new IndexConfig('index1', array(), array()))); + + $this->indexManager->expects($this->once()) + ->method('getIndex') + ->with('index1') + ->will($this->returnValue()); + + /*$this->indexConfigsByName['foo']['index']->expects($this->once()) ->method('create') ->with($this->indexConfigsByName['foo']['config'], true); @@ -67,8 +106,8 @@ class ResetterTest extends \PHPUnit_Framework_TestCase ->method('create') ->with($this->indexConfigsByName['bar']['config'], true); - $resetter = new Resetter($this->indexConfigsByName); - $resetter->resetAllIndexes(); + $resetter = new Resetter($this->indexConfigsByName);*/ + $this->resetter->resetAllIndexes(); } public function testResetIndex() From c97f0f1ddf9a0590bf8a978ee1131ce564860356 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 19 Jun 2014 11:14:13 +1000 Subject: [PATCH 101/154] Fix provider bailing if the indexable service filters an entire batch of objects --- Doctrine/AbstractProvider.php | 5 +++++ Propel/Provider.php | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 0c43b2d..44be6c7 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -64,6 +64,11 @@ abstract class AbstractProvider extends BaseAbstractProvider $stepNbObjects = count($objects); } $objects = array_filter($objects, array($this, 'isObjectIndexable')); + if (!$objects) { + $loggerClosure('Entire batch was filtered away, skipping...'); + + continue; + } if (!$ignoreErrors) { $this->objectPersister->insertMany($objects); diff --git a/Propel/Provider.php b/Propel/Provider.php index c242d47..38f7a61 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -36,6 +36,11 @@ class Provider extends AbstractProvider $stepNbObjects = count($objects); } $objects = array_filter($objects, array($this, 'isObjectIndexable')); + if (!$objects) { + $loggerClosure('Entire batch was filtered away, skipping...'); + + continue; + } $this->objectPersister->insertMany($objects); From ca6991d494e78844ce1007fecef5479a2a40b739 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 23 Jun 2014 23:05:57 +1000 Subject: [PATCH 102/154] Fix serializer --- DependencyInjection/FOSElasticaExtension.php | 55 ++++++++++++------- Resources/config/serializer.xml | 14 +++++ Tests/Functional/app/ORM/bundles.php | 4 +- Tests/Functional/app/ORM/config.yml | 4 ++ .../ModelToElasticaIdentifierTransformer.php | 1 + composer.json | 1 + 6 files changed, 57 insertions(+), 22 deletions(-) create mode 100644 Resources/config/serializer.xml diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 158bea5..aa00b18 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -5,7 +5,6 @@ namespace FOS\ElasticaBundle\DependencyInjection; use Symfony\Component\HttpKernel\DependencyInjection\Extension; use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; use Symfony\Component\DependencyInjection\ContainerBuilder; -use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\DefinitionDecorator; use Symfony\Component\DependencyInjection\Reference; use Symfony\Component\Config\FileLocator; @@ -35,8 +34,6 @@ class FOSElasticaExtension extends Extension */ private $loadedDrivers = array(); - protected $serializerConfig = array(); - public function load(array $configs, ContainerBuilder $container) { $configuration = $this->getConfiguration($configs, $container); @@ -63,7 +60,11 @@ class FOSElasticaExtension extends Extension $config['default_index'] = reset($keys); } - $this->serializerConfig = isset($config['serializer']) ? $config['serializer'] : null; + if (isset($config['serializer'])) { + $loader->load('serializer.xml'); + + $this->loadSerializer($config['serializer'], $container); + } $this->loadClients($config['clients'], $container); $container->setAlias('fos_elastica.client', sprintf('fos_elastica.client.%s', $config['default_client'])); @@ -253,27 +254,20 @@ class FOSElasticaExtension extends Extension $indexableCallbacks[sprintf('%s/%s', $indexName, $name)] = $type['indexable_callback']; } - /*if ($this->serializerConfig) { - $callbackDef = new Definition($this->serializerConfig['callback_class']); - $callbackId = sprintf('%s.%s.serializer.callback', $indexId, $name); + if ($container->hasDefinition('fos_elastica.serializer_callback_prototype')) { + $typeSerializerId = sprintf('%s.serializer.callback', $typeId); + $typeSerializerDef = new DefinitionDecorator('fos_elastica.serializer_callback_prototype'); - $typeDef->addMethodCall('setSerializer', array(array(new Reference($callbackId), 'serialize'))); - $callbackDef->addMethodCall('setSerializer', array(new Reference($this->serializerConfig['serializer']))); if (isset($type['serializer']['groups'])) { - $callbackDef->addMethodCall('setGroups', array($type['serializer']['groups'])); + $typeSerializerDef->addMethodCall('setGroups', array($type['serializer']['groups'])); } if (isset($type['serializer']['version'])) { - $callbackDef->addMethodCall('setVersion', array($type['serializer']['version'])); - } - $callbackClassImplementedInterfaces = class_implements($this->serializerConfig['callback_class']); // PHP < 5.4 friendly - if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) { - $callbackDef->addMethodCall('setContainer', array(new Reference('service_container'))); + $typeSerializerDef->addMethodCall('setVersion', array($type['serializer']['version'])); } - $container->setDefinition($callbackId, $callbackDef); - - $typeDef->addMethodCall('setSerializer', array(array(new Reference($callbackId), 'serialize'))); - }*/ + $typeDef->addMethodCall('setSerializer', array(array(new Reference($typeSerializerId), 'serialize'))); + $container->setDefinition($typeSerializerId, $typeSerializerDef); + } } $indexable = $container->getDefinition('fos_elastica.indexable'); @@ -358,7 +352,7 @@ class FOSElasticaExtension extends Extension return $typeConfig['model_to_elastica_transformer']['service']; } - $abstractId = $this->serializerConfig ? + $abstractId = $container->hasDefinition('fos_elastica.serializer_callback_prototype') ? 'fos_elastica.model_to_elastica_identifier_transformer' : 'fos_elastica.model_to_elastica_transformer'; @@ -391,7 +385,7 @@ class FOSElasticaExtension extends Extension $typeConfig['model'], ); - if ($this->serializerConfig) { + if ($container->hasDefinition('fos_elastica.serializer_callback_prototype')) { $abstractId = 'fos_elastica.object_serializer_persister'; $callbackId = sprintf('%s.%s.serializer.callback', $this->indexConfigs[$indexName]['reference'], $typeName); $arguments[] = array(new Reference($callbackId), 'serialize'); @@ -588,6 +582,25 @@ class FOSElasticaExtension extends Extension $this->loadedDrivers[] = $driver; } + /** + * Loads and configures the serializer prototype. + * + * @param array $config + * @param ContainerBuilder $container + */ + private function loadSerializer($config, ContainerBuilder $container) + { + $container->setAlias('fos_elastica.serializer', $config['serializer']); + + $serializer = $container->getDefinition('fos_elastica.serializer_callback_prototype'); + $serializer->setClass($config['callback_class']); + + $callbackClassImplementedInterfaces = class_implements($config['callback_class']); + if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) { + $serializer->addMethodCall('setContainer', array(new Reference('service_container'))); + } + } + /** * Creates a default manager alias for defined default manager or the first loaded driver. * diff --git a/Resources/config/serializer.xml b/Resources/config/serializer.xml new file mode 100644 index 0000000..8ee9646 --- /dev/null +++ b/Resources/config/serializer.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/Tests/Functional/app/ORM/bundles.php b/Tests/Functional/app/ORM/bundles.php index d0b6efb..25db3fe 100644 --- a/Tests/Functional/app/ORM/bundles.php +++ b/Tests/Functional/app/ORM/bundles.php @@ -3,9 +3,11 @@ use Doctrine\Bundle\DoctrineBundle\DoctrineBundle; use FOS\ElasticaBundle\FOSElasticaBundle; use Symfony\Bundle\FrameworkBundle\FrameworkBundle; +use JMS\SerializerBundle\JMSSerializerBundle; return array( new FrameworkBundle(), new FOSElasticaBundle(), - new DoctrineBundle() + new DoctrineBundle(), + new JMSSerializerBundle(), ); diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 77e5f5b..340fa65 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -13,6 +13,7 @@ fos_elastica: clients: default: url: http://localhost:9200 + serializer: ~ indexes: index: index_name: foselastica_test_%kernel.environment% @@ -25,6 +26,9 @@ fos_elastica: model: FOS\ElasticaBundle\Tests\Functional\TypeObj listener: is_indexable_callback: 'object.isIndexable() && !object.isntIndexable()' + serializer: + groups: ['search'] + version: 1.1 type2: properties: field1: ~ diff --git a/Transformer/ModelToElasticaIdentifierTransformer.php b/Transformer/ModelToElasticaIdentifierTransformer.php index 654850f..7cf97e6 100644 --- a/Transformer/ModelToElasticaIdentifierTransformer.php +++ b/Transformer/ModelToElasticaIdentifierTransformer.php @@ -21,6 +21,7 @@ class ModelToElasticaIdentifierTransformer extends ModelToElasticaAutoTransforme public function transform($object, array $fields) { $identifier = $this->propertyAccessor->getValue($object, $this->options['identifier']); + return new Document($identifier); } } diff --git a/composer.json b/composer.json index bde8b3e..b65e0b2 100644 --- a/composer.json +++ b/composer.json @@ -24,6 +24,7 @@ "doctrine/orm": "~2.2", "doctrine/doctrine-bundle": "~1.2@beta", "doctrine/mongodb-odm": "1.0.*@beta", + "jms/serializer-bundle": "@stable", "phpunit/phpunit": "~4.1", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev", From 9a62187329c0c6f7079db468b381bb1c3b611000 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 23 Jun 2014 23:50:52 +1000 Subject: [PATCH 103/154] Fix undefined index_name notice --- DependencyInjection/FOSElasticaExtension.php | 2 +- Tests/Functional/app/ORM/config.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index aa00b18..9862076 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -130,7 +130,7 @@ class FOSElasticaExtension extends Extension { foreach ($indexes as $name => $index) { $indexId = sprintf('fos_elastica.index.%s', $name); - $indexName = $index['index_name'] ?: $name; + $indexName = isset($index['index_name']) ? $index['index_name']: $name; $indexDef = new DefinitionDecorator('fos_elastica.index_prototype'); $indexDef->replaceArgument(0, $indexName); diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 340fa65..2c7c692 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -15,6 +15,11 @@ fos_elastica: url: http://localhost:9200 serializer: ~ indexes: + fos_elastica_test: + types: + type: + properties: + field1: ~ index: index_name: foselastica_test_%kernel.environment% types: From ae3605828ea5deed3f4c2424d1dc8f8879c8b4c0 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 24 Jun 2014 10:19:26 +1000 Subject: [PATCH 104/154] Fix AbstractProvider test --- Doctrine/AbstractProvider.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 44be6c7..a662fd4 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -65,7 +65,9 @@ abstract class AbstractProvider extends BaseAbstractProvider } $objects = array_filter($objects, array($this, 'isObjectIndexable')); if (!$objects) { - $loggerClosure('Entire batch was filtered away, skipping...'); + if ($loggerClosure) { + $loggerClosure('Entire batch was filtered away, skipping...'); + } continue; } From 2437a098ba92422c0d5111e5f66fa46479cd1b3c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 24 Jun 2014 10:20:15 +1000 Subject: [PATCH 105/154] Fix anonymous service error --- Resources/config/mongodb.xml | 5 ++--- Resources/config/orm.xml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 114ac5e..048d799 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -14,12 +14,11 @@ - + - - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 1f547b9..ddc0e50 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -14,12 +14,11 @@ - + - - + From f6264f4149990f26c247419e4e6f26b34fadcd8d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 24 Jun 2014 10:30:31 +1000 Subject: [PATCH 106/154] Fix AbstractProvider tests --- Tests/Doctrine/AbstractProviderTest.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index cd0a58c..99ed2de 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -54,6 +54,11 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->with($queryBuilder) ->will($this->returnValue($nbObjects)); + $this->indexable->expects($this->any()) + ->method('isObjectIndexable') + ->with('index', 'type', $this->anything()) + ->will($this->returnValue(true)); + $providerInvocationOffset = 2; foreach ($objectsByIteration as $i => $objects) { @@ -106,6 +111,11 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->method('fetchSlice') ->will($this->returnValue($objects)); + $this->indexable->expects($this->any()) + ->method('isObjectIndexable') + ->with('index', 'type', $this->anything()) + ->will($this->returnValue(true)); + $this->objectManager->expects($this->never()) ->method('clear'); @@ -127,6 +137,11 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->method('fetchSlice') ->will($this->returnValue($objects)); + $this->indexable->expects($this->any()) + ->method('isObjectIndexable') + ->with('index', 'type', $this->anything()) + ->will($this->returnValue(true)); + $loggerClosureInvoked = false; $loggerClosure = function () use (&$loggerClosureInvoked) { $loggerClosureInvoked = true; @@ -154,6 +169,11 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->method('fetchSlice') ->will($this->returnValue($objects)); + $this->indexable->expects($this->any()) + ->method('isObjectIndexable') + ->with('index', 'type', $this->anything()) + ->will($this->returnValue(true)); + $this->objectPersister->expects($this->any()) ->method('insertMany') ->will($this->throwException($this->getMockBulkResponseException())); From ae03b3f3cff1744bf7e819dcecb9b808abf310af Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 25 Jun 2014 13:18:02 +1000 Subject: [PATCH 107/154] Fix propel service definition --- Resources/config/propel.xml | 2 +- Tests/bootstrap.php | 19 ------------------- phpunit.xml.dist | 2 +- 3 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 Tests/bootstrap.php diff --git a/Resources/config/propel.xml b/Resources/config/propel.xml index 102935c..297e735 100644 --- a/Resources/config/propel.xml +++ b/Resources/config/propel.xml @@ -19,7 +19,7 @@ - + diff --git a/Tests/bootstrap.php b/Tests/bootstrap.php deleted file mode 100644 index 30fb165..0000000 --- a/Tests/bootstrap.php +++ /dev/null @@ -1,19 +0,0 @@ - - + ./Tests From 4d52c327aa2c43edbc38cf7ef5af45d2fa5bf80c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 25 Jun 2014 13:24:02 +1000 Subject: [PATCH 108/154] Add back missing KnpPaginatorSubscriber service --- Resources/config/config.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 6d86963..023688f 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -27,6 +27,13 @@ + + + + + + + %kernel.debug% From a879d3c1c98b4e706148928355d2a114720783ab Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 26 Jun 2014 17:29:16 +1000 Subject: [PATCH 109/154] Fix ClassCastException when no settings are present --- Index/MappingBuilder.php | 6 +++++- Tests/Functional/MappingToElasticaTest.php | 24 ++++++++++++++++++++++ Tests/Functional/app/ORM/config.yml | 5 +++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 53524ac..0751ce7 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -31,10 +31,14 @@ class MappingBuilder $mapping = array( 'mappings' => $typeMappings, - 'settings' => $indexConfig->getSettings(), // 'warmers' => $indexConfig->getWarmers(), ); + $settings = $indexConfig->getSettings(); + if ($settings) { + $mapping['settings'] = $settings; + } + return $mapping; } diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index bffcf76..390f854 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -42,6 +42,30 @@ class MappingToElasticaTest extends WebTestCase $this->assertNotEmpty($mapping, 'Mapping was populated'); } + public function testORMResetIndexAddsMappings() + { + $client = $this->createClient(array('test_case' => 'ORM')); + $resetter = $this->getResetter($client); + $resetter->resetIndex('index'); + + $type = $this->getType($client); + $mapping = $type->getMapping(); + + $this->assertNotEmpty($mapping, 'Mapping was populated'); + } + + public function testORMResetType() + { + $client = $this->createClient(array('test_case' => 'ORM')); + $resetter = $this->getResetter($client); + $resetter->resetIndexType('index', 'type'); + + $type = $this->getType($client); + $mapping = $type->getMapping(); + + $this->assertNotEmpty($mapping, 'Mapping was populated'); + } + /** * @param Client $client * @return \FOS\ElasticaBundle\Resetter $resetter diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 2c7c692..448d62b 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -43,11 +43,12 @@ fos_elastica: listener: is_indexable_callback: 'object.isntIndexable()' type3: - properties: + mappings: field1: ~ persistence: driver: orm model: FOS\ElasticaBundle\Tests\Functional\TypeObj + finder: ~ + provider: ~ listener: is_indexable_callback: 'isntIndexable' - From 96dc613c7125670d237da725f8f28117a8bae573 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 26 Jun 2014 17:46:29 +1000 Subject: [PATCH 110/154] Fix array format for indexable_callback --- DependencyInjection/Configuration.php | 2 +- Provider/Indexable.php | 15 ++++++++++- Resources/config/index.xml | 1 + Resources/doc/types.md | 3 +++ Tests/Functional/IndexableCallbackTest.php | 3 ++- Tests/Functional/app/ORM/IndexableService.php | 25 +++++++++++++++++++ Tests/Functional/app/ORM/config.yml | 16 +++++++++++- 7 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 Tests/Functional/app/ORM/IndexableService.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 3564fd4..52088b6 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -205,7 +205,7 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() - ->scalarNode('indexable_callback')->end() + ->variableNode('indexable_callback')->end() ->append($this->getPersistenceNode()) ->append($this->getSerializerNode()) ->end() diff --git a/Provider/Indexable.php b/Provider/Indexable.php index 09168a1..827b3a4 100644 --- a/Provider/Indexable.php +++ b/Provider/Indexable.php @@ -11,6 +11,7 @@ namespace FOS\ElasticaBundle\Provider; +use Symfony\Component\DependencyInjection\ContainerInterface; use Symfony\Component\ExpressionLanguage\Expression; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; use Symfony\Component\ExpressionLanguage\SyntaxError; @@ -26,6 +27,11 @@ class Indexable implements IndexableInterface */ private $callbacks = array(); + /** + * @var \Symfony\Component\DependencyInjection\ContainerInterface + */ + private $container; + /** * An instance of ExpressionLanguage * @@ -50,9 +56,10 @@ class Indexable implements IndexableInterface /** * @param array $callbacks */ - public function __construct(array $callbacks) + public function __construct(array $callbacks, ContainerInterface $container) { $this->callbacks = $callbacks; + $this->container = $container; $this->propertyAccessor = PropertyAccess::createPropertyAccessor(); } @@ -105,6 +112,12 @@ class Indexable implements IndexableInterface if (is_array($callback)) { list($class, $method) = $callback + array(null, null); + if (strpos($class, '@') === 0) { + $service = $this->container->get(substr($class, 1)); + + return array($service, $method); + } + if (is_object($class)) { $class = get_class($class); } diff --git a/Resources/config/index.xml b/Resources/config/index.xml index 17a0ec9..11586ff 100644 --- a/Resources/config/index.xml +++ b/Resources/config/index.xml @@ -19,6 +19,7 @@ + diff --git a/Resources/doc/types.md b/Resources/doc/types.md index be687d7..80d295b 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -177,6 +177,9 @@ The callback option supports multiple approaches: * An array of a service id and a method which will be called with the object as the first and only argument. `[ @my_custom_service, 'userIndexable' ]` will call the userIndexable method on a service defined as my_custom_service. +* An array of a class and a static method to call on that class which will be called with + the object as the only argument. `[ 'Acme\DemoBundle\IndexableChecker', 'isIndexable' ]` + will call Acme\DemoBundle\IndexableChecker::isIndexable($object) * If you have the ExpressionLanguage component installed, A valid ExpressionLanguage expression provided as a string. The object being indexed will be supplied as `object` in the expression. `object.isEnabled() or object.shouldBeIndexedAnyway()`. For more diff --git a/Tests/Functional/IndexableCallbackTest.php b/Tests/Functional/IndexableCallbackTest.php index 89fca1d..41ed402 100644 --- a/Tests/Functional/IndexableCallbackTest.php +++ b/Tests/Functional/IndexableCallbackTest.php @@ -31,8 +31,9 @@ class IndexableCallbackTest extends WebTestCase $in = $client->getContainer()->get('fos_elastica.indexable'); $this->assertTrue($in->isObjectIndexable('index', 'type', new TypeObj())); - $this->assertFalse($in->isObjectIndexable('index', 'type2', new TypeObj())); + $this->assertTrue($in->isObjectIndexable('index', 'type2', new TypeObj())); $this->assertFalse($in->isObjectIndexable('index', 'type3', new TypeObj())); + $this->assertFalse($in->isObjectIndexable('index', 'type4', new TypeObj())); } protected function setUp() diff --git a/Tests/Functional/app/ORM/IndexableService.php b/Tests/Functional/app/ORM/IndexableService.php new file mode 100644 index 0000000..018451e --- /dev/null +++ b/Tests/Functional/app/ORM/IndexableService.php @@ -0,0 +1,25 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Tests\Functional\app\ORM; + +class IndexableService +{ + public function isIndexable($object) + { + return true; + } + + public static function isntIndexable($object) + { + return false; + } +} diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 448d62b..9ba6830 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -9,6 +9,10 @@ doctrine: auto_generate_proxy_classes: false auto_mapping: false +services: + indexableService: + class: FOS\ElasticaBundle\Tests\Functional\app\ORM\IndexableService + fos_elastica: clients: default: @@ -41,7 +45,7 @@ fos_elastica: driver: orm model: FOS\ElasticaBundle\Tests\Functional\TypeObj listener: - is_indexable_callback: 'object.isntIndexable()' + is_indexable_callback: [ @indexableService, 'isIndexable' ] type3: mappings: field1: ~ @@ -52,3 +56,13 @@ fos_elastica: provider: ~ listener: is_indexable_callback: 'isntIndexable' + type4: + mappings: + field1: ~ + persistence: + driver: orm + model: FOS\ElasticaBundle\Tests\Functional\TypeObj + finder: ~ + provider: ~ + listener: + is_indexable_callback: [ 'FOS\ElasticaBundle\Tests\Functional\app\ORM\IndexableService', 'isntIndexable' ] From 77f2b99a3e83f8cb52865c9ddadae0c3a7fbb8fa Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 26 Jun 2014 18:01:34 +1000 Subject: [PATCH 111/154] Fix Indexable tests --- Provider/Indexable.php | 9 +++++---- Tests/Provider/IndexableTest.php | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Provider/Indexable.php b/Provider/Indexable.php index 827b3a4..197aeb8 100644 --- a/Provider/Indexable.php +++ b/Provider/Indexable.php @@ -112,16 +112,17 @@ class Indexable implements IndexableInterface if (is_array($callback)) { list($class, $method) = $callback + array(null, null); + + if (is_object($class)) { + $class = get_class($class); + } + if (strpos($class, '@') === 0) { $service = $this->container->get(substr($class, 1)); return array($service, $method); } - if (is_object($class)) { - $class = get_class($class); - } - if ($class && $method) { throw new \InvalidArgumentException(sprintf('Callback for type "%s", "%s::%s()", is not callable.', $type, $class, $method)); } diff --git a/Tests/Provider/IndexableTest.php b/Tests/Provider/IndexableTest.php index aae6484..6ef5669 100644 --- a/Tests/Provider/IndexableTest.php +++ b/Tests/Provider/IndexableTest.php @@ -12,12 +12,15 @@ namespace FOS\ElasticaBundle\Tests\Provider; use FOS\ElasticaBundle\Provider\Indexable; +use Symfony\Component\DependencyInjection\Exception\InvalidArgumentException; class IndexableTest extends \PHPUnit_Framework_TestCase { + public $container; + public function testIndexableUnknown() { - $indexable = new Indexable(array()); + $indexable = new Indexable(array(), $this->container); $index = $indexable->isObjectIndexable('index', 'type', new Entity); $this->assertTrue($index); @@ -30,7 +33,7 @@ class IndexableTest extends \PHPUnit_Framework_TestCase { $indexable = new Indexable(array( 'index/type' => $callback - )); + ), $this->container); $index = $indexable->isObjectIndexable('index', 'type', new Entity); $this->assertEquals($return, $index); @@ -44,7 +47,7 @@ class IndexableTest extends \PHPUnit_Framework_TestCase { $indexable = new Indexable(array( 'index/type' => $callback - )); + ), $this->container); $indexable->isObjectIndexable('index', 'type', new Entity); } @@ -63,12 +66,24 @@ class IndexableTest extends \PHPUnit_Framework_TestCase return array( array('isIndexable', false), array(array(new IndexableDecider(), 'isIndexable'), true), + array(array('@indexableService', 'isIndexable'), true), array(function(Entity $entity) { return $entity->maybeIndex(); }, true), array('entity.maybeIndex()', true), array('!object.isIndexable() && entity.property == "abc"', true), array('entity.property != "abc"', false), ); } + + protected function setUp() + { + $this->container = $this->getMockBuilder('Symfony\\Component\\DependencyInjection\\ContainerInterface') + ->getMock(); + + $this->container->expects($this->any()) + ->method('get') + ->with('indexableService') + ->will($this->returnValue(new IndexableDecider())); + } } class Entity From 474cbfa979e327080c4155dc6253491bbecd1043 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 26 Jun 2014 19:51:16 +1000 Subject: [PATCH 112/154] Added test coverage for ES store --- Index/MappingBuilder.php | 1 - Tests/Functional/MappingToElasticaTest.php | 6 ++++++ Tests/Functional/app/Basic/config.yml | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 0751ce7..4a6c70a 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -69,7 +69,6 @@ class MappingBuilder return $mapping; } - /** * create type mapping object * diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index 390f854..f038212 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -28,6 +28,9 @@ class MappingToElasticaTest extends WebTestCase $mapping = $type->getMapping(); $this->assertNotEmpty($mapping, 'Mapping was populated'); + $this->assertArrayHasKey('store', $mapping['type']['properties']['field1']); + $this->assertTrue($mapping['type']['properties']['field1']['store']); + $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); } public function testResetType() @@ -40,6 +43,9 @@ class MappingToElasticaTest extends WebTestCase $mapping = $type->getMapping(); $this->assertNotEmpty($mapping, 'Mapping was populated'); + $this->assertArrayHasKey('store', $mapping['type']['properties']['field1']); + $this->assertTrue($mapping['type']['properties']['field1']['store']); + $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); } public function testORMResetIndexAddsMappings() diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 174f9d5..0f2403b 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -38,6 +38,7 @@ fos_elastica: field1: ~ field2: type: integer + store: false date: { boost: 5 } title: { boost: 8, analyzer: my_analyzer } content: ~ From ad37a2835688ec88171caa893e3ae0e7ac4a1191 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 27 Jun 2014 15:08:08 +1000 Subject: [PATCH 113/154] Fix populating command setting alias before population --- Command/PopulateCommand.php | 2 +- Index/AliasProcessor.php | 12 +++++++----- Index/Resetter.php | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index af5fd5d..f17ca4c 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -111,7 +111,7 @@ class PopulateCommand extends ContainerAwareCommand { if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); - $this->resetter->resetIndex($index); + $this->resetter->resetIndex($index, true); } /** @var $providers ProviderInterface[] */ diff --git a/Index/AliasProcessor.php b/Index/AliasProcessor.php index 5c4592d..93877cd 100644 --- a/Index/AliasProcessor.php +++ b/Index/AliasProcessor.php @@ -39,11 +39,13 @@ class AliasProcessor */ public function switchIndexAlias(IndexConfig $indexConfig, Index $index) { + $client = $index->getClient(); + $aliasName = $indexConfig->getElasticSearchName(); $oldIndexName = false; $newIndexName = $index->getName(); - $aliasedIndexes = $this->getAliasedIndexes($aliasName); + $aliasedIndexes = $this->getAliasedIndexes($client, $aliasName); if (count($aliasedIndexes) > 1) { throw new \RuntimeException( @@ -71,7 +73,7 @@ class AliasProcessor ); try { - $this->client->request('_aliases', 'POST', $aliasUpdateRequest); + $client->request('_aliases', 'POST', $aliasUpdateRequest); } catch (ExceptionInterface $renameAliasException) { $additionalError = ''; // if we failed to move the alias, delete the newly built index @@ -96,7 +98,7 @@ class AliasProcessor // Delete the old index after the alias has been switched if ($oldIndexName) { - $oldIndex = new Index($this->client, $oldIndexName); + $oldIndex = new Index($client, $oldIndexName); try { $oldIndex->delete(); } catch (ExceptionInterface $deleteOldIndexException) { @@ -117,9 +119,9 @@ class AliasProcessor * @param string $aliasName Alias name * @return array */ - private function getAliasedIndexes($aliasName) + private function getAliasedIndexes(Client $client, $aliasName) { - $aliasesInfo = $this->client->request('_aliases', 'GET')->getData(); + $aliasesInfo = $client->request('_aliases', 'GET')->getData(); $aliasedIndexes = array(); foreach ($aliasesInfo as $indexName => $indexInfo) { diff --git a/Index/Resetter.php b/Index/Resetter.php index 174c410..3f07fa1 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -45,10 +45,10 @@ class Resetter /** * Deletes and recreates all indexes */ - public function resetAllIndexes() + public function resetAllIndexes($populating = false) { foreach ($this->configManager->getIndexNames() as $name) { - $this->resetIndex($name); + $this->resetIndex($name, $populating); } } From 78db0b9b6393f56af79185bd55c1dbcc0cca9652 Mon Sep 17 00:00:00 2001 From: Ron van der Molen Date: Sun, 29 Jun 2014 22:14:32 +0200 Subject: [PATCH 114/154] [FEATURE] Use static instantiation with max depth check --- Serializer/Callback.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Serializer/Callback.php b/Serializer/Callback.php index 9fe7064..38d93dc 100644 --- a/Serializer/Callback.php +++ b/Serializer/Callback.php @@ -43,7 +43,7 @@ class Callback public function serialize($object) { - $context = $this->serializer instanceof SerializerInterface ? new SerializationContext() : array(); + $context = $this->serializer instanceof SerializerInterface ? SerializationContext::create()->enableMaxDepthChecks() : array(); if ($this->groups) { $context->setGroups($this->groups); From c9a24436f3f27f7174db43e206d2093282694494 Mon Sep 17 00:00:00 2001 From: Tobias Schultze Date: Mon, 30 Jun 2014 16:19:10 +0200 Subject: [PATCH 115/154] dynamic templates are a list of hashes and have a mapping key (not properties) --- DependencyInjection/Configuration.php | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 52088b6..49b7b97 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -250,16 +250,21 @@ class Configuration implements ConfigurationInterface $node = $builder->root('dynamic_templates'); $node - ->useAttributeAsKey('name') ->prototype('array') - ->children() - ->scalarNode('match')->end() - ->scalarNode('unmatch')->end() - ->scalarNode('match_mapping_type')->end() - ->scalarNode('path_match')->end() - ->scalarNode('path_unmatch')->end() - ->scalarNode('match_pattern')->end() - ->append($this->getPropertiesNode()) + ->prototype('array') + ->children() + ->scalarNode('match')->end() + ->scalarNode('unmatch')->end() + ->scalarNode('match_mapping_type')->end() + ->scalarNode('path_match')->end() + ->scalarNode('path_unmatch')->end() + ->scalarNode('match_pattern')->end() + ->arrayNode('mapping') + ->prototype('variable') + ->treatNullLike(array()) + ->end() + ->end() + ->end() ->end() ->end() ; From 5d6567665955136caf3af10978403f55fa5dde1f Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 1 Jul 2014 10:09:25 +1000 Subject: [PATCH 116/154] Add tests and normalisation to support old dynamic_templates format --- DependencyInjection/Configuration.php | 20 ++++++++++++++++++ Index/MappingBuilder.php | 29 ++++----------------------- Tests/Functional/app/Basic/config.yml | 15 ++++++++++++++ 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 49b7b97..ae27860 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -189,6 +189,7 @@ class Configuration implements ConfigurationInterface return $v; }) ->end() + // BC - Support the old is_indexable_callback property ->beforeNormalization() ->ifTrue(function ($v) { return isset($v['persistence']) && @@ -202,6 +203,25 @@ class Configuration implements ConfigurationInterface return $v; }) ->end() + // Support multiple dynamic_template formats to match the old bundle style + // and the way ElasticSearch expects them + ->beforeNormalization() + ->ifTrue(function ($v) { return isset($v['dynamic_templates']); }) + ->then(function ($v) { + $dt = array(); + foreach ($v['dynamic_templates'] as $key => $type) { + if (is_numeric($key)) { + $dt[$key] = $type; + } else { + $dt[][$key] = $type; + } + } + + $v['dynamic_templates'] = $dt; + + return $v; + }) + ->end() ->children() ->scalarNode('index_analyzer')->end() ->scalarNode('search_analyzer')->end() diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 4a6c70a..f3474be 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -60,6 +60,10 @@ class MappingBuilder // 'search_analyzer' => $typeConfig->getSearchAnalyzer(), )); + if (isset($mapping['dynamic_templates']) and empty($mapping['dynamic_templates'])) { + unset($mapping['dynamic_templates']); + } + $this->fixProperties($mapping['properties']); if ($typeConfig->getModel()) { @@ -69,31 +73,6 @@ class MappingBuilder return $mapping; } - /** - * create type mapping object - * - * @param array $indexConfig - * @return Mapping - */ - protected function createMapping($indexConfig) - { - /*$mapping = $this->createMapping($indexConfig['config']['properties'][$typeName]);*/ - $mapping = Mapping::create($indexConfig['properties']); - - $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); - foreach ($mappingSpecialFields as $specialField) { - if (isset($indexConfig[$specialField])) { - $mapping->setParam($specialField, $indexConfig[$specialField]); - } - } - - if (isset($indexConfig['_parent'])) { - $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); - } - - return $mapping; - } - /** * Fixes any properties and applies basic defaults for any field that does not have * required options. diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 0f2403b..cdc6c53 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -29,11 +29,26 @@ fos_elastica: max_gram: 5 types: parent: + dynamic_templates: + dates: + match: "date_*" + mapping: + type: date mappings: field1: ~ field2: ~ type: search_analyzer: my_analyzer + dynamic_templates: + - dates: + match: "date_*" + mapping: + type: date + - strings: + match: "*" + mapping: + analyzer: english + type: string mappings: field1: ~ field2: From 7fcbb64a15fdf0a66a87acf9419456d7ce88ffd5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 1 Jul 2014 18:02:30 +1000 Subject: [PATCH 117/154] Fix edge case for dynamic templates --- DependencyInjection/Configuration.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index ae27860..a63cfe6 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -210,8 +210,8 @@ class Configuration implements ConfigurationInterface ->then(function ($v) { $dt = array(); foreach ($v['dynamic_templates'] as $key => $type) { - if (is_numeric($key)) { - $dt[$key] = $type; + if (is_int($key)) { + $dt[] = $type; } else { $dt[][$key] = $type; } From 965b319d82f36469ee94163b28bcfe2591fe6196 Mon Sep 17 00:00:00 2001 From: Maksim Kotlyar Date: Tue, 1 Jul 2014 14:40:03 +0300 Subject: [PATCH 118/154] [transformers] tell container that first argument is collection. --- Resources/config/transformer.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resources/config/transformer.xml b/Resources/config/transformer.xml index e3abbbc..4ce5062 100644 --- a/Resources/config/transformer.xml +++ b/Resources/config/transformer.xml @@ -12,14 +12,14 @@ - + - + From c200e8fdfd9a8878525a06345f00d46de9d53598 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 3 Jul 2014 19:59:33 +1000 Subject: [PATCH 119/154] Add tests for attachment type --- .travis.yml | 2 ++ Tests/Functional/app/Basic/config.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d6ba18a..a884d86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ php: - 5.5 before_script: + - sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/2.0.0 + - sudo service elasticsearch restart - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - composer install --dev --prefer-source diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index cdc6c53..beb0ded 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -49,7 +49,7 @@ fos_elastica: mapping: analyzer: english type: string - mappings: + properties: field1: ~ field2: type: integer @@ -68,6 +68,8 @@ fos_elastica: type: "object" properties: date: { boost: 5 } + agreement: + type: "attachment" lastlogin: { type: date, format: basic_date_time } birthday: { type: date, format: "yyyy-MM-dd" } _parent: From 55dcf6859ab9616028d2da2355b581737efb3802 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 3 Jul 2014 21:58:54 +1000 Subject: [PATCH 120/154] Rename servers to connections --- DependencyInjection/Configuration.php | 37 ++++++++++--------- DependencyInjection/FOSElasticaExtension.php | 2 +- .../doc/cookbook/multiple-connections.md | 16 ++++++++ Resources/doc/index.md | 2 + .../DependencyInjection/ConfigurationTest.php | 26 ++++++------- Tests/Functional/app/Basic/config.yml | 5 +++ 6 files changed, 56 insertions(+), 32 deletions(-) create mode 100644 Resources/doc/cookbook/multiple-connections.md diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index a63cfe6..874f51e 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -73,27 +73,28 @@ class Configuration implements ConfigurationInterface ->arrayNode('clients') ->useAttributeAsKey('id') ->prototype('array') - ->performNoDeepMerging() + // BC - Renaming 'servers' node to 'connections' ->beforeNormalization() - ->ifTrue(function($v) { return (isset($v['host']) && isset($v['port'])) || isset($v['url']); }) - ->then(function($v) { - return array( - 'servers' => array( - array( - 'host' => isset($v['host']) ? $v['host'] : null, - 'port' => isset($v['port']) ? $v['port'] : null, - 'url' => isset($v['url']) ? $v['url'] : null, - 'logger' => isset($v['logger']) ? $v['logger'] : null, - 'headers' => isset($v['headers']) ? $v['headers'] : null, - 'timeout' => isset($v['timeout']) ? $v['timeout'] : null, - 'transport' => isset($v['transport']) ? $v['transport'] : null, - ) - ) - ); - }) + ->ifTrue(function($v) { return isset($v['servers']); }) + ->then(function($v) { + $v['connections'] = $v['servers']; + unset($v['servers']); + + return $v; + }) + ->end() + // If there is no connections array key defined, assume a single connection. + ->beforeNormalization() + ->ifTrue(function ($v) { return is_array($v) && !array_key_exists('connections', $v); }) + ->then(function ($v) { + return array( + 'connections' => array($v) + ); + }) ->end() ->children() - ->arrayNode('servers') + ->arrayNode('connections') + ->requiresAtLeastOneElement() ->prototype('array') ->fixXmlConfig('header') ->children() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 9862076..292a9a5 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -104,7 +104,7 @@ class FOSElasticaExtension extends Extension $clientDef = new DefinitionDecorator('fos_elastica.client_prototype'); $clientDef->replaceArgument(0, $clientConfig); - $logger = $clientConfig['servers'][0]['logger']; + $logger = $clientConfig['connections'][0]['logger']; if (false !== $logger) { $clientDef->addMethodCall('setLogger', array(new Reference($logger))); } diff --git a/Resources/doc/cookbook/multiple-connections.md b/Resources/doc/cookbook/multiple-connections.md new file mode 100644 index 0000000..7b5226c --- /dev/null +++ b/Resources/doc/cookbook/multiple-connections.md @@ -0,0 +1,16 @@ +Multiple Connections +==================== + +You can define multiple endpoints for an Elastica client by specifying them as +multiple connections in the client configuration: + +```yaml +fos_elastica: + clients: + default: + connections: + - url: http://es1.example.net:9200 + - url: http://es2.example.net:9200 +``` + +For more information on Elastica clustering see http://elastica.io/getting-started/installation.html#section-connect-cluster diff --git a/Resources/doc/index.md b/Resources/doc/index.md index 1bc093e..80a500c 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -15,4 +15,6 @@ Cookbook Entries * [Custom Repositories](cookbook/custom-repositories.md) * [HTTP Headers for Elastica](cookbook/elastica-client-http-headers.md) * Performance - [Logging](cookbook/logging.md) +* [Manual Providers](cookbook/manual-provider.md) +* [Clustering - Multiple Connections](cookbook/multiple-connections.md) * [Suppressing server errors](cookbook/suppress-server-errors.md) diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index b474117..7165052 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -46,7 +46,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase 'url' => 'http://localhost:9200', ), 'clustered' => array( - 'servers' => array( + 'connections' => array( array( 'url' => 'http://es1:9200', 'headers' => array( @@ -65,13 +65,13 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase )); $this->assertCount(2, $configuration['clients']); - $this->assertCount(1, $configuration['clients']['default']['servers']); - $this->assertCount(0, $configuration['clients']['default']['servers'][0]['headers']); + $this->assertCount(1, $configuration['clients']['default']['connections']); + $this->assertCount(0, $configuration['clients']['default']['connections'][0]['headers']); - $this->assertCount(2, $configuration['clients']['clustered']['servers']); - $this->assertEquals('http://es2:9200/', $configuration['clients']['clustered']['servers'][1]['url']); - $this->assertCount(1, $configuration['clients']['clustered']['servers'][1]['headers']); - $this->assertEquals('Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==', $configuration['clients']['clustered']['servers'][0]['headers'][0]); + $this->assertCount(2, $configuration['clients']['clustered']['connections']); + $this->assertEquals('http://es2:9200/', $configuration['clients']['clustered']['connections'][1]['url']); + $this->assertCount(1, $configuration['clients']['clustered']['connections'][1]['headers']); + $this->assertEquals('Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==', $configuration['clients']['clustered']['connections'][0]['headers'][0]); } public function testLogging() @@ -98,10 +98,10 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertCount(4, $configuration['clients']); - $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_enabled']['servers'][0]['logger']); - $this->assertFalse($configuration['clients']['logging_disabled']['servers'][0]['logger']); - $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_not_mentioned']['servers'][0]['logger']); - $this->assertEquals('custom.service', $configuration['clients']['logging_custom']['servers'][0]['logger']); + $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_enabled']['connections'][0]['logger']); + $this->assertFalse($configuration['clients']['logging_disabled']['connections'][0]['logger']); + $this->assertEquals('fos_elastica.logger', $configuration['clients']['logging_not_mentioned']['connections'][0]['logger']); + $this->assertEquals('custom.service', $configuration['clients']['logging_custom']['connections'][0]['logger']); } public function testSlashIsAddedAtTheEndOfServerUrl() @@ -113,7 +113,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase ); $configuration = $this->getConfigs($config); - $this->assertEquals('http://www.github.com/', $configuration['clients']['default']['servers'][0]['url']); + $this->assertEquals('http://www.github.com/', $configuration['clients']['default']['connections'][0]['url']); } public function testTypeConfig() @@ -172,7 +172,7 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase ) )); - $this->assertTrue(empty($configuration['clients']['default']['servers'][0]['url'])); + $this->assertTrue(empty($configuration['clients']['default']['connections'][0]['url'])); } public function testMappingsRenamedToProperties() diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index beb0ded..e880fb1 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -11,6 +11,11 @@ twig: fos_elastica: clients: default: + connections: + - url: http://localhost:9200 + - host: localhost + port: 9200 + second_server: url: http://localhost:9200 indexes: index: From 815b836cdc067fffe32f74b863c7a7c433ecb4ff Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 3 Jul 2014 23:20:18 +1000 Subject: [PATCH 121/154] Add stopwatch support for Elastica\Client --- Elastica/Client.php | 26 ++++++++++++++++++++++++++ Resources/config/config.xml | 4 ++++ 2 files changed, 30 insertions(+) diff --git a/Elastica/Client.php b/Elastica/Client.php index e921965..1131ba5 100644 --- a/Elastica/Client.php +++ b/Elastica/Client.php @@ -5,6 +5,7 @@ namespace FOS\ElasticaBundle\Elastica; use Elastica\Client as BaseClient; use Elastica\Request; use FOS\ElasticaBundle\Logger\ElasticaLogger; +use Symfony\Component\Stopwatch\Stopwatch; /** * Extends the default Elastica client to provide logging for errors that occur @@ -21,11 +22,22 @@ class Client extends BaseClient */ private $indexCache = array(); + /** + * Symfony's debugging Stopwatch + * + * @var Stopwatch|null + */ + private $stopwatch; + /** * {@inheritdoc} */ public function request($path, $method = Request::GET, $data = array(), array $query = array()) { + if ($this->stopwatch) { + $this->stopwatch->start('es_request', 'fos_elastica'); + } + $start = microtime(true); $response = parent::request($path, $method, $data, $query); @@ -44,6 +56,10 @@ class Client extends BaseClient $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); } + if ($this->stopwatch) { + $this->stopwatch->stop('es_request'); + } + return $response; } @@ -55,4 +71,14 @@ class Client extends BaseClient return $this->indexCache[$name] = new Index($this, $name); } + + /** + * Sets a stopwatch instance for debugging purposes. + * + * @param Stopwatch $stopwatch + */ + public function setStopwatch(Stopwatch $stopwatch = null) + { + $this->stopwatch = $stopwatch; + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 023688f..dcec2e8 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -16,6 +16,10 @@ + + + + From 21e5d906a7ec8fd696dbacd03eb176d3d053314c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 3 Jul 2014 23:37:44 +1000 Subject: [PATCH 122/154] Simplify *One methods in the persister --- Persister/ObjectPersister.php | 18 ++++-------------- Tests/Persister/ObjectPersisterTest.php | 9 +++------ .../ObjectSerializerPersisterTest.php | 9 +++------ 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 54d5fd1..9604f7e 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -71,8 +71,7 @@ class ObjectPersister implements ObjectPersisterInterface */ public function insertOne($object) { - $document = $this->transformToElasticaDocument($object); - $this->type->addDocument($document); + $this->insertMany(array($object)); } /** @@ -83,11 +82,7 @@ class ObjectPersister implements ObjectPersisterInterface **/ public function replaceOne($object) { - $document = $this->transformToElasticaDocument($object); - try { - $this->type->deleteById($document->getId()); - } catch (NotFoundException $e) {} - $this->type->addDocument($document); + $this->replaceMany(array($object)); } /** @@ -98,10 +93,7 @@ class ObjectPersister implements ObjectPersisterInterface **/ public function deleteOne($object) { - $document = $this->transformToElasticaDocument($object); - try { - $this->type->deleteById($document->getId()); - } catch (NotFoundException $e) {} + $this->deleteMany(array($object)); } /** @@ -113,9 +105,7 @@ class ObjectPersister implements ObjectPersisterInterface **/ public function deleteById($id) { - try { - $this->type->deleteById($id); - } catch (NotFoundException $e) {} + $this->deleteManyByIdentifiers(array($id)); } /** diff --git a/Tests/Persister/ObjectPersisterTest.php b/Tests/Persister/ObjectPersisterTest.php index 497c286..77a8809 100644 --- a/Tests/Persister/ObjectPersisterTest.php +++ b/Tests/Persister/ObjectPersisterTest.php @@ -47,10 +47,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); $typeMock->expects($this->once()) - ->method('deleteById') - ->with($this->equalTo(123)); - $typeMock->expects($this->once()) - ->method('addDocument'); + ->method('updateDocuments'); $fields = array('name' => array()); @@ -91,7 +88,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase $typeMock->expects($this->never()) ->method('deleteById'); $typeMock->expects($this->once()) - ->method('addDocument'); + ->method('addDocuments'); $fields = array('name' => array()); @@ -130,7 +127,7 @@ class ObjectPersisterTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); $typeMock->expects($this->once()) - ->method('deleteById'); + ->method('deleteDocuments'); $typeMock->expects($this->never()) ->method('addDocument'); diff --git a/Tests/Persister/ObjectSerializerPersisterTest.php b/Tests/Persister/ObjectSerializerPersisterTest.php index aae3a64..fe15c0c 100644 --- a/Tests/Persister/ObjectSerializerPersisterTest.php +++ b/Tests/Persister/ObjectSerializerPersisterTest.php @@ -42,10 +42,7 @@ class ObjectSerializerPersisterTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); $typeMock->expects($this->once()) - ->method('deleteById') - ->with($this->equalTo(123)); - $typeMock->expects($this->once()) - ->method('addDocument'); + ->method('updateDocuments'); $serializerMock = $this->getMockBuilder('FOS\ElasticaBundle\Serializer\Callback')->getMock(); $serializerMock->expects($this->once())->method('serialize'); @@ -65,7 +62,7 @@ class ObjectSerializerPersisterTest extends \PHPUnit_Framework_TestCase $typeMock->expects($this->never()) ->method('deleteById'); $typeMock->expects($this->once()) - ->method('addDocument'); + ->method('addDocuments'); $serializerMock = $this->getMockBuilder('FOS\ElasticaBundle\Serializer\Callback')->getMock(); $serializerMock->expects($this->once())->method('serialize'); @@ -83,7 +80,7 @@ class ObjectSerializerPersisterTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); $typeMock->expects($this->once()) - ->method('deleteById'); + ->method('deleteDocuments'); $typeMock->expects($this->never()) ->method('addDocument'); From d57d430ab3e29a7502ac31500ed65f6c962a4674 Mon Sep 17 00:00:00 2001 From: tamirvs Date: Wed, 18 Jun 2014 17:02:50 +0000 Subject: [PATCH 123/154] Ignore iterator keys when converting to array --- Tests/Functional/MappingToElasticaTest.php | 18 +++++++ Tests/Functional/SerializerTest.php | 48 +++++++++++++++++++ Tests/Functional/TypeObj.php | 8 ++++ Tests/Functional/app/Basic/config.yml | 2 +- Tests/Functional/app/ORM/config.yml | 9 ++-- Tests/Functional/app/Serializer/TypeObj.yml | 8 ++++ Tests/Functional/app/Serializer/bundles.php | 13 +++++ Tests/Functional/app/Serializer/config.yml | 42 ++++++++++++++++ .../ModelToElasticaAutoTransformer.php | 2 +- 9 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 Tests/Functional/SerializerTest.php create mode 100644 Tests/Functional/app/Serializer/TypeObj.yml create mode 100644 Tests/Functional/app/Serializer/bundles.php create mode 100644 Tests/Functional/app/Serializer/config.yml diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index f038212..2474a1c 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -72,6 +72,22 @@ class MappingToElasticaTest extends WebTestCase $this->assertNotEmpty($mapping, 'Mapping was populated'); } + public function testMappingIteratorToArrayField() + { + $client = $this->createClient(array('test_case' => 'ORM')); + $persister = $client->getContainer()->get('fos_elastica.object_persister.index.type'); + + $object = new TypeObj(); + $object->id = 1; + $object->coll = new \ArrayIterator(array('foo', 'bar')); + $persister->insertOne($object); + + $object->coll = new \ArrayIterator(array('foo', 'bar', 'bazz')); + $object->coll->offsetUnset(1); + + $persister->replaceOne($object); + } + /** * @param Client $client * @return \FOS\ElasticaBundle\Resetter $resetter @@ -95,6 +111,7 @@ class MappingToElasticaTest extends WebTestCase parent::setUp(); $this->deleteTmpDir('Basic'); + $this->deleteTmpDir('ORM'); } protected function tearDown() @@ -102,5 +119,6 @@ class MappingToElasticaTest extends WebTestCase parent::tearDown(); $this->deleteTmpDir('Basic'); + $this->deleteTmpDir('ORM'); } } diff --git a/Tests/Functional/SerializerTest.php b/Tests/Functional/SerializerTest.php new file mode 100644 index 0000000..3a3b8cb --- /dev/null +++ b/Tests/Functional/SerializerTest.php @@ -0,0 +1,48 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +/** + * @group functional + */ +class SerializerTest extends WebTestCase +{ + public function testMappingIteratorToArrayField() + { + $client = $this->createClient(array('test_case' => 'Serializer')); + $persister = $client->getContainer()->get('fos_elastica.object_persister.index.type'); + + $object = new TypeObj(); + $object->id = 1; + $object->coll = new \ArrayIterator(array('foo', 'bar')); + $persister->insertOne($object); + + $object->coll = new \ArrayIterator(array('foo', 'bar', 'bazz')); + $object->coll->offsetUnset(1); + + $persister->replaceOne($object); + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('Serializer'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('Serializer'); + } +} diff --git a/Tests/Functional/TypeObj.php b/Tests/Functional/TypeObj.php index c264e7b..46e5968 100644 --- a/Tests/Functional/TypeObj.php +++ b/Tests/Functional/TypeObj.php @@ -13,6 +13,9 @@ namespace FOS\ElasticaBundle\Tests\Functional; class TypeObj { + public $coll; + public $field1; + public function isIndexable() { return true; @@ -22,4 +25,9 @@ class TypeObj { return false; } + + public function getSerializableColl() + { + return iterator_to_array($this->coll, false); + } } diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index cdc6c53..a7550b9 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -14,7 +14,7 @@ fos_elastica: url: http://localhost:9200 indexes: index: - index_name: foselastica_test_%kernel.environment% + index_name: foselastica_basic_test_%kernel.environment% settings: analysis: analyzer: diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 9ba6830..3dc0e63 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -17,27 +17,24 @@ fos_elastica: clients: default: url: http://localhost:9200 - serializer: ~ indexes: - fos_elastica_test: + fos_elastica_orm_test: types: type: properties: field1: ~ index: - index_name: foselastica_test_%kernel.environment% + index_name: foselastica_orm_test_%kernel.environment% types: type: properties: field1: ~ + coll: ~ persistence: driver: orm model: FOS\ElasticaBundle\Tests\Functional\TypeObj listener: is_indexable_callback: 'object.isIndexable() && !object.isntIndexable()' - serializer: - groups: ['search'] - version: 1.1 type2: properties: field1: ~ diff --git a/Tests/Functional/app/Serializer/TypeObj.yml b/Tests/Functional/app/Serializer/TypeObj.yml new file mode 100644 index 0000000..67d6335 --- /dev/null +++ b/Tests/Functional/app/Serializer/TypeObj.yml @@ -0,0 +1,8 @@ +FOS\ElasticaBundle\Tests\Functional\TypeObj: + properties: + field1: + type: string + virtualProperties: + getSerializableColl: + serializedName: coll + type: array diff --git a/Tests/Functional/app/Serializer/bundles.php b/Tests/Functional/app/Serializer/bundles.php new file mode 100644 index 0000000..25db3fe --- /dev/null +++ b/Tests/Functional/app/Serializer/bundles.php @@ -0,0 +1,13 @@ + Date: Fri, 4 Jul 2014 13:06:57 +1000 Subject: [PATCH 124/154] Documentation on aliased repopulation --- Resources/doc/cookbook/aliased-indexes.md | 45 +++++++++++++++++++++++ Resources/doc/index.md | 1 + 2 files changed, 46 insertions(+) create mode 100644 Resources/doc/cookbook/aliased-indexes.md diff --git a/Resources/doc/cookbook/aliased-indexes.md b/Resources/doc/cookbook/aliased-indexes.md new file mode 100644 index 0000000..b9049c5 --- /dev/null +++ b/Resources/doc/cookbook/aliased-indexes.md @@ -0,0 +1,45 @@ +Aliased Indexes +=============== + +You can set up FOSElasticaBundle to use aliases for indexes which allows you to run an +index population without resetting the index currently being used by the application. + +> *Note*: When you're using an alias, resetting an individual type will still cause a +> reset for that type. + +To configure FOSElasticaBundle to use aliases for an index, set the use_alias option to +true. + +```yaml +fos_elastica: + indexes: + website: + use_alias: true +``` + +The process for setting up aliases on an existing application is slightly more complicated +because the bundle is not able to set an alias as the same name as an index. You have some +options on how to handle this: + +1) Delete the index from Elasticsearch. This option will make searching unavailable in your + application until a population has completed itself, and an alias is created. + +2) Change the index_name parameter for your index to something new, and manually alias the + current index to the new index_name, which will then be replaced when you run a repopulate. + +```yaml +fos_elastica: + indexes: + website: + use_alias: true + index_name: website_prod +``` + +```bash +$ curl -XPOST 'http://localhost:9200/_aliases' -d ' +{ + "actions" : [ + { "add" : { "index" : "website", "alias" : "website_prod" } } + ] +}' +``` diff --git a/Resources/doc/index.md b/Resources/doc/index.md index 80a500c..349723b 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -12,6 +12,7 @@ Available documentation for FOSElasticaBundle Cookbook Entries ---------------- +* [Aliased Indexes](cookbook/aliased-indexes.md) * [Custom Repositories](cookbook/custom-repositories.md) * [HTTP Headers for Elastica](cookbook/elastica-client-http-headers.md) * Performance - [Logging](cookbook/logging.md) From d88d96bf55a1ba6b603d429d9bf6a217d945b1c5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 4 Jul 2014 22:10:24 +1000 Subject: [PATCH 125/154] Fix invalid service reference in production --- Resources/config/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index dcec2e8..06f0cda 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -18,7 +18,7 @@ - + From cdaf7105e03d6437e534f4d9a402f1674bab7cfc Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sat, 5 Jul 2014 15:14:29 +1000 Subject: [PATCH 126/154] Bump dev version to 3.1 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 372320f..a1734dd 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ }, "extra": { "branch-alias": { - "dev-master": "3.0.x-dev" + "dev-master": "3.1.x-dev" } } } From d797af5b80ae98936c78eea4a6db7a9d80b2eaab Mon Sep 17 00:00:00 2001 From: Floran Brutel Date: Mon, 7 Jul 2014 18:35:23 +0200 Subject: [PATCH 127/154] Fix "Faceted Searching" doc --- Resources/doc/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index 55d90ab..37514b3 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -45,7 +45,7 @@ $companies = $finder->findPaginated($query); $companies->setMaxPerPage($params['limit']); $companies->setCurrentPage($params['page']); -$facets = $companies->getAdapter()->getFacets()); +$facets = $companies->getAdapter()->getFacets(); ``` Searching the entire index From fad481d8222be4e6b49f3bcdaf120be76958d008 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 23 Jul 2014 20:00:14 +1000 Subject: [PATCH 128/154] Fix completion type --- Index/MappingBuilder.php | 16 +++++++++++++--- Tests/Functional/app/Basic/config.yml | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index f3474be..fc67420 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -16,6 +16,13 @@ use FOS\ElasticaBundle\Configuration\TypeConfig; class MappingBuilder { + /** + * Skip adding default information to certain fields. + * + * @var array + */ + private $skipTypes = array('completion'); + /** * Builds mappings for an entire index. * @@ -85,12 +92,15 @@ class MappingBuilder if (!isset($property['type'])) { $property['type'] = 'string'; } - if (!isset($property['store'])) { - $property['store'] = true; - } if (isset($property['properties'])) { $this->fixProperties($property['properties']); } + if (in_array($property['type'], $this->skipTypes)) { + continue; + } + if (!isset($property['store'])) { + $property['store'] = true; + } } } } diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 3552b3d..607e3cc 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -60,6 +60,8 @@ fos_elastica: type: integer store: false date: { boost: 5 } + completion: + type: completion title: { boost: 8, analyzer: my_analyzer } content: ~ comments: From e5410a5b650cddd3e1b9f128c7701496b3bda5b9 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 23 Jul 2014 21:38:46 +1000 Subject: [PATCH 129/154] Fix indexable callbacks being overwritten by another index closes #663 --- DependencyInjection/FOSElasticaExtension.php | 15 ++++++++------- Tests/Functional/app/ORM/config.yml | 12 ++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 292a9a5..e523bf4 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -128,6 +128,8 @@ class FOSElasticaExtension extends Extension */ private function loadIndexes(array $indexes, ContainerBuilder $container) { + $indexableCallbacks = array(); + foreach ($indexes as $name => $index) { $indexId = sprintf('fos_elastica.index.%s', $name); $indexName = isset($index['index_name']) ? $index['index_name']: $name; @@ -159,8 +161,11 @@ class FOSElasticaExtension extends Extension $this->loadIndexFinder($container, $name, $reference); } - $this->loadTypes((array) $index['types'], $container, $this->indexConfigs[$name]); + $this->loadTypes((array) $index['types'], $container, $this->indexConfigs[$name], $indexableCallbacks); } + + $indexable = $container->getDefinition('fos_elastica.indexable'); + $indexable->replaceArgument(0, $indexableCallbacks); } /** @@ -194,11 +199,10 @@ class FOSElasticaExtension extends Extension * @param array $types * @param ContainerBuilder $container * @param array $indexConfig + * @param array $indexableCallbacks */ - private function loadTypes(array $types, ContainerBuilder $container, array $indexConfig) + private function loadTypes(array $types, ContainerBuilder $container, array $indexConfig, array &$indexableCallbacks) { - $indexableCallbacks = array(); - foreach ($types as $name => $type) { $indexName = $indexConfig['name']; @@ -269,9 +273,6 @@ class FOSElasticaExtension extends Extension $container->setDefinition($typeSerializerId, $typeSerializerDef); } } - - $indexable = $container->getDefinition('fos_elastica.indexable'); - $indexable->replaceArgument(0, $indexableCallbacks); } /** diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 3dc0e63..02d7a92 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -63,3 +63,15 @@ fos_elastica: provider: ~ listener: is_indexable_callback: [ 'FOS\ElasticaBundle\Tests\Functional\app\ORM\IndexableService', 'isntIndexable' ] + second_index: + index_name: foselastica_orm_test_second_%kernel.environment% + types: + type: + properties: + field1: ~ + coll: ~ + persistence: + driver: orm + model: FOS\ElasticaBundle\Tests\Functional\TypeObj + listener: + is_indexable_callback: 'object.isIndexable() && !object.isntIndexable()' From 11ee25cfea23a1d2743bc875eb1ecacdaf3f5d8c Mon Sep 17 00:00:00 2001 From: Tobias Nyholm Date: Wed, 9 Jul 2014 12:29:01 +0200 Subject: [PATCH 130/154] Added PHP 5.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a884d86..034cd84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ php: - 5.3 - 5.4 - 5.5 + - 5.6 before_script: - sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/2.0.0 From 9befa90f4192701779181862fee5f7965af2b95d Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 25 Jul 2014 17:59:24 +0200 Subject: [PATCH 131/154] Return repository in Manager Class MongoDB --- Doctrine/MongoDB/ElasticaToModelTransformer.php | 1 + 1 file changed, 1 insertion(+) diff --git a/Doctrine/MongoDB/ElasticaToModelTransformer.php b/Doctrine/MongoDB/ElasticaToModelTransformer.php index 855a093..cea737f 100644 --- a/Doctrine/MongoDB/ElasticaToModelTransformer.php +++ b/Doctrine/MongoDB/ElasticaToModelTransformer.php @@ -22,6 +22,7 @@ class ElasticaToModelTransformer extends AbstractElasticaToModelTransformer { return $this->registry ->getManagerForClass($this->objectClass) + ->getRepository($this->objectClass) ->{$this->options['query_builder_method']}($this->objectClass) ->field($this->options['identifier'])->in($identifierValues) ->hydrate($hydrate) From 9a5b80e723f306e89f8d658d900b21276e010edf Mon Sep 17 00:00:00 2001 From: Lukasz Cybula Date: Thu, 31 Jul 2014 14:06:34 +0200 Subject: [PATCH 132/154] Ignore missing Doctrine results during hybridTransform() --- .../AbstractElasticaToModelTransformer.php | 10 ++- ...AbstractElasticaToModelTransformerTest.php | 66 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 Tests/Doctrine/AbstractElasticaToModelTransformerTest.php diff --git a/Doctrine/AbstractElasticaToModelTransformer.php b/Doctrine/AbstractElasticaToModelTransformer.php index 147067d..96f73bd 100755 --- a/Doctrine/AbstractElasticaToModelTransformer.php +++ b/Doctrine/AbstractElasticaToModelTransformer.php @@ -120,11 +120,17 @@ abstract class AbstractElasticaToModelTransformer implements ElasticaToModelTran public function hybridTransform(array $elasticaObjects) { + $indexedElasticaResults = array(); + foreach ($elasticaObjects as $elasticaObject) { + $indexedElasticaResults[$elasticaObject->getId()] = $elasticaObject; + } + $objects = $this->transform($elasticaObjects); $result = array(); - for ($i = 0; $i < count($elasticaObjects); $i++) { - $result[] = new HybridResult($elasticaObjects[$i], $objects[$i]); + foreach ($objects as $object) { + $id = $this->propertyAccessor->getValue($object, $this->options['identifier']); + $result[] = new HybridResult($indexedElasticaResults[$id], $object); } return $result; diff --git a/Tests/Doctrine/AbstractElasticaToModelTransformerTest.php b/Tests/Doctrine/AbstractElasticaToModelTransformerTest.php new file mode 100644 index 0000000..325171b --- /dev/null +++ b/Tests/Doctrine/AbstractElasticaToModelTransformerTest.php @@ -0,0 +1,66 @@ +getMock( + 'FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer', + array('findByIdentifiers'), + array($this->registry, $this->objectClass, array('ignore_missing' => true)) + ); + + $transformer->setPropertyAccessor(PropertyAccess::createPropertyAccessor()); + + $firstOrmResult = new \stdClass(); + $firstOrmResult->id = 1; + $secondOrmResult = new \stdClass(); + $secondOrmResult->id = 3; + $transformer->expects($this->once()) + ->method('findByIdentifiers') + ->with(array(1, 2, 3)) + ->willReturn(array($firstOrmResult, $secondOrmResult)); + + $firstElasticaResult = new Result(array('_id' => 1)); + $secondElasticaResult = new Result(array('_id' => 2)); + $thirdElasticaResult = new Result(array('_id' => 3)); + + $hybridResults = $transformer->hybridTransform(array($firstElasticaResult, $secondElasticaResult, $thirdElasticaResult)); + + $this->assertCount(2, $hybridResults); + $this->assertEquals($firstOrmResult, $hybridResults[0]->getTransformed()); + $this->assertEquals($firstElasticaResult, $hybridResults[0]->getResult()); + $this->assertEquals($secondOrmResult, $hybridResults[1]->getTransformed()); + $this->assertEquals($thirdElasticaResult, $hybridResults[1]->getResult()); + } + + protected function setUp() + { + if (!interface_exists('Doctrine\Common\Persistence\ManagerRegistry')) { + $this->markTestSkipped('Doctrine Common is not present'); + } + + $this->registry = $this->getMockBuilder('Doctrine\Common\Persistence\ManagerRegistry') + ->disableOriginalConstructor() + ->getMock(); + } +} From 9296534d306f6535e52e0d377fe1d5904f23a083 Mon Sep 17 00:00:00 2001 From: Luis Cordova Date: Wed, 6 Aug 2014 17:25:05 -0500 Subject: [PATCH 133/154] Update setup.md --- Resources/doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 485f290..912c5d2 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -7,7 +7,7 @@ A) Install FOSElasticaBundle FOSElasticaBundle is installed using [Composer](https://getcomposer.org). ```bash -$ php composer.phar require friendsofsymfony/elastica-bundle "3.0.*@alpha" +$ php composer.phar require friendsofsymfony/elastica-bundle "~3.0.2" ``` ### Elasticsearch From c44f676224dbf532e14aecb05d381e3c26b9fee8 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 7 Aug 2014 09:25:09 +1000 Subject: [PATCH 134/154] Test mappings key being null still causes appropriate configuration changes --- DependencyInjection/Configuration.php | 4 ++-- Tests/Functional/ConfigurationManagerTest.php | 2 +- Tests/Functional/app/Basic/config.yml | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 874f51e..dd41b09 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -182,7 +182,7 @@ class Configuration implements ConfigurationInterface ->treatNullLike(array()) // BC - Renaming 'mappings' node to 'properties' ->beforeNormalization() - ->ifTrue(function($v) { return isset($v['mappings']); }) + ->ifTrue(function($v) { return array_key_exists('mappings', $v); }) ->then(function($v) { $v['properties'] = $v['mappings']; unset($v['mappings']); @@ -213,7 +213,7 @@ class Configuration implements ConfigurationInterface foreach ($v['dynamic_templates'] as $key => $type) { if (is_int($key)) { $dt[] = $type; - } else { + } else { $dt[][$key] = $type; } } diff --git a/Tests/Functional/ConfigurationManagerTest.php b/Tests/Functional/ConfigurationManagerTest.php index 6fdc1d7..7ef02c5 100644 --- a/Tests/Functional/ConfigurationManagerTest.php +++ b/Tests/Functional/ConfigurationManagerTest.php @@ -26,7 +26,7 @@ class ConfigurationManagerTest extends WebTestCase $index = $manager->getIndexConfiguration('index'); $this->assertEquals('index', $index->getName()); - $this->assertCount(2, $index->getTypes()); + $this->assertGreaterThanOrEqual(2, count($index->getTypes())); $this->assertInstanceOf('FOS\\ElasticaBundle\\Configuration\\TypeConfig', $index->getType('type')); $this->assertInstanceOf('FOS\\ElasticaBundle\\Configuration\\TypeConfig', $index->getType('parent')); } diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 607e3cc..9feed34 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -83,3 +83,5 @@ fos_elastica: type: "parent" property: "parent" identifier: "id" + null_mappings: + mappings: ~ From f9eb6577d1556a6c071186a3750a8c56f7fd6001 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 7 Aug 2014 09:25:49 +1000 Subject: [PATCH 135/154] Add tests to cover no mappings for serializer enabled type --- Tests/Functional/SerializerTest.php | 7 +++++++ Tests/Functional/app/Serializer/config.yml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/Tests/Functional/SerializerTest.php b/Tests/Functional/SerializerTest.php index 3a3b8cb..81fbc8f 100644 --- a/Tests/Functional/SerializerTest.php +++ b/Tests/Functional/SerializerTest.php @@ -32,6 +32,13 @@ class SerializerTest extends WebTestCase $persister->replaceOne($object); } + public function testUnmappedType() + { + $client = $this->createClient(array('test_case' => 'Serializer')); + $resetter = $client->getContainer()->get('fos_elastica.resetter'); + $resetter->resetIndex('index'); + } + protected function setUp() { parent::setUp(); diff --git a/Tests/Functional/app/Serializer/config.yml b/Tests/Functional/app/Serializer/config.yml index ccd18f4..9bea4ba 100644 --- a/Tests/Functional/app/Serializer/config.yml +++ b/Tests/Functional/app/Serializer/config.yml @@ -40,3 +40,11 @@ fos_elastica: serializer: groups: ['search', 'Default'] version: 1.1 + unmapped: + persistence: + driver: orm + model: FOS\ElasticaBundle\Tests\Functional\TypeObj + serializer: + groups: ['search', 'Default'] + version: 1.1 + From dafe8abe0e2794d7bf13ff05eb5ef5bc030ad241 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 7 Aug 2014 09:32:30 +1000 Subject: [PATCH 136/154] Output ES version --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 034cd84..76a2969 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ php: - 5.6 before_script: + - /usr/share/elasticsearch -v - sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/2.0.0 - sudo service elasticsearch restart - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini From f5987a48b95d4dbd1e481878f0749cf3491bfa90 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Thu, 7 Aug 2014 16:33:14 -0500 Subject: [PATCH 137/154] BC Break: Restored noDeepMerging to Configuration When performNoDeepMerging is not used, Symfony environment-specific server configurations no longer work. --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 874f51e..df223a7 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -73,6 +73,7 @@ class Configuration implements ConfigurationInterface ->arrayNode('clients') ->useAttributeAsKey('id') ->prototype('array') + ->performNoDeepMerging() // BC - Renaming 'servers' node to 'connections' ->beforeNormalization() ->ifTrue(function($v) { return isset($v['servers']); }) From 20033709cf8ce84b169235de4e20bd888a6f8206 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 7 Aug 2014 09:36:23 +1000 Subject: [PATCH 138/154] Fix empty mappings for old ES versions --- .travis.yml | 2 +- Configuration/TypeConfig.php | 26 ++++++++++++++++++ DependencyInjection/FOSElasticaExtension.php | 6 ++-- Index/MappingBuilder.php | 29 +++++++++++++++----- Tests/Functional/MappingToElasticaTest.php | 10 +++++-- Tests/Functional/app/Basic/config.yml | 2 ++ 6 files changed, 62 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 76a2969..8ccaa8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ php: - 5.6 before_script: - - /usr/share/elasticsearch -v + - /usr/share/elasticsearch/bin/elasticsearch -v - sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/2.0.0 - sudo service elasticsearch restart - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini diff --git a/Configuration/TypeConfig.php b/Configuration/TypeConfig.php index 5d3f084..3f1c939 100644 --- a/Configuration/TypeConfig.php +++ b/Configuration/TypeConfig.php @@ -35,6 +35,14 @@ class TypeConfig $this->name = $name; } + /** + * @return string|null + */ + public function getIndexAnalyzer() + { + return $this->getConfig('index_analyzer'); + } + /** * @return array */ @@ -43,6 +51,9 @@ class TypeConfig return $this->mapping; } + /** + * @return string|null + */ public function getModel() { return isset($this->config['persistence']['model']) ? @@ -57,4 +68,19 @@ class TypeConfig { return $this->name; } + + /** + * @return string|null + */ + public function getSearchAnalyzer() + { + return $this->getConfig('search_analyzer'); + } + + private function getConfig($key) + { + return isset($this->config[$key]) ? + $this->config[$key] : + null; + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index e523bf4..aefa7fe 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -220,9 +220,7 @@ class FOSElasticaExtension extends Extension foreach (array( 'dynamic_templates', - 'index_analyzer', 'properties', - 'search_analyzer', '_all', '_boost', '_id', @@ -239,7 +237,9 @@ class FOSElasticaExtension extends Extension foreach (array( 'persistence', - 'serializer' + 'serializer', + 'index_analyzer', + 'search_analyzer', ) as $field) { $typeConfig['config'][$field] = array_key_exists($field, $type) ? $type[$field] : diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index fc67420..21ae871 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -18,7 +18,7 @@ class MappingBuilder { /** * Skip adding default information to certain fields. - * + * * @var array */ private $skipTypes = array('completion'); @@ -36,10 +36,11 @@ class MappingBuilder $typeMappings[$typeConfig->getName()] = $this->buildTypeMapping($typeConfig); } - $mapping = array( - 'mappings' => $typeMappings, - // 'warmers' => $indexConfig->getWarmers(), - ); + $mapping = array(); + if ($typeMappings) { + $mapping['mappings'] = $typeMappings; + } + // 'warmers' => $indexConfig->getWarmers(), $settings = $indexConfig->getSettings(); if ($settings) { @@ -61,22 +62,36 @@ class MappingBuilder // 'date_detection' => true, // 'dynamic_date_formats' => array() // 'dynamic_templates' => $typeConfig->getDynamicTemplates(), - // 'index_analyzer' => $typeConfig->getIndexAnalyzer(), // 'numeric_detection' => false, // 'properties' => array(), - // 'search_analyzer' => $typeConfig->getSearchAnalyzer(), )); + if ($typeConfig->getIndexAnalyzer()) { + $mapping['index_analyzer'] = $typeConfig->getIndexAnalyzer(); + } + + if ($typeConfig->getSearchAnalyzer()) { + $mapping['search_analyzer'] = $typeConfig->getSearchAnalyzer(); + } + if (isset($mapping['dynamic_templates']) and empty($mapping['dynamic_templates'])) { unset($mapping['dynamic_templates']); } $this->fixProperties($mapping['properties']); + if (!$mapping['properties']) { + unset($mapping['properties']); + } if ($typeConfig->getModel()) { $mapping['_meta']['model'] = $typeConfig->getModel(); } + if (!$mapping) { + // Empty mapping, we want it encoded as a {} instead of a [] + $mapping = new \stdClass; + } + return $mapping; } diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index 2474a1c..f42df61 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -31,6 +31,12 @@ class MappingToElasticaTest extends WebTestCase $this->assertArrayHasKey('store', $mapping['type']['properties']['field1']); $this->assertTrue($mapping['type']['properties']['field1']['store']); $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); + + $parent = $this->getType($client, 'parent'); + $mapping = $parent->getMapping(); + + $this->assertEquals('my_analyzer', $mapping['parent']['index_analyzer']); + $this->assertEquals('whitespace', $mapping['parent']['search_analyzer']); } public function testResetType() @@ -101,9 +107,9 @@ class MappingToElasticaTest extends WebTestCase * @param Client $client * @return \Elastica\Type */ - private function getType(Client $client) + private function getType(Client $client, $type = 'type') { - return $client->getContainer()->get('fos_elastica.index.index.type'); + return $client->getContainer()->get('fos_elastica.index.index.' . $type); } protected function setUp() diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 9feed34..3c3d369 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -42,6 +42,8 @@ fos_elastica: mappings: field1: ~ field2: ~ + search_analyzer: whitespace + index_analyzer: my_analyzer type: search_analyzer: my_analyzer dynamic_templates: From 27385046cae6a560ca778b0f682fde7cb395494c Mon Sep 17 00:00:00 2001 From: Oleg Andreyev Date: Fri, 8 Aug 2014 09:29:30 +0300 Subject: [PATCH 139/154] changing AliasProcessor::setRootName, so to use "Y-m-d-His" instead of random string --- Index/AliasProcessor.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Index/AliasProcessor.php b/Index/AliasProcessor.php index 93877cd..6b7da75 100644 --- a/Index/AliasProcessor.php +++ b/Index/AliasProcessor.php @@ -26,7 +26,12 @@ class AliasProcessor */ public function setRootName(IndexConfig $indexConfig, Index $index) { - $index->overrideName(sprintf('%s_%s', $indexConfig->getElasticSearchName(), uniqid())); + $index->overrideName( + sprintf('%s_%s', + $indexConfig->getElasticSearchName(), + date('Y-m-d-His') + ) + ); } /** @@ -116,7 +121,9 @@ class AliasProcessor /** * Returns array of indexes which are mapped to given alias * + * @param Client $client * @param string $aliasName Alias name + * * @return array */ private function getAliasedIndexes(Client $client, $aliasName) From 33ee047f83175ada2ef471db8ce5f8e5ab4a6050 Mon Sep 17 00:00:00 2001 From: Luis Cordova Date: Sat, 9 Aug 2014 07:50:00 -0500 Subject: [PATCH 140/154] fix dependency on elastic extension i had a weird error in which just installing the bundle with "friendsofsymfony/elastica-bundle": "~3.0.2", game me a version back even before it had a composer! :blush: i checked and found out composer gets really confused or glitchy with packages not following semver. In any case 1.3.0 is out, and i remove the 4th digit which i think composer ignores totally or gets confused about. --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index a1734dd..2fd9d3c 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.10.0, <1.3-dev", + "ruflin/elastica": ">=0.90.10.0, <1.4-dev", "psr/log": "~1.0" }, "require-dev":{ From 22a2a223cc6a7b1915054ee9fedf2f46b1a801c1 Mon Sep 17 00:00:00 2001 From: Floran Brutel Date: Sun, 17 Aug 2014 12:16:15 +0200 Subject: [PATCH 141/154] Update version in setup.md Use "~3.0" instead of "~3.0.2" to get version 3.0.3 and future minor versions --- Resources/doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 912c5d2..6a1c2ae 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -7,7 +7,7 @@ A) Install FOSElasticaBundle FOSElasticaBundle is installed using [Composer](https://getcomposer.org). ```bash -$ php composer.phar require friendsofsymfony/elastica-bundle "~3.0.2" +$ php composer.phar require friendsofsymfony/elastica-bundle "~3.0" ``` ### Elasticsearch From 69c2214bc5c9b910fb0021bd69fda8e6ae9e3503 Mon Sep 17 00:00:00 2001 From: Floran Brutel Date: Mon, 18 Aug 2014 13:30:45 +0200 Subject: [PATCH 142/154] Use the new Search annotation Use "FOS\ElasticaBundle\Annotation\Search" instead of "FOS\ElasticaBundle\Configuration\Search" in the repository manager. Update the cookbook --- Manager/RepositoryManager.php | 2 +- Resources/doc/cookbook/custom-repositories.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Manager/RepositoryManager.php b/Manager/RepositoryManager.php index 3cf8e96..be07b42 100644 --- a/Manager/RepositoryManager.php +++ b/Manager/RepositoryManager.php @@ -59,7 +59,7 @@ class RepositoryManager implements RepositoryManagerInterface } $refClass = new \ReflectionClass($entityName); - $annotation = $this->reader->getClassAnnotation($refClass, 'FOS\\ElasticaBundle\\Configuration\\Search'); + $annotation = $this->reader->getClassAnnotation($refClass, 'FOS\\ElasticaBundle\\Annotation\\Search'); if ($annotation) { $this->entities[$entityName]['repositoryName'] = $annotation->repositoryClass; diff --git a/Resources/doc/cookbook/custom-repositories.md b/Resources/doc/cookbook/custom-repositories.md index 47dc3fe..9eff5f7 100644 --- a/Resources/doc/cookbook/custom-repositories.md +++ b/Resources/doc/cookbook/custom-repositories.md @@ -58,7 +58,7 @@ Alternatively you can specify the custom repository using an annotation in the e namespace Application\UserBundle\Entity; -use FOS\ElasticaBundle\Configuration\Search; +use FOS\ElasticaBundle\Annotation\Search; /** * @Search(repositoryClass="Acme\ElasticaBundle\SearchRepository\UserRepository") From 0425379420ff69e9633d8579dea7a2200e7e547b Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Mon, 11 Aug 2014 17:17:38 +0100 Subject: [PATCH 143/154] add back fos_elastica.client tag that was removed in e78950ddb7b3e6c3ae402a042276dc81c4ee0dac --- DependencyInjection/FOSElasticaExtension.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index aefa7fe..cdf109b 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -108,6 +108,7 @@ class FOSElasticaExtension extends Extension if (false !== $logger) { $clientDef->addMethodCall('setLogger', array(new Reference($logger))); } + $clientDef->addTag('fos_elastica.client'); $container->setDefinition($clientId, $clientDef); From 598a59927ef9ed318a4ab9cfbc9672dea4947894 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 21 Aug 2014 21:48:19 +1000 Subject: [PATCH 144/154] Update travis testing --- .scrutinizer.yml | 5 +++++ .travis.yml | 19 ++++++++++++++++--- composer.json | 1 - 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 .scrutinizer.yml diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100644 index 0000000..e2cb043 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,5 @@ +imports: + - php + +tools: + external_code_coverage: true diff --git a/.travis.yml b/.travis.yml index 8ccaa8d..02f3ab8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,31 @@ language: php php: - 5.3 - - 5.4 - 5.5 - 5.6 +matrix: + include: + - php: 5.5 + env: SYMFONY_VERSION='2.3.*' + - php: 5.5 + env: SYMFONY_VERSION='2.5.*' + - php: 5.5 + env: SYMFONY_VERSION='dev-master' + before_script: - /usr/share/elasticsearch/bin/elasticsearch -v - sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-mapper-attachments/2.0.0 - sudo service elasticsearch restart - - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + - sh -c 'if [ "$TRAVIS_PHP_VERSION" != "hhvm" ]; then echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini; fi;' + - sh -c 'if [ "$SYMFONY_VERSION" != "" ]; then composer require --dev --no-update symfony/symfony=$SYMFONY_VERSION; fi;' - composer install --dev --prefer-source -script: vendor/bin/phpunit +script: vendor/bin/phpunit --coverage-clover=coverage.clover services: - elasticsearch + +after_script: + - wget https://scrutinizer-ci.com/ocular.phar + - php ocular.phar code-coverage:upload --format=php-clover coverage.clover diff --git a/composer.json b/composer.json index 2fd9d3c..bb30928 100644 --- a/composer.json +++ b/composer.json @@ -23,7 +23,6 @@ "require-dev":{ "doctrine/orm": "~2.2", "doctrine/doctrine-bundle": "~1.2@beta", - "doctrine/mongodb-odm": "1.0.*@beta", "jms/serializer-bundle": "@stable", "phpunit/phpunit": "~4.1", "propel/propel1": "1.6.*", From 6bea3c2154176bc68fc80cd6a9d8b0b7e9272a67 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 21 Aug 2014 23:23:20 +1000 Subject: [PATCH 145/154] Add code quality badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 631951a..01afa6d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Symfony2. Features include: > **Note** Propel support is limited and contributions fixing issues are welcome! [![Build Status](https://secure.travis-ci.org/FriendsOfSymfony/FOSElasticaBundle.png?branch=master)](http://travis-ci.org/FriendsOfSymfony/FOSElasticaBundle) [![Total Downloads](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/downloads.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Stable Version](https://poser.pugx.org/FriendsOfSymfony/elastica-bundle/v/stable.png)](https://packagist.org/packages/FriendsOfSymfony/elastica-bundle) [![Latest Unstable Version](https://poser.pugx.org/friendsofsymfony/elastica-bundle/v/unstable.svg)](https://packagist.org/packages/friendsofsymfony/elastica-bundle) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/FriendsOfSymfony/FOSElasticaBundle/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/FriendsOfSymfony/FOSElasticaBundle/?branch=master) Documentation ------------- From 2958833012b977ab6f6a831e03bdc28c8c5d6279 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Thu, 28 Aug 2014 17:59:58 +0100 Subject: [PATCH 146/154] Ability to delete an index if expecting an alias --- Command/ResetCommand.php | 4 +++- Exception/AliasIsIndexException.php | 12 ++++++++++++ Index/AliasProcessor.php | 29 +++++++++++++++++++++++++++-- Index/Resetter.php | 9 +++++---- 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 Exception/AliasIsIndexException.php diff --git a/Command/ResetCommand.php b/Command/ResetCommand.php index 06cfe48..ed14e6c 100755 --- a/Command/ResetCommand.php +++ b/Command/ResetCommand.php @@ -33,6 +33,7 @@ class ResetCommand extends ContainerAwareCommand ->setName('fos:elastica:reset') ->addOption('index', null, InputOption::VALUE_OPTIONAL, 'The index to reset') ->addOption('type', null, InputOption::VALUE_OPTIONAL, 'The type to reset') + ->addOption('force', null, InputOption::VALUE_NONE, 'Force index deletion if same name as alias') ->setDescription('Reset search indexes') ; } @@ -53,6 +54,7 @@ class ResetCommand extends ContainerAwareCommand { $index = $input->getOption('index'); $type = $input->getOption('type'); + $force = (true == $input->getOption('force')); if (null === $index && null !== $type) { throw new \InvalidArgumentException('Cannot specify type option without an index.'); @@ -69,7 +71,7 @@ class ResetCommand extends ContainerAwareCommand foreach ($indexes as $index) { $output->writeln(sprintf('Resetting %s', $index)); - $this->resetter->resetIndex($index); + $this->resetter->resetIndex($index, false, $force); } } } diff --git a/Exception/AliasIsIndexException.php b/Exception/AliasIsIndexException.php new file mode 100644 index 0000000..87f546b --- /dev/null +++ b/Exception/AliasIsIndexException.php @@ -0,0 +1,12 @@ +getClient(); @@ -45,7 +47,16 @@ class AliasProcessor $oldIndexName = false; $newIndexName = $index->getName(); - $aliasedIndexes = $this->getAliasedIndexes($client, $aliasName); + $aliasedIndexes = array(); + try { + $aliasedIndexes = $this->getAliasedIndexes($client, $aliasName); + } catch(AliasIsIndexException $e) { + if ($force) { + $this->deleteIndex($client, $aliasName); + } else { + throw new \RuntimeException($e->getMessage()); + } + } if (count($aliasedIndexes) > 1) { throw new \RuntimeException( @@ -125,6 +136,9 @@ class AliasProcessor $aliasedIndexes = array(); foreach ($aliasesInfo as $indexName => $indexInfo) { + if ($indexName == $aliasName) { + throw new AliasIsIndexException($indexName); + } $aliases = array_keys($indexInfo['aliases']); if (in_array($aliasName, $aliases)) { $aliasedIndexes[] = $indexName; @@ -133,4 +147,15 @@ class AliasProcessor return $aliasedIndexes; } + + /** + * Delete an index + * + * @param string $indexName Index name to delete + */ + private function deleteIndex($client, $indexName) + { + $path = sprintf("%s", $indexName); + $client->request($path, \Elastica\Request::DELETE); + } } diff --git a/Index/Resetter.php b/Index/Resetter.php index 3f07fa1..c93ae2d 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -45,10 +45,10 @@ class Resetter /** * Deletes and recreates all indexes */ - public function resetAllIndexes($populating = false) + public function resetAllIndexes($populating = false, $force = false) { foreach ($this->configManager->getIndexNames() as $name) { - $this->resetIndex($name, $populating); + $this->resetIndex($name, $populating, $force); } } @@ -58,9 +58,10 @@ class Resetter * * @param string $indexName * @param bool $populating + * @param bool $force If index exists with same name as alias, remove it * @throws \InvalidArgumentException if no index exists for the given name */ - public function resetIndex($indexName, $populating = false) + public function resetIndex($indexName, $populating = false, $force = false) { $indexConfig = $this->configManager->getIndexConfiguration($indexName); $index = $this->indexManager->getIndex($indexName); @@ -73,7 +74,7 @@ class Resetter $index->create($mapping, true); if (!$populating and $indexConfig->isUseAlias()) { - $this->aliasProcessor->switchIndexAlias($indexConfig, $index); + $this->aliasProcessor->switchIndexAlias($indexConfig, $index, $force); } } From 76dcd2f62e0da9188011002db62ce692f0052a16 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Fri, 29 Aug 2014 15:52:46 +0100 Subject: [PATCH 147/154] fix warning if no aliases PHP Warning: array_keys() expects parameter 1 to be array, AliasProcessor.php on line 128 --- Index/AliasProcessor.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Index/AliasProcessor.php b/Index/AliasProcessor.php index 93877cd..a900b33 100644 --- a/Index/AliasProcessor.php +++ b/Index/AliasProcessor.php @@ -125,9 +125,11 @@ class AliasProcessor $aliasedIndexes = array(); foreach ($aliasesInfo as $indexName => $indexInfo) { - $aliases = array_keys($indexInfo['aliases']); - if (in_array($aliasName, $aliases)) { - $aliasedIndexes[] = $indexName; + if (isset($indexInfo['aliases'])) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } } } From c4210a5c6dbab548342765936b8079ff4840ba13 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 4 Sep 2014 09:37:27 +1000 Subject: [PATCH 148/154] Fix previous merge --- Index/AliasProcessor.php | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Index/AliasProcessor.php b/Index/AliasProcessor.php index b38b5e3..29cfdc5 100644 --- a/Index/AliasProcessor.php +++ b/Index/AliasProcessor.php @@ -11,10 +11,10 @@ namespace FOS\ElasticaBundle\Index; +use Elastica\Client; use Elastica\Exception\ExceptionInterface; use Elastica\Request; use FOS\ElasticaBundle\Configuration\IndexConfig; -use FOS\ElasticaBundle\Elastica\Client; use FOS\ElasticaBundle\Elastica\Index; use FOS\ElasticaBundle\Exception\AliasIsIndexException; @@ -54,13 +54,12 @@ class AliasProcessor try { $aliasedIndexes = $this->getAliasedIndexes($client, $aliasName); } catch(AliasIsIndexException $e) { - if ($force) { - $this->deleteIndex($client, $aliasName); - - return; + if (!$force) { + throw $e; } - throw $e; + $this->deleteIndex($client, $aliasName); + $aliasedIndexes = array(); } if (count($aliasedIndexes) > 1) { @@ -75,7 +74,7 @@ class AliasProcessor } $aliasUpdateRequest = array('actions' => array()); - if (count($aliasedIndexes) == 1) { + if (count($aliasedIndexes) === 1) { // if the alias is set - add an action to remove it $oldIndexName = $aliasedIndexes[0]; $aliasUpdateRequest['actions'][] = array( @@ -135,7 +134,7 @@ class AliasProcessor * @param Client $client * @param string $aliasName Alias name * @return array - * @throws \FOS\ElasticaBundle\Exception\AliasIsIndexException + * @throws AliasIsIndexException */ private function getAliasedIndexes(Client $client, $aliasName) { @@ -146,11 +145,13 @@ class AliasProcessor if ($indexName === $aliasName) { throw new AliasIsIndexException($indexName); } - if (isset($indexInfo['aliases'])) { - $aliases = array_keys($indexInfo['aliases']); - if (in_array($aliasName, $aliases)) { - $aliasedIndexes[] = $indexName; - } + if (!isset($indexInfo['aliases'])) { + continue; + } + + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; } } From 1d5fe44ca4e8a70359dc108f2fd364cf30d93990 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 24 Aug 2014 19:50:56 +1000 Subject: [PATCH 149/154] Fix CS from scrutiniser-ci --- Configuration/TypeConfig.php | 3 +++ DependencyInjection/Configuration.php | 2 +- DependencyInjection/FOSElasticaExtension.php | 4 ++-- Doctrine/Listener.php | 19 +++++++++------- Doctrine/ORM/Provider.php | 1 - Elastica/Index.php | 4 +++- Index/Resetter.php | 4 +--- Manager/RepositoryManager.php | 3 +++ Paginator/RawPaginatorAdapter.php | 4 ++-- Persister/ObjectPersister.php | 1 - Persister/ObjectSerializerPersister.php | 3 +++ Propel/ElasticaToModelTransformer.php | 1 + Repository.php | 22 +++++++++++++++++++ Tests/Doctrine/AbstractListenerTest.php | 20 +++++++++++++++++ Tests/FOSElasticaBundleTest.php | 1 - .../ObjectSerializerPersisterTest.php | 2 -- ...asticaToModelTransformerCollectionTest.php | 3 +++ 17 files changed, 75 insertions(+), 22 deletions(-) diff --git a/Configuration/TypeConfig.php b/Configuration/TypeConfig.php index 3f1c939..fc9041d 100644 --- a/Configuration/TypeConfig.php +++ b/Configuration/TypeConfig.php @@ -77,6 +77,9 @@ class TypeConfig return $this->getConfig('search_analyzer'); } + /** + * @param string $key + */ private function getConfig($key) { return isset($this->config[$key]) ? diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 552f61b..3c5d18c 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -30,7 +30,7 @@ class Configuration implements ConfigurationInterface /** * Generates the configuration tree. * - * @return \Symfony\Component\Config\Definition\NodeInterface + * @return TreeBuilder */ public function getConfigTreeBuilder() { diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index cdf109b..804be44 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -82,7 +82,7 @@ class FOSElasticaExtension extends Extension /** * @param array $config * @param ContainerBuilder $container - * @return Configuration|null|\Symfony\Component\Config\Definition\ConfigurationInterface + * @return Configuration */ public function getConfiguration(array $config, ContainerBuilder $container) { @@ -523,7 +523,7 @@ class FOSElasticaExtension extends Extension * * @param array $typeConfig * @param ContainerBuilder $container - * @param $elasticaToModelId + * @param string $elasticaToModelId * @param Reference $typeRef * @param string $indexName * @param string $typeName diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 73a271d..bd8b2c0 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -174,27 +174,30 @@ class Listener implements EventSubscriber } /** - * Iterate through scheduled actions before flushing to emulate 2.x behavior. Note that the ElasticSearch index - * will fall out of sync with the source data in the event of a crash during flush. + * Iterate through scheduled actions before flushing to emulate 2.x behavior. + * Note that the ElasticSearch index will fall out of sync with the source + * data in the event of a crash during flush. + * + * This method is only called in legacy configurations of the listener. */ - public function preFlush(EventArgs $eventArgs) + public function preFlush() { $this->persistScheduled(); } /** - * Iterating through scheduled actions *after* flushing ensures that the ElasticSearch index will be affected - * only if the query is successful + * Iterating through scheduled actions *after* flushing ensures that the + * ElasticSearch index will be affected only if the query is successful. */ - public function postFlush(EventArgs $eventArgs) + public function postFlush() { $this->persistScheduled(); } /** * Record the specified identifier to delete. Do not need to entire object. - * @param mixed $object - * @return mixed + * + * @param object $object */ protected function scheduleForDeletion($object) { diff --git a/Doctrine/ORM/Provider.php b/Doctrine/ORM/Provider.php index dfd6700..7e2ac12 100644 --- a/Doctrine/ORM/Provider.php +++ b/Doctrine/ORM/Provider.php @@ -3,7 +3,6 @@ namespace FOS\ElasticaBundle\Doctrine\ORM; use Doctrine\ORM\QueryBuilder; -use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Doctrine\AbstractProvider; use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException; diff --git a/Elastica/Index.php b/Elastica/Index.php index bf37c51..49c656e 100644 --- a/Elastica/Index.php +++ b/Elastica/Index.php @@ -3,7 +3,6 @@ namespace FOS\ElasticaBundle\Elastica; use Elastica\Index as BaseIndex; -use Elastica\Type; /** * Overridden Elastica Index class that provides dynamic index name changes. @@ -32,6 +31,9 @@ class Index extends BaseIndex return $this->originalName ?: $this->_name; } + /** + * @param string $type + */ public function getType($type) { if (isset($this->typeCache[$type])) { diff --git a/Index/Resetter.php b/Index/Resetter.php index c93ae2d..9b65a8f 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -5,9 +5,7 @@ namespace FOS\ElasticaBundle\Index; use Elastica\Index; use Elastica\Exception\ResponseException; use Elastica\Type\Mapping; -use FOS\ElasticaBundle\Configuration\IndexConfig; use FOS\ElasticaBundle\Configuration\ConfigManager; -use FOS\ElasticaBundle\Elastica\Client; /** * Deletes and recreates indexes @@ -110,7 +108,7 @@ class Resetter /** * A command run when a population has finished. * - * @param $indexName + * @param string $indexName */ public function postPopulate($indexName) { diff --git a/Manager/RepositoryManager.php b/Manager/RepositoryManager.php index be07b42..7697b58 100644 --- a/Manager/RepositoryManager.php +++ b/Manager/RepositoryManager.php @@ -69,6 +69,9 @@ class RepositoryManager implements RepositoryManagerInterface return 'FOS\ElasticaBundle\Repository'; } + /** + * @param string $entityName + */ private function createRepository($entityName) { if (!class_exists($repositoryName = $this->getRepositoryName($entityName))) { diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index 9136bc0..f05205a 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -54,8 +54,8 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface /** * Returns the paginated results. * - * @param $offset - * @param $itemCountPerPage + * @param integer $offset + * @param integer $itemCountPerPage * @throws \InvalidArgumentException * @return ResultSet */ diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 9604f7e..0fe40c3 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -4,7 +4,6 @@ namespace FOS\ElasticaBundle\Persister; use Psr\Log\LoggerInterface; use Elastica\Exception\BulkException; -use Elastica\Exception\NotFoundException; use FOS\ElasticaBundle\Transformer\ModelToElasticaTransformerInterface; use Elastica\Type; use Elastica\Document; diff --git a/Persister/ObjectSerializerPersister.php b/Persister/ObjectSerializerPersister.php index 1a15656..3e33f8d 100644 --- a/Persister/ObjectSerializerPersister.php +++ b/Persister/ObjectSerializerPersister.php @@ -17,6 +17,9 @@ class ObjectSerializerPersister extends ObjectPersister { protected $serializer; + /** + * @param string $objectClass + */ public function __construct(Type $type, ModelToElasticaTransformerInterface $transformer, $objectClass, $serializer) { parent::__construct($type, $transformer, $objectClass, array()); diff --git a/Propel/ElasticaToModelTransformer.php b/Propel/ElasticaToModelTransformer.php index af5f8ab..e3602e5 100644 --- a/Propel/ElasticaToModelTransformer.php +++ b/Propel/ElasticaToModelTransformer.php @@ -170,6 +170,7 @@ class ElasticaToModelTransformer implements ElasticaToModelTransformerInterface /** * @see https://github.com/doctrine/common/blob/master/lib/Doctrine/Common/Util/Inflector.php + * @param string $str */ private function camelize($str) { diff --git a/Repository.php b/Repository.php index 70b2a21..fcc2784 100644 --- a/Repository.php +++ b/Repository.php @@ -19,21 +19,43 @@ class Repository $this->finder = $finder; } + /** + * @param mixed $query + * @param integer $limit + * @param array $options + * @return array + */ public function find($query, $limit = null, $options = array()) { return $this->finder->find($query, $limit, $options); } + /** + * @param mixed $query + * @param integer $limit + * @param array $options + * @return mixed + */ public function findHybrid($query, $limit = null, $options = array()) { return $this->finder->findHybrid($query, $limit, $options); } + /** + * @param mixed $query + * @param array $options + * @return \Pagerfanta\Pagerfanta + */ public function findPaginated($query, $options = array()) { return $this->finder->findPaginated($query, $options); } + /** + * @param string $query + * @param array $options + * @return Paginator\PaginatorAdapterInterface + */ public function createPaginatorAdapter($query, $options = array()) { return $this->finder->createPaginatorAdapter($query, $options); diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index 1f238d6..7242255 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -173,8 +173,14 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase abstract protected function getListenerClass(); + /** + * @return string + */ abstract protected function getObjectManagerClass(); + /** + * @return string + */ abstract protected function getClassMetadataClass(); private function createLifecycleEventArgs() @@ -205,6 +211,11 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase ->getMock(); } + /** + * @param Listener\Entity $object + * @param string $indexName + * @param string $typeName + */ private function getMockPersister($object, $indexName, $typeName) { $mock = $this->getMockBuilder('FOS\ElasticaBundle\Persister\ObjectPersister') @@ -235,6 +246,12 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase return $mock; } + /** + * @param string $indexName + * @param string $typeName + * @param Listener\Entity $object + * @param boolean $return + */ private function getMockIndexable($indexName, $typeName, $object, $return = null) { $mock = $this->getMock('FOS\ElasticaBundle\Provider\IndexableInterface'); @@ -256,6 +273,9 @@ class Entity { private $id; + /** + * @param integer $id + */ public function __construct($id) { $this->id = $id; diff --git a/Tests/FOSElasticaBundleTest.php b/Tests/FOSElasticaBundleTest.php index 3828e8b..4290e1d 100644 --- a/Tests/FOSElasticaBundleTest.php +++ b/Tests/FOSElasticaBundleTest.php @@ -3,7 +3,6 @@ namespace FOS\ElasticaBundle\Tests\Resetter; use FOS\ElasticaBundle\FOSElasticaBundle; -use Symfony\Component\DependencyInjection\Compiler\PassConfig; class FOSElasticaBundleTest extends \PHPUnit_Framework_TestCase { diff --git a/Tests/Persister/ObjectSerializerPersisterTest.php b/Tests/Persister/ObjectSerializerPersisterTest.php index fe15c0c..914b5dd 100644 --- a/Tests/Persister/ObjectSerializerPersisterTest.php +++ b/Tests/Persister/ObjectSerializerPersisterTest.php @@ -2,9 +2,7 @@ namespace FOS\ElasticaBundle\Tests\ObjectSerializerPersister; -use FOS\ElasticaBundle\Persister\ObjectPersister; use FOS\ElasticaBundle\Persister\ObjectSerializerPersister; -use FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer; use FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer; use Symfony\Component\PropertyAccess\PropertyAccess; diff --git a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php index eb4d8e4..c3fc323 100644 --- a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php +++ b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php @@ -157,6 +157,9 @@ class POPO public $id; public $data; + /** + * @param integer $id + */ public function __construct($id, $data) { $this->data = $data; From 428a1014ca244c5d864252f44ce336b9a0774452 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 24 Aug 2014 19:53:24 +1000 Subject: [PATCH 150/154] Move query logging into its own method --- Elastica/Client.php | 49 +++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/Elastica/Client.php b/Elastica/Client.php index 1131ba5..372b395 100644 --- a/Elastica/Client.php +++ b/Elastica/Client.php @@ -30,7 +30,11 @@ class Client extends BaseClient private $stopwatch; /** - * {@inheritdoc} + * @param string $path + * @param string $method + * @param array $data + * @param array $query + * @return \Elastica\Response */ public function request($path, $method = Request::GET, $data = array(), array $query = array()) { @@ -41,20 +45,7 @@ class Client extends BaseClient $start = microtime(true); $response = parent::request($path, $method, $data, $query); - if ($this->_logger and $this->_logger instanceof ElasticaLogger) { - $time = microtime(true) - $start; - - $connection = $this->getLastRequest()->getConnection(); - - $connection_array = array( - 'host' => $connection->getHost(), - 'port' => $connection->getPort(), - 'transport' => $connection->getTransport(), - 'headers' => $connection->hasConfig('headers') ? $connection->getConfig('headers') : array(), - ); - - $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); - } + $this->logQuery($path, $method, $data, $query, $start); if ($this->stopwatch) { $this->stopwatch->stop('es_request'); @@ -81,4 +72,32 @@ class Client extends BaseClient { $this->stopwatch = $stopwatch; } + + /** + * Log the query if we have an instance of ElasticaLogger. + * + * @param string $path + * @param string $method + * @param array $data + * @param array $query + * @param int $start + */ + private function logQuery($path, $method, $data, array $query, $start) + { + if (!$this->_logger or !$this->_logger instanceof ElasticaLogger) { + return; + } + + $time = microtime(true) - $start; + $connection = $this->getLastRequest()->getConnection(); + + $connection_array = array( + 'host' => $connection->getHost(), + 'port' => $connection->getPort(), + 'transport' => $connection->getTransport(), + 'headers' => $connection->hasConfig('headers') ? $connection->getConfig('headers') : array(), + ); + + $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); + } } From d0ce82ac2afb8ba685b200276e5eab84d94c4ced Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 21 Sep 2014 20:05:35 +1000 Subject: [PATCH 151/154] Adjust DoctrineListener to remove unnecessary method getDoctrineObject --- CHANGELOG-3.1.md | 14 ++++++++++ Doctrine/Listener.php | 60 ++++++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 27 deletions(-) create mode 100644 CHANGELOG-3.1.md diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md new file mode 100644 index 0000000..0170373 --- /dev/null +++ b/CHANGELOG-3.1.md @@ -0,0 +1,14 @@ +CHANGELOG for 3.0.x +=================== + +This changelog references the relevant changes (bug and security fixes) done +in 3.1 versions. + +To get the diff for a specific change, go to +https://github.com/FriendsOfSymfony/FOSElasticaBundle/commit/XXX where XXX is +the commit hash. To get the diff between two versions, go to +https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 + +* 3.1.0 + +* BC BREAK: `DoctrineListener#scheduleForDeletion` access changed to private. diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index bd8b2c0..039ddaa 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -2,8 +2,8 @@ namespace FOS\ElasticaBundle\Doctrine; -use Doctrine\Common\EventArgs; use Doctrine\Common\EventSubscriber; +use Doctrine\Common\Persistence\Event\LifecycleEventArgs; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Persister\ObjectPersister; use FOS\ElasticaBundle\Provider\IndexableInterface; @@ -38,13 +38,23 @@ class Listener implements EventSubscriber private $config; /** - * Objects scheduled for insertion and replacement + * Objects scheduled for insertion. + * + * @var array */ public $scheduledForInsertion = array(); + + /** + * Objects scheduled to be updated or removed. + * + * @var array + */ public $scheduledForUpdate = array(); /** * IDs of objects scheduled for removal + * + * @var array */ public $scheduledForDeletion = array(); @@ -56,7 +66,7 @@ class Listener implements EventSubscriber protected $propertyAccessor; /** - * @var \FOS\ElasticaBundle\Provider\IndexableInterface + * @var IndexableInterface */ private $indexable; @@ -98,37 +108,27 @@ class Listener implements EventSubscriber } /** - * Provides unified method for retrieving a doctrine object from an EventArgs instance + * Looks for new objects that should be indexed. * - * @param EventArgs $eventArgs - * @return object Entity | Document - * @throws \RuntimeException if no valid getter is found. + * @param LifecycleEventArgs $eventArgs */ - private function getDoctrineObject(EventArgs $eventArgs) + public function postPersist(LifecycleEventArgs $eventArgs) { - if (method_exists($eventArgs, 'getObject')) { - return $eventArgs->getObject(); - } elseif (method_exists($eventArgs, 'getEntity')) { - return $eventArgs->getEntity(); - } elseif (method_exists($eventArgs, 'getDocument')) { - return $eventArgs->getDocument(); - } - - throw new \RuntimeException('Unable to retrieve object from EventArgs.'); - } - - public function postPersist(EventArgs $eventArgs) - { - $entity = $this->getDoctrineObject($eventArgs); + $entity = $eventArgs->getObject(); if ($this->objectPersister->handlesObject($entity) && $this->isObjectIndexable($entity)) { $this->scheduledForInsertion[] = $entity; } } - public function postUpdate(EventArgs $eventArgs) + /** + * Looks for objects being updated that should be indexed or removed from the index. + * + * @param LifecycleEventArgs $eventArgs + */ + public function postUpdate(LifecycleEventArgs $eventArgs) { - $entity = $this->getDoctrineObject($eventArgs); + $entity = $eventArgs->getObject(); if ($this->objectPersister->handlesObject($entity)) { if ($this->isObjectIndexable($entity)) { @@ -143,10 +143,12 @@ class Listener implements EventSubscriber /** * Delete objects preRemove instead of postRemove so that we have access to the id. Because this is called * preRemove, first check that the entity is managed by Doctrine + * + * @param LifecycleEventArgs $eventArgs */ - public function preRemove(EventArgs $eventArgs) + public function preRemove(LifecycleEventArgs $eventArgs) { - $entity = $this->getDoctrineObject($eventArgs); + $entity = $eventArgs->getObject(); if ($this->objectPersister->handlesObject($entity)) { $this->scheduleForDeletion($entity); @@ -179,6 +181,10 @@ class Listener implements EventSubscriber * data in the event of a crash during flush. * * This method is only called in legacy configurations of the listener. + * + * @deprecated This method should only be called in applications that depend + * on the behaviour that entities are indexed regardless of if a + * flush is successful. */ public function preFlush() { @@ -199,7 +205,7 @@ class Listener implements EventSubscriber * * @param object $object */ - protected function scheduleForDeletion($object) + private function scheduleForDeletion($object) { if ($identifierValue = $this->propertyAccessor->getValue($object, $this->config['identifier'])) { $this->scheduledForDeletion[] = $identifierValue; From 71a86cada54ebc94d5723808b0616c7e378a3a27 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 21 Sep 2014 20:06:06 +1000 Subject: [PATCH 152/154] BC BREAK: Add `handlesObject` method to ObjectPersisterInterface --- CHANGELOG-3.1.md | 2 ++ Persister/ObjectPersisterInterface.php | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index 0170373..ee9af70 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -12,3 +12,5 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 * 3.1.0 * BC BREAK: `DoctrineListener#scheduleForDeletion` access changed to private. +* BC BREAK: `ObjectPersisterInterface` gains the method `handlesObject` that + returns a boolean value if it will handle a given object or not. diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index 2b4c8ee..0df7f7e 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -68,4 +68,12 @@ interface ObjectPersisterInterface * @param array $identifiers array of domain model object identifiers */ public function deleteManyByIdentifiers(array $identifiers); + + /** + * If the object persister handles the given object. + * + * @param object $object + * @return bool + */ + public function handlesObject($object); } From b3f87e414f5a5b04a351e741d437a40c988bdbdb Mon Sep 17 00:00:00 2001 From: Michael Schramm Date: Fri, 5 Sep 2014 02:52:37 +0200 Subject: [PATCH 153/154] move classes to parametes --- Resources/config/mongodb.xml | 16 ++++++++++++---- Resources/config/orm.xml | 15 +++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 048d799..8e15533 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -4,8 +4,16 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> + + FOS\ElasticaBundle\Doctrine\MongoDB\Provider + FOS\ElasticaBundle\Doctrine\Listener + FOS\ElasticaBundle\Doctrine\MongoDB\ElasticaToModelTransformer + FOS\ElasticaBundle\Doctrine\RepositoryManager + + + - + @@ -13,7 +21,7 @@ - + @@ -21,7 +29,7 @@ - + @@ -30,7 +38,7 @@ - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index ddc0e50..94f21d4 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -4,8 +4,15 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd"> + + FOS\ElasticaBundle\Doctrine\ORM\Provider + FOS\ElasticaBundle\Doctrine\Listener + FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer + FOS\ElasticaBundle\Doctrine\RepositoryManager + + - + @@ -13,7 +20,7 @@ - + @@ -21,7 +28,7 @@ - + @@ -30,7 +37,7 @@ - + From 25d56d0a0f4a7a9bfe65390220ed567204cd030b Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 3 Oct 2014 08:21:02 +1000 Subject: [PATCH 154/154] 3.1.x requires doctrine 2.4+ --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index bb30928..833621a 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,7 @@ "psr/log": "~1.0" }, "require-dev":{ - "doctrine/orm": "~2.2", + "doctrine/orm": "~2.4", "doctrine/doctrine-bundle": "~1.2@beta", "jms/serializer-bundle": "@stable", "phpunit/phpunit": "~4.1", @@ -34,7 +34,7 @@ "symfony/twig-bundle": "~2.3" }, "suggest": { - "doctrine/orm": "~2.2", + "doctrine/orm": "~2.4", "doctrine/mongodb-odm": "1.0.*@dev", "propel/propel1": "1.6.*", "pagerfanta/pagerfanta": "1.0.*@dev",