From 714502fa1f51bf7e9b372c799d811483b5db3b1c Mon Sep 17 00:00:00 2001 From: esodin Date: Thu, 17 Jul 2014 16:01:10 +0300 Subject: [PATCH 01/41] Issue: Parent is missing in the fields list that causes RoutingMissingException on flushing --- DependencyInjection/FOSElasticaExtension.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 292a9a5..89ac3dd 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -391,7 +391,13 @@ class FOSElasticaExtension extends Extension $arguments[] = array(new Reference($callbackId), 'serialize'); } else { $abstractId = 'fos_elastica.object_persister'; - $arguments[] = $this->indexConfigs[$indexName]['types'][$typeName]['mapping']['properties']; + $mapping = $this->indexConfigs[$indexName]['types'][$typeName]['mapping']; + $argument = $mapping['properties']; + if(isset($mapping['_parent'])){ + $argument['_parent'] = $mapping['_parent']; + } + $arguments[] = $argument; + } $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName); From 1ef55b1239a5b7c161da512d3a611d4cab9d30f9 Mon Sep 17 00:00:00 2001 From: esodin Date: Thu, 24 Jul 2014 17:03:26 +0300 Subject: [PATCH 02/41] Issue: Parent is missing in the fields list that causes RoutingMissingException on flushing - test --- .../FOSElasticaExtensionTest.php | 36 +++++++++++++++++++ .../DependencyInjection/config/config.yml | 21 +++++++++++ 2 files changed, 57 insertions(+) create mode 100644 Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php create mode 100644 Tests/Functional/DependencyInjection/config/config.yml diff --git a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php new file mode 100644 index 0000000..f210136 --- /dev/null +++ b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php @@ -0,0 +1,36 @@ +setParameter('kernel.debug', true); + + $extension = new FOSElasticaExtension; + + $extension->load($config, $containerBuilder); + + $this->assertTrue($containerBuilder->hasDefinition('fos_elastica.object_persister.test_index.child_field')); + + $persisterCallDefinition = $containerBuilder->getDefinition('fos_elastica.object_persister.test_index.child_field'); + + $this->assertArrayHasKey('_parent', $persisterCallDefinition->getArguments()['index_3']); + } + +} \ No newline at end of file diff --git a/Tests/Functional/DependencyInjection/config/config.yml b/Tests/Functional/DependencyInjection/config/config.yml new file mode 100644 index 0000000..5528d18 --- /dev/null +++ b/Tests/Functional/DependencyInjection/config/config.yml @@ -0,0 +1,21 @@ +fos_elastica: + clients: + default: + url: http://localhost:9200 + indexes: + test_index: + client: default + types: + parent_field: + mappings: + text: ~ + persistence: + driver: orm + model: foo_model + child_field: + mappings: + text: ~ + persistence: + driver: orm + model: foo_model + _parent: { type: "parent_field", property: "parent" } From 001b38cf59839d7acf230d76e765e77235af0842 Mon Sep 17 00:00:00 2001 From: esodin Date: Thu, 24 Jul 2014 17:07:22 +0300 Subject: [PATCH 03/41] Issue: Parent is missing in the fields list that causes RoutingMissingException on flushing - test --- .../DependencyInjection/FOSElasticaExtensionTest.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php index f210136..a02c35e 100644 --- a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php +++ b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php @@ -30,7 +30,10 @@ class FOSElasticaExtensionTest extends \PHPUnit_Framework_TestCase $persisterCallDefinition = $containerBuilder->getDefinition('fos_elastica.object_persister.test_index.child_field'); - $this->assertArrayHasKey('_parent', $persisterCallDefinition->getArguments()['index_3']); + $arguments = $persisterCallDefinition->getArguments()['index_3']; + + $this->assertArrayHasKey('_parent', $arguments); + $this->assertEquals('parent_field', $arguments['_parent']['type']); } } \ No newline at end of file From 659468ae3a47605d236bd726695a2bd65782cdfd Mon Sep 17 00:00:00 2001 From: esodin Date: Thu, 24 Jul 2014 17:36:45 +0300 Subject: [PATCH 04/41] Issue: Parent is missing in the fields list that causes RoutingMissingException on flushing - test --- .../DependencyInjection/FOSElasticaExtensionTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php index a02c35e..06676ef 100644 --- a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php +++ b/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php @@ -30,7 +30,8 @@ class FOSElasticaExtensionTest extends \PHPUnit_Framework_TestCase $persisterCallDefinition = $containerBuilder->getDefinition('fos_elastica.object_persister.test_index.child_field'); - $arguments = $persisterCallDefinition->getArguments()['index_3']; + $arguments = $persisterCallDefinition->getArguments(); + $arguments = $arguments['index_3']; $this->assertArrayHasKey('_parent', $arguments); $this->assertEquals('parent_field', $arguments['_parent']['type']); From eaa32cbf22acf6594c816a5b99abb92152c004ad Mon Sep 17 00:00:00 2001 From: Christophe Coevoet Date: Tue, 23 Dec 2014 15:19:52 +0100 Subject: [PATCH 05/41] Fix the BC layer for indexable callbacks Using services was not based on a @ prefix in the string in the old API but based on the existence of the class. --- DependencyInjection/Configuration.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b204f0c..a506853 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -199,7 +199,17 @@ class Configuration implements ConfigurationInterface isset($v['persistence']['listener']['is_indexable_callback']); }) ->then(function ($v) { - $v['indexable_callback'] = $v['persistence']['listener']['is_indexable_callback']; + $callback = $v['persistence']['listener']['is_indexable_callback']; + + if (is_array($callback)) { + list($class) = $callback + array(null); + + if (is_string($class) && !class_exists($class)) { + $callback[0] = '@'.$class; + } + } + + $v['indexable_callback'] = $callback; unset($v['persistence']['listener']['is_indexable_callback']); return $v; From 1f7acc563a04102cf64bf9d2eb4e691c69f08ce7 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Wed, 24 Dec 2014 09:09:45 +1100 Subject: [PATCH 06/41] Ignore strings starting with @ --- DependencyInjection/Configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index a506853..fa2c048 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -204,7 +204,7 @@ class Configuration implements ConfigurationInterface if (is_array($callback)) { list($class) = $callback + array(null); - if (is_string($class) && !class_exists($class)) { + if ($class[0] !== '@' && is_string($class) && !class_exists($class)) { $callback[0] = '@'.$class; } } From 3975ed3d5b69d88a3654d3421df9a096ee12f152 Mon Sep 17 00:00:00 2001 From: Danijel Brkic Date: Wed, 19 Nov 2014 14:21:31 +0100 Subject: [PATCH 07/41] Built-in templates use a base64 encoded image for the toolbar --- Resources/views/Collector/elastica.html.twig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/views/Collector/elastica.html.twig b/Resources/views/Collector/elastica.html.twig index e6d7072..82a3dcf 100644 --- a/Resources/views/Collector/elastica.html.twig +++ b/Resources/views/Collector/elastica.html.twig @@ -23,7 +23,7 @@ {% block menu %} - + Elastica {{ collector.querycount }} From 5eaff9e61ba19a3af16614a767193a001308c930 Mon Sep 17 00:00:00 2001 From: DjangoFR Date: Thu, 4 Dec 2014 16:40:10 +0100 Subject: [PATCH 08/41] removed unused image (being base64 encoded - see #742) --- Resources/public/images/elastica.png | Bin 1049 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 Resources/public/images/elastica.png diff --git a/Resources/public/images/elastica.png b/Resources/public/images/elastica.png deleted file mode 100644 index dbde0146fc9dab7535af943404cd5ef7e35128ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1049 zcmV+!1m^pRP)Px#0%A)?L;(MXkIcUS000SaNLh0L01FcU01FcV0GgZ_00007bV*G`2ipn+4h=D% z5t_#U00W&#L_t(I%XQOBY}8d0$MNsE_doL=W;(Rf4wmw;MQo`&)Tj#)f|`&Rf?^^Y zSx7=$sL_Qcw4!Ar*pQ$L1x$d5u`XDs!G+O4XiP9X)G81N1ZWKf5omanPCM`ay~jn1 zu_XQ$XZ6X+$-Ut$V0z;{X0RiIZiIW%*xS3S9@xInBAYvr@WPTCO`Ge z)ZypF+9UC2>PVT!8M7X0ZfTRD(<2XRv%6v+p#8qK*IU=M zPU}ChxJPHjPPO$lxs-&&B_7mFOSY!6jl<{1ijA_!&a-sTt&eMA)j`>zQNyv;t7#i2 z5Tg_e1EG(%g}EEU*PApO5a>cxHdgn^0s~{oTxbF;1}ia%CB{yK&046xB}JDoiW+XR zzxSM`4AuwEVANm;uvp?II#<3Bjey4}idQF48BhqLDc*<`4X^|RSc(2D4UHzFn8SEf z6$N$-R_^#zt|>V^kr9jrjE@t!yuUWPQWGng17%m^Z~{=-US;Hw!PeYMEBGtvBa4r}n9tNvyJqGwdAiGQ^aJ|R(f)}q-#Pl4)j$Gb zC`CMwxSi?j+5^F%?j=v?60IGld!95a`x%mWQ420Dn4aIAjUqcJ5%dHb&yCZC$* zIqDY`Kgoth@pzJapPIr#7WHMm%&~m39*ugUThYrtZXz5iHN3lY_r^c2Wp;l(w4UBc zww!gFq6YV7RG>X<|8Cy(w#WG2WE$&+(ub4NAW#(2nw=YZ5h5U<1Q;w;JQ1agCY@*u z0#zN=q<-peg9C&r22jdGI3j#1s_|pK3V~MiD_!l|!2{z}@Hir(3GPb%8*acqd)&1> T1X+Rr00000NkvXXu0mjf+yUwb From 156884527c78c2f6d72df016b86c35a5d9048380 Mon Sep 17 00:00:00 2001 From: Evan Owens Date: Tue, 30 Dec 2014 08:43:50 -0500 Subject: [PATCH 09/41] Update example FOS\ElasticaBundle\Client has been deprecated. --- Resources/doc/cookbook/suppress-server-errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/cookbook/suppress-server-errors.md b/Resources/doc/cookbook/suppress-server-errors.md index e4e371e..72c7b38 100644 --- a/Resources/doc/cookbook/suppress-server-errors.md +++ b/Resources/doc/cookbook/suppress-server-errors.md @@ -23,7 +23,7 @@ namespace Acme\ElasticaBundle; use Elastica\Exception\ExceptionInterface; use Elastica\Request; use Elastica\Response; -use FOS\ElasticaBundle\Client as BaseClient; +use FOS\ElasticaBundle\Elastica\Client as BaseClient; class Client extends BaseClient { From e361b7c53b4f1381c0551b1c20f12b0b915b5145 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 4 Jan 2015 21:58:53 +1100 Subject: [PATCH 10/41] Implement additional configuration options for types. --- CHANGELOG-3.1.md | 4 +++- Configuration/TypeConfig.php | 24 ++++++++++++++++++++ DependencyInjection/Configuration.php | 3 +++ DependencyInjection/FOSElasticaExtension.php | 3 +++ Index/MappingBuilder.php | 20 ++++++++++------ Index/Resetter.php | 3 +++ Tests/Functional/MappingToElasticaTest.php | 4 ++++ Tests/Functional/app/Basic/config.yml | 3 +++ Tests/Functional/app/Serializer/config.yml | 2 +- 9 files changed, 57 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index 0ce44ad..a03de37 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -14,6 +14,8 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 * BC BREAK: `Doctrine\Listener#scheduleForDeletion` access changed to private. * BC BREAK: `ObjectPersisterInterface` gains the method `handlesObject` that returns a boolean value if it will handle a given object or not. - * Removed `Doctrine\Listener#getSubscribedEvents`. The container + * BC BREAK: Removed `Doctrine\Listener#getSubscribedEvents`. The container configuration now configures tags with the methods to call to avoid loading this class on every request where doctrine is active. + * Added ability to configure `date_detection`, `numeric_detection` and + `dynamic_date_formats` for types. diff --git a/Configuration/TypeConfig.php b/Configuration/TypeConfig.php index fc9041d..a46cd34 100644 --- a/Configuration/TypeConfig.php +++ b/Configuration/TypeConfig.php @@ -35,6 +35,22 @@ class TypeConfig $this->name = $name; } + /** + * @return bool|null + */ + public function getDateDetection() + { + return $this->getConfig('date_detection'); + } + + /** + * @return array + */ + public function getDynamicDateFormats() + { + return $this->getConfig('dynamic_date_formats'); + } + /** * @return string|null */ @@ -61,6 +77,14 @@ class TypeConfig null; } + /** + * @return bool|null + */ + public function getNumericDetection() + { + return $this->getConfig('numeric_detection'); + } + /** * @return string */ diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index b204f0c..760966e 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -225,7 +225,10 @@ class Configuration implements ConfigurationInterface }) ->end() ->children() + ->booleanNode('date_detection')->end() + ->arrayNode('dynamic_date_formats')->prototype('scalar')->end()->end() ->scalarNode('index_analyzer')->end() + ->booleanNode('numeric_detection')->end() ->scalarNode('search_analyzer')->end() ->variableNode('indexable_callback')->end() ->append($this->getPersistenceNode()) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 565ba33..529bd29 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -241,6 +241,9 @@ class FOSElasticaExtension extends Extension 'serializer', 'index_analyzer', 'search_analyzer', + 'date_detection', + 'dynamic_date_formats', + 'numeric_detection', ) as $field) { $typeConfig['config'][$field] = array_key_exists($field, $type) ? $type[$field] : diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 21ae871..996db5f 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -58,13 +58,19 @@ class MappingBuilder */ public function buildTypeMapping(TypeConfig $typeConfig) { - $mapping = array_merge($typeConfig->getMapping(), array( - // 'date_detection' => true, - // 'dynamic_date_formats' => array() - // 'dynamic_templates' => $typeConfig->getDynamicTemplates(), - // 'numeric_detection' => false, - // 'properties' => array(), - )); + $mapping = $typeConfig->getMapping(); + + if (null !== $typeConfig->getDynamicDateFormats()) { + $mapping['dynamic_date_formats'] = $typeConfig->getDynamicDateFormats(); + } + + if (null !== $typeConfig->getDateDetection()) { + $mapping['date_detection'] = $typeConfig->getDateDetection(); + } + + if (null !== $typeConfig->getNumericDetection()) { + $mapping['numeric_detection'] = $typeConfig->getNumericDetection(); + } if ($typeConfig->getIndexAnalyzer()) { $mapping['index_analyzer'] = $typeConfig->getIndexAnalyzer(); diff --git a/Index/Resetter.php b/Index/Resetter.php index 9b65a8f..68a43dd 100644 --- a/Index/Resetter.php +++ b/Index/Resetter.php @@ -42,6 +42,9 @@ class Resetter /** * Deletes and recreates all indexes + * + * @param bool $populating + * @param bool $force */ public function resetAllIndexes($populating = false, $force = false) { diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index f42df61..229a1a2 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -49,6 +49,9 @@ class MappingToElasticaTest extends WebTestCase $mapping = $type->getMapping(); $this->assertNotEmpty($mapping, 'Mapping was populated'); + $this->assertFalse($mapping['type']['date_detection']); + $this->assertTrue($mapping['type']['numeric_detection']); + $this->assertEquals(array('yyyy-MM-dd'), $mapping['type']['dynamic_date_formats']); $this->assertArrayHasKey('store', $mapping['type']['properties']['field1']); $this->assertTrue($mapping['type']['properties']['field1']['store']); $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); @@ -105,6 +108,7 @@ class MappingToElasticaTest extends WebTestCase /** * @param Client $client + * @param string $type * @return \Elastica\Type */ private function getType(Client $client, $type = 'type') diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 3c3d369..690bf5e 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -46,6 +46,8 @@ fos_elastica: index_analyzer: my_analyzer type: search_analyzer: my_analyzer + date_detection: false + dynamic_date_formats: [ 'yyyy-MM-dd' ] dynamic_templates: - dates: match: "date_*" @@ -56,6 +58,7 @@ fos_elastica: mapping: analyzer: english type: string + numeric_detection: true properties: field1: ~ field2: diff --git a/Tests/Functional/app/Serializer/config.yml b/Tests/Functional/app/Serializer/config.yml index 9bea4ba..de7caec 100644 --- a/Tests/Functional/app/Serializer/config.yml +++ b/Tests/Functional/app/Serializer/config.yml @@ -28,7 +28,7 @@ fos_elastica: serializer: ~ indexes: index: - index_name: foselastica_test_%kernel.environment% + index_name: foselastica_ser_test_%kernel.environment% types: type: properties: From 92aab4bcf66ccb992195ad574457d706fcad63e6 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 4 Jan 2015 22:02:54 +1100 Subject: [PATCH 11/41] Add PR# to changelog --- CHANGELOG-3.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index a03de37..8880675 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -16,6 +16,6 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 returns a boolean value if it will handle a given object or not. * BC BREAK: Removed `Doctrine\Listener#getSubscribedEvents`. The container configuration now configures tags with the methods to call to avoid loading - this class on every request where doctrine is active. + this class on every request where doctrine is active. #729 * Added ability to configure `date_detection`, `numeric_detection` and - `dynamic_date_formats` for types. + `dynamic_date_formats` for types. #753 From 6ef6092f3f532a219517e94c7c3fc1fe147b9de0 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 4 Jan 2015 22:03:19 +1100 Subject: [PATCH 12/41] Update 3.0 changeling --- CHANGELOG-3.0.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 2596a29..1a49023 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,6 +12,11 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` +* 3.0.6 (Unreleased) + + * Removed unused public image asset for the web development toolbar + * Fixed is_indexable_callback BC code to support array notation + * 3.0.0-ALPHA6 * Moved `is_indexable_callback` from the listener properties to a type property called From 9f5ce217dcf5e628274a7767e20a413e5e0b1197 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 4 Jan 2015 22:08:20 +1100 Subject: [PATCH 13/41] Release 3.0.6 --- CHANGELOG-3.0.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 1a49023..3b9b2ac 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,10 +12,12 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.6 (Unreleased) +* 3.0.6 (2015-01-04) - * Removed unused public image asset for the web development toolbar - * Fixed is_indexable_callback BC code to support array notation + * Removed unused public image asset for the web development toolbar #742 + * Fixed is_indexable_callback BC code to support array notation #761 + * Fixed debug_logger for type providers #724 + * Clean the OM if we filter away the entire batch #737 * 3.0.0-ALPHA6 From a0f11ff36f012455df8ba081fce36913832a460d Mon Sep 17 00:00:00 2001 From: Vladimir Kartaviy Date: Wed, 7 Jan 2015 16:45:18 +0200 Subject: [PATCH 14/41] Fix ODM listener service arguments It became broken after https://github.com/FriendsOfSymfony/FOSElasticaBundle/commit/d731443aa5ad7a12c133bd1d7b97fa242c546e1e commit --- Resources/config/mongodb.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/Resources/config/mongodb.xml b/Resources/config/mongodb.xml index 8e15533..703c1d2 100644 --- a/Resources/config/mongodb.xml +++ b/Resources/config/mongodb.xml @@ -23,7 +23,6 @@ - From 905265ea0ebd0f0166de3f8ace63bcc995cb1cee Mon Sep 17 00:00:00 2001 From: Vladimir Kartaviy Date: Wed, 7 Jan 2015 16:49:46 +0200 Subject: [PATCH 15/41] "multi_field" type fields are not normalized Fix for #764 --- Index/MappingBuilder.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 21ae871..9782e84 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -107,6 +107,9 @@ class MappingBuilder if (!isset($property['type'])) { $property['type'] = 'string'; } + if ($property['type'] == 'multi_field' && isset($property['fields'])) { + $this->fixProperties($property['fields']); + } if (isset($property['properties'])) { $this->fixProperties($property['properties']); } From 2ce2d7e61053a7b7efbce73113ec455d23409eca Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 9 Jan 2015 08:55:57 +1100 Subject: [PATCH 16/41] Add test for multi_field --- CHANGELOG-3.0.md | 4 ++++ Tests/Functional/app/Basic/config.yml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 3b9b2ac..ecb5dad 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,6 +12,10 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` +* 3.0.7 (Unreleased) + + * Fixed multi_field properties not being normalised #769 + * 3.0.6 (2015-01-04) * Removed unused public image asset for the web development toolbar #742 diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 3c3d369..ebd1ea0 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -71,6 +71,11 @@ fos_elastica: properties: date: { boost: 5 } content: ~ + multiple: + type: "multi_field" + properties: + name: ~ + position: ~ user: type: "object" approver: From a28b9d3069f2baf3af57cef2925319ae10d81cbf Mon Sep 17 00:00:00 2001 From: Wouter J Date: Thu, 25 Sep 2014 10:52:51 +0200 Subject: [PATCH 17/41] Applied standard installation template --- Resources/doc/setup.md | 46 +++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 16e427f..c4f6784 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -1,40 +1,50 @@ Step 1: Setting up the bundle ============================= -A) Install FOSElasticaBundle ----------------------------- +A: Download the Bundle +---------------------- -FOSElasticaBundle is installed using [Composer](https://getcomposer.org). +Open a command console, enter your project directory and execute the +following command to download the latest stable version of this bundle: ```bash -$ php composer.phar require friendsofsymfony/elastica-bundle +$ composer require friendsofsymfony/elastica-bundle "~3.0" ``` +This command requires you to have Composer installed globally, as explained +in the [installation chapter](https://getcomposer.org/doc/00-intro.md) +of the Composer documentation. + ### Elasticsearch -Instructions for installing and deploying Elasticsearch may be found -[here](http://www.elasticsearch.org/guide/reference/setup/installation/). +Instructions for installing and deploying Elasticsearch may be found [here](http://www.elasticsearch.org/guide/reference/setup/installation/). +Step 2: Enable the Bundle +------------------------- -B) Enable FOSElasticaBundle ---------------------------- - -Enable FOSElasticaBundle in your AppKernel: +Then, enable the bundle by adding the following line in the `app/AppKernel.php` +file of your project: ```php Date: Tue, 20 Jan 2015 14:00:10 +1100 Subject: [PATCH 18/41] Dispatch an event when transforming objects --- CHANGELOG-3.1.md | 2 + Event/TransformEvent.php | 78 +++++++++++++++++++ Resources/config/transformer.xml | 3 +- Resources/doc/cookbook/custom-properties.md | 33 ++++++++ Resources/doc/index.md | 1 + .../ModelToElasticaAutoTransformerTest.php | 25 +++++- .../ModelToElasticaAutoTransformer.php | 18 ++++- 7 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 Event/TransformEvent.php create mode 100644 Resources/doc/cookbook/custom-properties.md diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index 8880675..12fbb05 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -19,3 +19,5 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 this class on every request where doctrine is active. #729 * Added ability to configure `date_detection`, `numeric_detection` and `dynamic_date_formats` for types. #753 + * New event `POST_TRANSFORM` which allows developers to add custom properties to + Elastica Documents for indexing. diff --git a/Event/TransformEvent.php b/Event/TransformEvent.php new file mode 100644 index 0000000..4f6871f --- /dev/null +++ b/Event/TransformEvent.php @@ -0,0 +1,78 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace FOS\ElasticaBundle\Event; + +use Symfony\Component\EventDispatcher\Event; + +class TransformEvent extends Event +{ + const POST_TRANSFORM = 'fos_elastica.post_transform'; + + /** + * @var mixed + */ + private $document; + + /** + * @var array + */ + private $fields; + + /** + * @var mixed + */ + private $object; + + /** + * @param mixed $document + * @param array $fields + * @param mixed $object + */ + public function __construct($document, array $fields, $object) + { + $this->document = $document; + $this->fields = $fields; + $this->object = $object; + } + + /** + * @return mixed + */ + public function getDocument() + { + return $this->document; + } + + /** + * @return array + */ + public function getFields() + { + return $this->fields; + } + + /** + * @return mixed + */ + public function getObject() + { + return $this->object; + } + + /** + * @param mixed $document + */ + public function setDocument($document) + { + $this->document = $document; + } +} diff --git a/Resources/config/transformer.xml b/Resources/config/transformer.xml index 4ce5062..0957152 100644 --- a/Resources/config/transformer.xml +++ b/Resources/config/transformer.xml @@ -12,7 +12,8 @@ - + + diff --git a/Resources/doc/cookbook/custom-properties.md b/Resources/doc/cookbook/custom-properties.md new file mode 100644 index 0000000..cc7751e --- /dev/null +++ b/Resources/doc/cookbook/custom-properties.md @@ -0,0 +1,33 @@ +##### Custom Repositories + +Since FOSElasticaBundle 3.1.0, we now dispatch an event for each transformation of an +object into an Elastica document which allows you to set custom properties on the Elastica +document for indexing. + +Set up an event listener or subscriber for +`FOS\ElasticaBundle\Event\TransformEvent::POST_TRANSFORM` to be able to inject your own +parameters. + +```php +class CustomPropertyListener implements EventSubscriberInterface +{ + private $anotherService; + + // ... + + public function addCustomProperty(TransformEvent $event) + { + $document = $event->getDocument(); + $custom = $this->anotherService->calculateCustom($event->getObject()); + + $document->set('custom', $custom); + } + + public static function getSubscribedEvents() + { + return array( + TransformEvent::POST_TRANSFORM => 'addCustomProperty', + ); + } +} +``` diff --git a/Resources/doc/index.md b/Resources/doc/index.md index 349723b..c856798 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -13,6 +13,7 @@ Cookbook Entries ---------------- * [Aliased Indexes](cookbook/aliased-indexes.md) +* [Custom Indexed Properties](cookbook/custom-properties.md) * [Custom Repositories](cookbook/custom-repositories.md) * [HTTP Headers for Elastica](cookbook/elastica-client-http-headers.md) * Performance - [Logging](cookbook/logging.md) diff --git a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php index 1fa6a8e..bb52129 100644 --- a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php +++ b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle\Tests\Transformer\ModelToElasticaAutoTransformer; +use FOS\ElasticaBundle\Event\TransformEvent; use FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer; use Symfony\Component\PropertyAccess\PropertyAccess; @@ -132,6 +133,21 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase } } + public function testTransformerDispatches() + { + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcherInterface') + ->getMock(); + $dispatcher->expects($this->once()) + ->method('dispatch') + ->with( + TransformEvent::POST_TRANSFORM, + $this->isInstanceOf('FOS\ElasticaBundle\Event\TransformEvent') + ); + + $transformer = $this->getTransformer($dispatcher); + $transformer->transform(new POPO(), array()); + } + public function testThatCanTransformObject() { $transformer = $this->getTransformer(); @@ -295,8 +311,8 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase $this->assertTrue(array_key_exists('obj', $data)); $this->assertInternalType('array', $data['obj']); $this->assertEquals(array( - 'foo' => 'foo', - 'bar' => 'foo', + 'foo' => 'foo', + 'bar' => 'foo', 'id' => 1 ), $data['obj']); } @@ -387,11 +403,12 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase } /** + * @param null|\Symfony\Component\EventDispatcher\EventDispatcherInterface $dispatcher * @return ModelToElasticaAutoTransformer */ - private function getTransformer() + private function getTransformer($dispatcher = null) { - $transformer = new ModelToElasticaAutoTransformer(); + $transformer = new ModelToElasticaAutoTransformer(array(), $dispatcher); $transformer->setPropertyAccessor(PropertyAccess::getPropertyAccessor()); return $transformer; diff --git a/Transformer/ModelToElasticaAutoTransformer.php b/Transformer/ModelToElasticaAutoTransformer.php index 106db15..eff29d2 100644 --- a/Transformer/ModelToElasticaAutoTransformer.php +++ b/Transformer/ModelToElasticaAutoTransformer.php @@ -2,6 +2,8 @@ namespace FOS\ElasticaBundle\Transformer; +use FOS\ElasticaBundle\Event\TransformEvent; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\PropertyAccess\PropertyAccessorInterface; use Elastica\Document; @@ -12,6 +14,11 @@ use Elastica\Document; */ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterface { + /** + * @var EventDispatcherInterface + */ + protected $dispatcher; + /** * Optional parameters * @@ -32,10 +39,12 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf * Instanciates a new Mapper * * @param array $options + * @param EventDispatcherInterface $dispatcher */ - public function __construct(array $options = array()) + public function __construct(array $options = array(), EventDispatcherInterface $dispatcher = null) { $this->options = array_merge($this->options, $options); + $this->dispatcher = $dispatcher; } /** @@ -92,6 +101,13 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf $document->set($key, $this->normalizeValue($value)); } + if ($this->dispatcher) { + $event = new TransformEvent($document, $fields, $object); + $this->dispatcher->dispatch(TransformEvent::POST_TRANSFORM, $event); + + $document = $event->getDocument(); + } + return $document; } From 5060fa4d4aaf790bda59180abb8b945535117239 Mon Sep 17 00:00:00 2001 From: Christophe Coevoet Date: Wed, 21 Jan 2015 17:51:56 +0100 Subject: [PATCH 19/41] Move the test file to a better location The test of the DI extension is not a functional test. --- .../FOSElasticaExtensionTest.php | 16 ++++------------ .../fixtures}/config.yml | 0 2 files changed, 4 insertions(+), 12 deletions(-) rename Tests/{Functional => }/DependencyInjection/FOSElasticaExtensionTest.php (79%) rename Tests/{Functional/DependencyInjection/config => DependencyInjection/fixtures}/config.yml (100%) diff --git a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php b/Tests/DependencyInjection/FOSElasticaExtensionTest.php similarity index 79% rename from Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php rename to Tests/DependencyInjection/FOSElasticaExtensionTest.php index 06676ef..feb520f 100644 --- a/Tests/Functional/DependencyInjection/FOSElasticaExtensionTest.php +++ b/Tests/DependencyInjection/FOSElasticaExtensionTest.php @@ -1,8 +1,6 @@ setParameter('kernel.debug', true); @@ -36,5 +29,4 @@ class FOSElasticaExtensionTest extends \PHPUnit_Framework_TestCase $this->assertArrayHasKey('_parent', $arguments); $this->assertEquals('parent_field', $arguments['_parent']['type']); } - -} \ No newline at end of file +} diff --git a/Tests/Functional/DependencyInjection/config/config.yml b/Tests/DependencyInjection/fixtures/config.yml similarity index 100% rename from Tests/Functional/DependencyInjection/config/config.yml rename to Tests/DependencyInjection/fixtures/config.yml From 32d190f554d4a9efa68cadeae3974c77d14b349f Mon Sep 17 00:00:00 2001 From: Christophe Coevoet Date: Wed, 21 Jan 2015 18:10:33 +0100 Subject: [PATCH 20/41] Bump the changelog for 3.0.7 --- CHANGELOG-3.0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index ecb5dad..70bd181 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,8 +12,9 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.7 (Unreleased) +* 3.0.7 (2015-01-21) + * Fixed the indexing of parent/child relations, broken since 3.0 #774 * Fixed multi_field properties not being normalised #769 * 3.0.6 (2015-01-04) From 401446e1c4129b37838461f9aec848e049805b9e Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 09:06:21 +1100 Subject: [PATCH 21/41] Add integration testing around _parent mapping --- Tests/Functional/MappingToElasticaTest.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index f42df61..2c9235a 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -32,11 +32,15 @@ class MappingToElasticaTest extends WebTestCase $this->assertTrue($mapping['type']['properties']['field1']['store']); $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); - $parent = $this->getType($client, 'parent'); - $mapping = $parent->getMapping(); + $type = $this->getType($client, 'type'); + $mapping = $type->getMapping()['type']; + $this->assertEquals('parent', $mapping['_parent']['type']); - $this->assertEquals('my_analyzer', $mapping['parent']['index_analyzer']); - $this->assertEquals('whitespace', $mapping['parent']['search_analyzer']); + $parent = $this->getType($client, 'parent'); + $mapping = $parent->getMapping()['parent']; + + $this->assertEquals('my_analyzer', $mapping['index_analyzer']); + $this->assertEquals('whitespace', $mapping['search_analyzer']); } public function testResetType() From c901d605529eb7a4b5a2cf7552640b082f0b19c5 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 09:10:46 +1100 Subject: [PATCH 22/41] Clarified what index_name does. Closes #731 --- Resources/doc/setup.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Resources/doc/setup.md b/Resources/doc/setup.md index 6a1c2ae..a977ff7 100644 --- a/Resources/doc/setup.md +++ b/Resources/doc/setup.md @@ -48,25 +48,28 @@ fos_elastica: clients: default: { host: localhost, port: 9200 } indexes: - search: ~ + app: ~ ``` In this example, an Elastica index (an instance of `Elastica\Index`) is available as a -service with the key `fos_elastica.index.search`. +service with the key `fos_elastica.index.app`. -If the Elasticsearch index name needs to be different to the service name in your -application, for example, renaming the search index based on different environments. +You may want the index `app` to be named something else on ElasticSearch depending on +if your application is running in a different env or other conditions that suit your +application. To set your customer index to a name that depends on the environment of your +Symfony application, use the example below: ```yaml #app/config/config.yml fos_elastica: indexes: - search: - index_name: search_dev + app: + index_name: app_%kernel.env% ``` -In this case, the service `fos_elastica.index.search` will be using an Elasticsearch -index of search_dev. +In this case, the service `fos_elastica.index.app` will relate to an ElasticSearch index +that varies depending on your kernel's environment. For example, in dev it will relate to +`app_dev`. D) Defining index types ----------------------- @@ -81,7 +84,7 @@ will end up being indexed. ```yaml fos_elastica: indexes: - search: + app: types: user: mappings: @@ -92,7 +95,7 @@ fos_elastica: ``` Each defined type is made available as a service, and in this case the service key is -`fos_elastica.index.search.user` and is an instance of `Elastica\Type`. +`fos_elastica.index.app.user` and is an instance of `Elastica\Type`. FOSElasticaBundle requires a provider for each type that will notify when an object that maps to a type has been modified. The bundle ships with support for Doctrine and From 7471c13d75f2c98b30775acbc4f13422ba25558a Mon Sep 17 00:00:00 2001 From: CedCannes Date: Mon, 19 Jan 2015 23:55:52 +0100 Subject: [PATCH 23/41] Update manual-provider.md Typo in class path --- Resources/doc/cookbook/manual-provider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/cookbook/manual-provider.md b/Resources/doc/cookbook/manual-provider.md index f4365da..ed5568e 100644 --- a/Resources/doc/cookbook/manual-provider.md +++ b/Resources/doc/cookbook/manual-provider.md @@ -8,7 +8,7 @@ index and type for which the service will provide. # app/config/config.yml services: acme.search_provider.user: - class: Acme\UserBundle\Search\UserProvider + class: Acme\UserBundle\Provider\UserProvider arguments: - @fos_elastica.index.website.user tags: From 64fa26e3d9fa38b9fdda1280a72bf5f820046fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michae=CC=88l=20Perrin?= Date: Wed, 17 Dec 2014 12:44:57 +0100 Subject: [PATCH 24/41] Fix PHP notice when using indexes without defined types --- Configuration/Source/ContainerSource.php | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Configuration/Source/ContainerSource.php b/Configuration/Source/ContainerSource.php index 8d094c7..da77009 100644 --- a/Configuration/Source/ContainerSource.php +++ b/Configuration/Source/ContainerSource.php @@ -41,13 +41,16 @@ class ContainerSource implements SourceInterface $indexes = array(); foreach ($this->configArray as $config) { $types = array(); - foreach ($config['types'] as $typeConfig) { - $types[$typeConfig['name']] = new TypeConfig( - $typeConfig['name'], - $typeConfig['mapping'], - $typeConfig['config'] - ); - // TODO: handle prototypes.. + + if (isset($config['types'])) { + foreach ($config['types'] as $typeConfig) { + $types[$typeConfig['name']] = new TypeConfig( + $typeConfig['name'], + $typeConfig['mapping'], + $typeConfig['config'] + ); + // TODO: handle prototypes.. + } } $index = new IndexConfig($config['name'], $types, array( From b9b0c1b961d8cfaac02e0787af0ee7f4cbc5c9a0 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 09:33:46 +1100 Subject: [PATCH 25/41] Move TypeConfig creation to its own method --- Configuration/Source/ContainerSource.php | 38 ++++++++++++++++-------- Tests/Functional/app/Basic/config.yml | 1 + 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Configuration/Source/ContainerSource.php b/Configuration/Source/ContainerSource.php index da77009..abcdf1b 100644 --- a/Configuration/Source/ContainerSource.php +++ b/Configuration/Source/ContainerSource.php @@ -40,19 +40,7 @@ class ContainerSource implements SourceInterface { $indexes = array(); foreach ($this->configArray as $config) { - $types = array(); - - if (isset($config['types'])) { - foreach ($config['types'] as $typeConfig) { - $types[$typeConfig['name']] = new TypeConfig( - $typeConfig['name'], - $typeConfig['mapping'], - $typeConfig['config'] - ); - // TODO: handle prototypes.. - } - } - + $types = $this->getTypes($config); $index = new IndexConfig($config['name'], $types, array( 'elasticSearchName' => $config['elasticsearch_name'], 'settings' => $config['settings'], @@ -64,4 +52,28 @@ class ContainerSource implements SourceInterface return $indexes; } + + /** + * Builds TypeConfig objects for each type. + * + * @param array $config + * @return array + */ + protected function getTypes($config) + { + $types = array(); + + if (isset($config['types'])) { + foreach ($config['types'] as $typeConfig) { + $types[$typeConfig['name']] = new TypeConfig( + $typeConfig['name'], + $typeConfig['mapping'], + $typeConfig['config'] + ); + // TODO: handle prototypes.. + } + } + + return $types; + } } diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index ebd1ea0..3d0e120 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -92,3 +92,4 @@ fos_elastica: identifier: "id" null_mappings: mappings: ~ + empty_index: ~ From 55abe132c658ce6612a72a22fb7f7df46f838b0c Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 09:36:33 +1100 Subject: [PATCH 26/41] Update changelog --- CHANGELOG-3.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 70bd181..d7a826f 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,6 +12,10 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` +* 3.0.8 (Unreleased) + + * Fixed handling of empty indexes #760 + * 3.0.7 (2015-01-21) * Fixed the indexing of parent/child relations, broken since 3.0 #774 From e772ca645017285bc7bfaa29f63b54a60fd1f622 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 11:23:51 +1100 Subject: [PATCH 27/41] Fix php 5.3 compatibility --- Tests/Functional/MappingToElasticaTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index 2c9235a..8fec8a9 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -33,14 +33,14 @@ class MappingToElasticaTest extends WebTestCase $this->assertArrayNotHasKey('store', $mapping['type']['properties']['field2']); $type = $this->getType($client, 'type'); - $mapping = $type->getMapping()['type']; - $this->assertEquals('parent', $mapping['_parent']['type']); + $mapping = $type->getMapping(); + $this->assertEquals('parent', $mapping['type']['_parent']['type']); $parent = $this->getType($client, 'parent'); - $mapping = $parent->getMapping()['parent']; + $mapping = $parent->getMapping(); - $this->assertEquals('my_analyzer', $mapping['index_analyzer']); - $this->assertEquals('whitespace', $mapping['search_analyzer']); + $this->assertEquals('my_analyzer', $mapping['type']['index_analyzer']); + $this->assertEquals('whitespace', $mapping['type']['search_analyzer']); } public function testResetType() From 030b194c7b884a435e98b31bfbc4358a43312a03 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 11:49:28 +1100 Subject: [PATCH 28/41] Fix tests --- Tests/Functional/MappingToElasticaTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/Functional/MappingToElasticaTest.php b/Tests/Functional/MappingToElasticaTest.php index 8fec8a9..11c0db6 100644 --- a/Tests/Functional/MappingToElasticaTest.php +++ b/Tests/Functional/MappingToElasticaTest.php @@ -39,8 +39,8 @@ class MappingToElasticaTest extends WebTestCase $parent = $this->getType($client, 'parent'); $mapping = $parent->getMapping(); - $this->assertEquals('my_analyzer', $mapping['type']['index_analyzer']); - $this->assertEquals('whitespace', $mapping['type']['search_analyzer']); + $this->assertEquals('my_analyzer', $mapping['parent']['index_analyzer']); + $this->assertEquals('whitespace', $mapping['parent']['search_analyzer']); } public function testResetType() From 81f5f983c0f4c9727eefcec36a5301d6619f7564 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sun, 25 Jan 2015 19:06:41 +1100 Subject: [PATCH 29/41] Fix line break --- DependencyInjection/FOSElasticaExtension.php | 1 - 1 file changed, 1 deletion(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 046a5ac..5d30739 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -399,7 +399,6 @@ class FOSElasticaExtension extends Extension $argument['_parent'] = $mapping['_parent']; } $arguments[] = $argument; - } $serviceId = sprintf('fos_elastica.object_persister.%s.%s', $indexName, $typeName); From 906e2e0749cb93ffe00f50416d5f369356e79272 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sat, 1 Nov 2014 19:33:05 +1100 Subject: [PATCH 30/41] Ability to set connectionStrategy for elastica clients --- CHANGELOG-3.0.md | 1 + DependencyInjection/Configuration.php | 11 +++++ .../doc/cookbook/multiple-connections.md | 5 ++ Tests/Functional/ClientTest.php | 48 +++++++++++++++++++ Tests/Functional/app/Basic/config.yml | 5 ++ 5 files changed, 70 insertions(+) create mode 100644 Tests/Functional/ClientTest.php diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index d7a826f..599d1f9 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -15,6 +15,7 @@ To generate a changelog summary since the last version, run * 3.0.8 (Unreleased) * Fixed handling of empty indexes #760 + * Added support for `connectionStrategy` Elastica configuration #732 * 3.0.7 (2015-01-21) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index fa2c048..7ebce1b 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -84,6 +84,16 @@ class Configuration implements ConfigurationInterface return $v; }) ->end() + // Elastica names its properties with camel case, support both + ->beforeNormalization() + ->ifTrue(function ($v) { return isset($v['connection_strategy']); }) + ->then(function ($v) { + $v['connectionStrategy'] = $v['connection_strategy']; + unset($v['connection_strategy']); + + return $v; + }) + ->end() // If there is no connections array key defined, assume a single connection. ->beforeNormalization() ->ifTrue(function ($v) { return is_array($v) && !array_key_exists('connections', $v); }) @@ -124,6 +134,7 @@ class Configuration implements ConfigurationInterface ->end() ->scalarNode('timeout')->end() ->scalarNode('headers')->end() + ->scalarNode('connectionStrategy')->defaultValue('Simple')->end() ->end() ->end() ->end() diff --git a/Resources/doc/cookbook/multiple-connections.md b/Resources/doc/cookbook/multiple-connections.md index 7b5226c..9544359 100644 --- a/Resources/doc/cookbook/multiple-connections.md +++ b/Resources/doc/cookbook/multiple-connections.md @@ -11,6 +11,11 @@ fos_elastica: connections: - url: http://es1.example.net:9200 - url: http://es2.example.net:9200 + connection_strategy: RoundRobin ``` +Elastica allows for definition of different connection strategies and by default +supports `RoundRobin` and `Simple`. You can see definitions for these strategies +in the `Elastica\Connection\Strategy` namespace. + For more information on Elastica clustering see http://elastica.io/getting-started/installation.html#section-connect-cluster diff --git a/Tests/Functional/ClientTest.php b/Tests/Functional/ClientTest.php new file mode 100644 index 0000000..8a6357a --- /dev/null +++ b/Tests/Functional/ClientTest.php @@ -0,0 +1,48 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +use Symfony\Bundle\FrameworkBundle\Client; + +/** + * @group functional + */ +class ClientTest extends WebTestCase +{ + public function testContainerSource() + { + $client = $this->createClient(array('test_case' => 'Basic')); + + $es = $client->getContainer()->get('fos_elastica.client.default'); + $this->assertInstanceOf('Elastica\\Connection\\Strategy\\RoundRobin', $es->getConnectionStrategy()); + + $es = $client->getContainer()->get('fos_elastica.client.second_server'); + $this->assertInstanceOf('Elastica\\Connection\\Strategy\\RoundRobin', $es->getConnectionStrategy()); + + $es = $client->getContainer()->get('fos_elastica.client.third'); + $this->assertInstanceOf('Elastica\\Connection\\Strategy\\Simple', $es->getConnectionStrategy()); + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('Basic'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('Basic'); + } +} diff --git a/Tests/Functional/app/Basic/config.yml b/Tests/Functional/app/Basic/config.yml index 3d0e120..41a869e 100644 --- a/Tests/Functional/app/Basic/config.yml +++ b/Tests/Functional/app/Basic/config.yml @@ -15,7 +15,12 @@ fos_elastica: - url: http://localhost:9200 - host: localhost port: 9200 + connectionStrategy: RoundRobin second_server: + connections: + - url: http://localhost:9200 + connection_strategy: RoundRobin + third: url: http://localhost:9200 indexes: index: From 6bb2def21ea467b3eda1c9542b4716a85f698c88 Mon Sep 17 00:00:00 2001 From: Pablo Martelletti Date: Wed, 7 Jan 2015 14:42:15 -0300 Subject: [PATCH 31/41] Use ProgressBar in populate commande when available. --- Command/PopulateCommand.php | 27 +++++++++++++++++++++------ Doctrine/AbstractProvider.php | 14 +++++++++++++- Propel/Provider.php | 5 ++++- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index f17ca4c..6f27e5b 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -11,6 +11,7 @@ use FOS\ElasticaBundle\IndexManager; use FOS\ElasticaBundle\Provider\ProviderRegistry; use FOS\ElasticaBundle\Resetter; use FOS\ElasticaBundle\Provider\ProviderInterface; +use Symfony\Component\Console\Helper\ProgressBar; /** * Populate the search index @@ -109,20 +110,34 @@ class PopulateCommand extends ContainerAwareCommand */ private function populateIndex(OutputInterface $output, $index, $reset, $options) { + + /** @var $providers ProviderInterface[] */ + $providers = $this->providerRegistry->getIndexProviders($index); + if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index, true); } - /** @var $providers ProviderInterface[] */ - $providers = $this->providerRegistry->getIndexProviders($index); - foreach ($providers as $type => $provider) { - $loggerClosure = function($message) use ($output, $index, $type) { - $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); - }; + if (class_exists('Symfony\Component\Console\Helper\ProgressBar')) { + $output->writeln(sprintf('Populating %s/%s', $index, $type)); + $progressBar = new ProgressBar($output, $provider->getTotalObjects()); + $progressBar->setFormat('debug'); + $progressBar->start(); + $loggerClosure = function($number) use ($progressBar) { + $progressBar->advance($number); + }; + } else { + $loggerClosure = function($message) use ($output, $index, $type) { + $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); + }; + } + $options['progress-bar'] = true; $provider->populate($loggerClosure, $options); + + if (isset($progressBar)) $progressBar->finish(); } $output->writeln(sprintf('Refreshing %s', $index)); diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 92be6ce..9901fef 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -7,6 +7,7 @@ use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Provider\AbstractProvider as BaseAbstractProvider; use FOS\ElasticaBundle\Provider\IndexableInterface; +use Symfony\Component\Console\Helper\ProgressBar; abstract class AbstractProvider extends BaseAbstractProvider { @@ -53,6 +54,7 @@ abstract class AbstractProvider extends BaseAbstractProvider $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors']; + $progressBar = isset($options['progress-bar']) ? boolval($options['progress-bar']) : false; $manager = $this->managerRegistry->getManagerForClass($this->objectClass); for (; $offset < $nbObjects; $offset += $batchSize) { @@ -94,12 +96,14 @@ abstract class AbstractProvider extends BaseAbstractProvider usleep($sleep); - if ($loggerClosure) { + if ($loggerClosure && !$progressBar) { $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $timeDifference = microtime(true) - $stepStartTime; $objectsPerSecond = $timeDifference ? ($stepNbObjects / $timeDifference) : $stepNbObjects; $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); + } else if ($loggerClosure && $progressBar) { + $loggerClosure($stepNbObjects); } } @@ -108,6 +112,14 @@ abstract class AbstractProvider extends BaseAbstractProvider } } + /** + * @return int|mixed + */ + public function getTotalObjects() + { + return $this->countObjects($this->createQueryBuilder()); + } + /** * Counts objects that would be indexed using the query builder. * diff --git a/Propel/Provider.php b/Propel/Provider.php index 38f7a61..f4966da 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -20,6 +20,7 @@ class Provider extends AbstractProvider $nbObjects = $queryClass::create()->count(); $offset = isset($options['offset']) ? intval($options['offset']) : 0; $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; + $progressBar = isset($options['progress-bar']) ? boolval($options['progress-bar']) : false; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; for (; $offset < $nbObjects; $offset += $batchSize) { @@ -46,11 +47,13 @@ class Provider extends AbstractProvider usleep($sleep); - if ($loggerClosure) { + if ($loggerClosure && !$progressBar) { $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); + } else if ($loggerClosure && $progressBar) { + $loggerClosure($stepNbObjects); } } } From 67c0b7950571b02c5627641e6e97c7b1f14332e7 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Thu, 22 Jan 2015 11:18:51 +1100 Subject: [PATCH 32/41] Tidy up ProgressBar use, move most calculations for loggerClosure into PopulateCommand rather than in AbstractProvider --- CHANGELOG-3.1.md | 4 ++ Command/PopulateCommand.php | 127 ++++++++++++++++++++++++++-------- Doctrine/AbstractProvider.php | 55 ++++----------- Propel/Provider.php | 28 ++------ Provider/AbstractProvider.php | 1 + 5 files changed, 120 insertions(+), 95 deletions(-) diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index 12fbb05..fe42514 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -21,3 +21,7 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 `dynamic_date_formats` for types. #753 * New event `POST_TRANSFORM` which allows developers to add custom properties to Elastica Documents for indexing. + * When available, the `fos:elastica:populate` command will now use the + ProgressBar helper instead of outputting strings. You can use verbosity + controls on the command to output additional information like memory + usage, runtime and estimated time. diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 6f27e5b..ac37cab 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -100,6 +100,81 @@ class PopulateCommand extends ContainerAwareCommand } } + /** + * @param ProviderInterface $provider + * @param OutputInterface $output + * @param string $input + * @param string $type + * @param array $options + */ + private function doPopulateType(ProviderInterface $provider, OutputInterface $output, $input, $type, $options) + { + $loggerClosure = $this->getLoggerClosure($output, $input, $type); + + $provider->populate($loggerClosure, $options); + } + + /** + * Builds a loggerClosure to be called from inside the Provider to update the command + * line. + * + * @param OutputInterface $output + * @param string $index + * @param string $type + * @return callable + */ + private function getLoggerClosure(OutputInterface $output, $index, $type) + { + if (!class_exists('Symfony\Component\Console\Helper\ProgressBar')) { + $lastStep = null; + $current = 0; + + return function ($increment, $totalObjects) use ($output, $index, $type, &$lastStep, &$current) { + if ($increment > $totalObjects) { + $increment = $totalObjects; + } + + $currentTime = microtime(true); + $timeDifference = $currentTime - $lastStep; + $objectsPerSecond = $lastStep ? ($increment / $timeDifference) : $increment; + $lastStep = $currentTime; + $current += $increment; + $percent = 100 * $current / $totalObjects; + + $output->writeln(sprintf( + 'Populating %s/%s %0.1f%% (%d/%d), %d objects/s (RAM: current=%uMo peak=%uMo)', + $index, + $type, + $percent, + $current, + $totalObjects, + $objectsPerSecond, + round(memory_get_usage() / (1024 * 1024)), + round(memory_get_peak_usage() / (1024 * 1024)) + )); + }; + } + + ProgressBar::setFormatDefinition('normal', " %current%/%max% [%bar%] %percent:3s%%\n%message%"); + ProgressBar::setFormatDefinition('verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%\n%message%"); + ProgressBar::setFormatDefinition('very_verbose', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s%\n%message%"); + ProgressBar::setFormatDefinition('debug', " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%\n%message%"); + $progress = null; + + return function ($increment, $totalObjects) use (&$progress, $output, $index, $type) { + if (null === $progress) { + $progress = new ProgressBar($output, $totalObjects); + } + + $progress->setMessage(sprintf('Populating %s/%s', $index, $type)); + $progress->advance($increment); + + if ($progress->getProgress() >= $progress->getMaxSteps()) { + $progress->finish(); + } + }; + } + /** * Recreates an index, populates its types, and refreshes the index. * @@ -110,39 +185,19 @@ class PopulateCommand extends ContainerAwareCommand */ private function populateIndex(OutputInterface $output, $index, $reset, $options) { - - /** @var $providers ProviderInterface[] */ - $providers = $this->providerRegistry->getIndexProviders($index); - if ($reset) { $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index, true); } + /** @var $providers ProviderInterface[] */ + $providers = $this->providerRegistry->getIndexProviders($index); + foreach ($providers as $type => $provider) { - if (class_exists('Symfony\Component\Console\Helper\ProgressBar')) { - $output->writeln(sprintf('Populating %s/%s', $index, $type)); - $progressBar = new ProgressBar($output, $provider->getTotalObjects()); - $progressBar->setFormat('debug'); - $progressBar->start(); - $loggerClosure = function($number) use ($progressBar) { - $progressBar->advance($number); - }; - } else { - $loggerClosure = function($message) use ($output, $index, $type) { - $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); - }; - } - - $options['progress-bar'] = true; - $provider->populate($loggerClosure, $options); - - if (isset($progressBar)) $progressBar->finish(); + $this->doPopulateType($provider, $output, $index, $type, $options); } - $output->writeln(sprintf('Refreshing %s', $index)); - $this->resetter->postPopulate($index); - $this->indexManager->getIndex($index)->refresh(); + $this->refreshIndex($output, $index); } /** @@ -161,12 +216,24 @@ class PopulateCommand extends ContainerAwareCommand $this->resetter->resetIndexType($index, $type); } - $loggerClosure = function($message) use ($output, $index, $type) { - $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); - }; - $provider = $this->providerRegistry->getProvider($index, $type); - $provider->populate($loggerClosure, $options); + $this->doPopulateType($provider, $output, $index, $type, $options); + + $this->refreshIndex($output, $index, false); + } + + /** + * Refreshes an index. + * + * @param OutputInterface $output + * @param string $index + * @param bool $postPopulate + */ + private function refreshIndex(OutputInterface $output, $index, $postPopulate = true) + { + if ($postPopulate) { + $this->resetter->postPopulate($index); + } $output->writeln(sprintf('Refreshing %s', $index)); $this->indexManager->getIndex($index)->refresh(); diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 9901fef..80d0716 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -7,7 +7,6 @@ use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Provider\AbstractProvider as BaseAbstractProvider; use FOS\ElasticaBundle\Provider\IndexableInterface; -use Symfony\Component\Console\Helper\ProgressBar; abstract class AbstractProvider extends BaseAbstractProvider { @@ -40,7 +39,7 @@ abstract class AbstractProvider extends BaseAbstractProvider } /** - * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() + * {@inheritDoc} */ public function populate(\Closure $loggerClosure = null, array $options = array()) { @@ -54,38 +53,22 @@ abstract class AbstractProvider extends BaseAbstractProvider $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors']; - $progressBar = isset($options['progress-bar']) ? boolval($options['progress-bar']) : false; $manager = $this->managerRegistry->getManagerForClass($this->objectClass); for (; $offset < $nbObjects; $offset += $batchSize) { - if ($loggerClosure) { - $stepStartTime = microtime(true); - } $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); - if ($loggerClosure) { - $stepNbObjects = count($objects); - } $objects = array_filter($objects, array($this, 'isObjectIndexable')); - if (!$objects) { - if ($loggerClosure) { - $loggerClosure('Entire batch was filtered away, skipping...'); - } - if ($this->options['clear_object_manager']) { - $manager->clear(); - } - - continue; - } - - if (!$ignoreErrors) { - $this->objectPersister->insertMany($objects); - } else { - try { + if ($objects) { + if (!$ignoreErrors) { $this->objectPersister->insertMany($objects); - } catch(BulkResponseException $e) { - if ($loggerClosure) { - $loggerClosure(sprintf('%s',$e->getMessage())); + } else { + try { + $this->objectPersister->insertMany($objects); + } catch(BulkResponseException $e) { + if ($loggerClosure) { + $loggerClosure(sprintf('%s',$e->getMessage())); + } } } } @@ -96,14 +79,8 @@ abstract class AbstractProvider extends BaseAbstractProvider usleep($sleep); - if ($loggerClosure && !$progressBar) { - $stepCount = $stepNbObjects + $offset; - $percentComplete = 100 * $stepCount / $nbObjects; - $timeDifference = microtime(true) - $stepStartTime; - $objectsPerSecond = $timeDifference ? ($stepNbObjects / $timeDifference) : $stepNbObjects; - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); - } else if ($loggerClosure && $progressBar) { - $loggerClosure($stepNbObjects); + if ($loggerClosure) { + $loggerClosure($batchSize, $nbObjects); } } @@ -112,14 +89,6 @@ abstract class AbstractProvider extends BaseAbstractProvider } } - /** - * @return int|mixed - */ - public function getTotalObjects() - { - return $this->countObjects($this->createQueryBuilder()); - } - /** * Counts objects that would be indexed using the query builder. * diff --git a/Propel/Provider.php b/Propel/Provider.php index f4966da..a3af1bd 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -12,7 +12,7 @@ use FOS\ElasticaBundle\Provider\AbstractProvider; class Provider extends AbstractProvider { /** - * @see FOS\ElasticaBundle\Provider\ProviderInterface::populate() + * {@inheritDoc} */ public function populate(\Closure $loggerClosure = null, array $options = array()) { @@ -20,40 +20,24 @@ class Provider extends AbstractProvider $nbObjects = $queryClass::create()->count(); $offset = isset($options['offset']) ? intval($options['offset']) : 0; $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; - $progressBar = isset($options['progress-bar']) ? boolval($options['progress-bar']) : false; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; for (; $offset < $nbObjects; $offset += $batchSize) { - if ($loggerClosure) { - $stepStartTime = microtime(true); - } - $objects = $queryClass::create() ->limit($batchSize) ->offset($offset) ->find() ->getArrayCopy(); - if ($loggerClosure) { - $stepNbObjects = count($objects); - } + $objects = array_filter($objects, array($this, 'isObjectIndexable')); - if (!$objects) { - $loggerClosure('Entire batch was filtered away, skipping...'); - - continue; + if ($objects) { + $this->objectPersister->insertMany($objects); } - $this->objectPersister->insertMany($objects); - usleep($sleep); - if ($loggerClosure && !$progressBar) { - $stepCount = $stepNbObjects + $offset; - $percentComplete = 100 * $stepCount / $nbObjects; - $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); - } else if ($loggerClosure && $progressBar) { - $loggerClosure($stepNbObjects); + if ($loggerClosure) { + $loggerClosure($batchSize, $nbObjects); } } } diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 82ea914..842518d 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -70,6 +70,7 @@ abstract class AbstractProvider implements ProviderInterface /** * Get string with RAM usage information (current and peak) * + * @deprecated To be removed in 4.0 * @return string */ protected function getMemoryUsage() From 58eed2dc7f06a86507477805e23e32b0d029293a Mon Sep 17 00:00:00 2001 From: Allan Brault Date: Mon, 26 Jan 2015 16:59:55 +0100 Subject: [PATCH 33/41] Update PopulateCommand.php in the case i have 110 objects, it was doing : Populating abc/Index 90.9% (100/110) Populating abc/index 181.8% (200/110) now : Populating abc/Index 90.9% (100/110) Populating abc/index 100.0% (110/110) --- Command/PopulateCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index ac37cab..3f24a7f 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -130,8 +130,8 @@ class PopulateCommand extends ContainerAwareCommand $current = 0; return function ($increment, $totalObjects) use ($output, $index, $type, &$lastStep, &$current) { - if ($increment > $totalObjects) { - $increment = $totalObjects; + if ($current + $increment > $totalObjects) { + $increment = $totalObjects - $current; } $currentTime = microtime(true); From 6992beeb471b6d8a9622fc4d3761fbbc5714bada Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sat, 31 Jan 2015 18:24:47 +1100 Subject: [PATCH 34/41] Update Elastica dependency --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index bb30928..b0f4b13 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": ">=0.90.10.0, <1.4-dev", + "ruflin/elastica": ">=0.90.10.0, <1.5-dev", "psr/log": "~1.0" }, "require-dev":{ From 2401b1083c2113a52f9846488f7bca264da23f4f Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Sat, 31 Jan 2015 18:31:57 +1100 Subject: [PATCH 35/41] Bump version --- CHANGELOG-3.0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 599d1f9..973269d 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,10 +12,11 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.8 (Unreleased) +* 3.0.8 (2014-01-31) * Fixed handling of empty indexes #760 * Added support for `connectionStrategy` Elastica configuration #732 + * Allow Elastica 1.4 * 3.0.7 (2015-01-21) From 72589f8341d9dc3ad78f07a45fb2b8a158a3c350 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 6 Feb 2015 22:07:13 +1100 Subject: [PATCH 36/41] Fix ProgressBar for Symfony 2.5 --- Command/PopulateCommand.php | 1 + 1 file changed, 1 insertion(+) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 3f24a7f..5869c9c 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -164,6 +164,7 @@ class PopulateCommand extends ContainerAwareCommand return function ($increment, $totalObjects) use (&$progress, $output, $index, $type) { if (null === $progress) { $progress = new ProgressBar($output, $totalObjects); + $progress->start(); } $progress->setMessage(sprintf('Populating %s/%s', $index, $type)); From 797d06628639cc649515985076b992287142697d Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 6 Feb 2015 23:50:39 +1100 Subject: [PATCH 37/41] Additional change for 2.5 ProgressBar support getProgress() is not available and getStep() throws a deprecation warning. --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 5869c9c..5817c4c 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -170,7 +170,7 @@ class PopulateCommand extends ContainerAwareCommand $progress->setMessage(sprintf('Populating %s/%s', $index, $type)); $progress->advance($increment); - if ($progress->getProgress() >= $progress->getMaxSteps()) { + if ($progress->getProgressPercent() >= 1.0) { $progress->finish(); } }; From c5185a0307dded8f73bf30898f87386a60f0a644 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 9 Feb 2015 09:32:39 +1100 Subject: [PATCH 38/41] Added capability to define property paths --- CHANGELOG-3.1.md | 6 +++ Index/MappingBuilder.php | 2 + Resources/doc/cookbook/custom-properties.md | 2 +- Resources/doc/types.md | 28 ++++++++++ Tests/Functional/PropertyPathTest.php | 54 +++++++++++++++++++ Tests/Functional/TypeObj.php | 2 + Tests/Functional/app/ORM/config.yml | 12 +++++ .../ModelToElasticaAutoTransformerTest.php | 14 +++++ .../ModelToElasticaAutoTransformer.php | 11 +++- 9 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 Tests/Functional/PropertyPathTest.php diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md index fe42514..19bec1f 100644 --- a/CHANGELOG-3.1.md +++ b/CHANGELOG-3.1.md @@ -25,3 +25,9 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0 ProgressBar helper instead of outputting strings. You can use verbosity controls on the command to output additional information like memory usage, runtime and estimated time. + * Added new option `property_path` to a type property definition to allow + customisation of the property path used to retrieve data from objects. + Setting `property_path` to `false` will configure the Transformer to ignore + that property while transforming. Combined with the above POST_TRANSFORM event + developers can now create calculated dynamic properties on Elastica documents + for indexing. #794 diff --git a/Index/MappingBuilder.php b/Index/MappingBuilder.php index 5756751..92beaf7 100644 --- a/Index/MappingBuilder.php +++ b/Index/MappingBuilder.php @@ -110,6 +110,8 @@ class MappingBuilder private function fixProperties(&$properties) { foreach ($properties as $name => &$property) { + unset($property['property_path']); + if (!isset($property['type'])) { $property['type'] = 'string'; } diff --git a/Resources/doc/cookbook/custom-properties.md b/Resources/doc/cookbook/custom-properties.md index cc7751e..1d7687e 100644 --- a/Resources/doc/cookbook/custom-properties.md +++ b/Resources/doc/cookbook/custom-properties.md @@ -1,4 +1,4 @@ -##### Custom Repositories +##### Custom Properties Since FOSElasticaBundle 3.1.0, we now dispatch an event for each transformation of an object into an Elastica document which allows you to set custom properties on the Elastica diff --git a/Resources/doc/types.md b/Resources/doc/types.md index 80d295b..2d575cd 100644 --- a/Resources/doc/types.md +++ b/Resources/doc/types.md @@ -1,6 +1,34 @@ Type configuration ================== +Custom Property Paths +--------------------- + +Since FOSElasticaBundle 3.1.0, it is now possible to define custom property paths +to be used for data retrieval from the underlying model. + +```yaml + user: + mappings: + username: + property_path: indexableUsername + firstName: + property_path: names[first] +``` + +This feature uses the Symfony PropertyAccessor component and supports all features +that the component supports. + +The above example would retrieve an indexed field `username` from the property +`User->indexableUsername`, and the indexed field `firstName` would be populated from a +key `first` from an array on `User->names`. + +Setting the property path to `false` will disable transformation of that value. In this +case the mapping will be created but no value will be populated while indexing. You can +populate this value by listening to the `POST_TRANSFORM` event emitted by this bundle. +See [cookbook/custom-properties.md](cookbook/custom-properties.md) for more information +about this event. + Handling missing results with FOSElasticaBundle ----------------------------------------------- diff --git a/Tests/Functional/PropertyPathTest.php b/Tests/Functional/PropertyPathTest.php new file mode 100644 index 0000000..860cb86 --- /dev/null +++ b/Tests/Functional/PropertyPathTest.php @@ -0,0 +1,54 @@ + + * + * This source file is subject to the MIT license that is bundled + * with this source code in the file LICENSE. + */ + +namespace FOS\ElasticaBundle\Tests\Functional; + +use Elastica\Query\Match; + +/** + * @group functional + */ +class PropertyPathTest extends WebTestCase +{ + public function testContainerSource() + { + $client = $this->createClient(array('test_case' => 'ORM')); + /** @var \FOS\ElasticaBundle\Persister\ObjectPersister $persister */ + $persister = $client->getContainer()->get('fos_elastica.object_persister.index.property_paths_type'); + $obj = new TypeObj(); + $obj->coll = 'Hello'; + $persister->insertOne($obj); + + /** @var \Elastica\Index $elClient */ + $index = $client->getContainer()->get('fos_elastica.index.index'); + $index->flush(true); + + $query = new Match(); + $query->setField('something', 'Hello'); + $search = $index->createSearch($query); + + $this->assertEquals(1, $search->count()); + } + + protected function setUp() + { + parent::setUp(); + + $this->deleteTmpDir('Basic'); + } + + protected function tearDown() + { + parent::tearDown(); + + $this->deleteTmpDir('Basic'); + } +} diff --git a/Tests/Functional/TypeObj.php b/Tests/Functional/TypeObj.php index 46e5968..39e9fe9 100644 --- a/Tests/Functional/TypeObj.php +++ b/Tests/Functional/TypeObj.php @@ -13,8 +13,10 @@ namespace FOS\ElasticaBundle\Tests\Functional; class TypeObj { + public $id = 5; public $coll; public $field1; + public $field2; public function isIndexable() { diff --git a/Tests/Functional/app/ORM/config.yml b/Tests/Functional/app/ORM/config.yml index 98c9221..d2ff931 100644 --- a/Tests/Functional/app/ORM/config.yml +++ b/Tests/Functional/app/ORM/config.yml @@ -65,6 +65,18 @@ fos_elastica: provider: ~ listener: is_indexable_callback: [ 'FOS\ElasticaBundle\Tests\Functional\app\ORM\IndexableService', 'isntIndexable' ] + property_paths_type: + persistence: + driver: orm + model: FOS\ElasticaBundle\Tests\Functional\TypeObj + provider: ~ + properties: + field1: + property_path: field2 + something: + property_path: coll + dynamic: + property_path: false second_index: index_name: foselastica_orm_test_second_%kernel.environment% types: diff --git a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php index bb52129..4852037 100644 --- a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php +++ b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php @@ -148,6 +148,20 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase $transformer->transform(new POPO(), array()); } + public function testPropertyPath() + { + $transformer = $this->getTransformer(); + + $document = $transformer->transform(new POPO(), array('name' => array('property_path' => false))); + $this->assertInstanceOf('Elastica\Document', $document); + $this->assertFalse($document->has('name')); + + $document = $transformer->transform(new POPO(), array('realName' => array('property_path' => 'name'))); + $this->assertInstanceOf('Elastica\Document', $document); + $this->assertTrue($document->has('realName')); + $this->assertEquals('someName', $document->get('realName')); + } + public function testThatCanTransformObject() { $transformer = $this->getTransformer(); diff --git a/Transformer/ModelToElasticaAutoTransformer.php b/Transformer/ModelToElasticaAutoTransformer.php index eff29d2..6a9fbca 100644 --- a/Transformer/ModelToElasticaAutoTransformer.php +++ b/Transformer/ModelToElasticaAutoTransformer.php @@ -75,16 +75,24 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf $property = (null !== $mapping['property'])?$mapping['property']:$mapping['type']; $value = $this->propertyAccessor->getValue($object, $property); $document->setParent($this->propertyAccessor->getValue($value, $mapping['identifier'])); + continue; } - $value = $this->propertyAccessor->getValue($object, $key); + $path = isset($mapping['property_path']) ? + $mapping['property_path'] : + $key; + if (false === $path) { + continue; + } + $value = $this->propertyAccessor->getValue($object, $path); if (isset($mapping['type']) && in_array($mapping['type'], array('nested', 'object')) && isset($mapping['properties']) && !empty($mapping['properties'])) { /* $value is a nested document or object. Transform $value into * an array of documents, respective the mapped properties. */ $document->set($key, $this->transformNested($value, $mapping['properties'])); + continue; } @@ -95,6 +103,7 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf } else { $document->addFileContent($key, $value); } + continue; } From 70fe702ccf0670db1ffc72a9f24550302b4451ca Mon Sep 17 00:00:00 2001 From: Marichez Pierre Date: Sat, 7 Mar 2015 22:06:32 +0100 Subject: [PATCH 39/41] Update usage.md Fix indent in yml --- Resources/doc/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/doc/usage.md b/Resources/doc/usage.md index 37514b3..588532c 100644 --- a/Resources/doc/usage.md +++ b/Resources/doc/usage.md @@ -160,7 +160,7 @@ fos_elastica: site: settings: index: - analysis: + analysis: analyzer: my_analyzer: type: snowball From 72a981ab51cad1d4b85f186c850876a9ff669f82 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 10 Mar 2015 21:49:26 +1100 Subject: [PATCH 40/41] Attempted fix for php 5.3 --- Tests/Transformer/ModelToElasticaAutoTransformerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php index 4852037..1dbf5fd 100644 --- a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php +++ b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php @@ -280,7 +280,7 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase $document = $transformer->transform(new POPO(), array( 'sub' => array( 'type' => 'nested', - 'properties' => array('foo' => '~') + 'properties' => array('foo' => array()) ) )); $data = $document->getData(); From 55bfee22e8bb38c3248c22b2e8f97c69aeeec010 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 10 Mar 2015 21:54:07 +1100 Subject: [PATCH 41/41] Cache composer --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a83f9b9..fbb22d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: php +cache: + directories: + - $HOME/.composer/cache + php: - 5.3 - 5.4 @@ -21,7 +25,8 @@ before_script: - sh -c 'if [ "$SYMFONY_VERSION" != "" ]; then composer require --dev --no-update symfony/symfony=$SYMFONY_VERSION; fi;' - composer install --dev --prefer-source -script: vendor/bin/phpunit --coverage-clover=coverage.clover +script: + - vendor/bin/phpunit --coverage-clover=coverage.clover services: - elasticsearch