diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index ba6affd..53ee7ee 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -91,7 +91,7 @@ class PopulateCommand extends ContainerAwareCommand private function populateIndex(OutputInterface $output, $index, $reset) { if ($reset) { - $output->writeln(sprintf('Resetting: %s', $index)); + $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index); } @@ -99,13 +99,13 @@ class PopulateCommand extends ContainerAwareCommand foreach ($providers as $type => $provider) { $loggerClosure = function($message) use ($output, $index, $type) { - $output->writeln(sprintf('Populating: %s/%s, %s', $index, $type, $message)); + $output->writeln(sprintf('Populating %s/%s, %s', $index, $type, $message)); }; $provider->populate($loggerClosure); } - $output->writeln(sprintf('Refreshing: %s', $index)); + $output->writeln(sprintf('Refreshing %s', $index)); $this->indexManager->getIndex($index)->refresh(); } diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index ebd006c..6aa179d 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -45,9 +45,41 @@ class Configuration ->useAttributeAsKey('id') ->prototype('array') ->performNoDeepMerging() + ->beforeNormalization() + ->ifTrue(function($v) { return isset($v['host']) && isset($v['port']); }) + ->then(function($v) { + return array( + 'servers' => array( + array( + 'host' => $v['host'], + 'port' => $v['port'], + ) + ) + ); + }) + ->end() + ->beforeNormalization() + ->ifTrue(function($v) { return isset($v['url']); }) + ->then(function($v) { + return array( + 'servers' => array( + array( + 'url' => $v['url'], + ) + ) + ); + }) + ->end() ->children() - ->scalarNode('host')->defaultValue('localhost')->end() - ->scalarNode('port')->defaultValue('9000')->end() + ->arrayNode('servers') + ->prototype('array') + ->children() + ->scalarNode('url')->end() + ->scalarNode('host')->end() + ->scalarNode('port')->end() + ->end() + ->end() + ->end() ->scalarNode('timeout')->end() ->scalarNode('headers')->end() ->end() @@ -214,6 +246,7 @@ class Configuration ->end() ->append($this->getMappingsNode()) ->append($this->getSourceNode()) + ->append($this->getBoostNode()) ->end() ; @@ -265,6 +298,33 @@ class Configuration ->end() ->end() ->end() + ->arrayNode('_parent') + ->treatNullLike(array()) + ->children() + ->scalarNode('type')->end() + ->scalarNode('identifier')->defaultValue('id')->end() + ->end() + ->end() + ->arrayNode('properties') + ->useAttributeAsKey('name') + ->prototype('array') + ->treatNullLike(array()) + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('type')->defaultValue('string')->end() + ->scalarNode('boost')->end() + ->scalarNode('store')->end() + ->scalarNode('index')->end() + ->scalarNode('index_analyzer')->end() + ->scalarNode('search_analyzer')->end() + ->scalarNode('analyzer')->end() + ->scalarNode('term_vector')->end() + ->scalarNode('null_value')->end() + ->booleanNode('include_in_all')->defaultValue('true')->end() + ->scalarNode('lat_lon')->end() + ->end() + ->end() + ->end() ->end() ->end() ; @@ -295,4 +355,22 @@ class Configuration return $node; } + + /** + * Returns the array node used for "_boost". + */ + protected function getBoostNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('_boost'); + + $node + ->children() + ->scalarNode('name')->end() + ->scalarNode('null_value')->end() + ->end() + ; + + return $node; + } } diff --git a/DependencyInjection/FOQElasticaExtension.php b/DependencyInjection/FOQElasticaExtension.php index 12658ba..92310cc 100644 --- a/DependencyInjection/FOQElasticaExtension.php +++ b/DependencyInjection/FOQElasticaExtension.php @@ -170,6 +170,9 @@ class FOQElasticaExtension extends Extension if (isset($type['_source'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['_source'] = $type['_source']; } + if (isset($type['_boost'])) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['_boost'] = $type['_boost']; + } if (isset($type['mappings'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['properties'] = $type['mappings']; $typeName = sprintf('%s/%s', $indexName, $name); @@ -184,6 +187,9 @@ class FOQElasticaExtension extends Extension if (isset($type['search_analyzer'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['search_analyzer'] = $type['search_analyzer']; } + if (isset($type['index'])) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['index'] = $type['index']; + } } } @@ -317,7 +323,7 @@ class FOQElasticaExtension extends Extension $listenerDef->replaceArgument(2, $this->getDoctrineEvents($typeConfig)); switch ($typeConfig['driver']) { case 'orm': $listenerDef->addTag('doctrine.event_subscriber'); break; - case 'mongodb': $listenerDef->addTag('doctrine.odm.mongodb.event_subscriber'); break; + case 'mongodb': $listenerDef->addTag('doctrine_mongodb.odm.event_subscriber'); break; } if (isset($typeConfig['listener']['is_indexable_callback'])) { $callback = $typeConfig['listener']['is_indexable_callback']; diff --git a/Doctrine/AbstractElasticaToModelTransformer.php b/Doctrine/AbstractElasticaToModelTransformer.php index 8774106..9a74cfa 100755 --- a/Doctrine/AbstractElasticaToModelTransformer.php +++ b/Doctrine/AbstractElasticaToModelTransformer.php @@ -111,6 +111,14 @@ abstract class AbstractElasticaToModelTransformer implements ElasticaToModelTran return $result; } + /** + * {@inheritdoc} + */ + public function getIdentifierField() + { + return $this->options['identifier']; + } + /** * Fetches objects by theses identifier values * diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 8f51e57..6a3e008 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -40,7 +40,6 @@ abstract class AbstractProvider extends BaseAbstractProvider if ($loggerClosure) { $stepStartTime = microtime(true); } - $objects = $this->fetchSlice($queryBuilder, $this->options['batch_size'], $offset); $this->objectPersister->insertMany($objects); diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index e2f9fb5..77d0527 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -65,6 +65,6 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface */ public function getTotalHits() { - return $this->searchable->count($this->query); + return $this->searchable->search($this->query)->getTotalHits(); } } diff --git a/Propel/ElasticaToModelTransformer.php b/Propel/ElasticaToModelTransformer.php index 16499c6..824b523 100644 --- a/Propel/ElasticaToModelTransformer.php +++ b/Propel/ElasticaToModelTransformer.php @@ -100,6 +100,14 @@ class ElasticaToModelTransformer implements ElasticaToModelTransformerInterface return $this->objectClass; } + /** + * {@inheritdoc} + */ + public function getIdentifierField() + { + return $this->options['identifier']; + } + /** * Fetch objects for theses identifier values * diff --git a/README.md b/README.md index f472e5a..774e7ec 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,41 @@ Elasticsearch type is comparable to Doctrine entity repository. Our type is now available as a service: `foq_elastica.index.website.user`. It is an instance of `Elastica_Type`. +### Declaring parent field + + foq_elastica: + clients: + default: { host: localhost, port: 9200 } + indexes: + website: + client: default + types: + comment: + mappings: + post: {_parent: { type: "post", identifier: "id" } } + date: { boost: 5 } + content: ~ + +### Declaring `nested` or `object` + + foq_elastica: + clients: + default: { host: localhost, port: 9200 } + indexes: + website: + client: default + types: + post: + mappings: + date: { boost: 5 } + title: { boost: 3 } + content: ~ + comments: + type: "nested" + properties: + date: { boost: 5 } + content: ~ + ### Populate the types php app/console foq:elastica:populate diff --git a/Resetter.php b/Resetter.php index f8d94ed..2773140 100644 --- a/Resetter.php +++ b/Resetter.php @@ -58,7 +58,27 @@ class Resetter $type = $indexConfig['index']->getType($typeName); $type->delete(); - $type->setMapping($indexConfig['config']['mappings'][$typeName]['properties']); + $mapping = $this->createMapping($indexConfig['config']['mappings'][$typeName]); + $type->setMapping($mapping); + } + + /** + * create type mapping object + * + * @param array $indexConfig + * @return Elastica_Type_Mapping + */ + protected function createMapping($indexConfig) + { + $mapping = \Elastica_Type_Mapping::create($indexConfig['properties']); + + foreach($indexConfig['properties'] as $field => $type) { + if (!empty($type['_parent']) && $type['_parent'] !== '~') { + $mapping->setParam('_parent', array('type' => $type['_parent']['type'])); + } + } + + return $mapping; } /** diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 95c5406..b6b9c92 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -61,7 +61,6 @@ - diff --git a/Tests/ResetterTest.php b/Tests/ResetterTest.php index d59da18..754310c 100644 --- a/Tests/ResetterTest.php +++ b/Tests/ResetterTest.php @@ -29,6 +29,17 @@ class ResetterTest extends \PHPUnit_Framework_TestCase ), ), ), + 'parent' => array( + 'index' => $this->getMockElasticaIndex(), + 'config' => array( + 'mappings' => array( + 'a' => array('properties' => array( + 'field_1' => array('_parent' => array('type' => 'b', 'identifier' => 'id')), + 'field_2' => array())), + 'b' => array('properties' => array()), + ), + ), + ), ); } @@ -80,9 +91,10 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $type->expects($this->once()) ->method('delete'); + $mapping = \Elastica_Type_Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); $type->expects($this->once()) ->method('setMapping') - ->with($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); + ->with($mapping); $resetter = new Resetter($this->indexConfigsByName); $resetter->resetIndexType('foo', 'a'); @@ -106,6 +118,28 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $resetter->resetIndexType('foo', 'c'); } + public function testIndexMappingForParent() + { + $type = $this->getMockElasticaType(); + + $this->indexConfigsByName['parent']['index']->expects($this->once()) + ->method('getType') + ->with('a') + ->will($this->returnValue($type)); + + $type->expects($this->once()) + ->method('delete'); + + $mapping = \Elastica_Type_Mapping::create($this->indexConfigsByName['parent']['config']['mappings']['a']['properties']); + $mapping->setParam('_parent', array('type' => 'b')); + $type->expects($this->once()) + ->method('setMapping') + ->with($mapping); + + $resetter = new Resetter($this->indexConfigsByName); + $resetter->resetIndexType('parent', 'a'); + } + /** * @return Elastica_Index */ diff --git a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php index e739959..05ddaaa 100644 --- a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php +++ b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php @@ -19,11 +19,19 @@ class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCa ->method('getObjectClass') ->will($this->returnValue('FOQ\ElasticaBundle\Tests\Transformer\POPO')); + $transformer1->expects($this->any()) + ->method('getIdentifierField') + ->will($this->returnValue('id')); + $transformer2 = $this->getMock('FOQ\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface'); $transformer2->expects($this->any()) ->method('getObjectClass') ->will($this->returnValue('FOQ\ElasticaBundle\Tests\Transformer\POPO2')); + $transformer2->expects($this->any()) + ->method('getIdentifierField') + ->will($this->returnValue('id')); + $this->collection = new ElasticaToModelTransformerCollection($this->transformers = array( 'type1' => $transformer1, 'type2' => $transformer2, diff --git a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php index 7f2595c..6f2aaea 100644 --- a/Tests/Transformer/ModelToElasticaAutoTransformerTest.php +++ b/Tests/Transformer/ModelToElasticaAutoTransformerTest.php @@ -92,6 +92,19 @@ class POPO { return $this->file; } + + public function getSub() + { + return array( + (object) array('foo' => 'foo', 'bar' => 'foo', 'id' => 1), + (object) array('foo' => 'bar', 'bar' => 'bar', 'id' => 2), + ); + } + + public function getUpper() + { + return (object) array('id' => 'parent', 'name' => 'a random name'); + } } class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase @@ -215,4 +228,66 @@ class ModelToElasticaAutoTransformerTest extends \PHPUnit_Framework_TestCase base64_encode(file_get_contents(__DIR__ . '/../fixtures/attachment.odt')), $data['fileContents'] ); } + + public function testNestedMapping() + { + $transformer = new ModelToElasticaAutoTransformer(); + $document = $transformer->transform(new POPO(), array( + 'sub' => array( + 'type' => 'nested', + 'properties' => array('foo' => '~') + ) + )); + $data = $document->getData(); + + $this->assertTrue(array_key_exists('sub', $data)); + $this->assertInternalType('array', $data['sub']); + $this->assertEquals(array( + array('foo' => 'foo'), + array('foo' => 'bar') + ), $data['sub']); + } + + public function tesObjectMapping() + { + $transformer = new ModelToElasticaAutoTransformer(); + $document = $transformer->transform(new POPO(), array( + 'sub' => array( + 'type' => 'object', + 'properties' => array('bar') + ) + )); + $data = $document->getData(); + + $this->assertTrue(array_key_exists('sub', $data)); + $this->assertInternalType('array', $data['sub']); + $this->assertEquals(array( + array('bar' => 'foo'), + array('bar' => 'bar') + ), $data['sub']); + } + + public function testParentMapping() + { + $transformer = new ModelToElasticaAutoTransformer(); + $document = $transformer->transform(new POPO(), array( + 'upper' => array( + '_parent' => array('type' => 'upper', 'identifier' => 'id'), + ) + )); + + $this->assertEquals("parent", $document->getParent()); + } + + public function testParentMappingWithCustomIdentifier() + { + $transformer = new ModelToElasticaAutoTransformer(); + $document = $transformer->transform(new POPO(), array( + 'upper' => array( + '_parent' => array('type' => 'upper', 'identifier' => 'name'), + ) + )); + + $this->assertEquals("a random name", $document->getParent()); + } } diff --git a/Transformer/ElasticaToModelTransformerCollection.php b/Transformer/ElasticaToModelTransformerCollection.php index 2418e2e..539f03d 100644 --- a/Transformer/ElasticaToModelTransformerCollection.php +++ b/Transformer/ElasticaToModelTransformerCollection.php @@ -13,14 +13,10 @@ use Symfony\Component\Form\Util\PropertyPath; class ElasticaToModelTransformerCollection implements ElasticaToModelTransformerInterface { protected $transformers = array(); - protected $options = array( - 'identifier' => 'id' - ); - public function __construct(array $transformers, array $options) + public function __construct(array $transformers) { $this->transformers = $transformers; - $this->options = array_merge($this->options, $options); } public function getObjectClass() @@ -30,6 +26,16 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer }, $this->transformers); } + /** + * {@inheritdoc} + */ + public function getIdentifierField() + { + return array_map(function ($transformer) { + return $transformer->getIdentifierField(); + }, $this->transformers); + } + public function transform(array $elasticaObjects) { $sorted = array(); @@ -37,12 +43,19 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer $sorted[$object->getType()][] = $object; } - $identifierProperty = new PropertyPath($this->options['identifier']); - $transformed = array(); foreach ($sorted AS $type => $objects) { $transformedObjects = $this->transformers[$type]->transform($objects); - $transformed[$type] = array_combine(array_map(function($o) use ($identifierProperty) {return $identifierProperty->getValue($o);},$transformedObjects),$transformedObjects); + $identifierGetter = 'get' . ucfirst($this->transformers[$type]->getIdentifierField()); + $transformed[$type] = array_combine( + array_map( + function($o) use ($identifierGetter) { + return $o->$identifierGetter(); + }, + $transformedObjects + ), + $transformedObjects + ); } $result = array(); @@ -71,4 +84,4 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer return $transformer->getObjectClass(); }, $this->transformers); } -} \ No newline at end of file +} diff --git a/Transformer/ElasticaToModelTransformerInterface.php b/Transformer/ElasticaToModelTransformerInterface.php index 5f3f5aa..971bdfe 100644 --- a/Transformer/ElasticaToModelTransformerInterface.php +++ b/Transformer/ElasticaToModelTransformerInterface.php @@ -24,4 +24,11 @@ interface ElasticaToModelTransformerInterface * @return string */ function getObjectClass(); + + /** + * Returns the identifier field from the options + * + * @return string the identifier field + */ + function getIdentifierField(); } diff --git a/Transformer/ModelToElasticaAutoTransformer.php b/Transformer/ModelToElasticaAutoTransformer.php index df72d6f..aa649e2 100644 --- a/Transformer/ModelToElasticaAutoTransformer.php +++ b/Transformer/ModelToElasticaAutoTransformer.php @@ -45,7 +45,15 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf $document = new \Elastica_Document($identifier); foreach ($fields as $key => $mapping) { $property = new PropertyPath($key); - if (isset($mapping['type']) && $mapping['type'] == 'attachment') { + if (!empty($mapping['_parent']) && $mapping['_parent'] !== '~') { + $parent = $property->getValue($object); + $identifierProperty = new PropertyPath($mapping['_parent']['identifier']); + $document->setParent($identifierProperty->getValue($parent)); + } else if (isset($mapping['type']) && in_array($mapping['type'], array('nested', 'object'))) { + $submapping = $mapping['properties']; + $subcollection = $property->getValue($object); + $document->add($key, $this->transformNested($subcollection, $submapping, $document)); + } else if (isset($mapping['type']) && $mapping['type'] == 'attachment') { $attachment = $property->getValue($object); if ($attachment instanceof \SplFileInfo) { $document->addFile($key, $attachment->getPathName()); @@ -59,6 +67,25 @@ class ModelToElasticaAutoTransformer implements ModelToElasticaTransformerInterf return $document; } + /** + * transform a nested document or an object property into an array of ElasticaDocument + * + * @param array $objects the object to convert + * @param array $fields the keys we want to have in the returned array + * @param Elastica_Document $parent the parent document + * @return array + */ + protected function transformNested($objects, array $fields, $parent) + { + $documents = array(); + foreach($objects as $object) { + $document = $this->transform($object, $fields); + $documents[] = $document->getData(); + } + + return $documents; + } + /** * Attempts to convert any type to a string or an array of strings * diff --git a/composer.json b/composer.json index f7efcf4..ad0b1fe 100644 --- a/composer.json +++ b/composer.json @@ -12,9 +12,9 @@ ], "require": { "php": ">=5.3.2", - "symfony/framework-bundle": "2.1.*", - "symfony/console": "2.1.*", - "symfony/form": "2.1.*", + "symfony/framework-bundle": ">=2.1.0,<2.3-dev", + "symfony/console": ">=2.1.0,<2.3-dev", + "symfony/form": ">=2.1.0,<2.3-dev", "ruflin/elastica": "0.19.8" }, "require-dev":{