From f258c9ddc0f1adf3a61fb2e5b3b2a267794a4f6f Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sun, 1 Dec 2013 21:17:55 +0000 Subject: [PATCH 01/67] Exploratory development for https://github.com/FriendsOfSymfony/FOSElasticaBundle/issues/410 --- DependencyInjection/FOSElasticaExtension.php | 4 + Doctrine/AbstractListener.php | 94 ++++++++++++-------- Doctrine/MongoDB/Listener.php | 40 ++------- Doctrine/ORM/Listener.php | 39 ++------ 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 9bf732e..3caaf57 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -405,11 +405,15 @@ class FOSElasticaExtension extends Extension private function getDoctrineEvents(array $typeConfig) { + // Flush event always fires; not configurable + $typeConfig['listener']['flush'] = true; + $events = array(); $eventMapping = array( 'insert' => array('postPersist'), 'update' => array('postUpdate'), 'delete' => array('postRemove', 'preRemove') + 'flush' => array('postFlush') ); foreach ($eventMapping as $event => $doctrineEvents) { diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php index 3b62444..9a4385a 100644 --- a/Doctrine/AbstractListener.php +++ b/Doctrine/AbstractListener.php @@ -2,8 +2,8 @@ namespace FOS\ElasticaBundle\Doctrine; +use Doctrine\Common\EventArgs; use Doctrine\Common\EventSubscriber; -use Doctrine\Common\Persistence\ObjectManager; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Persister\ObjectPersister; use Symfony\Component\ExpressionLanguage\Expression; @@ -48,11 +48,11 @@ abstract class AbstractListener implements EventSubscriber protected $isIndexableCallback; /** - * Objects scheduled for removal - * - * @var array + * Objects scheduled for insertion, replacement, or removal */ - private $scheduledForRemoval = array(); + protected $scheduledForInsertion = array(); + protected $scheduledForUpdate = array(); + protected $scheduledForDeletion = array(); /** * An instance of ExpressionLanguage @@ -149,37 +149,6 @@ abstract class AbstractListener implements EventSubscriber : call_user_func($this->isIndexableCallback, $object); } - /** - * Schedules the object for removal. - * - * This is usually called during the pre-remove event. - * - * @param object $object - * @param ObjectManager $objectManager - */ - protected function scheduleForRemoval($object, ObjectManager $objectManager) - { - $metadata = $objectManager->getClassMetadata($this->objectClass); - $esId = $metadata->getFieldValue($object, $this->esIdentifierField); - $this->scheduledForRemoval[spl_object_hash($object)] = $esId; - } - - /** - * Removes the object if it was scheduled for removal. - * - * This is usually called during the post-remove event. - * - * @param object $object - */ - protected function removeIfScheduled($object) - { - $objectHash = spl_object_hash($object); - if (isset($this->scheduledForRemoval[$objectHash])) { - $this->objectPersister->deleteById($this->scheduledForRemoval[$objectHash]); - unset($this->scheduledForRemoval[$objectHash]); - } - } - /** * @param mixed $object * @return string @@ -207,4 +176,57 @@ abstract class AbstractListener implements EventSubscriber return $this->expressionLanguage; } + + public function postPersist(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { + $this->scheduledForInsertion[] = $entity; + } + } + + public function postUpdate(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + if ($this->isObjectIndexable($entity)) { + $this->scheduledForUpdate[] = $entity; + } else { + // Delete if no longer indexable + $this->scheduledForDeletion[] = $entity; + } + } + } + + public function preRemove(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + $this->scheduledForDeletion[] = $entity; + } + } + + public function postRemove(EventArgs $eventArgs) + { + } + + /** + * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + foreach ($this->scheduledForInsertion as $entity) { + $this->objectPersister->insertOne($entity); + } + foreach ($this->scheduledForUpdate as $entity) { + $this->objectPersister->replaceOne($entity); + } + foreach ($this->scheduledForDeletion as $entity) { + $this->objectPersister->deleteOne($entity); + } + } } diff --git a/Doctrine/MongoDB/Listener.php b/Doctrine/MongoDB/Listener.php index 9fa3536..13e1d25 100644 --- a/Doctrine/MongoDB/Listener.php +++ b/Doctrine/MongoDB/Listener.php @@ -2,49 +2,23 @@ namespace FOS\ElasticaBundle\Doctrine\MongoDB; -use Doctrine\ODM\MongoDB\Event\LifecycleEventArgs; +use Doctrine\Common\EventArgs; use FOS\ElasticaBundle\Doctrine\AbstractListener; class Listener extends AbstractListener { - public function postPersist(LifecycleEventArgs $eventArgs) + public function postPersist(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass && $this->isObjectIndexable($document)) { - $this->objectPersister->insertOne($document); - } + parent::postPersist($eventArgs); } - public function postUpdate(LifecycleEventArgs $eventArgs) + public function postUpdate(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - if ($this->isObjectIndexable($document)) { - $this->objectPersister->replaceOne($document); - } else { - $this->scheduleForRemoval($document, $eventArgs->getDocumentManager()); - $this->removeIfScheduled($document); - } - } + parent::postUpdate($eventArgs); } - public function preRemove(LifecycleEventArgs $eventArgs) + public function postRemove(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - $this->scheduleForRemoval($document, $eventArgs->getDocumentManager()); - } - } - - public function postRemove(LifecycleEventArgs $eventArgs) - { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - $this->removeIfScheduled($document); - } + parent::postRemove($eventArgs); } } diff --git a/Doctrine/ORM/Listener.php b/Doctrine/ORM/Listener.php index 790ddb8..97f8792 100644 --- a/Doctrine/ORM/Listener.php +++ b/Doctrine/ORM/Listener.php @@ -2,49 +2,28 @@ namespace FOS\ElasticaBundle\Doctrine\ORM; -use Doctrine\ORM\Event\LifecycleEventArgs; +use Doctrine\Common\EventArgs; use FOS\ElasticaBundle\Doctrine\AbstractListener; class Listener extends AbstractListener { - public function postPersist(LifecycleEventArgs $eventArgs) + public function postPersist(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { - $this->objectPersister->insertOne($entity); - } + parent::postPersist($eventArgs); } - public function postUpdate(LifecycleEventArgs $eventArgs) + public function postUpdate(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - if ($this->isObjectIndexable($entity)) { - $this->objectPersister->replaceOne($entity); - } else { - $this->scheduleForRemoval($entity, $eventArgs->getEntityManager()); - $this->removeIfScheduled($entity); - } - } + parent::postUpdate($eventArgs); } - public function preRemove(LifecycleEventArgs $eventArgs) + public function preRemove(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->scheduleForRemoval($entity, $eventArgs->getEntityManager()); - } + parent::preRemove($eventArgs); } - public function postRemove(LifecycleEventArgs $eventArgs) + public function postRemove(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->removeIfScheduled($entity); - } + parent::postRemove($eventArgs); } } From 5ec652063de1c381400488f5607870ba490b21ba Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sun, 1 Dec 2013 21:48:41 +0000 Subject: [PATCH 02/67] Don't need postRemove. --- DependencyInjection/FOSElasticaExtension.php | 2 +- Doctrine/AbstractListener.php | 4 ---- Doctrine/ORM/Listener.php | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 3caaf57..30ad07a 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -412,7 +412,7 @@ class FOSElasticaExtension extends Extension $eventMapping = array( 'insert' => array('postPersist'), 'update' => array('postUpdate'), - 'delete' => array('postRemove', 'preRemove') + 'delete' => array('preRemove'), 'flush' => array('postFlush') ); diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php index 9a4385a..0fa15df 100644 --- a/Doctrine/AbstractListener.php +++ b/Doctrine/AbstractListener.php @@ -209,10 +209,6 @@ abstract class AbstractListener implements EventSubscriber } } - public function postRemove(EventArgs $eventArgs) - { - } - /** * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected * only if the query is successful diff --git a/Doctrine/ORM/Listener.php b/Doctrine/ORM/Listener.php index 97f8792..722b89d 100644 --- a/Doctrine/ORM/Listener.php +++ b/Doctrine/ORM/Listener.php @@ -21,9 +21,4 @@ class Listener extends AbstractListener { parent::preRemove($eventArgs); } - - public function postRemove(EventArgs $eventArgs) - { - parent::postRemove($eventArgs); - } } From 1d700261abb7b7f39c56550cac0732e807240b35 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Mon, 2 Dec 2013 12:42:04 +0000 Subject: [PATCH 03/67] Refactor to a single Listener class. Update tests. --- Doctrine/AbstractListener.php | 228 ------------------------ Doctrine/MongoDB/Listener.php | 24 --- Doctrine/ORM/Listener.php | 24 --- Resources/config/mongodb.xml | 2 +- Resources/config/orm.xml | 2 +- Tests/Doctrine/AbstractListenerTest.php | 75 +++++--- Tests/Doctrine/MongoDB/ListenerTest.php | 6 +- Tests/Doctrine/ORM/ListenerTest.php | 6 +- 8 files changed, 57 insertions(+), 310 deletions(-) delete mode 100644 Doctrine/AbstractListener.php delete mode 100644 Doctrine/MongoDB/Listener.php delete mode 100644 Doctrine/ORM/Listener.php diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php deleted file mode 100644 index 0fa15df..0000000 --- a/Doctrine/AbstractListener.php +++ /dev/null @@ -1,228 +0,0 @@ -objectPersister = $objectPersister; - $this->objectClass = $objectClass; - $this->events = $events; - $this->esIdentifierField = $esIdentifierField; - } - - /** - * @see Doctrine\Common\EventSubscriber::getSubscribedEvents() - */ - public function getSubscribedEvents() - { - return $this->events; - } - - /** - * Set the callback for determining object index eligibility. - * - * If callback is a string, it must be public method on the object class - * that expects no arguments and returns a boolean. Otherwise, the callback - * should expect the object for consideration as its only argument and - * return a boolean. - * - * @param callback $callback - * @throws \RuntimeException if the callback is not callable - */ - public function setIsIndexableCallback($callback) - { - if (is_string($callback)) { - if (!is_callable(array($this->objectClass, $callback))) { - if (false !== ($expression = $this->getExpressionLanguage())) { - $callback = new Expression($callback); - try { - $expression->compile($callback, array($this->getExpressionVar())); - } catch (SyntaxError $e) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e); - } - } else { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback)); - } - } - } elseif (!is_callable($callback)) { - if (is_array($callback)) { - list($class, $method) = $callback + array(null, null); - if (is_object($class)) { - $class = get_class($class); - } - - if ($class && $method) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method)); - } - } - throw new \RuntimeException('Indexable callback is not callable.'); - } - - $this->isIndexableCallback = $callback; - } - - /** - * Return whether the object is indexable with respect to the callback. - * - * @param object $object - * @return boolean - */ - protected function isObjectIndexable($object) - { - if (!$this->isIndexableCallback) { - return true; - } - - if ($this->isIndexableCallback instanceof Expression) { - return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object)); - } - - return is_string($this->isIndexableCallback) - ? call_user_func(array($object, $this->isIndexableCallback)) - : call_user_func($this->isIndexableCallback, $object); - } - - /** - * @param mixed $object - * @return string - */ - private function getExpressionVar($object = null) - { - $class = $object ?: $this->objectClass; - $ref = new \ReflectionClass($class); - - return strtolower($ref->getShortName()); - } - - /** - * @return bool|ExpressionLanguage - */ - private function getExpressionLanguage() - { - if (null === $this->expressionLanguage) { - if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { - return false; - } - - $this->expressionLanguage = new ExpressionLanguage(); - } - - return $this->expressionLanguage; - } - - public function postPersist(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { - $this->scheduledForInsertion[] = $entity; - } - } - - public function postUpdate(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - if ($this->isObjectIndexable($entity)) { - $this->scheduledForUpdate[] = $entity; - } else { - // Delete if no longer indexable - $this->scheduledForDeletion[] = $entity; - } - } - } - - public function preRemove(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->scheduledForDeletion[] = $entity; - } - } - - /** - * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected - * only if the query is successful - */ - public function postFlush(EventArgs $eventArgs) - { - foreach ($this->scheduledForInsertion as $entity) { - $this->objectPersister->insertOne($entity); - } - foreach ($this->scheduledForUpdate as $entity) { - $this->objectPersister->replaceOne($entity); - } - foreach ($this->scheduledForDeletion as $entity) { - $this->objectPersister->deleteOne($entity); - } - } -} diff --git a/Doctrine/MongoDB/Listener.php b/Doctrine/MongoDB/Listener.php deleted file mode 100644 index 13e1d25..0000000 --- a/Doctrine/MongoDB/Listener.php +++ /dev/null @@ -1,24 +0,0 @@ - - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 4fd6ae7..5bd16e5 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -13,7 +13,7 @@ - + diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index e99e26d..a6ad2aa 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -3,9 +3,11 @@ namespace FOS\ElasticaBundle\Tests\Doctrine; /** + * See concrete MongoDB/ORM instances of this abstract test + * * @author Richard Miller */ -abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase +abstract class ListenerTest extends \PHPUnit_Framework_TestCase { public function testObjectInsertedOnPersist() { @@ -14,12 +16,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $listener = $this->createListener($persister, get_class($entity), array()); + $listener->postPersist($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForInsertion)); + $persister->expects($this->once()) ->method('insertOne') ->with($entity); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->postPersist($eventArgs); + $listener->postFlush($eventArgs); } /** @@ -32,12 +38,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1, false); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); - $persister->expects($this->never()) - ->method('insertOne'); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->setIsIndexableCallback($isIndexableCallback); $listener->postPersist($eventArgs); + + $this->assertEmpty($listener->scheduledForInsertion); + + $persister->expects($this->never()) + ->method('insertOne'); + + $listener->postFlush($eventArgs); } public function testObjectReplacedOnUpdate() @@ -47,15 +57,18 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $listener = $this->createListener($persister, get_class($entity), array()); + $listener->postUpdate($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForUpdate)); + $persister->expects($this->once()) ->method('replaceOne') ->with($entity); - $persister->expects($this->never()) ->method('deleteById'); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->postUpdate($eventArgs); + $listener->postFlush($eventArgs); } /** @@ -80,16 +93,20 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $persister->expects($this->never()) - ->method('replaceOne'); - - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->setIsIndexableCallback($isIndexableCallback); $listener->postUpdate($eventArgs); + + $this->assertEmpty($listener->scheduledForUpdate); + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->never()) + ->method('replaceOne'); + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } public function testObjectDeletedOnRemove() @@ -111,13 +128,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->preRemove($eventArgs); - $listener->postRemove($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } public function testObjectWithNonStandardIdentifierDeletedOnRemove() @@ -139,13 +159,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'identifier') ->will($this->returnValue($entity->getId())); - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array(), 'identifier'); $listener->preRemove($eventArgs); - $listener->postRemove($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } /** diff --git a/Tests/Doctrine/MongoDB/ListenerTest.php b/Tests/Doctrine/MongoDB/ListenerTest.php index 7f1a9ab..37a0203 100644 --- a/Tests/Doctrine/MongoDB/ListenerTest.php +++ b/Tests/Doctrine/MongoDB/ListenerTest.php @@ -2,9 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Doctrine\MongoDB; -use FOS\ElasticaBundle\Tests\Doctrine\AbstractListenerTest; +use FOS\ElasticaBundle\Tests\Doctrine\ListenerTest as BaseListenerTest; -class ListenerTest extends AbstractListenerTest +class ListenerTest extends BaseListenerTest { public function setUp() { @@ -25,7 +25,7 @@ class ListenerTest extends AbstractListenerTest protected function getListenerClass() { - return 'FOS\ElasticaBundle\Doctrine\MongoDB\Listener'; + return 'FOS\ElasticaBundle\Doctrine\Listener'; } protected function getObjectManagerClass() diff --git a/Tests/Doctrine/ORM/ListenerTest.php b/Tests/Doctrine/ORM/ListenerTest.php index 48702c0..12a89b2 100644 --- a/Tests/Doctrine/ORM/ListenerTest.php +++ b/Tests/Doctrine/ORM/ListenerTest.php @@ -2,9 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Doctrine\ORM; -use FOS\ElasticaBundle\Tests\Doctrine\AbstractListenerTest; +use FOS\ElasticaBundle\Tests\Doctrine\ListenerTest as BaseListenerTest; -class ListenerTest extends AbstractListenerTest +class ListenerTest extends BaseListenerTest { public function setUp() { @@ -25,7 +25,7 @@ class ListenerTest extends AbstractListenerTest protected function getListenerClass() { - return 'FOS\ElasticaBundle\Doctrine\ORM\Listener'; + return 'FOS\ElasticaBundle\Doctrine\Listener'; } protected function getObjectManagerClass() From af2827df016feea3463ba911454f12ab3b1e6b00 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 14:01:21 +0000 Subject: [PATCH 04/67] Re-add renamed Listener. --- Doctrine/Listener.php | 228 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 Doctrine/Listener.php diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php new file mode 100644 index 0000000..6584207 --- /dev/null +++ b/Doctrine/Listener.php @@ -0,0 +1,228 @@ +objectPersister = $objectPersister; + $this->objectClass = $objectClass; + $this->events = $events; + $this->esIdentifierField = $esIdentifierField; + } + + /** + * @see Doctrine\Common\EventSubscriber::getSubscribedEvents() + */ + public function getSubscribedEvents() + { + return $this->events; + } + + /** + * Set the callback for determining object index eligibility. + * + * If callback is a string, it must be public method on the object class + * that expects no arguments and returns a boolean. Otherwise, the callback + * should expect the object for consideration as its only argument and + * return a boolean. + * + * @param callback $callback + * @throws \RuntimeException if the callback is not callable + */ + public function setIsIndexableCallback($callback) + { + if (is_string($callback)) { + if (!is_callable(array($this->objectClass, $callback))) { + if (false !== ($expression = $this->getExpressionLanguage())) { + $callback = new Expression($callback); + try { + $expression->compile($callback, array($this->getExpressionVar())); + } catch (SyntaxError $e) { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e); + } + } else { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback)); + } + } + } elseif (!is_callable($callback)) { + if (is_array($callback)) { + list($class, $method) = $callback + array(null, null); + if (is_object($class)) { + $class = get_class($class); + } + + if ($class && $method) { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method)); + } + } + throw new \RuntimeException('Indexable callback is not callable.'); + } + + $this->isIndexableCallback = $callback; + } + + /** + * Return whether the object is indexable with respect to the callback. + * + * @param object $object + * @return boolean + */ + protected function isObjectIndexable($object) + { + if (!$this->isIndexableCallback) { + return true; + } + + if ($this->isIndexableCallback instanceof Expression) { + return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object)); + } + + return is_string($this->isIndexableCallback) + ? call_user_func(array($object, $this->isIndexableCallback)) + : call_user_func($this->isIndexableCallback, $object); + } + + /** + * @param mixed $object + * @return string + */ + private function getExpressionVar($object = null) + { + $class = $object ?: $this->objectClass; + $ref = new \ReflectionClass($class); + + return strtolower($ref->getShortName()); + } + + /** + * @return bool|ExpressionLanguage + */ + private function getExpressionLanguage() + { + if (null === $this->expressionLanguage) { + if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { + return false; + } + + $this->expressionLanguage = new ExpressionLanguage(); + } + + return $this->expressionLanguage; + } + + public function postPersist(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { + $this->scheduledForInsertion[] = $entity; + } + } + + public function postUpdate(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + if ($this->isObjectIndexable($entity)) { + $this->scheduledForUpdate[] = $entity; + } else { + // Delete if no longer indexable + $this->scheduledForDeletion[] = $entity; + } + } + } + + public function preRemove(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + $this->scheduledForDeletion[] = $entity; + } + } + + /** + * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + foreach ($this->scheduledForInsertion as $entity) { + $this->objectPersister->insertOne($entity); + } + foreach ($this->scheduledForUpdate as $entity) { + $this->objectPersister->replaceOne($entity); + } + foreach ($this->scheduledForDeletion as $entity) { + $this->objectPersister->deleteOne($entity); + } + } +} From 22a5d67d05d2df4fe46ca6e80f4b21198f2d3b19 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 20:41:26 +0000 Subject: [PATCH 05/67] pre/postFlush configuration. Update documentation. --- DependencyInjection/Configuration.php | 2 ++ DependencyInjection/FOSElasticaExtension.php | 6 +++-- Doctrine/Listener.php | 23 +++++++++++++++++--- README.md | 22 ++++++++++++++++--- UPGRADE-3.0.md | 11 ++++++++++ 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 UPGRADE-3.0.md diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index fe7e9a4..50d4b89 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -165,6 +165,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->scalarNode('persist')->defaultValue('postFlush')->end() ->scalarNode('service')->end() ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() @@ -257,6 +258,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('service')->end() ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 30ad07a..4a7ebbc 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -405,7 +405,7 @@ class FOSElasticaExtension extends Extension private function getDoctrineEvents(array $typeConfig) { - // Flush event always fires; not configurable + // Flush always calls depending on actions scheduled in lifecycle listeners $typeConfig['listener']['flush'] = true; $events = array(); @@ -413,9 +413,11 @@ class FOSElasticaExtension extends Extension 'insert' => array('postPersist'), 'update' => array('postUpdate'), 'delete' => array('preRemove'), - 'flush' => array('postFlush') + 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') ); + var_dump($eventMapping); + foreach ($eventMapping as $event => $doctrineEvents) { if (isset($typeConfig['listener'][$event]) && $typeConfig['listener'][$event]) { $events = array_merge($events, $doctrineEvents); diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 6584207..683070e 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -210,10 +210,9 @@ class Listener implements EventSubscriber } /** - * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected - * only if the query is successful + * Persist scheduled action to ElasticSearch */ - public function postFlush(EventArgs $eventArgs) + private function persistScheduled() { foreach ($this->scheduledForInsertion as $entity) { $this->objectPersister->insertOne($entity); @@ -225,4 +224,22 @@ class Listener implements EventSubscriber $this->objectPersister->deleteOne($entity); } } + + /** + * Iterate through scheduled actions before flushing to emulate 2.x behavior. Note that the ElasticSearch index + * will fall out of sync with the data source in event of a crash on flush. + */ + public function preFlush(EventArgs $eventArgs) + { + $this->persistScheduled(); + } + + /** + * Iterating through scheduled actions *after* flushing ensures that the ElasticSearch index will be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + $this->persistScheduled(); + } } diff --git a/README.md b/README.md index 9792139..9f77a1b 100644 --- a/README.md +++ b/README.md @@ -576,7 +576,11 @@ class User ### Realtime, selective index update If you use the Doctrine integration, you can let ElasticaBundle update the indexes automatically -when an object is added, updated or removed. It uses Doctrine lifecycle events. +when an object is added, updated or removed. It uses Doctrine lifecycle events to schedule updates +and then synchronizes changes either before or after flush. + +> **Propel** doesn't support this feature yet. + Declare that you want to update the index in real time: fos_elastica: @@ -592,7 +596,7 @@ Declare that you want to update the index in real time: persistence: driver: orm model: Application\UserBundle\Entity\User - listener: ~ # by default, listens to "insert", "update" and "delete" + listener: ~ # by default, listens to "insert", "update" and "delete" and updates `postFlush` Now the index is automatically updated each time the state of the bound Doctrine repository changes. No need to repopulate the whole "user" index when a new `User` is created. @@ -605,7 +609,19 @@ You can also choose to only listen for some of the events: update: false delete: true -> **Propel** doesn't support this feature yet. +By default, the ElasticSearch index will be updated after flush. To update before flushing, set `immediate` +to `true`: + + persistence: + listener: + insert: true + update: false + delete: true + immediate: true + +> Updating ElasticSearch before flushing may cause the ElasticSearch index to fall out of sync with the +> original data in the event of a crash. + ### Checking an entity method for listener diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md new file mode 100644 index 0000000..0b0d370 --- /dev/null +++ b/UPGRADE-3.0.md @@ -0,0 +1,11 @@ +UPGRADE FROM 2.x to 3.0 +======================= + +### ElasticSearch Synchronization Event + + * Prior to 3.0, the ElasticSearch index was synchronized in the `postInsert`, + `postUpdate`, and `pre/postRemove` events which fire before flush. Because + of this, exceptions thrown when flushing would cause the data source and + ElasticSearch index to fall out of sync. + + As of 3.0, ElasticSearch is updated `postFlush` by default. From ed21e60869e86a5372f3d1a69ee97c4f61c65829 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 23:40:00 +0000 Subject: [PATCH 06/67] Remove upgrade doc. Merk will add manually. --- UPGRADE-3.0.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 UPGRADE-3.0.md diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md deleted file mode 100644 index 0b0d370..0000000 --- a/UPGRADE-3.0.md +++ /dev/null @@ -1,11 +0,0 @@ -UPGRADE FROM 2.x to 3.0 -======================= - -### ElasticSearch Synchronization Event - - * Prior to 3.0, the ElasticSearch index was synchronized in the `postInsert`, - `postUpdate`, and `pre/postRemove` events which fire before flush. Because - of this, exceptions thrown when flushing would cause the data source and - ElasticSearch index to fall out of sync. - - As of 3.0, ElasticSearch is updated `postFlush` by default. From 3a279f8edbff1eed66c224516904917725ccfc6b Mon Sep 17 00:00:00 2001 From: nurikabe Date: Wed, 4 Dec 2013 10:46:05 +0000 Subject: [PATCH 07/67] Remove debug. --- DependencyInjection/FOSElasticaExtension.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 4a7ebbc..6380873 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -416,8 +416,6 @@ class FOSElasticaExtension extends Extension 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') ); - var_dump($eventMapping); - foreach ($eventMapping as $event => $doctrineEvents) { if (isset($typeConfig['listener'][$event]) && $typeConfig['listener'][$event]) { $events = array_merge($events, $doctrineEvents); From 090f02b05d827b0f242d70434f17b46138564f6b Mon Sep 17 00:00:00 2001 From: Craig Marvelley Date: Wed, 11 Dec 2013 22:52:25 +0000 Subject: [PATCH 08/67] Improve test coverage --- Logger/ElasticaLogger.php | 2 +- Tests/ClientTest.php | 42 ++++++++++++++ Tests/FOSElasticaBundleTest.php | 39 +++++++++++++ Tests/HybridResultTest.php | 19 ++++++ Tests/Logger/ElasticaLoggerTest.php | 90 +++++++++++++++++++++++++++-- Tests/RepositoryTest.php | 16 +++++ 6 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 Tests/ClientTest.php create mode 100644 Tests/FOSElasticaBundleTest.php create mode 100644 Tests/HybridResultTest.php diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index bf7694e..7aacac5 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -147,6 +147,6 @@ class ElasticaLogger implements LoggerInterface */ public function log($level, $message, array $context = array()) { - return $this->logger->log($message, $context); + return $this->logger->log($level, $message, $context); } } diff --git a/Tests/ClientTest.php b/Tests/ClientTest.php new file mode 100644 index 0000000..c8509cf --- /dev/null +++ b/Tests/ClientTest.php @@ -0,0 +1,42 @@ +getMock('Elastica\Connection'); + $connection->expects($this->any())->method('getTransportObject')->will($this->returnValue($transport)); + $connection->expects($this->any())->method('toArray')->will($this->returnValue(array())); + + $logger = $this->getMock('FOS\ElasticaBundle\Logger\ElasticaLogger'); + $logger + ->expects($this->once()) + ->method('logQuery') + ->with( + 'foo', + Request::GET, + $this->isType('array'), + $this->isType('float'), + $this->isType('array') + ); + + $client = $this->getMockBuilder('FOS\ElasticaBundle\Client') + ->setMethods(array('getConnection')) + ->getMock(); + + $client->expects($this->any())->method('getConnection')->will($this->returnValue($connection)); + + $client->setLogger($logger); + + $response = $client->request('foo'); + + $this->assertInstanceOf('Elastica\Response', $response); + } +} diff --git a/Tests/FOSElasticaBundleTest.php b/Tests/FOSElasticaBundleTest.php new file mode 100644 index 0000000..2bfc7f9 --- /dev/null +++ b/Tests/FOSElasticaBundleTest.php @@ -0,0 +1,39 @@ +getMock('Symfony\Component\DependencyInjection\ContainerBuilder'); + + $container + ->expects($this->at(0)) + ->method('addCompilerPass') + ->with($this->isInstanceOf($passes[0][0]), $passes[0][1]); + + $container + ->expects($this->at(1)) + ->method('addCompilerPass') + ->with($this->isInstanceOf($passes[1][0])); + + $bundle = new FOSElasticaBundle(); + + $bundle->build($container); + } +} diff --git a/Tests/HybridResultTest.php b/Tests/HybridResultTest.php new file mode 100644 index 0000000..cb382d1 --- /dev/null +++ b/Tests/HybridResultTest.php @@ -0,0 +1,19 @@ +assertSame($result, $hybridResult->getResult()); + $this->assertNull($hybridResult->getTransformed()); + } +} diff --git a/Tests/Logger/ElasticaLoggerTest.php b/Tests/Logger/ElasticaLoggerTest.php index 05624cc..30ce30c 100644 --- a/Tests/Logger/ElasticaLoggerTest.php +++ b/Tests/Logger/ElasticaLoggerTest.php @@ -9,6 +9,40 @@ use FOS\ElasticaBundle\Logger\ElasticaLogger; */ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase { + /** + * @return \PHPUnit_Framework_MockObject_MockObject|\Symfony\Component\HttpKernel\Log\LoggerInterface + */ + private function getMockLogger() + { + return $this->getMockBuilder('Symfony\Component\HttpKernel\Log\LoggerInterface') + ->disableOriginalConstructor() + ->getMock(); + } + + /** + * @param string $level + * @param string $message + * @param array $context + * @return ElasticaLogger + */ + private function getMockLoggerForLevelMessageAndContext($level, $message, $context) + { + $loggerMock = $this->getMockBuilder('Symfony\Component\HttpKernel\Log\LoggerInterface') + ->disableOriginalConstructor() + ->getMock(); + + $loggerMock->expects($this->once()) + ->method($level) + ->with( + $this->equalTo($message), + $this->equalTo($context) + ); + + $elasticaLogger = new ElasticaLogger($loggerMock); + + return $elasticaLogger; + } + public function testGetZeroIfNoQueriesAdded() { $elasticaLogger = new ElasticaLogger; @@ -64,10 +98,7 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase public function testQueryIsLogged() { - /** @var $loggerMock \PHPUnit_Framework_MockObject_MockObject|\Symfony\Component\HttpKernel\Log\LoggerInterface */ - $loggerMock = $this->getMockBuilder('Symfony\Component\HttpKernel\Log\LoggerInterface') - ->disableOriginalConstructor() - ->getMock(); + $loggerMock = $this->getMockLogger(); $elasticaLogger = new ElasticaLogger($loggerMock); @@ -87,4 +118,55 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase $elasticaLogger->logQuery($path, $method, $data, $time); } + + /** + * @return array + */ + public function logLevels() + { + return array( + array('emergency'), + array('alert'), + array('critical'), + array('error'), + array('warning'), + array('notice'), + array('info'), + array('debug'), + ); + } + + /** + * @dataProvider logLevels + */ + public function testMessagesCanBeLoggedAtSpecificLogLevels($level) + { + $message = 'foo'; + $context = array('data'); + + $loggerMock = $this->getMockLoggerForLevelMessageAndContext($level, $message, $context); + + call_user_func(array($loggerMock, $level), $message, $context); + } + + public function testMessagesCanBeLoggedToArbitraryLevels() + { + $loggerMock = $this->getMockLogger(); + + $level = 'info'; + $message = 'foo'; + $context = array('data'); + + $loggerMock->expects($this->once()) + ->method('log') + ->with( + $level, + $message, + $context + ); + + $elasticaLogger = new ElasticaLogger($loggerMock); + + $elasticaLogger->log($level, $message, $context); + } } diff --git a/Tests/RepositoryTest.php b/Tests/RepositoryTest.php index 6eabe7b..c4d4efc 100644 --- a/Tests/RepositoryTest.php +++ b/Tests/RepositoryTest.php @@ -58,6 +58,22 @@ class RepositoryTest extends \PHPUnit_Framework_TestCase $repository->findPaginated($testQuery); } + public function testThatCreatePaginatorCreatesAPaginatorViaFinder() + { + $testQuery = 'Test Query'; + + /** @var $typeMock \PHPUnit_Framework_MockObject_MockObject|\FOS\ElasticaBundle\Finder\TransformedFinder */ + $finderMock = $this->getMockBuilder('FOS\ElasticaBundle\Finder\TransformedFinder') + ->disableOriginalConstructor() + ->getMock(); + $finderMock->expects($this->once()) + ->method('createPaginatorAdapter') + ->with($this->equalTo($testQuery)); + + $repository = new Repository($finderMock); + $repository->createPaginatorAdapter($testQuery); + } + public function testThatFindHybridCallsFindHybridOnFinder() { $testQuery = 'Test Query'; From 11a87c5ce3fbe1b7a4e240b494d2be12f4973000 Mon Sep 17 00:00:00 2001 From: Craig Marvelley Date: Thu, 12 Dec 2013 20:48:15 +0000 Subject: [PATCH 09/67] Add test for ElasticaToModelTransformerCollection::hybridTransform --- ...asticaToModelTransformerCollectionTest.php | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php index d33708e..bb5f6a3 100644 --- a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php +++ b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Tests\Transformer; use Elastica\Document; +use Elastica\Result; use FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerCollection; class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCase @@ -98,6 +99,57 @@ class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCa $result2, ), $results); } + + public function testGetIdentifierFieldReturnsAMapOfIdentifiers() + { + $collection = new ElasticaToModelTransformerCollection(array()); + $identifiers = $collection->getIdentifierField(); + $this->assertInternalType('array', $identifiers); + $this->assertEmpty($identifiers); + + $this->collectionSetup(); + $identifiers = $this->collection->getIdentifierField(); + $this->assertInternalType('array', $identifiers); + $this->assertEquals(array('type1' => 'id', 'type2' => 'id'), $identifiers); + } + + public function elasticaResults() + { + $document = new Result(array('_id' => 123, '_type' => 'type1')); + $result = new POPO(123, array()); + + return array( + array( + $document, $result + ) + ); + } + + /** + * @dataProvider elasticaResults + */ + public function testHybridTransformDecoratesResultsWithHybridResultObjects($document, $result) + { + $transformer = $this->getMock('FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface'); + $transformer->expects($this->any())->method('getIdentifierField')->will($this->returnValue('id')); + + $transformer + ->expects($this->any()) + ->method('transform') + ->will($this->returnValue(array($result))); + + $collection = new ElasticaToModelTransformerCollection(array('type1' => $transformer)); + + $hybridResults = $collection->hybridTransform(array($document)); + + $this->assertInternalType('array', $hybridResults); + $this->assertNotEmpty($hybridResults); + $this->assertContainsOnlyInstancesOf('FOS\ElasticaBundle\HybridResult', $hybridResults); + + $hybridResult = array_pop($hybridResults); + $this->assertEquals($document, $hybridResult->getResult()); + $this->assertEquals($result, $hybridResult->getTransformed()); + } } class POPO From 5d16ffb1bf31c01d0748e2f6b6950af3989d2b2c Mon Sep 17 00:00:00 2001 From: Craig Marvelley Date: Thu, 12 Dec 2013 20:50:15 +0000 Subject: [PATCH 10/67] Remove unused method --- Transformer/ElasticaToModelTransformerCollection.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Transformer/ElasticaToModelTransformerCollection.php b/Transformer/ElasticaToModelTransformerCollection.php index 6264959..a261e81 100644 --- a/Transformer/ElasticaToModelTransformerCollection.php +++ b/Transformer/ElasticaToModelTransformerCollection.php @@ -84,11 +84,4 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer return $result; } - - protected function getTypeToClassMap() - { - return array_map(function (ElasticaToModelTransformerInterface $transformer) { - return $transformer->getObjectClass(); - }, $this->transformers); - } } From e0ef8dff23dd7d100905bb8a8a28ee07ef89a660 Mon Sep 17 00:00:00 2001 From: Burkhard Reffeling Date: Thu, 12 Dec 2013 22:41:20 +0000 Subject: [PATCH 11/67] removed duplicate methods --- Persister/ObjectSerializerPersister.php | 63 ------------------------- 1 file changed, 63 deletions(-) diff --git a/Persister/ObjectSerializerPersister.php b/Persister/ObjectSerializerPersister.php index 8938a2f..1a15656 100644 --- a/Persister/ObjectSerializerPersister.php +++ b/Persister/ObjectSerializerPersister.php @@ -23,69 +23,6 @@ class ObjectSerializerPersister extends ObjectPersister $this->serializer = $serializer; } - /** - * Insert one object into the type - * The object will be transformed to an elastica document - * - * @param object $object - */ - public function insertOne($object) - { - $document = $this->transformToElasticaDocument($object); - $this->type->addDocument($document); - } - - /** - * Replaces one object in the type - * - * @param object $object - * @return null - */ - public function replaceOne($object) - { - $document = $this->transformToElasticaDocument($object); - $this->type->deleteById($document->getId()); - $this->type->addDocument($document); - } - - /** - * Deletes one object in the type - * - * @param object $object - * @return null - **/ - public function deleteOne($object) - { - $document = $this->transformToElasticaDocument($object); - $this->type->deleteById($document->getId()); - } - - /** - * Deletes one object in the type by id - * - * @param mixed $id - * - * @return null - **/ - public function deleteById($id) - { - $this->type->deleteById($id); - } - - /** - * Inserts an array of objects in the type - * - * @param array $objects array of domain model objects - **/ - public function insertMany(array $objects) - { - $docs = array(); - foreach ($objects as $object) { - $docs[] = $this->transformToElasticaDocument($object); - } - $this->type->addDocuments($docs); - } - /** * Transforms an object to an elastica document * with just the identifier set From 292af9f039e2c52b4d02fb9c0f2f916b068f67f9 Mon Sep 17 00:00:00 2001 From: Craig Marvelley Date: Thu, 12 Dec 2013 22:55:37 +0000 Subject: [PATCH 12/67] Better variable names --- .../ElasticaToModelTransformerCollectionTest.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php index bb5f6a3..eb4d8e4 100644 --- a/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php +++ b/Tests/Transformer/ElasticaToModelTransformerCollectionTest.php @@ -115,12 +115,12 @@ class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCa public function elasticaResults() { - $document = new Result(array('_id' => 123, '_type' => 'type1')); - $result = new POPO(123, array()); + $result = new Result(array('_id' => 123, '_type' => 'type1')); + $transformedObject = new POPO(123, array()); return array( array( - $document, $result + $result, $transformedObject ) ); } @@ -128,7 +128,7 @@ class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCa /** * @dataProvider elasticaResults */ - public function testHybridTransformDecoratesResultsWithHybridResultObjects($document, $result) + public function testHybridTransformDecoratesResultsWithHybridResultObjects($result, $transformedObject) { $transformer = $this->getMock('FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerInterface'); $transformer->expects($this->any())->method('getIdentifierField')->will($this->returnValue('id')); @@ -136,19 +136,19 @@ class ElasticaToModelTransformerCollectionTest extends \PHPUnit_Framework_TestCa $transformer ->expects($this->any()) ->method('transform') - ->will($this->returnValue(array($result))); + ->will($this->returnValue(array($transformedObject))); $collection = new ElasticaToModelTransformerCollection(array('type1' => $transformer)); - $hybridResults = $collection->hybridTransform(array($document)); + $hybridResults = $collection->hybridTransform(array($result)); $this->assertInternalType('array', $hybridResults); $this->assertNotEmpty($hybridResults); $this->assertContainsOnlyInstancesOf('FOS\ElasticaBundle\HybridResult', $hybridResults); $hybridResult = array_pop($hybridResults); - $this->assertEquals($document, $hybridResult->getResult()); - $this->assertEquals($result, $hybridResult->getTransformed()); + $this->assertEquals($result, $hybridResult->getResult()); + $this->assertEquals($transformedObject, $hybridResult->getTransformed()); } } From 3bd9155f467b6d46896d88d29050ddc593c53324 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Fri, 13 Dec 2013 17:51:15 +0000 Subject: [PATCH 13/67] Use constants of corresponding events classes rather than making assumption about string values. --- DependencyInjection/FOSElasticaExtension.php | 23 ++++++++++++++++---- Doctrine/Listener.php | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 6380873..733a039 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -403,17 +403,32 @@ class FOSElasticaExtension extends Extension return $listenerId; } + /** + * Map Elastica to Doctrine events for the current driver + */ private function getDoctrineEvents(array $typeConfig) { // Flush always calls depending on actions scheduled in lifecycle listeners $typeConfig['listener']['flush'] = true; + switch ($typeConfig['driver']) { + case 'orm': + $eventsClass = '\Doctrine\ORM\Events'; + break; + case 'mongodb': + $eventsClass = '\Doctrine\ODM\MongoDB\Events'; + break; + default: + throw new InvalidArgumentException(sprintf('Cannot determine events for driver "%s"', $typeConfig['driver'])); + break; + } + $events = array(); $eventMapping = array( - 'insert' => array('postPersist'), - 'update' => array('postUpdate'), - 'delete' => array('preRemove'), - 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') + 'insert' => array(constant($eventsClass.'::postPersist')), + 'update' => array(constant($eventsClass.'::postUpdate')), + 'delete' => array(constant($eventsClass.'::preRemove')), + 'flush' => array($typeConfig['listener']['immediate'] ? constant($eventsClass.'::preFlush') : constant($eventsClass.'::postFlush')) ); foreach ($eventMapping as $event => $doctrineEvents) { diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 683070e..383e02f 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -27,7 +27,7 @@ class Listener implements EventSubscriber protected $objectClass; /** - * List of su]bscribed events + * List of subscribed events * * @var array */ From 05ee300ddb2c1d13ec44c8f65b79808e06fbfe2a Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 16 Dec 2013 11:58:58 +0100 Subject: [PATCH 14/67] Fixing reset command to allow resetting of just one type, starting to write test --- Command/ResetCommand.php | 2 +- Tests/Command/ResetCommandTest.php | 53 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 Tests/Command/ResetCommandTest.php diff --git a/Command/ResetCommand.php b/Command/ResetCommand.php index 280f9aa..06cfe48 100755 --- a/Command/ResetCommand.php +++ b/Command/ResetCommand.php @@ -60,7 +60,7 @@ class ResetCommand extends ContainerAwareCommand if (null !== $type) { $output->writeln(sprintf('Resetting %s/%s', $index, $type)); - $this->resetter->resetIndex($index, $type); + $this->resetter->resetIndexType($index, $type); } else { $indexes = null === $index ? array_keys($this->indexManager->getAllIndexes()) diff --git a/Tests/Command/ResetCommandTest.php b/Tests/Command/ResetCommandTest.php new file mode 100644 index 0000000..44d493e --- /dev/null +++ b/Tests/Command/ResetCommandTest.php @@ -0,0 +1,53 @@ +resetter = $this->getMockBuilder('\FOS\ElasticaBundle\Resetter') + ->disableOriginalConstructor() + ->setMethods(array('resetIndex', 'resetIndexType')) + ->getMock(); + + $container->set('fos_elastica.resetter', $this->resetter); + + $this->indexManager = $this->getMockBuilder('\FOS\ElasticaBundle\IndexManager') + ->disableOriginalConstructor() + ->setMethods(array('getAllIndexes')) + ->getMock(); + + $this->command = new ResetCommand(); + $this->command->setContainer($container); + } + + public function testResetAllIndexes() + { + $this->indexManager->expects($this->any()) + ->method('getAllIndexes') + ->will($this->returnValue(array('index1', 'index2'))); + + $this->resetter->expects($this->at(1)) + ->method('resetIndex') + ->with($this->equalTo('index1')); + + $this->resetter->expects($this->at(2)) + ->method('resetIndex') + ->with($this->equalTo('index2')); + + $this->command->run( + new ArrayInput(array()), + new NullOutput() + ); + } +} \ No newline at end of file From e906d780ad019d4196b4e0817337f5a5a31eba17 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 16 Dec 2013 12:06:53 +0100 Subject: [PATCH 15/67] Adding tests --- Tests/Command/ResetCommandTest.php | 44 ++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/Tests/Command/ResetCommandTest.php b/Tests/Command/ResetCommandTest.php index 44d493e..b6548aa 100644 --- a/Tests/Command/ResetCommandTest.php +++ b/Tests/Command/ResetCommandTest.php @@ -4,6 +4,9 @@ namespace FOS\ElasticaBundle\Tests\Command; use FOS\ElasticaBundle\Command\ResetCommand; +use Symfony\Component\Console\Input\ArrayInput; +use Symfony\Component\Console\Output\NullOutput; +use Symfony\Component\DependencyInjection\Container; class ResetCommandTest extends \PHPUnit_Framework_TestCase { @@ -27,6 +30,8 @@ class ResetCommandTest extends \PHPUnit_Framework_TestCase ->setMethods(array('getAllIndexes')) ->getMock(); + $container->set('fos_elastica.index_manager', $this->indexManager); + $this->command = new ResetCommand(); $this->command->setContainer($container); } @@ -35,13 +40,13 @@ class ResetCommandTest extends \PHPUnit_Framework_TestCase { $this->indexManager->expects($this->any()) ->method('getAllIndexes') - ->will($this->returnValue(array('index1', 'index2'))); + ->will($this->returnValue(array('index1' => true, 'index2' => true))); - $this->resetter->expects($this->at(1)) + $this->resetter->expects($this->at(0)) ->method('resetIndex') ->with($this->equalTo('index1')); - $this->resetter->expects($this->at(2)) + $this->resetter->expects($this->at(1)) ->method('resetIndex') ->with($this->equalTo('index2')); @@ -50,4 +55,37 @@ class ResetCommandTest extends \PHPUnit_Framework_TestCase new NullOutput() ); } + + public function testResetIndex() + { + $this->indexManager->expects($this->never()) + ->method('getAllIndexes'); + + $this->resetter->expects($this->at(0)) + ->method('resetIndex') + ->with($this->equalTo('index1')); + + $this->command->run( + new ArrayInput(array('--index' => 'index1')), + new NullOutput() + ); + } + + public function testResetIndexType() + { + $this->indexManager->expects($this->never()) + ->method('getAllIndexes'); + + $this->resetter->expects($this->never()) + ->method('resetIndex'); + + $this->resetter->expects($this->at(0)) + ->method('resetIndexType') + ->with($this->equalTo('index1'), $this->equalTo('type1')); + + $this->command->run( + new ArrayInput(array('--index' => 'index1', '--type' => 'type1')), + new NullOutput() + ); + } } \ No newline at end of file From d33c5d0ece19f5758f4bc6679f26e4fdd17d7f49 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 17 Dec 2013 21:38:08 +1100 Subject: [PATCH 16/67] Fix tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ad1f366..8f6a9d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ php: before_script: - echo "extension = mongo.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - composer install --dev + - composer install --dev --prefer-source From 4ee81dc010f987497169128f891bc4139b90fea9 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 6 Jan 2014 09:20:44 +0100 Subject: [PATCH 17/67] Adding support for enabling timestamps --- DependencyInjection/Configuration.php | 20 ++++++++++++++++++++ DependencyInjection/FOSElasticaExtension.php | 3 +++ 2 files changed, 23 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e9088e8..dd55ba1 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -299,6 +299,7 @@ class Configuration implements ConfigurationInterface ->append($this->getRoutingNode()) ->append($this->getParentNode()) ->append($this->getAllNode()) + ->append($this->getTimestampNode()) ->end() ; @@ -647,4 +648,23 @@ class Configuration implements ConfigurationInterface return $node; } + + /** + * Returns the array node used for "_timestamp" + */ + protected function getTimestampNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('_timestamp'); + + $node + ->children() + ->scalarNode('enabled')->defaultValue(true)->end() + ->scalarNode('path')->end() + ->scalarNode('format')->end() + ->end() + ; + + return $node; + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 8b85629..a38b90a 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -246,6 +246,9 @@ class FOSElasticaExtension extends Extension if (isset($type['_all'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['_all'] = $type['_all']; } + if (isset($type['_timestamp'])) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['_timestamp'] = $type['_timestamp']; + } if (!empty($type['dynamic_templates'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'] = array(); foreach ($type['dynamic_templates'] as $templateName => $templateData) { From 274fc0099186980d730da1c6f7249b1b899125ad Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Mon, 6 Jan 2014 10:19:11 +0100 Subject: [PATCH 18/67] Add store and index options to timestamp --- DependencyInjection/Configuration.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index dd55ba1..5cdd52a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -662,6 +662,8 @@ class Configuration implements ConfigurationInterface ->scalarNode('enabled')->defaultValue(true)->end() ->scalarNode('path')->end() ->scalarNode('format')->end() + ->scalarNode('store')->end() + ->scalarNode('index')->end() ->end() ; From 73ee750515b0f62a83c3198b2327589f3f644e98 Mon Sep 17 00:00:00 2001 From: Vermi Date: Fri, 10 Jan 2014 16:25:45 +0100 Subject: [PATCH 19/67] Removing "->performNoDeepMerging()"in indexes configuration, allowing custom Bundle defining new types in an existing index. Configuration is injected in the new Bundle using prependExtensionConfig() --- DependencyInjection/Configuration.php | 1 - 1 file changed, 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e9088e8..bea6cc0 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -132,7 +132,6 @@ class Configuration implements ConfigurationInterface ->arrayNode('indexes') ->useAttributeAsKey('name') ->prototype('array') - ->performNoDeepMerging() ->children() ->scalarNode('index_name')->end() ->scalarNode('client')->end() From 74813768358f42b3348e6f07a4b7a4179ee73259 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 11 Jan 2014 16:28:15 +0000 Subject: [PATCH 20/67] Use bulk insert. Still working on bulk update/delete which is not yet suppored in Elastica. --- Doctrine/Listener.php | 5 +++-- Tests/Doctrine/AbstractListenerTest.php | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 383e02f..8c1d99f 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -214,9 +214,10 @@ class Listener implements EventSubscriber */ private function persistScheduled() { - foreach ($this->scheduledForInsertion as $entity) { - $this->objectPersister->insertOne($entity); + if (count($this->scheduledForInsertion)) { + $this->objectPersister->insertMany($this->scheduledForInsertion); } + foreach ($this->scheduledForUpdate as $entity) { $this->objectPersister->replaceOne($entity); } diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index a6ad2aa..3278be1 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -22,8 +22,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForInsertion)); $persister->expects($this->once()) - ->method('insertOne') - ->with($entity); + ->method('insertMany') + ->with($listener->scheduledForInsertion); $listener->postFlush($eventArgs); } @@ -46,6 +46,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $persister->expects($this->never()) ->method('insertOne'); + $persister->expects($this->never()) + ->method('insertMany'); $listener->postFlush($eventArgs); } From 4287a91d506cbc92a1436d21175775ee2dea8c0d Mon Sep 17 00:00:00 2001 From: Tom A Date: Thu, 16 Jan 2014 23:05:44 -0500 Subject: [PATCH 21/67] Add similarity as a valid field mapping. http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#similarity --- DependencyInjection/Configuration.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e9088e8..43f9db4 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -399,7 +399,8 @@ class Configuration implements ConfigurationInterface ->scalarNode('identifier')->defaultValue('id')->end() ->end() ->end() - ->scalarNode('format')->end(); + ->scalarNode('format')->end() + ->scalarNode('similarity')->end(); ; if (isset($nestings['fields'])) { From bbacad4bab2f6375a1bf9d841b7f64de17811047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B3th=20G=C3=A1bor?= Date: Tue, 10 Dec 2013 18:07:22 +0100 Subject: [PATCH 22/67] no-stop-on-error option added to populate command --- Command/PopulateCommand.php | 1 + Doctrine/AbstractProvider.php | 15 +++++++++++- Tests/Doctrine/AbstractProviderTest.php | 32 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) mode change 100755 => 100644 Command/PopulateCommand.php diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php old mode 100755 new mode 100644 index 7297523..661f70d --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -45,6 +45,7 @@ class PopulateCommand extends ContainerAwareCommand ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0) ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0) ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)') + ->addOption('no-stop-on-error', null, InputOption::VALUE_NONE, 'Do not stop on errors') ->setDescription('Populates search indexes from providers') ; } diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 1fb41f5..2346c80 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Doctrine; use Doctrine\Common\Persistence\ManagerRegistry; +use Elastica\Exception\Bulk\ResponseException as BulkResponseException; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Provider\AbstractProvider as BaseAbstractProvider; @@ -23,6 +24,7 @@ abstract class AbstractProvider extends BaseAbstractProvider parent::__construct($objectPersister, $objectClass, array_merge(array( 'clear_object_manager' => true, 'query_builder_method' => 'createQueryBuilder', + 'stop_on_error' => true, ), $options)); $this->managerRegistry = $managerRegistry; @@ -38,6 +40,7 @@ abstract class AbstractProvider extends BaseAbstractProvider $offset = isset($options['offset']) ? intval($options['offset']) : 0; $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; + $stopOnError = isset($options['no-stop-on-error']) ? empty($options['no-stop-on-error']) : $this->options['stop_on_error']; for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { @@ -45,7 +48,17 @@ abstract class AbstractProvider extends BaseAbstractProvider } $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); - $this->objectPersister->insertMany($objects); + if (!$stopOnError) { + $this->objectPersister->insertMany($objects); + } else { + try { + $this->objectPersister->insertMany($objects); + } catch(BulkResponseException $e) { + if ($loggerClosure) { + $loggerClosure(sprintf('%s',$e->getMessage())); + } + } + } if ($this->options['clear_object_manager']) { $this->managerRegistry->getManagerForClass($this->objectClass)->clear(); diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index 0a9aceb..a3836bd 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -135,6 +135,28 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase $this->assertTrue($loggerClosureInvoked); } + public function testPopulateNotStopOnError() + { + $nbObjects = 1; + $objects = array(1); + + $provider = $this->getMockAbstractProvider(); + + $provider->expects($this->any()) + ->method('countObjects') + ->will($this->returnValue($nbObjects)); + + $provider->expects($this->any()) + ->method('fetchSlice') + ->will($this->returnValue($objects)); + + $this->objectPersister->expects($this->any()) + ->method('insertMany') + ->will($this->throwException($this->getMockBulkResponseException())); + + $provider->populate(null, array('no-stop-on-error' => true)); + } + /** * @return \FOS\ElasticaBundle\Doctrine\AbstractProvider|\PHPUnit_Framework_MockObject_MockObject */ @@ -148,6 +170,16 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase )); } + /** + * @return \Elastica\Exception\Bulk\ResponseException + */ + private function getMockBulkResponseException() + { + return $this->getMockBuilder('Elastica\Exception\Bulk\ResponseException') + ->disableOriginalConstructor() + ->getMock(); + } + /** * @return \Doctrine\Common\Persistence\ManagerRegistry|\PHPUnit_Framework_MockObject_MockObject */ From b6d010a9d7e7801c17cb618d8984d70eacff8f15 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Thu, 23 Jan 2014 16:20:11 +0000 Subject: [PATCH 23/67] Bulk update. Still working on bulk delete for indexes and types in Elastica. --- Doctrine/Listener.php | 8 ++------ Persister/ObjectPersister.php | 17 +++++++++++------ Persister/ObjectPersisterInterface.php | 10 +++++++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 8c1d99f..d6314dc 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -214,13 +214,9 @@ class Listener implements EventSubscriber */ private function persistScheduled() { - if (count($this->scheduledForInsertion)) { - $this->objectPersister->insertMany($this->scheduledForInsertion); - } + $this->objectPersister->bulkPersist($this->scheduledForInsertion, ObjectPersisterInterface::BULK_INSERT); + $this->objectPersister->bulkPersist($this->scheduledForUpdate, ObjectPersisterInterface::BULK_REPLACE); - foreach ($this->scheduledForUpdate as $entity) { - $this->objectPersister->replaceOne($entity); - } foreach ($this->scheduledForDeletion as $entity) { $this->objectPersister->deleteOne($entity); } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 450e43b..b5a1be7 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -83,19 +83,24 @@ class ObjectPersister implements ObjectPersisterInterface } catch (NotFoundException $e) {} } - /** - * Inserts an array of objects in the type + * Bulk update an array of objects in the type for the given method * * @param array $objects array of domain model objects - **/ - public function insertMany(array $objects) + * @param string Method to call + */ + public function bulkPersist(array $objects, $method) { + if (!count($objects)) { + return; + } + $documents = array(); foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } - $this->type->addDocuments($documents); + + $this->type->$method($documents); } /** @@ -108,4 +113,4 @@ class ObjectPersister implements ObjectPersisterInterface { return $this->transformer->transform($object, $this->fields); } -} +} \ No newline at end of file diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index a50bcc8..5c4ecd2 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -10,6 +10,9 @@ namespace FOS\ElasticaBundle\Persister; */ interface ObjectPersisterInterface { + const BULK_INSERT = 'addDocuments'; + const BULK_REPLACE = 'updateDocuments'; + /** * Insert one object into the type * The object will be transformed to an elastica document @@ -42,9 +45,10 @@ interface ObjectPersisterInterface function deleteById($id); /** - * Inserts an array of objects in the type + * Bulk update an array of objects in the type for the given method * * @param array $objects array of domain model objects - **/ - function insertMany(array $objects); + * @param string Method to call + */ + function bulkPersist(array $objects, $method); } From 62dc3aaca04c1dc2fe62e763d8dde20d897d6f0d Mon Sep 17 00:00:00 2001 From: pkraeutli Date: Fri, 24 Jan 2014 14:20:10 +0100 Subject: [PATCH 24/67] Populate command: use isInteractive() instead of option --- Command/PopulateCommand.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7297523..38bff70 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -67,10 +67,9 @@ class PopulateCommand extends ContainerAwareCommand $index = $input->getOption('index'); $type = $input->getOption('type'); $reset = $input->getOption('no-reset') ? false : true; - $noInteraction = $input->getOption('no-interaction'); $options = $input->getOptions(); - if (!$noInteraction && $reset && $input->getOption('offset')) { + if ($input->isInteractive() && $reset && $input->getOption('offset')) { /** @var DialogHelper $dialog */ $dialog = $this->getHelperSet()->get('dialog'); if (!$dialog->askConfirmation($output, 'You chose to reset the index and start indexing with an offset. Do you really want to do that?', true)) { From 1c7da33526dea333dfd2eed81bfb04de5031a261 Mon Sep 17 00:00:00 2001 From: Simon Perdrisat Date: Tue, 28 Jan 2014 11:34:54 +0100 Subject: [PATCH 25/67] [doc] repository methode has to return the query --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3006bd1..f78c97f 100644 --- a/README.md +++ b/README.md @@ -526,7 +526,7 @@ class UserRepository extends Repository public function findWithCustomQuery($searchText) { // build $query with Elastica objects - $this->find($query); + return $this->find($query); } } ``` From 076d51ac40dcbbe8fdd69223ab2d7a0377b60ced Mon Sep 17 00:00:00 2001 From: Piotr Antosik Date: Wed, 29 Jan 2014 17:50:42 +0100 Subject: [PATCH 26/67] Services classes as parameters --- Resources/config/config.xml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 1fb2c1d..85cfbed 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -14,6 +14,10 @@ FOS\ElasticaBundle\Transformer\ElasticaToModelTransformerCollection FOS\ElasticaBundle\Provider\ProviderRegistry Symfony\Component\PropertyAccess\PropertyAccessor + FOS\ElasticaBundle\Persister\ObjectPersister + FOS\ElasticaBundle\Persister\ObjectSerializerPersister + FOS\ElasticaBundle\Transformer\ModelToElasticaAutoTransformer + FOS\ElasticaBundle\Transformer\ModelToElasticaIdentifierTransformer @@ -37,7 +41,7 @@ - + @@ -49,21 +53,21 @@ - + - + - + From 5480e037e3a005edd548d44a7bc31fb66e406db9 Mon Sep 17 00:00:00 2001 From: Laszlo Horvath Date: Thu, 30 Jan 2014 12:39:48 +0100 Subject: [PATCH 27/67] adding ttl for documents --- DependencyInjection/Configuration.php | 21 ++++++++++++++++++++ DependencyInjection/FOSElasticaExtension.php | 3 +++ 2 files changed, 24 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index ca97a4d..4ed88af 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -299,6 +299,7 @@ class Configuration implements ConfigurationInterface ->append($this->getParentNode()) ->append($this->getAllNode()) ->append($this->getTimestampNode()) + ->append($this->getTtlNode()) ->end() ; @@ -669,4 +670,24 @@ class Configuration implements ConfigurationInterface return $node; } + + /** + * Returns the array node used for "_ttl" + */ + protected function getTtlNode() + { + $builder = new TreeBuilder(); + $node = $builder->root('_ttl'); + + $node + ->children() + ->scalarNode('enabled')->defaultValue(true)->end() + ->scalarNode('default')->end() + ->scalarNode('store')->end() + ->scalarNode('index')->end() + ->end() + ; + + return $node; + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index a38b90a..52070fd 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -249,6 +249,9 @@ class FOSElasticaExtension extends Extension if (isset($type['_timestamp'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['_timestamp'] = $type['_timestamp']; } + if (isset($type['_ttl'])) { + $this->indexConfigs[$indexName]['config']['mappings'][$name]['_ttl'] = $type['_ttl']; + } if (!empty($type['dynamic_templates'])) { $this->indexConfigs[$indexName]['config']['mappings'][$name]['dynamic_templates'] = array(); foreach ($type['dynamic_templates'] as $templateName => $templateData) { From 5e54dcd9558efe686523dae6e0d1cfc0662d429c Mon Sep 17 00:00:00 2001 From: Fabien Somnier Date: Thu, 30 Jan 2014 13:01:13 +0100 Subject: [PATCH 28/67] Add RAM (current & peak) in populate evolution information --- Doctrine/AbstractProvider.php | 5 ++++- Propel/Provider.php | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 1fb41f5..eebfc79 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -54,11 +54,14 @@ abstract class AbstractProvider extends BaseAbstractProvider usleep($sleep); if ($loggerClosure) { + $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo + $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get max usage in Mo + $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond)); + $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $message)); } } } diff --git a/Propel/Provider.php b/Propel/Provider.php index c319691..3eb1c4f 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -37,11 +37,14 @@ class Provider extends AbstractProvider usleep($sleep); if ($loggerClosure) { + $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo + $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get usage in Mo + $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond)); + $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $message)); } } } From 1dcaadbe6f99b42b346e04239c0c8fd7812bbf51 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 1 Feb 2014 02:14:21 +0000 Subject: [PATCH 29/67] Bulk delete --- Doctrine/Listener.php | 17 +++++++----- Persister/ObjectPersister.php | 37 +++++++++++++++++++++----- Persister/ObjectPersisterInterface.php | 24 ++++++++++++----- README.md | 4 +-- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index d6314dc..c254513 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -210,21 +210,24 @@ class Listener implements EventSubscriber } /** - * Persist scheduled action to ElasticSearch + * Persist scheduled objects to ElasticSearch */ private function persistScheduled() { - $this->objectPersister->bulkPersist($this->scheduledForInsertion, ObjectPersisterInterface::BULK_INSERT); - $this->objectPersister->bulkPersist($this->scheduledForUpdate, ObjectPersisterInterface::BULK_REPLACE); - - foreach ($this->scheduledForDeletion as $entity) { - $this->objectPersister->deleteOne($entity); + if (count($this->scheduledForInsertion)) { + $this->objectPersister->insertMany($this->scheduledForInsertion); + } + if (count($this->scheduledForUpdate)) { + $this->objectPersister->replaceMany($this->scheduledForUpdate); + } + if (count($this->scheduledForDeletion)) { + $this->objectPersister->deleteMany($this->scheduledForDeletion); } } /** * Iterate through scheduled actions before flushing to emulate 2.x behavior. Note that the ElasticSearch index - * will fall out of sync with the data source in event of a crash on flush. + * will fall out of sync with the source data in the event of a crash during flush. */ public function preFlush(EventArgs $eventArgs) { diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index b5a1be7..3592a78 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -84,23 +84,46 @@ class ObjectPersister implements ObjectPersisterInterface } /** - * Bulk update an array of objects in the type for the given method + * Bulk insert an array of objects in the type for the given method * * @param array $objects array of domain model objects * @param string Method to call */ - public function bulkPersist(array $objects, $method) + public function insertMany(array $objects) { - if (!count($objects)) { - return; - } - $documents = array(); foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } + $this->type->addDocuments($documents); + } - $this->type->$method($documents); + /** + * Bulk updates an array of objects in the type + * + * @param array $objects array of domain model objects + */ + public function replaceMany(array $objects) + { + $documents = array(); + foreach ($objects as $object) { + $documents[] = $this->transformToElasticaDocument($object); + } + $this->type->updateDocuments($documents); + } + + /** + * Bulk deletes an array of objects in the type + * + * @param array $objects array of domain model objects + */ + public function deleteMany(array $objects) + { + $documents = array(); + foreach ($objects as $object) { + $documents[] = $this->transformToElasticaDocument($object); + } + $this->type->deleteDocuments($documents); } /** diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index 5c4ecd2..a25aafc 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -10,9 +10,6 @@ namespace FOS\ElasticaBundle\Persister; */ interface ObjectPersisterInterface { - const BULK_INSERT = 'addDocuments'; - const BULK_REPLACE = 'updateDocuments'; - /** * Insert one object into the type * The object will be transformed to an elastica document @@ -41,14 +38,27 @@ interface ObjectPersisterInterface * @param mixed $id * * @return null - **/ + */ function deleteById($id); /** - * Bulk update an array of objects in the type for the given method + * Bulk inserts an array of objects in the type * * @param array $objects array of domain model objects - * @param string Method to call */ - function bulkPersist(array $objects, $method); + function insertMany(array $objects); + + /** + * Bulk updates an array of objects in the type + * + * @param array $objects array of domain model objects + */ + function replaceMany(array $objects); + + /** + * Bulk deletes an array of objects in the type + * + * @param array $objects array of domain model objects + */ + function deleteMany(array $objects); } diff --git a/README.md b/README.md index b3a4c5d..cb6e5f8 100644 --- a/README.md +++ b/README.md @@ -627,8 +627,8 @@ to `true`: delete: true immediate: true -> Updating ElasticSearch before flushing may cause the ElasticSearch index to fall out of sync with the -> original data in the event of a crash. +> Using `immediate` to update ElasticSearch before flush completes may cause the ElasticSearch index to fall out of +> sync with the source database in the event of a crash during the flush itself, such as in the case of a bad query. ### Checking an entity method for listener From 559d7c13f277d534fbf883ae5d989c0fe4e1adce Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 1 Feb 2014 03:37:47 +0000 Subject: [PATCH 30/67] Update tests --- Tests/Doctrine/AbstractListenerTest.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index 3278be1..a9eff66 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -65,8 +65,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForUpdate)); $persister->expects($this->once()) - ->method('replaceOne') - ->with($entity); + ->method('replaceMany') + ->with(array($entity)); $persister->expects($this->never()) ->method('deleteById'); @@ -105,8 +105,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $persister->expects($this->never()) ->method('replaceOne'); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } @@ -136,8 +136,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } @@ -167,8 +167,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } From f348ebd0265aaa1d46e09c8aca5f6d646f27263c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=B6nthal?= Date: Mon, 3 Feb 2014 09:47:16 +0100 Subject: [PATCH 31/67] fixes ignore_missing option and unknown index --- Transformer/ElasticaToModelTransformerCollection.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Transformer/ElasticaToModelTransformerCollection.php b/Transformer/ElasticaToModelTransformerCollection.php index a261e81..f65f8db 100644 --- a/Transformer/ElasticaToModelTransformerCollection.php +++ b/Transformer/ElasticaToModelTransformerCollection.php @@ -67,7 +67,9 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer $result = array(); foreach ($elasticaObjects as $object) { - $result[] = $transformed[$object->getType()][$object->getId()]; + if (array_key_exists($object->getId(), $transformed[$object->getType()])) { + $result[] = $transformed[$object->getType()][$object->getId()]; + } } return $result; From a121a777743525f46c65dc29d983c8eb92720664 Mon Sep 17 00:00:00 2001 From: Fabien Somnier Date: Mon, 3 Feb 2014 17:50:35 +0100 Subject: [PATCH 32/67] Avoid index reset error in case of unexistant index --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7297523..9d00db1 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -107,7 +107,7 @@ class PopulateCommand extends ContainerAwareCommand */ private function populateIndex(OutputInterface $output, $index, $reset, $options) { - if ($reset) { + if ($reset && $this->indexManager->getIndex($index)->exists()) { $output->writeln(sprintf('Resetting %s', $index)); $this->resetter->resetIndex($index); } From 48b785e8764a887f903cbb81476b4368f5588bd5 Mon Sep 17 00:00:00 2001 From: Fabien Somnier Date: Mon, 3 Feb 2014 18:12:20 +0100 Subject: [PATCH 33/67] New method getMemoryUsage to get RAM information message --- Doctrine/AbstractProvider.php | 5 +---- Propel/Provider.php | 5 +---- Provider/AbstractProvider.php | 12 ++++++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index eebfc79..796a5e4 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -54,14 +54,11 @@ abstract class AbstractProvider extends BaseAbstractProvider usleep($sleep); if ($loggerClosure) { - $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo - $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get max usage in Mo - $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $message)); + $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); } } } diff --git a/Propel/Provider.php b/Propel/Provider.php index 3eb1c4f..393beba 100644 --- a/Propel/Provider.php +++ b/Propel/Provider.php @@ -37,14 +37,11 @@ class Provider extends AbstractProvider usleep($sleep); if ($loggerClosure) { - $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo - $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get usage in Mo - $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; $stepNbObjects = count($objects); $stepCount = $stepNbObjects + $offset; $percentComplete = 100 * $stepCount / $nbObjects; $objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime); - $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $message)); + $loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage())); } } } diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 06883a3..48da5d7 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -26,4 +26,16 @@ abstract class AbstractProvider implements ProviderInterface 'batch_size' => 100, ), $options); } + + /** + * Get string with RAM usage information (current and peak) + * + * @return string + */ + protected function getMemoryUsage() { + $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo + $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get max usage in Mo + $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; + return $message; + } } From 04390b37d1bacb16cce87384e0c33914ace7b967 Mon Sep 17 00:00:00 2001 From: tgallice Date: Tue, 4 Feb 2014 10:07:58 +0100 Subject: [PATCH 34/67] Force slash at the end of the url parameter --- DependencyInjection/Configuration.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4ed88af..0c6ca82 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -101,7 +101,12 @@ class Configuration implements ConfigurationInterface ->arrayNode('servers') ->prototype('array') ->children() - ->scalarNode('url')->end() + ->scalarNode('url') + ->validate() + ->ifTrue(function($v) { return substr($v['url'], -1) !== '/'; }) + ->then(function($v) { return $v['url'].'/'; }) + ->end() + ->end() ->scalarNode('host')->end() ->scalarNode('port')->end() ->scalarNode('logger') From d04a403f71f82b46c3a3314031595b0c60b9dee9 Mon Sep 17 00:00:00 2001 From: Hung Tran Date: Tue, 4 Feb 2014 19:41:38 -0600 Subject: [PATCH 35/67] added support for setting search options --- Finder/FinderInterface.php | 3 ++- Finder/PaginatedFinderInterface.php | 6 ++++-- Finder/TransformedFinder.php | 23 +++++++++++++---------- Paginator/RawPaginatorAdapter.php | 10 ++++++++-- Paginator/TransformedPaginatorAdapter.php | 4 ++-- Repository.php | 16 ++++++++-------- 6 files changed, 37 insertions(+), 25 deletions(-) diff --git a/Finder/FinderInterface.php b/Finder/FinderInterface.php index 4805d58..7c257de 100644 --- a/Finder/FinderInterface.php +++ b/Finder/FinderInterface.php @@ -9,7 +9,8 @@ interface FinderInterface * * @param mixed $query Can be a string, an array or an \Elastica\Query object * @param int $limit How many results to get + * @param array $options * @return array results */ - function find($query, $limit = null); + function find($query, $limit = null, $options = array()); } diff --git a/Finder/PaginatedFinderInterface.php b/Finder/PaginatedFinderInterface.php index 3581cda..fa10b70 100644 --- a/Finder/PaginatedFinderInterface.php +++ b/Finder/PaginatedFinderInterface.php @@ -12,15 +12,17 @@ interface PaginatedFinderInterface extends FinderInterface * Searches for query results and returns them wrapped in a paginator * * @param mixed $query Can be a string, an array or an \Elastica\Query object + * @param array $options * @return Pagerfanta paginated results */ - function findPaginated($query); + function findPaginated($query, $options = array()); /** * Creates a paginator adapter for this query * * @param mixed $query + * @param array $options * @return PaginatorAdapterInterface */ - function createPaginatorAdapter($query); + function createPaginatorAdapter($query, $options = array()); } diff --git a/Finder/TransformedFinder.php b/Finder/TransformedFinder.php index 4c8aa98..9080701 100644 --- a/Finder/TransformedFinder.php +++ b/Finder/TransformedFinder.php @@ -29,18 +29,19 @@ class TransformedFinder implements PaginatedFinderInterface * * @param string $query * @param integer $limit + * @param array $options * @return array of model objects **/ - public function find($query, $limit = null) + public function find($query, $limit = null, $options = array()) { - $results = $this->search($query, $limit); + $results = $this->search($query, $limit, $options); return $this->transformer->transform($results); } - public function findHybrid($query, $limit = null) + public function findHybrid($query, $limit = null, $options = array()) { - $results = $this->search($query, $limit); + $results = $this->search($query, $limit, $options); return $this->transformer->hybridTransform($results); } @@ -64,15 +65,16 @@ class TransformedFinder implements PaginatedFinderInterface /** * @param $query * @param null|int $limit + * @param array $options * @return array */ - protected function search($query, $limit = null) + protected function search($query, $limit = null, $options = array()) { $queryObject = Query::create($query); if (null !== $limit) { $queryObject->setSize($limit); } - $results = $this->searchable->search($queryObject)->getResults(); + $results = $this->searchable->search($queryObject, $options)->getResults(); return $results; } @@ -81,12 +83,13 @@ class TransformedFinder implements PaginatedFinderInterface * Gets a paginator wrapping the result of a search * * @param string $query + * @param array $options * @return Pagerfanta */ - public function findPaginated($query) + public function findPaginated($query, $options = array()) { $queryObject = Query::create($query); - $paginatorAdapter = $this->createPaginatorAdapter($queryObject); + $paginatorAdapter = $this->createPaginatorAdapter($queryObject, $options); return new Pagerfanta(new FantaPaginatorAdapter($paginatorAdapter)); } @@ -94,10 +97,10 @@ class TransformedFinder implements PaginatedFinderInterface /** * {@inheritdoc} */ - public function createPaginatorAdapter($query) + public function createPaginatorAdapter($query, $options = array()) { $query = Query::create($query); - return new TransformedPaginatorAdapter($this->searchable, $query, $this->transformer); + return new TransformedPaginatorAdapter($this->searchable, $query, $options, $this->transformer); } } diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index d4b5357..e99746f 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -22,6 +22,11 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface */ private $query; + /** + * @var array search options + */ + private $options; + /** * @var integer the number of hits */ @@ -38,10 +43,11 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface * @param SearchableInterface $searchable the object to search in * @param Query $query the query to search */ - public function __construct(SearchableInterface $searchable, Query $query) + public function __construct(SearchableInterface $searchable, Query $query, array $options = array()) { $this->searchable = $searchable; $this->query = $query; + $this->options = $options; } /** @@ -72,7 +78,7 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface $query->setFrom($offset); $query->setSize($itemCountPerPage); - $resultSet = $this->searchable->search($query); + $resultSet = $this->searchable->search($query, $this->options); $this->totalHits = $resultSet->getTotalHits(); $this->facets = $resultSet->getFacets(); return $resultSet; diff --git a/Paginator/TransformedPaginatorAdapter.php b/Paginator/TransformedPaginatorAdapter.php index 7bc038a..10976f7 100644 --- a/Paginator/TransformedPaginatorAdapter.php +++ b/Paginator/TransformedPaginatorAdapter.php @@ -18,9 +18,9 @@ class TransformedPaginatorAdapter extends RawPaginatorAdapter * @param Query $query the query to search * @param ElasticaToModelTransformerInterface $transformer the transformer for fetching the results */ - public function __construct(SearchableInterface $searchable, Query $query, ElasticaToModelTransformerInterface $transformer) + public function __construct(SearchableInterface $searchable, Query $query, array $options = array(), ElasticaToModelTransformerInterface $transformer) { - parent::__construct($searchable, $query); + parent::__construct($searchable, $query, $options); $this->transformer = $transformer; } diff --git a/Repository.php b/Repository.php index 413f1e4..70b2a21 100644 --- a/Repository.php +++ b/Repository.php @@ -19,23 +19,23 @@ class Repository $this->finder = $finder; } - public function find($query, $limit=null) + public function find($query, $limit = null, $options = array()) { - return $this->finder->find($query, $limit); + return $this->finder->find($query, $limit, $options); } - public function findHybrid($query, $limit=null) + public function findHybrid($query, $limit = null, $options = array()) { - return $this->finder->findHybrid($query, $limit); + return $this->finder->findHybrid($query, $limit, $options); } - public function findPaginated($query) + public function findPaginated($query, $options = array()) { - return $this->finder->findPaginated($query); + return $this->finder->findPaginated($query, $options); } - public function createPaginatorAdapter($query) + public function createPaginatorAdapter($query, $options = array()) { - return $this->finder->createPaginatorAdapter($query); + return $this->finder->createPaginatorAdapter($query, $options); } } From 5be3a8c22ea9577f12200b42b14318899c1260de Mon Sep 17 00:00:00 2001 From: Hung Tran Date: Wed, 5 Feb 2014 11:21:14 -0600 Subject: [PATCH 36/67] added query string to collector and updated tests --- Client.php | 2 +- Logger/ElasticaLogger.php | 6 ++++-- Resources/views/Collector/elastica.html.twig | 3 ++- Tests/ClientTest.php | 1 + Tests/Logger/ElasticaLoggerTest.php | 4 +++- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Client.php b/Client.php index ea0c572..7719095 100644 --- a/Client.php +++ b/Client.php @@ -27,7 +27,7 @@ class Client extends ElasticaClient 'transport' => $connection->getTransport(), ); - $this->_logger->logQuery($path, $method, $data, $time, $connection_array); + $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); } return $response; diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index 7aacac5..60f28db 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -37,10 +37,11 @@ class ElasticaLogger implements LoggerInterface * @param string $path Path to call * @param string $method Rest method to use (GET, POST, DELETE, PUT) * @param array $data arguments + * @param array $query arguments * @param float $time execution time * @param array $connection host, port and transport of the query */ - public function logQuery($path, $method, $data, $time, $connection = array()) + public function logQuery($path, $method, $data, $time, $connection = array(), $query = array()) { if ($this->debug) { $this->queries[] = array( @@ -48,7 +49,8 @@ class ElasticaLogger implements LoggerInterface 'method' => $method, 'data' => $data, 'executionMS' => $time, - 'connection' => $connection + 'connection' => $connection, + 'queryString' => $query, ); } diff --git a/Resources/views/Collector/elastica.html.twig b/Resources/views/Collector/elastica.html.twig index 0eb50a6..637dae7 100644 --- a/Resources/views/Collector/elastica.html.twig +++ b/Resources/views/Collector/elastica.html.twig @@ -44,6 +44,7 @@ {% for key, query in collector.queries %}
  • Path: {{ query.path }}
    + Query: {{ query.queryString|url_encode }}
    Method: {{ query.method }} ({{ query.connection.transport }} on {{ query.connection.host }}:{{ query.connection.port }})
    {{ query.data|json_encode }} @@ -60,7 +61,7 @@ {% endif %}
    • diff --git a/Tests/ClientTest.php b/Tests/ClientTest.php index c8509cf..8a9d91a 100644 --- a/Tests/ClientTest.php +++ b/Tests/ClientTest.php @@ -24,6 +24,7 @@ class ClientTest extends \PHPUnit_Framework_TestCase Request::GET, $this->isType('array'), $this->isType('float'), + $this->isType('array'), $this->isType('array') ); diff --git a/Tests/Logger/ElasticaLoggerTest.php b/Tests/Logger/ElasticaLoggerTest.php index 30ce30c..96adf53 100644 --- a/Tests/Logger/ElasticaLoggerTest.php +++ b/Tests/Logger/ElasticaLoggerTest.php @@ -70,6 +70,7 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase $data = array('data'); $time = 12; $connection = array('host' => 'localhost', 'port' => '8999', 'transport' => 'https'); + $query = array('search_type' => 'dfs_query_then_fetch'); $expected = array( 'path' => $path, @@ -77,9 +78,10 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase 'data' => $data, 'executionMS' => $time, 'connection' => $connection, + 'queryString' => $query, ); - $elasticaLogger->logQuery($path, $method, $data, $time, $connection); + $elasticaLogger->logQuery($path, $method, $data, $time, $connection, $query); $returnedQueries = $elasticaLogger->getQueries(); $this->assertEquals($expected, $returnedQueries[0]); } From 3b1a756e6f89e828b2f9f2c5b280d0f86a72637c Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Thu, 6 Feb 2014 21:11:12 +0000 Subject: [PATCH 37/67] Add support for using aliases to allow hot swapping of indexes when populating --- Client.php | 5 + Command/PopulateCommand.php | 1 + DependencyInjection/Configuration.php | 5 +- DependencyInjection/FOSElasticaExtension.php | 5 + DynamicIndex.php | 28 ++++ Resetter.php | 136 ++++++++++++++++++- Resources/config/config.xml | 2 +- 7 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 DynamicIndex.php diff --git a/Client.php b/Client.php index ea0c572..67252ec 100644 --- a/Client.php +++ b/Client.php @@ -32,4 +32,9 @@ class Client extends ElasticaClient return $response; } + + public function getIndex($name) + { + return new DynamicIndex($this, $name); + } } diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7297523..31ecc4b 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -124,6 +124,7 @@ class PopulateCommand extends ContainerAwareCommand } $output->writeln(sprintf('Refreshing %s', $index)); + $this->resetter->postPopulate($index); $this->indexManager->getIndex($index)->refresh(); } diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4ed88af..403de85 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -134,6 +134,7 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->children() ->scalarNode('index_name')->end() + ->booleanNode('use_alias')->defaultValue(false)->end() ->scalarNode('client')->end() ->scalarNode('finder') ->treatNullLike(true) @@ -670,7 +671,7 @@ class Configuration implements ConfigurationInterface return $node; } - + /** * Returns the array node used for "_ttl" */ @@ -689,5 +690,5 @@ class Configuration implements ConfigurationInterface ; return $node; - } + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 52070fd..a67be0f 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -124,6 +124,7 @@ class FOSElasticaExtension extends Extension $indexIds[$name] = $indexId; $this->indexConfigs[$name] = array( 'index' => new Reference($indexId), + 'name_or_alias' => $indexName, 'config' => array( 'mappings' => array() ) @@ -134,6 +135,10 @@ class FOSElasticaExtension extends Extension if (!empty($index['settings'])) { $this->indexConfigs[$name]['config']['settings'] = $index['settings']; } + if ($index['use_alias']) { + $this->indexConfigs[$name]['use_alias'] = true; + } + $this->loadTypes(isset($index['types']) ? $index['types'] : array(), $container, $name, $indexId, $typePrototypeConfig); } diff --git a/DynamicIndex.php b/DynamicIndex.php new file mode 100644 index 0000000..cbec8e9 --- /dev/null +++ b/DynamicIndex.php @@ -0,0 +1,28 @@ + + */ +class DynamicIndex extends Index +{ + /** + * Reassign index name + * + * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName + * since it's used for a very specific case and normally should not be used + * + * @param string $name Index name + * + * @return void + */ + public function overrideName($name) + { + $this->_name = $name; + } +} diff --git a/Resetter.php b/Resetter.php index 26a6bb5..b8e7b80 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,6 +2,8 @@ namespace FOS\ElasticaBundle; +use Elastica\Exception\ExceptionInterface; +use Elastica\Index; use Elastica\Type\Mapping; /** @@ -26,8 +28,8 @@ class Resetter */ public function resetAllIndexes() { - foreach ($this->indexConfigsByName as $indexConfig) { - $indexConfig['index']->create($indexConfig['config'], true); + foreach (array_keys($this->indexConfigsByName) as $name) { + $this->resetIndex($name); } } @@ -40,7 +42,17 @@ class Resetter public function resetIndex($indexName) { $indexConfig = $this->getIndexConfig($indexName); - $indexConfig['index']->create($indexConfig['config'], true); + $esIndex = $indexConfig['index']; + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $name = $indexConfig['name_or_alias']; + $name .= date('-Y-m-d-Gis'); + $esIndex->overrideName($name); + $esIndex->create($indexConfig['config']); + + return; + } + + $esIndex->create($indexConfig['config'], true); } /** @@ -102,4 +114,122 @@ class Resetter return $this->indexConfigsByName[$indexName]; } + + public function postPopulate($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $this->switchIndexAlias($indexName); + } + } + + /** + * Switches the alias for given index to the newly populated index + * and deletes the old index + * + * @param string $indexName Index name + * + * @throws \RuntimeException + */ + private function switchIndexAlias($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + $esIndex = $indexConfig['index']; + $aliasName = $indexConfig['name_or_alias']; + $oldIndexName = false; + $newIndexName = $esIndex->getName(); + + $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); + + if (count($aliasedIndexes) > 1) { + throw new \RuntimeException( + sprintf( + 'Alias %s is used for multiple indexes: [%s]. + Make sure it\'s either not used or is assigned to one index only', + $aliasName, + join(', ', $aliasedIndexes) + ) + ); + } + + // Change the alias to point to the new index + // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic + // In 0.20.x it's atomic, but it doesn't return the old index name + $aliasUpdateRequest = array('actions' => array()); + if (count($aliasedIndexes) == 1) { + // if the alias is set - add an action to remove it + $oldIndexName = $aliasedIndexes[0]; + $aliasUpdateRequest['actions'][] = array( + 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) + ); + } + + // add an action to point the alias to the new index + $aliasUpdateRequest['actions'][] = array( + 'add' => array('index' => $newIndexName, 'alias' => $aliasName) + ); + + try { + $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); + } catch (ExceptionInterface $renameAliasException) { + $additionalError = ''; + // if we failed to move the alias, delete the newly built index + try { + $esIndex->delete(); + } catch (ExceptionInterface $deleteNewIndexException) { + $additionalError = sprintf( + 'Tried to delete newly built index %s, but also failed: %s', + $newIndexName, + $deleteNewIndexException->getError() + ); + } + + throw new \RuntimeException( + sprintf( + 'Failed to updated index alias: %s. %s', + $renameAliasException->getMessage(), + $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) + ) + ); + } + + // Delete the old index after the alias has been switched + if ($oldIndexName) { + $oldIndex = new Index($esIndex->getClient(), $oldIndexName); + try { + $oldIndex->delete(); + } catch (ExceptionInterface $deleteOldIndexException) { + throw new \RuntimeException( + sprintf( + 'Failed to delete old index %s with message: %s', + $oldIndexName, + $deleteOldIndexException->getMessage() + ) + ); + } + } + } + + /** + * Returns array of indexes which are mapped to given alias + * + * @param Index $esIndex ES Index + * @param string $aliasName Alias name + * + * @return array + */ + private function getAliasedIndexes(Index $esIndex, $aliasName) + { + $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); + $aliasedIndexes = array(); + + foreach ($aliasesInfo as $indexName => $indexInfo) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } + } + + return $aliasedIndexes; + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 85cfbed..4419b4a 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -6,7 +6,7 @@ FOS\ElasticaBundle\Client - Elastica\Index + FOS\ElasticaBundle\DynamicIndex Elastica\Type FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector From 03bf793b656a11646ae74a351ce46dd3de74b045 Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Mon, 10 Feb 2014 10:55:13 +0000 Subject: [PATCH 38/67] Make real index name use uniqid so can be reset within a second --- Resetter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resetter.php b/Resetter.php index b8e7b80..4a7f13b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -45,7 +45,7 @@ class Resetter $esIndex = $indexConfig['index']; if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { $name = $indexConfig['name_or_alias']; - $name .= date('-Y-m-d-Gis'); + $name .= uniqid(); $esIndex->overrideName($name); $esIndex->create($indexConfig['config']); @@ -124,7 +124,7 @@ class Resetter } /** - * Switches the alias for given index to the newly populated index + * Switches the alias for given index (by key) to the newly populated index * and deletes the old index * * @param string $indexName Index name From 1402bdc9e63911510e093d22deb7b8290856719c Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Feb 2014 09:11:05 +0100 Subject: [PATCH 39/67] Keep all special mapping fields for types when resetting a single type. Field list is according to the current documentation of elasticsearch 0.90 --- Resetter.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Resetter.php b/Resetter.php index 26a6bb5..d0f3014 100644 --- a/Resetter.php +++ b/Resetter.php @@ -74,12 +74,11 @@ class Resetter { $mapping = Mapping::create($indexConfig['properties']); - if (isset($indexConfig['_parent'])) { - $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); - } - - if (isset($indexConfig['dynamic_templates'])) { - $mapping->setParam('dynamic_templates', $indexConfig['dynamic_templates']); + $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_parent', '_routing', '_index', '_size', '_timestamp', '_ttl'); + foreach ($mappingSpecialFields as $specialField) { + if (isset($indexConfig[$specialField])) { + $mapping->setParam($specialField, $indexConfig[$specialField]); + } } return $mapping; From ead3c95c88ed11756d0fdfbf11f9c7a8dfe84271 Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Feb 2014 09:45:27 +0100 Subject: [PATCH 40/67] Keep dynamic templates --- Resetter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index d0f3014..d193c9d 100644 --- a/Resetter.php +++ b/Resetter.php @@ -74,7 +74,7 @@ class Resetter { $mapping = Mapping::create($indexConfig['properties']); - $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_parent', '_routing', '_index', '_size', '_timestamp', '_ttl'); + $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_parent', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); foreach ($mappingSpecialFields as $specialField) { if (isset($indexConfig[$specialField])) { $mapping->setParam($specialField, $indexConfig[$specialField]); From 765e400278fff2e59030e47bdead2f8240ef1fef Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Feb 2014 10:06:10 +0100 Subject: [PATCH 41/67] handle _parent separately --- Resetter.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index d193c9d..69620a2 100644 --- a/Resetter.php +++ b/Resetter.php @@ -74,13 +74,17 @@ class Resetter { $mapping = Mapping::create($indexConfig['properties']); - $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_parent', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); + $mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates'); foreach ($mappingSpecialFields as $specialField) { if (isset($indexConfig[$specialField])) { $mapping->setParam($specialField, $indexConfig[$specialField]); } } + if (isset($indexConfig['_parent'])) { + $mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type'])); + } + return $mapping; } From 3065c96a2ca51ec03f5aa32877a75079faa8b413 Mon Sep 17 00:00:00 2001 From: Patrick McAndrew Date: Fri, 14 Feb 2014 17:32:22 +0000 Subject: [PATCH 42/67] Add ability to specify server timeout --- DependencyInjection/Configuration.php | 1 + 1 file changed, 1 insertion(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 79cadf2..b2a1756 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -102,6 +102,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('url')->end() ->scalarNode('host')->end() ->scalarNode('port')->end() + ->scalarNode('timeout')->end() ->end() ->end() ->end() From 7d2776e27aad52dc7808a5db55e795b436790aef Mon Sep 17 00:00:00 2001 From: Hung Tran Date: Tue, 18 Feb 2014 10:12:31 -0600 Subject: [PATCH 43/67] fixed phpdoc --- Logger/ElasticaLogger.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index 60f28db..0e0a9c0 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -37,9 +37,9 @@ class ElasticaLogger implements LoggerInterface * @param string $path Path to call * @param string $method Rest method to use (GET, POST, DELETE, PUT) * @param array $data arguments - * @param array $query arguments * @param float $time execution time * @param array $connection host, port and transport of the query + * @param array $query arguments */ public function logQuery($path, $method, $data, $time, $connection = array(), $query = array()) { From b8cc6d758c3fb8a6397184bb423deebbae38208b Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 09:31:22 +0100 Subject: [PATCH 44/67] CS fix in AbstractProvider --- Provider/AbstractProvider.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 48da5d7..8f5ad60 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -32,10 +32,12 @@ abstract class AbstractProvider implements ProviderInterface * * @return string */ - protected function getMemoryUsage() { + protected function getMemoryUsage() + { $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get max usage in Mo $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; + return $message; } } From 41bf07ec5995e47d95fb4b01d0434eb04e283680 Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 11:01:54 +0100 Subject: [PATCH 45/67] Fixed issues in #426 --- Command/PopulateCommand.php | 4 +++- Doctrine/AbstractProvider.php | 6 +++--- Tests/Doctrine/AbstractProviderTest.php | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 07badd7..b13ddc3 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -45,7 +45,7 @@ class PopulateCommand extends ContainerAwareCommand ->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0) ->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0) ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)') - ->addOption('no-stop-on-error', null, InputOption::VALUE_NONE, 'Do not stop on errors') + ->addOption('ignore-errors', null, InputOption::VALUE_NONE, 'Do not stop on errors') ->setDescription('Populates search indexes from providers') ; } @@ -70,6 +70,8 @@ class PopulateCommand extends ContainerAwareCommand $reset = $input->getOption('no-reset') ? false : true; $options = $input->getOptions(); + $options['ignore-errors'] = $input->hasOption('ignore-errors'); + if ($input->isInteractive() && $reset && $input->getOption('offset')) { /** @var DialogHelper $dialog */ $dialog = $this->getHelperSet()->get('dialog'); diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php index 109cfa0..9d1575c 100644 --- a/Doctrine/AbstractProvider.php +++ b/Doctrine/AbstractProvider.php @@ -23,8 +23,8 @@ abstract class AbstractProvider extends BaseAbstractProvider { parent::__construct($objectPersister, $objectClass, array_merge(array( 'clear_object_manager' => true, + 'ignore_errors' => false, 'query_builder_method' => 'createQueryBuilder', - 'stop_on_error' => true, ), $options)); $this->managerRegistry = $managerRegistry; @@ -40,7 +40,7 @@ abstract class AbstractProvider extends BaseAbstractProvider $offset = isset($options['offset']) ? intval($options['offset']) : 0; $sleep = isset($options['sleep']) ? intval($options['sleep']) : 0; $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size']; - $stopOnError = isset($options['no-stop-on-error']) ? empty($options['no-stop-on-error']) : $this->options['stop_on_error']; + $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors']; for (; $offset < $nbObjects; $offset += $batchSize) { if ($loggerClosure) { @@ -48,7 +48,7 @@ abstract class AbstractProvider extends BaseAbstractProvider } $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset); - if (!$stopOnError) { + if (!$ignoreErrors) { $this->objectPersister->insertMany($objects); } else { try { diff --git a/Tests/Doctrine/AbstractProviderTest.php b/Tests/Doctrine/AbstractProviderTest.php index a3836bd..2492eed 100644 --- a/Tests/Doctrine/AbstractProviderTest.php +++ b/Tests/Doctrine/AbstractProviderTest.php @@ -154,7 +154,9 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase ->method('insertMany') ->will($this->throwException($this->getMockBulkResponseException())); - $provider->populate(null, array('no-stop-on-error' => true)); + $this->setExpectedException('Elastica\Exception\Bulk\ResponseException'); + + $provider->populate(null, array('ignore-errors' => false)); } /** From 44180793fcef2b15671d2d75364c69154dc6e447 Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 12:29:51 +0100 Subject: [PATCH 46/67] Add changelog file for 3.0 --- CHANGELOG-3.0.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 CHANGELOG-3.0.md diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md new file mode 100644 index 0000000..534a25e --- /dev/null +++ b/CHANGELOG-3.0.md @@ -0,0 +1,17 @@ +CHANGELOG for 3.0.x +=================== + +This changelog references the relevant changes (bug and security fixes) done +in 3.0 minor versions. + +To get the diff for a specific change, go to +https://github.com/FriendsOfSymfony/FOSElasticaBundle/commit/XXX where XXX is +the commit hash. To get the diff between two versions, go to +https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 + +To generate a changelog summary since the last version, run +`git log --no-merges --oneline v3.0.0...3.0.x` + +* 3.0.0-ALPHA2 (2014-xx-xx) + + * 41bf07e: Renamed the `no-stop-on-error` option in PopulateCommand to `ignore-errors` From 63cca11a0b0a0a89f368d71a5afcb3944800d2eb Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 12:43:44 +0100 Subject: [PATCH 47/67] Tweak: use hasOption() in PopulateCommand instead of checking for booleans --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index b13ddc3..7d8a46e 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -67,7 +67,7 @@ class PopulateCommand extends ContainerAwareCommand { $index = $input->getOption('index'); $type = $input->getOption('type'); - $reset = $input->getOption('no-reset') ? false : true; + $reset = $input->hasOption('no-reset'); $options = $input->getOptions(); $options['ignore-errors'] = $input->hasOption('ignore-errors'); From 891dd51abebcdeaaa7ac4de017a8ef0a5868fa42 Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 12:50:48 +0100 Subject: [PATCH 48/67] Fixed bug made in #479 --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7d8a46e..afed5ed 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -67,7 +67,7 @@ class PopulateCommand extends ContainerAwareCommand { $index = $input->getOption('index'); $type = $input->getOption('type'); - $reset = $input->hasOption('no-reset'); + $reset = !$input->hasOption('no-reset'); $options = $input->getOptions(); $options['ignore-errors'] = $input->hasOption('ignore-errors'); From 1c74f61b4ef5cf1b2d1c9b83c5814f173d7edddf Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Wed, 19 Feb 2014 12:58:02 +0100 Subject: [PATCH 49/67] AbstractProvider cleanup --- Provider/AbstractProvider.php | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Provider/AbstractProvider.php b/Provider/AbstractProvider.php index 8f5ad60..2761a25 100644 --- a/Provider/AbstractProvider.php +++ b/Provider/AbstractProvider.php @@ -4,10 +4,24 @@ namespace FOS\ElasticaBundle\Provider; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; +/** + * AbstractProvider + */ abstract class AbstractProvider implements ProviderInterface { - protected $objectClass; + /** + * @var ObjectPersisterInterface + */ protected $objectPersister; + + /** + * @var string + */ + protected $objectClass; + + /** + * @var array + */ protected $options; /** @@ -34,10 +48,9 @@ abstract class AbstractProvider implements ProviderInterface */ protected function getMemoryUsage() { - $memory = round(memory_get_usage() / (1024*1024),0); // to get usage in Mo - $memoryMax = round(memory_get_peak_usage() / (1024*1024)); // to get max usage in Mo - $message = '(RAM : current='.$memory.'Mo peak='.$memoryMax.'Mo)'; + $memory = round(memory_get_usage() / (1024 * 1024)); // to get usage in Mo + $memoryMax = round(memory_get_peak_usage() / (1024 * 1024)); // to get max usage in Mo - return $message; + return sprintf('(RAM : current=%uMo peak=%uMo)', $memory, $memoryMax); } } From 6f8b3a5a0f028edfb99ad714b0c448c05c36ca00 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 21 Feb 2014 08:58:43 +1100 Subject: [PATCH 50/67] Fix populate command option --- Command/PopulateCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index afed5ed..a67df94 100644 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -67,7 +67,7 @@ class PopulateCommand extends ContainerAwareCommand { $index = $input->getOption('index'); $type = $input->getOption('type'); - $reset = !$input->hasOption('no-reset'); + $reset = !$input->getOption('no-reset'); $options = $input->getOptions(); $options['ignore-errors'] = $input->hasOption('ignore-errors'); From eecdd3474a2ddb2b46dc98fffc4b4bb641dfaed6 Mon Sep 17 00:00:00 2001 From: Matteo Galli Date: Fri, 21 Feb 2014 16:47:42 +0100 Subject: [PATCH 51/67] Fixes #459 --- DependencyInjection/FOSElasticaExtension.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 52070fd..96e1fdf 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -4,6 +4,7 @@ namespace FOS\ElasticaBundle\DependencyInjection; use Symfony\Component\HttpKernel\DependencyInjection\Extension; use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; +use Symfony\Component\DependencyInjection\ContainerAwareInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\DefinitionDecorator; @@ -198,6 +199,10 @@ class FOSElasticaExtension extends Extension if (isset($type['serializer']['version'])) { $callbackDef->addMethodCall('setVersion', array($type['serializer']['version'])); } + $callbackClassImplementedInterfaces = class_implements($this->serializerConfig['callback_class']); // PHP < 5.4 friendly + if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) { + $callbackDef->addMethodCall('setContainer', array(new Reference('service_container'))); + } $container->setDefinition($callbackId, $callbackDef); From 2b04f6cf34807854226794ff78dd3f4f207f73ed Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Tue, 25 Feb 2014 11:05:26 +0100 Subject: [PATCH 52/67] ElasticaLogger cleanup --- Logger/ElasticaLogger.php | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index 0e0a9c0..5b1269f 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -14,32 +14,42 @@ use Psr\Log\LoggerInterface; */ class ElasticaLogger implements LoggerInterface { + /** + * @var LoggerInterface + */ protected $logger; - protected $queries; + + /** + * @var array + */ + protected $queries = array(); + + /** + * @var boolean + */ protected $debug; /** * Constructor. * * @param LoggerInterface|null $logger The Symfony logger - * @param bool $debug + * @param boolean $debug */ public function __construct(LoggerInterface $logger = null, $debug = false) { $this->logger = $logger; - $this->queries = array(); $this->debug = $debug; } /** * Logs a query. * - * @param string $path Path to call - * @param string $method Rest method to use (GET, POST, DELETE, PUT) - * @param array $data arguments - * @param float $time execution time - * @param array $connection host, port and transport of the query - * @param array $query arguments + * @param string $path Path to call + * @param string $method Rest method to use (GET, POST, DELETE, PUT) + * @param array $data Arguments + * @param float $time Execution time + * @param array $connection Host, port and transport of the query + * @param array $query Arguments */ public function logQuery($path, $method, $data, $time, $connection = array(), $query = array()) { From 418b9d72cec839d1b5f2dcb73cbd83bdd276bdbe Mon Sep 17 00:00:00 2001 From: tgallice Date: Tue, 4 Mar 2014 17:54:49 +0100 Subject: [PATCH 53/67] Rework configuration validation to fix #461 --- DependencyInjection/Configuration.php | 4 ++-- .../DependencyInjection/ConfigurationTest.php | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 107744d..4488bf2 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -103,8 +103,8 @@ class Configuration implements ConfigurationInterface ->children() ->scalarNode('url') ->validate() - ->ifTrue(function($v) { return substr($v['url'], -1) !== '/'; }) - ->then(function($v) { return $v['url'].'/'; }) + ->ifTrue(function($url) { return substr($url, -1) !== '/'; }) + ->then(function($url) { return $url.'/'; }) ->end() ->end() ->scalarNode('host')->end() diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 35b2af3..0f0d374 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -3,6 +3,7 @@ namespace FOS\ElasticaBundle\Tests\Resetter\DependencyInjection; use FOS\ElasticaBundle\DependencyInjection\Configuration; +use Symfony\Component\Config\Definition\Processor; /** * ConfigurationTest @@ -85,4 +86,21 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['index']); $this->assertNull($mapping['index']->getDefaultValue()); } + + public function testSlashIsAddedAtTheEndOfServerUrl() + { + $config = array( + 'clients' => array( + 'default' => array( + 'url' => 'http://www.github.com', + ), + ), + ); + + $processor = new Processor(); + + $configuration = $processor->processConfiguration($this->configuration, array($config)); + + $this->assertEquals('http://www.github.com/', $configuration['clients']['default']['servers'][0]['url']); + } } From a1f7449efc7e8190726812f987e6cba98e4bccbe Mon Sep 17 00:00:00 2001 From: Bastien Jaillot Date: Wed, 5 Mar 2014 15:59:38 +0100 Subject: [PATCH 54/67] [doc] inform about the override of Elastica logger Add a documentation related to this change: https://github.com/FriendsOfSymfony/FOSElasticaBundle/pull/395#issuecomment-27729759 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f78c97f..aaaf0a5 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Most of the time, you will need only one. clients: default: { host: localhost, port: 9200 } +A client configuration can also override the Elastica logger to change the used class ```logger: ``` or to simply disable it ```logger: false```. Disabling the logger should be done on production because it can cause a memory leak. #### Declare a serializer From 7f53badad545a9e1d5b04a9b02c238fb7a5e30b7 Mon Sep 17 00:00:00 2001 From: Berny Cantos Date: Thu, 5 Sep 2013 13:45:36 +0200 Subject: [PATCH 55/67] Add support for include_in_{parent,root} for nested and objects --- DependencyInjection/Configuration.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4488bf2..adcaf6a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -415,6 +415,10 @@ class Configuration implements ConfigurationInterface } if (isset($nestings['properties'])) { + $node + ->booleanNode('include_in_parent')->end() + ->booleanNode('include_in_root')->end() + ; $this->addNestedFieldConfig($node, $nestings, 'properties'); } } From 0116a6ac4f88da2210660c0e6ca55515b0450656 Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 6 Mar 2014 12:38:23 -0430 Subject: [PATCH 56/67] Add support for clients requiring basic HTTP authentication --- Client.php | 1 + DependencyInjection/Configuration.php | 8 +++++++- Logger/ElasticaLogger.php | 2 +- README.md | 17 +++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Client.php b/Client.php index 7719095..601fc77 100644 --- a/Client.php +++ b/Client.php @@ -25,6 +25,7 @@ class Client extends ElasticaClient 'host' => $connection->getHost(), 'port' => $connection->getPort(), 'transport' => $connection->getTransport(), + 'headers' => $connection->getConfig('headers'), ); $this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query); diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4488bf2..a01d1cb 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -78,7 +78,8 @@ class Configuration implements ConfigurationInterface array( 'host' => $v['host'], 'port' => $v['port'], - 'logger' => isset($v['logger']) ? $v['logger'] : null + 'logger' => isset($v['logger']) ? $v['logger'] : null, + 'headers' => isset($v['headers']) ? $v['headers'] : null, ) ) ); @@ -114,6 +115,11 @@ class Configuration implements ConfigurationInterface ->treatNullLike('fos_elastica.logger') ->treatTrueLike('fos_elastica.logger') ->end() + ->arrayNode('headers') + ->children() + ->scalarNode('Authorization')->end() + ->end() + ->end() ->scalarNode('timeout')->end() ->end() ->end() diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index 0e0a9c0..85acf70 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -38,7 +38,7 @@ class ElasticaLogger implements LoggerInterface * @param string $method Rest method to use (GET, POST, DELETE, PUT) * @param array $data arguments * @param float $time execution time - * @param array $connection host, port and transport of the query + * @param array $connection host, port, transport and headers of the query * @param array $query arguments */ public function logQuery($path, $method, $data, $time, $connection = array(), $query = array()) diff --git a/README.md b/README.md index f78c97f..cd1a1ba 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,23 @@ Most of the time, you will need only one. clients: default: { host: localhost, port: 9200 } +If your client requires Basic HTTP Authentication, you can specify an Authorization Header to +include in HTTP requests. The Authorization Header value is a ``base64`` encoded string that +includes the authentication username and password, and can be obtained by running the following +command in your terminal: + + php -r "Print 'Basic ' . base64_encode('your_auth_username' . ':' . 'your_auth_password');" + +A sample configuration with Basic HTTP Authentication is: + + #app/config/config.yml + fos_elastica: + clients: + default: + host: example.com + port: 80 + headers: + Authorization: "Basic jdumrGK7rY9TMuQOPng7GZycmxyMHNoir==" #### Declare a serializer From fe871c5ac48d2be18a2efef46cd2b14ca001fe1a Mon Sep 17 00:00:00 2001 From: Miha Vrhovnik Date: Tue, 11 Mar 2014 14:50:00 +0100 Subject: [PATCH 57/67] Fix wrong annotation --- Paginator/RawPaginatorAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index e99746f..b74a9e4 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -33,7 +33,7 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface private $totalHits; /** - * @array for the facets + * @var array for the facets */ private $facets; From 726892c586949b8cf13ccca7fa7c1f3c3ca43f2d Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Mar 2014 10:43:35 +0100 Subject: [PATCH 58/67] Ignore TypeMissingException when resetting a single type. This allows to create new types without having to recreate the whole index. --- Resetter.php | 9 ++++++++- Tests/ResetterTest.php | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index 69620a2..b22f6a1 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle; +use Elastica\Exception\ResponseException; use Elastica\Type\Mapping; /** @@ -59,7 +60,13 @@ class Resetter } $type = $indexConfig['index']->getType($typeName); - $type->delete(); + try { + $type->delete(); + } catch (ResponseException $e) { + if (strpos($e->getMessage(), 'TypeMissingException') !== 0) { + throw $e; + } + } $mapping = $this->createMapping($indexConfig['config']['mappings'][$typeName]); $type->setMapping($mapping); } diff --git a/Tests/ResetterTest.php b/Tests/ResetterTest.php index aa0fbcc..b4e5649 100644 --- a/Tests/ResetterTest.php +++ b/Tests/ResetterTest.php @@ -2,6 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Resetter; +use Elastica\Exception\ResponseException; +use Elastica\Request; +use Elastica\Response; use FOS\ElasticaBundle\Resetter; use Elastica\Type\Mapping; @@ -130,6 +133,32 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $resetter->resetIndexType('foo', 'c'); } + public function testResetIndexTypeIgnoreTypeMissingException() + { + $type = $this->getMockElasticaType(); + + $this->indexConfigsByName['foo']['index']->expects($this->once()) + ->method('getType') + ->with('a') + ->will($this->returnValue($type)); + + $type->expects($this->once()) + ->method('delete') + ->will($this->throwException(new ResponseException( + new Request(''), + new Response(array('error' => 'TypeMissingException[[de_20131022] type[bla] missing]', 'status' => 404))) + )); + + $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); + $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']); + $type->expects($this->once()) + ->method('setMapping') + ->with($mapping); + + $resetter = new Resetter($this->indexConfigsByName); + $resetter->resetIndexType('foo', 'a'); + } + public function testIndexMappingForParent() { $type = $this->getMockElasticaType(); From 2f9896c893063b7142a0ac1d2980e05c9050f2df Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Mar 2014 10:45:57 +0100 Subject: [PATCH 59/67] Check for TypeMissingException anywhere in exception message --- Resetter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index b22f6a1..863524b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -63,7 +63,7 @@ class Resetter try { $type->delete(); } catch (ResponseException $e) { - if (strpos($e->getMessage(), 'TypeMissingException') !== 0) { + if (strpos($e->getMessage(), 'TypeMissingException') === false) { throw $e; } } From cdd6e3af45a7fb9f97b53887416e0b3511763452 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 14 Mar 2014 08:59:59 +1100 Subject: [PATCH 60/67] Bump elastica version --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 094eba5..e588494 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": "~0.20", + "ruflin/elastica": ">=0.20, <1.1-dev", "psr/log": "~1.0" }, "require-dev":{ From 6a26c63b2c5d0d56b9d4346e523b12ce25af6576 Mon Sep 17 00:00:00 2001 From: Piotr Antosik Date: Fri, 14 Mar 2014 18:41:49 +0100 Subject: [PATCH 61/67] fix some block comment --- Client.php | 3 +++ FOSElasticaBundle.php | 6 +++++- Paginator/RawPaginatorAdapter.php | 3 ++- Paginator/TransformedPaginatorAdapter.php | 5 +++-- Resetter.php | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Client.php b/Client.php index 7719095..ab819f8 100644 --- a/Client.php +++ b/Client.php @@ -11,6 +11,9 @@ use FOS\ElasticaBundle\Logger\ElasticaLogger; */ class Client extends ElasticaClient { + /** + * {@inheritdoc} + */ public function request($path, $method = Request::GET, $data = array(), array $query = array()) { $start = microtime(true); diff --git a/FOSElasticaBundle.php b/FOSElasticaBundle.php index 5ac1f2a..44424d3 100644 --- a/FOSElasticaBundle.php +++ b/FOSElasticaBundle.php @@ -8,10 +8,14 @@ use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Compiler\PassConfig; use Symfony\Component\HttpKernel\Bundle\Bundle; +/** + * Bundle. + * + */ class FOSElasticaBundle extends Bundle { /** - * @see Symfony\Component\HttpKernel\Bundle\Bundle::build() + * {@inheritdoc} */ public function build(ContainerBuilder $container) { diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index b74a9e4..8bd4ee2 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -41,7 +41,8 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface * @see PaginatorAdapterInterface::__construct * * @param SearchableInterface $searchable the object to search in - * @param Query $query the query to search + * @param Query $query the query to search + * @param array $options */ public function __construct(SearchableInterface $searchable, Query $query, array $options = array()) { diff --git a/Paginator/TransformedPaginatorAdapter.php b/Paginator/TransformedPaginatorAdapter.php index 10976f7..3b4716f 100644 --- a/Paginator/TransformedPaginatorAdapter.php +++ b/Paginator/TransformedPaginatorAdapter.php @@ -14,8 +14,9 @@ class TransformedPaginatorAdapter extends RawPaginatorAdapter private $transformer; /** - * @param SearchableInterface $searchable the object to search in - * @param Query $query the query to search + * @param SearchableInterface $searchable the object to search in + * @param Query $query the query to search + * @param array $options * @param ElasticaToModelTransformerInterface $transformer the transformer for fetching the results */ public function __construct(SearchableInterface $searchable, Query $query, array $options = array(), ElasticaToModelTransformerInterface $transformer) diff --git a/Resetter.php b/Resetter.php index 863524b..c97cd34 100644 --- a/Resetter.php +++ b/Resetter.php @@ -50,6 +50,7 @@ class Resetter * @param string $indexName * @param string $typeName * @throws \InvalidArgumentException if no index or type mapping exists for the given names + * @throws ResponseException */ public function resetIndexType($indexName, $typeName) { From dc01b189ed8ecb3670bd205fa2c8940547e60c40 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 17 Mar 2014 09:26:16 +1100 Subject: [PATCH 62/67] Update CHANGELOG-3.0.md --- CHANGELOG-3.0.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 534a25e..eb3e8c8 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,6 +12,14 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.0-ALPHA2 (2014-xx-xx) +* 3.0.0-ALPHA3 (xxxx-xx-xx) + + * #463: allowing hot swappable reindexing + * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. + +* 3.0.0-ALPHA2 (2014-03-17) * 41bf07e: Renamed the `no-stop-on-error` option in PopulateCommand to `ignore-errors` + * 418b9d7: Fixed validation of url configuration + * 726892c: Ignore TypeMissingException when resetting a single type. This allows to create new types without having to recreate the whole index. + * 7f53bad Add support for include_in_{parent,root} for nested and objects From 7d13823488c414cd42f794c1f3cacb82b6c96542 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 18 Mar 2014 09:10:44 +1100 Subject: [PATCH 63/67] Bump symfony requirement --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e588494..8f22393 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,7 @@ ], "require": { "php": ">=5.3.2", - "symfony/framework-bundle": "~2.1", + "symfony/framework-bundle": "~2.3", "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", From babe20800e6bff5af8bfd0de54bb72175dca0ff4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 18 Mar 2014 09:12:11 +1100 Subject: [PATCH 64/67] Updated changelog --- CHANGELOG-3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index eb3e8c8..a80d09d 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -16,6 +16,7 @@ To generate a changelog summary since the last version, run * #463: allowing hot swappable reindexing * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. + * 7d13823: Dropped (broken) support for Symfony <2.3 * 3.0.0-ALPHA2 (2014-03-17) From de70e78b5370f6f9f014351e3169dfb035ed8415 Mon Sep 17 00:00:00 2001 From: rayrigam Date: Tue, 18 Mar 2014 18:14:03 -0430 Subject: [PATCH 65/67] Update Configuration.php Updated to support any HTTP request header type in the "headers" section. --- DependencyInjection/Configuration.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index a01d1cb..d629ea6 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -101,6 +101,7 @@ class Configuration implements ConfigurationInterface ->children() ->arrayNode('servers') ->prototype('array') + ->fixXmlConfig('header') ->children() ->scalarNode('url') ->validate() @@ -116,9 +117,8 @@ class Configuration implements ConfigurationInterface ->treatTrueLike('fos_elastica.logger') ->end() ->arrayNode('headers') - ->children() - ->scalarNode('Authorization')->end() - ->end() + ->useAttributeAsKey('name') + ->prototype('scalar')->end() ->end() ->scalarNode('timeout')->end() ->end() From 4dc08cd006f3260411d7bb26e74c61ec6268d959 Mon Sep 17 00:00:00 2001 From: Ray Date: Tue, 18 Mar 2014 20:07:30 -0430 Subject: [PATCH 66/67] Added notice about added support for HTTP headers. --- CHANGELOG-3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index a80d09d..fd2ecd5 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -17,6 +17,7 @@ To generate a changelog summary since the last version, run * #463: allowing hot swappable reindexing * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. * 7d13823: Dropped (broken) support for Symfony <2.3 + * #496: Added support for HTTP headers * 3.0.0-ALPHA2 (2014-03-17) From 455ff9e0f7dea27c45570859218211a45a9fe0ea Mon Sep 17 00:00:00 2001 From: Joris van de Sande Date: Mon, 24 Mar 2014 13:13:57 +0100 Subject: [PATCH 67/67] Unset fields if no nested fields are defined --- DependencyInjection/Configuration.php | 4 +++ .../DependencyInjection/ConfigurationTest.php | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 0445748..40b26c6 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -334,6 +334,10 @@ class Configuration implements ConfigurationInterface $childrenNode = $node ->useAttributeAsKey('name') ->prototype('array') + ->validate() + ->ifTrue(function($v) { return isset($v['fields']) && empty($v['fields']); }) + ->then(function($v) { unset($v['fields']); return $v; }) + ->end() ->treatNullLike(array()) ->addDefaultsIfNotSet() ->children(); diff --git a/Tests/DependencyInjection/ConfigurationTest.php b/Tests/DependencyInjection/ConfigurationTest.php index 0f0d374..93143ec 100644 --- a/Tests/DependencyInjection/ConfigurationTest.php +++ b/Tests/DependencyInjection/ConfigurationTest.php @@ -103,4 +103,34 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase $this->assertEquals('http://www.github.com/', $configuration['clients']['default']['servers'][0]['url']); } + + public function testEmptyFieldsIndexIsUnset() + { + $config = array( + 'indexes' => array( + 'test' => array( + 'types' => array( + 'test' => array( + 'mappings' => array( + 'title' => array( + 'type' => 'string', + 'fields' => array( + 'autocomplete' => null + ) + ), + 'content' => null + ) + ) + ) + ) + ) + ); + + $processor = new Processor(); + + $configuration = $processor->processConfiguration(new Configuration(array($config)), array($config)); + + $this->assertArrayNotHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['content']); + $this->assertArrayHasKey('fields', $configuration['indexes']['test']['types']['test']['mappings']['title']); + } }