From f258c9ddc0f1adf3a61fb2e5b3b2a267794a4f6f Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sun, 1 Dec 2013 21:17:55 +0000 Subject: [PATCH 01/25] Exploratory development for https://github.com/FriendsOfSymfony/FOSElasticaBundle/issues/410 --- DependencyInjection/FOSElasticaExtension.php | 4 + Doctrine/AbstractListener.php | 94 ++++++++++++-------- Doctrine/MongoDB/Listener.php | 40 ++------- Doctrine/ORM/Listener.php | 39 ++------ 4 files changed, 78 insertions(+), 99 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 9bf732e..3caaf57 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -405,11 +405,15 @@ class FOSElasticaExtension extends Extension private function getDoctrineEvents(array $typeConfig) { + // Flush event always fires; not configurable + $typeConfig['listener']['flush'] = true; + $events = array(); $eventMapping = array( 'insert' => array('postPersist'), 'update' => array('postUpdate'), 'delete' => array('postRemove', 'preRemove') + 'flush' => array('postFlush') ); foreach ($eventMapping as $event => $doctrineEvents) { diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php index 3b62444..9a4385a 100644 --- a/Doctrine/AbstractListener.php +++ b/Doctrine/AbstractListener.php @@ -2,8 +2,8 @@ namespace FOS\ElasticaBundle\Doctrine; +use Doctrine\Common\EventArgs; use Doctrine\Common\EventSubscriber; -use Doctrine\Common\Persistence\ObjectManager; use FOS\ElasticaBundle\Persister\ObjectPersisterInterface; use FOS\ElasticaBundle\Persister\ObjectPersister; use Symfony\Component\ExpressionLanguage\Expression; @@ -48,11 +48,11 @@ abstract class AbstractListener implements EventSubscriber protected $isIndexableCallback; /** - * Objects scheduled for removal - * - * @var array + * Objects scheduled for insertion, replacement, or removal */ - private $scheduledForRemoval = array(); + protected $scheduledForInsertion = array(); + protected $scheduledForUpdate = array(); + protected $scheduledForDeletion = array(); /** * An instance of ExpressionLanguage @@ -149,37 +149,6 @@ abstract class AbstractListener implements EventSubscriber : call_user_func($this->isIndexableCallback, $object); } - /** - * Schedules the object for removal. - * - * This is usually called during the pre-remove event. - * - * @param object $object - * @param ObjectManager $objectManager - */ - protected function scheduleForRemoval($object, ObjectManager $objectManager) - { - $metadata = $objectManager->getClassMetadata($this->objectClass); - $esId = $metadata->getFieldValue($object, $this->esIdentifierField); - $this->scheduledForRemoval[spl_object_hash($object)] = $esId; - } - - /** - * Removes the object if it was scheduled for removal. - * - * This is usually called during the post-remove event. - * - * @param object $object - */ - protected function removeIfScheduled($object) - { - $objectHash = spl_object_hash($object); - if (isset($this->scheduledForRemoval[$objectHash])) { - $this->objectPersister->deleteById($this->scheduledForRemoval[$objectHash]); - unset($this->scheduledForRemoval[$objectHash]); - } - } - /** * @param mixed $object * @return string @@ -207,4 +176,57 @@ abstract class AbstractListener implements EventSubscriber return $this->expressionLanguage; } + + public function postPersist(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { + $this->scheduledForInsertion[] = $entity; + } + } + + public function postUpdate(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + if ($this->isObjectIndexable($entity)) { + $this->scheduledForUpdate[] = $entity; + } else { + // Delete if no longer indexable + $this->scheduledForDeletion[] = $entity; + } + } + } + + public function preRemove(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + $this->scheduledForDeletion[] = $entity; + } + } + + public function postRemove(EventArgs $eventArgs) + { + } + + /** + * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + foreach ($this->scheduledForInsertion as $entity) { + $this->objectPersister->insertOne($entity); + } + foreach ($this->scheduledForUpdate as $entity) { + $this->objectPersister->replaceOne($entity); + } + foreach ($this->scheduledForDeletion as $entity) { + $this->objectPersister->deleteOne($entity); + } + } } diff --git a/Doctrine/MongoDB/Listener.php b/Doctrine/MongoDB/Listener.php index 9fa3536..13e1d25 100644 --- a/Doctrine/MongoDB/Listener.php +++ b/Doctrine/MongoDB/Listener.php @@ -2,49 +2,23 @@ namespace FOS\ElasticaBundle\Doctrine\MongoDB; -use Doctrine\ODM\MongoDB\Event\LifecycleEventArgs; +use Doctrine\Common\EventArgs; use FOS\ElasticaBundle\Doctrine\AbstractListener; class Listener extends AbstractListener { - public function postPersist(LifecycleEventArgs $eventArgs) + public function postPersist(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass && $this->isObjectIndexable($document)) { - $this->objectPersister->insertOne($document); - } + parent::postPersist($eventArgs); } - public function postUpdate(LifecycleEventArgs $eventArgs) + public function postUpdate(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - if ($this->isObjectIndexable($document)) { - $this->objectPersister->replaceOne($document); - } else { - $this->scheduleForRemoval($document, $eventArgs->getDocumentManager()); - $this->removeIfScheduled($document); - } - } + parent::postUpdate($eventArgs); } - public function preRemove(LifecycleEventArgs $eventArgs) + public function postRemove(EventArgs $eventArgs) { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - $this->scheduleForRemoval($document, $eventArgs->getDocumentManager()); - } - } - - public function postRemove(LifecycleEventArgs $eventArgs) - { - $document = $eventArgs->getDocument(); - - if ($document instanceof $this->objectClass) { - $this->removeIfScheduled($document); - } + parent::postRemove($eventArgs); } } diff --git a/Doctrine/ORM/Listener.php b/Doctrine/ORM/Listener.php index 790ddb8..97f8792 100644 --- a/Doctrine/ORM/Listener.php +++ b/Doctrine/ORM/Listener.php @@ -2,49 +2,28 @@ namespace FOS\ElasticaBundle\Doctrine\ORM; -use Doctrine\ORM\Event\LifecycleEventArgs; +use Doctrine\Common\EventArgs; use FOS\ElasticaBundle\Doctrine\AbstractListener; class Listener extends AbstractListener { - public function postPersist(LifecycleEventArgs $eventArgs) + public function postPersist(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { - $this->objectPersister->insertOne($entity); - } + parent::postPersist($eventArgs); } - public function postUpdate(LifecycleEventArgs $eventArgs) + public function postUpdate(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - if ($this->isObjectIndexable($entity)) { - $this->objectPersister->replaceOne($entity); - } else { - $this->scheduleForRemoval($entity, $eventArgs->getEntityManager()); - $this->removeIfScheduled($entity); - } - } + parent::postUpdate($eventArgs); } - public function preRemove(LifecycleEventArgs $eventArgs) + public function preRemove(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->scheduleForRemoval($entity, $eventArgs->getEntityManager()); - } + parent::preRemove($eventArgs); } - public function postRemove(LifecycleEventArgs $eventArgs) + public function postRemove(EventArgs $eventArgs) { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->removeIfScheduled($entity); - } + parent::postRemove($eventArgs); } } From 5ec652063de1c381400488f5607870ba490b21ba Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sun, 1 Dec 2013 21:48:41 +0000 Subject: [PATCH 02/25] Don't need postRemove. --- DependencyInjection/FOSElasticaExtension.php | 2 +- Doctrine/AbstractListener.php | 4 ---- Doctrine/ORM/Listener.php | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 3caaf57..30ad07a 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -412,7 +412,7 @@ class FOSElasticaExtension extends Extension $eventMapping = array( 'insert' => array('postPersist'), 'update' => array('postUpdate'), - 'delete' => array('postRemove', 'preRemove') + 'delete' => array('preRemove'), 'flush' => array('postFlush') ); diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php index 9a4385a..0fa15df 100644 --- a/Doctrine/AbstractListener.php +++ b/Doctrine/AbstractListener.php @@ -209,10 +209,6 @@ abstract class AbstractListener implements EventSubscriber } } - public function postRemove(EventArgs $eventArgs) - { - } - /** * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected * only if the query is successful diff --git a/Doctrine/ORM/Listener.php b/Doctrine/ORM/Listener.php index 97f8792..722b89d 100644 --- a/Doctrine/ORM/Listener.php +++ b/Doctrine/ORM/Listener.php @@ -21,9 +21,4 @@ class Listener extends AbstractListener { parent::preRemove($eventArgs); } - - public function postRemove(EventArgs $eventArgs) - { - parent::postRemove($eventArgs); - } } From 1d700261abb7b7f39c56550cac0732e807240b35 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Mon, 2 Dec 2013 12:42:04 +0000 Subject: [PATCH 03/25] Refactor to a single Listener class. Update tests. --- Doctrine/AbstractListener.php | 228 ------------------------ Doctrine/MongoDB/Listener.php | 24 --- Doctrine/ORM/Listener.php | 24 --- Resources/config/mongodb.xml | 2 +- Resources/config/orm.xml | 2 +- Tests/Doctrine/AbstractListenerTest.php | 75 +++++--- Tests/Doctrine/MongoDB/ListenerTest.php | 6 +- Tests/Doctrine/ORM/ListenerTest.php | 6 +- 8 files changed, 57 insertions(+), 310 deletions(-) delete mode 100644 Doctrine/AbstractListener.php delete mode 100644 Doctrine/MongoDB/Listener.php delete mode 100644 Doctrine/ORM/Listener.php diff --git a/Doctrine/AbstractListener.php b/Doctrine/AbstractListener.php deleted file mode 100644 index 0fa15df..0000000 --- a/Doctrine/AbstractListener.php +++ /dev/null @@ -1,228 +0,0 @@ -objectPersister = $objectPersister; - $this->objectClass = $objectClass; - $this->events = $events; - $this->esIdentifierField = $esIdentifierField; - } - - /** - * @see Doctrine\Common\EventSubscriber::getSubscribedEvents() - */ - public function getSubscribedEvents() - { - return $this->events; - } - - /** - * Set the callback for determining object index eligibility. - * - * If callback is a string, it must be public method on the object class - * that expects no arguments and returns a boolean. Otherwise, the callback - * should expect the object for consideration as its only argument and - * return a boolean. - * - * @param callback $callback - * @throws \RuntimeException if the callback is not callable - */ - public function setIsIndexableCallback($callback) - { - if (is_string($callback)) { - if (!is_callable(array($this->objectClass, $callback))) { - if (false !== ($expression = $this->getExpressionLanguage())) { - $callback = new Expression($callback); - try { - $expression->compile($callback, array($this->getExpressionVar())); - } catch (SyntaxError $e) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e); - } - } else { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback)); - } - } - } elseif (!is_callable($callback)) { - if (is_array($callback)) { - list($class, $method) = $callback + array(null, null); - if (is_object($class)) { - $class = get_class($class); - } - - if ($class && $method) { - throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method)); - } - } - throw new \RuntimeException('Indexable callback is not callable.'); - } - - $this->isIndexableCallback = $callback; - } - - /** - * Return whether the object is indexable with respect to the callback. - * - * @param object $object - * @return boolean - */ - protected function isObjectIndexable($object) - { - if (!$this->isIndexableCallback) { - return true; - } - - if ($this->isIndexableCallback instanceof Expression) { - return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object)); - } - - return is_string($this->isIndexableCallback) - ? call_user_func(array($object, $this->isIndexableCallback)) - : call_user_func($this->isIndexableCallback, $object); - } - - /** - * @param mixed $object - * @return string - */ - private function getExpressionVar($object = null) - { - $class = $object ?: $this->objectClass; - $ref = new \ReflectionClass($class); - - return strtolower($ref->getShortName()); - } - - /** - * @return bool|ExpressionLanguage - */ - private function getExpressionLanguage() - { - if (null === $this->expressionLanguage) { - if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { - return false; - } - - $this->expressionLanguage = new ExpressionLanguage(); - } - - return $this->expressionLanguage; - } - - public function postPersist(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { - $this->scheduledForInsertion[] = $entity; - } - } - - public function postUpdate(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - if ($this->isObjectIndexable($entity)) { - $this->scheduledForUpdate[] = $entity; - } else { - // Delete if no longer indexable - $this->scheduledForDeletion[] = $entity; - } - } - } - - public function preRemove(EventArgs $eventArgs) - { - $entity = $eventArgs->getEntity(); - - if ($entity instanceof $this->objectClass) { - $this->scheduledForDeletion[] = $entity; - } - } - - /** - * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected - * only if the query is successful - */ - public function postFlush(EventArgs $eventArgs) - { - foreach ($this->scheduledForInsertion as $entity) { - $this->objectPersister->insertOne($entity); - } - foreach ($this->scheduledForUpdate as $entity) { - $this->objectPersister->replaceOne($entity); - } - foreach ($this->scheduledForDeletion as $entity) { - $this->objectPersister->deleteOne($entity); - } - } -} diff --git a/Doctrine/MongoDB/Listener.php b/Doctrine/MongoDB/Listener.php deleted file mode 100644 index 13e1d25..0000000 --- a/Doctrine/MongoDB/Listener.php +++ /dev/null @@ -1,24 +0,0 @@ - - + diff --git a/Resources/config/orm.xml b/Resources/config/orm.xml index 4fd6ae7..5bd16e5 100644 --- a/Resources/config/orm.xml +++ b/Resources/config/orm.xml @@ -13,7 +13,7 @@ - + diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index e99e26d..a6ad2aa 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -3,9 +3,11 @@ namespace FOS\ElasticaBundle\Tests\Doctrine; /** + * See concrete MongoDB/ORM instances of this abstract test + * * @author Richard Miller */ -abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase +abstract class ListenerTest extends \PHPUnit_Framework_TestCase { public function testObjectInsertedOnPersist() { @@ -14,12 +16,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $listener = $this->createListener($persister, get_class($entity), array()); + $listener->postPersist($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForInsertion)); + $persister->expects($this->once()) ->method('insertOne') ->with($entity); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->postPersist($eventArgs); + $listener->postFlush($eventArgs); } /** @@ -32,12 +38,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1, false); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); - $persister->expects($this->never()) - ->method('insertOne'); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->setIsIndexableCallback($isIndexableCallback); $listener->postPersist($eventArgs); + + $this->assertEmpty($listener->scheduledForInsertion); + + $persister->expects($this->never()) + ->method('insertOne'); + + $listener->postFlush($eventArgs); } public function testObjectReplacedOnUpdate() @@ -47,15 +57,18 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase $entity = new Listener\Entity(1); $eventArgs = $this->createLifecycleEventArgs($entity, $this->getMockObjectManager()); + $listener = $this->createListener($persister, get_class($entity), array()); + $listener->postUpdate($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForUpdate)); + $persister->expects($this->once()) ->method('replaceOne') ->with($entity); - $persister->expects($this->never()) ->method('deleteById'); - $listener = $this->createListener($persister, get_class($entity), array()); - $listener->postUpdate($eventArgs); + $listener->postFlush($eventArgs); } /** @@ -80,16 +93,20 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $persister->expects($this->never()) - ->method('replaceOne'); - - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->setIsIndexableCallback($isIndexableCallback); $listener->postUpdate($eventArgs); + + $this->assertEmpty($listener->scheduledForUpdate); + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->never()) + ->method('replaceOne'); + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } public function testObjectDeletedOnRemove() @@ -111,13 +128,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'id') ->will($this->returnValue($entity->getId())); - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array()); $listener->preRemove($eventArgs); - $listener->postRemove($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } public function testObjectWithNonStandardIdentifierDeletedOnRemove() @@ -139,13 +159,16 @@ abstract class AbstractListenerTest extends \PHPUnit_Framework_TestCase ->with($entity, 'identifier') ->will($this->returnValue($entity->getId())); - $persister->expects($this->once()) - ->method('deleteById') - ->with($entity->getId()); - $listener = $this->createListener($persister, get_class($entity), array(), 'identifier'); $listener->preRemove($eventArgs); - $listener->postRemove($eventArgs); + + $this->assertEquals($entity, current($listener->scheduledForDeletion)); + + $persister->expects($this->once()) + ->method('deleteOne') + ->with($entity); + + $listener->postFlush($eventArgs); } /** diff --git a/Tests/Doctrine/MongoDB/ListenerTest.php b/Tests/Doctrine/MongoDB/ListenerTest.php index 7f1a9ab..37a0203 100644 --- a/Tests/Doctrine/MongoDB/ListenerTest.php +++ b/Tests/Doctrine/MongoDB/ListenerTest.php @@ -2,9 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Doctrine\MongoDB; -use FOS\ElasticaBundle\Tests\Doctrine\AbstractListenerTest; +use FOS\ElasticaBundle\Tests\Doctrine\ListenerTest as BaseListenerTest; -class ListenerTest extends AbstractListenerTest +class ListenerTest extends BaseListenerTest { public function setUp() { @@ -25,7 +25,7 @@ class ListenerTest extends AbstractListenerTest protected function getListenerClass() { - return 'FOS\ElasticaBundle\Doctrine\MongoDB\Listener'; + return 'FOS\ElasticaBundle\Doctrine\Listener'; } protected function getObjectManagerClass() diff --git a/Tests/Doctrine/ORM/ListenerTest.php b/Tests/Doctrine/ORM/ListenerTest.php index 48702c0..12a89b2 100644 --- a/Tests/Doctrine/ORM/ListenerTest.php +++ b/Tests/Doctrine/ORM/ListenerTest.php @@ -2,9 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Doctrine\ORM; -use FOS\ElasticaBundle\Tests\Doctrine\AbstractListenerTest; +use FOS\ElasticaBundle\Tests\Doctrine\ListenerTest as BaseListenerTest; -class ListenerTest extends AbstractListenerTest +class ListenerTest extends BaseListenerTest { public function setUp() { @@ -25,7 +25,7 @@ class ListenerTest extends AbstractListenerTest protected function getListenerClass() { - return 'FOS\ElasticaBundle\Doctrine\ORM\Listener'; + return 'FOS\ElasticaBundle\Doctrine\Listener'; } protected function getObjectManagerClass() From af2827df016feea3463ba911454f12ab3b1e6b00 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 14:01:21 +0000 Subject: [PATCH 04/25] Re-add renamed Listener. --- Doctrine/Listener.php | 228 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 Doctrine/Listener.php diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php new file mode 100644 index 0000000..6584207 --- /dev/null +++ b/Doctrine/Listener.php @@ -0,0 +1,228 @@ +objectPersister = $objectPersister; + $this->objectClass = $objectClass; + $this->events = $events; + $this->esIdentifierField = $esIdentifierField; + } + + /** + * @see Doctrine\Common\EventSubscriber::getSubscribedEvents() + */ + public function getSubscribedEvents() + { + return $this->events; + } + + /** + * Set the callback for determining object index eligibility. + * + * If callback is a string, it must be public method on the object class + * that expects no arguments and returns a boolean. Otherwise, the callback + * should expect the object for consideration as its only argument and + * return a boolean. + * + * @param callback $callback + * @throws \RuntimeException if the callback is not callable + */ + public function setIsIndexableCallback($callback) + { + if (is_string($callback)) { + if (!is_callable(array($this->objectClass, $callback))) { + if (false !== ($expression = $this->getExpressionLanguage())) { + $callback = new Expression($callback); + try { + $expression->compile($callback, array($this->getExpressionVar())); + } catch (SyntaxError $e) { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable or a valid expression.', $this->objectClass, $callback), 0, $e); + } + } else { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $this->objectClass, $callback)); + } + } + } elseif (!is_callable($callback)) { + if (is_array($callback)) { + list($class, $method) = $callback + array(null, null); + if (is_object($class)) { + $class = get_class($class); + } + + if ($class && $method) { + throw new \RuntimeException(sprintf('Indexable callback %s::%s() is not callable.', $class, $method)); + } + } + throw new \RuntimeException('Indexable callback is not callable.'); + } + + $this->isIndexableCallback = $callback; + } + + /** + * Return whether the object is indexable with respect to the callback. + * + * @param object $object + * @return boolean + */ + protected function isObjectIndexable($object) + { + if (!$this->isIndexableCallback) { + return true; + } + + if ($this->isIndexableCallback instanceof Expression) { + return $this->getExpressionLanguage()->evaluate($this->isIndexableCallback, array($this->getExpressionVar($object) => $object)); + } + + return is_string($this->isIndexableCallback) + ? call_user_func(array($object, $this->isIndexableCallback)) + : call_user_func($this->isIndexableCallback, $object); + } + + /** + * @param mixed $object + * @return string + */ + private function getExpressionVar($object = null) + { + $class = $object ?: $this->objectClass; + $ref = new \ReflectionClass($class); + + return strtolower($ref->getShortName()); + } + + /** + * @return bool|ExpressionLanguage + */ + private function getExpressionLanguage() + { + if (null === $this->expressionLanguage) { + if (!class_exists('Symfony\Component\ExpressionLanguage\ExpressionLanguage')) { + return false; + } + + $this->expressionLanguage = new ExpressionLanguage(); + } + + return $this->expressionLanguage; + } + + public function postPersist(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass && $this->isObjectIndexable($entity)) { + $this->scheduledForInsertion[] = $entity; + } + } + + public function postUpdate(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + if ($this->isObjectIndexable($entity)) { + $this->scheduledForUpdate[] = $entity; + } else { + // Delete if no longer indexable + $this->scheduledForDeletion[] = $entity; + } + } + } + + public function preRemove(EventArgs $eventArgs) + { + $entity = $eventArgs->getEntity(); + + if ($entity instanceof $this->objectClass) { + $this->scheduledForDeletion[] = $entity; + } + } + + /** + * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + foreach ($this->scheduledForInsertion as $entity) { + $this->objectPersister->insertOne($entity); + } + foreach ($this->scheduledForUpdate as $entity) { + $this->objectPersister->replaceOne($entity); + } + foreach ($this->scheduledForDeletion as $entity) { + $this->objectPersister->deleteOne($entity); + } + } +} From 22a5d67d05d2df4fe46ca6e80f4b21198f2d3b19 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 20:41:26 +0000 Subject: [PATCH 05/25] pre/postFlush configuration. Update documentation. --- DependencyInjection/Configuration.php | 2 ++ DependencyInjection/FOSElasticaExtension.php | 6 +++-- Doctrine/Listener.php | 23 +++++++++++++++++--- README.md | 22 ++++++++++++++++--- UPGRADE-3.0.md | 11 ++++++++++ 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 UPGRADE-3.0.md diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index fe7e9a4..50d4b89 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -165,6 +165,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->scalarNode('persist')->defaultValue('postFlush')->end() ->scalarNode('service')->end() ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() @@ -257,6 +258,7 @@ class Configuration implements ConfigurationInterface ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() + ->booleanNode('immediate')->defaultFalse()->end() ->scalarNode('service')->end() ->variableNode('is_indexable_callback')->defaultNull()->end() ->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 30ad07a..4a7ebbc 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -405,7 +405,7 @@ class FOSElasticaExtension extends Extension private function getDoctrineEvents(array $typeConfig) { - // Flush event always fires; not configurable + // Flush always calls depending on actions scheduled in lifecycle listeners $typeConfig['listener']['flush'] = true; $events = array(); @@ -413,9 +413,11 @@ class FOSElasticaExtension extends Extension 'insert' => array('postPersist'), 'update' => array('postUpdate'), 'delete' => array('preRemove'), - 'flush' => array('postFlush') + 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') ); + var_dump($eventMapping); + foreach ($eventMapping as $event => $doctrineEvents) { if (isset($typeConfig['listener'][$event]) && $typeConfig['listener'][$event]) { $events = array_merge($events, $doctrineEvents); diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 6584207..683070e 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -210,10 +210,9 @@ class Listener implements EventSubscriber } /** - * Iterate through scheduled actions *after* flushing to ensure that the ElasticSearch index will only be affected - * only if the query is successful + * Persist scheduled action to ElasticSearch */ - public function postFlush(EventArgs $eventArgs) + private function persistScheduled() { foreach ($this->scheduledForInsertion as $entity) { $this->objectPersister->insertOne($entity); @@ -225,4 +224,22 @@ class Listener implements EventSubscriber $this->objectPersister->deleteOne($entity); } } + + /** + * Iterate through scheduled actions before flushing to emulate 2.x behavior. Note that the ElasticSearch index + * will fall out of sync with the data source in event of a crash on flush. + */ + public function preFlush(EventArgs $eventArgs) + { + $this->persistScheduled(); + } + + /** + * Iterating through scheduled actions *after* flushing ensures that the ElasticSearch index will be affected + * only if the query is successful + */ + public function postFlush(EventArgs $eventArgs) + { + $this->persistScheduled(); + } } diff --git a/README.md b/README.md index 9792139..9f77a1b 100644 --- a/README.md +++ b/README.md @@ -576,7 +576,11 @@ class User ### Realtime, selective index update If you use the Doctrine integration, you can let ElasticaBundle update the indexes automatically -when an object is added, updated or removed. It uses Doctrine lifecycle events. +when an object is added, updated or removed. It uses Doctrine lifecycle events to schedule updates +and then synchronizes changes either before or after flush. + +> **Propel** doesn't support this feature yet. + Declare that you want to update the index in real time: fos_elastica: @@ -592,7 +596,7 @@ Declare that you want to update the index in real time: persistence: driver: orm model: Application\UserBundle\Entity\User - listener: ~ # by default, listens to "insert", "update" and "delete" + listener: ~ # by default, listens to "insert", "update" and "delete" and updates `postFlush` Now the index is automatically updated each time the state of the bound Doctrine repository changes. No need to repopulate the whole "user" index when a new `User` is created. @@ -605,7 +609,19 @@ You can also choose to only listen for some of the events: update: false delete: true -> **Propel** doesn't support this feature yet. +By default, the ElasticSearch index will be updated after flush. To update before flushing, set `immediate` +to `true`: + + persistence: + listener: + insert: true + update: false + delete: true + immediate: true + +> Updating ElasticSearch before flushing may cause the ElasticSearch index to fall out of sync with the +> original data in the event of a crash. + ### Checking an entity method for listener diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md new file mode 100644 index 0000000..0b0d370 --- /dev/null +++ b/UPGRADE-3.0.md @@ -0,0 +1,11 @@ +UPGRADE FROM 2.x to 3.0 +======================= + +### ElasticSearch Synchronization Event + + * Prior to 3.0, the ElasticSearch index was synchronized in the `postInsert`, + `postUpdate`, and `pre/postRemove` events which fire before flush. Because + of this, exceptions thrown when flushing would cause the data source and + ElasticSearch index to fall out of sync. + + As of 3.0, ElasticSearch is updated `postFlush` by default. From ed21e60869e86a5372f3d1a69ee97c4f61c65829 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Tue, 3 Dec 2013 23:40:00 +0000 Subject: [PATCH 06/25] Remove upgrade doc. Merk will add manually. --- UPGRADE-3.0.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 UPGRADE-3.0.md diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md deleted file mode 100644 index 0b0d370..0000000 --- a/UPGRADE-3.0.md +++ /dev/null @@ -1,11 +0,0 @@ -UPGRADE FROM 2.x to 3.0 -======================= - -### ElasticSearch Synchronization Event - - * Prior to 3.0, the ElasticSearch index was synchronized in the `postInsert`, - `postUpdate`, and `pre/postRemove` events which fire before flush. Because - of this, exceptions thrown when flushing would cause the data source and - ElasticSearch index to fall out of sync. - - As of 3.0, ElasticSearch is updated `postFlush` by default. From 3a279f8edbff1eed66c224516904917725ccfc6b Mon Sep 17 00:00:00 2001 From: nurikabe Date: Wed, 4 Dec 2013 10:46:05 +0000 Subject: [PATCH 07/25] Remove debug. --- DependencyInjection/FOSElasticaExtension.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 4a7ebbc..6380873 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -416,8 +416,6 @@ class FOSElasticaExtension extends Extension 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') ); - var_dump($eventMapping); - foreach ($eventMapping as $event => $doctrineEvents) { if (isset($typeConfig['listener'][$event]) && $typeConfig['listener'][$event]) { $events = array_merge($events, $doctrineEvents); From 3bd9155f467b6d46896d88d29050ddc593c53324 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Fri, 13 Dec 2013 17:51:15 +0000 Subject: [PATCH 08/25] Use constants of corresponding events classes rather than making assumption about string values. --- DependencyInjection/FOSElasticaExtension.php | 23 ++++++++++++++++---- Doctrine/Listener.php | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 6380873..733a039 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -403,17 +403,32 @@ class FOSElasticaExtension extends Extension return $listenerId; } + /** + * Map Elastica to Doctrine events for the current driver + */ private function getDoctrineEvents(array $typeConfig) { // Flush always calls depending on actions scheduled in lifecycle listeners $typeConfig['listener']['flush'] = true; + switch ($typeConfig['driver']) { + case 'orm': + $eventsClass = '\Doctrine\ORM\Events'; + break; + case 'mongodb': + $eventsClass = '\Doctrine\ODM\MongoDB\Events'; + break; + default: + throw new InvalidArgumentException(sprintf('Cannot determine events for driver "%s"', $typeConfig['driver'])); + break; + } + $events = array(); $eventMapping = array( - 'insert' => array('postPersist'), - 'update' => array('postUpdate'), - 'delete' => array('preRemove'), - 'flush' => array($typeConfig['listener']['immediate'] ? 'preFlush' : 'postFlush') + 'insert' => array(constant($eventsClass.'::postPersist')), + 'update' => array(constant($eventsClass.'::postUpdate')), + 'delete' => array(constant($eventsClass.'::preRemove')), + 'flush' => array($typeConfig['listener']['immediate'] ? constant($eventsClass.'::preFlush') : constant($eventsClass.'::postFlush')) ); foreach ($eventMapping as $event => $doctrineEvents) { diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 683070e..383e02f 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -27,7 +27,7 @@ class Listener implements EventSubscriber protected $objectClass; /** - * List of su]bscribed events + * List of subscribed events * * @var array */ From 74813768358f42b3348e6f07a4b7a4179ee73259 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 11 Jan 2014 16:28:15 +0000 Subject: [PATCH 09/25] Use bulk insert. Still working on bulk update/delete which is not yet suppored in Elastica. --- Doctrine/Listener.php | 5 +++-- Tests/Doctrine/AbstractListenerTest.php | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 383e02f..8c1d99f 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -214,9 +214,10 @@ class Listener implements EventSubscriber */ private function persistScheduled() { - foreach ($this->scheduledForInsertion as $entity) { - $this->objectPersister->insertOne($entity); + if (count($this->scheduledForInsertion)) { + $this->objectPersister->insertMany($this->scheduledForInsertion); } + foreach ($this->scheduledForUpdate as $entity) { $this->objectPersister->replaceOne($entity); } diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index a6ad2aa..3278be1 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -22,8 +22,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForInsertion)); $persister->expects($this->once()) - ->method('insertOne') - ->with($entity); + ->method('insertMany') + ->with($listener->scheduledForInsertion); $listener->postFlush($eventArgs); } @@ -46,6 +46,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $persister->expects($this->never()) ->method('insertOne'); + $persister->expects($this->never()) + ->method('insertMany'); $listener->postFlush($eventArgs); } From b6d010a9d7e7801c17cb618d8984d70eacff8f15 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Thu, 23 Jan 2014 16:20:11 +0000 Subject: [PATCH 10/25] Bulk update. Still working on bulk delete for indexes and types in Elastica. --- Doctrine/Listener.php | 8 ++------ Persister/ObjectPersister.php | 17 +++++++++++------ Persister/ObjectPersisterInterface.php | 10 +++++++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index 8c1d99f..d6314dc 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -214,13 +214,9 @@ class Listener implements EventSubscriber */ private function persistScheduled() { - if (count($this->scheduledForInsertion)) { - $this->objectPersister->insertMany($this->scheduledForInsertion); - } + $this->objectPersister->bulkPersist($this->scheduledForInsertion, ObjectPersisterInterface::BULK_INSERT); + $this->objectPersister->bulkPersist($this->scheduledForUpdate, ObjectPersisterInterface::BULK_REPLACE); - foreach ($this->scheduledForUpdate as $entity) { - $this->objectPersister->replaceOne($entity); - } foreach ($this->scheduledForDeletion as $entity) { $this->objectPersister->deleteOne($entity); } diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index 450e43b..b5a1be7 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -83,19 +83,24 @@ class ObjectPersister implements ObjectPersisterInterface } catch (NotFoundException $e) {} } - /** - * Inserts an array of objects in the type + * Bulk update an array of objects in the type for the given method * * @param array $objects array of domain model objects - **/ - public function insertMany(array $objects) + * @param string Method to call + */ + public function bulkPersist(array $objects, $method) { + if (!count($objects)) { + return; + } + $documents = array(); foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } - $this->type->addDocuments($documents); + + $this->type->$method($documents); } /** @@ -108,4 +113,4 @@ class ObjectPersister implements ObjectPersisterInterface { return $this->transformer->transform($object, $this->fields); } -} +} \ No newline at end of file diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index a50bcc8..5c4ecd2 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -10,6 +10,9 @@ namespace FOS\ElasticaBundle\Persister; */ interface ObjectPersisterInterface { + const BULK_INSERT = 'addDocuments'; + const BULK_REPLACE = 'updateDocuments'; + /** * Insert one object into the type * The object will be transformed to an elastica document @@ -42,9 +45,10 @@ interface ObjectPersisterInterface function deleteById($id); /** - * Inserts an array of objects in the type + * Bulk update an array of objects in the type for the given method * * @param array $objects array of domain model objects - **/ - function insertMany(array $objects); + * @param string Method to call + */ + function bulkPersist(array $objects, $method); } From 1dcaadbe6f99b42b346e04239c0c8fd7812bbf51 Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 1 Feb 2014 02:14:21 +0000 Subject: [PATCH 11/25] Bulk delete --- Doctrine/Listener.php | 17 +++++++----- Persister/ObjectPersister.php | 37 +++++++++++++++++++++----- Persister/ObjectPersisterInterface.php | 24 ++++++++++++----- README.md | 4 +-- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/Doctrine/Listener.php b/Doctrine/Listener.php index d6314dc..c254513 100644 --- a/Doctrine/Listener.php +++ b/Doctrine/Listener.php @@ -210,21 +210,24 @@ class Listener implements EventSubscriber } /** - * Persist scheduled action to ElasticSearch + * Persist scheduled objects to ElasticSearch */ private function persistScheduled() { - $this->objectPersister->bulkPersist($this->scheduledForInsertion, ObjectPersisterInterface::BULK_INSERT); - $this->objectPersister->bulkPersist($this->scheduledForUpdate, ObjectPersisterInterface::BULK_REPLACE); - - foreach ($this->scheduledForDeletion as $entity) { - $this->objectPersister->deleteOne($entity); + if (count($this->scheduledForInsertion)) { + $this->objectPersister->insertMany($this->scheduledForInsertion); + } + if (count($this->scheduledForUpdate)) { + $this->objectPersister->replaceMany($this->scheduledForUpdate); + } + if (count($this->scheduledForDeletion)) { + $this->objectPersister->deleteMany($this->scheduledForDeletion); } } /** * Iterate through scheduled actions before flushing to emulate 2.x behavior. Note that the ElasticSearch index - * will fall out of sync with the data source in event of a crash on flush. + * will fall out of sync with the source data in the event of a crash during flush. */ public function preFlush(EventArgs $eventArgs) { diff --git a/Persister/ObjectPersister.php b/Persister/ObjectPersister.php index b5a1be7..3592a78 100644 --- a/Persister/ObjectPersister.php +++ b/Persister/ObjectPersister.php @@ -84,23 +84,46 @@ class ObjectPersister implements ObjectPersisterInterface } /** - * Bulk update an array of objects in the type for the given method + * Bulk insert an array of objects in the type for the given method * * @param array $objects array of domain model objects * @param string Method to call */ - public function bulkPersist(array $objects, $method) + public function insertMany(array $objects) { - if (!count($objects)) { - return; - } - $documents = array(); foreach ($objects as $object) { $documents[] = $this->transformToElasticaDocument($object); } + $this->type->addDocuments($documents); + } - $this->type->$method($documents); + /** + * Bulk updates an array of objects in the type + * + * @param array $objects array of domain model objects + */ + public function replaceMany(array $objects) + { + $documents = array(); + foreach ($objects as $object) { + $documents[] = $this->transformToElasticaDocument($object); + } + $this->type->updateDocuments($documents); + } + + /** + * Bulk deletes an array of objects in the type + * + * @param array $objects array of domain model objects + */ + public function deleteMany(array $objects) + { + $documents = array(); + foreach ($objects as $object) { + $documents[] = $this->transformToElasticaDocument($object); + } + $this->type->deleteDocuments($documents); } /** diff --git a/Persister/ObjectPersisterInterface.php b/Persister/ObjectPersisterInterface.php index 5c4ecd2..a25aafc 100644 --- a/Persister/ObjectPersisterInterface.php +++ b/Persister/ObjectPersisterInterface.php @@ -10,9 +10,6 @@ namespace FOS\ElasticaBundle\Persister; */ interface ObjectPersisterInterface { - const BULK_INSERT = 'addDocuments'; - const BULK_REPLACE = 'updateDocuments'; - /** * Insert one object into the type * The object will be transformed to an elastica document @@ -41,14 +38,27 @@ interface ObjectPersisterInterface * @param mixed $id * * @return null - **/ + */ function deleteById($id); /** - * Bulk update an array of objects in the type for the given method + * Bulk inserts an array of objects in the type * * @param array $objects array of domain model objects - * @param string Method to call */ - function bulkPersist(array $objects, $method); + function insertMany(array $objects); + + /** + * Bulk updates an array of objects in the type + * + * @param array $objects array of domain model objects + */ + function replaceMany(array $objects); + + /** + * Bulk deletes an array of objects in the type + * + * @param array $objects array of domain model objects + */ + function deleteMany(array $objects); } diff --git a/README.md b/README.md index b3a4c5d..cb6e5f8 100644 --- a/README.md +++ b/README.md @@ -627,8 +627,8 @@ to `true`: delete: true immediate: true -> Updating ElasticSearch before flushing may cause the ElasticSearch index to fall out of sync with the -> original data in the event of a crash. +> Using `immediate` to update ElasticSearch before flush completes may cause the ElasticSearch index to fall out of +> sync with the source database in the event of a crash during the flush itself, such as in the case of a bad query. ### Checking an entity method for listener From 559d7c13f277d534fbf883ae5d989c0fe4e1adce Mon Sep 17 00:00:00 2001 From: nurikabe Date: Sat, 1 Feb 2014 03:37:47 +0000 Subject: [PATCH 12/25] Update tests --- Tests/Doctrine/AbstractListenerTest.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tests/Doctrine/AbstractListenerTest.php b/Tests/Doctrine/AbstractListenerTest.php index 3278be1..a9eff66 100644 --- a/Tests/Doctrine/AbstractListenerTest.php +++ b/Tests/Doctrine/AbstractListenerTest.php @@ -65,8 +65,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForUpdate)); $persister->expects($this->once()) - ->method('replaceOne') - ->with($entity); + ->method('replaceMany') + ->with(array($entity)); $persister->expects($this->never()) ->method('deleteById'); @@ -105,8 +105,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $persister->expects($this->never()) ->method('replaceOne'); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } @@ -136,8 +136,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } @@ -167,8 +167,8 @@ abstract class ListenerTest extends \PHPUnit_Framework_TestCase $this->assertEquals($entity, current($listener->scheduledForDeletion)); $persister->expects($this->once()) - ->method('deleteOne') - ->with($entity); + ->method('deleteMany') + ->with(array($entity)); $listener->postFlush($eventArgs); } From 3b1a756e6f89e828b2f9f2c5b280d0f86a72637c Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Thu, 6 Feb 2014 21:11:12 +0000 Subject: [PATCH 13/25] Add support for using aliases to allow hot swapping of indexes when populating --- Client.php | 5 + Command/PopulateCommand.php | 1 + DependencyInjection/Configuration.php | 5 +- DependencyInjection/FOSElasticaExtension.php | 5 + DynamicIndex.php | 28 ++++ Resetter.php | 136 ++++++++++++++++++- Resources/config/config.xml | 2 +- 7 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 DynamicIndex.php diff --git a/Client.php b/Client.php index ea0c572..67252ec 100644 --- a/Client.php +++ b/Client.php @@ -32,4 +32,9 @@ class Client extends ElasticaClient return $response; } + + public function getIndex($name) + { + return new DynamicIndex($this, $name); + } } diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7297523..31ecc4b 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -124,6 +124,7 @@ class PopulateCommand extends ContainerAwareCommand } $output->writeln(sprintf('Refreshing %s', $index)); + $this->resetter->postPopulate($index); $this->indexManager->getIndex($index)->refresh(); } diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4ed88af..403de85 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -134,6 +134,7 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->children() ->scalarNode('index_name')->end() + ->booleanNode('use_alias')->defaultValue(false)->end() ->scalarNode('client')->end() ->scalarNode('finder') ->treatNullLike(true) @@ -670,7 +671,7 @@ class Configuration implements ConfigurationInterface return $node; } - + /** * Returns the array node used for "_ttl" */ @@ -689,5 +690,5 @@ class Configuration implements ConfigurationInterface ; return $node; - } + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 52070fd..a67be0f 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -124,6 +124,7 @@ class FOSElasticaExtension extends Extension $indexIds[$name] = $indexId; $this->indexConfigs[$name] = array( 'index' => new Reference($indexId), + 'name_or_alias' => $indexName, 'config' => array( 'mappings' => array() ) @@ -134,6 +135,10 @@ class FOSElasticaExtension extends Extension if (!empty($index['settings'])) { $this->indexConfigs[$name]['config']['settings'] = $index['settings']; } + if ($index['use_alias']) { + $this->indexConfigs[$name]['use_alias'] = true; + } + $this->loadTypes(isset($index['types']) ? $index['types'] : array(), $container, $name, $indexId, $typePrototypeConfig); } diff --git a/DynamicIndex.php b/DynamicIndex.php new file mode 100644 index 0000000..cbec8e9 --- /dev/null +++ b/DynamicIndex.php @@ -0,0 +1,28 @@ + + */ +class DynamicIndex extends Index +{ + /** + * Reassign index name + * + * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName + * since it's used for a very specific case and normally should not be used + * + * @param string $name Index name + * + * @return void + */ + public function overrideName($name) + { + $this->_name = $name; + } +} diff --git a/Resetter.php b/Resetter.php index 26a6bb5..b8e7b80 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,6 +2,8 @@ namespace FOS\ElasticaBundle; +use Elastica\Exception\ExceptionInterface; +use Elastica\Index; use Elastica\Type\Mapping; /** @@ -26,8 +28,8 @@ class Resetter */ public function resetAllIndexes() { - foreach ($this->indexConfigsByName as $indexConfig) { - $indexConfig['index']->create($indexConfig['config'], true); + foreach (array_keys($this->indexConfigsByName) as $name) { + $this->resetIndex($name); } } @@ -40,7 +42,17 @@ class Resetter public function resetIndex($indexName) { $indexConfig = $this->getIndexConfig($indexName); - $indexConfig['index']->create($indexConfig['config'], true); + $esIndex = $indexConfig['index']; + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $name = $indexConfig['name_or_alias']; + $name .= date('-Y-m-d-Gis'); + $esIndex->overrideName($name); + $esIndex->create($indexConfig['config']); + + return; + } + + $esIndex->create($indexConfig['config'], true); } /** @@ -102,4 +114,122 @@ class Resetter return $this->indexConfigsByName[$indexName]; } + + public function postPopulate($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $this->switchIndexAlias($indexName); + } + } + + /** + * Switches the alias for given index to the newly populated index + * and deletes the old index + * + * @param string $indexName Index name + * + * @throws \RuntimeException + */ + private function switchIndexAlias($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + $esIndex = $indexConfig['index']; + $aliasName = $indexConfig['name_or_alias']; + $oldIndexName = false; + $newIndexName = $esIndex->getName(); + + $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); + + if (count($aliasedIndexes) > 1) { + throw new \RuntimeException( + sprintf( + 'Alias %s is used for multiple indexes: [%s]. + Make sure it\'s either not used or is assigned to one index only', + $aliasName, + join(', ', $aliasedIndexes) + ) + ); + } + + // Change the alias to point to the new index + // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic + // In 0.20.x it's atomic, but it doesn't return the old index name + $aliasUpdateRequest = array('actions' => array()); + if (count($aliasedIndexes) == 1) { + // if the alias is set - add an action to remove it + $oldIndexName = $aliasedIndexes[0]; + $aliasUpdateRequest['actions'][] = array( + 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) + ); + } + + // add an action to point the alias to the new index + $aliasUpdateRequest['actions'][] = array( + 'add' => array('index' => $newIndexName, 'alias' => $aliasName) + ); + + try { + $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); + } catch (ExceptionInterface $renameAliasException) { + $additionalError = ''; + // if we failed to move the alias, delete the newly built index + try { + $esIndex->delete(); + } catch (ExceptionInterface $deleteNewIndexException) { + $additionalError = sprintf( + 'Tried to delete newly built index %s, but also failed: %s', + $newIndexName, + $deleteNewIndexException->getError() + ); + } + + throw new \RuntimeException( + sprintf( + 'Failed to updated index alias: %s. %s', + $renameAliasException->getMessage(), + $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) + ) + ); + } + + // Delete the old index after the alias has been switched + if ($oldIndexName) { + $oldIndex = new Index($esIndex->getClient(), $oldIndexName); + try { + $oldIndex->delete(); + } catch (ExceptionInterface $deleteOldIndexException) { + throw new \RuntimeException( + sprintf( + 'Failed to delete old index %s with message: %s', + $oldIndexName, + $deleteOldIndexException->getMessage() + ) + ); + } + } + } + + /** + * Returns array of indexes which are mapped to given alias + * + * @param Index $esIndex ES Index + * @param string $aliasName Alias name + * + * @return array + */ + private function getAliasedIndexes(Index $esIndex, $aliasName) + { + $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); + $aliasedIndexes = array(); + + foreach ($aliasesInfo as $indexName => $indexInfo) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } + } + + return $aliasedIndexes; + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 85cfbed..4419b4a 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -6,7 +6,7 @@ FOS\ElasticaBundle\Client - Elastica\Index + FOS\ElasticaBundle\DynamicIndex Elastica\Type FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector From 03bf793b656a11646ae74a351ce46dd3de74b045 Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Mon, 10 Feb 2014 10:55:13 +0000 Subject: [PATCH 14/25] Make real index name use uniqid so can be reset within a second --- Resetter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resetter.php b/Resetter.php index b8e7b80..4a7f13b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -45,7 +45,7 @@ class Resetter $esIndex = $indexConfig['index']; if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { $name = $indexConfig['name_or_alias']; - $name .= date('-Y-m-d-Gis'); + $name .= uniqid(); $esIndex->overrideName($name); $esIndex->create($indexConfig['config']); @@ -124,7 +124,7 @@ class Resetter } /** - * Switches the alias for given index to the newly populated index + * Switches the alias for given index (by key) to the newly populated index * and deletes the old index * * @param string $indexName Index name From eecdd3474a2ddb2b46dc98fffc4b4bb641dfaed6 Mon Sep 17 00:00:00 2001 From: Matteo Galli Date: Fri, 21 Feb 2014 16:47:42 +0100 Subject: [PATCH 15/25] Fixes #459 --- DependencyInjection/FOSElasticaExtension.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 52070fd..96e1fdf 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -4,6 +4,7 @@ namespace FOS\ElasticaBundle\DependencyInjection; use Symfony\Component\HttpKernel\DependencyInjection\Extension; use Symfony\Component\DependencyInjection\Loader\XmlFileLoader; +use Symfony\Component\DependencyInjection\ContainerAwareInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\DefinitionDecorator; @@ -198,6 +199,10 @@ class FOSElasticaExtension extends Extension if (isset($type['serializer']['version'])) { $callbackDef->addMethodCall('setVersion', array($type['serializer']['version'])); } + $callbackClassImplementedInterfaces = class_implements($this->serializerConfig['callback_class']); // PHP < 5.4 friendly + if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) { + $callbackDef->addMethodCall('setContainer', array(new Reference('service_container'))); + } $container->setDefinition($callbackId, $callbackDef); From 2b04f6cf34807854226794ff78dd3f4f207f73ed Mon Sep 17 00:00:00 2001 From: Karel Souffriau Date: Tue, 25 Feb 2014 11:05:26 +0100 Subject: [PATCH 16/25] ElasticaLogger cleanup --- Logger/ElasticaLogger.php | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Logger/ElasticaLogger.php b/Logger/ElasticaLogger.php index 0e0a9c0..5b1269f 100644 --- a/Logger/ElasticaLogger.php +++ b/Logger/ElasticaLogger.php @@ -14,32 +14,42 @@ use Psr\Log\LoggerInterface; */ class ElasticaLogger implements LoggerInterface { + /** + * @var LoggerInterface + */ protected $logger; - protected $queries; + + /** + * @var array + */ + protected $queries = array(); + + /** + * @var boolean + */ protected $debug; /** * Constructor. * * @param LoggerInterface|null $logger The Symfony logger - * @param bool $debug + * @param boolean $debug */ public function __construct(LoggerInterface $logger = null, $debug = false) { $this->logger = $logger; - $this->queries = array(); $this->debug = $debug; } /** * Logs a query. * - * @param string $path Path to call - * @param string $method Rest method to use (GET, POST, DELETE, PUT) - * @param array $data arguments - * @param float $time execution time - * @param array $connection host, port and transport of the query - * @param array $query arguments + * @param string $path Path to call + * @param string $method Rest method to use (GET, POST, DELETE, PUT) + * @param array $data Arguments + * @param float $time Execution time + * @param array $connection Host, port and transport of the query + * @param array $query Arguments */ public function logQuery($path, $method, $data, $time, $connection = array(), $query = array()) { From a1f7449efc7e8190726812f987e6cba98e4bccbe Mon Sep 17 00:00:00 2001 From: Bastien Jaillot Date: Wed, 5 Mar 2014 15:59:38 +0100 Subject: [PATCH 17/25] [doc] inform about the override of Elastica logger Add a documentation related to this change: https://github.com/FriendsOfSymfony/FOSElasticaBundle/pull/395#issuecomment-27729759 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f78c97f..aaaf0a5 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Most of the time, you will need only one. clients: default: { host: localhost, port: 9200 } +A client configuration can also override the Elastica logger to change the used class ```logger: ``` or to simply disable it ```logger: false```. Disabling the logger should be done on production because it can cause a memory leak. #### Declare a serializer From 7f53badad545a9e1d5b04a9b02c238fb7a5e30b7 Mon Sep 17 00:00:00 2001 From: Berny Cantos Date: Thu, 5 Sep 2013 13:45:36 +0200 Subject: [PATCH 18/25] Add support for include_in_{parent,root} for nested and objects --- DependencyInjection/Configuration.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4488bf2..adcaf6a 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -415,6 +415,10 @@ class Configuration implements ConfigurationInterface } if (isset($nestings['properties'])) { + $node + ->booleanNode('include_in_parent')->end() + ->booleanNode('include_in_root')->end() + ; $this->addNestedFieldConfig($node, $nestings, 'properties'); } } From fe871c5ac48d2be18a2efef46cd2b14ca001fe1a Mon Sep 17 00:00:00 2001 From: Miha Vrhovnik Date: Tue, 11 Mar 2014 14:50:00 +0100 Subject: [PATCH 19/25] Fix wrong annotation --- Paginator/RawPaginatorAdapter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Paginator/RawPaginatorAdapter.php b/Paginator/RawPaginatorAdapter.php index e99746f..b74a9e4 100644 --- a/Paginator/RawPaginatorAdapter.php +++ b/Paginator/RawPaginatorAdapter.php @@ -33,7 +33,7 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface private $totalHits; /** - * @array for the facets + * @var array for the facets */ private $facets; From 726892c586949b8cf13ccca7fa7c1f3c3ca43f2d Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Mar 2014 10:43:35 +0100 Subject: [PATCH 20/25] Ignore TypeMissingException when resetting a single type. This allows to create new types without having to recreate the whole index. --- Resetter.php | 9 ++++++++- Tests/ResetterTest.php | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index 69620a2..b22f6a1 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,6 +2,7 @@ namespace FOS\ElasticaBundle; +use Elastica\Exception\ResponseException; use Elastica\Type\Mapping; /** @@ -59,7 +60,13 @@ class Resetter } $type = $indexConfig['index']->getType($typeName); - $type->delete(); + try { + $type->delete(); + } catch (ResponseException $e) { + if (strpos($e->getMessage(), 'TypeMissingException') !== 0) { + throw $e; + } + } $mapping = $this->createMapping($indexConfig['config']['mappings'][$typeName]); $type->setMapping($mapping); } diff --git a/Tests/ResetterTest.php b/Tests/ResetterTest.php index aa0fbcc..b4e5649 100644 --- a/Tests/ResetterTest.php +++ b/Tests/ResetterTest.php @@ -2,6 +2,9 @@ namespace FOS\ElasticaBundle\Tests\Resetter; +use Elastica\Exception\ResponseException; +use Elastica\Request; +use Elastica\Response; use FOS\ElasticaBundle\Resetter; use Elastica\Type\Mapping; @@ -130,6 +133,32 @@ class ResetterTest extends \PHPUnit_Framework_TestCase $resetter->resetIndexType('foo', 'c'); } + public function testResetIndexTypeIgnoreTypeMissingException() + { + $type = $this->getMockElasticaType(); + + $this->indexConfigsByName['foo']['index']->expects($this->once()) + ->method('getType') + ->with('a') + ->will($this->returnValue($type)); + + $type->expects($this->once()) + ->method('delete') + ->will($this->throwException(new ResponseException( + new Request(''), + new Response(array('error' => 'TypeMissingException[[de_20131022] type[bla] missing]', 'status' => 404))) + )); + + $mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']); + $mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']); + $type->expects($this->once()) + ->method('setMapping') + ->with($mapping); + + $resetter = new Resetter($this->indexConfigsByName); + $resetter->resetIndexType('foo', 'a'); + } + public function testIndexMappingForParent() { $type = $this->getMockElasticaType(); From 2f9896c893063b7142a0ac1d2980e05c9050f2df Mon Sep 17 00:00:00 2001 From: Lea Haensenberger Date: Thu, 13 Mar 2014 10:45:57 +0100 Subject: [PATCH 21/25] Check for TypeMissingException anywhere in exception message --- Resetter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resetter.php b/Resetter.php index b22f6a1..863524b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -63,7 +63,7 @@ class Resetter try { $type->delete(); } catch (ResponseException $e) { - if (strpos($e->getMessage(), 'TypeMissingException') !== 0) { + if (strpos($e->getMessage(), 'TypeMissingException') === false) { throw $e; } } From cdd6e3af45a7fb9f97b53887416e0b3511763452 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Fri, 14 Mar 2014 08:59:59 +1100 Subject: [PATCH 22/25] Bump elastica version --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 094eba5..e588494 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", - "ruflin/elastica": "~0.20", + "ruflin/elastica": ">=0.20, <1.1-dev", "psr/log": "~1.0" }, "require-dev":{ From dc01b189ed8ecb3670bd205fa2c8940547e60c40 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Mon, 17 Mar 2014 09:26:16 +1100 Subject: [PATCH 23/25] Update CHANGELOG-3.0.md --- CHANGELOG-3.0.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 534a25e..eb3e8c8 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -12,6 +12,14 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1 To generate a changelog summary since the last version, run `git log --no-merges --oneline v3.0.0...3.0.x` -* 3.0.0-ALPHA2 (2014-xx-xx) +* 3.0.0-ALPHA3 (xxxx-xx-xx) + + * #463: allowing hot swappable reindexing + * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. + +* 3.0.0-ALPHA2 (2014-03-17) * 41bf07e: Renamed the `no-stop-on-error` option in PopulateCommand to `ignore-errors` + * 418b9d7: Fixed validation of url configuration + * 726892c: Ignore TypeMissingException when resetting a single type. This allows to create new types without having to recreate the whole index. + * 7f53bad Add support for include_in_{parent,root} for nested and objects From 7d13823488c414cd42f794c1f3cacb82b6c96542 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 18 Mar 2014 09:10:44 +1100 Subject: [PATCH 24/25] Bump symfony requirement --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e588494..8f22393 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,7 @@ ], "require": { "php": ">=5.3.2", - "symfony/framework-bundle": "~2.1", + "symfony/framework-bundle": "~2.3", "symfony/console": "~2.1", "symfony/form": "~2.1", "symfony/property-access": "~2.2", From babe20800e6bff5af8bfd0de54bb72175dca0ff4 Mon Sep 17 00:00:00 2001 From: Tim Nagel Date: Tue, 18 Mar 2014 09:12:11 +1100 Subject: [PATCH 25/25] Updated changelog --- CHANGELOG-3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index eb3e8c8..a80d09d 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -16,6 +16,7 @@ To generate a changelog summary since the last version, run * #463: allowing hot swappable reindexing * #415: BC BREAK: document indexing occurs in postFlush rather than the pre* events previously. + * 7d13823: Dropped (broken) support for Symfony <2.3 * 3.0.0-ALPHA2 (2014-03-17)