From 7fa7e44beea138994e49fc52633ffa701ecc58ac Mon Sep 17 00:00:00 2001
From: Gnucki <tprelot@gmail.com>
Date: Wed, 8 Oct 2014 10:44:49 +0200
Subject: [PATCH] Fix mongodb populate falling down performances for big
 collections

---
 Doctrine/AbstractProvider.php |  6 ++++--
 Doctrine/MongoDB/Provider.php | 14 ++++++++++++--
 Doctrine/ORM/Provider.php     |  2 +-
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/Doctrine/AbstractProvider.php b/Doctrine/AbstractProvider.php
index a662fd4..d897c96 100644
--- a/Doctrine/AbstractProvider.php
+++ b/Doctrine/AbstractProvider.php
@@ -54,12 +54,13 @@ abstract class AbstractProvider extends BaseAbstractProvider
         $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
         $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
         $manager = $this->managerRegistry->getManagerForClass($this->objectClass);
+        $objects = array();
 
         for (; $offset < $nbObjects; $offset += $batchSize) {
             if ($loggerClosure) {
                 $stepStartTime = microtime(true);
             }
-            $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
+            $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset, $objects);
             if ($loggerClosure) {
                 $stepNbObjects = count($objects);
             }
@@ -133,9 +134,10 @@ abstract class AbstractProvider extends BaseAbstractProvider
      * @param object  $queryBuilder
      * @param integer $limit
      * @param integer $offset
+     * @param array   $previousSlice
      * @return array
      */
-    protected abstract function fetchSlice($queryBuilder, $limit, $offset);
+    protected abstract function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice);
 
     /**
      * Creates the query builder, which will be used to fetch objects to index.
diff --git a/Doctrine/MongoDB/Provider.php b/Doctrine/MongoDB/Provider.php
index 9e1c5dd..ae711ec 100644
--- a/Doctrine/MongoDB/Provider.php
+++ b/Doctrine/MongoDB/Provider.php
@@ -59,15 +59,25 @@ class Provider extends AbstractProvider
     /**
      * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
      */
-    protected function fetchSlice($queryBuilder, $limit, $offset)
+    protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
     {
         if (!$queryBuilder instanceof Builder) {
             throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
         }
 
+        $lastObject = array_pop($previousSlice);
+
+        if ($lastObject) {
+            $queryBuilder
+                ->field('_id')->gt($lastObject->getId())
+                ->skip(0);
+        } else {
+            $queryBuilder->skip($offset);
+        }
+
         return $queryBuilder
             ->limit($limit)
-            ->skip($offset)
+            ->sort(array('_id' => 'asc'))
             ->getQuery()
             ->execute()
             ->toArray();
diff --git a/Doctrine/ORM/Provider.php b/Doctrine/ORM/Provider.php
index 7e2ac12..701c6c5 100644
--- a/Doctrine/ORM/Provider.php
+++ b/Doctrine/ORM/Provider.php
@@ -70,7 +70,7 @@ class Provider extends AbstractProvider
     /**
      * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
      */
-    protected function fetchSlice($queryBuilder, $limit, $offset)
+    protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
     {
         if (!$queryBuilder instanceof QueryBuilder) {
             throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');