Fix mongodb populate falling down performances for big collections

Doctrine/AbstractProvider.php

@@ -54,12 +54,13 @@ abstract class AbstractProvider extends BaseAbstractProvider
         $batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
         $ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
         $manager = $this->managerRegistry->getManagerForClass($this->objectClass);
+        $objects = array();
 
         for (; $offset < $nbObjects; $offset += $batchSize) {
             if ($loggerClosure) {
                 $stepStartTime = microtime(true);
             }
-            $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
+            $objects = $this->fetchSlice($queryBuilder, $batchSize, $offset, $objects);
             if ($loggerClosure) {
                 $stepNbObjects = count($objects);
             }
@@ -133,9 +134,10 @@ abstract class AbstractProvider extends BaseAbstractProvider
      * @param object  $queryBuilder
      * @param integer $limit
      * @param integer $offset
+     * @param array   $previousSlice
      * @return array
      */
-    protected abstract function fetchSlice($queryBuilder, $limit, $offset);
+    protected abstract function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice);
 
     /**
      * Creates the query builder, which will be used to fetch objects to index.

Doctrine/MongoDB/Provider.php

@@ -59,15 +59,25 @@ class Provider extends AbstractProvider
     /**
      * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
      */
-    protected function fetchSlice($queryBuilder, $limit, $offset)
+    protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
     {
         if (!$queryBuilder instanceof Builder) {
             throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
         }
 
+        $lastObject = array_pop($previousSlice);
+
+        if ($lastObject) {
+            $queryBuilder
+                ->field('_id')->gt($lastObject->getId())
+                ->skip(0);
+        } else {
+            $queryBuilder->skip($offset);
+        }
+
         return $queryBuilder
             ->limit($limit)
-            ->skip($offset)
+            ->sort(array('_id' => 'asc'))
             ->getQuery()
             ->execute()
             ->toArray();

Doctrine/ORM/Provider.php

@@ -70,7 +70,7 @@ class Provider extends AbstractProvider
     /**
      * @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
      */
-    protected function fetchSlice($queryBuilder, $limit, $offset)
+    protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
     {
         if (!$queryBuilder instanceof QueryBuilder) {
             throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');