Fix mongodb populate falling down performances for big collections
This commit is contained in:
parent
25d56d0a0f
commit
7fa7e44bee
|
@ -54,12 +54,13 @@ abstract class AbstractProvider extends BaseAbstractProvider
|
|||
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
|
||||
$ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
|
||||
$manager = $this->managerRegistry->getManagerForClass($this->objectClass);
|
||||
$objects = array();
|
||||
|
||||
for (; $offset < $nbObjects; $offset += $batchSize) {
|
||||
if ($loggerClosure) {
|
||||
$stepStartTime = microtime(true);
|
||||
}
|
||||
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
|
||||
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset, $objects);
|
||||
if ($loggerClosure) {
|
||||
$stepNbObjects = count($objects);
|
||||
}
|
||||
|
@ -133,9 +134,10 @@ abstract class AbstractProvider extends BaseAbstractProvider
|
|||
* @param object $queryBuilder
|
||||
* @param integer $limit
|
||||
* @param integer $offset
|
||||
* @param array $previousSlice
|
||||
* @return array
|
||||
*/
|
||||
protected abstract function fetchSlice($queryBuilder, $limit, $offset);
|
||||
protected abstract function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice);
|
||||
|
||||
/**
|
||||
* Creates the query builder, which will be used to fetch objects to index.
|
||||
|
|
|
@ -59,15 +59,25 @@ class Provider extends AbstractProvider
|
|||
/**
|
||||
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
||||
*/
|
||||
protected function fetchSlice($queryBuilder, $limit, $offset)
|
||||
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
|
||||
{
|
||||
if (!$queryBuilder instanceof Builder) {
|
||||
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
|
||||
}
|
||||
|
||||
$lastObject = array_pop($previousSlice);
|
||||
|
||||
if ($lastObject) {
|
||||
$queryBuilder
|
||||
->field('_id')->gt($lastObject->getId())
|
||||
->skip(0);
|
||||
} else {
|
||||
$queryBuilder->skip($offset);
|
||||
}
|
||||
|
||||
return $queryBuilder
|
||||
->limit($limit)
|
||||
->skip($offset)
|
||||
->sort(array('_id' => 'asc'))
|
||||
->getQuery()
|
||||
->execute()
|
||||
->toArray();
|
||||
|
|
|
@ -70,7 +70,7 @@ class Provider extends AbstractProvider
|
|||
/**
|
||||
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
||||
*/
|
||||
protected function fetchSlice($queryBuilder, $limit, $offset)
|
||||
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
|
||||
{
|
||||
if (!$queryBuilder instanceof QueryBuilder) {
|
||||
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');
|
||||
|
|
Loading…
Reference in a new issue