Fix mongodb populate falling down performances for big collections
This commit is contained in:
parent
25d56d0a0f
commit
7fa7e44bee
|
@ -54,12 +54,13 @@ abstract class AbstractProvider extends BaseAbstractProvider
|
||||||
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
|
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
|
||||||
$ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
|
$ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
|
||||||
$manager = $this->managerRegistry->getManagerForClass($this->objectClass);
|
$manager = $this->managerRegistry->getManagerForClass($this->objectClass);
|
||||||
|
$objects = array();
|
||||||
|
|
||||||
for (; $offset < $nbObjects; $offset += $batchSize) {
|
for (; $offset < $nbObjects; $offset += $batchSize) {
|
||||||
if ($loggerClosure) {
|
if ($loggerClosure) {
|
||||||
$stepStartTime = microtime(true);
|
$stepStartTime = microtime(true);
|
||||||
}
|
}
|
||||||
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
|
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset, $objects);
|
||||||
if ($loggerClosure) {
|
if ($loggerClosure) {
|
||||||
$stepNbObjects = count($objects);
|
$stepNbObjects = count($objects);
|
||||||
}
|
}
|
||||||
|
@ -133,9 +134,10 @@ abstract class AbstractProvider extends BaseAbstractProvider
|
||||||
* @param object $queryBuilder
|
* @param object $queryBuilder
|
||||||
* @param integer $limit
|
* @param integer $limit
|
||||||
* @param integer $offset
|
* @param integer $offset
|
||||||
|
* @param array $previousSlice
|
||||||
* @return array
|
* @return array
|
||||||
*/
|
*/
|
||||||
protected abstract function fetchSlice($queryBuilder, $limit, $offset);
|
protected abstract function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates the query builder, which will be used to fetch objects to index.
|
* Creates the query builder, which will be used to fetch objects to index.
|
||||||
|
|
|
@ -59,15 +59,25 @@ class Provider extends AbstractProvider
|
||||||
/**
|
/**
|
||||||
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
||||||
*/
|
*/
|
||||||
protected function fetchSlice($queryBuilder, $limit, $offset)
|
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
|
||||||
{
|
{
|
||||||
if (!$queryBuilder instanceof Builder) {
|
if (!$queryBuilder instanceof Builder) {
|
||||||
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
|
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$lastObject = array_pop($previousSlice);
|
||||||
|
|
||||||
|
if ($lastObject) {
|
||||||
|
$queryBuilder
|
||||||
|
->field('_id')->gt($lastObject->getId())
|
||||||
|
->skip(0);
|
||||||
|
} else {
|
||||||
|
$queryBuilder->skip($offset);
|
||||||
|
}
|
||||||
|
|
||||||
return $queryBuilder
|
return $queryBuilder
|
||||||
->limit($limit)
|
->limit($limit)
|
||||||
->skip($offset)
|
->sort(array('_id' => 'asc'))
|
||||||
->getQuery()
|
->getQuery()
|
||||||
->execute()
|
->execute()
|
||||||
->toArray();
|
->toArray();
|
||||||
|
|
|
@ -70,7 +70,7 @@ class Provider extends AbstractProvider
|
||||||
/**
|
/**
|
||||||
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
|
||||||
*/
|
*/
|
||||||
protected function fetchSlice($queryBuilder, $limit, $offset)
|
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
|
||||||
{
|
{
|
||||||
if (!$queryBuilder instanceof QueryBuilder) {
|
if (!$queryBuilder instanceof QueryBuilder) {
|
||||||
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');
|
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');
|
||||||
|
|
Loading…
Reference in a new issue