deblan/FOSElasticaBundle
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'pr/725'

Browse source 
Conflicts:
	CHANGELOG-3.1.md
	Doctrine/AbstractProvider.php
This commit is contained in:
Tim Nagel 2015-03-11 22:10:54 +11:00
parent 181b5a0ac0 f834499a2c
commit b7c7f77383
9 changed files with 269 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG-3.1.md
View file

@ -35,3 +35,6 @@ https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.4...v3.1.0
for indexing. #794
* Fixed a case where ProgressCommand would always ignore errors regardless of
--ignore-errors being passed or not.
* Added a `SliceFetcher` abstraction for Doctrine providers that get more
information about the previous slice allowing for optimising queries during
population. #725

46
Doctrine/AbstractProvider.php
View file

@ -10,6 +10,14 @@ use FOS\ElasticaBundle\Provider\IndexableInterface;
abstract class AbstractProvider extends BaseAbstractProvider
{
/**
* @var SliceFetcherInterface
*/
private $sliceFetcher;
/**
* @var ManagerRegistry
*/
protected $managerRegistry;
/**
@ -20,13 +28,15 @@ abstract class AbstractProvider extends BaseAbstractProvider
* @param string $objectClass
* @param array $options
* @param ManagerRegistry $managerRegistry
* @param SliceFetcherInterface $sliceFetcher
*/
public function __construct(
ObjectPersisterInterface $objectPersister,
IndexableInterface $indexable,
$objectClass,
array $options,
ManagerRegistry $managerRegistry
ManagerRegistry $managerRegistry,
SliceFetcherInterface $sliceFetcher = null
) {
parent::__construct($objectPersister, $indexable, $objectClass, array_merge(array(
'clear_object_manager' => true,
@ -36,6 +46,7 @@ abstract class AbstractProvider extends BaseAbstractProvider
), $options));
$this->managerRegistry = $managerRegistry;
$this->sliceFetcher = $sliceFetcher;
}
/**
@ -55,8 +66,9 @@ abstract class AbstractProvider extends BaseAbstractProvider
$ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
$manager = $this->managerRegistry->getManagerForClass($this->objectClass);
$objects = array();
for (; $offset < $nbObjects; $offset += $batchSize) {
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
$objects = $this->getSlice($queryBuilder, $batchSize, $offset, $objects);
$objects = array_filter($objects, array($this, 'isObjectIndexable'));
if ($objects) {
@ -89,6 +101,36 @@ abstract class AbstractProvider extends BaseAbstractProvider
}
}
/**
* If this Provider has a SliceFetcher defined, we use it instead of falling back to
* the fetchSlice methods defined in the ORM/MongoDB subclasses.
*
* @param $queryBuilder
* @param int $limit
* @param int $offset
* @param array $lastSlice
* @return array
*/
protected function getSlice($queryBuilder, $limit, $offset, $lastSlice)
{
if (!$this->sliceFetcher) {
return $this->fetchSlice($queryBuilder, $limit, $offset);
}
$manager = $this->managerRegistry->getManagerForClass($this->objectClass);
$identifierFieldNames = $manager
->getClassMetadata($this->objectClass)
->getIdentifierFieldNames();
return $this->sliceFetcher->fetch(
$queryBuilder,
$limit,
$offset,
$lastSlice,
$identifierFieldNames
);
}
/**
* Counts objects that would be indexed using the query builder.
*

2
Doctrine/MongoDB/Provider.php
View file

@ -66,8 +66,8 @@ class Provider extends AbstractProvider
}
return $queryBuilder
->limit($limit)
->skip($offset)
->limit($limit)
->getQuery()
->execute()
->toArray();

44
Doctrine/MongoDB/SliceFetcher.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace FOS\ElasticaBundle\Doctrine\MongoDB;
use Doctrine\ODM\MongoDB\Query\Builder;
use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException;
use FOS\ElasticaBundle\Doctrine\SliceFetcherInterface;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
class SliceFetcher implements SliceFetcherInterface
{
/**
* {@inheritdoc}
*/
public function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames)
{
if (!$queryBuilder instanceof Builder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
}
$lastObject = array_pop($previousSlice);
if ($lastObject) {
$queryBuilder
->field('_id')->gt($lastObject->getId())
->skip(0)
;
} else {
$queryBuilder->skip($offset);
}
return $queryBuilder
->limit($limit)
->sort(array('_id' => 'asc'))
->getQuery()
->execute()
->toArray()
;
}
}

47
Doctrine/ORM/SliceFetcher.php Normal file
View file

@ -0,0 +1,47 @@
<?php
namespace FOS\ElasticaBundle\Doctrine\ORM;
use Doctrine\ORM\QueryBuilder;
use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException;
use FOS\ElasticaBundle\Doctrine\SliceFetcherInterface;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
class SliceFetcher implements SliceFetcherInterface
{
/**
* {@inheritdoc}
*/
public function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames)
{
if (!$queryBuilder instanceof QueryBuilder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');
}
/**
* An orderBy DQL part is required to avoid feching the same row twice.
* @see http://stackoverflow.com/questions/6314879/does-limit-offset-length-require-order-by-for-pagination
* @see http://www.postgresql.org/docs/current/static/queries-limit.html
* @see http://www.sqlite.org/lang_select.html#orderby
*/
$orderBy = $queryBuilder->getDQLPart('orderBy');
if (empty($orderBy)) {
$rootAliases = $queryBuilder->getRootAliases();
foreach ($identifierFieldNames as $fieldName) {
$queryBuilder->addOrderBy($rootAliases[0].'.'.$fieldName);
}
}
return $queryBuilder
->setFirstResult($offset)
->setMaxResults($limit)
->getQuery()
->getResult()
;
}
}

23
Doctrine/SliceFetcherInterface.php Normal file
View file

@ -0,0 +1,23 @@
<?php
namespace FOS\ElasticaBundle\Doctrine;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
interface SliceFetcherInterface
{
/**
* Fetches a slice of objects using the query builder.
*
* @param object $queryBuilder
* @param integer $limit
* @param integer $offset
* @param array $previousSlice
* @param array $identifierFieldNames
* @return array
*/
function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames);
}

8
Resources/config/mongodb.xml
View file

@ -5,20 +5,24 @@
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
<parameters>
<parameter key="fos_elastica.slice_fetcher.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\SliceFetcher</parameter>
<parameter key="fos_elastica.provider.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.mongodb.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<services>
<service id="fos_elastica.slice_fetcher.mongodb" class="%fos_elastica.slice_fetcher.mongodb.class%">
</service>
<service id="fos_elastica.provider.prototype.mongodb" class="%fos_elastica.provider.prototype.mongodb.class%" public="true" abstract="true">
<argument /> <!-- object persister -->
<argument type="service" id="fos_elastica.indexable" />
<argument /> <!-- model -->
<argument type="collection" /> <!-- options -->
<argument type="service" id="doctrine_mongodb" />
<argument type="service" id="doctrine_mongodb" /> <!-- manager registry -->
<argument type="service" id="fos_elastica.slice_fetcher.mongodb" /> <!-- slice fetcher -->
</service>
<service id="fos_elastica.listener.prototype.mongodb" class="%fos_elastica.listener.prototype.mongodb.class%" public="false" abstract="true">

21
Resources/config/orm.xml
View file

@ -4,20 +4,25 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
<parameters>
<parameter key="fos_elastica.provider.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.orm.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.orm.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<parameters>
<parameter key="fos_elastica.slice_fetcher.orm.class">FOS\ElasticaBundle\Doctrine\ORM\SliceFetcher</parameter>
<parameter key="fos_elastica.provider.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.orm.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.orm.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<services>
<service id="fos_elastica.slice_fetcher.orm" class="%fos_elastica.slice_fetcher.orm.class%">
</service>
<service id="fos_elastica.provider.prototype.orm" class="%fos_elastica.provider.prototype.orm.class%" public="true" abstract="true">
<argument /> <!-- object persister -->
<argument type="service" id="fos_elastica.indexable" />
<argument /> <!-- model -->
<argument type="collection" /> <!-- options -->
<argument type="service" id="doctrine" />
<argument type="service" id="doctrine" /> <!-- manager registry -->
<argument type="service" id="fos_elastica.slice_fetcher.orm" /> <!-- slice fetcher -->
</service>
<service id="fos_elastica.listener.prototype.orm" class="%fos_elastica.listener.prototype.orm.class%" public="false" abstract="true">

101
Tests/Doctrine/AbstractProviderTest.php
View file

@ -13,6 +13,7 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
private $options;
private $managerRegistry;
private $indexable;
private $sliceFetcher;
public function setUp()
{
@ -32,6 +33,8 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
->method('getManagerForClass')
->with($this->objectClass)
->will($this->returnValue($this->objectManager));
$this->sliceFetcher = $this->getMockSliceFetcher();
}
/**
@ -45,6 +48,54 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
$queryBuilder = new \stdClass();
$provider->expects($this->once())
->method('createQueryBuilder')
->will($this->returnValue($queryBuilder));
$provider->expects($this->once())
->method('countObjects')
->with($queryBuilder)
->will($this->returnValue($nbObjects));
$this->indexable->expects($this->any())
->method('isObjectIndexable')
->with('index', 'type', $this->anything())
->will($this->returnValue(true));
$providerInvocationOffset = 2;
$previousSlice = array();
foreach ($objectsByIteration as $i => $objects) {
$offset = $objects[0] - 1;
$this->sliceFetcher->expects($this->at($i))
->method('fetch')
->with($queryBuilder, $batchSize, $offset, $previousSlice, array('id'))
->will($this->returnValue($objects));
$this->objectManager->expects($this->at($i))
->method('clear');
$previousSlice = $objects;
}
$this->objectPersister->expects($this->exactly(count($objectsByIteration)))
->method('insertMany');
$provider->populate();
}
/**
* @dataProvider providePopulateIterations
*/
public function testPopulateIterationsWithoutSliceFetcher($nbObjects, $objectsByIteration, $batchSize)
{
$this->options['batch_size'] = $batchSize;
$provider = $this->getMockAbstractProvider(false);
$queryBuilder = new \stdClass();
$provider->expects($this->once())
->method('createQueryBuilder')
->will($this->returnValue($queryBuilder));
@ -107,8 +158,8 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
$this->sliceFetcher->expects($this->any())
->method('fetch')
->will($this->returnValue($objects));
$this->indexable->expects($this->any())
@ -127,14 +178,14 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
$nbObjects = 1;
$objects = array(1);
$provider = $this->getMockAbstractProvider();
$provider = $this->getMockAbstractProvider(true);
$provider->expects($this->any())
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
$this->sliceFetcher->expects($this->any())
->method('fetch')
->will($this->returnValue($objects));
$this->indexable->expects($this->any())
@ -159,8 +210,8 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
$this->sliceFetcher->expects($this->any())
->method('fetch')
->will($this->returnValue($objects));
$this->indexable->expects($this->any())
@ -191,8 +242,8 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
$this->sliceFetcher->expects($this->any())
->method('fetch')
->will($this->returnValue($objects));
$this->indexable->expects($this->any())
@ -218,8 +269,9 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
$provider->expects($this->any())
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
$this->sliceFetcher->expects($this->any())
->method('fetch')
->will($this->returnValue($objects));
$this->indexable->expects($this->at(0))
@ -240,9 +292,11 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
}
/**
* @param boolean $setSliceFetcher Whether or not to set the slice fetcher.
*
* @return \FOS\ElasticaBundle\Doctrine\AbstractProvider|\PHPUnit_Framework_MockObject_MockObject
*/
private function getMockAbstractProvider()
private function getMockAbstractProvider($setSliceFetcher = true)
{
return $this->getMockForAbstractClass('FOS\ElasticaBundle\Doctrine\AbstractProvider', array(
$this->objectPersister,
@ -250,6 +304,7 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
$this->objectClass,
$this->options,
$this->managerRegistry,
$setSliceFetcher ? $this->sliceFetcher : null
));
}
@ -276,7 +331,17 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
*/
private function getMockObjectManager()
{
return $this->getMock(__NAMESPACE__ . '\ObjectManager');
$mock = $this->getMock(__NAMESPACE__ . '\ObjectManager');
$mock->expects($this->any())
->method('getClassMetadata')
->will($this->returnSelf());
$mock->expects($this->any())
->method('getIdentifierFieldNames')
->will($this->returnValue(array('id')));
return $mock;
}
/**
@ -294,6 +359,14 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
{
return $this->getMock('FOS\ElasticaBundle\Provider\IndexableInterface');
}
/**
* @return \FOS\ElasticaBundle\Doctrine\SliceFetcherInterface|\PHPUnit_Framework_MockObject_MockObject
*/
private function getMockSliceFetcher()
{
return $this->getMock('FOS\ElasticaBundle\Doctrine\SliceFetcherInterface');
}
}
/**
@ -303,4 +376,6 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
interface ObjectManager
{
function clear();
function getClassMetadata();
function getIdentifierFieldNames();
}