deblan/FOSElasticaBundle
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add slice fetching abstraction

Browse source
This commit is contained in:
Gnucki 2014-10-13 15:41:29 +02:00
parent 7fa7e44bee
commit be75b387a5
8 changed files with 160 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
Doctrine/AbstractProvider.php
View file

@ -11,6 +11,7 @@ use FOS\ElasticaBundle\Provider\IndexableInterface;
abstract class AbstractProvider extends BaseAbstractProvider
{
protected $managerRegistry;
protected $sliceFetcher;
/**
* Constructor.
@ -20,13 +21,15 @@ abstract class AbstractProvider extends BaseAbstractProvider
* @param string $objectClass
* @param array $options
* @param ManagerRegistry $managerRegistry
* @param SliceFetcherInterface $sliceFetcher
*/
public function __construct(
ObjectPersisterInterface $objectPersister,
IndexableInterface $indexable,
$objectClass,
array $options,
ManagerRegistry $managerRegistry
ManagerRegistry $managerRegistry,
SliceFetcherInterface $sliceFetcher = null
) {
parent::__construct($objectPersister, $indexable, $objectClass, array_merge(array(
'clear_object_manager' => true,
@ -36,6 +39,7 @@ abstract class AbstractProvider extends BaseAbstractProvider
), $options));
$this->managerRegistry = $managerRegistry;
$this->sliceFetcher = $sliceFetcher;
}
/**
@ -60,7 +64,24 @@ abstract class AbstractProvider extends BaseAbstractProvider
if ($loggerClosure) {
$stepStartTime = microtime(true);
}
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset, $objects);
if ($this->sliceFetcher) {
$identifierFieldNames = $this->managerRegistry
->getManagerForClass($this->objectClass)
->getClassMetadata($this->objectClass)
->getIdentifierFieldNames();
$objects = $this->sliceFetcher->fetch(
$queryBuilder,
$batchSize,
$offset,
$objects,
$identifierFieldNames
);
} else {
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
}
if ($loggerClosure) {
$stepNbObjects = count($objects);
}
@ -134,10 +155,9 @@ abstract class AbstractProvider extends BaseAbstractProvider
* @param object $queryBuilder
* @param integer $limit
* @param integer $offset
* @param array $previousSlice
* @return array
*/
protected abstract function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice);
protected abstract function fetchSlice($queryBuilder, $limit, $offset);
/**
* Creates the query builder, which will be used to fetch objects to index.

14
Doctrine/MongoDB/Provider.php
View file

@ -59,25 +59,15 @@ class Provider extends AbstractProvider
/**
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
*/
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
protected function fetchSlice($queryBuilder, $limit, $offset)
{
if (!$queryBuilder instanceof Builder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
}
$lastObject = array_pop($previousSlice);
if ($lastObject) {
$queryBuilder
->field('_id')->gt($lastObject->getId())
->skip(0);
} else {
$queryBuilder->skip($offset);
}
return $queryBuilder
->skip($offset)
->limit($limit)
->sort(array('_id' => 'asc'))
->getQuery()
->execute()
->toArray();

44
Doctrine/MongoDB/SliceFetcher.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace FOS\ElasticaBundle\Doctrine\MongoDB;
use Doctrine\ODM\MongoDB\Query\Builder;
use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException;
use FOS\ElasticaBundle\Doctrine\SliceFetcherInterface;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
class SliceFetcher implements SliceFetcherInterface
{
/**
* {@inheritdoc}
*/
public function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames)
{
if (!$queryBuilder instanceof Builder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ODM\MongoDB\Query\Builder');
}
$lastObject = array_pop($previousSlice);
if ($lastObject) {
$queryBuilder
->field('_id')->gt($lastObject->getId())
->skip(0)
;
} else {
$queryBuilder->skip($offset);
}
return $queryBuilder
->limit($limit)
->sort(array('_id' => 'asc'))
->getQuery()
->execute()
->toArray()
;
}
}

2
Doctrine/ORM/Provider.php
View file

@ -70,7 +70,7 @@ class Provider extends AbstractProvider
/**
* @see FOS\ElasticaBundle\Doctrine\AbstractProvider::fetchSlice()
*/
protected function fetchSlice($queryBuilder, $limit, $offset, array $previousSlice)
protected function fetchSlice($queryBuilder, $limit, $offset)
{
if (!$queryBuilder instanceof QueryBuilder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');

47
Doctrine/ORM/SliceFetcher.php Normal file
View file

@ -0,0 +1,47 @@
<?php
namespace FOS\ElasticaBundle\Doctrine\ORM;
use Doctrine\ORM\QueryBuilder;
use FOS\ElasticaBundle\Exception\InvalidArgumentTypeException;
use FOS\ElasticaBundle\Doctrine\SliceFetcherInterface;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
class SliceFetcher implements SliceFetcherInterface
{
/**
* {@inheritdoc}
*/
public function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames)
{
if (!$queryBuilder instanceof QueryBuilder) {
throw new InvalidArgumentTypeException($queryBuilder, 'Doctrine\ORM\QueryBuilder');
}
/**
* An orderBy DQL part is required to avoid feching the same row twice.
* @see http://stackoverflow.com/questions/6314879/does-limit-offset-length-require-order-by-for-pagination
* @see http://www.postgresql.org/docs/current/static/queries-limit.html
* @see http://www.sqlite.org/lang_select.html#orderby
*/
$orderBy = $queryBuilder->getDQLPart('orderBy');
if (empty($orderBy)) {
$rootAliases = $queryBuilder->getRootAliases();
foreach ($identifierFieldNames as $fieldName) {
$queryBuilder->addOrderBy($rootAliases[0].'.'.$fieldName);
}
}
return $queryBuilder
->setFirstResult($offset)
->setMaxResults($limit)
->getQuery()
->getResult()
;
}
}

23
Doctrine/SliceFetcherInterface.php Normal file
View file

@ -0,0 +1,23 @@
<?php
namespace FOS\ElasticaBundle\Doctrine;
/**
* Fetches a slice of objects
*
* @author Thomas Prelot <tprelot@gmail.com>
*/
interface SliceFetcherInterface
{
/**
* Fetches a slice of objects using the query builder.
*
* @param object $queryBuilder
* @param integer $limit
* @param integer $offset
* @param array $previousSlice
* @param array $identifierFieldNames
* @return array
*/
function fetch($queryBuilder, $limit, $offset, array $previousSlice, array $identifierFieldNames);
}

8
Resources/config/mongodb.xml
View file

@ -5,20 +5,24 @@
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
<parameters>
<parameter key="fos_elastica.slice_fetcher.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\SliceFetcher</parameter>
<parameter key="fos_elastica.provider.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.mongodb.class">FOS\ElasticaBundle\Doctrine\MongoDB\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.mongodb.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<services>
<service id="fos_elastica.slice_fetcher.mongodb" class="%fos_elastica.slice_fetcher.mongodb.class%">
</service>
<service id="fos_elastica.provider.prototype.mongodb" class="%fos_elastica.provider.prototype.mongodb.class%" public="true" abstract="true">
<argument /> <!-- object persister -->
<argument type="service" id="fos_elastica.indexable" />
<argument /> <!-- model -->
<argument type="collection" /> <!-- options -->
<argument type="service" id="doctrine_mongodb" />
<argument type="service" id="doctrine_mongodb" /> <!-- manager registry -->
<argument type="service" id="fos_elastica.slice_fetcher.mongodb" /> <!-- slice fetcher -->
</service>
<service id="fos_elastica.listener.prototype.mongodb" class="%fos_elastica.listener.prototype.mongodb.class%" public="false" abstract="true">

21
Resources/config/orm.xml
View file

@ -4,20 +4,25 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd">
<parameters>
<parameter key="fos_elastica.provider.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.orm.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.orm.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<parameters>
<parameter key="fos_elastica.slice_fetcher.orm.class">FOS\ElasticaBundle\Doctrine\ORM\SliceFetcher</parameter>
<parameter key="fos_elastica.provider.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\Provider</parameter>
<parameter key="fos_elastica.listener.prototype.orm.class">FOS\ElasticaBundle\Doctrine\Listener</parameter>
<parameter key="fos_elastica.elastica_to_model_transformer.prototype.orm.class">FOS\ElasticaBundle\Doctrine\ORM\ElasticaToModelTransformer</parameter>
<parameter key="fos_elastica.manager.orm.class">FOS\ElasticaBundle\Doctrine\RepositoryManager</parameter>
</parameters>
<services>
<service id="fos_elastica.slice_fetcher.orm" class="%fos_elastica.slice_fetcher.orm.class%">
</service>
<service id="fos_elastica.provider.prototype.orm" class="%fos_elastica.provider.prototype.orm.class%" public="true" abstract="true">
<argument /> <!-- object persister -->
<argument type="service" id="fos_elastica.indexable" />
<argument /> <!-- model -->
<argument type="collection" /> <!-- options -->
<argument type="service" id="doctrine" />
<argument type="service" id="doctrine" /> <!-- manager registry -->
<argument type="service" id="fos_elastica.slice_fetcher.orm" /> <!-- slice fetcher -->
</service>
<service id="fos_elastica.listener.prototype.orm" class="%fos_elastica.listener.prototype.orm.class%" public="false" abstract="true">