Merge branch 'master' into add-hotswapping-aliased-indexes-on-populate

Conflicts:
	Resetter.php
This commit is contained in:
Tim Nagel 2014-03-17 09:18:57 +11:00
commit 49521e9fc4
25 changed files with 251 additions and 55 deletions

17
CHANGELOG-3.0.md Normal file
View file

@ -0,0 +1,17 @@
CHANGELOG for 3.0.x
===================
This changelog references the relevant changes (bug and security fixes) done
in 3.0 minor versions.
To get the diff for a specific change, go to
https://github.com/FriendsOfSymfony/FOSElasticaBundle/commit/XXX where XXX is
the commit hash. To get the diff between two versions, go to
https://github.com/FriendsOfSymfony/FOSElasticaBundle/compare/v3.0.0...v3.0.1
To generate a changelog summary since the last version, run
`git log --no-merges --oneline v3.0.0...3.0.x`
* 3.0.0-ALPHA2 (2014-xx-xx)
* 41bf07e: Renamed the `no-stop-on-error` option in PopulateCommand to `ignore-errors`

View file

@ -27,7 +27,7 @@ class Client extends ElasticaClient
'transport' => $connection->getTransport(),
);
$this->_logger->logQuery($path, $method, $data, $time, $connection_array);
$this->_logger->logQuery($path, $method, $data, $time, $connection_array, $query);
}
return $response;

10
Command/PopulateCommand.php Executable file → Normal file
View file

@ -45,6 +45,7 @@ class PopulateCommand extends ContainerAwareCommand
->addOption('offset', null, InputOption::VALUE_REQUIRED, 'Start indexing at offset', 0)
->addOption('sleep', null, InputOption::VALUE_REQUIRED, 'Sleep time between persisting iterations (microseconds)', 0)
->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Index packet size (overrides provider config option)')
->addOption('ignore-errors', null, InputOption::VALUE_NONE, 'Do not stop on errors')
->setDescription('Populates search indexes from providers')
;
}
@ -66,11 +67,12 @@ class PopulateCommand extends ContainerAwareCommand
{
$index = $input->getOption('index');
$type = $input->getOption('type');
$reset = $input->getOption('no-reset') ? false : true;
$noInteraction = $input->getOption('no-interaction');
$reset = !$input->getOption('no-reset');
$options = $input->getOptions();
if (!$noInteraction && $reset && $input->getOption('offset')) {
$options['ignore-errors'] = $input->hasOption('ignore-errors');
if ($input->isInteractive() && $reset && $input->getOption('offset')) {
/** @var DialogHelper $dialog */
$dialog = $this->getHelperSet()->get('dialog');
if (!$dialog->askConfirmation($output, '<question>You chose to reset the index and start indexing with an offset. Do you really want to do that?</question>', true)) {
@ -107,7 +109,7 @@ class PopulateCommand extends ContainerAwareCommand
*/
private function populateIndex(OutputInterface $output, $index, $reset, $options)
{
if ($reset) {
if ($reset && $this->indexManager->getIndex($index)->exists()) {
$output->writeln(sprintf('<info>Resetting</info> <comment>%s</comment>', $index));
$this->resetter->resetIndex($index);
}

View file

@ -101,7 +101,12 @@ class Configuration implements ConfigurationInterface
->arrayNode('servers')
->prototype('array')
->children()
->scalarNode('url')->end()
->scalarNode('url')
->validate()
->ifTrue(function($url) { return substr($url, -1) !== '/'; })
->then(function($url) { return $url.'/'; })
->end()
->end()
->scalarNode('host')->end()
->scalarNode('port')->end()
->scalarNode('logger')
@ -109,6 +114,7 @@ class Configuration implements ConfigurationInterface
->treatNullLike('fos_elastica.logger')
->treatTrueLike('fos_elastica.logger')
->end()
->scalarNode('timeout')->end()
->end()
->end()
->end()
@ -410,6 +416,10 @@ class Configuration implements ConfigurationInterface
}
if (isset($nestings['properties'])) {
$node
->booleanNode('include_in_parent')->end()
->booleanNode('include_in_root')->end()
;
$this->addNestedFieldConfig($node, $nestings, 'properties');
}
}

View file

@ -4,6 +4,7 @@ namespace FOS\ElasticaBundle\DependencyInjection;
use Symfony\Component\HttpKernel\DependencyInjection\Extension;
use Symfony\Component\DependencyInjection\Loader\XmlFileLoader;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerBuilder;
use Symfony\Component\DependencyInjection\Definition;
use Symfony\Component\DependencyInjection\DefinitionDecorator;
@ -203,6 +204,10 @@ class FOSElasticaExtension extends Extension
if (isset($type['serializer']['version'])) {
$callbackDef->addMethodCall('setVersion', array($type['serializer']['version']));
}
$callbackClassImplementedInterfaces = class_implements($this->serializerConfig['callback_class']); // PHP < 5.4 friendly
if (isset($callbackClassImplementedInterfaces['Symfony\Component\DependencyInjection\ContainerAwareInterface'])) {
$callbackDef->addMethodCall('setContainer', array(new Reference('service_container')));
}
$container->setDefinition($callbackId, $callbackDef);

View file

@ -3,6 +3,7 @@
namespace FOS\ElasticaBundle\Doctrine;
use Doctrine\Common\Persistence\ManagerRegistry;
use Elastica\Exception\Bulk\ResponseException as BulkResponseException;
use FOS\ElasticaBundle\Persister\ObjectPersisterInterface;
use FOS\ElasticaBundle\Provider\AbstractProvider as BaseAbstractProvider;
@ -22,6 +23,7 @@ abstract class AbstractProvider extends BaseAbstractProvider
{
parent::__construct($objectPersister, $objectClass, array_merge(array(
'clear_object_manager' => true,
'ignore_errors' => false,
'query_builder_method' => 'createQueryBuilder',
), $options));
@ -38,6 +40,7 @@ abstract class AbstractProvider extends BaseAbstractProvider
$offset = isset($options['offset']) ? intval($options['offset']) : 0;
$sleep = isset($options['sleep']) ? intval($options['sleep']) : 0;
$batchSize = isset($options['batch-size']) ? intval($options['batch-size']) : $this->options['batch_size'];
$ignoreErrors = isset($options['ignore-errors']) ? $options['ignore-errors'] : $this->options['ignore_errors'];
for (; $offset < $nbObjects; $offset += $batchSize) {
if ($loggerClosure) {
@ -45,7 +48,17 @@ abstract class AbstractProvider extends BaseAbstractProvider
}
$objects = $this->fetchSlice($queryBuilder, $batchSize, $offset);
$this->objectPersister->insertMany($objects);
if (!$ignoreErrors) {
$this->objectPersister->insertMany($objects);
} else {
try {
$this->objectPersister->insertMany($objects);
} catch(BulkResponseException $e) {
if ($loggerClosure) {
$loggerClosure(sprintf('<error>%s</error>',$e->getMessage()));
}
}
}
if ($this->options['clear_object_manager']) {
$this->managerRegistry->getManagerForClass($this->objectClass)->clear();
@ -58,7 +71,7 @@ abstract class AbstractProvider extends BaseAbstractProvider
$stepCount = $stepNbObjects + $offset;
$percentComplete = 100 * $stepCount / $nbObjects;
$objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime);
$loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond));
$loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage()));
}
}
}

View file

@ -9,7 +9,8 @@ interface FinderInterface
*
* @param mixed $query Can be a string, an array or an \Elastica\Query object
* @param int $limit How many results to get
* @param array $options
* @return array results
*/
function find($query, $limit = null);
function find($query, $limit = null, $options = array());
}

View file

@ -12,15 +12,17 @@ interface PaginatedFinderInterface extends FinderInterface
* Searches for query results and returns them wrapped in a paginator
*
* @param mixed $query Can be a string, an array or an \Elastica\Query object
* @param array $options
* @return Pagerfanta paginated results
*/
function findPaginated($query);
function findPaginated($query, $options = array());
/**
* Creates a paginator adapter for this query
*
* @param mixed $query
* @param array $options
* @return PaginatorAdapterInterface
*/
function createPaginatorAdapter($query);
function createPaginatorAdapter($query, $options = array());
}

View file

@ -29,18 +29,19 @@ class TransformedFinder implements PaginatedFinderInterface
*
* @param string $query
* @param integer $limit
* @param array $options
* @return array of model objects
**/
public function find($query, $limit = null)
public function find($query, $limit = null, $options = array())
{
$results = $this->search($query, $limit);
$results = $this->search($query, $limit, $options);
return $this->transformer->transform($results);
}
public function findHybrid($query, $limit = null)
public function findHybrid($query, $limit = null, $options = array())
{
$results = $this->search($query, $limit);
$results = $this->search($query, $limit, $options);
return $this->transformer->hybridTransform($results);
}
@ -64,15 +65,16 @@ class TransformedFinder implements PaginatedFinderInterface
/**
* @param $query
* @param null|int $limit
* @param array $options
* @return array
*/
protected function search($query, $limit = null)
protected function search($query, $limit = null, $options = array())
{
$queryObject = Query::create($query);
if (null !== $limit) {
$queryObject->setSize($limit);
}
$results = $this->searchable->search($queryObject)->getResults();
$results = $this->searchable->search($queryObject, $options)->getResults();
return $results;
}
@ -81,12 +83,13 @@ class TransformedFinder implements PaginatedFinderInterface
* Gets a paginator wrapping the result of a search
*
* @param string $query
* @param array $options
* @return Pagerfanta
*/
public function findPaginated($query)
public function findPaginated($query, $options = array())
{
$queryObject = Query::create($query);
$paginatorAdapter = $this->createPaginatorAdapter($queryObject);
$paginatorAdapter = $this->createPaginatorAdapter($queryObject, $options);
return new Pagerfanta(new FantaPaginatorAdapter($paginatorAdapter));
}
@ -94,10 +97,10 @@ class TransformedFinder implements PaginatedFinderInterface
/**
* {@inheritdoc}
*/
public function createPaginatorAdapter($query)
public function createPaginatorAdapter($query, $options = array())
{
$query = Query::create($query);
return new TransformedPaginatorAdapter($this->searchable, $query, $this->transformer);
return new TransformedPaginatorAdapter($this->searchable, $query, $options, $this->transformer);
}
}

View file

@ -14,33 +14,44 @@ use Psr\Log\LoggerInterface;
*/
class ElasticaLogger implements LoggerInterface
{
/**
* @var LoggerInterface
*/
protected $logger;
protected $queries;
/**
* @var array
*/
protected $queries = array();
/**
* @var boolean
*/
protected $debug;
/**
* Constructor.
*
* @param LoggerInterface|null $logger The Symfony logger
* @param bool $debug
* @param boolean $debug
*/
public function __construct(LoggerInterface $logger = null, $debug = false)
{
$this->logger = $logger;
$this->queries = array();
$this->debug = $debug;
}
/**
* Logs a query.
*
* @param string $path Path to call
* @param string $method Rest method to use (GET, POST, DELETE, PUT)
* @param array $data arguments
* @param float $time execution time
* @param array $connection host, port and transport of the query
* @param string $path Path to call
* @param string $method Rest method to use (GET, POST, DELETE, PUT)
* @param array $data Arguments
* @param float $time Execution time
* @param array $connection Host, port and transport of the query
* @param array $query Arguments
*/
public function logQuery($path, $method, $data, $time, $connection = array())
public function logQuery($path, $method, $data, $time, $connection = array(), $query = array())
{
if ($this->debug) {
$this->queries[] = array(
@ -48,7 +59,8 @@ class ElasticaLogger implements LoggerInterface
'method' => $method,
'data' => $data,
'executionMS' => $time,
'connection' => $connection
'connection' => $connection,
'queryString' => $query,
);
}

View file

@ -22,13 +22,18 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface
*/
private $query;
/**
* @var array search options
*/
private $options;
/**
* @var integer the number of hits
*/
private $totalHits;
/**
* @array for the facets
* @var array for the facets
*/
private $facets;
@ -38,10 +43,11 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface
* @param SearchableInterface $searchable the object to search in
* @param Query $query the query to search
*/
public function __construct(SearchableInterface $searchable, Query $query)
public function __construct(SearchableInterface $searchable, Query $query, array $options = array())
{
$this->searchable = $searchable;
$this->query = $query;
$this->options = $options;
}
/**
@ -72,7 +78,7 @@ class RawPaginatorAdapter implements PaginatorAdapterInterface
$query->setFrom($offset);
$query->setSize($itemCountPerPage);
$resultSet = $this->searchable->search($query);
$resultSet = $this->searchable->search($query, $this->options);
$this->totalHits = $resultSet->getTotalHits();
$this->facets = $resultSet->getFacets();
return $resultSet;

View file

@ -18,9 +18,9 @@ class TransformedPaginatorAdapter extends RawPaginatorAdapter
* @param Query $query the query to search
* @param ElasticaToModelTransformerInterface $transformer the transformer for fetching the results
*/
public function __construct(SearchableInterface $searchable, Query $query, ElasticaToModelTransformerInterface $transformer)
public function __construct(SearchableInterface $searchable, Query $query, array $options = array(), ElasticaToModelTransformerInterface $transformer)
{
parent::__construct($searchable, $query);
parent::__construct($searchable, $query, $options);
$this->transformer = $transformer;
}

View file

@ -41,7 +41,7 @@ class Provider extends AbstractProvider
$stepCount = $stepNbObjects + $offset;
$percentComplete = 100 * $stepCount / $nbObjects;
$objectsPerSecond = $stepNbObjects / (microtime(true) - $stepStartTime);
$loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond));
$loggerClosure(sprintf('%0.1f%% (%d/%d), %d objects/s %s', $percentComplete, $stepCount, $nbObjects, $objectsPerSecond, $this->getMemoryUsage()));
}
}
}

View file

@ -4,10 +4,24 @@ namespace FOS\ElasticaBundle\Provider;
use FOS\ElasticaBundle\Persister\ObjectPersisterInterface;
/**
* AbstractProvider
*/
abstract class AbstractProvider implements ProviderInterface
{
protected $objectClass;
/**
* @var ObjectPersisterInterface
*/
protected $objectPersister;
/**
* @var string
*/
protected $objectClass;
/**
* @var array
*/
protected $options;
/**
@ -26,4 +40,17 @@ abstract class AbstractProvider implements ProviderInterface
'batch_size' => 100,
), $options);
}
/**
* Get string with RAM usage information (current and peak)
*
* @return string
*/
protected function getMemoryUsage()
{
$memory = round(memory_get_usage() / (1024 * 1024)); // to get usage in Mo
$memoryMax = round(memory_get_peak_usage() / (1024 * 1024)); // to get max usage in Mo
return sprintf('(RAM : current=%uMo peak=%uMo)', $memory, $memoryMax);
}
}

View file

@ -58,6 +58,7 @@ Most of the time, you will need only one.
clients:
default: { host: localhost, port: 9200 }
A client configuration can also override the Elastica logger to change the used class ```logger: <your logger class>``` or to simply disable it ```logger: false```. Disabling the logger should be done on production because it can cause a memory leak.
#### Declare a serializer

View file

@ -19,23 +19,23 @@ class Repository
$this->finder = $finder;
}
public function find($query, $limit=null)
public function find($query, $limit = null, $options = array())
{
return $this->finder->find($query, $limit);
return $this->finder->find($query, $limit, $options);
}
public function findHybrid($query, $limit=null)
public function findHybrid($query, $limit = null, $options = array())
{
return $this->finder->findHybrid($query, $limit);
return $this->finder->findHybrid($query, $limit, $options);
}
public function findPaginated($query)
public function findPaginated($query, $options = array())
{
return $this->finder->findPaginated($query);
return $this->finder->findPaginated($query, $options);
}
public function createPaginatorAdapter($query)
public function createPaginatorAdapter($query, $options = array())
{
return $this->finder->createPaginatorAdapter($query);
return $this->finder->createPaginatorAdapter($query, $options);
}
}

View file

@ -4,6 +4,7 @@ namespace FOS\ElasticaBundle;
use Elastica\Exception\ExceptionInterface;
use Elastica\Index;
use Elastica\Exception\ResponseException;
use Elastica\Type\Mapping;
/**
@ -71,7 +72,13 @@ class Resetter
}
$type = $indexConfig['index']->getType($typeName);
$type->delete();
try {
$type->delete();
} catch (ResponseException $e) {
if (strpos($e->getMessage(), 'TypeMissingException') === false) {
throw $e;
}
}
$mapping = $this->createMapping($indexConfig['config']['mappings'][$typeName]);
$type->setMapping($mapping);
}
@ -86,12 +93,15 @@ class Resetter
{
$mapping = Mapping::create($indexConfig['properties']);
if (isset($indexConfig['_parent'])) {
$mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type']));
$mappingSpecialFields = array('_uid', '_id', '_source', '_all', '_analyzer', '_boost', '_routing', '_index', '_size', '_timestamp', '_ttl', 'dynamic_templates');
foreach ($mappingSpecialFields as $specialField) {
if (isset($indexConfig[$specialField])) {
$mapping->setParam($specialField, $indexConfig[$specialField]);
}
}
if (isset($indexConfig['dynamic_templates'])) {
$mapping->setParam('dynamic_templates', $indexConfig['dynamic_templates']);
if (isset($indexConfig['_parent'])) {
$mapping->setParam('_parent', array('type' => $indexConfig['_parent']['type']));
}
return $mapping;

View file

@ -44,6 +44,7 @@
{% for key, query in collector.queries %}
<li class="{{ cycle(['odd', 'even'], loop.index) }}">
<strong>Path</strong>: {{ query.path }}<br />
<strong>Query</strong>: {{ query.queryString|url_encode }}<br />
<strong>Method</strong>: {{ query.method }} <small>({{ query.connection.transport }} on {{ query.connection.host }}:{{ query.connection.port }})</small>
<div>
<code>{{ query.data|json_encode }}</code>
@ -60,7 +61,7 @@
</a>
<div style="display: none;" id="elastica_curl_query_{{ key }}">
<code>curl -X{{ query.method }} '{{ query.connection.transport|lower }}://{{ query.connection.host }}:{{ query.connection.port }}/{{ query.path }}' -d '{{ query.data|json_encode }}'</code>
<code>curl -X{{ query.method }} '{{ query.connection.transport|lower }}://{{ query.connection.host }}:{{ query.connection.port }}/{{ query.path }}{% if query.queryString|length %}?{{ query.queryString|url_encode }}{% endif %}' -d '{{ query.data|json_encode }}'</code>
</div>
{% endif %}
</li>

View file

@ -24,6 +24,7 @@ class ClientTest extends \PHPUnit_Framework_TestCase
Request::GET,
$this->isType('array'),
$this->isType('float'),
$this->isType('array'),
$this->isType('array')
);

View file

@ -3,6 +3,7 @@
namespace FOS\ElasticaBundle\Tests\Resetter\DependencyInjection;
use FOS\ElasticaBundle\DependencyInjection\Configuration;
use Symfony\Component\Config\Definition\Processor;
/**
* ConfigurationTest
@ -85,4 +86,21 @@ class ConfigurationTest extends \PHPUnit_Framework_TestCase
$this->assertInstanceOf('Symfony\Component\Config\Definition\ScalarNode', $mapping['index']);
$this->assertNull($mapping['index']->getDefaultValue());
}
public function testSlashIsAddedAtTheEndOfServerUrl()
{
$config = array(
'clients' => array(
'default' => array(
'url' => 'http://www.github.com',
),
),
);
$processor = new Processor();
$configuration = $processor->processConfiguration($this->configuration, array($config));
$this->assertEquals('http://www.github.com/', $configuration['clients']['default']['servers'][0]['url']);
}
}

View file

@ -135,6 +135,30 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
$this->assertTrue($loggerClosureInvoked);
}
public function testPopulateNotStopOnError()
{
$nbObjects = 1;
$objects = array(1);
$provider = $this->getMockAbstractProvider();
$provider->expects($this->any())
->method('countObjects')
->will($this->returnValue($nbObjects));
$provider->expects($this->any())
->method('fetchSlice')
->will($this->returnValue($objects));
$this->objectPersister->expects($this->any())
->method('insertMany')
->will($this->throwException($this->getMockBulkResponseException()));
$this->setExpectedException('Elastica\Exception\Bulk\ResponseException');
$provider->populate(null, array('ignore-errors' => false));
}
/**
* @return \FOS\ElasticaBundle\Doctrine\AbstractProvider|\PHPUnit_Framework_MockObject_MockObject
*/
@ -148,6 +172,16 @@ class AbstractProviderTest extends \PHPUnit_Framework_TestCase
));
}
/**
* @return \Elastica\Exception\Bulk\ResponseException
*/
private function getMockBulkResponseException()
{
return $this->getMockBuilder('Elastica\Exception\Bulk\ResponseException')
->disableOriginalConstructor()
->getMock();
}
/**
* @return \Doctrine\Common\Persistence\ManagerRegistry|\PHPUnit_Framework_MockObject_MockObject
*/

View file

@ -70,6 +70,7 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase
$data = array('data');
$time = 12;
$connection = array('host' => 'localhost', 'port' => '8999', 'transport' => 'https');
$query = array('search_type' => 'dfs_query_then_fetch');
$expected = array(
'path' => $path,
@ -77,9 +78,10 @@ class ElasticaLoggerTest extends \PHPUnit_Framework_TestCase
'data' => $data,
'executionMS' => $time,
'connection' => $connection,
'queryString' => $query,
);
$elasticaLogger->logQuery($path, $method, $data, $time, $connection);
$elasticaLogger->logQuery($path, $method, $data, $time, $connection, $query);
$returnedQueries = $elasticaLogger->getQueries();
$this->assertEquals($expected, $returnedQueries[0]);
}

View file

@ -2,6 +2,9 @@
namespace FOS\ElasticaBundle\Tests\Resetter;
use Elastica\Exception\ResponseException;
use Elastica\Request;
use Elastica\Response;
use FOS\ElasticaBundle\Resetter;
use Elastica\Type\Mapping;
@ -130,6 +133,32 @@ class ResetterTest extends \PHPUnit_Framework_TestCase
$resetter->resetIndexType('foo', 'c');
}
public function testResetIndexTypeIgnoreTypeMissingException()
{
$type = $this->getMockElasticaType();
$this->indexConfigsByName['foo']['index']->expects($this->once())
->method('getType')
->with('a')
->will($this->returnValue($type));
$type->expects($this->once())
->method('delete')
->will($this->throwException(new ResponseException(
new Request(''),
new Response(array('error' => 'TypeMissingException[[de_20131022] type[bla] missing]', 'status' => 404)))
));
$mapping = Mapping::create($this->indexConfigsByName['foo']['config']['mappings']['a']['properties']);
$mapping->setParam('dynamic_templates', $this->indexConfigsByName['foo']['config']['mappings']['a']['dynamic_templates']);
$type->expects($this->once())
->method('setMapping')
->with($mapping);
$resetter = new Resetter($this->indexConfigsByName);
$resetter->resetIndexType('foo', 'a');
}
public function testIndexMappingForParent()
{
$type = $this->getMockElasticaType();

View file

@ -67,7 +67,9 @@ class ElasticaToModelTransformerCollection implements ElasticaToModelTransformer
$result = array();
foreach ($elasticaObjects as $object) {
$result[] = $transformed[$object->getType()][$object->getId()];
if (array_key_exists($object->getId(), $transformed[$object->getType()])) {
$result[] = $transformed[$object->getType()][$object->getId()];
}
}
return $result;

View file

@ -16,7 +16,7 @@
"symfony/console": "~2.1",
"symfony/form": "~2.1",
"symfony/property-access": "~2.2",
"ruflin/elastica": "~0.20",
"ruflin/elastica": ">=0.20, <1.1-dev",
"psr/log": "~1.0"
},
"require-dev":{