From 3b1a756e6f89e828b2f9f2c5b280d0f86a72637c Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Thu, 6 Feb 2014 21:11:12 +0000 Subject: [PATCH 1/2] Add support for using aliases to allow hot swapping of indexes when populating --- Client.php | 5 + Command/PopulateCommand.php | 1 + DependencyInjection/Configuration.php | 5 +- DependencyInjection/FOSElasticaExtension.php | 5 + DynamicIndex.php | 28 ++++ Resetter.php | 136 ++++++++++++++++++- Resources/config/config.xml | 2 +- 7 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 DynamicIndex.php diff --git a/Client.php b/Client.php index ea0c572..67252ec 100644 --- a/Client.php +++ b/Client.php @@ -32,4 +32,9 @@ class Client extends ElasticaClient return $response; } + + public function getIndex($name) + { + return new DynamicIndex($this, $name); + } } diff --git a/Command/PopulateCommand.php b/Command/PopulateCommand.php index 7297523..31ecc4b 100755 --- a/Command/PopulateCommand.php +++ b/Command/PopulateCommand.php @@ -124,6 +124,7 @@ class PopulateCommand extends ContainerAwareCommand } $output->writeln(sprintf('Refreshing %s', $index)); + $this->resetter->postPopulate($index); $this->indexManager->getIndex($index)->refresh(); } diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 4ed88af..403de85 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -134,6 +134,7 @@ class Configuration implements ConfigurationInterface ->prototype('array') ->children() ->scalarNode('index_name')->end() + ->booleanNode('use_alias')->defaultValue(false)->end() ->scalarNode('client')->end() ->scalarNode('finder') ->treatNullLike(true) @@ -670,7 +671,7 @@ class Configuration implements ConfigurationInterface return $node; } - + /** * Returns the array node used for "_ttl" */ @@ -689,5 +690,5 @@ class Configuration implements ConfigurationInterface ; return $node; - } + } } diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 52070fd..a67be0f 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -124,6 +124,7 @@ class FOSElasticaExtension extends Extension $indexIds[$name] = $indexId; $this->indexConfigs[$name] = array( 'index' => new Reference($indexId), + 'name_or_alias' => $indexName, 'config' => array( 'mappings' => array() ) @@ -134,6 +135,10 @@ class FOSElasticaExtension extends Extension if (!empty($index['settings'])) { $this->indexConfigs[$name]['config']['settings'] = $index['settings']; } + if ($index['use_alias']) { + $this->indexConfigs[$name]['use_alias'] = true; + } + $this->loadTypes(isset($index['types']) ? $index['types'] : array(), $container, $name, $indexId, $typePrototypeConfig); } diff --git a/DynamicIndex.php b/DynamicIndex.php new file mode 100644 index 0000000..cbec8e9 --- /dev/null +++ b/DynamicIndex.php @@ -0,0 +1,28 @@ + + */ +class DynamicIndex extends Index +{ + /** + * Reassign index name + * + * While it's technically a regular setter for name property, it's specifically named overrideName, but not setName + * since it's used for a very specific case and normally should not be used + * + * @param string $name Index name + * + * @return void + */ + public function overrideName($name) + { + $this->_name = $name; + } +} diff --git a/Resetter.php b/Resetter.php index 26a6bb5..b8e7b80 100644 --- a/Resetter.php +++ b/Resetter.php @@ -2,6 +2,8 @@ namespace FOS\ElasticaBundle; +use Elastica\Exception\ExceptionInterface; +use Elastica\Index; use Elastica\Type\Mapping; /** @@ -26,8 +28,8 @@ class Resetter */ public function resetAllIndexes() { - foreach ($this->indexConfigsByName as $indexConfig) { - $indexConfig['index']->create($indexConfig['config'], true); + foreach (array_keys($this->indexConfigsByName) as $name) { + $this->resetIndex($name); } } @@ -40,7 +42,17 @@ class Resetter public function resetIndex($indexName) { $indexConfig = $this->getIndexConfig($indexName); - $indexConfig['index']->create($indexConfig['config'], true); + $esIndex = $indexConfig['index']; + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $name = $indexConfig['name_or_alias']; + $name .= date('-Y-m-d-Gis'); + $esIndex->overrideName($name); + $esIndex->create($indexConfig['config']); + + return; + } + + $esIndex->create($indexConfig['config'], true); } /** @@ -102,4 +114,122 @@ class Resetter return $this->indexConfigsByName[$indexName]; } + + public function postPopulate($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { + $this->switchIndexAlias($indexName); + } + } + + /** + * Switches the alias for given index to the newly populated index + * and deletes the old index + * + * @param string $indexName Index name + * + * @throws \RuntimeException + */ + private function switchIndexAlias($indexName) + { + $indexConfig = $this->getIndexConfig($indexName); + $esIndex = $indexConfig['index']; + $aliasName = $indexConfig['name_or_alias']; + $oldIndexName = false; + $newIndexName = $esIndex->getName(); + + $aliasedIndexes = $this->getAliasedIndexes($esIndex, $aliasName); + + if (count($aliasedIndexes) > 1) { + throw new \RuntimeException( + sprintf( + 'Alias %s is used for multiple indexes: [%s]. + Make sure it\'s either not used or is assigned to one index only', + $aliasName, + join(', ', $aliasedIndexes) + ) + ); + } + + // Change the alias to point to the new index + // Elastica's addAlias can't be used directly, because in current (0.19.x) version it's not atomic + // In 0.20.x it's atomic, but it doesn't return the old index name + $aliasUpdateRequest = array('actions' => array()); + if (count($aliasedIndexes) == 1) { + // if the alias is set - add an action to remove it + $oldIndexName = $aliasedIndexes[0]; + $aliasUpdateRequest['actions'][] = array( + 'remove' => array('index' => $oldIndexName, 'alias' => $aliasName) + ); + } + + // add an action to point the alias to the new index + $aliasUpdateRequest['actions'][] = array( + 'add' => array('index' => $newIndexName, 'alias' => $aliasName) + ); + + try { + $esIndex->getClient()->request('_aliases', 'POST', $aliasUpdateRequest); + } catch (ExceptionInterface $renameAliasException) { + $additionalError = ''; + // if we failed to move the alias, delete the newly built index + try { + $esIndex->delete(); + } catch (ExceptionInterface $deleteNewIndexException) { + $additionalError = sprintf( + 'Tried to delete newly built index %s, but also failed: %s', + $newIndexName, + $deleteNewIndexException->getError() + ); + } + + throw new \RuntimeException( + sprintf( + 'Failed to updated index alias: %s. %s', + $renameAliasException->getMessage(), + $additionalError ?: sprintf('Newly built index %s was deleted', $newIndexName) + ) + ); + } + + // Delete the old index after the alias has been switched + if ($oldIndexName) { + $oldIndex = new Index($esIndex->getClient(), $oldIndexName); + try { + $oldIndex->delete(); + } catch (ExceptionInterface $deleteOldIndexException) { + throw new \RuntimeException( + sprintf( + 'Failed to delete old index %s with message: %s', + $oldIndexName, + $deleteOldIndexException->getMessage() + ) + ); + } + } + } + + /** + * Returns array of indexes which are mapped to given alias + * + * @param Index $esIndex ES Index + * @param string $aliasName Alias name + * + * @return array + */ + private function getAliasedIndexes(Index $esIndex, $aliasName) + { + $aliasesInfo = $esIndex->getClient()->request('_aliases', 'GET')->getData(); + $aliasedIndexes = array(); + + foreach ($aliasesInfo as $indexName => $indexInfo) { + $aliases = array_keys($indexInfo['aliases']); + if (in_array($aliasName, $aliases)) { + $aliasedIndexes[] = $indexName; + } + } + + return $aliasedIndexes; + } } diff --git a/Resources/config/config.xml b/Resources/config/config.xml index 85cfbed..4419b4a 100644 --- a/Resources/config/config.xml +++ b/Resources/config/config.xml @@ -6,7 +6,7 @@ FOS\ElasticaBundle\Client - Elastica\Index + FOS\ElasticaBundle\DynamicIndex Elastica\Type FOS\ElasticaBundle\Logger\ElasticaLogger FOS\ElasticaBundle\DataCollector\ElasticaDataCollector From 03bf793b656a11646ae74a351ce46dd3de74b045 Mon Sep 17 00:00:00 2001 From: Richard Miller Date: Mon, 10 Feb 2014 10:55:13 +0000 Subject: [PATCH 2/2] Make real index name use uniqid so can be reset within a second --- Resetter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Resetter.php b/Resetter.php index b8e7b80..4a7f13b 100644 --- a/Resetter.php +++ b/Resetter.php @@ -45,7 +45,7 @@ class Resetter $esIndex = $indexConfig['index']; if (isset($indexConfig['use_alias']) && $indexConfig['use_alias']) { $name = $indexConfig['name_or_alias']; - $name .= date('-Y-m-d-Gis'); + $name .= uniqid(); $esIndex->overrideName($name); $esIndex->create($indexConfig['config']); @@ -124,7 +124,7 @@ class Resetter } /** - * Switches the alias for given index to the newly populated index + * Switches the alias for given index (by key) to the newly populated index * and deletes the old index * * @param string $indexName Index name