Merge pull request #1067 from Block8/dc/workers
Adding support for a beanstalkd-based queue
This commit is contained in:
commit
de1c058f83
|
@ -36,37 +36,42 @@ class BuildFactory
|
|||
|
||||
/**
|
||||
* Takes a generic build and returns a type-specific build model.
|
||||
* @param Build $base The build from which to get a more specific build type.
|
||||
* @param Build $build The build from which to get a more specific build type.
|
||||
* @return Build
|
||||
*/
|
||||
public static function getBuild(Build $base)
|
||||
public static function getBuild(Build $build)
|
||||
{
|
||||
switch ($base->getProject()->getType()) {
|
||||
case 'remote':
|
||||
$type = 'RemoteGitBuild';
|
||||
break;
|
||||
case 'local':
|
||||
$type = 'LocalBuild';
|
||||
break;
|
||||
case 'github':
|
||||
$type = 'GithubBuild';
|
||||
break;
|
||||
case 'bitbucket':
|
||||
$type = 'BitbucketBuild';
|
||||
break;
|
||||
case 'gitlab':
|
||||
$type = 'GitlabBuild';
|
||||
break;
|
||||
case 'hg':
|
||||
$type = 'MercurialBuild';
|
||||
break;
|
||||
case 'svn':
|
||||
$type = 'SubversionBuild';
|
||||
break;
|
||||
$project = $build->getProject();
|
||||
|
||||
if (!empty($project)) {
|
||||
switch ($project->getType()) {
|
||||
case 'remote':
|
||||
$type = 'RemoteGitBuild';
|
||||
break;
|
||||
case 'local':
|
||||
$type = 'LocalBuild';
|
||||
break;
|
||||
case 'github':
|
||||
$type = 'GithubBuild';
|
||||
break;
|
||||
case 'bitbucket':
|
||||
$type = 'BitbucketBuild';
|
||||
break;
|
||||
case 'gitlab':
|
||||
$type = 'GitlabBuild';
|
||||
break;
|
||||
case 'hg':
|
||||
$type = 'MercurialBuild';
|
||||
break;
|
||||
case 'svn':
|
||||
$type = 'SubversionBuild';
|
||||
break;
|
||||
}
|
||||
|
||||
$class = '\\PHPCI\\Model\\Build\\' . $type;
|
||||
$build = new $class($build->getDataArray());
|
||||
}
|
||||
|
||||
$type = '\\PHPCI\\Model\\Build\\' . $type;
|
||||
|
||||
return new $type($base->getDataArray());
|
||||
return $build;
|
||||
}
|
||||
}
|
||||
|
|
80
PHPCI/Command/WorkerCommand.php
Normal file
80
PHPCI/Command/WorkerCommand.php
Normal file
|
@ -0,0 +1,80 @@
|
|||
<?php
|
||||
/**
|
||||
* PHPCI - Continuous Integration for PHP
|
||||
*
|
||||
* @copyright Copyright 2015, Block 8 Limited.
|
||||
* @license https://github.com/Block8/PHPCI/blob/master/LICENSE.md
|
||||
* @link https://www.phptesting.org/
|
||||
*/
|
||||
|
||||
namespace PHPCI\Command;
|
||||
|
||||
use b8\Config;
|
||||
use Monolog\Logger;
|
||||
use PHPCI\Logging\OutputLogHandler;
|
||||
use PHPCI\Worker\BuildWorker;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
|
||||
/**
|
||||
* Worker Command - Starts the BuildWorker, which pulls jobs from beanstalkd
|
||||
* @author Dan Cryer <dan@block8.co.uk>
|
||||
* @package PHPCI
|
||||
* @subpackage Console
|
||||
*/
|
||||
class WorkerCommand extends Command
|
||||
{
|
||||
/**
|
||||
* @var OutputInterface
|
||||
*/
|
||||
protected $output;
|
||||
|
||||
/**
|
||||
* @var Logger
|
||||
*/
|
||||
protected $logger;
|
||||
|
||||
/**
|
||||
* @param \Monolog\Logger $logger
|
||||
* @param string $name
|
||||
*/
|
||||
public function __construct(Logger $logger, $name = null)
|
||||
{
|
||||
parent::__construct($name);
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
protected function configure()
|
||||
{
|
||||
$this
|
||||
->setName('phpci:worker')
|
||||
->setDescription('Runs the PHPCI build worker.');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output)
|
||||
{
|
||||
$this->output = $output;
|
||||
|
||||
// For verbose mode we want to output all informational and above
|
||||
// messages to the symphony output interface.
|
||||
if ($input->hasOption('verbose') && $input->getOption('verbose')) {
|
||||
$this->logger->pushHandler(
|
||||
new OutputLogHandler($this->output, Logger::INFO)
|
||||
);
|
||||
}
|
||||
|
||||
$config = Config::getInstance()->get('phpci.worker', []);
|
||||
|
||||
if (empty($config['host']) || empty($config['queue'])) {
|
||||
$error = 'The worker is not configured. You must set a host and queue in your config.yml file.';
|
||||
throw new \Exception($error);
|
||||
}
|
||||
|
||||
$worker = new BuildWorker($config['host'], $config['queue']);
|
||||
$worker->setLogger($this->logger);
|
||||
$worker->setMaxJobs(Config::getInstance()->get('phpci.worker.max_jobs', -1));
|
||||
$worker->startWorker();
|
||||
}
|
||||
}
|
|
@ -364,10 +364,6 @@ class WebhookController extends \b8\Controller
|
|||
|
||||
// If not, create a new build job for it:
|
||||
$build = $this->buildService->createBuild($project, $commitId, $branch, $committer, $commitMessage, $extra);
|
||||
$build = BuildFactory::getBuild($build);
|
||||
|
||||
// Send a status postback if the build type provides one:
|
||||
$build->sendStatusPostback();
|
||||
|
||||
return array('status' => 'ok', 'buildID' => $build->getID());
|
||||
}
|
||||
|
|
|
@ -233,7 +233,7 @@ class Build extends BuildBase
|
|||
if (!$this->getId()) {
|
||||
return null;
|
||||
}
|
||||
return PHPCI_BUILD_ROOT_DIR . $this->getId();
|
||||
return PHPCI_BUILD_ROOT_DIR . $this->getId() . '_' . substr(md5(microtime(true)), 0, 5);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -45,12 +45,16 @@ class GithubBuild extends RemoteGitBuild
|
|||
{
|
||||
$token = \b8\Config::getInstance()->get('phpci.github.token');
|
||||
|
||||
if (empty($token)) {
|
||||
if (empty($token) || empty($this->data['id'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$project = $this->getProject();
|
||||
|
||||
if (empty($project)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$url = 'https://api.github.com/repos/'.$project->getReference().'/statuses/'.$this->getCommitId();
|
||||
$http = new \b8\HttpClient();
|
||||
|
||||
|
@ -114,10 +118,14 @@ class GithubBuild extends RemoteGitBuild
|
|||
{
|
||||
$rtn = parent::getCommitMessage($this->data['commit_message']);
|
||||
|
||||
$reference = $this->getProject()->getReference();
|
||||
$commitLink = '<a target="_blank" href="https://github.com/' . $reference . '/issues/$1">#$1</a>';
|
||||
$rtn = preg_replace('/\#([0-9]+)/', $commitLink, $rtn);
|
||||
$rtn = preg_replace('/\@([a-zA-Z0-9_]+)/', '<a target="_blank" href="https://github.com/$1">@$1</a>', $rtn);
|
||||
$project = $this->getProject();
|
||||
|
||||
if (!is_null($project)) {
|
||||
$reference = $project->getReference();
|
||||
$commitLink = '<a target="_blank" href="https://github.com/' . $reference . '/issues/$1">#$1</a>';
|
||||
$rtn = preg_replace('/\#([0-9]+)/', $commitLink, $rtn);
|
||||
$rtn = preg_replace('/\@([a-zA-Z0-9_]+)/', '<a target="_blank" href="https://github.com/$1">@$1</a>', $rtn);
|
||||
}
|
||||
|
||||
return $rtn;
|
||||
}
|
||||
|
|
|
@ -9,6 +9,10 @@
|
|||
|
||||
namespace PHPCI\Service;
|
||||
|
||||
use b8\Config;
|
||||
use Pheanstalk\Pheanstalk;
|
||||
use Pheanstalk\PheanstalkInterface;
|
||||
use PHPCI\BuildFactory;
|
||||
use PHPCI\Helper\Lang;
|
||||
use PHPCI\Model\Build;
|
||||
use PHPCI\Model\Project;
|
||||
|
@ -81,7 +85,13 @@ class BuildService
|
|||
$build->setExtra(json_encode($extra));
|
||||
}
|
||||
|
||||
return $this->buildStore->save($build);
|
||||
$build = $this->buildStore->save($build);
|
||||
|
||||
$build = BuildFactory::getBuild($build);
|
||||
$build->sendStatusPostback();
|
||||
$this->addBuildToQueue($build);
|
||||
|
||||
return $build;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -104,7 +114,13 @@ class BuildService
|
|||
$build->setCreated(new \DateTime());
|
||||
$build->setStatus(0);
|
||||
|
||||
return $this->buildStore->save($build);
|
||||
$build = $this->buildStore->save($build);
|
||||
|
||||
$build = BuildFactory::getBuild($build);
|
||||
$build->sendStatusPostback();
|
||||
$this->addBuildToQueue($build);
|
||||
|
||||
return $build;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -117,4 +133,40 @@ class BuildService
|
|||
$build->removeBuildDirectory();
|
||||
return $this->buildStore->delete($build);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a build and puts it into the queue to be run (if using a queue)
|
||||
* @param Build $build
|
||||
*/
|
||||
public function addBuildToQueue(Build $build)
|
||||
{
|
||||
$buildId = $build->getId();
|
||||
|
||||
if (empty($buildId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$config = Config::getInstance();
|
||||
|
||||
$settings = $config->get('phpci.worker', []);
|
||||
|
||||
if (!empty($settings['host']) && !empty($settings['queue'])) {
|
||||
$jobData = array(
|
||||
'build_id' => $build->getId(),
|
||||
);
|
||||
|
||||
if ($config->get('using_custom_file')) {
|
||||
$jobData['config'] = $config->getArray();
|
||||
}
|
||||
|
||||
$pheanstalk = new Pheanstalk($settings['host']);
|
||||
$pheanstalk->useTube($settings['queue']);
|
||||
$pheanstalk->put(
|
||||
json_encode($jobData),
|
||||
PheanstalkInterface::DEFAULT_PRIORITY,
|
||||
PheanstalkInterface::DEFAULT_DELAY,
|
||||
$config->get('phpci.worker.job_timeout', 600)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
163
PHPCI/Worker/BuildWorker.php
Normal file
163
PHPCI/Worker/BuildWorker.php
Normal file
|
@ -0,0 +1,163 @@
|
|||
<?php
|
||||
|
||||
namespace PHPCI\Worker;
|
||||
|
||||
use b8\Config;
|
||||
use b8\Database;
|
||||
use b8\Store\Factory;
|
||||
use Monolog\Logger;
|
||||
use Pheanstalk\Pheanstalk;
|
||||
use PHPCI\Builder;
|
||||
use PHPCI\BuildFactory;
|
||||
use PHPCI\Logging\BuildDBLogHandler;
|
||||
use PHPCI\Model\Build;
|
||||
|
||||
/**
|
||||
* Class BuildWorker
|
||||
* @package PHPCI\Worker
|
||||
*/
|
||||
class BuildWorker
|
||||
{
|
||||
/**
|
||||
* If this variable changes to false, the worker will stop after the current build.
|
||||
* @var bool
|
||||
*/
|
||||
protected $run = true;
|
||||
|
||||
/**
|
||||
* The maximum number of jobs this worker should run before exiting.
|
||||
* Use -1 for no limit.
|
||||
* @var int
|
||||
*/
|
||||
protected $maxJobs = -1;
|
||||
|
||||
/**
|
||||
* The logger for builds to use.
|
||||
* @var \Monolog\Logger
|
||||
*/
|
||||
protected $logger;
|
||||
|
||||
/**
|
||||
* beanstalkd host
|
||||
* @var string
|
||||
*/
|
||||
protected $host;
|
||||
|
||||
/**
|
||||
* beanstalkd queue to watch
|
||||
* @var string
|
||||
*/
|
||||
protected $queue;
|
||||
|
||||
/**
|
||||
* @param $host
|
||||
* @param $queue
|
||||
*/
|
||||
public function __construct($host, $queue)
|
||||
{
|
||||
$this->host = $host;
|
||||
$this->queue = $queue;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $maxJobs
|
||||
*/
|
||||
public function setMaxJobs($maxJobs = -1)
|
||||
{
|
||||
$this->maxJobs = $maxJobs;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Logger $logger
|
||||
*/
|
||||
public function setLogger(Logger $logger)
|
||||
{
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the worker.
|
||||
*/
|
||||
public function startWorker()
|
||||
{
|
||||
$pheanstalk = new Pheanstalk($this->host);
|
||||
$pheanstalk->watch($this->queue);
|
||||
$buildStore = Factory::getStore('Build');
|
||||
|
||||
$jobs = 0;
|
||||
|
||||
while ($this->run) {
|
||||
// Get a job from the queue:
|
||||
$job = $pheanstalk->reserve();
|
||||
|
||||
// Make sure we don't run more than maxJobs jobs on this worker:
|
||||
$jobs++;
|
||||
|
||||
if ($this->maxJobs != -1 && $this->maxJobs <= $jobs) {
|
||||
$this->run = false;
|
||||
}
|
||||
|
||||
// Get the job data and run the job:
|
||||
$jobData = json_decode($job->getData(), true);
|
||||
$this->logger->addInfo('Received build #'.$jobData['build_id'].' from Beanstalkd');
|
||||
|
||||
// If the job comes with config data, reset our config and database connections
|
||||
// and then make sure we kill the worker afterwards:
|
||||
if (!empty($jobData['config'])) {
|
||||
$this->logger->addDebug('Using job-specific config.');
|
||||
$currentConfig = Config::getInstance()->getArray();
|
||||
$config = new Config($jobData['config']);
|
||||
Database::reset($config);
|
||||
}
|
||||
|
||||
$build = BuildFactory::getBuildById($jobData['build_id']);
|
||||
|
||||
if (empty($build)) {
|
||||
$this->logger->addWarning('Build #' . $jobData['build_id'] . ' does not exist in the database.');
|
||||
$pheanstalk->delete($job);
|
||||
}
|
||||
|
||||
try {
|
||||
// Logging relevant to this build should be stored
|
||||
// against the build itself.
|
||||
$buildDbLog = new BuildDBLogHandler($build, Logger::INFO);
|
||||
$this->logger->pushHandler($buildDbLog);
|
||||
|
||||
$builder = new Builder($build, $this->logger);
|
||||
$builder->execute();
|
||||
|
||||
// After execution we no longer want to record the information
|
||||
// back to this specific build so the handler should be removed.
|
||||
$this->logger->popHandler($buildDbLog);
|
||||
} catch (\PDOException $ex) {
|
||||
// If we've caught a PDO Exception, it is probably not the fault of the build, but of a failed
|
||||
// connection or similar. Release the job and kill the worker.
|
||||
$this->run = false;
|
||||
$pheanstalk->release($job);
|
||||
} catch (\Exception $ex) {
|
||||
$build->setStatus(Build::STATUS_FAILED);
|
||||
$build->setFinished(new \DateTime());
|
||||
$build->setLog($build->getLog() . PHP_EOL . PHP_EOL . $ex->getMessage());
|
||||
$buildStore->save($build);
|
||||
$build->sendStatusPostback();
|
||||
}
|
||||
|
||||
// Reset the config back to how it was prior to running this job:
|
||||
if (!empty($currentConfig)) {
|
||||
$config = new Config($currentConfig);
|
||||
Database::reset($config);
|
||||
}
|
||||
|
||||
// Delete the job when we're done:
|
||||
$pheanstalk->delete($job);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the worker after the current build.
|
||||
*/
|
||||
public function stopWorker()
|
||||
{
|
||||
$this->run = false;
|
||||
}
|
||||
}
|
|
@ -17,9 +17,11 @@ if (empty($timezone)) {
|
|||
|
||||
$configFile = dirname(__FILE__) . '/PHPCI/config.yml';
|
||||
$configEnv = getenv('phpci_config_file');
|
||||
$usingCustomConfigFile = false;
|
||||
|
||||
if (!empty($configEnv) && file_exists($configEnv)) {
|
||||
$configFile = $configEnv;
|
||||
$usingCustomConfigFile = true;
|
||||
}
|
||||
|
||||
// If we don't have a config file at all, fail at this point and tell the user to install:
|
||||
|
@ -53,6 +55,7 @@ $conf = array();
|
|||
$conf['b8']['app']['namespace'] = 'PHPCI';
|
||||
$conf['b8']['app']['default_controller'] = 'Home';
|
||||
$conf['b8']['view']['path'] = dirname(__FILE__) . '/PHPCI/View/';
|
||||
$conf['using_custom_file'] = $usingCustomConfigFile;
|
||||
|
||||
$config = new b8\Config($conf);
|
||||
|
||||
|
|
|
@ -47,7 +47,8 @@
|
|||
"monolog/monolog": "~1.6",
|
||||
"pimple/pimple": "~1.1",
|
||||
"robmorgan/phinx": "~0.4",
|
||||
"sensiolabs/ansi-to-html": "~1.1"
|
||||
"sensiolabs/ansi-to-html": "~1.1",
|
||||
"pda/pheanstalk": "~3.1"
|
||||
},
|
||||
|
||||
"require-dev": {
|
||||
|
|
761
composer.lock
generated
761
composer.lock
generated
File diff suppressed because it is too large
Load diff
2
console
2
console
|
@ -21,6 +21,7 @@ use PHPCI\Command\DaemonCommand;
|
|||
use PHPCI\Command\PollCommand;
|
||||
use PHPCI\Command\CreateAdminCommand;
|
||||
use PHPCI\Command\CreateBuildCommand;
|
||||
use PHPCI\Command\WorkerCommand;
|
||||
use PHPCI\Service\BuildService;
|
||||
use Symfony\Component\Console\Application;
|
||||
use b8\Store\Factory;
|
||||
|
@ -36,5 +37,6 @@ $application->add(new DaemonCommand($loggerConfig->getFor('DaemonCommand')));
|
|||
$application->add(new PollCommand($loggerConfig->getFor('PollCommand')));
|
||||
$application->add(new CreateAdminCommand(Factory::getStore('User')));
|
||||
$application->add(new CreateBuildCommand(Factory::getStore('Project'), new BuildService(Factory::getStore('Build'))));
|
||||
$application->add(new WorkerCommand($loggerConfig->getFor('WorkerCommand')));
|
||||
|
||||
$application->run();
|
||||
|
|
Loading…
Reference in a new issue