Adding support for beanstalkd-based workers.

PHPCI/Command/WorkerCommand.php

@@ -0,0 +1,80 @@
+<?php
+/**
+ * PHPCI - Continuous Integration for PHP
+ *
+ * @copyright    Copyright 2015, Block 8 Limited.
+ * @license      https://github.com/Block8/PHPCI/blob/master/LICENSE.md
+ * @link         https://www.phptesting.org/
+ */
+
+namespace PHPCI\Command;
+
+use b8\Config;
+use Monolog\Logger;
+use PHPCI\Logging\OutputLogHandler;
+use PHPCI\Worker\BuildWorker;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+/**
+* Worker Command - Starts the BuildWorker, which pulls jobs from beanstalkd
+* @author       Dan Cryer <dan@block8.co.uk>
+* @package      PHPCI
+* @subpackage   Console
+*/
+class WorkerCommand extends Command
+{
+    /**
+     * @var OutputInterface
+     */
+    protected $output;
+
+    /**
+     * @var Logger
+     */
+    protected $logger;
+
+    /**
+     * @param \Monolog\Logger $logger
+     * @param string $name
+     */
+    public function __construct(Logger $logger, $name = null)
+    {
+        parent::__construct($name);
+        $this->logger = $logger;
+    }
+
+    protected function configure()
+    {
+        $this
+            ->setName('phpci:worker')
+            ->setDescription('Runs the PHPCI build worker.');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $this->output = $output;
+
+        // For verbose mode we want to output all informational and above
+        // messages to the symphony output interface.
+        if ($input->hasOption('verbose') && $input->getOption('verbose')) {
+            $this->logger->pushHandler(
+                new OutputLogHandler($this->output, Logger::INFO)
+            );
+        }
+
+        $config = Config::getInstance()->get('phpci.worker', []);
+
+        if (empty($config['host']) || empty($config['queue'])) {
+            $error = 'The worker is not configured. You must set a host and queue in your config.yml file.';
+            throw new \Exception($error);
+        }
+
+        $worker = new BuildWorker($config['host'], $config['queue']);
+        $worker->setLogger($this->logger);
+        $worker->setMaxJobs(Config::getInstance()->get('phpci.worker.max_jobs', -1));
+        $worker->startWorker();
+    }
+}

PHPCI/Controller/WebhookController.php

@@ -364,10 +364,6 @@ class WebhookController extends \b8\Controller
 
         // If not, create a new build job for it:
         $build = $this->buildService->createBuild($project, $commitId, $branch, $committer, $commitMessage, $extra);
-        $build = BuildFactory::getBuild($build);
-
-        // Send a status postback if the build type provides one:
-        $build->sendStatusPostback();
 
         return array('status' => 'ok', 'buildID' => $build->getID());
     }

PHPCI/Model/Build.php

@@ -228,7 +228,7 @@ class Build extends BuildBase
         if (!$this->getId()) {
             return null;
         }
-        return PHPCI_BUILD_ROOT_DIR . $this->getId();
+        return PHPCI_BUILD_ROOT_DIR . $this->getId() . '_' . substr(md5(microtime(true)), 0, 5);
     }
 
     /**

PHPCI/Service/BuildService.php

@@ -9,6 +9,10 @@
 
 namespace PHPCI\Service;
 
+use b8\Config;
+use Pheanstalk\Pheanstalk;
+use Pheanstalk\PheanstalkInterface;
+use PHPCI\BuildFactory;
 use PHPCI\Helper\Lang;
 use PHPCI\Model\Build;
 use PHPCI\Model\Project;
@@ -81,7 +85,13 @@ class BuildService
             $build->setExtra(json_encode($extra));
         }
 
-        return $this->buildStore->save($build);
+        $build = $this->buildStore->save($build);
+
+        $build = BuildFactory::getBuild($build);
+        $build->sendStatusPostback();
+        $this->addBuildToQueue($build);
+
+        return $build;
     }
 
     /**
@@ -117,4 +127,31 @@ class BuildService
         $build->removeBuildDirectory();
         return $this->buildStore->delete($build);
     }
+
+
+    public function addBuildToQueue(Build $build)
+    {
+        $config = Config::getInstance();
+
+        $settings = $config->get('phpci.worker', []);
+
+        if (!empty($settings['host']) && !empty($settings['queue'])) {
+            $jobData = array(
+                'build_id' => $build->getId(),
+            );
+
+            if ($config->get('using_custom_file')) {
+                $jobData['config'] = $config->getArray();
+            }
+
+            $pheanstalk = new Pheanstalk($settings['host']);
+            $pheanstalk->useTube($settings['queue']);
+            $pheanstalk->put(
+                json_encode($jobData),
+                PheanstalkInterface::DEFAULT_PRIORITY,
+                PheanstalkInterface::DEFAULT_DELAY,
+                $config->get('phpci.worker.job_timeout', 600)
+            );
+        }
+    }
 }

PHPCI/Worker/BuildWorker.php

@@ -0,0 +1,160 @@
+<?php
+
+namespace PHPCI\Worker;
+
+use b8\Config;
+use b8\Database;
+use b8\Store\Factory;
+use Monolog\Logger;
+use Pheanstalk\Pheanstalk;
+use PHPCI\Builder;
+use PHPCI\BuildFactory;
+use PHPCI\Logging\BuildDBLogHandler;
+use PHPCI\Model\Build;
+
+
+class BuildWorker
+{
+    /**
+     * If this variable changes to false, the worker will stop after the current build.
+     * @var bool
+     */
+    protected $run = true;
+
+    /**
+     * The maximum number of jobs this worker should run before exiting.
+     * Use -1 for no limit.
+     * @var int
+     */
+    protected $maxJobs = -1;
+
+    /**
+     * The logger for builds to use.
+     * @var \Monolog\Logger
+     */
+    protected $logger;
+
+    /**
+     * beanstalkd host
+     * @var string
+     */
+    protected $host;
+
+    /**
+     * beanstalkd queue to watch
+     * @var string
+     */
+    protected $queue;
+
+    /**
+     * @param $host
+     * @param $queue
+     */
+    public function __construct($host, $queue)
+    {
+        $this->host = $host;
+        $this->queue = $queue;
+    }
+
+    /**
+     * @param int $maxJobs
+     */
+    public function setMaxJobs($maxJobs = -1)
+    {
+        $this->maxJobs = $maxJobs;
+    }
+
+    /**
+     * @param Logger $logger
+     */
+    public function setLogger(Logger $logger)
+    {
+        $this->logger = $logger;
+    }
+
+    /**
+     * Start the worker.
+     */
+    public function startWorker()
+    {
+        $pheanstalk = new Pheanstalk($this->host);
+        $pheanstalk->watch($this->queue);
+        $buildStore = Factory::getStore('Build');
+
+        $jobs = 0;
+
+        while ($this->run) {
+            // Get a job from the queue:
+            $job = $pheanstalk->reserve();
+
+            // Make sure we don't run more than maxJobs jobs on this worker:
+            $jobs++;
+
+            if ($this->maxJobs != -1 && $this->maxJobs <= $jobs) {
+                $this->run = false;
+            }
+
+            // Get the job data and run the job:
+            $jobData = json_decode($job->getData(), true);
+            $this->logger->addInfo('Received build #'.$jobData['build_id'].' from Beanstalkd');
+
+            // If the job comes with config data, reset our config and database connections
+            // and then make sure we kill the worker afterwards:
+            if (!empty($jobData['config'])) {
+                $this->logger->addDebug('Using job-specific config.');
+                $currentConfig = Config::getInstance()->getArray();
+                $config = new Config($jobData['config']);
+                Database::reset($config);
+            }
+
+            $build = BuildFactory::getBuildById($jobData['build_id']);
+
+            if (empty($build)) {
+                $this->logger->addWarning('Build #' . $jobData['build_id'] . ' does not exist in the database.');
+                $pheanstalk->delete($job);
+            }
+
+            try {
+                // Logging relevant to this build should be stored
+                // against the build itself.
+                $buildDbLog = new BuildDBLogHandler($build, Logger::INFO);
+                $this->logger->pushHandler($buildDbLog);
+
+                $builder = new Builder($build, $this->logger);
+                $builder->execute();
+
+                // After execution we no longer want to record the information
+                // back to this specific build so the handler should be removed.
+                $this->logger->popHandler($buildDbLog);
+            } catch (\PDOException $ex) {
+                // If we've caught a PDO Exception, it is probably not the fault of the build, but of a failed
+                // connection or similar. Release the job and kill the worker.
+                $this->run = false;
+                $pheanstalk->release($job);
+            } catch (\Exception $ex) {
+                $build->setStatus(Build::STATUS_FAILED);
+                $build->setFinished(new \DateTime());
+                $build->setLog($build->getLog() . PHP_EOL . PHP_EOL . $ex->getMessage());
+                $buildStore->save($build);
+                $build->sendStatusPostback();
+            }
+
+            // Reset the config back to how it was prior to running this job:
+            if (!empty($currentConfig)) {
+                $config = new Config($currentConfig);
+                Database::reset($config);
+            }
+
+            // Delete the job when we're done:
+            $pheanstalk->delete($job);
+        }
+    }
+
+    /**
+     * Stops the worker after the current build.
+     */
+    public function stopWorker()
+    {
+        $this->run = false;
+    }
+}

bootstrap.php

@@ -17,9 +17,11 @@ if (empty($timezone)) {
 
 $configFile = dirname(__FILE__) . '/PHPCI/config.yml';
 $configEnv = getenv('phpci_config_file');
+$usingCustomConfigFile = false;
 
 if (!empty($configEnv) && file_exists($configEnv)) {
     $configFile = $configEnv;
+    $usingCustomConfigFile = true;
 }
 
 // If we don't have a config file at all, fail at this point and tell the user to install:
@@ -53,6 +55,7 @@ $conf = array();
 $conf['b8']['app']['namespace'] = 'PHPCI';
 $conf['b8']['app']['default_controller'] = 'Home';
 $conf['b8']['view']['path'] = dirname(__FILE__) . '/PHPCI/View/';
+$conf['using_custom_file'] = $usingCustomConfigFile;
 
 $config = new b8\Config($conf);
 

composer.json

@@ -47,7 +47,8 @@
         "monolog/monolog": "~1.6",
         "pimple/pimple": "~1.1",
         "robmorgan/phinx": "~0.4",
-        "sensiolabs/ansi-to-html": "~1.1"
+        "sensiolabs/ansi-to-html": "~1.1",
+        "pda/pheanstalk": "~3.1"
     },
 
     "require-dev": {

console

@@ -21,6 +21,7 @@ use PHPCI\Command\DaemonCommand;
 use PHPCI\Command\PollCommand;
 use PHPCI\Command\CreateAdminCommand;
 use PHPCI\Command\CreateBuildCommand;
+use PHPCI\Command\WorkerCommand;
 use PHPCI\Service\BuildService;
 use Symfony\Component\Console\Application;
 use b8\Store\Factory;
@@ -36,5 +37,6 @@ $application->add(new DaemonCommand($loggerConfig->getFor('DaemonCommand')));
 $application->add(new PollCommand($loggerConfig->getFor('PollCommand')));
 $application->add(new CreateAdminCommand(Factory::getStore('User')));
 $application->add(new CreateBuildCommand(Factory::getStore('Project'), new BuildService(Factory::getStore('Build'))));
+$application->add(new WorkerCommand($loggerConfig->getFor('WorkerCommand')));
 
 $application->run();