Merge branch 'feature-workers'

This commit is contained in:
Dmitry Khomutov 2017-01-20 21:56:44 +07:00
commit 285853efad
No known key found for this signature in database
GPG key ID: 7EB36C9576F9ECB9
20 changed files with 75 additions and 419 deletions

View file

@ -77,8 +77,8 @@ with web-server (Nginx or Apache2), MySQL (or MariaDB) database and Composer.
* [Add a virtual host to your web server](docs/en/virtual_host.md), pointing to the `public` directory within your new
PHP Censor directory. You'll need to set up rewrite rules to point all non-existent requests to PHP Censor;
* [Set up the PHP Censor Worker](docs/en/workers/worker.md), or you can run builds using the
[daemon](docs/en/workers/daemon.md) or [a cron-job](docs/en/workers/cron.md) to run PHP Censor builds;
* [Set up the PHP Censor Worker](docs/en/workers/worker.md), or [a cron-job](docs/en/workers/cron.md) to run PHP
Censor builds;
More details about [installation](docs/en/installing.md).

View file

@ -13,6 +13,15 @@ php-censor:
url: 'http://php-censor.local'
email_settings:
from_address: 'no-reply@php-censor.local'
smtp_address:
worker:
host: localhost
queue: php-censor-queue
github:
token: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
comments:
commit: false
pull_request: false
authentication_settings:
state: false
user_id: 1

View file

@ -12,7 +12,6 @@
use PHPCensor\Command\RunCommand;
use PHPCensor\Command\RebuildCommand;
use PHPCensor\Command\InstallCommand;
use PHPCensor\Command\DaemonCommand;
use PHPCensor\Command\PollCommand;
use PHPCensor\Command\CreateAdminCommand;
use PHPCensor\Command\CreateBuildCommand;
@ -34,7 +33,6 @@ $application = new Application();
$application->add(new RunCommand($loggerConfig->getFor('RunCommand')));
$application->add(new RebuildCommand($loggerConfig->getFor('RunCommand')));
$application->add(new InstallCommand);
$application->add(new DaemonCommand($loggerConfig->getFor('DaemonCommand')));
$application->add(new PollCommand($loggerConfig->getFor('PollCommand')));
$application->add(new CreateAdminCommand(Factory::getStore('User')));
$application->add(new CreateBuildCommand(Factory::getStore('Project'), new BuildService(Factory::getStore('Build'))));

View file

@ -1,21 +0,0 @@
#!/usr/bin/env php
<?php
/**
* PHPCI - Continuous Integration for PHP
*
* @copyright Copyright 2013, Block 8 Limited.
* @license https://github.com/Block8/PHPCI/blob/master/LICENSE.md
* @link http://www.phptesting.org/
*/
use PHPCensor\Command\DaemoniseCommand;
use Symfony\Component\Console\Application;
define('IS_CONSOLE', true);
require_once(dirname(__DIR__) . '/bootstrap.php');
$application = new Application();
$application->add(new DaemoniseCommand($loggerConfig->getFor('DaemoniseCommand')));
$application->run();

View file

@ -7,7 +7,6 @@ Getting Started
* [Installing PHP Censor](installing.md)
* [Adding a Virtual Host](virtual_host.md)
* [Run builds using a worker](workers/worker.md)
* [Run builds using a daemon](workers/daemon.md)
* [Run builds using cronjob](workers/cron.md)
* [Adding PHP Censor Support to Your Projects](config.md)
* [Setting up Logging](logging.md)

View file

@ -1,13 +1,18 @@
Configuring PHP Censor
======================
The PHP Censor configuration on the server is automatically generated into the `config.yml` file during installation. One might need to also edit the file manually.
The PHP Censor configuration on the server is automatically generated into the `config.yml` file during installation.
One might need to also edit the file manually.
For example, one could log into PHP Censor and go into the settings to disable it. But if you have already set up a username/password pair and have forgotten the password, and if the server is on a local network, and it's not sending the `forgot password` email, then editing the config file manually would be handy. To do so, just edit the `php-censor` section in the config file (which is in [yaml format](https://en.wikipedia.org/wiki/YAML)), and add
For example, one could log into PHP Censor and go into the settings to disable it. But if you have already set up a
username/password pair and have forgotten the password, and if the server is on a local network, and it's not sending
the `forgot password` email, then editing the config file manually would be handy. To do so, just edit the `php-censor`
section in the config file (which is in [yaml format](https://en.wikipedia.org/wiki/YAML)), and add
php-censor:
authentication_settings:
state: 1
user_id: 1
where you can get the user_id by logging into the mysql database and selecting your user ID from the `users` table in the PHP Censor database.
where you can get the user_id by logging into the mysql database and selecting your user ID from the `users` table in
the PHP Censor database.

View file

@ -20,5 +20,6 @@ Installing PHP Censor Manually
* Go to the PHP Censor directory: `cd /var/www/php-censor`
* Install dependencies using Composer: `composer install`
* Install PHP Censor itself: `./bin/console php-censor:install`
* [Add a virtual host to your web server](virtual_host.md), pointing to the `public` directory within your new PHP Censor directory. You'll need to set up rewrite rules to point all non-existent requests to PHP Censor.
* [Set up the PHP Censor Worker](workers/worker.md), or you can run builds using the [daemon](workers/daemon.md) or [a cron-job](workers/cron.md) to run PHP Censor builds.
* [Add a virtual host to your web server](virtual_host.md), pointing to the `public` directory within your new PHP
Censor directory. You'll need to set up rewrite rules to point all non-existent requests to PHP Censor.
* [Set up the PHP Censor Worker](workers/worker.md), or [a cron-job](workers/cron.md) to run PHP Censor builds.

View file

@ -1,9 +1,8 @@
Run builds using cronjob
========================
Running builds using cron is a quick and simple method of getting up and running with PHP Censor. It also removes the need for PHP Censor to be running all the time.
If you want a little more control over how PHP Censor runs, you may want to [set up the daemon](workers/daemon.md) instead.
Running builds using cron is a quick and simple method of getting up and running with PHP Censor. It also removes the
need for PHP Censor to be running all the time.
Setting up the Cron Job
-----------------------

View file

@ -1,23 +0,0 @@
Run builds using a daemon
=========================
The PHP Censor daemon runs in the background on your server and continuously checks for new builds. Unless already running a build, the daemon should pick up and start running new builds within seconds of being created.
The daemon is also useful if you want to run multiple PHP Censor workers in a virtualised environment (i.e. Docker)
If you want to run PHP Censor builds on a regular schedule instead, you should [set up a cronjob](workers/cron.md).
Starting the Daemon
-------------------
On a Linux/Unix server, the following command will start the daemon and keep it running even when you log out of the server:
```sh
nohup php ./daemonise php-censor:daemonise >/dev/null 2>&1 &
```
If you need to debug what's going on with your builds, you can also run the daemon directly using the following command, which will output the daemon's log directly to your terminal:
```sh
php daemonise php-censor:daemonise
```

View file

@ -1,16 +1,21 @@
Run builds using a worker
=========================
The PHP Censor Worker runs in the background on your server and waits for new builds to be added to a Beanstalkd queue. Unless already running a build, the worker will pick up and start running new builds almost immediately after their creation.
The PHP Censor Worker runs in the background on your server and waits for new builds to be added to a Beanstalkd queue.
Unless already running a build, the worker will pick up and start running new builds almost immediately after their
creation.
The worker is the recommended way to run PHP Censor builds. You can run several workers all watching one queue, allowing jobs to be run simultaneously without the overhead of polling your MySQL database.
The worker is the recommended way to run PHP Censor builds. You can run several workers all watching one queue,
allowing jobs to be run simultaneously without the overhead of polling your MySQL database.
If you can't run Beanstalkd on your server, or would prefer to run builds on a regular schedule, you should consider using the [build daemon](workers/daemon.md) or [running builds via Cron](workers/cron.md).
If you can't run Beanstalkd on your server, or would prefer to run builds on a regular schedule, you should consider
using the [running builds via Cron](workers/cron.md).
Pre-Requisites
--------------
* You need to install [Beanstalkd](http://kr.github.io/beanstalkd/) - On Ubuntu, this is as simple as running `apt-get install beanstalkd`.
* You need to install [Beanstalkd](http://kr.github.io/beanstalkd/) - On Ubuntu, this is as simple as running
`apt-get install beanstalkd`.
* [Supervisord](http://supervisord.org/) needs to be installed and running on your server.
Setting up the PHP Censor worker
@ -20,14 +25,10 @@ Setting up the PHP Censor worker
Setting up the worker on a new installation of PHP Censor is as simple as entering the appropriate values for your Beanstalkd server hostname and queue name when running the PHP Censor installer. By default, the installer assumes that you'll be using beanstalkd on `localhost` and will use the queue name `php-censor-queue`.
![PHP Censor Worker Installer](https://www.phptesting.org/media/render/f48f63699a04444630352643af18b643)
### On an existing installation
On an existing installation, to set up the worker, you simply need to add the beanstalkd host and queue names directly into your `config.yml` file. You should add a `worker` key beneath the `php-censor` section, with the properties `host` and `queue` as outlined in the screenshot below:
![PHP Censor Worker Config](https://www.phptesting.org/media/render/9a88e9298670f2913f5798e68b94c9ed)
Running the PHP Censor worker
-----------------------------

View file

@ -159,8 +159,8 @@ class Application extends b8\Application
protected function shouldSkipAuth()
{
$config = b8\Config::getInstance();
$state = (bool)$config->get('php-censor.authentication_settings.state', false);
$userId = $config->get('php-censor.authentication_settings.user_id', 0);
$state = (bool)$config->get('php-censor.authentication_settings.state', false);
$userId = $config->get('php-censor.authentication_settings.user_id', 0);
if (false !== $state && 0 != (int)$userId) {
$user = b8\Store\Factory::getStore('User')

View file

@ -20,9 +20,10 @@ use Symfony\Component\Console\Output\OutputInterface;
/**
* Create build command - creates a build for a project
* @author Jérémy DECOOL (@jdecool)
* @package PHPCI
* @subpackage Console
*
* @author Jérémy DECOOL (@jdecool)
* @package PHPCI
* @subpackage Console
*/
class CreateBuildCommand extends Command
{

View file

@ -1,202 +0,0 @@
<?php
/**
* PHPCI - Continuous Integration for PHP
*
* @copyright Copyright 2014, Block 8 Limited.
* @license https://github.com/Block8/PHPCI/blob/master/LICENSE.md
* @link https://www.phptesting.org/
*/
namespace PHPCensor\Command;
use Monolog\Logger;
use PHPCensor\ProcessControl\Factory;
use PHPCensor\ProcessControl\ProcessControlInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Daemon that loops and call the run-command.
* @author Gabriel Baker <gabriel.baker@autonomicpilot.co.uk>
* @package PHPCI
* @subpackage Console
*/
class DaemonCommand extends Command
{
/**
* @var Logger
*/
protected $logger;
/**
* @var string
*/
protected $pidFilePath;
/**
* @var string
*/
protected $logFilePath;
/**
* @var ProcessControlInterface
*/
protected $processControl;
public function __construct(Logger $logger, ProcessControlInterface $processControl = null, $name = null)
{
parent::__construct($name);
$this->logger = $logger;
$this->processControl = $processControl ?: Factory::getInstance();
}
protected function configure()
{
$this
->setName('php-censor:daemon')
->setDescription('Initiates the daemon to run commands.')
->addArgument(
'state', InputArgument::REQUIRED, 'start|stop|status'
)->addOption(
'pid-file',
'p',
InputOption::VALUE_REQUIRED,
'Path of the PID file',
(ROOT_DIR . 'runtime' . DIRECTORY_SEPARATOR . 'daemon' . DIRECTORY_SEPARATOR . 'daemon.pid')
)->addOption(
'log-file',
'l',
InputOption::VALUE_REQUIRED,
'Path of the log file',
(ROOT_DIR . 'runtime' . DIRECTORY_SEPARATOR . 'daemon' . DIRECTORY_SEPARATOR . 'daemon.log')
);
}
/**
* Loops through running.
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->pidFilePath = $input->getOption('pid-file');
$this->logFilePath = $input->getOption('log-file');
$state = $input->getArgument('state');
switch ($state) {
case 'start':
$this->startDaemon();
break;
case 'stop':
$this->stopDaemon();
break;
case 'status':
$this->statusDaemon($output);
break;
default:
$this->output->writeln("<error>Not a valid choice, please use start, stop or status</error>");
break;
}
}
protected function startDaemon()
{
$pid = $this->getRunningPid();
if ($pid) {
$this->logger->notice("Daemon already started", ['pid' => $pid]);
return "alreadystarted";
}
$this->logger->info("Trying to start the daemon");
$cmd = "nohup %sdaemonise php-censor:daemonise > %s 2>&1 &";
$command = sprintf($cmd, BIN_DIR, $this->logFilePath);
$output = $exitCode = null;
exec($command, $output, $exitCode);
if ($exitCode !== 0) {
$this->logger->error(sprintf("daemonise exited with status %d", $exitCode));
return "notstarted";
}
for ($i = 0; !($pid = $this->getRunningPid()) && $i < 5; $i++) {
sleep(1);
}
if (!$pid) {
$this->logger->error("Could not start the daemon");
return "notstarted";
}
$this->logger->notice("Daemon started", ['pid' => $pid]);
return "started";
}
protected function stopDaemon()
{
$pid = $this->getRunningPid();
if (!$pid) {
$this->logger->notice("Cannot stop the daemon as it is not started");
return "notstarted";
}
$this->logger->info("Trying to terminate the daemon", ['pid' => $pid]);
$this->processControl->kill($pid);
for ($i = 0; ($pid = $this->getRunningPid()) && $i < 5; $i++) {
sleep(1);
}
if ($pid) {
$this->logger->warning("The daemon is resiting, trying to kill it", ['pid' => $pid]);
$this->processControl->kill($pid, true);
for ($i = 0; ($pid = $this->getRunningPid()) && $i < 5; $i++) {
sleep(1);
}
}
if (!$pid) {
$this->logger->notice("Daemon stopped");
return "stopped";
}
$this->logger->error("Could not stop the daemon");
}
protected function statusDaemon(OutputInterface $output)
{
$pid = $this->getRunningPid();
if ($pid) {
$output->writeln(sprintf('The daemon is running, PID: %d', $pid));
return "running";
}
$output->writeln('The daemon is not running');
return "notrunning";
}
/** Check if there is a running daemon
*
* @return int|null
*/
protected function getRunningPid()
{
if (!file_exists($this->pidFilePath)) {
return;
}
$pid = intval(trim(file_get_contents($this->pidFilePath)));
if($this->processControl->isRunning($pid)) {
return $pid;
}
// Not found, remove the stale PID file
unlink($this->pidFilePath);
}
}

View file

@ -1,111 +0,0 @@
<?php
/**
* PHPCI - Continuous Integration for PHP
*
* @copyright Copyright 2014, Block 8 Limited.
* @license https://github.com/Block8/PHPCI/blob/master/LICENSE.md
* @link https://www.phptesting.org/
*/
namespace PHPCensor\Command;
use Monolog\Logger;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\ArgvInput;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Daemon that loops and call the run-command.
* @author Gabriel Baker <gabriel.baker@autonomicpilot.co.uk>
* @package PHPCI
* @subpackage Console
*/
class DaemoniseCommand extends Command
{
/**
* @var Logger
*/
protected $logger;
/**
* @var OutputInterface
*/
protected $output;
/**
* @var boolean
*/
protected $run;
/**
* @var int
*/
protected $sleep;
/**
* @param \Monolog\Logger $logger
* @param string $name
*/
public function __construct(Logger $logger, $name = null)
{
parent::__construct($name);
$this->logger = $logger;
}
protected function configure()
{
$this
->setName('php-censor:daemonise')
->setDescription('Starts the daemon to run commands.');
}
/**
* Loops through running.
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$cmd = "echo %s > '%sdaemon/daemon.pid'";
$command = sprintf($cmd, getmypid(), RUNTIME_DIR);
exec($command);
$this->output = $output;
$this->run = true;
$this->sleep = 0;
$runner = new RunCommand($this->logger);
$runner->setMaxBuilds(1);
$runner->setDaemon(true);
$emptyInput = new ArgvInput([]);
while ($this->run) {
$buildCount = 0;
try {
$buildCount = $runner->run($emptyInput, $output);
} catch (\Exception $e) {
$output->writeln('<error>Exception: ' . $e->getMessage() . '</error>');
$output->writeln('<error>Line: ' . $e->getLine() . ' - File: ' . $e->getFile() . '</error>');
}
if (0 == $buildCount && $this->sleep < 15) {
$this->sleep++;
} elseif (1 < $this->sleep) {
$this->sleep--;
}
echo '.'.(0 === $buildCount?'':'build');
sleep($this->sleep);
}
}
/**
* Called when log entries are made in Builder / the plugins.
*
* @see \PHPCensor\Builder::log()
*/
public function logCallback($log)
{
$this->output->writeln($log);
}
}

View file

@ -27,9 +27,10 @@ use Symfony\Component\Yaml\Dumper;
/**
* Install console command - Installs PHPCI.
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
class InstallCommand extends Command
{

View file

@ -18,11 +18,12 @@ use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Re-runs the last run build.
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
* Re-runs the last run build.
*
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
class RebuildCommand extends Command
{
/**
@ -69,7 +70,6 @@ class RebuildCommand extends Command
{
$runner = new RunCommand($this->logger);
$runner->setMaxBuilds(1);
$runner->setDaemon(false);
/** @var \PHPCensor\Store\BuildStore $store */
$store = Factory::getStore('Build');

View file

@ -21,10 +21,10 @@ use Symfony\Component\Console\Output\OutputInterface;
/**
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
class RebuildQueueCommand extends Command
{
/**

View file

@ -24,11 +24,12 @@ use PHPCensor\BuildFactory;
use PHPCensor\Model\Build;
/**
* Run console command - Runs any pending builds.
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
* Run console command - Runs any pending builds.
*
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
class RunCommand extends Command
{
/**
@ -46,11 +47,6 @@ class RunCommand extends Command
*/
protected $maxBuilds = 100;
/**
* @var bool
*/
protected $isFromDaemon = false;
/**
* @param \Monolog\Logger $logger
* @param string $name
@ -105,7 +101,7 @@ class RunCommand extends Command
$build = BuildFactory::getBuild($build);
// Skip build (for now) if there's already a build running in that project:
if (!$this->isFromDaemon && in_array($build->getProjectId(), $running)) {
if (in_array($build->getProjectId(), $running)) {
$this->logger->addInfo(Lang::get('skipping_build', $build->getId()));
$result['items'][] = $build;
@ -147,11 +143,6 @@ class RunCommand extends Command
$this->maxBuilds = (int)$numBuilds;
}
public function setDaemon($fromDaemon)
{
$this->isFromDaemon = (bool)$fromDaemon;
}
protected function validateRunningBuilds()
{
/** @var \PHPCensor\Store\BuildStore $store */

View file

@ -19,11 +19,12 @@ use Symfony\Component\Console\Output\OutputInterface;
/**
* Worker Command - Starts the BuildWorker, which pulls jobs from beanstalkd
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
* Worker Command - Starts the BuildWorker, which pulls jobs from beanstalkd
*
* @author Dan Cryer <dan@block8.co.uk>
* @package PHPCI
* @subpackage Console
*/
class WorkerCommand extends Command
{
/**

View file

@ -211,10 +211,17 @@ class GithubBuild extends RemoteGitBuild
$prNumber = $this->getExtra('pull_request_number');
$commit = $this->getCommitId();
$allowCommentCommit = Config::getInstance()->get('php-censor.github.comments.commit');
$allowCommentPullRequest = Config::getInstance()->get('php-censor.github.comments.pull_request');
if (!empty($prNumber)) {
$helper->createPullRequestComment($repo, $prNumber, $commit, $file, $diffLineNumber, $message);
if ($allowCommentPullRequest) {
$helper->createPullRequestComment($repo, $prNumber, $commit, $file, $diffLineNumber, $message);
}
} else {
$helper->createCommitComment($repo, $commit, $file, $diffLineNumber, $message);
if ($allowCommentCommit) {
$helper->createCommitComment($repo, $commit, $file, $diffLineNumber, $message);
}
}
}