[TEST] Automatic screenshots comparing and error checking in IE11, Firefox and Chrome (#715)

* taking screenshots

* let's try

* save artifacts

* better exceptiosn

* try to install geskodriver

* fix edge name

* add check for safari

* check os

* try this

* again

* fix syntax

* try this

* try firefox on windows

* and again

* handle error

* and again

* try older macos

* add firefox screenshot

* switch back env

* switch back env

* add IE screenshot

* try sudo for safari

* try not install nuget

* try more

* more

* try this

* install firefox

* add chrome

* increase threshold

* add firefox-darwin

* push

* fixing

* let's go

* increase threeshold

* again

* try safari tp

* tap cask versions

* fix conditions

* try like this

* last run

* increase threshold

* reenable macos firefox

* last try for safari

* rename screenshots to snapshots

* check console

* console workarounds

* fix safari misspleings

* logging is not supported by everyone

* maximize and set rect

* errors only for Chrome

* remove safari for now

* try to decrease threshold

* cleanup

* increase threeshold

* sleep more and increase threeshold

* add pupeeter

* handle errors

* build

* add prettier end of line

* add gitattributes

* add png to binary

* more attributest

* limit run

* run on chages to snapshots

* hey!

* make artifacts named as snapshots

* just for fun: we don't need express here

* update pupeeter snapshot

* no audit

* don't wait for quit?

* try more IE capabilities

* add wait timeout

* use server.js
This commit is contained in:
Konstantin Vyatkin 2019-11-02 07:18:19 -04:00 committed by Josh Johnson
parent 5afe8b5a7f
commit dddba5b35b
12 changed files with 429 additions and 3 deletions

64
.gitattributes vendored Normal file
View File

@ -0,0 +1,64 @@
## GITATTRIBUTES FOR WEB PROJECTS
#
# These settings are for any web project.
#
# Details per file setting:
# text These files should be normalized (i.e. convert CRLF to LF).
# binary These files are binary and should be left untouched.
#
# Note that binary is a macro for -text -diff.
######################################################################
# Auto detect
## Handle line endings automatically for files detected as
## text and leave all files detected as binary untouched.
## This will handle all files NOT defined below.
* text eol=lf
# Source code
*.css text eol=lf
*.html text diff=html eol=lf
*.js text eol=lf
*.json text eol=lf
*.scss text diff=css eol=lf
*.ts text eol=lf
# Documentation
*.md text eol=lf
*.txt text eol=lf
AUTHORS text eol=lf
CHANGELOG text eol=lf
CHANGES text eol=lf
CONTRIBUTING text eol=lf
COPYING text eol=lf
copyright text eol=lf
*COPYRIGHT* text eol=lf
INSTALL text eol=lf
license text eol=lf
LICENSE text eol=lf
NEWS text eol=lf
readme text eol=lf
*README* text eol=lf
TODO text eol=lf
# Linters
.eslintrc text eol=lf
.stylelintrc text eol=lf
# Configs
.babelrc text eol=lf
.browserslistrc text eol=lf
.editorconfig text eol=lf
.env text eol=lf
.gitattributes text eol=lf
.gitconfig text eol=lf
package-lock.json text -diff eol=lf
*.npmignore text eol=lf
*.yaml text eol=lf
*.yml text eol=lf
browserslist text eol=lf
# Graphics
# SVG treated as an asset (binary) by default.
*.svg text eol=lf
*.png binary

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 294 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 300 KiB

88
.github/actions-scripts/puppeteer.js vendored Normal file
View File

@ -0,0 +1,88 @@
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
const path = require('path');
const { once } = require('events');
const puppeteer = require('puppeteer');
const pixelmatch = require('pixelmatch');
const { PNG } = require('pngjs');
const server = require('../../server');
async function test() {
const browser = await puppeteer.launch();
const page = await browser.newPage();
let error;
let pixelDifference;
if (!server.listening) await once(server, 'listening');
try {
page.on('console', msg => {
if (msg.type() === 'error') throw new Error(msg.text());
});
page.on('pageerror', err => {
throw err;
});
await page.goto(`http://127.0.0.1:${server.address().port}`, {
waitUntil: 'networkidle2',
});
await page.setViewport({ width: 640, height: 1000 });
await page.click('label[for="choices-single-custom-templates"]');
await page.keyboard.press('ArrowDown');
await page.keyboard.press('ArrowDown');
const snapshotName = `puppeteer-${process.platform}.png`;
const artifactsPath = 'screenshot';
mkdirSync(artifactsPath, { recursive: true });
const imageBuffer = await page.screenshot({
path: path.join(artifactsPath, snapshotName),
fullPage: true,
});
// compare with snapshot
const screenshot = PNG.sync.read(imageBuffer);
const snapshot = PNG.sync.read(
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
);
const { width, height } = screenshot;
const diff = new PNG({ width, height });
pixelDifference = pixelmatch(
screenshot.data,
snapshot.data,
diff.data,
width,
height,
{
threshold: 0.6,
},
);
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
} catch (err) {
console.error(err);
error = err;
} finally {
await Promise.all([
browser.close(),
new Promise(resolve => server.close(resolve)),
]);
}
if (pixelDifference > 200) {
console.error(
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
);
process.exit(1);
}
if (error) process.exit(1);
}
process.on('unhandledRejection', err => {
console.error(err);
process.exit(1);
});
process.once('uncaughtException', err => {
console.error(err);
process.exit(1);
});
setImmediate(test);

155
.github/actions-scripts/selenium.js vendored Normal file
View File

@ -0,0 +1,155 @@
const path = require('path');
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
const { once } = require('events');
const pixelmatch = require('pixelmatch');
const { PNG } = require('pngjs');
const {
Builder,
By,
Key,
until,
Capabilities,
logging,
} = require('selenium-webdriver');
const server = require('../../server');
async function test() {
let pixelDifference;
let error;
let capabilities;
switch (process.env.BROWSER) {
case 'ie':
capabilities = Capabilities.ie();
capabilities.set('ignoreProtectedModeSettings', true);
capabilities.set('ignoreZoomSetting', true);
capabilities.set('ie.options', {
enableFullPageScreenshot: true,
ensureCleanSession: true,
});
break;
case 'edge':
capabilities = Capabilities.edge();
break;
case 'safari':
capabilities = Capabilities.safari();
capabilities.set('safari.options', { technologyPreview: false });
break;
case 'firefox': {
capabilities = Capabilities.firefox().setLoggingPrefs({ browser: 'ALL' });
break;
}
case 'chrome': {
capabilities = Capabilities.chrome().setLoggingPrefs({ browser: 'ALL' });
capabilities.set('chromeOptions', {
args: ['--headless', '--no-sandbox', '--disable-gpu'],
});
break;
}
}
let driver = await new Builder().withCapabilities(capabilities).build();
if (!server.listening) await once(server, 'listening');
try {
await driver.get(`http://127.0.0.1:${server.address().port}`);
// wait for last choice to init
await driver.wait(
until.elementLocated(By.css('#reset-multiple ~ .choices__list')),
10000,
'waiting for all Choices instances to init',
);
// Resize window
await driver
.manage()
.window()
.maximize();
await driver
.manage()
.window()
// magic numbers here to make sure all demo page are fit inside
.setRect({ x: 0, y: 0, width: 630, height: 4000 });
// and click on press space on it, so it should open choices
await driver
.findElement(By.css('#reset-multiple ~ .choices__list button'))
.sendKeys(Key.SPACE);
await driver.sleep(1000);
// take screenshot
const image = await driver.takeScreenshot();
const imageBuffer = Buffer.from(image, 'base64');
const snapshotName = `${process.env.BROWSER}-${process.platform}.png`;
const artifactsPath = 'screenshot';
mkdirSync(artifactsPath, { recursive: true });
writeFileSync(path.join(artifactsPath, snapshotName), imageBuffer);
// compare with snapshot
const screenshot = PNG.sync.read(imageBuffer);
const snapshot = PNG.sync.read(
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
);
const { width, height } = screenshot;
const diff = new PNG({ width, height });
pixelDifference = pixelmatch(
screenshot.data,
snapshot.data,
diff.data,
width,
height,
{
threshold: 1,
},
);
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
// getting console logs
// ensure no errors in console (only supported in Chrome currently)
if (process.env.BROWSER === 'chrome') {
const entries = await driver
.manage()
.logs()
.get(logging.Type.BROWSER);
if (
Array.isArray(entries) &&
entries.some(entry => entry.level.name_ === 'SEVERE')
)
throw new Error(JSON.stringify(entries));
}
} catch (err) {
console.error(err);
error = err;
} finally {
await Promise.all([
driver.quit(),
new Promise(resolve => server.close(resolve)),
]);
}
if (pixelDifference > 200) {
console.error(
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
);
process.exit(1);
}
if (error) process.exit(1);
}
process.on('unhandledRejection', err => {
console.error(err);
process.exit(1);
});
process.once('uncaughtException', err => {
console.error(err);
process.exit(1);
});
setImmediate(test);

116
.github/workflows/browsers.yml vendored Normal file
View File

@ -0,0 +1,116 @@
name: Browsers
on:
pull_request:
paths:
- 'src/**'
- 'package-lock.json'
- '.browserslistrc'
- '.babelrc'
- 'webpack.config.*'
- 'public/index.html'
- '.github/actions-scripts/__snapshots__/**'
jobs:
selenium:
strategy:
fail-fast: false
matrix:
os: [windows-latest, macOS-10.14]
browser: [ie, firefox]
exclude:
# On Windows, run tests with only IE and Edge
- os: windows-latest
browser: safari
# On macOS, run tests with only on safari
- os: macOS-10.14
browser: ie
- os: macOS-10.14
browser: chrome
runs-on: ${{ matrix.os }}
steps:
- name: Dump GitHub context
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
run: echo "$GITHUB_CONTEXT"
- uses: actions/checkout@v1
with:
fetch-depth: 1
- uses: actions/setup-node@v1
with:
node-version: '12.x'
- run: |
npm ci
npm run build
env:
CYPRESS_INSTALL_BINARY: 0
HUSKY_SKIP_INSTALL: true
# install drivers
- name: Enable Safari Driver
run: |
# brew tap homebrew/cask-versions
# brew cask install safari-technology-preview
sudo safaridriver --enable
defaults write -app Safari IncludeDevelopMenu 1
defaults write -app Safari AllowJavaScriptFromAppleEvents 1
defaults write -app Safari IncludeInternalDebugMenu 1
defaults write -app Safari AllowRemoteAutomation 1
safaridriver -p 0 &
if: matrix.browser == 'safari'
- run: |
brew cask install firefox
brew install geckodriver
if: matrix.browser == 'firefox' && matrix.os == 'macOS-10.14'
- run: echo "::add-path::$env:GeckoWebDriver"
if: matrix.browser == 'firefox' && matrix.os == 'windows-latest'
- run: echo "::add-path::$env:IEWebDriver"
if: matrix.browser == 'ie' && matrix.os == 'windows-latest'
- run: echo "::add-path::$env:ChromeWebDriver"
if: matrix.browser == 'chrome' && matrix.os == 'windows-latest'
- run: npm i --no-optional --no-audit selenium-webdriver pixelmatch pngjs
- run: node .github/actions-scripts/selenium.js
env:
BROWSER: ${{ matrix.browser }}
PORT: 0
NODE_ENV: production # prevent watching
- uses: actions/upload-artifact@master
if: failure()
with:
name: screenshot-${{ matrix.browser }}-${{ matrix.os }}
path: screenshot
puppeteer:
runs-on: macos-latest
steps:
- uses: actions/checkout@v1
with:
fetch-depth: 1
- uses: actions/setup-node@v1
with:
node-version: '12.x'
- run: |
npm ci
npm run build
env:
CYPRESS_INSTALL_BINARY: 0
HUSKY_SKIP_INSTALL: true
- run: npm i --no-optional --no-audit puppeteer pixelmatch pngjs
- run: node .github/actions-scripts/puppeteer.js
env:
PORT: 0
NODE_ENV: production # prevent watching
- uses: actions/upload-artifact@master
if: failure()
with:
name: screenshot-puppeteer-darwin
path: screenshot

View File

@ -1,6 +1,7 @@
{
"singleQuote": true,
"trailingComma": "all",
"endOfLine": "lf",
"overrides": [
{
"files": ["*.svg"],

View File

@ -2,8 +2,8 @@
const express = require('express');
const path = require('path');
const PORT = 3001;
const DIST_DIR = path.join(__dirname, 'public');
const PORT = process.env.PORT || 3001;
const DIST_DIR = path.resolve(__dirname, 'public');
const app = express();
@ -51,7 +51,7 @@ const server = app.listen(PORT, err => {
console.log(err);
}
console.log(`Listening at http://localhost:${PORT} 👂`);
console.log(`Listening at http://localhost:${server.address().port} 👂`);
});
process.on('SIGTERM', () => {
@ -70,3 +70,5 @@ process.on('SIGTERM', () => {
process.exit(0);
}
});
module.exports = server;