mirror of
https://github.com/Choices-js/Choices.git
synced 2024-06-15 20:25:18 +02:00
[TEST] Automatic screenshots comparing and error checking in IE11, Firefox and Chrome (#715)
* taking screenshots * let's try * save artifacts * better exceptiosn * try to install geskodriver * fix edge name * add check for safari * check os * try this * again * fix syntax * try this * try firefox on windows * and again * handle error * and again * try older macos * add firefox screenshot * switch back env * switch back env * add IE screenshot * try sudo for safari * try not install nuget * try more * more * try this * install firefox * add chrome * increase threshold * add firefox-darwin * push * fixing * let's go * increase threeshold * again * try safari tp * tap cask versions * fix conditions * try like this * last run * increase threshold * reenable macos firefox * last try for safari * rename screenshots to snapshots * check console * console workarounds * fix safari misspleings * logging is not supported by everyone * maximize and set rect * errors only for Chrome * remove safari for now * try to decrease threshold * cleanup * increase threeshold * sleep more and increase threeshold * add pupeeter * handle errors * build * add prettier end of line * add gitattributes * add png to binary * more attributest * limit run * run on chages to snapshots * hey! * make artifacts named as snapshots * just for fun: we don't need express here * update pupeeter snapshot * no audit * don't wait for quit? * try more IE capabilities * add wait timeout * use server.js
This commit is contained in:
parent
5afe8b5a7f
commit
dddba5b35b
64
.gitattributes
vendored
Normal file
64
.gitattributes
vendored
Normal file
|
@ -0,0 +1,64 @@
|
||||||
|
## GITATTRIBUTES FOR WEB PROJECTS
|
||||||
|
#
|
||||||
|
# These settings are for any web project.
|
||||||
|
#
|
||||||
|
# Details per file setting:
|
||||||
|
# text These files should be normalized (i.e. convert CRLF to LF).
|
||||||
|
# binary These files are binary and should be left untouched.
|
||||||
|
#
|
||||||
|
# Note that binary is a macro for -text -diff.
|
||||||
|
######################################################################
|
||||||
|
|
||||||
|
# Auto detect
|
||||||
|
## Handle line endings automatically for files detected as
|
||||||
|
## text and leave all files detected as binary untouched.
|
||||||
|
## This will handle all files NOT defined below.
|
||||||
|
* text eol=lf
|
||||||
|
|
||||||
|
# Source code
|
||||||
|
*.css text eol=lf
|
||||||
|
*.html text diff=html eol=lf
|
||||||
|
*.js text eol=lf
|
||||||
|
*.json text eol=lf
|
||||||
|
*.scss text diff=css eol=lf
|
||||||
|
*.ts text eol=lf
|
||||||
|
|
||||||
|
# Documentation
|
||||||
|
*.md text eol=lf
|
||||||
|
*.txt text eol=lf
|
||||||
|
AUTHORS text eol=lf
|
||||||
|
CHANGELOG text eol=lf
|
||||||
|
CHANGES text eol=lf
|
||||||
|
CONTRIBUTING text eol=lf
|
||||||
|
COPYING text eol=lf
|
||||||
|
copyright text eol=lf
|
||||||
|
*COPYRIGHT* text eol=lf
|
||||||
|
INSTALL text eol=lf
|
||||||
|
license text eol=lf
|
||||||
|
LICENSE text eol=lf
|
||||||
|
NEWS text eol=lf
|
||||||
|
readme text eol=lf
|
||||||
|
*README* text eol=lf
|
||||||
|
TODO text eol=lf
|
||||||
|
|
||||||
|
# Linters
|
||||||
|
.eslintrc text eol=lf
|
||||||
|
.stylelintrc text eol=lf
|
||||||
|
|
||||||
|
# Configs
|
||||||
|
.babelrc text eol=lf
|
||||||
|
.browserslistrc text eol=lf
|
||||||
|
.editorconfig text eol=lf
|
||||||
|
.env text eol=lf
|
||||||
|
.gitattributes text eol=lf
|
||||||
|
.gitconfig text eol=lf
|
||||||
|
package-lock.json text -diff eol=lf
|
||||||
|
*.npmignore text eol=lf
|
||||||
|
*.yaml text eol=lf
|
||||||
|
*.yml text eol=lf
|
||||||
|
browserslist text eol=lf
|
||||||
|
|
||||||
|
# Graphics
|
||||||
|
# SVG treated as an asset (binary) by default.
|
||||||
|
*.svg text eol=lf
|
||||||
|
*.png binary
|
BIN
.github/actions-scripts/__snapshots__/chrome-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/chrome-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 25 KiB |
BIN
.github/actions-scripts/__snapshots__/firefox-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/firefox-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 48 KiB |
BIN
.github/actions-scripts/__snapshots__/firefox-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/firefox-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 108 KiB |
BIN
.github/actions-scripts/__snapshots__/ie-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/ie-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 22 KiB |
BIN
.github/actions-scripts/__snapshots__/puppeteer-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/puppeteer-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 294 KiB |
BIN
.github/actions-scripts/__snapshots__/safari-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/safari-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 300 KiB |
88
.github/actions-scripts/puppeteer.js
vendored
Normal file
88
.github/actions-scripts/puppeteer.js
vendored
Normal file
|
@ -0,0 +1,88 @@
|
||||||
|
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const { once } = require('events');
|
||||||
|
|
||||||
|
const puppeteer = require('puppeteer');
|
||||||
|
const pixelmatch = require('pixelmatch');
|
||||||
|
const { PNG } = require('pngjs');
|
||||||
|
|
||||||
|
const server = require('../../server');
|
||||||
|
|
||||||
|
async function test() {
|
||||||
|
const browser = await puppeteer.launch();
|
||||||
|
const page = await browser.newPage();
|
||||||
|
let error;
|
||||||
|
let pixelDifference;
|
||||||
|
|
||||||
|
if (!server.listening) await once(server, 'listening');
|
||||||
|
|
||||||
|
try {
|
||||||
|
page.on('console', msg => {
|
||||||
|
if (msg.type() === 'error') throw new Error(msg.text());
|
||||||
|
});
|
||||||
|
page.on('pageerror', err => {
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto(`http://127.0.0.1:${server.address().port}`, {
|
||||||
|
waitUntil: 'networkidle2',
|
||||||
|
});
|
||||||
|
await page.setViewport({ width: 640, height: 1000 });
|
||||||
|
await page.click('label[for="choices-single-custom-templates"]');
|
||||||
|
await page.keyboard.press('ArrowDown');
|
||||||
|
await page.keyboard.press('ArrowDown');
|
||||||
|
|
||||||
|
const snapshotName = `puppeteer-${process.platform}.png`;
|
||||||
|
const artifactsPath = 'screenshot';
|
||||||
|
mkdirSync(artifactsPath, { recursive: true });
|
||||||
|
const imageBuffer = await page.screenshot({
|
||||||
|
path: path.join(artifactsPath, snapshotName),
|
||||||
|
fullPage: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// compare with snapshot
|
||||||
|
const screenshot = PNG.sync.read(imageBuffer);
|
||||||
|
const snapshot = PNG.sync.read(
|
||||||
|
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
|
||||||
|
);
|
||||||
|
const { width, height } = screenshot;
|
||||||
|
const diff = new PNG({ width, height });
|
||||||
|
pixelDifference = pixelmatch(
|
||||||
|
screenshot.data,
|
||||||
|
snapshot.data,
|
||||||
|
diff.data,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
{
|
||||||
|
threshold: 0.6,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
error = err;
|
||||||
|
} finally {
|
||||||
|
await Promise.all([
|
||||||
|
browser.close(),
|
||||||
|
new Promise(resolve => server.close(resolve)),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pixelDifference > 200) {
|
||||||
|
console.error(
|
||||||
|
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (error) process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('unhandledRejection', err => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
process.once('uncaughtException', err => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
setImmediate(test);
|
155
.github/actions-scripts/selenium.js
vendored
Normal file
155
.github/actions-scripts/selenium.js
vendored
Normal file
|
@ -0,0 +1,155 @@
|
||||||
|
const path = require('path');
|
||||||
|
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
|
||||||
|
const { once } = require('events');
|
||||||
|
|
||||||
|
const pixelmatch = require('pixelmatch');
|
||||||
|
const { PNG } = require('pngjs');
|
||||||
|
const {
|
||||||
|
Builder,
|
||||||
|
By,
|
||||||
|
Key,
|
||||||
|
until,
|
||||||
|
Capabilities,
|
||||||
|
logging,
|
||||||
|
} = require('selenium-webdriver');
|
||||||
|
|
||||||
|
const server = require('../../server');
|
||||||
|
|
||||||
|
async function test() {
|
||||||
|
let pixelDifference;
|
||||||
|
let error;
|
||||||
|
|
||||||
|
let capabilities;
|
||||||
|
switch (process.env.BROWSER) {
|
||||||
|
case 'ie':
|
||||||
|
capabilities = Capabilities.ie();
|
||||||
|
capabilities.set('ignoreProtectedModeSettings', true);
|
||||||
|
capabilities.set('ignoreZoomSetting', true);
|
||||||
|
capabilities.set('ie.options', {
|
||||||
|
enableFullPageScreenshot: true,
|
||||||
|
ensureCleanSession: true,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'edge':
|
||||||
|
capabilities = Capabilities.edge();
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'safari':
|
||||||
|
capabilities = Capabilities.safari();
|
||||||
|
capabilities.set('safari.options', { technologyPreview: false });
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'firefox': {
|
||||||
|
capabilities = Capabilities.firefox().setLoggingPrefs({ browser: 'ALL' });
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'chrome': {
|
||||||
|
capabilities = Capabilities.chrome().setLoggingPrefs({ browser: 'ALL' });
|
||||||
|
capabilities.set('chromeOptions', {
|
||||||
|
args: ['--headless', '--no-sandbox', '--disable-gpu'],
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let driver = await new Builder().withCapabilities(capabilities).build();
|
||||||
|
|
||||||
|
if (!server.listening) await once(server, 'listening');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await driver.get(`http://127.0.0.1:${server.address().port}`);
|
||||||
|
|
||||||
|
// wait for last choice to init
|
||||||
|
await driver.wait(
|
||||||
|
until.elementLocated(By.css('#reset-multiple ~ .choices__list')),
|
||||||
|
10000,
|
||||||
|
'waiting for all Choices instances to init',
|
||||||
|
);
|
||||||
|
|
||||||
|
// Resize window
|
||||||
|
await driver
|
||||||
|
.manage()
|
||||||
|
.window()
|
||||||
|
.maximize();
|
||||||
|
await driver
|
||||||
|
.manage()
|
||||||
|
.window()
|
||||||
|
// magic numbers here to make sure all demo page are fit inside
|
||||||
|
.setRect({ x: 0, y: 0, width: 630, height: 4000 });
|
||||||
|
|
||||||
|
// and click on press space on it, so it should open choices
|
||||||
|
await driver
|
||||||
|
.findElement(By.css('#reset-multiple ~ .choices__list button'))
|
||||||
|
.sendKeys(Key.SPACE);
|
||||||
|
await driver.sleep(1000);
|
||||||
|
|
||||||
|
// take screenshot
|
||||||
|
const image = await driver.takeScreenshot();
|
||||||
|
const imageBuffer = Buffer.from(image, 'base64');
|
||||||
|
|
||||||
|
const snapshotName = `${process.env.BROWSER}-${process.platform}.png`;
|
||||||
|
const artifactsPath = 'screenshot';
|
||||||
|
mkdirSync(artifactsPath, { recursive: true });
|
||||||
|
|
||||||
|
writeFileSync(path.join(artifactsPath, snapshotName), imageBuffer);
|
||||||
|
|
||||||
|
// compare with snapshot
|
||||||
|
const screenshot = PNG.sync.read(imageBuffer);
|
||||||
|
const snapshot = PNG.sync.read(
|
||||||
|
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
|
||||||
|
);
|
||||||
|
const { width, height } = screenshot;
|
||||||
|
const diff = new PNG({ width, height });
|
||||||
|
pixelDifference = pixelmatch(
|
||||||
|
screenshot.data,
|
||||||
|
snapshot.data,
|
||||||
|
diff.data,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
{
|
||||||
|
threshold: 1,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
|
||||||
|
|
||||||
|
// getting console logs
|
||||||
|
// ensure no errors in console (only supported in Chrome currently)
|
||||||
|
if (process.env.BROWSER === 'chrome') {
|
||||||
|
const entries = await driver
|
||||||
|
.manage()
|
||||||
|
.logs()
|
||||||
|
.get(logging.Type.BROWSER);
|
||||||
|
if (
|
||||||
|
Array.isArray(entries) &&
|
||||||
|
entries.some(entry => entry.level.name_ === 'SEVERE')
|
||||||
|
)
|
||||||
|
throw new Error(JSON.stringify(entries));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
error = err;
|
||||||
|
} finally {
|
||||||
|
await Promise.all([
|
||||||
|
driver.quit(),
|
||||||
|
new Promise(resolve => server.close(resolve)),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
if (pixelDifference > 200) {
|
||||||
|
console.error(
|
||||||
|
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (error) process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('unhandledRejection', err => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
process.once('uncaughtException', err => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
setImmediate(test);
|
116
.github/workflows/browsers.yml
vendored
Normal file
116
.github/workflows/browsers.yml
vendored
Normal file
|
@ -0,0 +1,116 @@
|
||||||
|
name: Browsers
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- 'src/**'
|
||||||
|
- 'package-lock.json'
|
||||||
|
- '.browserslistrc'
|
||||||
|
- '.babelrc'
|
||||||
|
- 'webpack.config.*'
|
||||||
|
- 'public/index.html'
|
||||||
|
- '.github/actions-scripts/__snapshots__/**'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
selenium:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [windows-latest, macOS-10.14]
|
||||||
|
browser: [ie, firefox]
|
||||||
|
exclude:
|
||||||
|
# On Windows, run tests with only IE and Edge
|
||||||
|
- os: windows-latest
|
||||||
|
browser: safari
|
||||||
|
# On macOS, run tests with only on safari
|
||||||
|
- os: macOS-10.14
|
||||||
|
browser: ie
|
||||||
|
- os: macOS-10.14
|
||||||
|
browser: chrome
|
||||||
|
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
steps:
|
||||||
|
- name: Dump GitHub context
|
||||||
|
env:
|
||||||
|
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||||
|
run: echo "$GITHUB_CONTEXT"
|
||||||
|
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
with:
|
||||||
|
fetch-depth: 1
|
||||||
|
- uses: actions/setup-node@v1
|
||||||
|
with:
|
||||||
|
node-version: '12.x'
|
||||||
|
- run: |
|
||||||
|
npm ci
|
||||||
|
npm run build
|
||||||
|
env:
|
||||||
|
CYPRESS_INSTALL_BINARY: 0
|
||||||
|
HUSKY_SKIP_INSTALL: true
|
||||||
|
|
||||||
|
# install drivers
|
||||||
|
- name: Enable Safari Driver
|
||||||
|
run: |
|
||||||
|
# brew tap homebrew/cask-versions
|
||||||
|
# brew cask install safari-technology-preview
|
||||||
|
sudo safaridriver --enable
|
||||||
|
defaults write -app Safari IncludeDevelopMenu 1
|
||||||
|
defaults write -app Safari AllowJavaScriptFromAppleEvents 1
|
||||||
|
defaults write -app Safari IncludeInternalDebugMenu 1
|
||||||
|
defaults write -app Safari AllowRemoteAutomation 1
|
||||||
|
safaridriver -p 0 &
|
||||||
|
if: matrix.browser == 'safari'
|
||||||
|
|
||||||
|
- run: |
|
||||||
|
brew cask install firefox
|
||||||
|
brew install geckodriver
|
||||||
|
if: matrix.browser == 'firefox' && matrix.os == 'macOS-10.14'
|
||||||
|
|
||||||
|
- run: echo "::add-path::$env:GeckoWebDriver"
|
||||||
|
if: matrix.browser == 'firefox' && matrix.os == 'windows-latest'
|
||||||
|
|
||||||
|
- run: echo "::add-path::$env:IEWebDriver"
|
||||||
|
if: matrix.browser == 'ie' && matrix.os == 'windows-latest'
|
||||||
|
|
||||||
|
- run: echo "::add-path::$env:ChromeWebDriver"
|
||||||
|
if: matrix.browser == 'chrome' && matrix.os == 'windows-latest'
|
||||||
|
|
||||||
|
- run: npm i --no-optional --no-audit selenium-webdriver pixelmatch pngjs
|
||||||
|
- run: node .github/actions-scripts/selenium.js
|
||||||
|
env:
|
||||||
|
BROWSER: ${{ matrix.browser }}
|
||||||
|
PORT: 0
|
||||||
|
NODE_ENV: production # prevent watching
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@master
|
||||||
|
if: failure()
|
||||||
|
with:
|
||||||
|
name: screenshot-${{ matrix.browser }}-${{ matrix.os }}
|
||||||
|
path: screenshot
|
||||||
|
|
||||||
|
puppeteer:
|
||||||
|
runs-on: macos-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
with:
|
||||||
|
fetch-depth: 1
|
||||||
|
- uses: actions/setup-node@v1
|
||||||
|
with:
|
||||||
|
node-version: '12.x'
|
||||||
|
- run: |
|
||||||
|
npm ci
|
||||||
|
npm run build
|
||||||
|
env:
|
||||||
|
CYPRESS_INSTALL_BINARY: 0
|
||||||
|
HUSKY_SKIP_INSTALL: true
|
||||||
|
- run: npm i --no-optional --no-audit puppeteer pixelmatch pngjs
|
||||||
|
- run: node .github/actions-scripts/puppeteer.js
|
||||||
|
env:
|
||||||
|
PORT: 0
|
||||||
|
NODE_ENV: production # prevent watching
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@master
|
||||||
|
if: failure()
|
||||||
|
with:
|
||||||
|
name: screenshot-puppeteer-darwin
|
||||||
|
path: screenshot
|
|
@ -1,6 +1,7 @@
|
||||||
{
|
{
|
||||||
"singleQuote": true,
|
"singleQuote": true,
|
||||||
"trailingComma": "all",
|
"trailingComma": "all",
|
||||||
|
"endOfLine": "lf",
|
||||||
"overrides": [
|
"overrides": [
|
||||||
{
|
{
|
||||||
"files": ["*.svg"],
|
"files": ["*.svg"],
|
||||||
|
|
|
@ -2,8 +2,8 @@
|
||||||
const express = require('express');
|
const express = require('express');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
|
|
||||||
const PORT = 3001;
|
const PORT = process.env.PORT || 3001;
|
||||||
const DIST_DIR = path.join(__dirname, 'public');
|
const DIST_DIR = path.resolve(__dirname, 'public');
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
|
@ -51,7 +51,7 @@ const server = app.listen(PORT, err => {
|
||||||
console.log(err);
|
console.log(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Listening at http://localhost:${PORT} 👂`);
|
console.log(`Listening at http://localhost:${server.address().port} 👂`);
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on('SIGTERM', () => {
|
process.on('SIGTERM', () => {
|
||||||
|
@ -70,3 +70,5 @@ process.on('SIGTERM', () => {
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
module.exports = server;
|
||||||
|
|
Loading…
Reference in a new issue