mirror of
https://github.com/Choices-js/Choices.git
synced 2024-05-02 22:03:11 +02:00
[TEST] Automatic screenshots comparing and error checking in IE11, Firefox and Chrome (#715)
* taking screenshots * let's try * save artifacts * better exceptiosn * try to install geskodriver * fix edge name * add check for safari * check os * try this * again * fix syntax * try this * try firefox on windows * and again * handle error * and again * try older macos * add firefox screenshot * switch back env * switch back env * add IE screenshot * try sudo for safari * try not install nuget * try more * more * try this * install firefox * add chrome * increase threshold * add firefox-darwin * push * fixing * let's go * increase threeshold * again * try safari tp * tap cask versions * fix conditions * try like this * last run * increase threshold * reenable macos firefox * last try for safari * rename screenshots to snapshots * check console * console workarounds * fix safari misspleings * logging is not supported by everyone * maximize and set rect * errors only for Chrome * remove safari for now * try to decrease threshold * cleanup * increase threeshold * sleep more and increase threeshold * add pupeeter * handle errors * build * add prettier end of line * add gitattributes * add png to binary * more attributest * limit run * run on chages to snapshots * hey! * make artifacts named as snapshots * just for fun: we don't need express here * update pupeeter snapshot * no audit * don't wait for quit? * try more IE capabilities * add wait timeout * use server.js
This commit is contained in:
parent
5afe8b5a7f
commit
dddba5b35b
64
.gitattributes
vendored
Normal file
64
.gitattributes
vendored
Normal file
|
@ -0,0 +1,64 @@
|
|||
## GITATTRIBUTES FOR WEB PROJECTS
|
||||
#
|
||||
# These settings are for any web project.
|
||||
#
|
||||
# Details per file setting:
|
||||
# text These files should be normalized (i.e. convert CRLF to LF).
|
||||
# binary These files are binary and should be left untouched.
|
||||
#
|
||||
# Note that binary is a macro for -text -diff.
|
||||
######################################################################
|
||||
|
||||
# Auto detect
|
||||
## Handle line endings automatically for files detected as
|
||||
## text and leave all files detected as binary untouched.
|
||||
## This will handle all files NOT defined below.
|
||||
* text eol=lf
|
||||
|
||||
# Source code
|
||||
*.css text eol=lf
|
||||
*.html text diff=html eol=lf
|
||||
*.js text eol=lf
|
||||
*.json text eol=lf
|
||||
*.scss text diff=css eol=lf
|
||||
*.ts text eol=lf
|
||||
|
||||
# Documentation
|
||||
*.md text eol=lf
|
||||
*.txt text eol=lf
|
||||
AUTHORS text eol=lf
|
||||
CHANGELOG text eol=lf
|
||||
CHANGES text eol=lf
|
||||
CONTRIBUTING text eol=lf
|
||||
COPYING text eol=lf
|
||||
copyright text eol=lf
|
||||
*COPYRIGHT* text eol=lf
|
||||
INSTALL text eol=lf
|
||||
license text eol=lf
|
||||
LICENSE text eol=lf
|
||||
NEWS text eol=lf
|
||||
readme text eol=lf
|
||||
*README* text eol=lf
|
||||
TODO text eol=lf
|
||||
|
||||
# Linters
|
||||
.eslintrc text eol=lf
|
||||
.stylelintrc text eol=lf
|
||||
|
||||
# Configs
|
||||
.babelrc text eol=lf
|
||||
.browserslistrc text eol=lf
|
||||
.editorconfig text eol=lf
|
||||
.env text eol=lf
|
||||
.gitattributes text eol=lf
|
||||
.gitconfig text eol=lf
|
||||
package-lock.json text -diff eol=lf
|
||||
*.npmignore text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
browserslist text eol=lf
|
||||
|
||||
# Graphics
|
||||
# SVG treated as an asset (binary) by default.
|
||||
*.svg text eol=lf
|
||||
*.png binary
|
BIN
.github/actions-scripts/__snapshots__/chrome-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/chrome-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 25 KiB |
BIN
.github/actions-scripts/__snapshots__/firefox-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/firefox-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 48 KiB |
BIN
.github/actions-scripts/__snapshots__/firefox-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/firefox-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 108 KiB |
BIN
.github/actions-scripts/__snapshots__/ie-win32.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/ie-win32.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 22 KiB |
BIN
.github/actions-scripts/__snapshots__/puppeteer-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/puppeteer-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 294 KiB |
BIN
.github/actions-scripts/__snapshots__/safari-darwin.png
vendored
Normal file
BIN
.github/actions-scripts/__snapshots__/safari-darwin.png
vendored
Normal file
Binary file not shown.
After Width: | Height: | Size: 300 KiB |
88
.github/actions-scripts/puppeteer.js
vendored
Normal file
88
.github/actions-scripts/puppeteer.js
vendored
Normal file
|
@ -0,0 +1,88 @@
|
|||
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
|
||||
const path = require('path');
|
||||
const { once } = require('events');
|
||||
|
||||
const puppeteer = require('puppeteer');
|
||||
const pixelmatch = require('pixelmatch');
|
||||
const { PNG } = require('pngjs');
|
||||
|
||||
const server = require('../../server');
|
||||
|
||||
async function test() {
|
||||
const browser = await puppeteer.launch();
|
||||
const page = await browser.newPage();
|
||||
let error;
|
||||
let pixelDifference;
|
||||
|
||||
if (!server.listening) await once(server, 'listening');
|
||||
|
||||
try {
|
||||
page.on('console', msg => {
|
||||
if (msg.type() === 'error') throw new Error(msg.text());
|
||||
});
|
||||
page.on('pageerror', err => {
|
||||
throw err;
|
||||
});
|
||||
|
||||
await page.goto(`http://127.0.0.1:${server.address().port}`, {
|
||||
waitUntil: 'networkidle2',
|
||||
});
|
||||
await page.setViewport({ width: 640, height: 1000 });
|
||||
await page.click('label[for="choices-single-custom-templates"]');
|
||||
await page.keyboard.press('ArrowDown');
|
||||
await page.keyboard.press('ArrowDown');
|
||||
|
||||
const snapshotName = `puppeteer-${process.platform}.png`;
|
||||
const artifactsPath = 'screenshot';
|
||||
mkdirSync(artifactsPath, { recursive: true });
|
||||
const imageBuffer = await page.screenshot({
|
||||
path: path.join(artifactsPath, snapshotName),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
// compare with snapshot
|
||||
const screenshot = PNG.sync.read(imageBuffer);
|
||||
const snapshot = PNG.sync.read(
|
||||
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
|
||||
);
|
||||
const { width, height } = screenshot;
|
||||
const diff = new PNG({ width, height });
|
||||
pixelDifference = pixelmatch(
|
||||
screenshot.data,
|
||||
snapshot.data,
|
||||
diff.data,
|
||||
width,
|
||||
height,
|
||||
{
|
||||
threshold: 0.6,
|
||||
},
|
||||
);
|
||||
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
error = err;
|
||||
} finally {
|
||||
await Promise.all([
|
||||
browser.close(),
|
||||
new Promise(resolve => server.close(resolve)),
|
||||
]);
|
||||
}
|
||||
|
||||
if (pixelDifference > 200) {
|
||||
console.error(
|
||||
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (error) process.exit(1);
|
||||
}
|
||||
|
||||
process.on('unhandledRejection', err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
process.once('uncaughtException', err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
setImmediate(test);
|
155
.github/actions-scripts/selenium.js
vendored
Normal file
155
.github/actions-scripts/selenium.js
vendored
Normal file
|
@ -0,0 +1,155 @@
|
|||
const path = require('path');
|
||||
const { readFileSync, writeFileSync, mkdirSync } = require('fs');
|
||||
const { once } = require('events');
|
||||
|
||||
const pixelmatch = require('pixelmatch');
|
||||
const { PNG } = require('pngjs');
|
||||
const {
|
||||
Builder,
|
||||
By,
|
||||
Key,
|
||||
until,
|
||||
Capabilities,
|
||||
logging,
|
||||
} = require('selenium-webdriver');
|
||||
|
||||
const server = require('../../server');
|
||||
|
||||
async function test() {
|
||||
let pixelDifference;
|
||||
let error;
|
||||
|
||||
let capabilities;
|
||||
switch (process.env.BROWSER) {
|
||||
case 'ie':
|
||||
capabilities = Capabilities.ie();
|
||||
capabilities.set('ignoreProtectedModeSettings', true);
|
||||
capabilities.set('ignoreZoomSetting', true);
|
||||
capabilities.set('ie.options', {
|
||||
enableFullPageScreenshot: true,
|
||||
ensureCleanSession: true,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'edge':
|
||||
capabilities = Capabilities.edge();
|
||||
break;
|
||||
|
||||
case 'safari':
|
||||
capabilities = Capabilities.safari();
|
||||
capabilities.set('safari.options', { technologyPreview: false });
|
||||
break;
|
||||
|
||||
case 'firefox': {
|
||||
capabilities = Capabilities.firefox().setLoggingPrefs({ browser: 'ALL' });
|
||||
break;
|
||||
}
|
||||
case 'chrome': {
|
||||
capabilities = Capabilities.chrome().setLoggingPrefs({ browser: 'ALL' });
|
||||
capabilities.set('chromeOptions', {
|
||||
args: ['--headless', '--no-sandbox', '--disable-gpu'],
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let driver = await new Builder().withCapabilities(capabilities).build();
|
||||
|
||||
if (!server.listening) await once(server, 'listening');
|
||||
|
||||
try {
|
||||
await driver.get(`http://127.0.0.1:${server.address().port}`);
|
||||
|
||||
// wait for last choice to init
|
||||
await driver.wait(
|
||||
until.elementLocated(By.css('#reset-multiple ~ .choices__list')),
|
||||
10000,
|
||||
'waiting for all Choices instances to init',
|
||||
);
|
||||
|
||||
// Resize window
|
||||
await driver
|
||||
.manage()
|
||||
.window()
|
||||
.maximize();
|
||||
await driver
|
||||
.manage()
|
||||
.window()
|
||||
// magic numbers here to make sure all demo page are fit inside
|
||||
.setRect({ x: 0, y: 0, width: 630, height: 4000 });
|
||||
|
||||
// and click on press space on it, so it should open choices
|
||||
await driver
|
||||
.findElement(By.css('#reset-multiple ~ .choices__list button'))
|
||||
.sendKeys(Key.SPACE);
|
||||
await driver.sleep(1000);
|
||||
|
||||
// take screenshot
|
||||
const image = await driver.takeScreenshot();
|
||||
const imageBuffer = Buffer.from(image, 'base64');
|
||||
|
||||
const snapshotName = `${process.env.BROWSER}-${process.platform}.png`;
|
||||
const artifactsPath = 'screenshot';
|
||||
mkdirSync(artifactsPath, { recursive: true });
|
||||
|
||||
writeFileSync(path.join(artifactsPath, snapshotName), imageBuffer);
|
||||
|
||||
// compare with snapshot
|
||||
const screenshot = PNG.sync.read(imageBuffer);
|
||||
const snapshot = PNG.sync.read(
|
||||
readFileSync(path.resolve(__dirname, `./__snapshots__/${snapshotName}`)),
|
||||
);
|
||||
const { width, height } = screenshot;
|
||||
const diff = new PNG({ width, height });
|
||||
pixelDifference = pixelmatch(
|
||||
screenshot.data,
|
||||
snapshot.data,
|
||||
diff.data,
|
||||
width,
|
||||
height,
|
||||
{
|
||||
threshold: 1,
|
||||
},
|
||||
);
|
||||
writeFileSync(path.join(artifactsPath, 'diff.png'), PNG.sync.write(diff));
|
||||
|
||||
// getting console logs
|
||||
// ensure no errors in console (only supported in Chrome currently)
|
||||
if (process.env.BROWSER === 'chrome') {
|
||||
const entries = await driver
|
||||
.manage()
|
||||
.logs()
|
||||
.get(logging.Type.BROWSER);
|
||||
if (
|
||||
Array.isArray(entries) &&
|
||||
entries.some(entry => entry.level.name_ === 'SEVERE')
|
||||
)
|
||||
throw new Error(JSON.stringify(entries));
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
error = err;
|
||||
} finally {
|
||||
await Promise.all([
|
||||
driver.quit(),
|
||||
new Promise(resolve => server.close(resolve)),
|
||||
]);
|
||||
}
|
||||
if (pixelDifference > 200) {
|
||||
console.error(
|
||||
`Snapshot is different from screenshot by ${pixelDifference} pixels`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (error) process.exit(1);
|
||||
}
|
||||
|
||||
process.on('unhandledRejection', err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
process.once('uncaughtException', err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
setImmediate(test);
|
116
.github/workflows/browsers.yml
vendored
Normal file
116
.github/workflows/browsers.yml
vendored
Normal file
|
@ -0,0 +1,116 @@
|
|||
name: Browsers
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/**'
|
||||
- 'package-lock.json'
|
||||
- '.browserslistrc'
|
||||
- '.babelrc'
|
||||
- 'webpack.config.*'
|
||||
- 'public/index.html'
|
||||
- '.github/actions-scripts/__snapshots__/**'
|
||||
|
||||
jobs:
|
||||
selenium:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macOS-10.14]
|
||||
browser: [ie, firefox]
|
||||
exclude:
|
||||
# On Windows, run tests with only IE and Edge
|
||||
- os: windows-latest
|
||||
browser: safari
|
||||
# On macOS, run tests with only on safari
|
||||
- os: macOS-10.14
|
||||
browser: ie
|
||||
- os: macOS-10.14
|
||||
browser: chrome
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Dump GitHub context
|
||||
env:
|
||||
GITHUB_CONTEXT: ${{ toJson(github) }}
|
||||
run: echo "$GITHUB_CONTEXT"
|
||||
|
||||
- uses: actions/checkout@v1
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: '12.x'
|
||||
- run: |
|
||||
npm ci
|
||||
npm run build
|
||||
env:
|
||||
CYPRESS_INSTALL_BINARY: 0
|
||||
HUSKY_SKIP_INSTALL: true
|
||||
|
||||
# install drivers
|
||||
- name: Enable Safari Driver
|
||||
run: |
|
||||
# brew tap homebrew/cask-versions
|
||||
# brew cask install safari-technology-preview
|
||||
sudo safaridriver --enable
|
||||
defaults write -app Safari IncludeDevelopMenu 1
|
||||
defaults write -app Safari AllowJavaScriptFromAppleEvents 1
|
||||
defaults write -app Safari IncludeInternalDebugMenu 1
|
||||
defaults write -app Safari AllowRemoteAutomation 1
|
||||
safaridriver -p 0 &
|
||||
if: matrix.browser == 'safari'
|
||||
|
||||
- run: |
|
||||
brew cask install firefox
|
||||
brew install geckodriver
|
||||
if: matrix.browser == 'firefox' && matrix.os == 'macOS-10.14'
|
||||
|
||||
- run: echo "::add-path::$env:GeckoWebDriver"
|
||||
if: matrix.browser == 'firefox' && matrix.os == 'windows-latest'
|
||||
|
||||
- run: echo "::add-path::$env:IEWebDriver"
|
||||
if: matrix.browser == 'ie' && matrix.os == 'windows-latest'
|
||||
|
||||
- run: echo "::add-path::$env:ChromeWebDriver"
|
||||
if: matrix.browser == 'chrome' && matrix.os == 'windows-latest'
|
||||
|
||||
- run: npm i --no-optional --no-audit selenium-webdriver pixelmatch pngjs
|
||||
- run: node .github/actions-scripts/selenium.js
|
||||
env:
|
||||
BROWSER: ${{ matrix.browser }}
|
||||
PORT: 0
|
||||
NODE_ENV: production # prevent watching
|
||||
|
||||
- uses: actions/upload-artifact@master
|
||||
if: failure()
|
||||
with:
|
||||
name: screenshot-${{ matrix.browser }}-${{ matrix.os }}
|
||||
path: screenshot
|
||||
|
||||
puppeteer:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: '12.x'
|
||||
- run: |
|
||||
npm ci
|
||||
npm run build
|
||||
env:
|
||||
CYPRESS_INSTALL_BINARY: 0
|
||||
HUSKY_SKIP_INSTALL: true
|
||||
- run: npm i --no-optional --no-audit puppeteer pixelmatch pngjs
|
||||
- run: node .github/actions-scripts/puppeteer.js
|
||||
env:
|
||||
PORT: 0
|
||||
NODE_ENV: production # prevent watching
|
||||
|
||||
- uses: actions/upload-artifact@master
|
||||
if: failure()
|
||||
with:
|
||||
name: screenshot-puppeteer-darwin
|
||||
path: screenshot
|
|
@ -1,6 +1,7 @@
|
|||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all",
|
||||
"endOfLine": "lf",
|
||||
"overrides": [
|
||||
{
|
||||
"files": ["*.svg"],
|
||||
|
|
|
@ -2,8 +2,8 @@
|
|||
const express = require('express');
|
||||
const path = require('path');
|
||||
|
||||
const PORT = 3001;
|
||||
const DIST_DIR = path.join(__dirname, 'public');
|
||||
const PORT = process.env.PORT || 3001;
|
||||
const DIST_DIR = path.resolve(__dirname, 'public');
|
||||
|
||||
const app = express();
|
||||
|
||||
|
@ -51,7 +51,7 @@ const server = app.listen(PORT, err => {
|
|||
console.log(err);
|
||||
}
|
||||
|
||||
console.log(`Listening at http://localhost:${PORT} 👂`);
|
||||
console.log(`Listening at http://localhost:${server.address().port} 👂`);
|
||||
});
|
||||
|
||||
process.on('SIGTERM', () => {
|
||||
|
@ -70,3 +70,5 @@ process.on('SIGTERM', () => {
|
|||
process.exit(0);
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = server;
|
||||
|
|
Loading…
Reference in a new issue