Compare commits

...

11 commits

Author SHA1 Message Date
Alicia Sykes 24ca677021 Fix timeout parsing, and show timeout on Vercel 2024-03-22 06:40:36 +00:00
Alicia Sykes ed53c29a55 Reduce Vercel function timeout to 10 seconds 2024-03-21 12:10:52 +00:00
Alicia Sykes c4e29fda0f
Merge pull request #118 from Lissy93/FEAT/override-timeout
FEAT: Override timeout
2024-03-20 21:49:51 +00:00
Alicia Sykes ff65696729 Bump to V 1.1.2 2024-03-20 21:43:04 +00:00
Alicia Sykes 2304aaf17c Adds unique keys to itteratbles, remove react warn 2024-03-20 21:20:41 +00:00
Alicia Sykes d43a05a0ed Show timeout seperatley 2024-03-20 21:20:23 +00:00
Alicia Sykes ada1dccc5b Update timeout message for sitemap 2024-03-20 21:19:58 +00:00
Alicia Sykes 7a8e694abc Logging for timeout info, and adds key to map 2024-03-20 20:45:13 +00:00
Alicia Sykes c764bbfcd4 If response is a timeout, set type to timeout 2024-03-20 20:44:37 +00:00
Alicia Sykes 523419df11 Adds timeout functionality 2024-03-20 20:43:51 +00:00
liss-bot 62a213d74d docs: Updates contributors list 2024-03-17 01:58:43 +00:00
16 changed files with 192 additions and 86 deletions

34
.github/README.md vendored
View file

@ -1049,13 +1049,6 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
<sub><b>Peng1can</b></sub>
</a>
</td>
<td align="center">
<a href="https://github.com/alydemah">
<img src="https://avatars.githubusercontent.com/u/652035?u=ac2c04e474da37bfeafcfa25076cc1800997aedb&v=4" width="80;" alt="alydemah"/>
<br />
<sub><b>Aly Mohamed</b></sub>
</a>
</td>
<td align="center">
<a href="https://github.com/bgadrian">
<img src="https://avatars.githubusercontent.com/u/830001?u=69f115baad2fcd8c14eb05bdbf5cd80f4649a95a&v=4" width="80;" alt="bgadrian"/>
@ -1076,15 +1069,15 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
<br />
<sub><b>Eddy Lazzarin</b></sub>
</a>
</td></tr>
<tr>
</td>
<td align="center">
<a href="https://github.com/AnandChowdhary">
<img src="https://avatars.githubusercontent.com/u/2841780?u=747e554b3a7f12eb20b7910e1c87d817844f714f&v=4" width="80;" alt="AnandChowdhary"/>
<br />
<sub><b>Anand Chowdhary</b></sub>
</a>
</td>
</td></tr>
<tr>
<td align="center">
<a href="https://github.com/shrippen">
<img src="https://avatars.githubusercontent.com/u/2873570?v=4" width="80;" alt="shrippen"/>
@ -1119,15 +1112,15 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
<br />
<sub><b>InDieTasten</b></sub>
</a>
</td></tr>
<tr>
</td>
<td align="center">
<a href="https://github.com/bmcgonag">
<img src="https://avatars.githubusercontent.com/u/7346620?u=2a0f9284f3e12ac1cc15288c254d1ec68a5081e8&v=4" width="80;" alt="bmcgonag"/>
<br />
<sub><b>Brian McGonagill</b></sub>
</a>
</td>
</td></tr>
<tr>
<td align="center">
<a href="https://github.com/vlad-timofeev">
<img src="https://avatars.githubusercontent.com/u/11474041?u=eee43705b54d2ec9f51fc4fcce5ad18dd17c87e4&v=4" width="80;" alt="vlad-timofeev"/>
@ -1144,7 +1137,7 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
</td>
<td align="center">
<a href="https://github.com/mryesiller">
<img src="https://avatars.githubusercontent.com/u/24632172?v=4" width="80;" alt="mryesiller"/>
<img src="https://avatars.githubusercontent.com/u/24632172?u=0d20f2d615158f87cd60a3398d3efb026c32f291&v=4" width="80;" alt="mryesiller"/>
<br />
<sub><b>Göksel Yeşiller</b></sub>
</a>
@ -1162,15 +1155,15 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
<br />
<sub><b>Bastii717</b></sub>
</a>
</td></tr>
<tr>
</td>
<td align="center">
<a href="https://github.com/frankdez93">
<img src="https://avatars.githubusercontent.com/u/87549420?v=4" width="80;" alt="frankdez93"/>
<br />
<sub><b>Frankdez93</b></sub>
</a>
</td>
</td></tr>
<tr>
<td align="center">
<a href="https://github.com/ratty222">
<img src="https://avatars.githubusercontent.com/u/92832598?u=137b65530cbd5f5af9c24cde51baa6cc77cc934b&v=4" width="80;" alt="ratty222"/>
@ -1185,6 +1178,13 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
<sub><b>Hernanpopper</b></sub>
</a>
</td>
<td align="center">
<a href="https://github.com/terminaltrove">
<img src="https://avatars.githubusercontent.com/u/121595180?v=4" width="80;" alt="terminaltrove"/>
<br />
<sub><b>Terminal Trove</b></sub>
</a>
</td>
<td align="center">
<a href="https://github.com/NixyJuppie">
<img src="https://avatars.githubusercontent.com/u/138570196?u=b102c222487905728b858704962d32759df29ebe&v=4" width="80;" alt="NixyJuppie"/>

View file

@ -2,13 +2,47 @@ const normalizeUrl = (url) => {
return url.startsWith('http') ? url : `https://${url}`;
};
// If present, set a shorter timeout for API requests
const TIMEOUT = process.env.API_TIMEOUT_LIMIT ? parseInt(process.env.API_TIMEOUT_LIMIT, 10) : 60000;
// If present, set CORS allowed origins for responses
const ALLOWED_ORIGINS = process.env.API_CORS_ORIGIN || '*';
// Set the platform currently being used
let PLATFORM = 'NETLIFY';
if (process.env.PLATFORM) { PLATFORM = process.env.PLATFORM.toUpperCase(); }
else if (process.env.VERCEL) { PLATFORM = 'VERCEL'; }
else if (process.env.WC_SERVER) { PLATFORM = 'NODE'; }
// Define the headers to be returned with each response
const headers = {
'Access-Control-Allow-Origin': process.env.API_CORS_ORIGIN || '*',
'Access-Control-Allow-Origin': ALLOWED_ORIGINS,
'Access-Control-Allow-Credentials': true,
'Content-Type': 'application/json;charset=UTF-8',
};
const timeoutErrorMsg = 'You can re-trigger this request, by clicking "Retry"\n'
+ 'If you\'re running your own instance of Web Check, then you can '
+ 'resolve this issue, by increasing the timeout limit in the '
+ '`API_TIMEOUT_LIMIT` environmental variable to a higher value (in milliseconds), '
+ 'or if you\'re hosting on Vercel increase the maxDuration in vercel.json.\n\n'
+ `The public instance currently has a lower timeout of ${TIMEOUT}ms `
+ 'in order to keep running costs affordable, so that Web Check can '
+ 'remain freely available for everyone.';
// A middleware function used by all API routes on all platforms
const commonMiddleware = (handler) => {
// Create a timeout promise, to throw an error if a request takes too long
const createTimeoutPromise = (timeoutMs) => {
return new Promise((_, reject) => {
setTimeout(() => {
reject(new Error(`Request timed-out after ${timeoutMs} ms`));
}, timeoutMs);
});
};
// Vercel
const vercelHandler = async (request, response) => {
const queryParams = request.query || {};
@ -21,7 +55,12 @@ const commonMiddleware = (handler) => {
const url = normalizeUrl(rawUrl);
try {
const handlerResponse = await handler(url, request);
// Race the handler against the timeout
const handlerResponse = await Promise.race([
handler(url, request),
createTimeoutPromise(TIMEOUT)
]);
if (handlerResponse.body && handlerResponse.statusCode) {
response.status(handlerResponse.statusCode).json(handlerResponse.body);
} else {
@ -30,7 +69,12 @@ const commonMiddleware = (handler) => {
);
}
} catch (error) {
response.status(500).json({ error: error.message });
let errorCode = 500;
if (error.message.includes('timed-out') || response.statusCode === 504) {
errorCode = 408;
error.message = `${error.message}\n\n${timeoutErrorMsg}`;
}
response.status(errorCode).json({ error: error.message });
}
};
@ -51,7 +95,12 @@ const commonMiddleware = (handler) => {
const url = normalizeUrl(rawUrl);
try {
const handlerResponse = await handler(url, event, context);
// Race the handler against the timeout
const handlerResponse = await Promise.race([
handler(url, event, context),
createTimeoutPromise(TIMEOUT)
]);
if (handlerResponse.body && handlerResponse.statusCode) {
callback(null, handlerResponse);
} else {
@ -71,11 +120,7 @@ const commonMiddleware = (handler) => {
};
// The format of the handlers varies between platforms
// E.g. Netlify + AWS expect Lambda functions, but Vercel or Node needs standard handler
const platformEnv = (process.env.PLATFORM || '').toUpperCase(); // Has user set platform manually?
const nativeMode = (['VERCEL', 'NODE'].includes(platformEnv) || process.env.VERCEL || process.env.WC_SERVER);
const nativeMode = (['VERCEL', 'NODE'].includes(PLATFORM));
return nativeMode ? vercelHandler : netlifyHandler;
};

View file

@ -6,15 +6,17 @@ const xml2js = require('xml2js');
const handler = async (url) => {
let sitemapUrl = `${url}/sitemap.xml`;
const hardTimeOut = 5000;
try {
// Try to fetch sitemap directly
let sitemapRes;
try {
sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut });
} catch (error) {
if (error.response && error.response.status === 404) {
// If sitemap not found, try to fetch it from robots.txt
const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: 5000 });
const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: hardTimeOut });
const robotsTxt = robotsRes.data.split('\n');
for (let line of robotsTxt) {
@ -28,7 +30,7 @@ const handler = async (url) => {
return { skipped: 'No sitemap found' };
}
sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut });
} else {
throw error; // If other error, throw it
}
@ -40,7 +42,7 @@ const handler = async (url) => {
return sitemap;
} catch (error) {
if (error.code === 'ECONNABORTED') {
return { error: 'Request timed out after 5000ms' };
return { error: `Request timed-out after ${hardTimeOut}ms` };
} else {
return { error: error.message };
}

View file

@ -1,6 +1,6 @@
{
"name": "web-check",
"version": "1.1.0",
"version": "1.1.2",
"private": false,
"description": "All-in-one OSINT tool for analyzing any website",
"repository": "github:lissy93/web-check",

View file

@ -159,7 +159,7 @@ export const ExpandableRow = (props: RowProps) => {
{ rowList?.map((row: RowProps, index: number) => {
return (
<SubRow key={`${row.lbl}-${index}`}>
<span className="lbl" title={row.title}>{row.lbl}</span>
<span className="lbl" title={row.title?.toString()}>{row.lbl}</span>
<span className="val" title={row.val} onClick={() => copyToClipboard(row.val)}>
{formatValue(row.val)}
</span>
@ -199,7 +199,7 @@ const Row = (props: RowProps) => {
if (children) return <StyledRow key={`${lbl}-${val}`}>{children}</StyledRow>;
return (
<StyledRow key={`${lbl}-${val}`}>
{ lbl && <span className="lbl" title={title}>{lbl}</span> }
{ lbl && <span className="lbl" title={title?.toString()}>{lbl}</span> }
<span className="val" title={val} onClick={() => copyToClipboard(val)}>
{formatValue(val)}
</span>

View file

@ -6,11 +6,13 @@ const BlockListsCard = (props: {data: any, title: string, actionButtons: any }):
const blockLists = props.data.blocklists;
return (
<Card heading={props.title} actionButtons={props.actionButtons}>
{ blockLists.map((blocklist: any) => (
{ blockLists.map((blocklist: any, blockIndex: number) => (
<Row
title={blocklist.serverIp}
lbl={blocklist.server}
val={blocklist.isBlocked ? '❌ Blocked' : '✅ Not Blocked'} />
val={blocklist.isBlocked ? '❌ Blocked' : '✅ Not Blocked'}
key={`blocklist-${blockIndex}-${blocklist.serverIp}`}
/>
))}
</Card>
);

View file

@ -18,12 +18,12 @@ const DnsServerCard = (props: {data: any, title: string, actionButtons: any }):
return (
<Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
{dnsSecurity.dns.map((dns: any, index: number) => {
return (<>
return (<div key={`dns-${index}`}>
{ dnsSecurity.dns.length > 1 && <Heading as="h4" size="small" color={colors.primary}>DNS Server #{index+1}</Heading> }
<Row lbl="IP Address" val={dns.address} key={`ip-${index}`} />
{ dns.hostname && <Row lbl="Hostname" val={dns.hostname} key={`host-${index}`} /> }
<Row lbl="DoH Support" val={dns.dohDirectSupports ? '✅ Yes*' : '❌ No*'} key={`doh-${index}`} />
</>);
</div>);
})}
{dnsSecurity.dns.length > 0 && (<small>
* DoH Support is determined by the DNS server's response to a DoH query.

View file

@ -13,7 +13,7 @@ span.val {
const ServerStatusCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => {
const serverStatus = props.data;
return (
<Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
<Card heading={props.title.toString()} actionButtons={props.actionButtons} styles={cardStyles}>
<Row lbl="" val="">
<span className="lbl">Is Up?</span>
{ serverStatus.isUp ? <span className="val up"> Online</span> : <span className="val down"> Offline</span>}

View file

@ -52,7 +52,7 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
<Card heading={props.title} actionButtons={props.actionButtons}>
{ cipherSuites.length && cipherSuites.map((cipherSuite: any, index: number) => {
return (
<ExpandableRow lbl={cipherSuite.title} val="" rowList={cipherSuite.fields} />
<ExpandableRow key={`tls-${index}`} lbl={cipherSuite.title} val="" rowList={cipherSuite.fields} />
);
})}
{ !cipherSuites.length && (

View file

@ -59,8 +59,15 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
const scanId = props.data?.id;
return (
<Card heading={props.title} actionButtons={props.actionButtons}>
{clientSupport.map((support: any) => {
return (<ExpandableRow lbl={support.title} val={support.value || '?'} rowList={support.fields} />)
{clientSupport.map((support: any, index: number) => {
return (
<ExpandableRow
key={`tls-client-${index}`}
lbl={support.title}
val={support.value || '?'}
rowList={support.fields}
/>
)
})}
{ !clientSupport.length && (
<div>

View file

@ -99,7 +99,13 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
<Card heading={props.title} actionButtons={props.actionButtons}>
{ tlsResults.length > 0 && tlsResults.map((row: any, index: number) => {
return (
<Row lbl={row.lbl} val={row.val} plaintext={row.plaintext} listResults={row.list} />
<Row
lbl={row.lbl}
val={row.val}
plaintext={row.plaintext}
listResults={row.list}
key={`tls-issues-${index}`}
/>
);
})}
<Expandable>

View file

@ -7,6 +7,7 @@ import colors from 'styles/colors';
interface Props {
children: ReactNode;
title?: string;
key?: string;
}
interface State {

View file

@ -224,6 +224,52 @@ const jobNames = [
'carbon',
] as const;
interface JobListItemProps {
job: LoadingJob;
showJobDocs: (name: string) => void;
showErrorModal: (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => void;
barColors: Record<LoadingState, [string, string]>;
}
const getStatusEmoji = (state: LoadingState): string => {
switch (state) {
case 'success':
return '✅';
case 'loading':
return '🔄';
case 'error':
return '❌';
case 'timed-out':
return '⏸️';
case 'skipped':
return '⏭️';
default:
return '❓';
}
};
const JobListItem: React.FC<JobListItemProps> = ({ job, showJobDocs, showErrorModal, barColors }) => {
const { name, state, timeTaken, retry, error } = job;
const actionButton = retry && state !== 'success' && state !== 'loading' ?
<FailedJobActionButton onClick={retry}> Retry</FailedJobActionButton> : null;
const showModalButton = error && ['error', 'timed-out', 'skipped'].includes(state) &&
<FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error, state === 'skipped')}>
{state === 'timed-out' ? '■ Show Timeout Reason' : '■ Show Error'}
</FailedJobActionButton>;
return (
<li key={name}>
<b onClick={() => showJobDocs(name)}>{getStatusEmoji(state)} {name}</b>
<span style={{color: barColors[state][0]}}> ({state})</span>.
<i>{timeTaken && state !== 'loading' ? ` Took ${timeTaken} ms` : ''}</i>
{actionButton}
{showModalButton}
</li>
);
};
export const initialJobs = jobNames.map((job: string) => {
return {
name: job,
@ -239,9 +285,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec
const stateCount: Record<LoadingState, number> = {
'success': 0,
'loading': 0,
'skipped': 0,
'error': 0,
'timed-out': 0,
'error': 0,
'skipped': 0,
};
// Count the number of each state
@ -253,9 +299,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec
const statePercentage: Record<LoadingState, number> = {
'success': (stateCount['success'] / totalJobs) * 100,
'loading': (stateCount['loading'] / totalJobs) * 100,
'skipped': (stateCount['skipped'] / totalJobs) * 100,
'error': (stateCount['error'] / totalJobs) * 100,
'timed-out': (stateCount['timed-out'] / totalJobs) * 100,
'error': (stateCount['error'] / totalJobs) * 100,
'skipped': (stateCount['skipped'] / totalJobs) * 100,
};
return statePercentage;
@ -353,26 +399,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
const barColors: Record<LoadingState | string, [string, string]> = {
'success': isDone ? makeBarColor(colors.primary) : makeBarColor(colors.success),
'loading': makeBarColor(colors.info),
'skipped': makeBarColor(colors.warning),
'error': makeBarColor(colors.danger),
'timed-out': makeBarColor(colors.neutral),
};
const getStatusEmoji = (state: LoadingState): string => {
switch (state) {
case 'success':
return '✅';
case 'loading':
return '🔄';
case 'skipped':
return '⏭️';
case 'error':
return '❌';
case 'timed-out':
return '⏸️';
default:
return '❓';
}
'timed-out': makeBarColor(colors.warning),
'skipped': makeBarColor(colors.neutral),
};
const showErrorModal = (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => {
@ -416,20 +445,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
<Details>
<summary>Show Details</summary>
<ul>
{
loadStatus.map(({ name, state, timeTaken, retry, error }: LoadingJob) => {
return (
<li key={name}>
<b onClick={() => props.showJobDocs(name)}>{getStatusEmoji(state)} {name}</b>
<span style={{color : barColors[state][0]}}> ({state})</span>.
<i>{(timeTaken && state !== 'loading') ? ` Took ${timeTaken} ms` : '' }</i>
{ (retry && state !== 'success' && state !== 'loading') && <FailedJobActionButton onClick={retry}> Retry</FailedJobActionButton> }
{ (error && state === 'error') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error)}> Show Error</FailedJobActionButton> }
{ (error && state === 'skipped') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error, true)}> Show Reason</FailedJobActionButton> }
</li>
);
})
}
{loadStatus.map((job: LoadingJob) => (
<JobListItem key={job.name} job={job} showJobDocs={props.showJobDocs} showErrorModal={showErrorModal} barColors={barColors} />
))}
</ul>
{ loadStatus.filter((val: LoadingJob) => val.state === 'error').length > 0 &&
<p className="error">

View file

@ -35,16 +35,21 @@ const useMotherOfAllHooks = <ResultType = any>(params: UseIpAddressProps<ResultT
const [result, setResult] = useState<ResultType>();
// Fire off the HTTP fetch request, then set results and update loading / error state
const doTheFetch = () => {
return fetchRequest()
.then((res: any) => {
if (!res) { // No response :(
updateLoadingJobs(jobId, 'error', res.error || 'No response', reset);
updateLoadingJobs(jobId, 'error', 'No response', reset);
} else if (res.error) { // Response returned an error message
updateLoadingJobs(jobId, 'error', res.error, reset);
if (res.error.includes("timed-out")) { // Specific handling for timeout errors
updateLoadingJobs(jobId, 'timed-out', res.error, reset);
} else {
updateLoadingJobs(jobId, 'error', res.error, reset);
}
} else if (res.skipped) { // Response returned a skipped message
updateLoadingJobs(jobId, 'skipped', res.skipped, reset);
} else { // Yay, everything went to plan :)
} else { // Yay, everything went to plan :)
setResult(res);
updateLoadingJobs(jobId, 'success', '', undefined, res);
}

View file

@ -208,12 +208,24 @@ const Results = (): JSX.Element => {
console.log(
`%cFetch Error - ${job}%c\n\n${timeString}%c The ${job} job failed `
+`after ${timeTaken}ms, with the following error:%c\n${error}`,
`background: ${colors.danger}; padding: 4px 8px; font-size: 16px;`,
`background: ${colors.danger}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`,
`font-weight: bold; color: ${colors.danger};`,
`color: ${colors.danger};`,
`color: ${colors.warning};`,
);
}
if (newState === 'timed-out') {
console.log(
`%cFetch Timeout - ${job}%c\n\n${timeString}%c The ${job} job timed out `
+`after ${timeTaken}ms, with the following error:%c\n${error}`,
`background: ${colors.info}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`,
`font-weight: bold; color: ${colors.info};`,
`color: ${colors.info};`,
`color: ${colors.warning};`,
);
}
return newJobs;
});
});
@ -225,8 +237,9 @@ const Results = (): JSX.Element => {
.then(data => resolve(data))
.catch(error => resolve(
{ error: `Failed to get a valid response 😢\n`
+ `This is likely due the target not exposing the required data, `
+ `or limitations in how Netlify executes lambda functions, such as the 10-sec timeout.\n\n`
+ 'This is likely due the target not exposing the required data, '
+ 'or limitations in imposed by the infrastructure this instance '
+ 'of Web Check is running on.\n\n'
+ `Error info:\n${error}`}
));
});
@ -910,7 +923,7 @@ const Results = (): JSX.Element => {
&& title.toLowerCase().includes(searchTerm.toLowerCase())
&& (result && !result.error);
return show ? (
<ErrorBoundary title={title}>
<ErrorBoundary title={title} key={`eb-${index}`}>
<Component
key={`${title}-${index}`}
data={{...result}}

7
vercel.json Normal file
View file

@ -0,0 +1,7 @@
{
"functions": {
"api/*.js": {
"maxDuration": 10
}
}
}