Merge pull request #111 from Lissy93/FEAT/optional-rate-limiting

Feat/optional rate limiting

.env

@@ -18,8 +18,9 @@ REACT_APP_WHO_API_KEY=''
 
 # Configuration settings
 # CHROME_PATH='/usr/bin/chromium' # The path the the Chromium executable
-# PORT='3000' # Port to serve the API, when running server.js
-# DISABLE_GUI='false' # Disable the GUI, and only serve the API
-# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds
-# API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here
-# REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote)
+# PORT='3000'                     # Port to serve the API, when running server.js
+# DISABLE_GUI='false'             # Disable the GUI, and only serve the API
+# API_TIMEOUT_LIMIT='10000'       # The timeout limit for API requests, in milliseconds
+# API_CORS_ORIGIN='*'             # Enable CORS, by setting your allowed hostname(s) here
+# API_ENABLE_RATE_LIMIT='true'    # Enable rate limiting for the API
+# REACT_APP_API_ENDPOINT='/api'   # The endpoint for the API (can be local or remote)

.github/README.md

@@ -9,6 +9,22 @@
 
 </p>
 
+---
+<p align="center">
+  <sup>Kindly supported by:</sup><br>
+<a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+  <img src="https://i.ibb.co/8jrrcZ0/IMG-7210.jpg" width="300" alt="Terminal Trove">
+  <br>
+  <strong>The $HOME of all things in the terminal.</strong>
+</a>
+<br>
+<a href="https://terminaltrove.com/newsletter?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+  <sub>Find your next CLI / TUI tool and more at Terminal Trove,</sub>
+  <br>
+  <sup>Get updates on new tools on our newsletter.</sup>
+</a>
+</p>
+
 ---
 
 #### Contents
@@ -823,10 +839,12 @@ Key | Value
 
 Key | Value
 ---|---
-`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`)
 `PORT` | Port to serve the API, when running server.js (e.g. `3000`)
-`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`)
+`API_ENABLE_RATE_LIMIT` | Enable rate-limiting for the /api endpoints (e.g. `true`)
 `API_TIMEOUT_LIMIT` | The timeout limit for API requests, in milliseconds (e.g. `10000`)
+`API_CORS_ORIGIN` | Enable CORS, by setting your allowed hostname(s) here (e.g. `example.com`)
+`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`)
+`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`)
 `REACT_APP_API_ENDPOINT` | The endpoint for the API, either local or remote (e.g. `/api`)
 
 All values are optional.
@@ -907,6 +925,13 @@ Credit to the following users for contributing to Web-Check
             <sub><b>Mounir Samite</b></sub>
         </a>
     </td>
+    <td align="center">
+        <a href="https://github.com/ChrisCarini">
+            <img src="https://avatars.githubusercontent.com/u/6374067?v=4" width="80;" alt="ChrisCarini"/>
+            <br />
+            <sub><b>Chris Carini</b></sub>
+        </a>
+    </td>
     <td align="center">
         <a href="https://github.com/brianteeman">
             <img src="https://avatars.githubusercontent.com/u/1296369?v=4" width="80;" alt="brianteeman"/>
@@ -920,41 +945,27 @@ Credit to the following users for contributing to Web-Check
             <br />
             <sub><b>Jinna Baalu</b></sub>
         </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/0xflotus">
-            <img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="80;" alt="0xflotus"/>
-            <br />
-            <sub><b>0xflotus</b></sub>
-        </a>
     </td></tr>
 <tr>
     <td align="center">
-        <a href="https://github.com/eltociear">
-            <img src="https://avatars.githubusercontent.com/u/22633385?v=4" width="80;" alt="eltociear"/>
+        <a href="https://github.com/robinson">
+            <img src="https://avatars.githubusercontent.com/u/237874?v=4" width="80;" alt="robinson"/>
             <br />
-            <sub><b>Ikko Eltociear Ashimine</b></sub>
+            <sub><b>Lth</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/treatmesubj">
-            <img src="https://avatars.githubusercontent.com/u/39680353?v=4" width="80;" alt="treatmesubj"/>
+        <a href="https://github.com/abhishekMuge">
+            <img src="https://avatars.githubusercontent.com/u/49590582?v=4" width="80;" alt="abhishekMuge"/>
             <br />
-            <sub><b>John Hupperts</b></sub>
+            <sub><b>Abhishek Muge</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/t3chn0m4g3">
-            <img src="https://avatars.githubusercontent.com/u/4318452?v=4" width="80;" alt="t3chn0m4g3"/>
+        <a href="https://github.com/UlisesGascon">
+            <img src="https://avatars.githubusercontent.com/u/5110813?v=4" width="80;" alt="UlisesGascon"/>
             <br />
-            <sub><b>Marco Ochse</b></sub>
-        </a>
-    </td>
-    <td align="center">
-        <a href="https://github.com/murrple-1">
-            <img src="https://avatars.githubusercontent.com/u/5559656?v=4" width="80;" alt="murrple-1"/>
-            <br />
-            <sub><b>Murray Christopherson</b></sub>
+            <sub><b>Ulises Gascón</b></sub>
         </a>
     </td>
     <td align="center">
@@ -965,25 +976,53 @@ Credit to the following users for contributing to Web-Check
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/UlisesGascon">
-            <img src="https://avatars.githubusercontent.com/u/5110813?v=4" width="80;" alt="UlisesGascon"/>
+        <a href="https://github.com/murrple-1">
+            <img src="https://avatars.githubusercontent.com/u/5559656?v=4" width="80;" alt="murrple-1"/>
             <br />
-            <sub><b>Ulises Gascón</b></sub>
+            <sub><b>Murray Christopherson</b></sub>
+        </a>
+    </td>
+    <td align="center">
+        <a href="https://github.com/t3chn0m4g3">
+            <img src="https://avatars.githubusercontent.com/u/4318452?v=4" width="80;" alt="t3chn0m4g3"/>
+            <br />
+            <sub><b>Marco Ochse</b></sub>
         </a>
     </td></tr>
 <tr>
     <td align="center">
-        <a href="https://github.com/abhishekMuge">
-            <img src="https://avatars.githubusercontent.com/u/49590582?v=4" width="80;" alt="abhishekMuge"/>
+        <a href="https://github.com/treatmesubj">
+            <img src="https://avatars.githubusercontent.com/u/39680353?v=4" width="80;" alt="treatmesubj"/>
             <br />
-            <sub><b>Abhishek Muge</b></sub>
+            <sub><b>John Hupperts</b></sub>
         </a>
     </td>
     <td align="center">
-        <a href="https://github.com/robinson">
-            <img src="https://avatars.githubusercontent.com/u/237874?v=4" width="80;" alt="robinson"/>
+        <a href="https://github.com/eltociear">
+            <img src="https://avatars.githubusercontent.com/u/22633385?v=4" width="80;" alt="eltociear"/>
             <br />
-            <sub><b>Lth</b></sub>
+            <sub><b>Ikko Eltociear Ashimine</b></sub>
+        </a>
+    </td>
+    <td align="center">
+        <a href="https://github.com/Gertje823">
+            <img src="https://avatars.githubusercontent.com/u/36937387?v=4" width="80;" alt="Gertje823"/>
+            <br />
+            <sub><b>Gertje823</b></sub>
+        </a>
+    </td>
+    <td align="center">
+        <a href="https://github.com/epreston">
+            <img src="https://avatars.githubusercontent.com/u/347224?v=4" width="80;" alt="epreston"/>
+            <br />
+            <sub><b>Ed Preston</b></sub>
+        </a>
+    </td>
+    <td align="center">
+        <a href="https://github.com/0xflotus">
+            <img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="80;" alt="0xflotus"/>
+            <br />
+            <sub><b>0xflotus</b></sub>
         </a>
     </td></tr>
 </table>
@@ -1019,7 +1058,7 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
     </td>
     <td align="center">
         <a href="https://github.com/bgadrian">
-            <img src="https://avatars.githubusercontent.com/u/830001?v=4" width="80;" alt="bgadrian"/>
+            <img src="https://avatars.githubusercontent.com/u/830001?u=69f115baad2fcd8c14eb05bdbf5cd80f4649a95a&v=4" width="80;" alt="bgadrian"/>
             <br />
             <sub><b>B.G.Adrian</b></sub>
         </a>
@@ -1076,7 +1115,7 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
     </td>
     <td align="center">
         <a href="https://github.com/InDieTasten">
-            <img src="https://avatars.githubusercontent.com/u/7047377?v=4" width="80;" alt="InDieTasten"/>
+            <img src="https://avatars.githubusercontent.com/u/7047377?u=8d8f8017628b38bc46dcbf3620e194b01d3fb2d1&v=4" width="80;" alt="InDieTasten"/>
             <br />
             <sub><b>InDieTasten</b></sub>
         </a>
@@ -1103,6 +1142,13 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
             <sub><b>HeliXZz</b></sub>
         </a>
     </td>
+    <td align="center">
+        <a href="https://github.com/mryesiller">
+            <img src="https://avatars.githubusercontent.com/u/24632172?v=4" width="80;" alt="mryesiller"/>
+            <br />
+            <sub><b>Göksel Yeşiller</b></sub>
+        </a>
+    </td>
     <td align="center">
         <a href="https://github.com/forwardemail">
             <img src="https://avatars.githubusercontent.com/u/32481436?v=4" width="80;" alt="forwardemail"/>
@@ -1116,15 +1162,15 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
             <br />
             <sub><b>Bastii717</b></sub>
         </a>
-    </td>
+    </td></tr>
+<tr>
     <td align="center">
         <a href="https://github.com/frankdez93">
             <img src="https://avatars.githubusercontent.com/u/87549420?v=4" width="80;" alt="frankdez93"/>
             <br />
             <sub><b>Frankdez93</b></sub>
         </a>
-    </td></tr>
-<tr>
+    </td>
     <td align="center">
         <a href="https://github.com/ratty222">
             <img src="https://avatars.githubusercontent.com/u/92832598?u=137b65530cbd5f5af9c24cde51baa6cc77cc934b&v=4" width="80;" alt="ratty222"/>
@@ -1157,7 +1203,7 @@ Huge thanks to these wonderful people, who sponsor me on GitHub, their support h
         <a href="https://github.com/mezza93">
             <img src="https://avatars.githubusercontent.com/u/153599966?v=4" width="80;" alt="mezza93"/>
             <br />
-            <sub><b>mezza93</b></sub>
+            <sub><b>Mezza93</b></sub>
         </a>
     </td></tr>
 </table>

api/quality.js

@@ -5,14 +5,17 @@ const handler = async (url, event, context) => {
   const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
 
   if (!apiKey) {
-    throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set');
+    throw new Error(
+      'Missing Google API. You need to set the `GOOGLE_CLOUD_API_KEY` environment variable'
+    );
   }
 
-  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
+  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?`
+  + `url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY`
+  + `&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile`
+  + `&key=${apiKey}`;
 
-  const response = await axios.get(endpoint);
-  
-  return response.data;
+  return (await axios.get(endpoint)).data;
 };
 
 module.exports = middleware(handler);

api/sitemap.js

@@ -39,10 +39,8 @@ const handler = async (url) => {
 
     return sitemap;
   } catch (error) {
-    // If error occurs
-    console.log(error.message);
     if (error.code === 'ECONNABORTED') {
-      return { error: 'Request timed out' };
+      return { error: 'Request timed out after 5000ms' };
     } else {
       return { error: error.message };
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-check",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "private": false,
   "description": "All-in-one OSINT tool for analyzing any website",
   "repository": "github:lissy93/web-check",
@@ -45,6 +45,7 @@
     "cors": "^2.8.5",
     "csv-parser": "^3.0.0",
     "dotenv": "^16.3.1",
+    "express-rate-limit": "^7.2.0",
     "flatted": "^3.2.7",
     "follow-redirects": "^1.15.2",
     "got": "^13.0.0",

server.js

@@ -2,6 +2,7 @@ const express = require('express');
 const fs = require('fs');
 const path = require('path');
 const cors = require('cors');
+const rateLimit = require('express-rate-limit');
 const historyApiFallback = require('connect-history-api-fallback');
 require('dotenv').config();
 
@@ -20,6 +21,34 @@ app.use(cors({
   origin: process.env.API_CORS_ORIGIN || '*',
 }));
 
+// Define max requests within each time frame
+const limits = [
+  { timeFrame: 10 * 60, max: 100, messageTime: '10 minutes' },
+  { timeFrame: 60 * 60, max: 250, messageTime: '1 hour' },
+  { timeFrame: 12 * 60 * 60, max: 500, messageTime: '12 hours' },
+];
+
+// Construct a message to be returned if the user has been rate-limited
+const makeLimiterResponseMsg = (retryAfter) => {
+  const why = 'This keeps the service running smoothly for everyone. '
+  + 'You can get around these limits by running your own instance of Web Check.';
+  return `You've been rate-limited, please try again in ${retryAfter} seconds.\n${why}`;
+};
+
+// Create rate limiters for each time frame
+const limiters = limits.map(limit => rateLimit({
+  windowMs: limit.timeFrame * 1000,
+  max: limit.max,
+  standardHeaders: true,
+  legacyHeaders: false,
+  message: { error: makeLimiterResponseMsg(limit.messageTime) }
+}));
+
+// If rate-limiting enabled, then apply the limiters to the /api endpoint
+if (process.env.API_ENABLE_RATE_LIMIT === 'true') {
+  app.use('/api', limiters);
+}
+
 // Read and register each API function as an Express routes
 fs.readdirSync(dirPath, { withFileTypes: true })
   .filter(dirent => dirent.isFile() && dirent.name.endsWith('.js'))
@@ -85,7 +114,6 @@ fs.readdirSync(dirPath, { withFileTypes: true })
     await Promise.all(handlerPromises);
     res.json(results);
   });
-  
 
 // Handle SPA routing
 app.use(historyApiFallback({

src/App.tsx

@@ -12,7 +12,6 @@ const Container = Styled.main`
   background: ${colors.background};
   color: ${colors.textColor};
   width: 100vw;
-  height: 100vh;
   margin: 0;
 `;
 

src/pages/About.tsx

@@ -101,6 +101,18 @@ const Section = styled(StyledCard)`
   }
 `;
 
+const SponsorshipContainer = styled.div`
+  display: flex;
+  justify-content: space-between;
+  gap: 1rem;
+  flex-wrap: wrap;
+  align-items: center;
+  line-height: 1.5rem;
+  img {
+    border-radius: 4px;
+  }
+`;
+
 const makeAnchor = (title: string): string => {
   return title.toLowerCase().replace(/[^\w\s]|_/g, "").replace(/\s+/g, "-");
 };
@@ -121,6 +133,26 @@ const About = (): JSX.Element => {
           <p key={index}>{para}</p>
         ))}
         <hr />
+        <SponsorshipContainer>
+          <p>
+            Web-Check is kindly sponsored
+            by <a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+              Terminal Trove
+            </a>
+            <br />
+            The $HOME of all things in the terminal.
+            <br />
+            <small>
+              <a href="https://terminaltrove.com/newsletter?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+                Find your next CLI / TUI tool, and get updates to your inbox
+              </a>
+            </small>
+          </p>
+          <a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+            <img width="300" alt="Terminal Trove" src="https://i.ibb.co/T1KzVmR/terminal-trove-green.png" />
+          </a>
+        </SponsorshipContainer>
+        <hr />
         <p>
           Web-Check is developed and maintained by <a href="https://aliciasykes.com">Alicia Sykes</a>.
           It's licensed under the <a href="https://github.com/Lissy93/web-check/blob/master/LICENSE">MIT license</a>,

src/pages/Home.tsx

@@ -20,7 +20,7 @@ const HomeContainer = styled.section`
   align-items: center;
   height: 100%;
   font-family: 'PTMono';
-  padding: 0 1rem;
+  padding: 1.5rem 1rem 4rem 1rem;
   footer {
     z-index: 1;
   }
@@ -34,10 +34,52 @@ const UserInputMain = styled.form`
   z-index: 5;
   margin: 1rem;
   width: calc(100% - 2rem);
-  max-width: 50rem;
+  max-width: 60rem;
   z-index: 2;
 `;
 
+const SponsorCard = styled.div`
+  background: ${colors.backgroundLighter};
+  box-shadow: 4px 4px 0px ${colors.bgShadowColor};
+  border-radius: 8px;
+  padding: 1rem;
+  z-index: 5;
+  margin: 1rem;
+  width: calc(100% - 2rem);
+  max-width: 60rem;
+  z-index: 2;
+  .inner {
+    display: flex;
+    justify-content: space-between;
+    flex-wrap: wrap;
+    gap: 1rem;
+    p {
+      margin: 0.25rem 0;
+    }
+  }
+  a {
+    color: ${colors.textColor};
+  }
+  img {
+    border-radius: 0.25rem;
+    box-shadow: 2px 2px 0px ${colors.fgShadowColor};
+    transition: box-shadow 0.2s;
+    margin: 0 auto;
+    display: block;
+    width: 200px;
+    &:hover {
+      box-shadow: 4px 4px 0px ${colors.fgShadowColor};
+    }
+    &:active {
+      box-shadow: -2px -2px 0px ${colors.fgShadowColor};
+    }
+  }
+  .cta {
+    font-size: 0.78rem;
+    a { color: ${colors.primary}; }
+  }
+`;
+
 // const FindIpButton = styled.a`
 //   margin: 0.5rem;
 //   cursor: pointer;
@@ -55,7 +97,7 @@ const ErrorMessage = styled.p`
 const SiteFeaturesWrapper = styled(StyledCard)`
   margin: 1rem;
   width: calc(100% - 2rem);
-  max-width: 50rem;
+  max-width: 60rem;
   z-index: 2;
   .links {
     display: flex;
@@ -173,6 +215,28 @@ const Home = (): JSX.Element => {
         { errorMsg && <ErrorMessage>{errorMsg}</ErrorMessage>}
         <Button styles="width: calc(100% - 1rem);" size="large" onClick={submit}>Analyze!</Button>
       </UserInputMain>
+      <SponsorCard>
+        <Heading as="h2" size="small" color={colors.primary}>Sponsored by</Heading>
+        <div className="inner">
+          <p>
+            <a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+              Terminal Trove
+            </a> - The $HOME of all things in the terminal.
+            <br />
+            <span className="cta">
+              Get updates on the latest CLI/TUI tools via
+              the <a className="cta" href="https://terminaltrove.com/newsletter?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+                Terminal Trove newsletter
+              </a>
+            </span>
+            
+          </p>
+          <a href="https://terminaltrove.com/?utm_campaign=github&utm_medium=referral&utm_content=web-check&utm_source=wcgh">
+            <img width="120" alt="Terminal Trove" src="https://i.ibb.co/NKtYjJ1/terminal-trove-web-check.png" />
+          </a>
+        </div>
+
+      </SponsorCard>
       <SiteFeaturesWrapper>
         <div className="features">
           <Heading as="h2" size="small" color={colors.primary}>Supported Checks</Heading>

src/pages/Results.tsx

@@ -288,7 +288,7 @@ const Results = (): JSX.Element => {
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
     fetchRequest: () => fetch(`${api}/quality?url=${address}`)
       .then(res => parseJson(res))
-      .then(res => res?.lighthouseResult || { error: 'No Data'}),
+      .then(res => res?.lighthouseResult || { error: res.error || 'No Data' }),
   });
 
   // Get the technologies used to build site, using Wappalyzer

yarn.lock

@@ -9788,6 +9788,11 @@ express-logging@1.1.1:
   dependencies:
     on-headers "^1.0.0"
 
+express-rate-limit@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-7.2.0.tgz#06ce387dd5388f429cab8263c514fc07bf90a445"
+  integrity sha512-T7nul1t4TNyfZMJ7pKRKkdeVJWa2CqB8NA1P8BwYaoDI5QSBZARv5oMS43J7b7I5P+4asjVXjb7ONuwDKucahg==
+
 express@4.18.2, express@^4.17.3:
   version "4.18.2"
   resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz"