deblan-mirror/web-check
1
0
Fork 0
mirror of https://codeberg.org/alicia/web-check.git synced 2026-03-14 22:55:47 +01:00
Code Issues Projects Releases Packages Wiki Activity

Skip check if robots.txt not present

Browse source
This commit is contained in:
Alicia Sykes 2023-08-18 19:27:38 +01:00
commit dbcbd36874
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api/robots-txt.js
View file

@ -48,7 +48,11 @@ const handler = async function(url) {
const response = await axios.get(robotsURL);
if (response.status === 200) {
return parseRobotsTxt(response.data);
const parsedData = parseRobotsTxt(response.data);
if (!parsedData.robots || parsedData.robots.length === 0) {
return { skipped: 'No robots.txt file present, unable to continue' };
}
return parsedData;
} else {
return {
statusCode: response.status,