From 4b4b9687100eba4d9b499fbfe84fcd42118c5dc2 Mon Sep 17 00:00:00 2001 From: g0tmi1k Date: Mon, 14 May 2018 15:57:33 +0100 Subject: [PATCH] Check HTTP status of each value in /robots.txt --- lib/wpscan/web_site/robots_txt.rb | 6 ++++++ wpscan.rb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/wpscan/web_site/robots_txt.rb b/lib/wpscan/web_site/robots_txt.rb index d8d52cfb..2a10893d 100644 --- a/lib/wpscan/web_site/robots_txt.rb +++ b/lib/wpscan/web_site/robots_txt.rb @@ -15,6 +15,12 @@ class WebSite @uri.clone.merge('robots.txt').to_s end + # Check status code for each robots.txt entry + def header_robots_txt(url) + code = Browser.get(url).code + puts info("Interesting entry from robots.txt: #{url} [HTTP #{code}]") + end + # Parse robots.txt # @return [ Array ] URLs generated from robots.txt def parse_robots_txt diff --git a/wpscan.rb b/wpscan.rb index 6ef9aadd..f9b40364 100755 --- a/wpscan.rb +++ b/wpscan.rb @@ -240,7 +240,7 @@ def main puts info("robots.txt available under: #{wp_target.robots_url}") wp_target.parse_robots_txt.each do |dir| - puts info("Interesting entry from robots.txt: #{dir}") + wp_target.header_robots_txt(dir) end spacer() end