Check HTTP status of each value in /robots.txt

lib/wpscan/web_site/robots_txt.rb

@@ -15,6 +15,12 @@ class WebSite
       @uri.clone.merge('robots.txt').to_s
     end
 
+    # Check status code for each robots.txt entry
+    def header_robots_txt(url)
+      code = Browser.get(url).code
+      puts info("Interesting entry from robots.txt: #{url}   [HTTP #{code}]")
+    end
+
     # Parse robots.txt
     # @return [ Array ] URLs generated from robots.txt
     def parse_robots_txt

wpscan.rb

@@ -240,7 +240,7 @@ def main
       puts info("robots.txt available under: #{wp_target.robots_url}")
 
       wp_target.parse_robots_txt.each do |dir|
-        puts info("Interesting entry from robots.txt: #{dir}")
+        wp_target.header_robots_txt(dir)
       end
       spacer()
     end