parse robots.txt

This commit is contained in:
Christian Mehlmauer
2013-08-10 11:35:17 +02:00
parent 6c8e76060a
commit 6c008015e9
10 changed files with 171 additions and 33 deletions

View File

@@ -105,6 +105,10 @@ def main
if wp_target.has_robots?
puts green('[+]') + " robots.txt available under '#{wp_target.robots_url}'"
wp_target.parse_robots_txt.each do |dir|
puts "#{green('[+]')} Interesting entry from robots.txt: #{dir}"
end
end
if wp_target.has_readme?