fix duplicate robots.txt entries

2015-08-18 15:55:10 +02:00
parent 390f10e83f
commit 7ec095d708
4 changed files with 45 additions and 0 deletions
--- a/lib/wpscan/web_site/robots_txt.rb
+++ b/lib/wpscan/web_site/robots_txt.rb
@@ -28,6 +28,7 @@ class WebSite
      if entries
        entries.flatten!
        entries.compact.sort!
+        entries.uniq!
        wordpress_path = @uri.path
        RobotsTxt.known_dirs.each do |d|
          entries.delete(d)