fix duplicate robots.txt entries

lib/wpscan/web_site/robots_txt.rb

@@ -28,6 +28,7 @@ class WebSite
       if entries
         entries.flatten!
         entries.compact.sort!
+        entries.uniq!
         wordpress_path = @uri.path
         RobotsTxt.known_dirs.each do |d|
           entries.delete(d)

spec/samples/wpscan/web_site/robots_txt/robots_duplicate_1.txt

@@ -0,0 +1,17 @@
+User-agent: *
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-includes/
+Disallow: /wordpress/admin/
+Disallow: /wordpress/wp-admin/
+Disallow: /wordpress/secret/
+Disallow: /wordpress/secret/
+Disallow: /wordpress/
+Disallow: /wordpress/secret/
+Disallow: /Wordpress/wp-admin/
+Disallow: /wp-admin/tralling-space/
+Allow: /asdf/
+
+Sitemap: http://10.0.0.0/sitemap.xml.gz

spec/samples/wpscan/web_site/robots_txt/robots_duplicate_2.txt

@@ -0,0 +1,9 @@
+User-agent: *
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+Disallow: /wp-admin/
+
+Sitemap: http://10.0.0.0/sitemap.xml.gz

spec/shared_examples/web_site/robots_txt.rb

@@ -61,6 +61,24 @@ shared_examples 'WebSite::RobotsTxt' do
           http://example.localhost/asdf/
         )
       end
+
+      it 'removes duplicate entries from robots.txt test 1' do
+        @fixture = fixtures_dir + '/robots_txt/robots_duplicate_1.txt'
+        @expected = %w(
+          http://example.localhost/wordpress/
+          http://example.localhost/wordpress/admin/
+          http://example.localhost/wordpress/wp-admin/
+          http://example.localhost/wordpress/secret/
+          http://example.localhost/Wordpress/wp-admin/
+          http://example.localhost/wp-admin/tralling-space/
+          http://example.localhost/asdf/
+        )
+      end
+
+      it 'removes duplicate entries from robots.txt test 2' do
+        @fixture = fixtures_dir + '/robots_txt/robots_duplicate_2.txt'
+        @expected = nil
+      end
     end
 
     context 'installed in sub directory' do