Moved http response to a function

lib/common/common_helper.rb

@@ -326,4 +326,9 @@ def valid_json?(json)
     return true
   rescue JSON::ParserError => e
     return false
+end
+
+# Get the HTTP response code
+def get_http_status(url)
+  Browser.get(url.to_s).code
 end

lib/wpscan/web_site/robots_txt.rb

@@ -15,12 +15,6 @@ class WebSite
       @uri.clone.merge('robots.txt').to_s
     end
 
-    # Check status code for each robots.txt entry
-    def header_robots_txt(url)
-      code = Browser.get(url).code
-      puts info("Interesting entry from robots.txt: #{url}   [HTTP #{code}]")
-    end
-
     # Parse robots.txt
     # @return [ Array ] URLs generated from robots.txt
     def parse_robots_txt