diff --git a/README b/README
index 22812643..438b939e 100644
--- a/README
+++ b/README
@@ -252,9 +252,9 @@ Debug output...
-v, --verbose Verbose output
--check-vuln-ref-urls, --cvru Check all the vulnerabilities reference urls for 404
--check-local-vulnerable-files, --clvf LOCAL_DIRECTORY Perform a recursive scan in the LOCAL_DIRECTORY to find vulnerable files or shells
- --generate-plugin-list, --gpl [NUMBER_OF_PAGES] Generate a new data/plugins.txt file. (supply number of *pages* to parse, default : 150)
+ --generate-plugin-list, --gpl [NUMBER_OF_ITEMS] Generate a new data/plugins.txt file. (supply number of *items* to parse, default : 1500)
--generate-full-plugin-list, --gfpl Generate a new full data/plugins.txt file
- --generate-theme-list, --gtl [NUMBER_OF_PAGES] Generate a new data/themes.txt file. (supply number of *pages* to parse, default : 20)
+ --generate-theme-list, --gtl [NUMBER_OF_ITEMS] Generate a new data/themes.txt file. (supply number of *items* to parse, default : 200)
--generate-full-theme-list, --gftl Generate a new full data/themes.txt file
--generate-all, --ga Generate a new full plugins, full themes, popular plugins and popular themes list
-s, --stats Show WpScan Database statistics
@@ -262,8 +262,8 @@ Debug output...
==WPSTOOLS EXAMPLES==
-- Generate a new 'most popular' plugin list, up to 150 pages ...
-ruby wpstools.rb --generate-plugin-list 150
+- Generate a new 'most popular' plugin list, up to 1500 items ...
+ruby wpstools.rb --generate-plugin-list 1500
Locally scan a wordpress installation for vulnerable files or shells:
ruby wpstools.rb --check-local-vulnerable-files /var/www/wordpress/
diff --git a/README.md b/README.md
index 7d382786..8078c825 100644
--- a/README.md
+++ b/README.md
@@ -252,9 +252,9 @@ Debug output...
-v, --verbose Verbose output
--check-vuln-ref-urls, --cvru Check all the vulnerabilities reference urls for 404
--check-local-vulnerable-files, --clvf LOCAL_DIRECTORY Perform a recursive scan in the LOCAL_DIRECTORY to find vulnerable files or shells
- --generate-plugin-list, --gpl [NUMBER_OF_PAGES] Generate a new data/plugins.txt file. (supply number of *pages* to parse, default : 150)
+ --generate-plugin-list, --gpl [NUMBER_OF_ITEMS] Generate a new data/plugins.txt file. (supply number of *items* to parse, default : 1500)
--generate-full-plugin-list, --gfpl Generate a new full data/plugins.txt file
- --generate-theme-list, --gtl [NUMBER_OF_PAGES] Generate a new data/themes.txt file. (supply number of *pages* to parse, default : 20)
+ --generate-theme-list, --gtl [NUMBER_OF_ITEMS] Generate a new data/themes.txt file. (supply number of *items* to parse, default : 200)
--generate-full-theme-list, --gftl Generate a new full data/themes.txt file
--generate-all, --ga Generate a new full plugins, full themes, popular plugins and popular themes list
-s, --stats Show WpScan Database statistics.
@@ -263,9 +263,9 @@ Debug output...
#### WPSTOOLS EXAMPLES
-Generate a new 'most popular' plugin list, up to 150 pages...
+Generate a new 'most popular' plugin list, up to 1500 items...
-```ruby wpstools.rb --generate-plugin-list 150```
+```ruby wpstools.rb --generate-plugin-list 1500```
Locally scan a wordpress installation for vulnerable files or shells:
diff --git a/lib/wpstools/plugins/list_generator/generate_list.rb b/lib/wpstools/plugins/list_generator/generate_list.rb
index bc15678a..4211a9a1 100644
--- a/lib/wpstools/plugins/list_generator/generate_list.rb
+++ b/lib/wpstools/plugins/list_generator/generate_list.rb
@@ -10,13 +10,13 @@ class GenerateList
if type =~ /plugins/i
@type = 'plugin'
@svn_url = 'http://plugins.svn.wordpress.org/'
- @popular_url = 'http://wordpress.org/plugins/browse/popular/'
- @popular_regex = %r{}i
+ @popular_url = 'http://api.wordpress.org/plugins/info/1.0/'
+ @popular_action = 'query_plugins'
elsif type =~ /themes/i
@type = 'theme'
@svn_url = 'http://themes.svn.wordpress.org/'
- @popular_url = 'http://wordpress.org/themes/browse/popular/'
- @popular_regex = %r{}i
+ @popular_url = 'http://api.wordpress.org/themes/info/1.0/'
+ @popular_action = 'query_themes'
else
raise "Type #{type} not defined"
end
@@ -55,48 +55,46 @@ class GenerateList
save items
end
- def generate_popular_list(pages)
+ def generate_popular_list(items)
set_file_name(:popular)
- items = get_popular_items(pages)
+ items = get_popular_items(items)
save items
end
- # Send a HTTP request to the WordPress most popular theme or plugin webpage
- # parse the response for the names.
- def get_popular_items(pages)
+ # Fets most popular items via unofficial wordpress api
+ # see https://github.com/wpscanteam/wpscan/issues/657
+ def get_popular_items(items)
found_items = []
- page_count = 1
- retries = 0
- (1...(pages.to_i + 1)).each do |page|
- # First page has another URL
- url = (page == 1) ? @popular_url : @popular_url + 'page/' + page.to_s + '/'
- puts "[+] Parsing page #{page_count}" if @verbose
- code = 0
+ # in chunks of 100
+ step = 100
+ number_of_requests = (items.to_f / step.to_f).ceil
+ counter = 1
+ while items > 0
+ puts "[+] Request #{counter} / #{number_of_requests}"
+ rest = items < step ? items : step
+
+ # we need to fetch step entries every time, because the starting page
+ # is calculated: page * entries per page. If we would reduce the
+ # per page entries, the starting point will not match. So we are
+ # stripping down the array later
+ post_data = get_serialized(counter, step)
+ resp = Browser.post(@popular_url, { :body => { :action => @popular_action, :request => post_data } })
+ raise "Unknown reponse (code #{resp.code})" unless resp.code == 200
+ found = resp.body.scan(/"slug";s:[0-9]+:"([^"]+)";/).flatten
- while code != 200 && retries <= 3
- puts red("[!] Retrying request for page #{page} (Code: #{code})") unless code == 0
-
- request = @browser.forge_request(url)
- response = request.run
- code = response.code
-
- sleep(5) unless code == 200
- retries += 1
+ # too much entries? remove them
+ if found.length > rest
+ found = found[0,rest]
end
- page_count += 1
- found = 0
+ found_items << found
- response.body.scan(@popular_regex).each do |item|
- found_items << item[0]
- found = found + 1
- end
-
- retries = 0
- puts "[+] Found #{found} items on page #{page}" if @verbose
+ items -= rest
+ counter += 1
end
+ found_items.flatten!
found_items.sort!
found_items.uniq
end
@@ -111,4 +109,10 @@ class GenerateList
puts "New #@file_name file created"
end
+ private
+
+ def get_serialized(page_start, count)
+ 'O:8:"stdClass":4:{s:4:"page";i:' + page_start.to_s + ';s:8:"per_page";i:' + count.to_s + ';s:6:"browse";s:7:"popular";s:6:"fields";a:9:{s:11:"description";b:0;s:8:"sections";b:0;s:6:"tested";b:0;s:8:"requires";b:0;s:6:"rating";b:0;s:12:"downloadlink";b:0;s:12:"last_updated";b:0;s:8:"homepage";b:0;s:4:"tags";b:0;}}'
+ end
+
end
diff --git a/lib/wpstools/plugins/list_generator/list_generator_plugin.rb b/lib/wpstools/plugins/list_generator/list_generator_plugin.rb
index c2b08658..ef615f1c 100644
--- a/lib/wpstools/plugins/list_generator/list_generator_plugin.rb
+++ b/lib/wpstools/plugins/list_generator/list_generator_plugin.rb
@@ -6,10 +6,10 @@ class ListGeneratorPlugin < Plugin
super(author: 'WPScanTeam - @FireFart')
register_options(
- ['--generate-plugin-list [NUMBER_OF_PAGES]', '--gpl', Integer, 'Generate a new data/plugins.txt file. (supply number of *pages* to parse, default : 150)'],
+ ['--generate-plugin-list [NUMBER_OF_ITEMS]', '--gpl', Integer, 'Generate a new data/plugins.txt file. (supply number of *items* to parse, default : 1500)'],
['--generate-full-plugin-list', '--gfpl', 'Generate a new full data/plugins.txt file'],
- ['--generate-theme-list [NUMBER_OF_PAGES]', '--gtl', Integer, 'Generate a new data/themes.txt file. (supply number of *pages* to parse, default : 20)'],
+ ['--generate-theme-list [NUMBER_OF_ITEMS]', '--gtl', Integer, 'Generate a new data/themes.txt file. (supply number of *items* to parse, default : 200)'],
['--generate-full-theme-list', '--gftl', 'Generate a new full data/themes.txt file'],
['--generate-all', '--ga', 'Generate a new full plugins, full themes, popular plugins and popular themes list']
@@ -21,7 +21,7 @@ class ListGeneratorPlugin < Plugin
generate_all = options[:generate_all] || false
if options.has_key?(:generate_plugin_list) || generate_all
- most_popular('plugin', options[:generate_plugin_list] || 150)
+ most_popular('plugin', options[:generate_plugin_list] || 1500)
end
if options[:generate_full_plugin_list] || generate_all
@@ -29,7 +29,7 @@ class ListGeneratorPlugin < Plugin
end
if options.has_key?(:generate_theme_list) || generate_all
- most_popular('theme', options[:generate_theme_list] || 20)
+ most_popular('theme', options[:generate_theme_list] || 200)
end
if options[:generate_full_theme_list] || generate_all
@@ -39,10 +39,10 @@ class ListGeneratorPlugin < Plugin
private
- def most_popular(type, number_of_pages)
- puts "[+] Generating new most popular #{type} list"
+ def most_popular(type, number_of_items)
+ puts "[+] Generating new most popular #{type} list (#{number_of_items} items)"
puts
- GenerateList.new(type + 's', @verbose).generate_popular_list(number_of_pages)
+ GenerateList.new(type + 's', @verbose).generate_popular_list(number_of_items)
end
def full(type)
diff --git a/lib/wpstools/wpstools_helper.rb b/lib/wpstools/wpstools_helper.rb
index a478dca0..ac5ca124 100644
--- a/lib/wpstools/wpstools_helper.rb
+++ b/lib/wpstools/wpstools_helper.rb
@@ -12,14 +12,14 @@ def usage
puts
puts 'Examples:'
puts
- puts "- Generate a new 'most popular' plugin list, up to 150 pages ..."
- puts "ruby #{script_name} --generate-plugin-list 150"
+ puts "- Generate a new 'most popular' plugin list, up to 1500 items ..."
+ puts "ruby #{script_name} --generate-plugin-list 1500"
puts
puts '- Generate a new full plugin list'
puts "ruby #{script_name} --generate-full-plugin-list"
puts
- puts "- Generate a new 'most popular' theme list, up to 150 pages ..."
- puts "ruby #{script_name} --generate-theme-list 150"
+ puts "- Generate a new 'most popular' theme list, up to 1500 items ..."
+ puts "ruby #{script_name} --generate-theme-list 1500"
puts
puts '- Generate a new full theme list'
puts "ruby #{script_name} --generate-full-theme-list"