Generate_List

# File lib/wpstools/generate_list.rb, line 27
def initialize(type, verbose)
  if type =~ /plugins/
    @type           = "plugin"
    @svn_url        = 'http://plugins.svn.wordpress.org/'
    @file_name      = DATA_DIR + '/plugins.txt'
    @popular_url    = 'http://wordpress.org/extend/plugins/browse/popular/'
    @popular_regex  = %{<h3><a href="http://wordpress.org/extend/plugins/(.+)/">.+</a></h3>}
  elsif type =~ /themes/
    @type           = "theme"
    @svn_url        = 'http://themes.svn.wordpress.org/'
    @file_name      = DATA_DIR + '/themes.txt'
    @popular_url    = 'http://wordpress.org/extend/themes/browse/popular/'
    @popular_regex  = %{<h3><a href="http://wordpress.org/extend/themes/(.+)">.+</a></h3>}
  else
    raise "Type #{type} not defined"
  end
  @verbose  = verbose
  @browser  = Browser.instance
  @hydra    = @browser.hydra
end

# File lib/wpstools/generate_list.rb, line 48
def generate_full_list
  items = Svn_Parser.new(@svn_url, @verbose).parse
  save items
end

# File lib/wpstools/generate_list.rb, line 53
def generate_popular_list(pages)
  popular = get_popular_items(pages)
  items = Svn_Parser.new(@svn_url, @verbose).parse(popular)
  save items
end

# File lib/wpstools/generate_list.rb, line 62
def get_popular_items(pages)
  found_items = []
  page_count = 1
  queue_count = 0

  (1...(pages.to_i+1)).each do |page|
    # First page has another URL
    url = (page == 1) ? @popular_url : @popular_url + 'page/' + page.to_s + '/'
    request = @browser.forge_request(url)

    queue_count += 1

    request.on_complete do |response|
      puts "[+] Parsing page " + page_count.to_s if @verbose
      page_count += 1
      response.body.scan(@popular_regex).each do |item|
        puts "[+] Found popular #{@type}: #{item}" if @verbose
        found_items << item[0]
      end
    end

    @hydra.queue(request)

    if queue_count == @browser.max_threads
      @hydra.run
      queue_count = 0
    end

  end

  @hydra.run

  found_items.sort!
  found_items.uniq
end

# File lib/wpstools/generate_list.rb, line 99
def save(items)
  items.sort!
  items.uniq!
  puts "[*] We have parsed #{items.length} #{@type}s"
  File.open(@file_name, 'w') { |f| f.puts(items) }
  puts "New #{@file_name} file created"
end