fix issue #265 - remove base64 images before passive detection

lib/common/collections/wp_items/detectable.rb

@@ -73,6 +73,8 @@ class WpItems < Array
     def passive_detection(wp_target, options = {})
       results  = new(wp_target)
       body     = Browser.get(wp_target.url).body
+      # improves speed
+      body     = remove_base64_images_from_html(body)
       names    = body.scan(passive_detection_pattern(wp_target))
 
       names.flatten.uniq.each { |name| results.add(name) }

lib/common/common_helper.rb

@@ -149,3 +149,10 @@ def get_equal_string_end(stringarray = [''])
   end
   already_found
 end
+
+def remove_base64_images_from_html(html)
+  # remove data:image/png;base64, images
+  base64regex = %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}
+  imageregex = %r{data\s*:\s*image/[^\s;]+\s*;\s*base64\s*,\s*}
+  html.gsub(/["']\s*#{imageregex}#{base64regex}\s*["']/, '""')
+end

spec/lib/common/common_helper_spec.rb

@@ -71,4 +71,21 @@ describe 'common_helper' do
       @expected = ' | test'
     end
   end
+
+  describe '#remove_base64_images_from_html' do
+    after :each do
+      output = remove_base64_images_from_html(@html)
+      output.should == @expected
+    end
+
+    it 'removes the valid base64 image' do
+      @html = '<img alt="" src="data:image/x-png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsTAAALEwEAmpwYAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAABLElEQVR42qSTQStFURSFP7f3XygyoAwoYSYMPCIpk2egMFSmUvwCRpSRDIwYGbwyVuYykB9y914m951z7nHe6J26dc9u77XXWmdvJLF7/audqx9JYuvyW92LL0li8K2df2r17CPEVk7ftXTclyQqAMmRCwC5I3fS42a4W7y74VYDNAAuJA8AaXIsSACsDgAdAJeFrnnyoMBygKZJJ3b1It0AmsTMDPdEgrujJqHEwCxqznMaD2KgyCDRnEuo8qJhHvx/hcQDbzGoix5Yi4G1TcwZWNEDKwJU+WDkhg2ToDaD+M65YcVB8jg3Y5IY5VQAyyf9gLJw+CqAuYNnAczsPQpgevtBU937kDexcdssj8Ti0ZskMd97CRs3u//U2sjJzbtwH1+/Cf8jS/gbAMmWc42HzdIjAAAAAElFTkSuQmCC" />'
+      @expected = '<img alt="" src="" />'
+    end
+
+    it 'ignores invalid base64 content' do
+      @html = '<img alt="" src="data:image/x-png;base64,iVBORw0KGgo" />'
+      @expected = @html
+    end
+  end
 end