tasks::cve: Support compressed data from NVD

1 files changed, 71 insertions, 188 deletions

diff --git a/lib/tasks/cve.rake b/lib/tasks/cve.rake
index 4e7337e..dbe06cf 100644
--- a/lib/tasks/cve.rake
+++ b/lib/tasks/cve.rake
@@ -1,5 +1,5 @@
 # ===GLSAMaker v2
-#  Copyright (C) 2010 Alex Legler <a3li@gentoo.org>
+#  Copyright (C) 2010–15 Alex Legler <a3li@gentoo.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -8,49 +8,41 @@
 #
 # For more information, see the LICENSE file.
 
-libs = %w[ nokogiri ]
+libs = %w( nokogiri zlib stringio )
 libs << File.join(File.dirname(__FILE__), '..', 'bugzilla')
 libs << File.join(File.dirname(__FILE__), '..', 'glsamaker')
 libs << File.join(File.dirname(__FILE__), 'utils')
 
-#print "About to load libraries\n"
-libs.each { |lib| 
-#	print "Loading #{lib}\n"
-	require lib 
-}
+libs.each { |lib| require lib }
 
 TMPDIR = File.join(File.dirname(__FILE__), '..', '..', 'tmp')
 # What year are the first CVEs from?
-YEAR = 2004
-BASEURL = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml"
+YEAR = (ENV['START_YEAR'] || 2004).to_i
+BASEURL = 'https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz'
 
-DEBUG   = ENV.has_key? 'DEBUG'
-VERBOSE = ENV.has_key?('VERBOSE') or DEBUG
-QUIET   = ENV.has_key? 'QUIET'
+DEBUG   = ENV.key? 'DEBUG'
+VERBOSE = (ENV.key?('VERBOSE') || DEBUG)
+QUIET   = ENV.key? 'QUIET'
 
-raise "I can't be quiet and verbose at the same time..." if QUIET and VERBOSE
+fail "I can't be quiet and verbose at the same time..." if QUIET && VERBOSE
 
 namespace :cve do
-  desc "Full CVE data import"
-  task :full_import => [:environment, "db:load_config"] do
+  desc 'Full CVE data import'
+  task full_import: [:environment, 'db:load_config'] do
     start_ts = Time.now
 
     (YEAR..Date.today.year).each do |year|
-      info "Processing CVEs from ".bold + year.to_s.purple
+      info 'Processing CVEs from '.bold + year.to_s.purple
 
-      xmldata = status "Downloading" do
-        Glsamaker::HTTP.get(BASEURL % year)
+      xmldata = status 'Downloading' do
+        gunzip_str Glsamaker::HTTP.get(BASEURL % year)
       end
 
-      xml = status "Loading XML" do
+      xml = status 'Loading XML' do
         Nokogiri::XML(xmldata)
       end
 
-      #xml = status "Loading cached XML from a file" do
-      # Nokogiri::XML(File.open('/Users/alex/Desktop/nvdcve-2.0-2004.xml'))
-      #end
-
-      namespace = {'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0'}
+      namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
       processed_cves = 0
       cpe_cache = {}
 
@@ -58,12 +50,12 @@ namespace :cve do
       info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
 
       cves.each do |cve|
-        unless VERBOSE or QUIET
+        unless VERBOSE || QUIET
 
           if processed_cves % 500 == 0
-            print ".".purple
+            print '.'.purple
           else
-            print "." if processed_cves % 100 == 0
+            print '.' if processed_cves % 100 == 0
           end
         end
 
@@ -83,24 +75,24 @@ namespace :cve do
       puts
     end
 
-    info "done".green
+    info 'done'.green
     info "(#{Time.now - start_ts} seconds)"
   end
 
-  desc "Incremental CVE data update"
-  task :update => :environment do
+  desc 'Incremental CVE data update'
+  task update: :environment do
     start_ts = Time.now
-    info "Running incremental CVE data update..."
+    info 'Running incremental CVE data update...'
 
-    xmldata = status "Downloading" do
-      Glsamaker::HTTP.get(BASEURL % 'modified')
+    xmldata = status 'Downloading' do
+      gunzip_str Glsamaker::HTTP.get(BASEURL % 'modified')
     end
 
-    xml = status "Loading XML" do
+    xml = status 'Loading XML' do
       Nokogiri::XML(xmldata)
     end
 
-    namespace = {'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0'}
+    namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
     processed_cves = created_cves = updated_cves = 0
     cpe_cache = {}
 
@@ -108,11 +100,11 @@ namespace :cve do
     info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
 
     cves.each do |cve|
-      unless VERBOSE or QUIET
+      unless VERBOSE || QUIET
         if processed_cves % 500 == 0
-          print ".".purple
+          print '.'.purple
         else
-          print "." if processed_cves % 100 == 0
+          print '.' if processed_cves % 100 == 0
         end
       end
 
@@ -120,23 +112,23 @@ namespace :cve do
 
       c = Cve.find_by_cve_id cve['id']
 
-      if c == nil
-        debug "Creating CVE."
+      if c.nil?
+        debug 'Creating CVE.'
         create_cve(cve)
         created_cves += 1
       else
         last_changed_at = Time.parse(cve.xpath('vuln:last-modified-datetime').first.content).utc
         db_lca = c.last_changed_at
-        
-        if last_changed_at.to_i > c.last_changed_at.to_i          
-          debug "Updating CVE. Timestamp changed."
+
+        if last_changed_at.to_i > c.last_changed_at.to_i
+          debug 'Updating CVE. Timestamp changed.'
           summary = cve.xpath('vuln:summary').first.content
           c.attributes = {
-            :cve_id => cve['id'],
-            :summary => summary,
-            :cvss => cvss_xml2str(cve.xpath('vuln:cvss')),
-            :published_at => DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
-            :last_changed_at => DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content),
+            cve_id: cve['id'],
+            summary: summary,
+            cvss: cvss_xml2str(cve.xpath('vuln:cvss')),
+            published_at: DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
+            last_changed_at: DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content)
           }
 
           c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/
@@ -151,28 +143,29 @@ namespace :cve do
               ref.xpath('vuln:reference').first['href']
             ]
           end
-          
+
           c.references.each do |ref|
             db_references << [ref.source, ref.title, ref.uri]
           end
-          
+
           rem = db_references - xml_references
           debug "Removing references: #{rem.inspect}"
-          
+
           rem.each do |item|
             ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first
+            debug ref
             c.references.delete(ref)
             ref.destroy
           end
-          
+
           add = xml_references - db_references
           debug "Ading references:    #{add.inspect}"
-          
+
           add.each do |item|
             c.references.create(
-              :source => item[0],
-              :title =>  item[1],
-              :uri =>    item[2]
+              source: item[0],
+              title:  item[1],
+              uri:    item[2]
             )
           end
 
@@ -181,28 +174,28 @@ namespace :cve do
           cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod|
             xml_cpes << prod.content
           end
-          
+
           c.cpes.each do |prod|
             db_cpes << prod.cpe
           end
-          
+
           rem = db_cpes - xml_cpes
           debug "Removing CPEs: #{rem.inspect}"
-          
+
           rem.each do |item|
             c.cpes.delete(Cpe.find_by_cpe(item))
           end
-          
+
           add = xml_cpes - db_cpes
           debug "Ading CPEs:    #{add.inspect}"
-          
+
           add.each do |item|
-            cpe = Cpe.where(:cpe => item).first
-            cpe ||= Cpe.create(:cpe => item)
+            cpe = Cpe.where(cpe: item).first
+            cpe ||= Cpe.create(cpe: item)
 
             c.cpes << cpe
           end
-          
+
           c.save!
           updated_cves += 1
         end
@@ -212,135 +205,21 @@ namespace :cve do
       STDOUT.flush
     end
 
-    info ""
+    info ''
 
     info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)"
   end
-
-  desc "Import CVE resolutions from the old CVE tool"
-  task :oldimport => :environment do
-
-    file = status "Downloading" do
-      Glsamaker::HTTP.get("http://overlays.gentoo.org/svn/proj/security/data/CVE/list")
-    end
-
-    $saved_cve = {:id => "", :bugs => []}
-    sys_usr = User.find(0)
-
-    file.each_line do |l|
-      # Start a new entry
-      if l[0..2] == "CVE"
-        # Save the old entry before starting a new one
-        unless $saved_cve[:id] == ""
-          #
-          # Public Service Announcement:
-          # IN THE FOLLOWING CODE BLOCK: DO *NEVER* ADD ANY `next' STATEMENTS!
-          #                                                           -a3li
-          debug "CVE: #{$saved_cve[:id]}. Bugs: #{$saved_cve[:bugs].inspect}. State: #{$saved_cve[:state]}. Reason: #{$saved_cve[:reason]}"
-          c = Cve.find_by_cve_id($saved_cve[:id])
-          c = nil if $saved_cve[:skip]
-
-          if c == nil
-            $stderr.puts "#{$saved_cve[:id]} not found in the CVE databse OR forced skip. Skipping."
-          else
-
-            state = $saved_cve[:state]
-            state ||= "NEW"
-
-            if $saved_cve.has_key? :note
-              c.comments.create(
-                :user_id => 0,
-                :comment => $saved_cve[:note]
-             )
-            end
-
-            unless state == "NEW"
-              $saved_cve[:bugs].each do |bug|
-                c.assign(bug, sys_usr)
-              end
-
-              if state == "NFU"
-                c.nfu(sys_usr, $saved_cve[:reason])
-              end
-
-              if state == "REJECTED"
-                c.invalidate(sys_usr, "REJECTED")
-              end
-            end
-          end
-        end
-
-        l.match /^(CVE-\d{4}-\d{4})/
-        $saved_cve = {:id => $1, :bugs => []}
-        next
-      end
-
-      # We ignore reserved, as NVD doesn't support it.
-      if l =~ /^\tRESERVED$/
-        $saved_cve[:skip] = true
-        $saved_cve[:state] = "RESERVED"
-        next
-      end
-
-      if l =~ /^\tREJECTED$/
-        $saved_cve[:state] = "REJECTED"
-        next
-      end
-
-      if l =~ /^\tNOT-FOR-US: (.*)$/
-        $saved_cve[:state] = "NFU" unless $saved_cve[:state] == "REJECTED"
-        $saved_cve[:reason] = $1
-        next
-      end
-
-      if l =~ /^\tBUG: (\d+)$/
-        $saved_cve[:bugs] << Integer($1)
-        $saved_cve[:state] = "ASSIGNED"
-        next
-      end
-
-      if l =~ /^\tTODO: check$/
-        $saved_cve[:state] = "NEW"
-        next
-      end
-
-      if l =~ /^\tTODO: (.*)$/
-        if $1 == 'check-old'
-          $saved_cve[:state] = "NFU"
-          $saved_cve[:reason] = $1
-        else
-          $saved_cve[:state] = "NEW"
-          $saved_cve[:note] = $1
-        end
-        next
-      end
-
-      if l =~ /^\tNOTE: (.*)$/
-        $saved_cve[:note] = $1
-        next
-      end
-
-      puts "XXX: #{l}"
-      break
-    end # each line
-
-  end # task
-
-end # namespace cve
-
-# Misc. functions
+end
 
 # Run something, and display a status info around it
 def status(message)
-  unless block_given?
-    raise ArgumentError, "I want a block :("
-  end
+  fail ArgumentError, 'I want a block :(' unless block_given?
 
   print "#{message}....." unless QUIET
   STDOUT.flush
 
   stuff = yield
-  print "\b" * 5 + " done.".green + "\n" unless QUIET
+  print "\b" * 5 + ' done.'.green + "\n" unless QUIET
   stuff
 end
 
@@ -361,12 +240,12 @@ def cvss_xml2str(data)
   return nil if data.size == 0
 
   str  = "#{get_content(data, 'cvss:base_metrics/cvss:score')}/"
-  str += "AV:#{get_content(data, 'cvss:base_metrics/cvss:access-vector')[0,1]}/"
-  str += "AC:#{get_content(data, 'cvss:base_metrics/cvss:access-complexity')[0,1]}/"
-  str += "Au:#{get_content(data, 'cvss:base_metrics/cvss:authentication')[0,1]}/"
-  str += "C:#{get_content(data, 'cvss:base_metrics/cvss:confidentiality-impact')[0,1]}/"
-  str += "I:#{get_content(data, 'cvss:base_metrics/cvss:integrity-impact')[0,1]}/"
-  str += "A:#{get_content(data, 'cvss:base_metrics/cvss:availability-impact')[0,1]}"
+  str += "AV:#{get_content(data, 'cvss:base_metrics/cvss:access-vector')[0, 1]}/"
+  str += "AC:#{get_content(data, 'cvss:base_metrics/cvss:access-complexity')[0, 1]}/"
+  str += "Au:#{get_content(data, 'cvss:base_metrics/cvss:authentication')[0, 1]}/"
+  str += "C:#{get_content(data, 'cvss:base_metrics/cvss:confidentiality-impact')[0, 1]}/"
+  str += "I:#{get_content(data, 'cvss:base_metrics/cvss:integrity-impact')[0, 1]}/"
+  str += "A:#{get_content(data, 'cvss:base_metrics/cvss:availability-impact')[0, 1]}"
 
   str
 end
@@ -400,3 +279,7 @@ def create_cve(cve)
     _cve.cpes << cpe
   end
 end
+
+def gunzip_str(str)
+  Zlib::GzipReader.new(StringIO.new(str)).read
+end