#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Scan for new CVEs, and report on stdout. # # Author: Martin Pitt # Author: Kees Cook # Author: Jamie Strandboge # Author: Marc Deslauriers # Author: Steve Beattie # Copyright (C) 2005-2021 Canonical Ltd. # # This script is distributed under the terms and conditions of the GNU General # Public License, Version 2 or later. See http://www.gnu.org/copyleft/gpl.html # for details. import datetime import json import math import optparse import os import os.path import random import re import shutil import subprocess import sys import tempfile import time import unittest import urllib.request import xml.sax import xml.sax.handler import xml.sax.xmlreader from html import escape from functools import reduce import progressbar import cve_lib import source_map # load settings, if any cve_lib.read_config() parser = optparse.OptionParser() parser.add_option("-r", "--report", help="Just report CVEs that need checking", action="store_true") parser.add_option("-v", "--verbose", help="Report verbose XML details", action="store_true") parser.add_option("-k", "--known", help="Only report CVEs already known", action="store_true") parser.add_option("-N", "--skip-nfu", help="Skip any CVEs marked as NFU (used with -k)", action="store_true") parser.add_option("-R", "--refresh", help="Refresh CVE descriptions", action="store_true") parser.add_option("", "--test", help="Run regression tests", action="store_true") parser.add_option("--untriaged", help="Process untriaged CVEs from output of locate_cves.py", metavar="FILE") parser.add_option("--mbox", help="Process untriaged CVEs from mbox file", metavar="FILE") parser.add_option("--rhel8oval", help="Process untriaged RHEL8 CVEs", metavar="FILE") parser.add_option("--import-missing-debian", help="Process missing Debian CVEs", action="store_true") parser.add_option("--debug", help="Report debugging information", action="store_true") parser.add_option("--cve", help="Check only the listed comma-separated CVEs and ignore others", action="store", default="") parser.add_option("--mistriaged", help="Process the specified number of possible mistriaged CVEs compared to Debian\n" "Implies --import-missing-debian", action="store", type=int, default=0) (opt, args) = parser.parse_args() experimental = os.getenv('CHECK_CVES_EXPERIMENTAL', False) # Pull in package details source = source_map.load() allsrcs = set() for release in list(source.keys()): allsrcs.update(set(source[release].keys())) # remove common words which also happen to be names # of source packages since our ignore suggestion is # likely to sometimes contain these common_words = ['an', 'and', 'context', 'file', 'modules', 'the', 'when'] allsrcs.difference_update(set(common_words)) # add boilerplate names too so we can get mysql or postgresql # etc even if they don't exist as source package names allsrcs.update(set(cve_lib.load_boilerplates().keys())) built_using_map = None destdir = "." # Skip stuff older than 2005 cve_limit = 2004 mistriaged_hint = 'Previously triaged as ignored in Ubuntu\n\n' ignore_strings = [ "** REJECT **", "Internet Explorer", "Microsoft Edge", "Windows 98", "Windows 2000", "Windows XP", "Windows Server 2003", "Windows NT", "Mercury Board", "ZeroBoard", "AntiVirus", "Microsoft", "SGI IRIX", "FreeBSD", "IBM AIX", "SCO", "OS X", "Mac OS", "Apple QuickTime", "Cisco", "ActiveX", "Joomla!", "TYPO3", "Linksys", "Netgear", "TP-LINK", "Belkin", "Juniper", "Microsoft OneDrive", "IBM WebSphere", "Oracle Fusion Middleware", "Foxit Reader", "Oracle E-Business Suite", "Oracle PeopleSoft", "Oracle Hyperion", "Oracle Database Server", "Oracle Food and Beverage Applications", "Oracle Siebel CRM", "Oracle Financial Services Applications", "Oracle Construction and Engineering", "Oracle Commerce", "Oracle Enterprise Manager", "F5 BIG-IP", "Adobe Acrobat and Reader" ] def merge_list(list1, list2, intersection=None): """Write the union of list and list2 into list. If intersection is not None, that list will be filled with the intersection of list and list2.""" for item in list2: if item not in list1: list1.append(item) else: if intersection is not None: intersection.append(item) def subtract_list(list1, list2): """Remove all elements from list which occur in list2.""" for item in list2: if item in list1: list1.remove(item) def wordwrap(text, width): """ A word-wrap function that preserves existing line breaks and most spaces in the text. Expects that existing line breaks are posix newlines (\n). """ return reduce(lambda line, word, width=width: '%s%s%s' % (line, ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n', 1)[0] ) >= width)], word), text.split(' ') ) def _wrap_desc(desc): return wordwrap(desc, 75).replace(' \n', '\n') def _spawn_editor(path): editor = os.getenv('EDITOR', 'vi') subprocess.call([editor, path]) def debug(msg): global opt if opt.debug: print(msg, file=sys.stderr) def prompt_user(msg): '''function for prompting for user input; designed to cope with flushing output''' if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 3): print(msg, end='') sys.stdout.flush() else: # print function supports flush= argument print(msg, flush=True, end='') def add_CVE_to_tracker(cve, info, packages, priority=None, bug_urls=[], ref_urls=[]): src = '%s/active/00boilerplate' % (destdir) # Use the first boilerplate for template first_boiler = "" for b in packages: if os.path.exists(src + "." + b): src = src + "." + b first_boiler = src break dst = '%s/active/%s' % (destdir, cve) with open(src) as f: template = f.readlines() cve_file = open(dst, 'w') orig_priority = "" for line in template: line = line.rstrip() if line.startswith('Candidate:'): print('Candidate: %s' % (cve), file=cve_file) elif info['public'] and line.startswith('PublicDate:'): print('PublicDate: %s' % (info['public']), file=cve_file) elif info['cvss'] and line.startswith('CVSS:'): print('CVSS:', file=cve_file) for entry in info['cvss']: print(' %s: %s [%s %s]' % (entry['source'], entry['vector'], entry['baseScore'], entry['baseSeverity']), file=cve_file) elif line.startswith('References:'): print('References:\n https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s' % (cve), file=cve_file) for i in ref_urls: print(" %s" % i, file=cve_file) elif line.startswith('Bugs:'): print(line, file=cve_file) for i in bug_urls: print(" %s" % i, file=cve_file) elif line.startswith('Priority:'): orig_priority = line.split()[1] if priority: print('Priority: %s' % priority, file=cve_file) else: print(line, file=cve_file) elif not line.startswith('#'): print(line, file=cve_file) if line.startswith('Description:'): for desc_line in _wrap_desc(info['desc']).split('\n'): print(" %s" % (desc_line), file=cve_file) # Now add package information (with Priority_) from other boilers if len(packages) > 1: for p in packages: skip_emptyline = False boiler = '%s/active/00boilerplate.%s' % (destdir, p) if os.path.exists(boiler) and boiler != first_boiler: with open(boiler) as f: template = f.readlines() in_note_section = False for line in template: line = line.rstrip() # handle notes in templates if in_note_section: if line.startswith(' '): continue else: in_note_section = False if line.startswith('Notes:'): in_note_section = True continue if line.startswith('Candidate:') or \ line.startswith('PublicDate:') or \ line.startswith('References:') or \ line.startswith('Description:') or \ line.startswith('Ubuntu-Description:') or \ line.startswith('Bugs:') or \ line.startswith('Discovered-by:') or \ line.startswith('Assigned-to:') or \ line.startswith('#'): continue if line.startswith('Priority:'): new_priority = line.split()[1] if priority: continue elif new_priority == orig_priority: continue elif orig_priority == "": print(line, file=cve_file) else: print('Priority_%s: %s' % (p, new_priority), file=cve_file) skip_emptyline = True elif skip_emptyline and line == "": skip_emptyline = False continue else: print(line, file=cve_file) print("") cve_file.close() return dst class PercentageFile(object): def __init__(self, filename): self.name = filename self.size = os.stat(filename)[6] self.delivered = 0 self.f = open(filename, encoding='utf-8', errors='replace') widgets = [progressbar.Percentage(), ' ', progressbar.Bar(marker='=', left='[', right=']'), ' ', str(self.size), ' ', progressbar.ETA()] self.bar = progressbar.ProgressBar(widgets=widgets, maxval=self.size).start() def read(self, size=None): if size is None: data = self.f.read() else: data = self.f.read(size) self.delivered += len(data) if self.size != 0: self.bar.update(self.delivered) if self.size == self.delivered: self.bar.finish() return data def close(self): return self.f.close() def convert_to_nvd(cves=[], desc=""): # convert to nvd format dict (like nvd json) nvd = {"CVE_data_type": "CVE", "CVE_data_format": "MITRE", "CVE_data_version": "4.0", "CVE_data_timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%MZ"), "CVE_Items": []} keys = list(cves.keys()) keys.sort() for cve in keys: refs = [] if cve in cves and 'refs' in cves[cve] and len(cves[cve]['refs']) > 0: for r in cves[cve]['refs']: refs.append({"name": r, "url": r, "refsource": "MISC"}) item = {"cve": {"data_type": "CVE", "data_format": "MITRE", "data_version": "4.0", "CVE_data_meta": {"ID": cve}, "references": {"reference_data": refs}, "description": {"description_data": [ {"lang": "en", "value": desc(cve) if desc else ""} ]}}, "publishedDate": cves[cve]['date'].strftime("%Y-%m-%dT%H:%MZ")} if cve in cves and 'cvss' in cves[cve]: try: cvssV3 = cve_lib.parse_cvss(cves[cve]['cvss']) item["impact"] = cvssV3 except ValueError: pass nvd["CVE_Items"].append(item) return nvd def import_debian(handler): '''Import Debian CVEs and DSAs missing from the tracker''' cves = dict() today = datetime.date.today() known = set(CVEKnownList + CVEIgnoreList) def mistriaged(cve): if cve in CVEIgnoreNotForUsList and \ cve not in CVEIgnoreMistriagedList and \ handler.debian[cve]['state'] == 'FOUND': # check that at least one of the assigned packages exist # in Ubuntu pkgs = {} for pkg in handler.debian[cve]['pkgs'].keys(): # ignore package if debian says is not affected if handler.debian[cve]['pkgs'][pkg]['state'] == '': continue answer = source_map.madison(source, pkg) if len(answer) > 0: pkgs[pkg] = answer if len(pkgs) > 0: return True return False # pull in CVEs from data/DSA/list dsas = cve_lib.load_debian_dsas(cve_lib.config['secure_testing_path'] + '/data/DSA/list', opt.verbose) for dsa in dsas: for cve in dsas[dsa]['cves']: if not cve_lib.CVE_RE.match(cve): if opt.verbose: print("Skipping %s, not well-formed?" % cve, file=sys.stderr) continue year = int(re.split('-', cve)[1]) if year < cve_limit: continue # If we already know about the CVE, skip it unless is # mistriaged if cve in known: if mistriaged(cve): # add a note about how this was originally classified dsas[dsa]['desc'] = mistriaged_hint + dsas[dsa]['desc'] else: continue cves[cve] = dict() cves[cve]['subject'] = escape(dsas[dsa]['desc']) cves[cve]['date'] = dsas[dsa]['date'] if opt.verbose: print("Processing %s: %s (%s)" % (dsa, dsas[dsa]['desc'], cves[cve]['date']), file=sys.stderr) # Now pull in CVEs from the data/CVE/list for cve in handler.debian: if opt.verbose: print("[--- Processing %s ---]" % cve, file=sys.stderr) if cve in cves: if opt.verbose: print("Skipping %s, already found in DSA" % cve, file=sys.stderr) continue if not cve_lib.CVE_RE.match(cve): if opt.verbose: print("Skipping %s, not well-formed?" % cve, file=sys.stderr) continue year = int(re.split('-', cve)[1]) if year < cve_limit: if opt.verbose: print("Skipping %s, year %d predates %d" % (cve, year, cve_limit), file=sys.stderr) continue # If we already know about the CVE, skip it unless is mistriaged if cve in known: if mistriaged(cve): # add a note about how this was originally classified handler.debian[cve]['desc'] = mistriaged_hint + handler.debian[cve]['desc'] else: if opt.verbose: print("Skipping %s, already known" % cve, file=sys.stderr) continue if handler.debian[cve]['desc'] or handler.debian[cve]['state'] == 'FOUND': cves[cve] = dict() cves[cve]['subject'] = escape(handler.debian[cve]['desc']) if cves[cve]['subject'] == '': cves[cve]['subject'] = '[Unknown description]' # just make something up. It'll get adjusted whenever mitre adds it date = "%s-12-31" % year if year >= today.year: date = "%s-%s-%s" % (today.year, today.month, today.day) cves[cve]['date'] = datetime.datetime.strptime(date, "%Y-%m-%d") if opt.verbose: print("Processing %s: %s (%s)" % (cve, handler.debian[cve]['desc'], cves[cve]['date']), file=sys.stderr) nvd = convert_to_nvd(cves, lambda cve: cves[cve]['subject']) tmp = tempfile.NamedTemporaryFile(mode='w', prefix='debian-import_', suffix='.json', delete=False) tmpname = tmp.name tmp.file.write(json.dumps(nvd)) tmp.close() return tmpname class RHEL8OVALHandler(xml.sax.handler.ContentHandler): """SAX handler for processing rhel8 OVAL XML.""" def __init__(self, ignore=[]): # For per-hit processing self._curr_vuln = None self._curr_cve = None self._curr_pkgs = [] self._curr_url = [] self._curr_source = None self._curr_chars_collect = False self._curr_chars = "" self._timestamp = None self._cves = dict() def startElement(self, name, attrs): if name == 'oval:timestamp': if opt.verbose: print("Parsing RHEL8 OVAL schema", file=sys.stderr) self._curr_chars_collect = True self._curr_chars = "" if name == "definition" and attrs['class'] == 'vulnerability': self._curr_vuln = attrs['id'] self._curr_desc = None self._curr_cve = None self._curr_url = None if name == "title": self._curr_chars_collect = True self._curr_chars = "" if name == "reference": self._curr_cve = attrs['ref_id'] if 'ref_url' in attrs: self._curr_url = attrs['ref_url'] def characters(self, content): if self._curr_chars_collect: self._curr_chars += content def endElement(self, name): self._curr_chars_collect = False if name == 'oval:timestamp': self._timestamp = datetime.datetime.strptime(self._curr_chars, "%Y-%m-%dT%H:%M:%S") if name == "title": title = self._curr_chars # rhel oval titles are of form "CVE-XXXX-XXXX Name: Description # here (priority)" - we want to keep "Name: Description here" self._curr_desc = ' '.join(title.split(' ')[1:-1]) if name == "definition": self._cves.setdefault(self._curr_cve, dict()) self._cves[self._curr_cve].setdefault('desc', self._curr_desc) self._cves[self._curr_cve].setdefault('refs', [self._curr_url]) self._cves[self._curr_cve].setdefault('date', self._timestamp) def cves(self): return self._cves def read_rhel8oval_file(f): '''Read in rhel8 oval This is sneaky because we read in the oval and then output a fake JSON file for processing. ''' if not os.path.isfile(f): print("'%s' not a file" % f, file=sys.stderr) sys.exit(1) name = os.path.abspath(f + ".json") if os.path.exists(name): print("'%s' already exists" % name, file=sys.stderr) sys.exit(1) parser = xml.sax.make_parser() handler = RHEL8OVALHandler() parser.setContentHandler(handler) parser.parse(f) cves = handler.cves() nvd = convert_to_nvd(cves, lambda cve: cves[cve]['desc']) tmp = tempfile.NamedTemporaryFile(mode='w', prefix='rhel8oval-import_', suffix='.json', delete=False) tmpname = tmp.name tmp.file.write(json.dumps(nvd)) tmp.close() return tmpname def read_locate_cves_output(f): '''Read in output of UCT/scripts/locate_cves.py This is sneaky because we read in the output and then output a fake JSON file for processing. ''' if not os.path.isfile(f): print("'%s' not a file" % f, file=sys.stderr) sys.exit(1) name = os.path.abspath(f + ".json") if os.path.exists(name): print("'%s' already exists" % name, file=sys.stderr) sys.exit(1) with open(f) as _f: lines = _f.readlines() cves = dict() cve = None subject = "" for line in lines: if line == "\n": # record delimiter if cve is not None: cves[cve]['subject'] = subject cve = None subject = "" continue if line.startswith("Couldn't find CVE"): cve = line.split()[2] if not cve_lib.CVE_RE.match(cve): print("Skipping malformed CVE: '%s' from '%s'" % (cve, f), file=sys.stderr) cve = None elif cve in cves: if opt.verbose: print("Skipping duplicate '%s' from '%s'" % (cve, f), file=sys.stderr) cve = None else: if opt.verbose: print("Adding '%s'" % cve, file=sys.stderr) cves[cve] = dict() continue if cve is None: continue if line.startswith("Message date:"): try: date = " ".join(line.split(": ")[1].strip().split()[0:5]) cves[cve]['date'] = datetime.datetime.strptime(date, "%a, %d %b %Y %H:%M:%S") except Exception: print("Could not process date '%s', skipping %s from '%s'" % (line, cve, f), file=sys.stderr) del cves[cve] cve = None continue if line.startswith("Message subject:") or subject != "": s = re.sub(r'Message subject: ', "", line) if subject == "": subject = s.strip() else: subject += " " + s.strip() # Try to fake up some urls rhsa_regex = '\[RHSA-\d\d\d\d:\d+-\d+\]' osssec_regex = '\[oss-security\]' if re.search(r'' + rhsa_regex, subject): rhsa = re.sub(r'.*(%s).*' % rhsa_regex, r'\1', subject).strip('[|]') url = "https://rhn.redhat.com/errata/%s-%s.html" % (rhsa.split(':')[0], rhsa.split(':')[1].split('-')[0]) cves[cve].setdefault('refs', [] + [url]) elif re.search(r'' + osssec_regex, subject) and 'date' in cves[cve]: # NOTE: while we can determine the url for the year/month/day, we # cannot determine the specific message on that day. This gets us # close though, so use it. url = "http://www.openwall.com/lists/oss-security/%s" % (cves[cve]['date'].strftime("%Y/%m/%d")) cves[cve].setdefault('refs', [] + [url]) nvd = convert_to_nvd(cves, lambda c: '''ML-Date: %s, ML-Subject: %s''' % (cves[c]['date'], escape(cves[c]['subject']))) tmp = tempfile.NamedTemporaryFile(mode='w', prefix='locate-cves-import_', suffix='.json', delete=False) tmp.file.write(json.dumps(nvd)) tmp.close() shutil.move(tmp.name, name) return name def read_mbox_file(f): '''Run an mbox file through UCT/scripts/locate_cves.py And process through read_locate_cves_output() ''' if not os.path.isfile(f) and not os.path.isdir(f): print("'%s' not a file" % f, file=sys.stderr) sys.exit(1) child = subprocess.Popen(['./scripts/locate_cves.py', f], stdout=subprocess.PIPE, universal_newlines=True) out, err = child.communicate() tmp = tempfile.NamedTemporaryFile(mode='w', prefix='mbox-import_', suffix='.out', delete=False) tmpname = tmp.name tmp.file.write(out) tmp.close() json_file = read_locate_cves_output(tmpname) os.unlink(tmpname) return json_file def dpkg_compare_versions(v1, op, v2): import subprocess try: sp = subprocess.Popen(['dpkg', '--compare-versions', v1, op, v2], close_fds=True) except OSError as e: return [127, str(e)] sp.communicate() if sp.returncode == 0: return True return False def get_built_using(pkgs=[]): global built_using_map if built_using_map is None: built_using_map = source_map.load_built_using_collection( source_map.load(data_type='packages')) out = "" for pkg in pkgs: out += source_map.get_built_using(built_using_map, pkg) return out def cvss_source_from_filename(name): sources = ['nvd', 'rhel'] for source in sources: if source in name: return source return 'unknown' class CVEHandler(xml.sax.handler.ContentHandler): """SAX handler for processing mitre's CVE database XML.""" def __init__(self, ignore=[]): # For per-hit processing self.curr_cve = None self.curr_desc = None self.curr_cvss = [] self.curr_desc_ready = False self.curr_refs = [] self.curr_chars_collect = False self.curr_chars = "" self.num_ignored = 0 self.num_added = 0 self.num_skipped = 0 self.curr_public = None # For long-term (human) processing self.cve_ignore = set() for cve in ignore: self.cve_ignore.add(cve) self.cve_seen = set() self.cve_list = [] self.cve_data = dict() self.saved_ignore_reason = "" self.saved_package = "" self.saved_cve = "" self.debian = None # File-type detection # Load debian CVE states, if configured if 'secure_testing_path' in cve_lib.config: self.debian = cve_lib.load_debian_cves(cve_lib.config['secure_testing_path'] + '/data/CVE/list') def updateTimestamp(self): # Get UTC time timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) # Append to timestamp file list with open('%s/check-cves.log' % (destdir), 'a') as f: f.write('%s UTC - %s added, %s ignored, %s skipped, %s total - files: %s\n' % (timestamp, self.num_added, self.num_ignored, self.num_skipped, self.num_added + self.num_ignored, [os.path.basename(x) for x in args])) def printReport(self): print('\n============================ Triage summary =============================') print("\n %4d CVEs added" % self.num_added) print(" %4d CVEs ignored" % self.num_ignored) print(" %4d CVEs skipped" % self.num_skipped) print("---------------------------") print("%5d total CVEs triaged" % (self.num_added + self.num_ignored)) updates_detected, updates_details = self.detect_updates_to_external_repositories() if updates_detected: print('\n====================== External updates detected ========================') print(updates_details) print("\n Please remember to push the above changes if appropriate") def detect_updates_to_external_repositories(self): external_repositories = [cve_lib.subprojects_dir, cve_lib.embargoed_dir] updates_detected = False updates_details = "" for repository_dir in external_repositories: cmd = ['git', '-C', repository_dir, 'status', '--porcelain'] try: process = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True, ) if process.returncode == 0 and process.stdout: if process.stdout: updates_detected = True updates_details += "\n %s \n %s" % (repository_dir, process.stdout) except subprocess.CalledProcessError as exception: print(exception) updates_detected = False return updates_detected, updates_details def parse_json(self, fp): template_nvd = {"CVE_data_type": "CVE", "CVE_data_format": "MITRE", "CVE_data_version": "4.0"} nvd = json.load(fp) # check for expected fields for key in list(template_nvd.keys()): if key not in nvd or nvd[key] != template_nvd[key]: raise KeyError("NVD JSON in '%s' seems invalid" % fp.name) for item in nvd["CVE_Items"]: template_cve = {"data_type": "CVE", "data_format": "MITRE", "data_version": "4.0"} if "publishedDate" in item: # convert from YYYY-MM-DDTHH:MMZ to YYYY-MM-DD HH:MM:SS UTC self.curr_public = item["publishedDate"].replace("T", " ").replace("Z", ":00 UTC") else: self.curr_public = None cve = item["cve"] for key in list(template_cve.keys()): if key not in cve or cve[key] != template_cve[key]: raise KeyError("NVD JSON in '%s' seems invalid" % fp.name) metadata = cve["CVE_data_meta"] self.curr_cve = metadata["ID"] self.curr_refs = [] self.curr_desc = None self.curr_cvss = [] # Some MITRE CVE data that were REJECT has no more references field # so need to handle it. try: for ref in cve["references"]["reference_data"]: self.curr_refs += [(ref["refsource"], ref["name"], ref["url"])] except: pass # find an english description for desc in cve["description"]["description_data"]: if desc["lang"] == "en": if self.curr_desc is None: self.curr_desc = desc["value"] else: self.curr_desc += " " + desc["value"] # add cvss if available try: cvss = dict() # map filename to source cvss['source'] = cvss_source_from_filename(fp.name) cvss['vector'] = item["impact"]["baseMetricV3"]["cvssV3"]["vectorString"] cvss['baseScore'] = item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"] cvss['baseSeverity'] = item["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"] self.curr_cvss += [cvss] except KeyError: pass self.handle_cve() def startElement(self, name, attrs): if name == "item": if opt.verbose: print("Parsing Mitre XML schema", file=sys.stderr) self.curr_cve = attrs['name'] self.curr_refs = [] self.curr_url = None self.curr_desc = None self.curr_cvss = [] if name == "desc": self.curr_chars_collect = True self.curr_chars = "" if name == "ref": self.curr_chars_collect = True self.curr_chars = "" self.curr_url = None self.curr_source = None if 'url' in attrs: self.curr_url = attrs['url'] if 'source' in attrs: self.curr_source = attrs['source'] def characters(self, content): if self.curr_chars_collect: self.curr_chars += content def endElement(self, name): self.curr_chars_collect = False # Mitre items if name == "desc": self.curr_desc = self.curr_chars if name == "item": self.handle_cve() if name == "ref": self.curr_refs += [(self.curr_source, self.curr_chars, self.curr_url)] def handle_cve(self): # Skip CVEs we know about already unless this is a mistriaged CVE if self.curr_cve in self.cve_ignore and mistriaged_hint not in self.curr_desc: return limit = cve_limit if not opt.refresh: limit = 2005 if int(self.curr_cve.split("-")[1]) < limit: return # Check for ** RESERVED ** s = '** RESERVED **' if self.curr_desc.find(s) >= 0 and self.curr_desc.find("Linux") < 0: return if self.curr_cve in self.cve_seen: print("Skipping repeat: %s" % (self.curr_cve), file=sys.stderr) return self.cve_seen.add(self.curr_cve) self.cve_list += [self.curr_cve] self.cve_data.setdefault(self.curr_cve, dict()) self.cve_data[self.curr_cve].setdefault('desc', self.curr_desc) self.cve_data[self.curr_cve].setdefault('public', self.curr_public) self.cve_data[self.curr_cve].setdefault('refs', [] + self.curr_refs) self.cve_data[self.curr_cve].setdefault('cvss', [] + self.curr_cvss) def cves(self): return self.cve_list def get_ignore_suggestion(self, text): '''Try to find a reasonable suggestion for the user.''' suggestion = "" # strip out the added mailing list stuff (locate_cves.py importing) rev_text = re.sub(r'^ML-Date: .* ML-Subject: ', '', text) rev_text = re.sub(r'^(|Re: )CVE (r|R)equest: ', '', rev_text) first_sentence = re.split(r'\. ', rev_text)[0] # hunt for module/component match = re.search(r'(?: in t|^T)he (.*) (library|template|component|module|plug-?in|extension|application|theme) (?:.* )?for (Joomla!|Drupal|WordPress|TYPO3|Mambo|Android|jQuery)', first_sentence) if match: module = match.group(1) for marker in [" module", " ("]: if marker in module: module = module.split(marker)[0] return "%s %s for %s" % (module, match.group(2), match.group(3)) # drop commas-extensions if ',' in first_sentence: first_sentence = re.split(r',', first_sentence)[0] phrases = re.split(r' [io]n ', first_sentence) # default to the last phrase suggestion = phrases[-1] # move to earlier phrase if suggestion starts with "a" if suggestion.startswith('a ') and len(phrases) > 1: suggestion = phrases[-2] version_preps = '(\s+(before|through|prior to|versions?))+\s*' version_regex = '\s+([a-zA-Z\._\-]*[0-9]+[a-zA-Z\._\-]*)+' # prefer 'Apple iOS before ' or 'Apple Mac OS X through # max_length: suggestion = suggestion[:max_length] suggestion = re.sub(r'\s+\w+$', '', suggestion) return suggestion def display_command_file_usage(self, f, line_prefix=''): f.write('%sThe following commands can be used in this file:\n' % (line_prefix)) f.write('%s\n' % (line_prefix)) f.write('%s* Add a new CVE to the tracker:\n' % (line_prefix)) f.write('%s add [PACKAGE] ...\n' % (line_prefix)) f.write('%s* Add a new CVE to the tracker and open it in your editor:\n' % (line_prefix)) f.write('%s edit [PACKAGE] ...\n' % (line_prefix)) f.write('%s* Unembargo a CVE that is now public:\n' % (line_prefix)) f.write('%s unembargo>\n' % (line_prefix)) f.write('%s* Permanently ignore a CVE:\n' % (line_prefix)) f.write('%s ignore \n' % (line_prefix)) f.write('%s* Temporarily skip over a CVE:\n' % (line_prefix)) f.write('%s skip\n\n' % (line_prefix)) f.flush() def find_hint_in_external_subprojects(self, software_hints_from_cve_description): external_subprojects = {} for subproject in cve_lib.external_releases: if subproject in source: for hint in software_hints_from_cve_description: if hint in source[subproject] and hint not in common_words: external_subprojects.setdefault(subproject, set()).add(hint) return external_subprojects def display_cve(self, cve, file=sys.stdout, line_prefix=None, wrap_desc=False): class CVEOutput: def __init__(self, file, line_prefix=None): self.file = file self.line_prefix = line_prefix def write(self, buf): if not line_prefix: self.file.write(buf) else: for line in buf.splitlines(True): # splitlines is returning lines without trailing # newlines despite the keepends parameter being set to # True. Work around this issue by appending a newline # when needed. line_suffix = '' if not line.endswith('\n'): line_suffix = '\n' self.file.write(line_prefix + line + line_suffix) orig_stdout = sys.stdout if file != sys.stdout: sys.stdout = CVEOutput(file, line_prefix) # Check if this was once an embargoed issue if cve in EmbargoList: print('**!!** no longer embargoed **!!**') print('==========================details from embargo entry==========================') with open(os.path.join('embargoed', cve)) as f: print(f.read().rstrip()) print('=================================end details==================================') # Display CVE information if self.cve_data[cve]['public']: print(' Published: %s' % (self.cve_data[cve]['public'])) for ref in self.cve_data[cve]['refs']: print(' %s: %s' % (ref[0], ref[1]), end='') # Do not repeat URL if it matches the contents of the reference if ref[2] and ref[1].strip() != ref[2].strip(): print(' %s' % (ref[2]), end='') print() if wrap_desc: print('%s' % _wrap_desc(self.cve_data[cve]['desc'])) else: print('\n======================== CVE details ==========================') print(' %s' % cve) print(' %s' % (self.cve_data[cve]['desc'])) for cvss in self.cve_data[cve]['cvss']: print(' CVSS (%s): %s [%.1f]' % (cvss['source'], cvss['vector'], cvss['baseScore'])) if self.debian and cve in self.debian: print('\n======================= Debian details ========================') print(' Debian CVE Tracker: %s' % (self.debian[cve]['state'])) if len(self.debian[cve]['note']): print("\t" + "\n\t".join(self.debian[cve]['note'])) for pkg in self.debian[cve]['pkgs']: info = [] if self.debian[cve]['pkgs'][pkg]['priority']: info.append(self.debian[cve]['pkgs'][pkg]['priority']) if self.debian[cve]['pkgs'][pkg]['bug']: info.append(self.debian[cve]['pkgs'][pkg]['bug']) if self.debian[cve]['pkgs'][pkg]['note']: info.append(self.debian[cve]['pkgs'][pkg]['note']) print(" Debian: %s: %s (%s)" % (pkg, self.debian[cve]['pkgs'][pkg]['state'], "; ".join(info))) # Display version and component details for Ubuntu's pkg answer = source_map.madison(source, pkg) for name in sorted(answer.keys()): for pkg in sorted(answer[name].keys()): print(' Ubuntu: %s | %s | %s' % (pkg, answer[name][pkg], name)) # no debian info, display possible commented ignore command when # using command file (i.e. wrap_desc is true) if (self.debian[cve]['state'] == 'RESERVED' or self.debian[cve]['state'] == None) and wrap_desc: proposed_ignore = self.get_ignore_suggestion(self.cve_data[cve]['desc']) print('%s ignore "%s"' % (cve, proposed_ignore)) else: proposed_ignore = self.get_ignore_suggestion(self.cve_data[cve]['desc']) print('%s ignore %s' % (cve, proposed_ignore)) software_hints_from_cve_desc = self.get_software_hints_from_cve_description(self.cve_data[cve]["desc"]) if software_hints_from_cve_desc: software_hints_per_external_releases = self.find_hint_in_external_subprojects(software_hints_from_cve_desc) if software_hints_per_external_releases: print('\n==================== Subprojects details ======================') print(' External subprojects possibly affected: ') for affected_subproject in software_hints_per_external_releases: print(" - " + affected_subproject + ": " + " - ".join( software_hints_per_external_releases[affected_subproject])) # once again, announce formerly embargoed status if cve in EmbargoList: print('**!!** no longer embargoed **!!**') print('**!!** ensure this is correct before unembargoing **!!**') if file != orig_stdout: sys.stdout = orig_stdout file.flush() def get_cve_suggestions(self, cve): action = 'skip' reason = "" packages = [] # Skip CVEs that are obviously not about Ubuntu for s in ignore_strings: if re.search('(^| )%s' % re.escape(s), self.cve_data[cve]['desc'], flags=re.MULTILINE) and self.cve_data[cve]['desc'].find("Linux") < 0: action = 'ignore' reason = s # if cve is in embargo list (but now public), default to unembargo action if cve in EmbargoList: action = 'unembargo' reason = "" else: words = self.get_software_hints_from_cve_description(self.cve_data[cve]['desc']) # Default to Debian state, if available if self.debian and cve in self.debian and self.debian[cve]['state']: if self.debian[cve]['state'].startswith('NOT-FOR-US:'): action = 'ignore' reason = self.debian[cve]['state'].split(':', 1)[1].lstrip() if self.debian[cve]['state'] == 'FOUND': action = 'add' packages = list(self.debian[cve]['pkgs'].keys()) if len(self.debian[cve]['pkgs']) == 1: pkg = packages[0] if self.debian[cve]['pkgs'][pkg]['state'] == '': # software has not been admitted into debian action = 'ignore' reason = '%s: (dbug %s)' % (pkg, self.debian[cve]['pkgs'][pkg]['bug']) else: # try and hint if the detected product name contains any known # package names hints = words & allsrcs if len(hints) > 0: packages = list(hints) action = 'add' # Regardless of Ubuntu software, try and hint if the detected product # name contains any external software name external_software_hints = self.find_hint_in_external_subprojects(words) if external_software_hints: for software in external_software_hints.values(): packages.extend(list(software)) # remove any duplicates uniq = [] for p in packages: if p not in uniq: uniq.append(p) packages = uniq # ensure is always a string of package names packages = " ".join(packages) return (action, reason, packages) def get_software_hints_from_cve_description(self, cve_description): desc = self.get_ignore_suggestion(cve_description) words = set(desc.lower().split(" ")) return words def human_process_cve(self, cve, action='skip', reason='', package=''): info = '' while info == "" or not info[0] in ['i', 'a', 's', 'q', 'r']: prompt_user('\nA]dd (or R]epeat), I]gnore forever, S]kip for now, or Q]uit? [%s] ' % (action)) info = sys.stdin.readline().strip().lower() if info == "": info = action if info.startswith('q'): self.printReport() sys.exit(0) elif info.startswith('a') or info.startswith('r'): do_repeat = False if info.startswith('r'): info = self.saved_package do_repeat = True else: info = "" while info == "": prompt_user('Package(s) affected? ') if package == "": package = self.saved_package if package != "": prompt_user('[%s] ' % (package)) info = sys.stdin.readline().strip() if info == '': info = package self.saved_package = info dst = self.add_cve(cve, info.split(), None) if do_repeat: subprocess.call(['./scripts/active_dup', self.saved_cve, cve]) _spawn_editor(dst) self.saved_cve = cve print('\n===================== Dependant packages ======================') print(' Detecting packages built using: %s...' % info, end='') sys.stdout.flush() built_using = "" try: built_using = get_built_using(info) except Exception as e: print("ERROR: %s" % e, file=sys.stderr) pass # for now just show the error but don't break triage if built_using != "": print("found:\n") print("%s" % source_map.get_built_using_header()) print("%s" % built_using) print("IMPORTANT: the above packages are candidates for rebuilds when fixes are applied to:") print(" %s" % "\n ".join(info)) else: print("none detected") elif info.startswith('i'): info = "" while info == "": print('Reason to be ignored?') prompts = [] # Show debian reason first, then automatically determined # reason, then saved reason. This makes more sense # than sorting the reasons and is more predictable for choice in [reason, self.get_ignore_suggestion(self.cve_data[cve]['desc']), self.saved_ignore_reason]: if choice != "" and choice not in prompts: prompts.append(choice) for i in range(0, len(prompts)): print(' %s) %s' % (chr(97 + i), prompts[i])) prompt_user(' > ') info = sys.stdin.readline().strip() if len(info) == 1 and info.isalpha(): try: # ord('a') == 97 info = prompts[ord(info) - 97] except IndexError: print('\nError: invalid reason.\n') info = "" # Enter defaults to only suggestion if only one exists elif len(info) == 0 and len(prompts) == 1: info = prompts[0] elif len(info) < 3: # Fat fingers protection print('\nError: Reason must be at least 3 characters long!\n') info = "" self.saved_ignore_reason = info self.ignore_cve(cve, info) elif info.startswith('s'): self.skip_cve() print('') def process_command_file(self, f, preprocess=False): line_num = 0 cves = [] for line in f: line_num += 1 orig_line = line line = orig_line.strip() if not line or line.startswith('#'): continue # Remove any duplicated whitespace such as "CVE-2017-NNN1 skip" line = " ".join(line.split()).strip() args = line.split(None, 2) try: (cve, action) = (args[0].upper(), args[1].lower()) except IndexError: raise ValueError('Invalid formatting on line %d:\n%s' % (line_num, orig_line)) if not cve.startswith('CVE-'): # The first arg should look like a CVE ID raise ValueError('Invalid CVE ID formatting (%s) on line %d:\n%s' % (cve, line_num, orig_line)) elif cve in cves: raise ValueError('Performing multiple operations on the same CVE (%s) is not supported (line %d):\n%s' % (cve, line_num, orig_line)) cves.append(cve) if action == 'add' or action == 'edit': # If the CVE should be added, a package name and priority are # required try: args = args[2].split() priority = args[0].lower() packages = args[1:] except IndexError: raise ValueError('Invalid add command on line %d:\n%s' % (line_num, orig_line)) if not priority in cve_lib.priorities and not priority == 'untriaged': raise ValueError('Invalid priority on line %d:\n%s' % (line_num, orig_line)) if os.path.exists('%s/active/%s' % (destdir, cve)): raise ValueError('Updating an existing CVE is not supported (line %d):\n%s' % (line_num, orig_line)) if preprocess: continue if priority == 'untriaged': priority = None cve_path = self.add_cve(cve, packages, priority) if action == 'edit': _spawn_editor(cve_path) elif action == 'unembargo': if cve not in EmbargoList: raise ValueError('CVE %s is not in the embargo database (line %d):\n%s' % (cve, line_num, orig_line)) if os.path.exists(os.path.join('active', cve)): raise ValueError('Cannot unembargo %s because it already exists in the active directory (line %d):\n%s' % (cve, line_num, orig_line)) if preprocess: continue self.unembargo_cve(cve) elif action == 'ignore': # If the CVE should be ignored, a reason is required try: reason = args[2] except IndexError: raise ValueError('Invalid ignore command on line %d:\n%s' % (line_num, orig_line)) if preprocess: continue self.ignore_cve(cve, reason) elif action == 'skip': # If the CVE should be skipped, no arguments are allowed if len(args) > 2: raise ValueError('Invalid skip command on line %d:\n%s' % (line_num, orig_line)) if preprocess: continue self.skip_cve() else: # The second arg must be a known action raise ValueError('Unknown action (%s) on line %d:\n%s' % (action, line_num, orig_line)) def preprocess_cve(self, cve): desc = '' # Check if this was once an embargoed issue if cve in EmbargoList: desc += '# **!!** no longer embargoed **!!**\n' desc += '# ==========================details from embargo entry==========================\n' with open(os.path.join('embargoed', cve)) as f: desc += '# ' + f.read().rstrip() + '\n' desc += '# =================================end details==================================\n' # Display CVE information if self.cve_data[cve]['public']: desc += '# Published: %s\n' % (self.cve_data[cve]['public']) for ref in self.cve_data[cve]['refs']: desc += '# %s: %s' % (ref[0], ref[1]) # Do not repeat URL if it matches the contents of the reference if ref[2] and ref[1].strip() != ref[2].strip(): desc += ' %s' % (ref[2]) desc += '\n' for line in _wrap_desc(self.cve_data[cve]['desc']).split('\n'): desc += '# %s\n' % line if self.debian and cve in self.debian: desc += '# Debian CVE Tracker: %s\n' % (self.debian[cve]['state']) if len(self.debian[cve]['note']): for note in self.debian[cve]['note']: desc += '# \t%s\n' % note for pkg in self.debian[cve]['pkgs']: info = [] if self.debian[cve]['pkgs'][pkg]['priority']: info.append('# ' + self.debian[cve]['pkgs'][pkg]['priority']) if self.debian[cve]['pkgs'][pkg]['bug']: info.append('# ' + self.debian[cve]['pkgs'][pkg]['bug']) if self.debian[cve]['pkgs'][pkg]['note']: info.append('# ' + self.debian[cve]['pkgs'][pkg]['note']) desc += "# Debian: %s: %s (%s)\n" % (pkg, self.debian[cve]['pkgs'][pkg]['state'], "; ".join(info)) # Display version and component details for Ubuntu's pkg answer = source_map.madison(source, pkg) for name in sorted(answer.keys()): for pkg in sorted(answer[name].keys()): desc += '# Ubuntu: %s | %s | %s\n' % (pkg, answer[name][pkg], name) action = 'skip' data = "" # Skip CVEs that are obviously not about Ubuntu for s in ignore_strings: if self.cve_data[cve]['desc'].find(s) >= 0 and self.cve_data[cve]['desc'].find("Linux") < 0: action = 'ignore' data = s # Default to Debian state, if available if self.debian and cve in self.debian and self.debian[cve]['state']: if self.debian[cve]['state'].startswith('NOT-FOR-US:'): action = 'ignore' data = self.debian[cve]['state'].split(':', 1)[1].lstrip() if self.debian[cve]['state'] == 'FOUND': action = 'add' data = " ".join(self.debian[cve]['pkgs']) print('%s %s %s\n%s' % (action, cve, data, desc)) def add_cve(self, cve, packages, priority=None): # remove from not-for-us.txt if adding and ensure we remove any # mistriaged_hint from the description if cve in CVEIgnoreNotForUsList: cmd = ['sed', '-i', '/^%s #.*$/d' % cve, './ignored/not-for-us.txt'] subprocess.call(cmd) self.cve_data[cve]['desc'] = self.cve_data[cve]['desc'].replace(mistriaged_hint, '') # Build up list of reference urls ref_urls = [] if self.debian and \ cve in self.debian and \ 'note' in self.debian[cve]: for line in self.debian[cve]['note']: tmp = line.lstrip("NOTE: ") if tmp.startswith("http"): ref_urls.append(tmp) if 'refs' in self.cve_data[cve]: for ref in self.cve_data[cve]['refs']: url = "" if ref[1].strip().startswith("http"): url = ref[1].strip() elif ref[2] is not None and ref[2].strip().startswith("http"): url = ref[2].strip() else: # no urls continue if '//' not in url: # invalid url continue # ignore certain reference URLs which we don't use ignored_urls = ['www.securityfocus.com', 'www.osvdb.org'] if url.split('//')[1].split('/')[0] in ignored_urls: continue if url not in ref_urls: ref_urls.append(url) # Build up list of bug urls bug_urls = [] for pkg in packages: if self.debian and \ cve in self.debian and \ self.debian[cve]['pkgs'] and \ pkg in self.debian[cve]['pkgs']: bug = None if self.debian[cve]['pkgs'][pkg]['priority'] and \ re.search(r'^bug #[0-9]+$', self.debian[cve]['pkgs'][pkg]['priority']): bug = self.debian[cve]['pkgs'][pkg]['priority'].split('#')[1] elif self.debian[cve]['pkgs'][pkg]['bug']: bug = self.debian[cve]['pkgs'][pkg]['bug'] if bug: url = "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s" % bug if url not in bug_urls: bug_urls.append(url) # Add to tracker from 00boilerplate dst = add_CVE_to_tracker(cve, self.cve_data[cve], packages, priority, bug_urls, ref_urls) # Build up command line cmd = ['./scripts/active_edit', '-c', cve, '--yes'] # capture debian not-affected states not_affected = [] for pkg in packages: # The Debian convention is to specify the fixed version as the state # with the bug number as the priority for fixed bugs. Leverage this # with active_edit fixed_in = "" if self.debian and \ cve in self.debian and \ self.debian[cve]['pkgs'] and \ pkg in self.debian[cve]['pkgs'] and \ self.debian[cve]['pkgs'][pkg]['state']: if re.search(r'^[0-9]', self.debian[cve]['pkgs'][pkg]['state']): fixed_version = self.debian[cve]['pkgs'][pkg]['state'] fixed_in = ",%s" % fixed_version # Now see if we can correlate this to an Ubuntu version answer = source_map.madison(source, pkg) for name in sorted(answer.keys()): rel = name.split('/')[0].split('-')[0] # don't care about the pocket version = answer[name][pkg] # Try to compare apples to apples. Ie, if one of us has # an epoch and the other doesn't, don't try to be smart if (':' not in version and ':' not in fixed_version) or \ (':' in version and ':' in fixed_version): if dpkg_compare_versions(version, 'ge', fixed_version): if rel == cve_lib.devel_release: rel = 'devel' fixed_in += ",%s,%s" % (rel, version) break elif self.debian[cve]['pkgs'][pkg]['state'].startswith('') and \ len(self.debian[cve]['pkgs'][pkg]['priority']) > 0: # capture that debian believes their version is unaffected not_affected.append((pkg, "debian: %s" % self.debian[cve]['pkgs'][pkg]['priority'])) cmd += ['-p', "%s%s" % (pkg, fixed_in)] subprocess.call(cmd) for (pkg, reason) in not_affected: cmd = ['./scripts/mass-cve-edit', '-p', pkg, '-r', 'upstream', '-s', 'not-affected', '-v', reason, cve] subprocess.call(cmd) self.num_added += 1 return dst def unembargo_cve(self, cve): # unembargo a cve now public shutil.move(os.path.join('embargoed', cve), 'active/') self.num_added += 1 def ignore_cve(self, cve, reason): # Append to ignore list unless is already in CVEIgnoreList and then # append to the ignored/ignore-mistriaged.txt txtfile = 'ignore-mistriaged.txt' if cve in CVEIgnoreNotForUsList else 'not-for-us.txt' with open('%s/ignored/%s' % (destdir, txtfile), 'a') as f: f.write('%s # %s\n' % (cve, reason)) self.num_ignored += 1 def skip_cve(self): self.num_skipped += 1 class CheckCVETest(unittest.TestCase): def test_get_ignore_suggestion(self): '''"Ignore" suggestion text extraction''' # Re-use the global handler h = handler self.assertEqual("Courier-Authlib", h.get_ignore_suggestion('''SQL injection vulnerability in authpgsqllib.c in Courier-Authlib before 0.62.0, when a non-Latin locale Postgres database is used, allows remote attackers to execute arbitrary SQL commands via query parameters containing apostrophes.''')) self.assertEqual("Apple Mac OS X", h.get_ignore_suggestion('''Buffer overflow in the DirectoryService Proxy in DirectoryService in Apple Mac OS X through 10.6.8 allows remote attackers to execute arbitrary code or cause a denial of service (application crash) via unspecified vectors.''')) self.assertEqual("KDE", h.get_ignore_suggestion('''HTMLTokenizer::scriptHandler in Konqueror in KDE 3.5.9 and 3.5.10 allows remote attackers to cause a denial of service (application crash) via an invalid document.load call that triggers use of a deleted object. NOTE: some of these details are obtained from third party information.''')) self.assertEqual("Sun Solaris", h.get_ignore_suggestion('''The name service cache daemon (nscd) in Sun Solaris 10 and OpenSolaris snv_50 through snv_104 does not properly check permissions, which allows local users to gain privileges and obtain sensitive information via unspecified vectors.''')) self.assertEqual("Linux kernel", h.get_ignore_suggestion('''libata in the Linux kernel before 2.6.27.9 does not set minimum timeouts for SG_IO requests, which allows local users to cause a denial of service (Programmed I/O mode on drives) via multiple simultaneous invocations of an unspecified test program.''')) self.assertEqual("iGaming", h.get_ignore_suggestion('''Multiple SQL injection vulnerabilities in iGaming 1.5 and earlier allow remote attackers to execute arbitrary SQL commands via the browse parameter to (1) previews.php and (2) reviews.php, and the (3) id parameter to index.php in a viewarticle action.''')) self.assertEqual("PHP iCalendar", h.get_ignore_suggestion('''PHP iCalendar 2.24 and earlier allows remote attackers to bypass authentication by setting the phpicalendar and phpicalendar_login cookies to 1.' ''')) # Test length truncation, tweaked to avoid "has" matcher self.assertEqual("** TEST CVE ** This candidate HAS been reserved by an", h.get_ignore_suggestion('''** TEST CVE ** This candidate HAS been reserved by an organization or individual that will use it when announcing a new security problem. When the candidate has been publicized, the details for this candidate will be provided.''')) self.assertEqual("Sun OpenSolaris", h.get_ignore_suggestion('''Unspecified vulnerability in the root/boot archive tool in Sun OpenSolaris has unknown impact and local attack vectors, related to a "Temporary file vulnerability," aka Bug ID 6653455.''')) self.assertEqual("Red Hat Certificate System", h.get_ignore_suggestion('''Red Hat Certificate System 7.2 uses world-readable permissions for password.conf and unspecified other configuration files, which allows local users to discover passwords by reading these files.''')) self.assertEqual("Microsoft Internet Explorer", h.get_ignore_suggestion('''An unspecified function in the JavaScript implementation in Microsoft Internet Explorer creates and exposes a "temporary footprint" when there is a current login to a web site, which makes it easier for remote attackers to trick a user into acting upon a spoofed pop-up message, aka an "in-session phishing attack." NOTE: as of 20090116, the only disclosure is a vague pre-advisory with no actionable information. However, because it is from a well-known researcher, it is being assigned a CVE identifier for tracking purposes.''')) self.assertEqual("Umer Inc Songs Portal", h.get_ignore_suggestion('''SQL injection vulnerability in albums.php in Umer Inc Songs Portal allows remote attackers to execute arbitrary SQL commands via the id parameter.''')) self.assertEqual("Limbo CMS", h.get_ignore_suggestion('''SQL injection vulnerability in open.php in the Private Messaging (com_privmsg) component for Limbo CMS allows remote attackers to execute arbitrary SQL commands via the id parameter in a pms action to index.php.''')) self.assertEqual("phpscripts Ranking Script", h.get_ignore_suggestion('''phpscripts Ranking Script allows remote attackers to bypass authentication and gain administrative access by sending an admin=ja cookie.''')) self.assertEqual("A4Desk Event Calendar", h.get_ignore_suggestion('''PHP remote file inclusion vulnerability in index.php in A4Desk Event Calendar, when magic_quotes_gpc is disabled, allows remote attackers to execute arbitrary PHP code via a URL in the v parameter.''')) self.assertEqual("Galatolo WebManager", h.get_ignore_suggestion('''Cross-site scripting (XSS) vulnerability in result.php in Galatolo WebManager (GWM) 1.0 allows remote attackers to inject arbitrary web script or HTML via the key parameter.''')) self.assertEqual("Sun OpenSolaris", h.get_ignore_suggestion('''Unspecified vulnerability in the process (aka proc) filesystem in Sun OpenSolaris snv_85 through snv_100 allows local users to gain privileges via vectors related to the contract filesystem.''')) self.assertEqual("Dreampics Gallery Builder", h.get_ignore_suggestion('''SQL injection vulnerability in index.php in Dreampics Gallery Builder allows remote attackers to execute arbitrary SQL commands via the exhibition_id parameter in a gallery.viewPhotos action.''')) self.assertEqual("Omilen Photo Gallery component for Joomla!", h.get_ignore_suggestion('''Directory traversal vulnerability in the Omilen Photo Gallery (com_omphotogallery) component Beta 0.5 for Joomla! allows remote attackers to include and execute arbitrary local files via directory traversal sequences in the controller parameter to index.php.''')) self.assertEqual("Webform module for Drupal", h.get_ignore_suggestion('''Cross-site scripting (XSS) vulnerability in the Webform module 5.x before 5.x-2.7 and 6.x before 6.x-2.7, a module for Drupal, allows remote attackers to inject arbitrary web script or HTML via a submission.''')) self.assertEqual("Itamar Elharar MusicGallery component for Joomla!", h.get_ignore_suggestion('''SQL injection vulnerability in the Itamar Elharar MusicGallery (com_musicgallery) component for Joomla! allows remote attackers to execute arbitrary SQL commands via the id parameter in an itempage action to index.php. NOTE: the provenance of this information is unknown; the details are obtained solely from third party information.''')) self.assertEqual("Kide Shoutbox component for Joomla!", h.get_ignore_suggestion('''The Kide Shoutbox (com_kide) component 0.4.6 for Joomla! does not properly perform authentication, which allows remote attackers to post messages with an arbitrary account name via an insertar action to index.php. NOTE: the provenance of this information is unknown; the details are obtained solely from third party information.''')) self.assertEqual("WP-Forum plugin for WordPress", h.get_ignore_suggestion('''Multiple SQL injection vulnerabilities in the WP-Forum plugin before 2.4 for WordPress allow remote attackers to execute arbitrary SQL commands via (1) the search_max parameter in a search action to the default URI, related to wpf.class.php; (2) the forum parameter to an unspecified component, related to wpf.class.php; (3) the topic parameter in a viewforum action to the default URI, related to the remove_topic function in wpf.class.php; or the id parameter in a (4) editpost or (5) viewtopic action to the default URI, related to wpf-post.php.''')) self.assertEqual("ListMan extension for TYPO3", h.get_ignore_suggestion('''Cross-site scripting (XSS) vulnerability in the ListMan (nl_listman) extension 1.2.1 for TYPO3 allows remote attackers to inject arbitrary web script or HTML via unspecified vectors.''')) self.assertEqual("multiple status.net issues", h.get_ignore_suggestion('''ML-Date: 2011-01-25 12:08:05, ML-Subject: Re: CVE request: multiple status.net issues''')) self.assertEqual("multiple status.net issues", h.get_ignore_suggestion('''ML-Date: 2011-01-25 12:08:05, ML-Subject: CVE request: multiple status.net issues''')) self.assertEqual("multiple status.net issues", h.get_ignore_suggestion('''ML-Date: 2011-01-25 12:08:05, ML-Subject: multiple status.net issues''')) self.assertEqual("Mambo CMS", h.get_ignore_suggestion('''ML-Date: 2011-06-28 16:24:28, ML-Subject: Re: CVE Request: Mambo CMS 4.6.x | Multiple Cross Site Scripting Vulnerabilities''')) self.assertEqual("Apple iOS", h.get_ignore_suggestion('''The DNAv4 protocol implementation in the DHCP component in Apple iOS before 6 sends Wi-Fi packets containing a MAC address of a host on a previously used network, which might allow remote attackers to obtain sensitive information about previous device locations by sniffing an unencrypted Wi-Fi network for these packets.''')) self.assertEqual("Conceptronic", h.get_ignore_suggestion('''Multiple open redirect vulnerabilities on the Conceptronic C54APM access point with runtime code 1.26 allow remote attackers to redirect users to arbitrary web sites and conduct phishing attacks via (1) the submit-url parameter in a Refresh action to goform/formWlSiteSurvey or (2) the wlan-url parameter to goform/formWlanSetup.''')) self.assertEqual("Tapjoy library for Android", h.get_ignore_suggestion('''The Tapjoy library for Android does not verify X.509 certificates from SSL servers, which allows man-in-the-middle attackers to spoof servers and obtain sensitive information via a crafted certificate.''')) self.assertEqual("kamkomesan application for Android", h.get_ignore_suggestion('''The kamkomesan (aka com.anek.kamkomesan) application 1.0 for Android does not verify X.509 certificates from SSL servers, which allows man-in-the-middle attackers to spoof servers and obtain sensitive information via a crafted certificate.''')) self.assertEqual("DataTables plugin for jQuery", h.get_ignore_suggestion('''Cross-site scripting (XSS) vulnerability in the DataTables plugin 1.10.8 and earlier for jQuery allows remote attackers to inject arbitrary web script or HTML via the scripts parameter to media/unit_testing/templates/6776.php.''')) self.assertEqual("Cisco Application Policy Infrastructure Controller (APIC)", h.get_ignore_suggestion('''A vulnerability in Cisco Application Policy Infrastructure Controller (APIC) could allow an authenticated, remote attacker to gain higher privileges than the account is assigned.''')) self.assertEqual("Redirection", h.get_ignore_suggestion('''Redirection version 2.7.3 contains a ACE via file inclusion vulnerability in Pass-through mode that can result in allows admins to execute any PHP file in the filesystem.''')) self.assertEqual("topydo", h.get_ignore_suggestion('''topydo contains a CWE-20: Improper Input Validation vulnerability in ListFormatParser::parse, file topydo/lib/ListFormat.py line 292 as of d4f843dac71308b2f29a7c2cdc76f055c3841523 that can result in Injection of arbitrary bytes to the terminal,''')) self.assertEqual("Joplin", h.get_ignore_suggestion('''Joplin version prior to 1.0.90 contains a XSS evolving into code execution due to enabled nodeIntegration.''')) self.assertEqual("Foxit Reader", h.get_ignore_suggestion('''This vulnerability allows remote attackers to execute arbitrary code on vulnerable installations of Foxit Reader 9.2.0.9297.''')) self.assertEqual("Exquisite Ultimate Newspaper theme for WordPress", h.get_ignore_suggestion('''The Exquisite Ultimate Newspaper theme 1.3.3 for WordPress has XSS via the anchor identifier to assets/js/jquery.foundation.plugins.js.''')) self.assertEqual("browserless-chrome", h.get_ignore_suggestion('''This affects all versions of package browserless-chrome. User input flowing from the workspace endpoint gets used to create a file path filePath and this is fetched and then sent back to a user. This can be escaped to fetch arbitrary files from a server.''')) self.assertEqual("@absolunet/kafe", h.get_ignore_suggestion('''This affects the package @absolunet/kafe before 3.2.10. It allows cause a denial of service when validating crafted invalid emails.''')) ignored_notforus_path = 'ignored/not-for-us.txt' if destdir != './' and destdir != '.': ignored_notforus_path = os.path.join(destdir, ignored_notforus_path) # CVEIgnoreNotForUsList is a list of all CVEs that we have previously # chosen to ignore since they don't apply to software in Ubuntu CVEIgnoreNotForUsList = cve_lib.parse_CVEs_from_uri(ignored_notforus_path) ignored_mistriaged_path = 'ignored/ignore-mistriaged.txt' if destdir != './' and destdir != '.': ignored_mistriaged_path = os.path.join(destdir, ignored_mistriaged_path) # CVEIgnoreMistriagedList is a list of all CVEs that we want to definitely # ignore when doing mistriaged CVE detection - they should exist in both # CVEIgnoreNotForUsList and CVEIgnoreMistriagedList CVEIgnoreMistriagedList = cve_lib.parse_CVEs_from_uri(ignored_mistriaged_path) # CVEIgnoreList is a list of all CVEs we know about already. These will be # ignored when checking MITRE for new CVEs CVEIgnoreList = list(CVEIgnoreNotForUsList) CVEKnownList = [] CVEKnownList += [cve for cve in os.listdir(destdir + "/ignored/") if cve.startswith('CVE-')] CVEKnownList += [cve for cve in os.listdir(destdir + "/retired/") if cve.startswith('CVE-')] (ActiveList, EmbargoList) = cve_lib.get_cve_list() CVEKnownList += [cve for cve in ActiveList if cve not in EmbargoList] if not opt.refresh and not opt.mistriaged: CVEIgnoreList += CVEKnownList if opt.known: cvelist = CVEIgnoreList if opt.skip_nfu: cvelist = CVEKnownList for cve in sorted(cvelist): print(cve) sys.exit(0) parser = xml.sax.make_parser() handler = CVEHandler(CVEIgnoreList) parser.setContentHandler(handler) if opt.test: suite = unittest.TestSuite() suite.addTest(unittest.TestLoader().loadTestsFromTestCase(CheckCVETest)) unittest.TextTestRunner(verbosity=2).run(suite) sys.exit(0) # if has specified to triage only specific CVEs, check these are not # ignored specific_cves = set() for cve in opt.cve.split(","): # ignore empty CVE if cve.strip() == "": continue # error out if is ignored if cve in CVEIgnoreList: print("%s already exists in UCT - please remove it then retriage." % cve) sys.exit(1) specific_cves.add(cve) untriaged_json = "" if opt.untriaged: untriaged_json = read_locate_cves_output(opt.untriaged) args.append(untriaged_json) if opt.mbox: untriaged_json = read_mbox_file(opt.mbox) args.append(untriaged_json) rhel8oval_import_json = "" if opt.rhel8oval: untriaged_json = read_rhel8oval_file(opt.rhel8oval) args.append(untriaged_json) debian_import_json = "" if (opt.import_missing_debian or opt.mistriaged) and handler.debian is not None: debian_import_json = import_debian(handler) args.append(debian_import_json) if len(args) == 0: args.append("https://cve.mitre.org/cve/downloads/allitems.xml") for uri in args: print('Loading %s ...' % (uri), file=sys.stderr) if '://' in uri: readable = urllib.request.urlopen(uri) else: readable = PercentageFile(uri) try: if uri.endswith("json"): handler.parse_json(readable) else: parser.parse(readable) except xml.sax._exceptions.SAXParseException as e: print("\n\nWARNING: %s is malformed:\n%s" % (uri, e)) print("Aborting", file=sys.stderr) sys.exit(1) print('') # Leaving our fake json around is icky if os.path.exists(untriaged_json): os.unlink(untriaged_json) if os.path.exists(debian_import_json): os.unlink(debian_import_json) if opt.refresh: for cve in sorted(CVEKnownList): # Get new information from the XML desc = None public = None cvsss = [] try: desc = _wrap_desc(handler.cve_data[cve]['desc'].strip()) public = handler.cve_data[cve]['public'] cvsss = handler.cve_data[cve]['cvss'] except: if opt.verbose: print('%s not listed in XML' % (cve), file=sys.stderr) # Find the on-disk CVE file cvefile = "" for status in ['active', 'retired', 'ignored']: check = '%s/%s/%s' % (destdir, status, cve) if os.path.exists(check): cvefile = check break if cvefile == "": print('local dirs missing %s?!' % (cve), file=sys.stderr) continue # Load CVE try: data = cve_lib.load_cve(cvefile) except ValueError as e: print(e, file=sys.stderr) continue # Set defaults for required fields if no XML value exists if 'PublicDate' not in data and not public: public = "unknown" updated = False # Update description if it needs it if desc: if data['Description'].strip() != desc: cve_lib.update_multiline_field(cvefile, 'Description', desc) updated = True debug("updated description for %s" % (cvefile)) # Update Publication Date if it needs it if public: if 'PublicDate' not in data or ('PublicDate' in data and data['PublicDate'] != public): cve_lib.update_field(cvefile, 'PublicDate', public) updated = True debug("updated pubdate for %s" % (cvefile)) # Add CVE Reference, if it's missing if 'References' in data and re.match('^CVE-\d+-\d+$', cve): mitre_ref = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + cve if mitre_ref not in data['References']: cve_lib.add_reference(cvefile, mitre_ref) updated = True debug("updated reference for %s" % (cvefile)) # Update CVSS if needs it for entry in cvsss: try: updated |= cve_lib.add_cvss(cvefile, entry['source'], entry['vector']) debug("updated cvss for %s" % (cvefile)) except KeyError: # we might not have a vector, in this case continue pass if updated: print("Refreshed %s" % (cvefile), file=sys.stderr) sys.exit(0) new_cves = handler.cves() total = len(new_cves) count = 0 fout = sys.stdout if experimental: fout = tempfile.NamedTemporaryFile(prefix='check-cves.', mode='w+') handler.display_command_file_usage(fout, '# ') for cve in new_cves: if len(specific_cves) > 0 and cve not in specific_cves: # ignore this cve continue # if this got marked as mistriaged, probablistically choose it for # processing if mistriaged_hint in handler.cve_data[cve]['desc']: if opt.mistriaged == 0: # ignore this one continue else: # choose CVEs at random but pick most recent first - if year is # now then we want to pick it, if year is 1 year ago we want to # pick half as often, if year is 2 years ago we want to pick # 1/4 as often etc - so do 1/2^(diff) year = int(re.split('-', cve)[1]) now = datetime.datetime.utcnow().year prob = 1.0 / math.pow(2, now - year) rand = random.random() if rand > prob: continue # selected! opt.mistriaged = opt.mistriaged - 1 count += 1 if opt.report: print(cve) continue if not experimental: print('\n***********************************************************************') print(' %s (%d/%d: %d%%)' % (cve, count, total, (count * 100 / total))) print(' https://cve.mitre.org/cgi-bin/cvename.cgi?name=%s' % (cve)) print('***********************************************************************') handler.display_cve(cve, file=fout) (action, reason, packages) = handler.get_cve_suggestions(cve) if action == 'unembargo': action = 'skip' handler.human_process_cve(cve, action, reason, packages) else: line_prefix = '# ' fout.write('%s%s\n#\n' % (line_prefix, cve)) handler.display_cve(cve, file=fout, line_prefix=line_prefix, wrap_desc=True) (action, reason, packages) = handler.get_cve_suggestions(cve) suggestion = '%s %s' % (cve, action) if reason: suggestion += ' %s' % (reason) elif action == 'add': suggestion += ' untriaged %s' % (packages) fout.write('%s\n\n' % (suggestion)) if experimental: fout.flush() while True: assert((total == 0 and count == 0) or (total > 0 and count >= 0)) # no need to spawn editor if no CVEs listed if count > 0: prompt_user('Asking user to handle %d CVEs...' % count) _spawn_editor(fout.name) else: prompt_user('Not spawning editor as no CVEs to process') fout.seek(0) try: handler.process_command_file(fout, preprocess=True) break except Exception as e: print('Error encountered while processing the command file!', file=sys.stderr) print('%s' % e, file=sys.stderr) print('Enter "quit" to lose all work or anything else to try again: ', end='', file=sys.stderr) sys.stderr.flush() response = sys.stdin.readline().strip().lower() if response == 'quit': sys.exit(1) fout.seek(0) handler.process_command_file(fout) if not opt.report: handler.updateTimestamp() handler.printReport()