#!/usr/bin/env python3 # Author: Kees Cook # Author: Jamie Strandboge # Author: Marc Deslauriers # Copyright (C) 2005-2020 Canonical Ltd. # # This script is distributed under the terms and conditions of the GNU General # Public License, Version 2 or later. See http://www.gnu.org/copyleft/gpl.html # for details. # Set CVE_ALLOW_NEWER_PKGS=1 to skip new package checks from __future__ import print_function import datetime import glob import optparse import os import pprint import re import sys import subprocess import signal import cve_lib import kernel_lib import usn_lib import source_map source = source_map.load() dev_proposed = source_map.load(pockets=["-proposed"], releases=[cve_lib.devel_release]) # add to this list when adding tracking for kernels that have not been # published to updates/security yet. Once published, can be removed # (script should warn about that). If a kernel is published in one # release, but not yet in another release, add the unpublished release # after a '/' e.g. 'linux-aws/trusty' unpublished_kernels = [ "linux-fips", # XXX silencing warnings about xenial fips kernel, but # the code need to be updated to handle esm releases ] required_fields = [ 'Candidate', 'Description', 'Priority', 'PublicDate', 'Ubuntu-Description', ] def CVEs_from_CNA(): """Returns a dict of CVEs assigned from the README in the embargoed cna directory, using a space and text after CVE-NNNN-NNNN as indicator of which CVEs have been assigned. For each CVE in the dict, the value is a list containing the filename and line number where this was sourced from. """ lines = [] # Assumes embargoed symlink/ exists in parent directory and points to # ../embargoed/cves/ cna_source = os.path.realpath( os.path.join( os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "embargoed", "..", "cna", "README")) try: f = open(cna_source) lines = f.readlines() f.close() except IOError as e: print("Ignoring CNA sanity checks: %s\n" % e, file=sys.stderr) return {} # Assumes any text after CVE-NNNN-NNNN means "assigned" # Note the space is important, used for split(" ") later assigned_cves = re.compile(r"^(CVE-\d\d\d\d-\d{4,7}) .+") def find_assigned_cves(line): if "REJECTED" in line: return None if "IGNORED" in line: return None match = assigned_cves.match(line) if match: return match.group(1) return None cves = {} linenum = 1 for line in lines: cve = find_assigned_cves(line) if cve: cves[cve] = [cna_source, linenum] linenum += 1 return cves def ever_existed(pkg): for rel in cve_lib.all_releases: if rel in cve_lib.eol_releases: continue if rel in source and pkg in source[rel]: return True return False def is_active(cve): return os.path.exists(os.path.join(cve_lib.active_dir, cve)) def is_embargoed(cve): return os.path.exists(os.path.join(cve_lib.embargoed_dir, cve)) def is_retired(cve): return os.path.exists(os.path.join(cve_lib.retired_dir, cve)) def is_unpublished_kernel(kernel, release=None, debug=False): for u in unpublished_kernels: unpub = u.split("/") if debug: print("Checking %s/%s against %s" % (kernel, release, unpub)) if unpub[0] == kernel and ( len(unpub) == 1 or (len(unpub) == 2 and unpub[1] == release) ): return True return False def subprocess_setup(): # Python installs a SIGPIPE handler by default. This is usually not what # non-Python subprocesses expect. signal.signal(signal.SIGPIPE, signal.SIG_DFL) def cmd( command, input=None, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, stdin=None, timeout=None, ): """Try to execute given command (array) and return its stdout, or return a textual error if it failed.""" try: sp = subprocess.Popen( command, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=True, preexec_fn=subprocess_setup, universal_newlines=True, ) except OSError as e: return [127, str(e)] out, outerr = sp.communicate(input) # Handle redirection of stdout if out is None: out = "" # Handle redirection of stderr if outerr is None: outerr = "" return [sp.returncode, out + outerr] def get_bzr_filelist(): rc, report = cmd(["bzr", "stat", "-S"]) file_list = [] for line in report.splitlines(): if len(line.split()) == 2: if line.split()[0] != "D": file_list.append(line.split()[1]) return file_list def get_git_filelist(): rc, report = cmd(["git", "diff", "--name-only", "--diff-filter=AM", "HEAD"]) return report.splitlines() def cve_lib_consistency_checks(): # add additional consistency check calls here, if need be kernel_lib.meta_kernels.consistency_check() parser = optparse.OptionParser() parser.add_option( "-v", "--verbose", help="Enable verbose reporting", action="store_true", default=False, ) parser.add_option( "-d", "--debug", help="Enable debug reporting", action="store_true", default=False ) parser.add_option( "-u", "--usn-db", dest="usndb", help="Enable USN database on DB", metavar="DB" ) parser.add_option( "-n", "--newer", help="Allow versions newer than what is in the archive", action="store_true", default=False, ) parser.add_option( "-s", "--strict", help="Be extra strict in syntax", action="store_true", default=False, ) parser.add_option( "-f", "--filelist", help="Use a list of files to work on", metavar="FILE" ) parser.add_option( "-m", "--modified", help="Only check modified files", action="store_true", default=False, ) # parser.add_option("-c", "--cna", help="Ensure every CVE assigned by Canonical's CNA has an entry", action='store_true') (opt, args) = parser.parse_args() if opt.debug: pp = pprint.PrettyPrinter(indent=4) all_okay = True cves_updated = False warned_kernels = [] esm_warned = False cve_lib_consistency_checks() cves = dict() if opt.usndb: usn_db = usn_lib.load_database(opt.usndb) reverted = usn_lib.get_reverted() # create a dictionary of key=CVE value=USNs for usn in usn_db: if "cves" not in usn_db[usn]: continue for cve in usn_db[usn]["cves"]: if not cve.startswith("CVE-"): continue if usn in reverted and cve in reverted[usn]: continue if cve not in cves: cves[cve] = [] cves[cve].append(usn) else: cves[cve].append(usn) check_dirs = [cve_lib.active_dir, cve_lib.retired_dir, cve_lib.ignored_dir] if os.path.islink(cve_lib.embargoed_dir): check_dirs.append(cve_lib.embargoed_dir) if opt.debug: pp.pprint("check_dirs %s" % check_dirs) all_files = True if len(args) == 0: if opt.filelist: if opt.debug: pp.pprint("Using filelist %s" % opt.filelist) all_files = False with open(opt.filelist) as fh: for line in fh: for dir in check_dirs: if line.startswith("%s/CVE-" % dir) or line.startswith("%s/00boilerplate" % dir): args += [line.rstrip()] elif opt.modified: if opt.debug: pp.pprint("Checking modified files only") all_files = False if os.path.exists(".git"): file_list = get_git_filelist() else: file_list = get_bzr_filelist() for filename in file_list: filename = os.path.join(os.getcwd(), filename) for dir in check_dirs: if filename.startswith("%s/CVE-" % dir): args += [filename] else: for dir in check_dirs: # XXX might want to separate out boilerplate checks # as they might need to be less restrictive than regular CVE # file checks for cve in sorted(glob.glob("%s/CVE-*" % dir) + glob.glob("%s/00boilerplate.*" % dir)): args += [cve] else: all_files = False if opt.debug: pp.pprint("args %s" % args) ignored = cve_lib.parse_CVEs_from_uri("%s/not-for-us.txt" % cve_lib.ignored_dir) if opt.debug: pp.pprint("ignored %s" % ignored) cna_cves_set = CVEs_from_CNA() # Just run this if we're not specifying specific CVEs if len(cna_cves_set) > 0 and all_files: cve_name_re = re.compile(r"(CVE-\d\d\d\d-\d{4,7})") def filter_cves(fname): return re.search(cve_name_re, fname) def find_cves(fname): m = re.search(cve_name_re, fname) if m: return m.group(1) else: print("unmatched %s\n" % fname, file=sys.stderr) ignored_set = set(ignored) args_set = {find_cves(name) for name in filter(filter_cves, args)} if opt.debug: pp.pprint("cna_cves_set: %s\n" % cna_cves_set) pp.pprint("ignored_set: %s\n" % ignored_set) pp.pprint("args: %s" % args) pp.pprint("filtered_args: %s" % filter(filter_cves, args)) pp.pprint("args_set: %s\n" % args_set) # if we ever assign a CVE then ignore it (vbulletin comes to mind...) # we can use the "IGNORED" tag to skip these checks martians = cna_cves_set.keys() & ignored_set for cve in martians: print("%s: %d: %s that we assigned is now IGNORED.\n" % (cna_cves_set[cve][0], cna_cves_set[cve][1], cve)) too_private = cna_cves_set.keys() - args_set for cve in too_private: print("%s: %d: %s that we assigned needs an UCT entry.\n" % (cna_cves_set[cve][0], cna_cves_set[cve][1], cve)) for cve in args: if re.match(r"EMB-", cve): cvepath = os.path.join(cve_lib.embargoed_dir, cve) elif re.match(r"CVE-", cve): cvepath = os.path.join(cve_lib.active_dir, cve) else: cvepath = cve cve = os.path.basename(cvepath) cve_okay = True srcmap = dict() try: if "subprojects" in cvepath: data = cve_lib.load_cve(cve_lib.find_cve(cve), opt.strict, srcmap=srcmap) else: data = cve_lib.load_cve(cvepath, opt.strict, srcmap=srcmap) except ValueError as e: print(e, file=sys.stderr) all_okay = False continue if cve in ignored: print( "%s: %d: duplicate CVE found in not-for-us.txt" % (cvepath, 1), file=sys.stderr, ) all_okay = False matches = set() for dir in [ cve_lib.active_dir, cve_lib.retired_dir, cve_lib.ignored_dir, cve_lib.embargoed_dir, ]: if os.path.exists(os.path.join(dir, cve)): matches.add(dir) if len(matches) > 1: print( "%s: %d: found in multiple classes: %s" % (cvepath, 1, ", ".join(sorted(matches))), file=sys.stderr, ) all_okay = False # verify required fields are present for field in required_fields: if field not in data: print("%s: missing required field '%s'" % (cvepath, field), file=sys.stderr) cve_okay = False # verify candidate field matches the CVE file name if "stdin" not in cve and "boilerplate" not in cve and not data["Candidate"] == cve: filename = srcmap["Candidate"][0] linenum = srcmap["Candidate"][1] print( "%s: %d: Candidate field '%s' mismatch with cve pathname '%s'" % (filename, linenum, data["Candidate"], cve), file=sys.stderr, ) cve_okay = False supported = [] for pkg in sorted(data["pkgs"].keys()): # Verify have required releases for each package listed_releases = set(sorted(data["pkgs"][pkg].keys())) all_required_releases = (set(cve_lib.all_releases + ["devel"]) - set([cve_lib.devel_release])) - set(cve_lib.eol_releases) missing_releases = all_required_releases - listed_releases # get the name of a release which is listed in the CVE so we can # place the generated error message on this release's line etc nearby_rel = list(listed_releases - missing_releases)[0] for rel in missing_releases: # only warn on active CVEs but don't warn on boilerplate entries missing external # releases as this is not supported if is_active(cve) and \ ("boilerplate" not in cve or rel not in cve_lib.external_releases) and \ rel in source and pkg in source[rel]: filename = srcmap["pkgs"][pkg][nearby_rel][0] linenum = srcmap["pkgs"][pkg][nearby_rel][1] print( "%s: %d: %s missing release '%s'" # put the error on a line near where this entry should go % (filename, linenum, pkg, rel), file=sys.stderr, ) cve_okay = False unknown_releases = listed_releases - set(cve_lib.all_releases + ["devel", "upstream"]) for rel in unknown_releases: filename = srcmap["pkgs"][pkg][rel][0] linenum = srcmap["pkgs"][pkg][rel][1] print( "%s: %d: %s unknown release '%s'" % (filename, linenum, pkg, rel), file=sys.stderr, ) cve_okay = False for release in sorted(data["pkgs"][pkg].keys()): rel = release filename = srcmap["pkgs"][pkg][release][0] linenum = srcmap["pkgs"][pkg][release][1] # warn if the cve file contains external subproject releases # and it is not located in the subprojects folder if rel in cve_lib.external_releases and not "subprojects" in filename: print( "%s: %d: external release '%s' listed in internal CVE file" % (filename, linenum, rel), file=sys.stderr, ) cve_okay = False # Handle just after release, but before devel has opened if rel == "devel" and cve_lib.devel_release == "": continue # Adjust devel release name if rel == "devel": rel = cve_lib.devel_release # Don't syntax check upstream if rel == "upstream": continue # Don't syntax check end-of-lifed releases if rel in cve_lib.eol_releases: continue details = data["pkgs"][pkg][release] state = details[0] # Skip devel checks on retired CVEs if release == "devel": if "retired/" in cvepath: # but first check to ensure that state is not open if state in ["needed", "needs-triage", "pending"]: print( "%s: %d: retired but %s is listed as unfixed (%s) for '%s'" % (filename, linenum, pkg, state, rel), file=sys.stderr, ) cve_okay = False continue # Skip DNE's if state == "DNE": if pkg in source[rel]: # TODO: remove this when partner archive is back in sync # sigh, partner archive mirrors are out of date since removing sun-java6 if pkg == "sun-java6" and source[rel][pkg]["section"] == "partner": continue print( "%s: %d: package '%s' DOES exist in '%s'" % (filename, linenum, pkg, rel), file=sys.stderr, ) cve_okay = False continue # Check for released-esm as it is not a valid state in UCT anymore if state == "released-esm": print( "%s: %d: %s_%s has invalid state: '%s'" % (filename, linenum, rel, pkg, state) ) cve_okay = False # check any notes have balanced parentheses if len(details) > 1: note = details[1] if note.count("(") != note.count(")"): print( "%s: %d: %s_%s has unbalanced parentheses in state annotation: '%s %s'" % (filename, linenum, rel, pkg, state, note) ) cve_okay = False # Check that package exists in a given release if not ever_existed(pkg): if is_active(cve) and not cve_lib.is_active_esm_release(release): # forcibly skip linux-lts-backport packages and # # other derived kernels since want to track them # before they end up fully in the archive; # additional kernels can be added to # unpublished_kernels as needed if pkg.startswith("linux-lts-") or is_unpublished_kernel(pkg, rel): if opt.debug: print( "%s: %d: skipping unpublished kernel '%s'" % (filename, linenum, pkg), file=sys.stderr, ) continue print( "%s: %d: unknown package '%s' in %s" % (filename, linenum, pkg, rel), file=sys.stderr, ) cve_okay = False else: if rel in source: if pkg not in source[rel]: if is_active(cve): if is_unpublished_kernel(pkg, rel): if opt.debug: print( "%s: %d: skipping unpublished kernel '%s'" % (filename, linenum, pkg), file=sys.stderr, ) continue # is this package released for ESM, even though is not in # supported list? ignore it if "esm" in rel and cve_lib.is_active_esm_release(rel.split("/esm")[0]): if state == "released" and not cve_lib.is_supported( source, pkg, rel, data ): continue # Just a warning if this is the devel release, package may be # in -proposed. # TODO: actually check -proposed. if release == "devel" and pkg in dev_proposed[rel]: if opt.debug: print( "%s: %d: WARNING: package '%s' not in '%s' (found in -proposed)" % (filename, linenum, pkg, rel), file=sys.stderr, ) else: print( "%s: %d: package '%s' not in '%s'" % (filename, linenum, pkg, rel), file=sys.stderr, ) cve_okay = False elif opt.strict and not opt.newer: # Validate the version is <= version in release # Unfortuanely, had to move this to --strict, as # some older nvidia updates trigger this check. if state == "released": released = details[1] # XXX 'version' existence check should not be needed if "version" not in source[rel][pkg]: if not esm_warned: print( "%s: %d: unable to lookup source version for %s in %s" % (filename, linenum, pkg, rel), file=sys.stderr, ) print( "-- This is likely due to missing version info in source_map.py for the ESM release", file=sys.stderr, ) esm_warned = True elif ( source_map.version_compare( released, source[rel][pkg]["version"] ) > 0 ): print( "%s: %d: %s has %s > %s (in %s)" % ( filename, linenum, pkg, released, source[rel][pkg]["version"], rel, ), file=sys.stderr, ) cve_okay = False # this is a check-syntax self-sanity check; we're # looking for kernels that were in the # unpublished_kernels list that have subsequently been # published and should be checked for existence. if is_active(cve) and is_unpublished_kernel(pkg, rel): # we'll only issue a warning the first time we come # across a kernel that's been published if pkg not in warned_kernels: print( "kernel '%s/%s' is now published; please remove from unpublished_kernels list in check-syntax. (%s)" % (pkg, rel, cve), file=sys.stderr, ) warned_kernels.append(pkg) # Is this package unfixed and considered supported? if state not in ["released-esm", "released", "not-affected"] and (cve_lib.is_active_esm_release(rel) or cve_lib.is_supported( source, pkg, rel, data )): supported.append("%s/%s" % (pkg, rel)) if ( len(details) > 1 and state != "deferred" and "end of standard support" not in details[1] and "end of ESM support" not in details[1] and "deferred" in details[1] ): print( "%s: %d: %s uses 'deferred' with '%s'. Use: 'deferred [(YYYY-MM-DD)]'" % (filename, linenum, pkg, state), file=sys.stderr, ) cve_okay = False # Verify priority for any CVE with a supported package and when this is # not boilerplate if ( "boilerplate" not in cve and len(supported) and (is_active(cve) or is_embargoed(cve)) and ("Priority" not in data or data["Priority"] not in cve_lib.priorities) ): filename = srcmap["Priority"][0] if "Priority" in srcmap else cvepath linenum = srcmap["Priority"][1] if "Priority" in srcmap else 1 print( "%s: %d: Priority missing with supported packages (%s)" % ( filename, linenum, ", ".join(supported), ), file=sys.stderr, ) cve_okay = False # Verify per-package and per-package-release priorities for pkg_priority in [x for x in data if "Priority_" in x]: filename = srcmap[pkg_priority][0] linenum = srcmap[pkg_priority][1] fields = pkg_priority.split("_", 2) if len(fields) == 2: pkg = fields[1] if pkg not in data["pkgs"]: print( "%s: %d: per package priority (%s) does not match any package in cve" % (filename, linenum, pkg), file=sys.stderr, ) cve_okay = False elif len(fields) == 3: pkg, release = fields[1:3] if pkg not in data["pkgs"]: print( "%s: %d: per package/release priority (%s/%s) does not match a package/release pair in cve" % (filename, linenum, pkg, release), file=sys.stderr, ) cve_okay = False elif ( release not in cve_lib.all_releases and not release == "devel" ) or release not in data["pkgs"][pkg]: print( "%s: %d: invalid release in per package/release priority pair (%s/%s)" % (filename, linenum, pkg, release), file=sys.stderr, ) cve_okay = False # print(pkg_priority, file=sys.stderr) # check to see if the description has been changed to rejected by # MITRE, we should move CVE to ignored state. if ( is_active(cve) and "Description" in data and data["Description"].lstrip().startswith("** REJECT **") ): filename = srcmap["Description"][0] linenum = srcmap["Description"][1] print( "%s: %d: Rejected by MITRE (possibly a duplicate of another CVE?)" % (filename, linenum), file=sys.stderr, ) cve_okay = False # check for CVE reference if re.match(r"^CVE-\d+-\d+$", cve): found = False mitre_ref = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + cve if "References" in data: if mitre_ref in data["References"]: found = True if not found: filename = srcmap["References"][0] if "References" in srcmap else cvepath linenum = srcmap["References"][1] if "References" in srcmap else 1 print( "WARNING: %s: %d: does not contain reference to %s" % ( filename, linenum, mitre_ref, ), file=sys.stderr, ) cve_okay = False # check for URLs if using the USN database if opt.usndb and "ignored/" not in cvepath and cve in cves and len(cves[cve]) > 0: if "References" in data: for usn in cves[cve]: found = False usn_ref = "http://www.ubuntu.com/usn/usn-" + usn text = data["References"].strip() if len(text) != 0: for line in text.split("\n"): if line == usn_ref: found = True if not found: filename = srcmap["References"][0] if "References" in srcmap else cvepath linenum = srcmap["References"][1] if "References" in srcmap else 1 print( "%s: %d: does not contain reference to %s" % ( filename, linenum, usn_ref, ), file=sys.stderr, ) cve_okay = False # Either PublicDate or CRD must be set to something if ("boilerplate" not in cve and ("PublicDate" not in data or data["PublicDate"] == "") and ( "CRD" not in data or data["CRD"] == "" )): key = "PublicDate" if "PublicDate" in srcmap else "CRD" filename = srcmap[key][0] linenum = srcmap[key][1] print( "%s: %d: must specify at least one of PublicDate or CRD as a valid date or unknown" % ( filename, linenum, ), file=sys.stderr, ) for d in ["PublicDate", "PublicDateAtUSN", "CRD"]: if d in data and data[d] not in ["", "unknown"]: valid = False # Date patterns for formatted as YYYY-MM-DD HH:MM:SS date_only_format = "%Y-%m-%d" date_time_format = date_only_format + " %H:%M:%S" date_time_tz_offset_format = date_time_format + " %z" date_time_tz_name_format = date_time_format + " %Z" for fmt in [date_only_format, date_time_format, date_time_tz_offset_format, date_time_tz_name_format]: try: datetime.datetime.strptime(data[d], fmt) valid = True except ValueError: continue if not valid: filename = srcmap[d][0] linenum = srcmap[d][1] print( "%s: %d: does not contain a valid %s '%s' (need YYYY-MM-DD [HH:MM:SS] [TIMEZONE|OFFSET], an empty string, or 'unknown')" % (filename, linenum, d, data[d]), file=sys.stderr, ) cve_okay = False for pkg in data["patches"]: for index, value in enumerate(data["patches"][pkg]): patch_type, patch = data["patches"][pkg][index] # validate break-fix entries as 'I?hash|-|local-|URL' and # others should be a URL - but don't bother with retired # CVEs as these have a lot of old cruft filename = srcmap["patches"][pkg][index][0] linenum = srcmap["patches"][pkg][index][1] if patch_type == "break-fix": try: bfre = "^(-|I?[a-f0-9]{1,40}|local-[A-Za-z0-9-]+|https?://.*)$" breaks, fixes = patch.split(' ', 1) # breaks and fixes can contain multiple entries separated by | for brk in breaks.split('|'): if re.match(bfre, brk) is None: raise ValueError("invalid break entry '%s':" % brk) # fixes can contain multiple entries separated by | for fix in fixes.split('|'): if re.match(bfre, fix) is None: raise ValueError("invalid fix entry '%s':" % fix) except Exception as e: print( "%s: %d: invalid break-fix entry: '%s': %s" % (filename, linenum, patch, e), file=sys.stderr, ) cve_okay = False elif opt.strict or not "retired/" in cvepath: if "://" not in patch: print( "%s: %d: invalid patch URL '%s'" % (filename, linenum, patch), file=sys.stderr, ) cve_okay = False for entry in data["CVSS"]: srcname = entry['source'] filename = srcmap["CVSS"][srcname][0] linenum = srcmap["CVSS"][srcname][1] # check for cvss entries with unknown origin if srcname == 'unknown': print("%s: %d: CVSS with unknown origin" % (filename, linenum), file=sys.stderr) cve_okay = False if "baseScore" not in entry.keys(): print("%s: %d: CVSS missing baseScore" % (filename, linenum), file=sys.stderr) cve_okay = False if "baseSeverity" not in entry.keys(): print("%s: %d: CVSS missing baseSeverity" % (filename, linenum), file=sys.stderr) cve_okay = False cvss = cve_lib.parse_cvss(entry['vector']) # check baseScore and baseSeverity match baseScore = cvss['baseMetricV3']['cvssV3']['baseScore'] baseSeverity = cvss['baseMetricV3']['cvssV3']['baseSeverity'] if float(entry['baseScore']) != baseScore: print("%s: %d: baseScore %s is not correct, should be %02.1f" % (filename, linenum, entry['baseScore'], baseScore), file=sys.stderr) cve_okay = False if entry['baseSeverity'] != baseSeverity: print("%s: %d: baseSeverity %s is not correct, should be %s" % (filename, linenum, entry['baseSeverity'], baseSeverity), file=sys.stderr) cve_okay = False # Report on failures if not cve_okay: all_okay = False elif opt.debug: pp.pprint(data) if cves_updated: print("CVEs updated. Please run again.", file=sys.stderr) all_okay = False if all_okay and opt.verbose: print("OK: %d CVEs" % (len(args))) # Invert boolean for unix exit code sys.exit(not all_okay)