#!/usr/bin/env python3
# Copyright 2007-2020 Canonical, Ltd.
# Authors: Kees Cook <kees@ubuntu.com>
#          Jamie Strandboge <jamie@ubuntu.com>
#          Marc Deslauriers <marc.deslauriers@ubuntu.com>
# License: GPLv2
# Goes and finds all the files that make up a set of .changes files.
#
# Example for linux package binary filter:
#  --filter-bins '^linux-image-\d'

from __future__ import print_function

import apt_pkg
import json
import gzip
import lzma
import optparse
import os
import os.path
import re
import subprocess
import sys
import tempfile
import cve_lib

opter = optparse.OptionParser()
opter.add_option("--debug", help="Verbose processing output", action='store_true')
opter.add_option("--esm", help="Set to be an ESM release", default=None)
opter.add_option("--kernel-mode", help="Use special kernel mode", action='store_true')
opter.add_option("--no-new-warn", help="Do not warn about or error out on NEW binaries", action='store_true')
opter.add_option("--filter-bins", metavar="REGEX", help="Only include binary packages matching the REGEX in report", default=None)
opter.add_option("--cves", metavar="CVES", help="Comma separated list of CVEs to use instead of the normal *_source.changes autodetection (must be a superset).", default=None)
opter.add_option("--add-cves", metavar="CVES", help="Comma separated list of CVEs to use in addition to the normal *_source.changes autodetection.", default=None)
opter.add_option("--ignore-cves", metavar="CVES", help="Comma separated list of CVEs to ignore when doing CVE autodetection.", default=None)
opter.add_option("--embargoed", help="Include embargoed directory when looking for CVE descriptions", action='store_true')
opter.add_option("--include-eol", help="Include EoL releases", action='store_true')
opter.add_option("--binaries-json", help="Path to JSON mapping of binary packages to versions (can repeat, default: binaries.json)", action='append', default=[])
opter.add_option("--this-only-affected", help="Makes this only affected feature optional", action='store_true')
(opt, args) = opter.parse_args()

if len(args) < 2:
    raise ValueError("Must give USN and all changes files")

if opt.filter_bins:
    opt.filter_bins = re.compile(opt.filter_bins)
filter_dbg = re.compile("-dbg(sym)?$")

# since we can take multiple json files, have a sensile default if none
# are given
if opt.binaries_json == []:
    opt.binaries_json = ['binaries.json']

releasemap = dict()
for rel in cve_lib.releases:
    releasemap['%s-security' % (rel)] = rel

# Put -security first since it tends to have more correctly-validate
# components (-updates has gotten wrecked by things in -proposed before).
release_suffixes = ['','-security','-updates']

config = cve_lib.read_config()
archive = config['packages_mirror']
cve_lib.check_mirror_timestamp(config, mirror='packages_mirror')

pool = '%s/pool' % (archive)
dists = '%s/dists' % (archive)
published = 'http://security.ubuntu.com/ubuntu/pool/'
ports     = 'http://ports.ubuntu.com/pool/'

bin_locations = dict()
src_locations = dict()

def debug(message):
    if opt.debug:
        print("DEBUG: %s" % message, file=sys.stderr)

if not os.path.exists(dists):
    print("Could not find '%s'" % (dists), file=sys.stderr)
    sys.exit(1)

for release in releasemap.values():
    if release in cve_lib.eol_releases and not opt.include_eol:
        debug("skipping EOL release '%s'" % (release))
        continue

    def scan_packages(locations, component, release, arch, name="Packages", di=False):
        '''Used to locate which component a given source package and
           related binary packages live'''
        archloc = arch
        if archloc != 'source':
            archloc = 'binary-%s' % (arch)
        # It seems that the d-i files have a separate name-space ('-di_*.udeb')
        # so it is safe to merge them together on a per-release basis.
        if di:
            dipath = 'debian-installer/'
        else:
            dipath = ""
        filename = '%s/%s/%s/%s%s/%s' % (dists, release, component, dipath, archloc, name)
        debug("Loading %s" % (filename))
        if os.path.exists(filename):
            handle = open(filename)
        else:
            gz_filename = filename + ".gz"
            if os.path.exists(gz_filename):
                handle = tempfile.TemporaryFile()
                handle.write(gzip.open(gz_filename).read())
                handle.seek(0)
            else:
                debug("WARN: %s does not exist" % (gz_filename))
                xz_filename = filename + ".xz"
                if not os.path.exists(xz_filename):
                    return
                    debug("WARN: %s does not exist" % (xz_filename))
                handle = tempfile.TemporaryFile()
                handle.write(lzma.open(xz_filename).read())
                handle.seek(0)
        #print("INFO: loading %s" % (filename), file=sys.stderr)
        parser = apt_pkg.TagFile(handle)
        locations.setdefault(release, dict())
        locations[release].setdefault(arch, dict())
        while True:
            parser.step()
            s = parser.section
            if not s:
                break
            pkg = s['Package']
            pkgcomponent = component
            try:
                if '/' in s['Section']:
                    pkgcomponent = s['Section'].split('/')[0]
            except:
                pass
                #print("%s(%s): " % (filename, pkg) + " ".join(s.keys()), file=sys.stderr)
                #raise
            if pkg in locations[release][arch]:
                # This is very noisy as NBS (not built from source) packages
                # are not autocleaned. This can happen for any number of
                # reasons. See LP: #988017 for details.
                #msg = "DEBUG: %s/%s/%s has multiple entries for '%s' (arch='%s')" % (release, pkgcomponent, archloc, pkg, s['Architecture'])
                #print(msg, file=sys.stderr)
                pass
            locations[release][arch].setdefault(pkg, pkgcomponent)

    for suffix in release_suffixes:
        for component in cve_lib.components:
            for arch in cve_lib.official_architectures:
                if arch == 'source' or arch == 'all':
                    continue
                scan_packages(bin_locations, component, release+suffix, arch)
                scan_packages(bin_locations, component, release+suffix, arch, di=True)
            scan_packages(src_locations, component, release+suffix, "source", name="Sources")

def open_gpg_clearsign(path):
    '''Open a GPG-signed .changes file and strip off the GPG envelope, so that
    it becomes suitable for apt_pkg parsing. Returns a file object.'''

    dest = tempfile.TemporaryFile(mode='w+t')
    f = open(path)

    line = f.readline()
    clearsigned = False
    if line.startswith('-----BEGIN PGP SIGNED MESSAGE-----'):
        clearsigned = True
        for line in f:
            if line.strip() == '':
                break
    else:
        dest.write(line)

    for line in f:
        if clearsigned and line.find('BEGIN PGP SIGNATURE') >= 0:
            break
        dest.write(line)

    f.close()
    dest.seek(0)
    return dest

def check_cve_priority(cve):
    global opt
    cve_data = None
    states = ['active', 'retired']
    if opt.embargoed:
        states.append("embargoed")
    for state in states:
        cve_path = os.path.join(os.environ['UCT'], state, cve)
        if os.path.exists(cve_path):
            try:
                cve_data = cve_lib.load_cve(cve_path)
                break
            except:
                print("ERROR: {} has an unknown Priority".format(cve), file=sys.stderr)
                sys.exit(1)

    if cve_data is None:
        print("ERROR: {} does not exist in UCT in either {}".format(cve, state),
              file=sys.stderr)
        sys.exit(1)
    if cve_data['Priority'] in ['untriaged', 'not-for-us']:
        print("ERROR: {} has Priority {}".format(cve, cve_data['Priority']),
              file=sys.stderr)
        sys.exit(1)

def parse_CVEs(text, cvelines):
    '''Return a set of all CVEs mentioned in the given text.'''

    # Looks for all CVE/CAN mentions
    result = set([])
    # Removing lines from debian/patches, so avoiding two checks for
    # CVE number and bugs.
    text = ' '.join(text.split('/CVE'))
    cvere = re.compile("((?:CVE|cve)-\d\d\d\d-\d{4,7})")
    for cve in cvere.finditer(text):
        cve_number = cve.group().upper()
        check_cve_priority(cve_number)
        result.add(cve_number)

    # Record the lines the mentions came from
    for line in text.splitlines():
        line = line.rstrip()
        for cve in result:
            if cve in line:
                cvelines.setdefault(cve, [])
                cvelines[cve].append(line)

    return result

def find_archive_path_one_release(source, release, arch, filename, fuzzy=None):
    '''Try to find the local archive path for a given .deb filename, release, and architecture'''
    # When performing filename analysis for the component-match,
    # allow for a different filename (for handling guesses at NEW names).
    # This is distinct from "filename" so that the resulting archive path
    # contains the actual expected filename (we just do bin_location[]
    # look-ups with the "fuzzy" filename.
    if not fuzzy:
        fuzzy = filename

    if source.startswith('lib'):
        prefix = source[:4]
    else:
        prefix = source[:1]

    try:
        bin, version, archdeb = fuzzy.split('_',2)
    except:
        # non-deb: stored in the source package's location
        try:
            base = os.path.join(src_locations[release][arch][source], prefix, source, filename)
        except KeyError as e:
            raise KeyError("cannot find %s path for %s %s/%s/%s" % (arch, release, prefix, source, filename))
        filepath = os.path.join(pool, base)
        return filepath, base

    # deb: stored in the binary package's location
    if arch == 'all':
        arch = 'i386'
    try:
        base = os.path.join(bin_locations[release][arch][bin], prefix, source, filename)
    except KeyError as e:
        # strip is used here to get rid of enclosing single quotes
        failure = str(e).strip("'")
        if failure == release:
            raise KeyError("cannot find release '%s'" % (release))
        elif failure == arch:
            raise KeyError("cannot find arch '%s' in release '%s'" % (arch, release))
        elif failure == bin:
            raise KeyError("cannot find binary '%s' (arch '%s') in release '%s'" % (bin, arch, release))
        raise KeyError("unknown KeyError '%s'" % (failure))
    filepath = os.path.join(pool, base)
    return filepath, base

def find_archive_path_each_release(source, release, arch, filename, fuzzy=None):
    '''Try each release to find an archive path'''
    msgs = []
    for suffix in release_suffixes:
        try:
            filepath, base = find_archive_path_one_release(source, release+suffix, arch, filename, fuzzy)
            return filepath, base
        except KeyError as e:
            msgs += [str(e)]
            pass
    raise KeyError("\n".join(msgs))

def find_archive_path(source, release, arch, filename):
    try:
        filepath, base = find_archive_path_each_release(source, release, arch, filename)
    except KeyError as e:
        # Try to guess for an incremented NEW binary name
        found = False
        try:
            if filename.endswith('deb'):
                debug("missing: " + filename)
                bin, version, archdeb = filename.split('_',2)
                # skip leading numbers (i.e. the kernel version)
                match = re.search(r'[^\d]+(?:-[\d\.]+)?-(\d+)', bin)
                if not match:
                    # Handle incrementing libraries (i.e. libdnsXX)
                    match = re.search(r'[^\d]+(\d+)', bin)
                if match:
                    debug("guessing abi: " + match.group(1))
                    inc = int(match.group(1)) - 1
                    while not found and inc > 0:
                        # Attempt to locate a binary with a similar name.
                        # I.e. if it ends with a number, find the same filename
                        # but with the number part reduced by 1.
                        newbin = match.string[:match.start(1)] + '%d' % (inc) + match.string[match.end(1):]
                        debug("trying: " + newbin)
                        fuzzy_filename = "%s_%s_%s" % (newbin, version, archdeb)
                        try:
                            filepath, base = find_archive_path_each_release(source, release, arch, filename, fuzzy_filename)
                            found = True
                            if not opt.no_new_warn:
                                print("WARN: found '%s' in place of NEW '%s'" % (newbin, bin), file=sys.stderr)
                        except:
                            inc -= 1
                            pass
        except:
            pass
        if not found:
            if opt.no_new_warn:
                print("INFO: new binary '%s' in release '%s', is packages_mirror up to date?" % (filename,release), file=sys.stderr)
            else:
                print("ERROR: new binary '%s' in release '%s', is packages_mirror up to date?" % (filename,release), file=sys.stderr)
                raise e
            return '', ''

    return filepath, base

# return True if the passed origin archive or ppa is one of the ones
# that we publish packages to the ubuntu archive from.
def is_origin_ppa_for_archive(origin):
    default_ppas = [
        'ubuntu',  # ubuntu archive itself
        "~ubuntu-security/ubuntu/ppa",  # private security ppa
        "~ubuntu-security-proposed/ubuntu/ppa",  # public security-proposed ppa
        "~ubuntu-mozilla-security/ubuntu/ppa",  # public security ppa for Mozilla updates
    ]
    return origin in default_ppas


# Determine where to expect the files to live (archive vs ports)
def base_to_url(base, release, arch):
    if arch in cve_lib.ports_architectures:
        return os.path.join(ports, base)
    if arch in ['powerpc','sparc'] and release not in ['dapper','gutsy']:
        return os.path.join(ports,base)
    return os.path.join(published,base)

# This routine isn't useful for finding the expected download locations of debs
def lp_find_url(source, srcver, filename, release, arch):
    try:
        bin, version, archdeb = filename.split('_',2)
    except:
        return 'https://launchpad.net/ubuntu/%s/+source/%s/%s/+files/%s' % (release, source, srcver, filename)
    return 'https://launchpad.net/ubuntu/%s/%s/%s/%s' % (release, arch, bin, version)

def find_url(source, srcver, filename, release, arch):
    '''Try to find the archive URL for a given .deb filename'''

    debug('finding url for "%s/%s/%s"' % (source, filename, release))
    # For ESM there is no files in archive
    if re.match("^.*[+~]esm[0-9]*$", srcver):
        return ""

    filepath, base = find_archive_path(source, release, arch, filename)
    debug("found: %s" % (base))

    return base_to_url(base, release, arch)

def scan_dsc_for_orig(filename):
    '''Parse a .dsc file for orig size and sum'''
    parser = apt_pkg.TagFile(open_gpg_clearsign(filename))
    parser.step()
    s = parser.section

    for line in s['Files'].splitlines():
        sum, size, filename = line.strip().split()
        # TODO: adjust for multiple source tarballs as seen in:
        # http://wiki.debian.org/Projects/DebSrc3.0
        for type in ('gz', 'bz2', 'lzma', 'xz'):
            if filename.endswith('tar.' + type) and not filename.endswith('debian.tar.' + type):
                return size, sum, filename
    raise ValueError("Cannot find source tar.* file in %s" % (filename))

def parse_changes(changes, info, cveset, cvelines):
    '''Parse a .changes file, insert package information into the given info
    dictionary, and add the parsed CVEs to cveset.

    info has the following structure:

     release -> srcpkgname -> 'version' -> version,
                           -> 'binaries' -> set(),
                           -> 'arch' -> {arch -> {files -> (size,md5,url)}},
                           -> 'description' -> description
    '''

    # locate arch based on changes filename
    changes_arch = changes.split('_').pop().split('.')[0]
    if changes_arch not in cve_lib.official_architectures:
        return

    # parse .changes files into map s
    try:
        parser = apt_pkg.TagFile(open_gpg_clearsign(changes))
        parser.step()
        s = parser.section
    except SystemError:
        print('ERROR: could not parse', changes, file=sys.stderr)
        raise

    release = s['Distribution']
    if '-' in release:
        # This is a hack to pretend that changes from -proposed are
        # really coming through security. If we change releasemap
        # directly, we explode since it will pull multiple package
        # lists for the same release.
        if release.endswith('-proposed'):
            release = release.replace('-proposed','-security')
        release = releasemap[release]
    srcinfo = info.setdefault(release, {}).setdefault(s['Source'],{})
    srcinfo.setdefault('version', s['Version'])

    # Use hard-coded descriptions, or use the description for the binary
    # package name that matches the source package name, and if can't find
    # it, use # the first one.
    desc = cve_lib.lookup_package_override_description(s['Source'])

    if not desc:
        # source only uploads no longer include binary package
        # descriptions in their source.changes file
        if 'Description' in s:
            for line in s['Description'].split('\n'):
                tmp_pkg = line.split(' - ')[0].strip()
                tmp_desc = " - ".join(line.split(' - ')[1:])
                if s['Source'] == tmp_pkg:
                    desc = tmp_desc
                    break
                elif not desc or desc == '':
                    desc = tmp_desc
    srcinfo.setdefault('description', desc)

    # Add binaries
    try:
        srcinfo.setdefault('binaries', set()).update(set(s['Binary'].split()))
    except:
        # this is intentional that _source.changes will no longer
        # contain the Binary field because of debian bug 818618
        if not changes.endswith('_source.changes'):
            raise

    # Add files
    seen_tar = False
    for line in s['Files'].splitlines():
        sum, size, category, priority, filename = line.strip().split()
        debug('examining "%s"' % filename)
        # Skip 'raw-*' files per infinity. 'raw-uefi' are used for UEFI
        # signing, etc. and aren't published. This also should skip the
        # translations files, too
        if category.startswith('raw-'):
            debug('skipping "%s" -- raw category' % filename)
            continue
        # Skip translation bundles
        if filename.endswith('_translations.tar.gz'):
            debug('skipping "%s" -- translations bundle' % filename)
            continue
        # Skip .ddeb files
        if filename.endswith('.ddeb'):
            debug('skipping "%s" -- ddeb' % filename)
            continue
        # Skip .udeb files
        if filename.endswith('.udeb'):
            debug('skipping "%s" -- udeb' % filename)
            continue
        # Skip buildinfo files
        if filename.endswith('.buildinfo'):
            debug('skipping "%s" -- buildinfo' % filename)
            continue
        if filename.endswith('deb'):
            name, version, arch = filename.split('_')
            arch, deb = arch.split('.')
        else:
            arch = 'source'
            # TODO: adjust for multiple source tarballs as seen in:
            # http://wiki.debian.org/Projects/DebSrc3.0
            for type in ('gz', 'bz2', 'lzma'):
                if filename.endswith('tar.' + type) and not filename.endswith('debian.tar.' + type):
                    seen_tar = True
        files = srcinfo.setdefault('arch', {}).setdefault(arch,{})
        files.setdefault(filename, (size, sum, find_url(s['Source'], s['Version'], filename, release, arch)))

    # Special case for adding missing source tar.* files
    if not seen_tar and changes.endswith('_source.changes'):
        try:
            version = s['Version']
            if ':' in version and not version.endswith(':'):
                # strip out epoch, if it exists
                version = version[(version.find(':')+1):]
            size, sum, filename = scan_dsc_for_orig('%s_%s.dsc' % (s['Source'], version))
        except:
            print("source tar.* not found in changes files, please specific dsc file with --dsc", file=sys.stderr)
            raise

        # Build URL
        arch = "source"
        url = find_url(s['Source'], s['Version'], filename, release, arch)

        files = srcinfo.setdefault('arch', {}).setdefault(arch,{})
        files.setdefault(filename, (size,sum,url))

    # update CVE set
    if changes.endswith('_source.changes'):
        cveset.update(parse_CVEs(s['Changes'], cvelines))


info = {}
CVEs = set([])
cvelines = dict()

usn = args[0]

for changes in args[1:]:
    parse_changes(changes, info, CVEs, cvelines)

# dict is [source][release][binary] => {'version': version, 'pocket': pocket, 'origin': originating archive or ppa}
# see sis-changes::load_pkg_details_from_lp() comment near the bottom of
# the function
binaries = dict()

def merge_json(binaries, json_dict):
    for source in json_dict:
        if not source in binaries:
            binaries[source] = json_dict[source]
        else:
            for release in json_dict[source]:
                if release not in binaries[source]:
                    binaries[source][release] = json_dict[source][release]
                else:
                    raise ValueError("source package %s for release %s is in multiple json files" % (source, release), file=sys.stderr)

    return binaries

# don't try and catch this exception if binaries.json is not available for
# now since we want this to be noisy
for json_file in opt.binaries_json:
    with open(json_file) as fp:
        json_dict = json.load(fp)
        binaries = merge_json(binaries, json_dict)


if opt.ignore_cves:
    CVEs.difference_update(set(opt.ignore_cves.split(',')))

if opt.cves:
    superset = set(opt.cves.split(','))
    if not superset.issuperset(CVEs):
        unexpected = CVEs.difference(superset)
        msgs = ["CVEs found in changelog but not command line: %s" % (" ".join(unexpected))]
        for cve in unexpected:
            msgs.append("\n".join(cvelines[cve]))
        raise ValueError("\n".join(msgs))
    CVEs = superset

if opt.add_cves:
    extra_set = set(opt.add_cves.split(','))
    CVEs.update(extra_set)

local_pickle = os.path.join(config['usn_storage'], '$USN.pickle')
print('#!/bin/bash')
print('set -e')
print('export PATH=$PATH:%s' % (config['usn_tool']))
print('export USN=%s'%(usn))
print('export DB="%s"'%(local_pickle))
print('#export HIDDEN=True')
print('#export TIMESTAMP=')
print('umask 0002')
print()
print('# check for known command-line arguments')
print('FORCE=""')
print('case $@ in')
print('    --force)')
print('    FORCE="--force"')
print('    ;;')
print('    "")')
print('    ;;')
print('    *)')
print('    echo Ignoring unknown command-line argument "$@"...')
print('    ;;')
print('esac')
print()
print('echo Recording USN-$USN ...')
print('# Wipe out the old one -- we want to create a fresh one')
print('rm -f %s'%(local_pickle))
print('mkdir -p %s || true'%(config['usn_storage']))
print()

possible_regression = False
if len(CVEs):
    print('usn.py --db "$DB" $USN --cve ' + " --cve ".join(sorted(CVEs)))
    print('# Can also specify a URL with --cve "https://launchpad.net/bugs/XXXXXX"')
else:
    possible_regression = True
    print('# XXX FIX ME XXX No CVEs found!  Please include URL-based reference')
    print('usn.py --db "$DB" $USN --cve "https://launchpad.net/bugs/XXXXXX"')
print()
print('test "${HIDDEN,,}" = "true" && usn.py --db "$DB" --hidden "$HIDDEN" "$USN"')
print('test -n "${TIMESTAMP}" && usn.py --db "$DB" --timestamp "$TIMESTAMP" "$USN"')

# Is this an updated CVE?
addition = False
origin = usn
if not usn.endswith('-1'):
    addition = True
    origin = usn.split('-')[0] + "-1"

def is_lts_kernel(source):
    return source.startswith('linux-lts-') or source.startswith('linux-hwe')

srcs = set()
for release in info.keys():
    srcs.update(set(info[release].keys()))

titles = []
if opt.kernel_mode:
    for kernel_source in sorted(srcs):
        kernel_title = cve_lib.lookup_package_override_title(kernel_source)
        if kernel_title:
            titles.append(kernel_title)
        else:
            titles.append("XXX Unknown kernel")
else:
    for package_source in sorted(srcs):
        package_title = cve_lib.lookup_package_override_title(package_source)
        if package_title:
            if package_title in titles:
                break
            titles.append(package_title)
        else:
            titles.append(package_source)
    if not titles:
        titles = sorted(srcs)

title = ", ".join(titles)
app_title = title
summary = ", ".join(sorted(srcs))

if len(CVEs)==1:
    title += " vulnerability"
    summary += " vulnerability"
else:
    title += " vulnerabilities"
    summary += " vulnerabilities"

usn_db_tool = [os.path.join(config['usn_tool'],"usn.py"),'--db',config['usn_db_copy']]
if addition and possible_regression:
    # Fetch original CVE details
    title = subprocess.Popen(usn_db_tool + ['--show-title',origin], stdout=subprocess.PIPE).communicate()[0].decode("utf-8").rstrip()
    summary = subprocess.Popen(usn_db_tool + ['--show-summary',origin], stdout=subprocess.PIPE).communicate()[0].decode("utf-8").rstrip()
if addition:
    description = subprocess.Popen(usn_db_tool + ['--show-description',origin], stdout=subprocess.PIPE).communicate()[0].decode("utf-8").rstrip()
    action = subprocess.Popen(usn_db_tool + ['--show-action',origin], stdout=subprocess.PIPE).communicate()[0].decode("utf-8").rstrip()
    isummary = subprocess.Popen(usn_db_tool + ['--show-isummary',origin], stdout=subprocess.PIPE).communicate()[0].decode("utf-8").rstrip()

releases = []
for release in releasemap.values():
    if release in info:
        releases += [release]
releases.sort()

# Building a release filter list with the current info regarding if it is
# esm or not so we can properly filter in it in pull-usn-desc.py --releases
releases_filter = []
srcs_list =[]
if opt.this_only_affected:
    for pkg in binaries.keys():
        for release in  binaries[pkg].keys():
            for _pkg in binaries[pkg][release].keys():
                pocket = binaries[pkg][release][_pkg]['pocket']
                if 'esm' in pocket:
                    releases_filter.append(pocket + '/' + release)
                else:
                    releases_filter.append(release)
                # We just need one info as it reflects the release so skip all
                # pkgs into binaries
                break

    for src in srcs:
        srcs_list.append(src)

print('# title: used for Email Subject, Web title. XXX-EXPAND-TO-UPSTREAM-NAME-XXX')
print('# summary: used inside USN, should be package names')
print('usn.py --db "$DB" $USN --title "%s" --summary "%s" --description - <<EOM' % (title,summary))
print('XXX FILL ME IN: Detailed summary for admins XXX')
if opt.esm:
    print('')
    texting = "a vulnerability"
    if len(CVEs) > 1:
        texting = "several vulnerabilities"
    print('USN-%s-1 fixed %s in %s. This update provides' % (origin.split('-')[0], texting, title.split(' ')[0]))
    print('the corresponding update for Ubuntu XXX ESM.')
if addition and opt.kernel_mode and not ([x for x in srcs if is_lts_kernel(x)] == []):
    print('')
    print('USN-%s fixed vulnerabilities in the Linux kernel for Ubuntu XXX.XXX.' % origin)
    print('This update provides the corresponding updates for the Linux')
    print('Hardware Enablement (HWE) kernel from Ubuntu XXX.XXX for Ubuntu')
    print('XXX.XXX LTS.')
if len(CVEs):
    pull_usn_cmd = ['%s/scripts/pull-usn-desc.py' % os.environ['UCT'], '--prioritize']

    if opt.this_only_affected:
        # We need to pass all releases we are publishing in order to filter it
        # properly in the __this_only_affected__ feature.
        for release in releases_filter:
            pull_usn_cmd.append('--releases=%s' % release)
        for src in srcs_list:
            pull_usn_cmd.append('--src=%s' % src)

        pull_usn_cmd.append('--this-only-affected')
    else:
        pull_usn_cmd.append('--releases=%s' % release)

    if opt.embargoed:
        pull_usn_cmd.append('--embargoed')
    print(subprocess.Popen(pull_usn_cmd + sorted(CVEs), stdout=subprocess.PIPE).communicate()[0].decode("utf-8"))
if not opt.kernel_mode:
    print('XXX IF COMPILER PROTECTED XXX')
    print('The default compiler options for affected releases should reduce the')
    print('vulnerability to a denial of service.')
    print('XXX OR IF APPARMOR PROTECTED XXX')
    print('In the default installation, attackers would be isolated by the')
    print('XXX-APP-XXX AppArmor profile.')
if addition:
    print('XXX OR XXX')
    print('USN-%s fixed vulnerabilities in XXX-APP-XXX. XXX FILL ME IN XXX' % origin)
    print('This update fixes the problem.')
    print('  XXX OR XXX')
    print('This update provides the corresponding updates for XXX-APP-XXX.')
    print('')
    print('XXX We apologize for the inconvenience.')
    print('')
    print('Original advisory details:')
    print('')
    print(' ' + description.replace("\n","\n "))
print('EOM')
print()

print('usn.py --db "$DB" $USN --issue-summary - <<EOM')
if opt.kernel_mode and len(CVEs) > 1:
    print('Several security issues were fixed in the Linux kernel.')
else:
    if addition:
        print('XXX original summary - use it or remove it as your needs XXX')
        print(isummary)
        print('')

    print('XXX FILL ME IN: Summary for regular (non-admin) users XXX')
    print('')
    print('XXX LOCAL TEMPLATES XXX')
    print('%s could be made to crash or run programs as your login if it' % app_title)
    print('opened a specially crafted file.')
    print('XXX OR XXX')
    print('%s could be made to crash or run programs as an administrator' % app_title)
    print('if it opened a specially crafted file.')
    print('XXX OR XXX')
    print('%s could be made to crash if it opened a specially crafted' % app_title)
    print('file.')
    print('XXX OR XXX')
    print('%s could be made to crash if it received specially crafted' % app_title)
    print('input.')
    print('XXX OR XXX')
    print('%s could be made to overwrite files as the administrator.' % app_title)
    print('XXX OR XXX')
    print('%s could be made to overwrite files.' % app_title)
    print('')
    print('XXX NETWORK TEMPLATES XXX')
    print('%s could be made to crash or run programs as your login if it' % app_title)
    print('opened a malicious website.')
    print('XXX OR XXX')
    print('%s could be made to crash or run programs if it received' % app_title)
    print('specially crafted network traffic.')
    print('XXX OR XXX')
    print('%s could be made to crash or run programs if it received' % app_title)
    print('specially crafted network traffic from an authenticated user.')
    print('XXX OR XXX')
    print('%s would allow unintended access to files over the network.' % app_title)
    print('XXX OR XXX')
    print('%s could be made to expose sensitive information over the' % app_title)
    print('network.')
    print('XXX OR XXX')
    print('%s could allow unintended access to network services.' % app_title)
    print('XXX OR XXX')
    print('Fraudulent security certificates could allow sensitive information to')
    print('be exposed when accessing the Internet.')
    print('')
    print('XXX KERNEL/PLUMBING TEMPLATES XXX')
    print('The system could be made to crash under certain conditions.')
    print('XXX OR XXX')
    print('The system could be made to run programs as an administrator.')
    print('XXX OR XXX')
    print('The system could be made to crash or run programs as an administrator.')
    print('XXX OR XXX')
    print('The system could be made to crash if it received specially crafted')
    print('network traffic.')
    print('XXX OR XXX')
    print('The system could be made to expose sensitive information.')
    print('XXX OR XXX')
    print('A system hardening measure could be bypassed.')
    print('')
    print('XXX GENERIC TEMPLATES XXX')
    print('Several security issues were fixed in %s.' % app_title)

print('EOM')
print()


print('# Source descriptions for source packages. Defaults to what is found')
print('# in debian/control, but should be adjusted for readability. Templates')
print('# might only use one per source package, so in general it is best to')
print('# keep descriptions for the same source package across different')
print('# releases the same. XXX-CHECK-XXX')
if opt.kernel_mode:
    print('')
    print('# XXX for kernel updates delete meta and signed sources here')
for release in releases:
    for source in sorted(info[release].keys()):
        description = info[release][source]['description']
        print('usn.py --db "$DB" $USN --release %s --package %s --source-description \'%s\'' % (release,source,description))
print()

print('usn.py --db "$DB" $USN --action - <<EOM')
if addition:
    print(action)
    print('XXX OR XXX')
if opt.kernel_mode:
    print('After a standard system update you need to reboot your computer to make')
    print('all the necessary changes.')
    print('XXX MAYBE WITH XXX')
    print('''ATTENTION: Due to an unavoidable ABI change the kernel updates have
been given a new version number, which requires you to recompile and
reinstall all third party kernel modules you might have installed.
Unless you manually uninstalled the standard kernel metapackages
(e.g. linux-generic, linux-generic-lts-RELEASE, linux-virtual,
linux-powerpc), a standard system upgrade will automatically perform
this as well.''')
else:
    print('In general, a standard system update will make all the necessary changes.')
    print('XXX OR XXX')
    print('After a standard system update you need to restart XXX-APP-XXX to make')
    print('all the necessary changes.')
    print('XXX OR XXX')
    print('After a standard system update you need to reboot your computer to make')
    print('all the necessary changes.')
    print('XXX OR IF MICROVERSIONEXCEPTION XXX')
    print('This update uses a new upstream release, which includes additional bug')
    print('fixes. In general, a standard system update will make all the necessary')
    print('changes.')
    print('XXX OR IF MICROVERSIONEXCEPTION XXX')
    print('This update uses a new upstream release, which includes additional bug')
    print('fixes. After a standard system update you need to restart XXX-APP-XXX to')
    print('make all the necessary changes.')

print('EOM')
print()


# Sources
if opt.kernel_mode:
    print('# XXX For kernel updates, remove the -meta and -signed source package')
for release in releases:
    for source in sorted(info[release].keys()):
        version = info[release][source]['version']
        print('usn.py --db "$DB" $USN --release %s --package %s --source-version %s' % (release,source,version))
print()

# Affected Binaries
for release in releases:
    print('# Reduce to minimum binaries affected in %s' % (release))
    for source in sorted(info[release].keys()):
        version = info[release][source]['version']
        for deb in sorted(info[release][source]['binaries']):
            # binary packages can have different versions than their source
            # package, so override the source package version with the
            # binary package version if we have this available
            #
            # also some changes files include binaries that aren't built
            # (looking at you kernel packages) so if we have binaries
            # info from json files, then we have an accurate listing of
            # binary packages, so don't emit bad ones.
            if source in binaries and release in binaries[source] and deb in binaries[source][release]:
                try:
                    version = binaries[source][release][deb]['version']
                except KeyError as e:
                    pass
                if not filter_dbg.search(deb) and (not opt.filter_bins or opt.filter_bins.search(deb)):
                    pocket = ""
                    if binaries[source][release][deb]['pocket']:
                        pocket = "--pocket %s" % binaries[source][release][deb]['pocket']
                    elif is_origin_ppa_for_archive(binaries[source][release][deb]['origin']):
                        pocket = "--pocket security"
                    print('  usn.py --db "$DB" $USN --release %s --package %s --binary-version %s %s' % (release, deb, version, pocket))
        print()

# URLs
print('# URLs')
print('echo Recording URLs ...')
for release in releases:
    if cve_lib.is_active_esm_release(release):
        # Don't record URLs for ESM releases since they won't be valid links
        continue

    for source in sorted(info[release].keys()):
        for arch in cve_lib.official_architectures:
            if arch not in info[release][source]['arch']:
                continue
            for name in info[release][source]['arch'][arch]:
                size, md5, url = info[release][source]['arch'][arch][name]
                if size and md5 and url:
                    print('usn.py --db "$DB" $USN --release %s --arch %s --url %s --url-size %s --url-md5 %s' % (release, arch, url, size, md5))
                else:
                    print('# No valid URL found for %s %s' % (release, arch))
            print()
        print()
    print()

# Complete List of Binaries - place this after URLs above so is less likely to
# get modified and hence any binaries missed from the list
for release in releases:
    print('## DO NOT MODIFY - this is a complete list of all binaries for landscape etc')
    for source in sorted(info[release].keys()):
        version = info[release][source]['version']
        for deb in sorted(info[release][source]['binaries']):
            # binary packages can have different versions than their source
            # package, so override the source package version with the
            # binary package version if we have this available
            #
            # also some changes files include binaries that aren't built
            # (looking at you kernel packages) so if we have binaries
            # info from json files, then we have an accurate listing of
            # binary packages, so don't emit bad ones.
            if source in binaries and release in binaries[source] and deb in binaries[source][release]:
                try:
                    version = binaries[source][release][deb]['version']
                except KeyError as e:
                    pass
                if not filter_dbg.search(deb):
                    pocket = ""
                    if binaries[source][release][deb]['pocket']:
                        pocket = "--pocket-all %s" % binaries[source][release][deb]['pocket']
                    elif is_origin_ppa_for_archive(binaries[source][release][deb]['origin']):
                        pocket = "--pocket-all security"
                    print('  usn.py --db "$DB" $USN --release %s --package %s --all-binary-version %s --all-binary-source %s %s'
                          % (release, deb, version, source, pocket))
        print()

print("# Start of non-inclusive and consistent term checks for USN text fields")
print("usn_term_check_script=%s/usn-term-check.py" % config['usn_tool'])
print()
print("# Checking that the current user has updated usn-tool and has the usn-term-check.py script")
print("if [ -f $usn_term_check_script ]; then")
print()
print("    # Temporarily turning off in order to capture a potentially non-zero exit code for usn-term-check.py")
print("    set +e")
print()
print("    # Calling script to run checks against the USN text fields")
print('    usn-term-check.py --db "$DB" --usn $USN --path "$PATH"')
print("    term_check_exit_code=$?")
print()
print("    # Re-setting non-zero status exit option")
print("    set -e")
print()
print("    # If usn-term-check.py returned an exit code of 7, that means there are issues needing to be resolved")
print("    if [ $term_check_exit_code -eq 7 ]; then")
print("        echo")
print('        echo "***"')
print("        echo ATTENTION: Please fix the non-ignored above ISSUES/WARNINGS and re-run this script to proceed.")
print('        echo "***"')
print("        echo")
print()
print("        #Terminating execution of the new-usn bash script to allow user to resolve identified issues")
print("        exit")
print("    # If usn-term-check.py triggers an exception, pause execution to make user aware")
print("    elif [ $term_check_exit_code -ne 0 ]; then")
print("        echo")
print("        echo The above exception occurred during the execution of usn-term-check.py and needs to be addressed.")
print('        read -p "This means that a check for non-inclusive terms may have not been successful, <enter> to ACK and continue: "')
print("    fi")
print()
print("else")
print("    echo Attention: Please update usn-tool in order to include checks for non-inclusive terms!")
print("fi")
print()

print('if [ "${HIDDEN,,}" = "true" ]; then')
print('    "$UCT"/scripts/convert-pickle.py --input-file $DB --output-file $USN.json --prefix USN-')
print('    mkdir -p "$UCT/experimental/usns"')
print('    mv $USN.json "$UCT/experimental/usns/"')
print('    echo ""')
print('    echo This USN is set as HIDDEN so no information about it will be stored')
print('    echo in our database yet. Use the command below to update the website api:')
print('    echo ""')
print('    echo "    \$UCT/scripts/publish-usn-to-website-api.py --action add --json \$UCT/experimental/usns/$USN.json"')
print('    echo ""')
print('    echo IMPORTANT: Keep $USN.json until it becomes public.')
print('    echo ""')
print('    exit')
print('fi')
print('')
# refresh the local db so when we do an import we can catch cases where a
# duplicate USN is used before we go to push it to the remote master DB
print('# Refresh the local DB')
print('echo Refreshing local master USN database ...')
print('pushd %s' % (os.path.dirname(config['usn_db_copy'])))
print('%s/scripts/fetch-db %s.bz2' % (os.environ['UCT'], os.path.basename(config['usn_db_copy'])))
print('popd')

print('# Update the local DB')
print('echo Updating local master USN database ...')
print('usn.py --db "$DB" --export | usn.py --db %s --import $FORCE' % (config['usn_db_copy']))

print('# Check local DB for unfilled templates')
print('%s/scripts/check-unreplaced-templates.py %s $USN' % (os.environ['UCT'], config['usn_db_copy']))

print('# Update the remote DB')
print('echo Updating remote master USN database and sending template email ...')
print("usn.py --db \"$DB\" --export | ssh -C people.canonical.com '/home/ubuntu-security/bin/usn.sh --db /home/ubuntu-security/usn/database-all.pickle --import '\"$FORCE\"' && /home/ubuntu-security/bin/send-usn-email '\"$USN\"")

#import pprint
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(info)