#!/usr/bin/env python3
# Copyright 2007-2017, Canonical, Ltd.
# Author: Kees Cook <kees@ubuntu.com>
#         Jamie Strandboge <jamie@canonical.com>
#         Marc Deslauriers <marc.deslauriers@canonical.com>
# License: GPLv3
#
# Extract/download list of changes file links from a given LP name, pkg, version
#
# TODO: need to handle multiple orig tarballs for 3.0 format
# http://wiki.debian.org/Projects/DebSrc3.0


import copy
import optparse
import os.path
import progressbar
import re
import shutil
import sys
import tempfile
import urllib.request, urllib.error, urllib.parse
import cve_lib
import json
from source_map import version_compare, load

try:
    import lpl_common
except ImportError as e:
    print("lpl_common.py seems to be missing. Please create a symlink from $UQT/common/lpl_common.py to $UCT/scripts/", file=sys.stderr)
    print(e, file=sys.stderr)
    sys.exit(1)


def download(url):
    # Download file to tmpdir
    if not os.path.exists(tmpdir):
        print("Failed: '%s' does not exist" % (tmpdir), file=sys.stderr)
        sys.exit(1)

    # Initialize progressbar so it has a real view of the time taken to fetch
    widgets = [progressbar.Percentage(),
               ' ', progressbar.Bar(marker='=', left='[', right=']'),
               ' ', progressbar.FileTransferSpeed(),
               ' ', progressbar.ETA()]
    bar = progressbar.ProgressBar(widgets=widgets).start()

    # Open the URL
    urlfile = lpl_common.open_url(opener, url)
    received = 0

    # Extract expected file size, updating progress bar and widgets
    size = int(urlfile.info().get('Content-Length').strip())
    bar.maxval = size
    widgets.insert(1, ' of %d' % (size))
    bar.widgets = widgets
    bar.update(received)

    # See 'if opt.action == changes' section when adding replace() characters
    name = urllib.parse.unquote(os.path.join(tmpdir, os.path.basename(url)))
    try:
        tmp, tmpname = tempfile.mkstemp()
    except Exception:
        raise

    # Fetch data, updating progressbar in minimum 100K chunks
    while True:
        data = urlfile.read(1024 * 100)
        if not data:
            break
        received += len(data)

        os.write(tmp, data)
        bar.update(received)
        if received == size:
            bar.finish()

    # Close and rename
    os.close(tmp)
    shutil.move(tmpname, name)
    return name


def download_url(url):
    '''Display URL, and optionally download it, if requested and matches the re'''
    if url is None:
        print("download_url(): passed an empty url, skipping...", file=sys.stderr)
        return None

    filename = os.path.basename(url)
    if not opt.re or re.search(opt.re, filename):
            print(url)
            if opt.download:
                try:
                    return download(url)
                except urllib.error.HTTPError as e:
                    if e.getcode() == 500 and re.search('/\+sourcefiles/', url):
                        # launchpad devs: "stable apis are for losers"
                        newurl = gen_fallback_lp_url(url)
                        print('Launchpad coughed up a lung, falling back to alternate URL: %s' % newurl, file=sys.stderr)
                        return download(newurl)
                    else:
                        raise e
    return None


def gen_fallback_lp_url(source_url):
    '''hack to work around LP randomly dropping source urls'''

    # replace '.../+sourcefiles/build/../../FILE' with '/+files/FILE'
    filename = os.path.basename(source_url)
    return re.sub('/\+sourcefiles/.*', '/+files/%s' % filename, source_url)

#
# START SCRIPT
#

parser = optparse.OptionParser()
parser.add_option("--action", help="What action to take: 'changes'(default), 'check-build', 'binaries', 'source', 'buildlogs', 'list'", metavar="NAME", action='store', default='changes')
parser.add_option("--ppa", help="Which PPA to use (default is 'ubuntu-security/ppa')", metavar="PERSON[/PPA]", action='store', default='ubuntu-security/ppa')
parser.add_option("--esm-merge-ppa", help="If used will download packages from the given esm-ppa. Used to merge USN templates", action='append', default=None, dest='esm_ppas')
parser.add_option("--pocket", help="Which pocket to use (valid values are: 'Release', 'Security', 'Updates', 'Proposed', 'Backports')", metavar="POCKET", action='store', default=None)
parser.add_option("--superseded-version", help="Version of superseded files", metavar="NAME", action='store')
parser.add_option("--debug", help="Show debug output", action='store_true')
parser.add_option("--verbose", help="Verbose output", action='store_true')
parser.add_option("--uri", help="Use specific URI for API", action='store', default=None, metavar="URI")
parser.add_option("--beta", help="Use beta API instead of 1.0 LP API", action='store_true', default=False)
parser.add_option("-r", "--release", help="Limit to a specific set of comma-separate releases", metavar="SERIES", action='store', default=None)
parser.add_option("--skip-build-check", help="Skip binary package build check", action='store_true', default=False)

# Action-specific options
#   'changes'
parser.add_option("--dsc", help="Toggle fetching source .dsc files (default is True)", action='store_false', default=True)
#   'binaries'
parser.add_option("--arch", help="Limit 'binaries' and 'changes' action to comma-separated list of archs", metavar="ARCH[,ARCH...]", action='store')
parser.add_option("--re", help="When handling binaries, only include those matching this regular expression", metavar="RE", action='store')
parser.add_option("--include-debug", help="When handling binaries, skip .udeb, -dbg, -dbgsym and non-English -locale packages", action='store_true', default=False)
#   'changes', 'binaries', 'source'
parser.add_option("--download", help="Download to DIR", metavar="DIR", action='store', default='')
parser.add_option("--force-download", help="Force download to DIR if it exists (removes old DIR)", action='store_true', default=False)
#   'source'
parser.add_option("--fetch-orig", help="Download the orig.tar.gz when fetching source", action='store_true', default=False)
#   'include-devel'
parser.add_option("--include-devel", help="Include development release", action='store_true', default=False)
parser.add_option("--include-eol", help="Include end of life releases", action='store_true', default=False)
parser.add_option("--distribution", help="Distribution to use (eg, 'ubuntu-rtm')", metavar="DIST", action='store', default=None)

(opt, args) = parser.parse_args()

# Load configuration
cve_lib.read_config()

# API interface
lp = lpl_common.connect(beta=opt.beta, uri=opt.uri)

# Get authenticated URL fetcher
opener = lpl_common.opener_with_cookie(cve_lib.config["plb_authentication"])
if not opener:
    raise ValueError("Could not open cookies")

if len(args) < 1:
    print("Usage: %s [--download <dir>] SRCPKG" % (sys.argv[0]), file=sys.stderr)
    sys.exit(1)

serieses = []
if opt.release:
    for r in opt.release.split(','):
        serieses.append(r.lower())

download_dir = ""
if opt.download:
    if opt.download == '':
        print("Must specify a directory with '--download'", file=sys.stderr)
        sys.exit(1)
    else:
        download_dir = opt.download
        if os.path.exists(download_dir):
            if opt.force_download:
                cve_lib.recursive_rm(download_dir)
            else:
                print("Specified download directory exists:\n %s" % (download_dir), file=sys.stderr)
                print("\nPlease remove (or use --force-download) and try again.", file=sys.stderr)
                sys.exit(1)
        tmpdir = tempfile.mkdtemp(prefix='sis-changes-download-')


# split_package -> pkg_name, arch
def split_package(pkg):
    tmp = pkg.split('_')
    arch = tmp[-1].split('.')[0]
    pkg_name = tmp[-3].split('/')[-1]
    return (pkg_name, arch)


def get_arch_from_dsc(dsc):
    f = open(dsc, 'r')
    for line in f:
        vals = line.split(':')
        if len(vals) == 2 and vals[0] == 'Architecture':
            f.close()
            return vals[1].strip()
    f.close()
    return None


# pkg -> { release, release -> { version } }
def load_pkg_details_from_lp(archive, pkgs, binaries, pkg, item):

    rel = item.distro_series.name
    if opt.debug:
        print("Processing %s" % (rel), file=sys.stderr)
    if opt.distribution is None and rel not in cve_lib.releases:
        raise ValueError("Unknown release '%s':\n" % (rel))

    if serieses and rel not in serieses:
        if opt.debug:
            print("Skipping %s: not in %s" % (rel, serieses), file=sys.stderr)
        return

    version = item.source_package_version

    # This check is *probably* not needed, but it is not easy to verify,
    # so keep it.
    if opt.superseded_version and version != opt.superseded_version:
        print("Skipping %s: %s %s (we need %s)" % (rel, pkg, version, opt.superseded_version), file=sys.stderr)
        return

    if pkg in pkgs and rel in pkgs[pkg]:
        state = version_compare(version, pkgs[pkg][rel]['source']['version'])
        if state < 0:
            print("Skipping %s: %s %s (already have %s)" % (rel, pkg, version, pkgs[pkg][rel]['source']['version']), file=sys.stderr)
            return
        elif state == 0:
            if opt.verbose:
                print("Skipping %s: %s %s (same as %s)" % (rel, pkg, version, pkgs[pkg][rel]['source']['version']), file=sys.stderr)
            return
        else:
            print("Forgetting %s: %s %s (now have %s)" % (rel, pkg, pkgs[pkg][rel]['source']['version'], version), file=sys.stderr)
            pkgs[pkg][rel] = dict()
    pkgs.setdefault(pkg, dict())
    pkgs[pkg].setdefault(rel, dict())
    if opt.debug:
        print("Source(%s): %s %s" % (rel, pkg, version), file=sys.stderr)

    # Source details
    pkgs[pkg][rel].setdefault('source', dict())
    pkgs[pkg][rel]['source'].setdefault('version', version)

    # Handle transition to method (LP: #474876)
    if hasattr(item, 'changes_file_url'):
        src_changes = item.changes_file_url
    else:
        src_changes = item.changesFileUrl()

    pkgs[pkg][rel]['source'].setdefault('changes', src_changes)
    if opt.debug:
        print("Source(%s) changes: %s" % (rel, src_changes), file=sys.stderr)

    # Get per-build items
    builds = item.getBuilds()
    # If we didn't find a build, we're in trouble.
    # This can happen if something was pocket-copied from a different release
    # First, see if we can find the builds published in another release.
    # See LP: #783613
    if len(builds) == 0:
        print("Warning: no builds found for source(%s): %s %s" % (rel, pkg, version), file=sys.stderr)
        params = dict(
            source_name=pkg,
            exact_match=True,
            status="Published",
            version = item.source_package_version,
        )
        others = archive.getPublishedSources(**params)
        other = None
        for other in others:
            other_builds = other.getBuilds()
            if len(other_builds) > 0:
                builds = other_builds
                print("Warning: using builds from (%s/%s) for source(%s): %s %s" % (other.distro_series.name, other.pocket, rel, pkg, version), file=sys.stderr)
                print("=== Double check this is what you want ===", file=sys.stderr)
                break

    if len(builds) == 0:
        raise ValueError("Could not find any builds for %s." % (pkg))

    for build in builds:
        arch = build.arch_tag
        state = build.buildstate
        if opt.debug:
            print("Build(%s,%s) %s" % (rel, arch, state), file=sys.stderr)
        if state == 'Failed to build':
            # don't add records if the build failed
            continue
        elif state == 'Successful build':
            # Work around LP: #559591
            state = 'Successfully built'
        pkgs[pkg][rel].setdefault(arch, dict())
        pkgs[pkg][rel][arch].setdefault('build_state', state)
        bin_changes = build.changesfile_url
        pkgs[pkg][rel][arch].setdefault('changes', bin_changes)
        if opt.debug:
            print("Build(%s,%s) changes: %s" % (rel, arch, bin_changes), file=sys.stderr)
        build_log = build.build_log_url
        pkgs[pkg][rel][arch].setdefault('build_log', build_log)

    # Diff (we don't use this yet...)
    # diff_url = item.packageDiffUrl()
    # pkgs[pkg][rel]['source'].setdefault('ancestor-diff', diff_url)
    # if opt.debug:
    #     print("Diff(%s) URL: %s" % (rel, diff_url), file=sys.stderr)

    # Binary outputs
    # Handle transition to method (LP: #474876)
    if hasattr(item, 'binary_file_url'):
        bin_files = item.binary_file_urls
    else:
        bin_files = item.binaryFileUrls()
    for file_url in bin_files:
        if file_url.endswith('deb'):
            name, arch = split_package(file_url)
            if opt.debug:
                print("Binary(%s,%s) URL: %s" % (rel, arch, file_url), file=sys.stderr)
            # hack for "all": attach to all_arch
            if arch == 'all':
                all_arch = cve_lib.get_all_arch(rel)
                # if only building for one arch that's not the default
                # all arch, the all packages will be built under that arch
                # so check the all_arch has binary pkgs
                archs = [x for x in list(pkgs[pkg][rel].keys()) if x != 'source']
                if all_arch in archs:
                    arch = all_arch
                elif len(archs) == 1:
                    arch = archs[0]
                elif opt.debug:
                    print("Couldn't find 'all' arch for %s in %s/%s" % (name, arch, rel), file=sys.stderr)
            if not cve_lib.arch_is_valid_for_release(arch, rel):
                if opt.debug:
                    print("Skipping %s binary because %s is not a valid arch in %s" % (name, arch, rel), file=sys.stderr)
                continue
            if not arch in pkgs[pkg][rel]:
                # this architecture is not present from the build
                # records, this can happen when pocket copied from a
                # different pocket or release.
                pkgs[pkg][rel].setdefault(arch, dict())
            pkgs[pkg][rel][arch].setdefault('binaries', dict())
            pkgs[pkg][rel][arch]['binaries'].setdefault(name, file_url)
        else:
            raise ValueError("Unknown downloadable binary file from %s %s '%s'" % (pkg, version, file_url))

    # Source inputs
    # Handle transition to method (LP: #474876)
    if hasattr(item, 'source_file_url'):
        src_files = item.source_file_urls
    else:
        src_files = item.sourceFileUrls()
    for file_url in src_files:
        if file_url.endswith('.dsc'):
            pkgs[pkg][rel]['source'].setdefault('dsc', file_url)
            if opt.debug:
                print("Source(%s) dsc URL: %s" % (rel, file_url), file=sys.stderr)
        elif re.search('\.(diff\.gz|debian\.tar\.(gz|bz2|lzma|xz))$', file_url):
            pkgs[pkg][rel]['source'].setdefault('diff', file_url)
            if opt.debug:
                print("Source(%s) debian differences URL: %s" % (rel, file_url), file=sys.stderr)
        elif re.search('\.tar\.(gz|bz2|lzma|xz)$', file_url):
            pkgs[pkg][rel]['source'].setdefault('orig', file_url)
            if opt.debug:
                print("Source(%s) orig URL: %s" % (rel, file_url), file=sys.stderr)
        elif file_url.endswith('.asc'):
            pkgs[pkg][rel]['source'].setdefault('asc', file_url)
            if opt.debug:
                print("Source(%s) asc URL: %s" % (rel, file_url), file=sys.stderr)
        else:
            raise ValueError("Unknown downloadable source file from %s %s '%s'" % (pkg, version, file_url))

    binaries.setdefault(pkg, dict())
    binaries[pkg].setdefault(rel, dict())

    # map binaries as pkg -> rel -> binary -> dict(version, origin, pocket)
    # where:
    #    version = actual version
    #    pocket = main, esm-infra, esm-apps (based on guesswork)
    #    origin = originating archive/ppa name
    # this is used by sis-generate-usn to know what the actual
    # binary package version is since this may be different than
    # the version of the source package. Also check that all built
    # binaries have actually published into the PPA
    # it is also used to determine what the ACTUAL binary packages

    origin = archive.reference
    for binary in item.getPublishedBinaries():
        # collect binary versions

        # skip debian-installer packages, these are udebs we don't want.
        if binary.section_name == 'debian-installer':
            if opt.debug:
                print(
                    "skipping debian-instaler binary file %s from source %s"
                    % (binary.binary_package_name, item.display_name),
                    file=sys.stderr
                )
            continue

        url = ""
        # because a binary can be published for only one arch (and not
        # even the all_arch arch), we need to iterate over all the
        # arches until we find a valid one.
        for arch in cve_lib.official_architectures:
            if not cve_lib.arch_is_valid_for_release(arch, rel) or arch not in pkgs[pkg][rel]:
                continue
            if binary.binary_package_name in pkgs[pkg][rel][arch]['binaries']:
                url = pkgs[pkg][rel][arch]['binaries'][binary.binary_package_name]
                break
        pocket = ""
        for esm_pocket in ["esm-infra", "esm-apps"]:
            if esm_pocket in url:
                pocket = esm_pocket
        binaries[pkg][rel].setdefault(binary.binary_package_name, {
            'origin': origin,
            'pocket': pocket,
            'version': binary.binary_package_version,
        })
        if opt.skip_build_check is True:
            print("WARNING: skipping binary publication check. Please check manually.", file=sys.stderr)
        else:
            if binary.status != 'Published':
                if opt.debug:
                    print("BinaryPublication(%s,%s,%s) state: %s" % (rel, binary.distro_arch_series.architecture_tag, binary.binary_package_name, binary.status), file=sys.stderr)
                arch = binary.distro_arch_series.architecture_tag
                # Override binary target in the case of "all"
                if 'all' in pkgs[pkg][rel]:
                    arch = 'all'
                pkgs[pkg][rel][arch]['build_state'] = 'Binaries pending'


def is_debug_pkg(name):
    if name.endswith("-dbg") or name.endswith("-dbgsym") or name.endswith("-mozsymbols"):
        return True
    if name.endswith("-source") and name.startswith("openjdk-"):
        return True
    if ("-locale-" in name and not name.endswith("-locale-en") and
            (name.startswith("firefox") or name.startswith("thunderbird"))):
        return True


if opt.distribution is None:
    # We could default to this, but it would require changes elsewhere
    # distribution = lp.distributions['ubuntu']
    distribution = opt.distribution
else:
    distribution = lp.distributions[opt.distribution]
archive, group, ppa = lpl_common.get_archive(opt.ppa, lp, opt.debug, distribution=distribution)

# This does a download from both the given --ppa and --esm-ppa
# so we can merge them in an unique USN
# Usage sample: --ppa ubuntu-security-proposed/ppa --esm-ppa ubuntu-security/esm
# --esm-ppa ubuntu-esm/esm-infra-security-staging will download pkg info
# from all these ppa in /tmp/pending.
esm_archives = []
if opt.esm_ppas:
    for _esm_ppa in opt.esm_ppas:
        if _esm_ppa != opt.ppa:
            tmp_esm_archive, _, __ = lpl_common.get_archive(_esm_ppa, lp, opt.debug, distribution=distribution)
            esm_archives.append(tmp_esm_archive)

pkgs = dict()
binaries = dict()
if opt.superseded_version:
    status = "Superseded"
else:
    status = "Published"
for pkg_name in args:
    params = dict(source_name=pkg_name,
                  exact_match=True,
                  status=status)
    if opt.pocket:
        params['pocket'] = opt.pocket
    if opt.superseded_version:
        params['version'] = opt.superseded_version

    for item in archive.getPublishedSources(**params):
        load_pkg_details_from_lp(archive, pkgs, binaries, pkg_name, item)

    # If is esm_archives found so try each one of them to grab pkgs changes info
    if esm_archives:
        for _esm_archive in esm_archives:
            for item in _esm_archive.getPublishedSources(**params):
                load_pkg_details_from_lp(_esm_archive, pkgs, binaries, pkg_name, item)


if opt.action == 'changes':
    for pkg in args:
        if pkg not in pkgs:
            msg = "Source package '%s' not found in group %s PPA %s" % (pkg, group, ppa)
            if opt.pocket:
                msg += " pocket %s" % (opt.pocket)
            raise ValueError(msg)

        for rel in sorted(pkgs[pkg].keys()):
            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            version = pkgs[pkg][rel]['source']['version']
            if ':' in version and not version.endswith(':'):
                # strip out epoch, if it exists
                version = version[(version.find(':') + 1):]

            if opt.debug:
                print("Fetching %s %s ..." % (pkg, version), file=sys.stderr)

            download_url(pkgs[pkg][rel]['source']['changes'])

            archs = sorted(pkgs[pkg][rel].keys())
            if opt.arch:
                archs = archlist = opt.arch.split(',')

            if opt.dsc:
                dsc = download_url(pkgs[pkg][rel]['source']['dsc'])
                dsc_arch = get_arch_from_dsc(dsc)
                if dsc_arch == 'all':
                    archs = [cve_lib.get_all_arch(rel)]
                    print("Skipping non-i386 builds for 'Architecture: all' package %s %s" % (pkg, rel), file=sys.stderr)

            for arch in archs:
                # Ignore 'source' and 'item' for build states
                if arch in ['source', 'item']:
                    continue
                if arch not in pkgs[pkg][rel]:
                    print("Skipping missing %s build for %s %s" % (arch, pkg, rel), file=sys.stderr)
                    continue
                if 'build_state' not in pkgs[pkg][rel][arch]:
                    print("Warning, no build_state for build for %s %s %s" % (pkg, rel, arch), file=sys.stderr)
                    print("(This can happen for pocket copies from other releases", file=sys.stderr)
                elif pkgs[pkg][rel][arch]['build_state'] != 'Successfully built':
                    print("Skipping '%s' build for %s %s %s" % (pkgs[pkg][rel][arch]['build_state'], pkg, rel, arch), file=sys.stderr)
                    continue
                try:
                    download_url(pkgs[pkg][rel][arch]['changes'])
                except urllib.error.HTTPError as e:
                    if e.getcode() == 404:
                        print("Failed to download changes files, is this a copied package? %s" % pkgs[pkg][rel][arch]['changes'], file=sys.stderr)
                    else:
                        raise e
                except KeyError as e:
                    if str(e) == "'changes'":
                        print("Warning, no changes file for %s %s %s" % (pkg, rel, arch), file=sys.stderr)
                    else:
                        raise e


    # write out binaries as json so can be consumed by sis-generate-usn
    with open(os.path.join(tmpdir, "binaries.json"), 'w') as fp:
        json.dump(binaries, fp, indent=2)

elif opt.action == 'binaries':
    for pkg in args:
        if pkg not in pkgs:
            raise ValueError("Source package '%s' not found in PPA" % (pkg))
        for rel in sorted(pkgs[pkg].keys()):
            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            version = pkgs[pkg][rel]['source']['version']
            archlist = sorted(pkgs[pkg][rel].keys())
            if opt.arch:
                archlist = opt.arch.split(',')
            for arch in archlist:
                if arch not in pkgs[pkg][rel]:
                    continue
                if 'binaries' not in pkgs[pkg][rel][arch]:
                    continue
                for name in sorted(pkgs[pkg][rel][arch]['binaries'].keys()):
                    # If --include-debug is not specified, don't download:
                    # -dbg
                    # -dbgsym
                    # -mozsymbols
                    # non-english firefox-locale-*
                    # non-english thunderbird-locale-*
                    # openjdk-*-source
                    # .udeb
                    if not opt.include_debug and \
                            (is_debug_pkg(name) or
                             pkgs[pkg][rel][arch]['binaries'][name].endswith(".udeb")):
                        print("Skipping '%s' (use --include-debug)" % (name), file=sys.stderr)
                        continue
                    download_url(pkgs[pkg][rel][arch]['binaries'][name])

elif opt.action == 'buildlogs':
    for pkg in args:
        if pkg not in pkgs:
            raise ValueError("Source package '%s' not found in PPA" % (pkg))
        for rel in sorted(pkgs[pkg].keys()):
            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            archlist = sorted(pkgs[pkg][rel].keys())
            if opt.arch:
                archlist = opt.arch.split(',')
            for arch in archlist:
                if opt.debug:
                    print("Fetching build log for %s %s %s ..." % (pkg, rel, arch), file=sys.stderr)
                if not cve_lib.arch_is_valid_for_release(arch, rel):
                    continue
                if arch not in pkgs[pkg][rel]:
                    continue
                if 'build_log' not in pkgs[pkg][rel][arch]:
                    continue
                download_url(pkgs[pkg][rel][arch]['build_log'])

elif opt.action == 'list':
    for pkg in sorted(pkgs.keys()):
        print(pkg)

elif opt.action == 'source':
    for pkg in args:
        if pkg not in pkgs:
            raise ValueError("Source package '%s' not found in PPA" % (pkg))
        for rel in sorted(pkgs[pkg].keys()):
            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            version = pkgs[pkg][rel]['source']['version']
            download_url(pkgs[pkg][rel]['source']['dsc'])
            if 'diff' in pkgs[pkg][rel]['source']:
                download_url(pkgs[pkg][rel]['source']['diff'])
            if 'diff' not in pkgs[pkg][rel]['source'] or opt.fetch_orig:
                download_url(pkgs[pkg][rel]['source']['orig'])

elif opt.action == 'check-build':

    EXIT_OKAY = 0
    EXIT_FAIL = 1
    exit_code = EXIT_OKAY
    for pkg in args:
        if pkg not in pkgs:
            raise ValueError("Source package '%s' not found in PPA" % (pkg))
        found = dict()

        suffix = ""
        if len(args) > 1:
            suffix = " (%s)" % (pkg)

        for rel in sorted(pkgs[pkg].keys()):
            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            version = pkgs[pkg][rel]['source']['version']
            found.setdefault(rel, dict())
            for arch in cve_lib.arch_list:
                found[rel].setdefault(arch, False)
                if arch in pkgs[pkg][rel]:
                    state = pkgs[pkg][rel][arch]['build_state']
                    if state == 'Successfully built':
                        found[rel][arch] = True
                        if opt.verbose:
                            print('\t%s %s Built' % (rel, arch))

        code = EXIT_OKAY
        report_rel = []

        for rel in cve_lib.releases:
            complete = 1
            # Skip missing source.changes
            if rel not in list(found.keys()):
                continue

            if not opt.include_devel and rel == cve_lib.devel_release:
                print("Skipping '%s' (use --include-devel)" % (rel), file=sys.stderr)
                continue
            if not opt.include_eol and rel in cve_lib.eol_releases:
                print("Skipping '%s' (use --include-eol)" % (rel), file=sys.stderr)
                continue
            version = pkgs[pkg][rel]['source']['version']

            def drop_support(supported, arches):
                for drop_arch in arches:
                    for area in ['expected', 'required']:
                        if drop_arch in supported[area]:
                            supported[area].remove(drop_arch)
                            supported['bonus'].append(drop_arch)

            support = copy.deepcopy(cve_lib.release_expectations[rel])

            # Load a map of all binary packages for this release.
            packages_map = {}
            arches = support['required'] + support['expected']
            pockets = ['', '-updates', '-security']

            for arch in arches:
                packages_map[arch] = load(data_type="packages",
                                          pockets=pockets,
                                          releases=[rel],
                                          skip_eol_releases=not opt.include_eol,
                                          arch = arch)

            # Special-case the split kernel in intrepid and later
            if re.match('linux(-meta|-source-2.6.15|-(backports|ubuntu|restricted)-modules(-2.6.[0-9]+)?)?$', pkg):
                if 'lpia' in support['required'] and rel not in ['hardy']:
                    drop_support(support, ['lpia'])
                # Non-Dapper and Non-Hardy does not build sparc, ppc, hppa
                if rel not in ['dapper', 'hardy']:
                    drop_support(support, ['sparc', 'powerpc', 'hppa'])
                # Intrepid does not build armel or ia64
                if rel in ['intrepid']:
                    drop_support(support, ['lpia', 'ia64'])
                # Jaunty does not build armel or ia64
                if rel in ['jaunty']:
                    drop_support(support, ['armel', 'ia64'])
            if re.match('linux-(|meta-)ec2$', pkg):
                # EC2 is i386/amd64 only
                drop_support(support, ['sparc', 'powerpc', 'lpia', 'armel', 'armhf'])
            if re.match('linux-(|meta-)(fsl-imx51|mvl-dove|ti-omap4?|qcm-msm|armadaxp|raspi2)$', pkg):
                # ARM kernels are, shockingly, ARM-only
                drop_support(support, ['sparc', 'powerpc', 'lpia', 'i386', 'amd64'])
            if re.match('linux-(|meta-)(lts-.*)$', pkg):
                # LTS backports seem to be built only for i386 and amd64?
                drop_support(support, ['sparc', 'powerpc', 'lpia', 'armel', 'armhf'])

            # Detect the "all" case -- only all_arch in the build record
            all_arch = cve_lib.get_all_arch(rel)
            if all_arch in pkgs[pkg][rel] and 'source' in pkgs[pkg][rel] and len(list(pkgs[pkg][rel].keys())) == 2:
                support['bonus'] = []
                support['expected'] = []
                support['required'] = [all_arch]

            # Drop arches that aren't built in the archive
            for arch in cve_lib.arch_list:
                # Get a list of binary packages in the archive that were
                # built from our source package
                archive_binaries = []
                if arch in packages_map and rel in packages_map[arch]:
                    for binpkg in packages_map[arch][rel]:
                        if packages_map[arch][rel][binpkg]['source'] == pkg:
                            if packages_map[arch][rel][binpkg]['architecture'] == "all" and all_arch != arch:
                                continue
                            archive_binaries.append(binpkg)
                if len(archive_binaries) == 0:
                    if (arch in support['required']) or (arch in support['expected']):
                        print("INFO: no build for %s in archive for %s, skipping." % (arch, rel))
                        drop_support(support, [arch])

            for arch in cve_lib.arch_list:
                if arch in support['required'] and not found[rel][arch]:
                    build_state = "[no build for %s]" % (arch)
                    if arch in pkgs[pkg][rel]:
                        build_state = pkgs[pkg][rel][arch]['build_state']
                    print('ERROR: %s missing for %s (%s)' % (arch, rel, build_state) + suffix, file=sys.stderr)
                    code = EXIT_FAIL
                    complete = 0
            for arch in cve_lib.arch_list:
                if arch in support['expected'] and not found[rel][arch]:
                    build_state = "[no build for %s]" % (arch)
                    if arch in pkgs[pkg][rel]:
                        build_state = pkgs[pkg][rel][arch]['build_state']
                    print('WARN: %s missing for %s (%s)' % (arch, rel, build_state) + suffix, file=sys.stderr)
            for arch in cve_lib.arch_list:
                if arch in support['bonus'] and found[rel][arch]:
                    print('BONUS: %s found for %s' % (arch, rel) + suffix, file=sys.stderr)
            if complete:
                report_rel.append(rel)

        if code == EXIT_OKAY:
            print("OK: " + " ".join(report_rel) + suffix)
        else:
            if len(report_rel) > 0:
                print("READY: " + " ".join(report_rel) + suffix)
            print("FAIL: not all releases ready" + suffix)
            print("*** DO NOT PUBLISH YET *** There is no method to unembargo an architecture later")
            exit_code = EXIT_FAIL
    sys.exit(exit_code)

else:
    print("Unknown action '%s'" % (opt.action), file=sys.stderr)
    sys.exit(1)

if opt.download:
    # Can't use os.rename because of potential for:
    # OSError: [Errno 18] Invalid cross-device link'
    shutil.move(tmpdir, download_dir)
    print("Files downloaded to %s" % (download_dir))