%s

#!/usr/bin/env python2 # Copyright 2011, Canonical, Ltd. # Author: Jamie Strandboge # License: GPLv3 # # Various bug reports by team. Defaults to ubuntu-security # # Eg: # $ ./scripts/report-bugs-by-team --target=~/tmp/bugs --report-type=all --bug-cache=~/tmp/bugs/bug.cache --html # from __future__ import print_function import cve_lib from types import * import optparse import os import re import shutil import subprocess import sys import time try: import pickle from html import escape except ImportError: # python 2 import cPickle as pickle from cgi import escape parser = optparse.OptionParser() parser.add_option("--debug", help="Verbose processing output", action='store_true') parser.add_option("--html", help="HTML output", action='store_true') parser.add_option("--force-cache-update", help="Force cache update", action='store_true') parser.add_option("--show-private", help="Don't redact private fields (warning-- do not publish results publicly", action='store_true') parser.add_option("--target", help="Into which directory to write the data files", metavar="DIR", action='store') parser.add_option("--team", help="Find bugs for team", metavar="TEAM", action='store', default='ubuntu-security') parser.add_option("--limit", help="Limit number of bugs", metavar="NUM", action='store', default=None) parser.add_option("--report-type", help="Report type. Specify 'help' for details", metavar="TYPE", action='store', default=None) parser.add_option("--bug-cache", help="Cached pickle file for bugs (warning-- private bug data stored)", metavar="FILE", action='store') (opt, args) = parser.parse_args() # # Helpers # def debug(s): '''Print debug message''' if opt.debug: print("DEBUG: %s" % (s), file=sys.stderr) def get_bug_data(task, bug_id=None, bug_target_name=None): '''This is expensive and should be called only once per bug/task combination''' data = {} if bug_id != None: data['bug_id'] = bug_id else: data['bug_id'] = task.bug.id data['bug_status'] = task.status data['bug_private'] = task.bug.private data['bug_security'] = task.bug.security_related data['bug_summary'] = task.bug.title data['bug_priority'] = task.importance if bug_target_name != None: data['bug_source'] = bug_target_name else: data['bug_source'] = task.bug_target_name if len(data['bug_source'].split("(")) > 1: release = data['bug_source'].split("(")[1] data['bug_release'] = str(release.split()[len(release.split()) - 1].strip("()")).lower() else: data['bug_release'] = 'unknown' # # these are private # data['bug_summary'] = '' # data['bug_source'] = '' # data['bug_priority'] = '' # # if not data['bug_private']: # # Fill in private fields for public bugs # data['bug_summary'] = task.bug.title # data['bug_priority'] = task.importance # if bug_target_name != None: # data['bug_source'] = bug_target_name # else: # data['bug_source'] = task.bug_target_name return data def print_header(title, html, intro=''): out = "" if html: out = ''' %s

%s

''' % (title, title, title) else: out = "= %s =\n" % title if intro != '': if html: out += '

' % intro else: out += '== Introduction ==\n%s\n\n' % intro if not html: out += "== Bug list ==\n" return out def print_footer(html): out = "" if html: out = '''

Updated: %s

''' % (time.strftime('%Y-%m-%d %H:%M:%S %Z'), time.strftime('%Y')) return out def write_row(bug, html, table_header=False): order = ['bug_id', 'bug_summary', 'bug_source', 'bug_priority', 'bug_status', 'bug_security', 'bug_private'] private_fields = ['bug_summary', 'bug_source', 'bug_priority'] if opt.show_private: private_fields = [] out = "" if html: out = "" else: out += "|" if table_header: for h in order: t = h.split('_')[1].title() if opt.html: out += "%s" % t else: out += " %s |" % t else: for c in order: # Convert everything to a string here contents = "" if isinstance(bug[c], IntType): contents = str(bug[c]) elif isinstance(bug[c], BooleanType): contents = "N" if bug[c]: contents = "Y" elif isinstance(bug[c], UnicodeType): # we could do better, but this is at least safe contents = bug[c].encode("ascii", "ignore") elif isinstance(bug[c], StringType): contents = bug[c] else: contents = "Skipping unknown type '%s'" % str(type(bug[c])) # Redact sensitive info if bug['bug_private'] and c in private_fields: contents = "(private)" if opt.html: contents = escape(contents) if c == 'bug_id': contents = "%s" % (int(bug[c]), int(bug[c])) cls = '' if c == 'bug_summary': cls = ' id="summary"' out += '%s\n' % (cls, contents) else: out += ' %s |' % re.sub(r'\|', '%7C', contents) # enforce CSV-style if html: out += "\n" else: out += "\n" return out def print_bugs(bugs, html, sort_by=None): '''Output a collection of bugs''' sorted_keys = [] keys = sorted(list(bugs.keys()), key=lambda k: int(k.split(':')[0])) if sort_by == 'priority': importances = ['Unknown', 'Critical', 'High', 'Medium', 'Low', 'Wishlist', 'Undecided'] bugs_by_importance = dict((key, set([])) for key in importances) for k in keys: bugs_by_importance[bugs[k]['bug_priority']].add(k) for i in importances: sorted_keys += sorted(bugs_by_importance[i]) elif sort_by == 'status': statuses = ['New', 'Incomplete', 'Confirmed', 'Triaged', 'In Progress', 'Fix Committed'] bugs_by_status = dict((key, set([])) for key in statuses) for k in keys: bugs_by_status[bugs[k]['bug_status']].add(k) for i in statuses: sorted_keys += sorted(bugs_by_status[i]) elif sort_by == 'release': try: import cve_lib except Exception: return "Could not sort by release: import 'cve_lib' failed" bugs_by_release = dict((key, set([])) for key in cve_lib.releases + ['ubuntu', 'unknown']) for k in keys: bugs_by_release[bugs[k]['bug_release']].add(k) for i in cve_lib.releases + ['ubuntu', 'unknown']: sorted_keys += sorted(bugs_by_release[i]) else: sorted_keys = keys out = "" if html: out += "

Number bugs: %d

\n" % len(keys) out += '' else: out += "Number bugs: %d\n" % len(keys) out += write_row(None, html, table_header=True) for k in sorted_keys: out += write_row(bugs[k], html) if html: out += "\n

" return out def report_team(): title = "Bugs for %s" % opt.team output = print_header(title, opt.html) output += print_bugs(all_bugs, opt.html) output += print_footer(opt.html) return output def report_not_ubuntu(): title = "Bugs for %s without an Ubuntu task" % opt.team intro = "These are bugs for which there is only a non-Ubuntu task remaining. The team should typically not track these, but individual members may want to track them on their own. These can be unsubscribed by using $UQT/responses/security/unsub-security" output = print_header(title, opt.html, intro) _bugs = {} for k in other_bugs.keys(): if k in ubuntu_bugs: continue if other_bugs[k]['bug_source'] == "launchpad": debug("Skipping 'launchpad' bug '%s'" % str(k)) continue _bugs['%s' % k] = other_bugs[k] output += print_bugs(_bugs, opt.html) output += print_footer(opt.html) return output def report_ubuntu_only(sort_by=None): title = "Bugs for %s (duplicate tasks are filtered)" % opt.team intro = "This list contains bugs where the team is subscribed, but only lists one open task per bug (as opposed to all tasks per release). It also excludes kernel bugs since those are handled via the new kernel cadence where they have different reports for bugs." if sort_by != None: intro += " This list is sorted by '%s'." % sort_by.capitalize() output = print_header(title, opt.html, intro) _bugs = {} for k in ubuntu_bugs.keys(): if k in skipped_bugs: debug("Skipping '%s' (%s)" % (str(k), ubuntu_bugs[k]['bug_source'])) continue _bugs['%s' % k] = ubuntu_bugs[k] output += print_bugs(_bugs, opt.html, sort_by=sort_by) output += print_footer(opt.html) return output def get_report(t): if t == "team": return report_team() elif t == "not-ubuntu": return report_not_ubuntu() elif t == "only-ubuntu": return report_ubuntu_only() elif t == "priority": return report_ubuntu_only(sort_by='priority') elif t == "status": return report_ubuntu_only(sort_by='status') elif t == "release": return report_ubuntu_only(sort_by='release') return "(bad report '%s')" % t def load_cache(cachename): print("Loading bugs from cache ... ", end='', file=sys.stderr) with open(bug_database_fn, 'rb') as f: bug_database = pickle.load(f) bugs = bug_database['all_bugs'] print("done", file=sys.stderr) return bugs def write_cache(bugs, cachename): bug_database = {} bug_database['all_bugs'] = bugs with open(bug_database_fn, 'wb') as f: pickle.dump(bug_database, f) # # End helpers # # # Main # all_bugs = None ubuntu_bugs = {} other_bugs = {} skipped_bugs = {} valid_reports = ['all', 'not-ubuntu', 'only-ubuntu', 'priority', 'status', 'team', 'release'] if opt.report_type == None: print("Need to specify --report-type", file=sys.stderr) sys.exit(1) elif opt.report_type == "help" or opt.report_type not in valid_reports: print('''Valid report types are: all All bugs with team subscribed (must also specify --target) not-ubuntu Bugs with team subscribed but no Ubuntu tasks only-ubuntu Bugs with team subscribed with Ubuntu tasks, skipping certain bugs (eg, kernel and LP itself) priority Same as only-ubuntu except also sort by priority release Same as all, but sorted by release ''') sys.exit(0) elif opt.report_type == "all" and opt.target == None: print("Need to specify --target with --report-type=all", file=sys.stderr) sys.exit(1) elif opt.force_cache_update and not opt.bug_cache: print("Need to specify --bug-cache with --force-cache-update", file=sys.stderr) sys.exit(1) elif opt.target and not os.path.isdir(opt.target): print("'%s' is not a directory" % (opt.target), file=sys.stderr) sys.exit(1) # Try to load the cached database bug_database = None bug_database_fn = "" if opt.bug_cache: bug_database_fn = os.path.expanduser(opt.bug_cache) if not os.path.isfile(bug_database_fn): print("'%s' does not exist. Skipping load" % bug_database_fn, file=sys.stderr) elif opt.force_cache_update: print("Found '%s', but --force-cache-update specified" % bug_database_fn, file=sys.stderr) else: # Load pickle database all_bugs = load_cache(bug_database_fn) # Connect to LP only if cached db not available if all_bugs == None: # Connect to Launchpad all_bugs = {} try: import lpl_common except: print("lpl_common.py seems to be missing. Please create a symlink from $UQT/common/lpl_common.py to $UCT/scripts/", file=sys.stderr) sys.exit(1) # Load configuration cve_lib.read_config() # API interface debug("Team: %s" % opt.team) print("Connecting to LP ...", end=' ', file=sys.stderr) lp = lpl_common.connect() # Get authenticated URL fetcher opener = lpl_common.opener_with_cookie(cve_lib.config["plb_authentication"]) if not opener: raise ValueError("Could not open cookies") team = lp.people[opt.team] print("done", file=sys.stderr) print("Loading bugs from LP (takes a while)...", file=sys.stderr) task_collection = team.searchTasks(bug_subscriber=team, omit_targeted=False) count = 0 for task in task_collection: bugid = task.bug.id target = task.bug_target_name bug_data = get_bug_data(task, bugid, target) all_bugs['%s:%s' % (bugid, target)] = bug_data count += 1 debug("%d, %s" % (bugid, target)) debug("bug count: %d" % count) if opt.limit != None and opt.limit.isdigit() and count >= int(opt.limit): break print("done (loaded %d bugs, %d could be skipped)" % (count, len(skipped_bugs)), file=sys.stderr) if opt.bug_cache and opt.force_cache_update and os.path.exists(bug_database_fn): print("Removing old '%s' (--force-cache-update specified)" % bug_database_fn, file=sys.stderr) os.unlink(bug_database_fn) # Create the cache if it doesn't exist if opt.bug_cache and not os.path.exists(bug_database_fn): print("Saving cache to '%s'. To update cache, remove this file or use --force-cache-update" % (opt.bug_cache), file=sys.stderr) write_cache(all_bugs, bug_database_fn) skipped_sources = ['launchpad', 'usn-website-content', 'usn-website', 'ubuntu-website-content'] for k in all_bugs.keys(): if all_bugs[k]['bug_source'] in skipped_sources: skipped_bugs[all_bugs[k]['bug_id']] = True elif 'Ubuntu' in all_bugs[k]['bug_source']: ubuntu_bugs[all_bugs[k]['bug_id']] = all_bugs[k] for l in cve_lib.kernel_srcs: if all_bugs[k]['bug_source'].startswith("%s " % l): skipped_bugs[all_bugs[k]['bug_id']] = True else: other_bugs[all_bugs[k]['bug_id']] = all_bugs[k] # Now that we have the bugs, report on them if opt.report_type == "all": if not os.path.isdir(opt.target): print("Could not find '%s'. Please create." % opt.target, file=sys.stderr) sys.exit(1) for r in valid_reports: if r == "all": continue ext = ".txt" if opt.html: ext = ".html" fn = os.path.join(opt.target, r + ext) output = open(fn, 'w') print(get_report(r), file=output) output.close() shutil.copy('./scripts/html-top/toplevel.css', opt.target) else: print(get_report(opt.report_type)) sys.exit(0)