#!/usr/bin/python

import urllib, re, sys, optparse, os.path, datetime
import sqlite3 as dbapi2

blueprints_base_url = 'https://blueprints.launchpad.net'

valid_states = set(['todo', 'done', 'postponed'])

def get_db(dbpath):
    '''Open/initialize database.

    This creates the database if it does not exist.
    '''
    init = not os.path.exists(dbpath)

    db = dbapi2.connect(dbpath)

    if init:
        cur = db.cursor()
        cur.execute('''CREATE TABLE work_items (
            blueprint VARCHAR(255) NOT NULL,
            workitem VARCHAR(255) NOT NULL,
            status VARCHAR(20) NOT NULL,
            date TIMESTAMP NOT NULL)''')
        db.commit()

    return db

def parse_argv():
    '''Parse CLI arguments.

    Return (options, args) tuple.
    '''
    optparser = optparse.OptionParser()
    optparser.add_option('-d', '--database',
        help='Path to database', dest='database', metavar='PATH')
    optparser.add_option('-r', '--release',
        help='Release name', dest='release')
    optparser.add_option('-p', '--pattern', metavar='REGEX',
        help='Regex pattern for blueprint name', dest='pattern')
    optparser.add_option('-D', '--dump', action='store_true', default=False,
        help='Dump database', dest='dump')
    optparser.add_option('-m', '--moin', metavar='URL',
        help='moin URL for additional work items (can be given multiple times)', 
        action='append', dest='moin', default=[])
    optparser.add_option('-t', '--text', action='store_true', default=False,
        help='Print work item summary in text format', dest='text')
    optparser.add_option('-c', '--csv', action='store_true', default=False,
        help='Print work item summary in text format', dest='csv')
    optparser.add_option('--from', metavar='YYYY-MM-DD',
        help='Generate CSV data from this day on', dest='from_date')
    optparser.add_option('--to', metavar='YYYY-MM-DD',
        help='Generate CSV data until this day', dest='to_date')

    (opts, args) = optparser.parse_args()

    if not opts.database:
        optparser.error('No database given')
    if not opts.dump and not opts.text and not opts.csv:
        if not opts.release:
            optparser.error('No release given')
        if not opts.pattern:
            optparser.error('No pattern given')

    return (opts, args)

def get_blueprints(url, name_pattern):
    '''Return a list of blueprint URLs for the current release.'''

    blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
            name_pattern)

    result = []
    for l in urllib.urlopen(url):
        m = blueprint_name_filter.search(l)
        if m:
            result.append(blueprints_base_url + m.group(1))

    return result

def get_blueprint_workitems(blueprint_url):
    '''Collect work items from a particular blueprint URL.

    This will return a list of ('item', 'status') pairs.
    '''
    work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)

    found_workitems = False
    result = []
    for l in urllib.urlopen(blueprint_url):
        if not found_workitems:
            if work_items_re.search(l):
                found_workitems = True
            continue

        l = l.replace('<br />', '').replace('</div>', '').strip()

        # ends with empty line
        if l.endswith('</p>') or not l:
            break

        try:
            (desc, state) = l.rsplit(':', 1)
        except ValueError:
            print >> sys.stderr, 'ERROR: invalid work item format: ' + l
            continue
        desc = desc.strip()
        state = state.strip().lower()
        if not state:
            state = 'todo'
        if state not in valid_states:
            print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
                state, desc)
            continue
        result.append((desc, state))

    return result

def get_moin_workitems(url):
    '''Collect work items from a moin wiki URL.

    Every line starting with "|| " is treated as a work item.

    Return a list of ('item', 'status') pairs.
    '''
    result = []
    for line in urllib.urlopen(url):
        if line.startswith('|| '):
            fields = line.strip().split('||')
            assert not fields[0] # should be empty
            desc = fields[1].strip()
            for f in fields[2:]:
                if 'DONE' in f:
                    result.append((desc, 'done'))
                    break
                elif 'POSTPONED' in f:
                    result.append((desc, 'done'))
                    break
            else:
                result.append((desc, 'todo'))

    return result

def dump(db):
    '''Dump database contents.'''

    cur = db.cursor()
    cur.execute('SELECT * FROM work_items')
    for (blueprint, workitem, status, date) in cur:
        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)

def add_work_item(db, blueprint, item, status):
    '''Add work item to database.'''

    cur = db.cursor()
    cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
            (blueprint, item, status))

def import_lp(db, name_pattern, release):
    '''Collect blueprint work items from Launchpad into DB.'''

    blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
        opts.release), name_pattern)

    cur = db.cursor()
    cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')

    for bp in blueprints:
        #print 'Checking', bp
        bpname = bp.split('/')[-1]
        work_items = get_blueprint_workitems(bp)
        if not work_items:
            print >> sys.stderr, 'WARNING: %s has no work items' % bpname
        for (item, status) in work_items:
            add_work_item(db, bpname, item, status)

def workitems_over_time(db):
    '''Calculate work item development over time.

    Return date -> state -> count mapping.
    '''
    data = {}
    for s in valid_states:
        cur = db.cursor()
        cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
                (s,))
        for (date, num) in cur:
            data.setdefault(date, {})[s] = num
    return data

def blueprint_status(db):
    '''Determine current blueprint status.

    Return blueprint -> [todo, done, postponed] mapping.
    '''
    data = {}

    # last date
    cur = db.cursor()
    cur.execute('SELECT max(date) FROM work_items')
    (last_date,) = cur.fetchone()

    index = 0
    for s in valid_states:
        cur = db.cursor()
        cur.execute('SELECT blueprint, count(workitem) FROM work_items '
                'WHERE status = ? AND date = ? GROUP BY blueprint', 
                (s, last_date))
        for (bp, num) in cur:
            data.setdefault(bp, [0, 0, 0])[index] = num
        index += 1

    return data

def text(db):
    '''Print work item status as text.'''

    data = workitems_over_time(db)

    print 'History:'
    for d in sorted(data.keys()):
        print d, data[d]

    print '\nBlueprint status:'
    data = blueprint_status(db)
    for (bp, (todo, done, postponed)) in data.iteritems():
        print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed, 
                int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))

def csv(db, from_date, to_date):
    '''Print work item status as csv.'''

    def _fmtdate(d):
        '''Convert datetime.date into MM/DD/YYYY'''

        return '%s/%s/%s' % (d.month, d.day, d.year)

    def _fromstr(s):
        '''Convert YYYY-MM-DD string to datetime.date'''

        (y, m, d) = s.split('-')
        return datetime.date(int(y), int(m), int(d))

    data = workitems_over_time(db)

    dates = sorted(data.keys())

    f = _fromstr(from_date or dates[0])
    t = _fromstr(to_date or dates[-1])

    d = f
    while d <= t:
        entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
        print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year, 
                entry.get('todo', 0), entry.get('done', 0),
                entry.get('postponed', 0))
        d += datetime.timedelta(days=1)

def import_moin(db, urls):
    '''Collect blueprint work items from a moin wiki.'''

    for url in urls:
        for (d, s) in get_moin_workitems(url):
            add_work_item(db, url, d, s)

#
# main
#

(opts, args) = parse_argv()

db = get_db(opts.database)

if opts.dump:
    dump(db)
elif opts.text:
    text(db)
elif opts.csv:
    csv(db, opts.from_date, opts.to_date)
else:
    import_lp(db, opts.pattern, opts.release)
    import_moin(db, opts.moin)
    db.commit()