Roundup Tracker

   1     #!/usr/bin/env python
   2 
   3     """Bugzilla to roundup
   4 
   5     Converts entries in bugzilla to roundup issues
   6     """
   7 
   8     import os
   9     from optparse import OptionParser
  10     import urllib2
  11     import sys
  12     import logging
  13 
  14     from elementtree import ElementTree
  15     from roundup import instance
  16     from roundup.date import Date
  17 
  18     logging.basicConfig()
  19     logger = logging.getLogger()
  20     logger.setLevel(logging.INFO)
  21 
  22     # Map bugzilla states to ours
  23     bug_status_mapping = {
  24         "UNCONFIRMED": "open",
  25         "NEW": "open",
  26         "ASSIGNED": "accepted",
  27         "REOPENED": "open", # this our chatting equiv?
  28         "RESOLVED": "complete",
  29         "VERIFIED": "verified",
  30         "CLOSED": "cancelled",
  31         }
  32 
  33     # Map severity to ours
  34     bug_severity_mapping = {
  35         "blocker": "critical",
  36         "critical": "critical",
  37         "major": "urgent",
  38         "normal": "bug",
  39         "minor": "bug",
  40         "trivial": "bug",
  41         "enhancement": "feature",
  42         }
  43 
  44     # Map properties, this doesn't cover severity and status
  45     bug_property_mapping = {
  46         "product": "product",
  47         "version": "version",
  48         "assigned_to": "assignedto",
  49         "reporter": "creator",
  50         "creation_ts": "creation",
  51         "short_desc": "title"
  52         }
  53 
  54     usage="""%prog bugzilla_base_url roundup_instace
  55 
  56     e.g. %prog http://example.com/cgi-bin/bugzilla/ $HOME/Servers/roundup/issues
  57     """
  58 
  59     def get_bugzilla_bug_list(url_prefix):
  60         """Uses bugzilla to get a list of bug ids.
  61 
  62         Uses the RDF output of bugzilla.
  63         """
  64         result = []
  65     
  66         url = "%sbuglist.cgi?short_desc_type=allwordssubstr&short_desc=&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_status=RESOLVED&bug_status=VERIFIED&bug_status=CLOSED&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&changedin=&chfieldfrom=&chfieldto=Now&chfieldvalue=&cmdtype=doit&newqueryname=&order=Reuse+same+sort+as+last+time&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf" % url_prefix
  67 
  68         f = urllib2.urlopen(url)
  69         rdf_text = f.read()
  70         fp = open("bugzilla.rdf", "w")
  71         fp.write(rdf_text)
  72         fp.close()
  73         tree = ElementTree.parse("bugzilla.rdf")
  74         root = tree.getroot()
  75     
  76         for bug in root.findall(".//{http://www.bugzilla.org/rdf#}id"):
  77             result.append(bug.text)
  78 
  79         return result
  80 
  81     def download_bugs(url_prefix, bug_ids, output_dir):
  82         """Downloads all the bugs given from bugzilla to XML files"""
  83         for bug_id in bug_ids:
  84             f = urllib2.urlopen("%s/xml.cgi?id=%s" % (url_prefix, bug_id))
  85             bug_text = f.read()
  86             fp = open(os.path.join(output_dir, "%s.xml" % bug_id), "w")
  87             fp.write(bug_text)
  88             fp.close()
  89 
  90     class BugzillaBug:
  91         def __init__(self, tree):
  92             self._tree = tree
  93             self._root = self._tree.getroot()
  94 
  95         def __getattr__(self, name):
  96             # Special case for all the long_desc's there
  97             if name in ["long_desc",]:
  98                 return self._root.findall("bug/%s" % name)
  99             return self._root.find("bug/%s" % name).text
 100 
 101         def __getitem__(self, key):
 102             return self.__getattr__(key)
 103 
 104         def keys(self):
 105             return [x.tag for x in self._root.find("bug")]
 106 
 107     def parse_bug(bug_filename):
 108         """Parses a bug"""
 109         logger.debug("Parsing %s" % bug_filename)
 110         tree = ElementTree.parse(bug_filename)
 111         return BugzillaBug(tree)
 112 
 113     def dump_bugs(bug_ids, cache_dir):
 114         """Dumps out info on the given bugs"""
 115         for bug_id in bug_ids:
 116             bug = parse_bug(os.path.join(cache_dir, "%s.xml" % bug_id))
 117             for key in bug.keys():
 118                 if key == "long_desc":
 119                     continue
 120                 print "%s: %s = %s" % (bug_id, key, bug[key])
 121             for long_desc in bug.long_desc:
 122                 print "%s:long_desc %s = %s" % (bug_id, "who", long_desc.find("who").text)
 123                 print "%s:long_desc %s = %s" % (bug_id, "bug_when", long_desc.find("bug_when").text)
 124                 thetext = long_desc.find("thetext").text
 125                 thetext = thetext.replace("\n", "\\n")
 126                 print "%s:long_desc %s = %s" % (bug_id, "thetext", thetext)
 127 
 128     def convert_bugs(bug_ids, cache_dir, roundup_instance):
 129         """Converts the bugzilla dumped XML into roundup bugs"""
 130         tracker = instance.open(roundup_instance)
 131         # NOTE: Is it worth sorting the bugs so the ids increase in chronological order?
 132         for bug_id in bug_ids:
 133             filename = os.path.join(cache_dir, "%s.xml" % bug_id)
 134             logger.info("Processing bug %s" % bug_id)
 135             bz_bug = parse_bug(filename)
 136             roundup_bug = {}
 137             for bz_prop, roundup_prop in bug_property_mapping.items():
 138                 roundup_bug[roundup_prop] = bz_bug[bz_prop]
 139 
 140             roundup_bug['status'] = bug_status_mapping[bz_bug.bug_status]
 141             roundup_bug['priority'] = bug_severity_mapping[bz_bug.bug_severity]
 142 
 143             roundup_bug['messages'] = []
 144             for long_desc in bz_bug.long_desc:
 145                 message = {}
 146                 message['creator'] = long_desc.find('who').text
 147                 message['author'] = long_desc.find('who').text
 148                 message['creation'] = long_desc.find('bug_when').text
 149                 message['file'] = long_desc.find('thetext').text
 150                 roundup_bug['messages'].append(message)
 151 
 152             bug_id = add_bug_to_roundup(roundup_bug, tracker)
 153             for message in roundup_bug['messages']:
 154                 add_message_to_roundup(bug_id, message, tracker)
 155 
 156             # Append the bug's XML as a file
 157             xml = open(filename, "r").read()
 158             add_file_to_roundup(bug_id, filename, xml, "text/xml", tracker)
 159 
 160     def add_bug_to_roundup(bug, tracker):
 161         """Add a roundup bug (dict) to a roundup instance
 162 
 163         :param bug: A dictionary of bug data. Pretty much a plain set of
 164         property -> value mappings, with the only exception being messages
 165         which maps to a list of message dictionaries.
 166 
 167         :param tracker: The roundup tracker instance, this is obtained via
 168         `roundup.instance.open`.
 169 
 170         """
 171         try:
 172                     # Open up the db and get the user, creating if necessary
 173             db = tracker.open('admin')
 174             username = get_roundup_user(db, bug['creator'])
 175             db.commit()
 176             db.close()
 177 
 178             db = tracker.open(username)
 179 
 180             # Get the issue's properties
 181             product_id = get_roundup_property(db, 'product', bug['product'])
 182             version_id = get_roundup_property(db, 'version', bug['version'])
 183             status_id = get_roundup_property(db, 'status', bug['status'])
 184             priority_id = get_roundup_property(db, 'priority', bug['priority'])
 185 
 186             # Get the users relating to the issue
 187             # From bugzilla the username is the email, so can use that as a starting point
 188             creator_id = get_roundup_user(db, bug['creator'])
 189             assignedto_id = get_roundup_user(db, bug['assignedto'])
 190 
 191             bug_id = db.issue.create(
 192                 title=bug['title'],
 193                 product=bug['product'],
 194                 version=bug['version'],
 195                 status=bug['status'],
 196                 priority=bug['priority'],
 197                 #creator=bug['creator'],
 198                 assignedto=bug['assignedto'].split("@")[0],
 199                 #creation=bug['creation'],
 200                 )
 201             db.commit()
 202         finally:
 203             #Ensure the db is always closed no matter what
 204             db.close()
 205             logger.info("Roundup db connection closed")
 206 
 207         logger.info("Added issue: %s" % bug_id)
 208         return bug_id
 209 
 210     def add_message_to_roundup(bug_id, message, tracker):
 211         """Adds the given message to to roundup"""
 212         try:
 213             # Open up the db and get the user, creating if necessary
 214             db = tracker.open('admin')
 215             username = get_roundup_user(db, message['creator'])
 216             db.commit()
 217             db.close()
 218 
 219             # Re-open as the user
 220             db = tracker.open(username)
 221 
 222             # Create the author if not there
 223             get_roundup_user(db, message['author'])
 224 
 225             # Create the message
 226             message_id = db.msg.create(
 227                 content=message['file'],
 228                 author=message['author'].split("@")[0],
 229                 date=Date(message['creation']),
 230                 #creation=message['creation']
 231                 )
 232 
 233             # Add the message to the bug
 234             bug = db.issue.getnode(bug_id)
 235             messages = bug.messages
 236             messages.append(message_id)
 237             bug.messages = messages # Force a setattr
 238             db.commit()
 239         finally:
 240             #Ensure the db is always closed no matter what
 241             db.close()
 242             logger.info("Roundup db connection closed")
 243 
 244         logger.info("Added message: %s to issue: %s" % (message_id, bug_id))
 245         return message_id
 246 
 247     def add_file_to_roundup(bug_id, filename, content, mimetype, tracker):
 248         filename = os.path.basename(filename) # ensure there are no directory parts
 249         try:
 250             db = tracker.open("admin")
 251 
 252             # Create the file
 253             file_id = db.file.create(
 254                 content=content,
 255                 name=filename,
 256                 type=mimetype
 257                 )
 258 
 259             #Append the file to the bug
 260             bug = db.issue.getnode(bug_id)
 261             files = bug.files
 262             files.append(file_id)
 263             bug.files = files
 264 
 265             db.commit()
 266         finally:
 267             db.close()
 268             logger.debug("Closed db connection")
 269 
 270         logger.info("Added file %s to issue %s" % (file_id, bug_id))
 271         return file_id
 272         
 273     def get_roundup_property(db, propname, value):
 274         """Obtains the id of the given property.
 275 
 276         If it doesn't exist it is created.
 277         """
 278         klass = db.getclass(propname)
 279         try:
 280             result = klass.lookup(value)
 281         except KeyError:
 282             result = klass.create(name=value)
 283         return result
 284 
 285     def get_roundup_user(db, email):
 286         """Find (or create) a user based on their email address
 287 
 288         This assumes everyone has a username which is also their email
 289         address prefix.
 290         """
 291         username = email.split("@")[0]
 292         try:
 293             db.user.lookup(username)
 294         except KeyError:
 295             db.user.create(username=username, address=email)
 296         return username
 297 
 298     def main():
 299         parser = OptionParser(usage=usage)
 300         parser.add_option("", "--download", dest="download", default=False, action="store_true",
 301                           help="Download bugs only. Downloads the individual bug xml files to dir called 'cache'.")
 302         parser.add_option("", "--dump", dest="dump", default=False, action="store_true",
 303                           help="Dumps out details on the parse entries")
 304         options, args = parser.parse_args()
 305 
 306         if len(args) != 2:
 307             parser.error("You need to give the bugzilla URL and the roundup instance home dir")
 308         bugzilla_url, roundup_instance = args
 309 
 310         bug_ids = get_bugzilla_bug_list(bugzilla_url)
 311 
 312         if options.download:
 313             download_bugs(bugzilla_url, bug_ids, "cache")
 314             sys.exit(0)
 315 
 316         if options.dump:
 317             dump_bugs(bug_ids, "cache")
 318             sys.exit(0)
 319 
 320         convert_bugs(bug_ids, "cache", roundup_instance)
 321 
 322     
 323 
 324     if __name__ == "__main__":
 325         main()