def setUp(cls):
        # Create two unique labs
        lab1 = data_setup.create_labcontroller(fqdn=u'lab_%d' %
                                               int(time.time() * 1000))
        lab2 = data_setup.create_labcontroller(fqdn=u'lab_%d' %
                                               int(time.time() * 1000))

        # Create two distros and only put one in each lab.
        cls.distro_tree1 = data_setup.create_distro_tree()
        cls.distro_tree2 = data_setup.create_distro_tree()
        session.flush()
        cls.distro_tree1.lab_controller_assocs = [LabControllerDistroTree(
                lab_controller=lab2, url=u'http://notimportant')]
        cls.distro_tree2.lab_controller_assocs = [LabControllerDistroTree(
                lab_controller=lab1, url=u'http://notimportant')]

        # Create a user
        user = data_setup.create_user()

        # Create two systems but only put them in lab1.
        system1 = data_setup.create_system(owner=user)
        system2 = data_setup.create_system(owner=user)
        system1.lab_controller = lab1
        system2.lab_controller = lab1

        session.flush()

        # Create two jobs, one requiring distro_tree1 and one requiring distro_tree2
        job = '''
            <job>
                <whiteboard>%s</whiteboard>
                <recipeSet>
                    <recipe>
                        <distroRequires>
                            <distro_name op="=" value="%s" />
                        </distroRequires>
                        <hostRequires/>
                        <task name="/distribution/install" role="STANDALONE">
                            <params/>
                        </task>
                    </recipe>
                </recipeSet>
            </job>
                 ''' 
        xmljob1 = XmlJob(xmltramp.parse(job % (cls.distro_tree1.distro.name,
                cls.distro_tree1.distro.name)))
        xmljob2 = XmlJob(xmltramp.parse(job % (cls.distro_tree2.distro.name,
                cls.distro_tree2.distro.name)))

        cls.job1 = Jobs().process_xmljob(xmljob1, user)
        cls.job2 = Jobs().process_xmljob(xmljob2, user)

    def test_namespaces_repr(self):
        doc = Namespace("http://example.org/bar")
        bbc = Namespace("http://example.org/bbc")
        dc = Namespace("http://purl.org/dc/elements/1.1/")
        d = parse("""<doc version="2.7182818284590451"
        xmlns="http://example.org/bar"
        xmlns:dc="http://purl.org/dc/elements/1.1/"
        xmlns:bbc="http://example.org/bbc">
            <author>John Polk and John Palfrey</author>
            <dc:creator>John Polk</dc:creator>
            <dc:creator>John Palfrey</dc:creator>
            <bbc:show bbc:station="4">Buffy</bbc:show>
        </doc>""")
        assert repr(d) == '<doc version="2.7182818284590451">...</doc>'
        # I supect py3 does not see equality in type below.
        #assert d.__repr__(1) == '<doc xmlns:bbc="http://example.org/bbc" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://example.org/bar" version="2.7182818284590451"><author>John Polk and John Palfrey</author><dc:creator>John Polk</dc:creator><dc:creator>John Palfrey</dc:creator><bbc:show bbc:station="4">Buffy</bbc:show></doc>'
        #assert d.__repr__(1, 1) == '<doc xmlns:bbc="http://example.org/bbc" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://example.org/bar" version="2.7182818284590451">\n\t<author>John Polk and John Palfrey</author>\n\t<dc:creator>John Polk</dc:creator>\n\t<dc:creator>John Palfrey</dc:creator>\n\t<bbc:show bbc:station="4">Buffy</bbc:show>\n</doc>'
        assert repr(parse("<doc xml:lang='en' />")) == '<doc xml:lang="en"></doc>'
        self.assertEqual(d.author, d['author'])
        assert d.author == "John Polk and John Palfrey"
        assert d.author._name == doc.author
        assert str(d[dc.creator]) == "John Polk"
        assert d[dc.creator]._name == dc.creator
        assert str(d[dc.creator:][1]) == "John Palfrey"
        d[dc.creator] = "Me!!!"
        assert str(d[dc.creator]) == "Me!!!"
        assert len(d[dc.creator:]) == 1
        d[dc.creator:] = "You!!!"
        assert len(d[dc.creator:]) == 2
        assert d[bbc.show](bbc.station) == "4"
        d[bbc.show](bbc.station, "5")
        assert d[bbc.show](bbc.station) == "5"

        e = Element('e')
        e.c = '<img src="foo">'
        assert e.__repr__(1) == '<e><c>&lt;img src="foo"></c></e>'
        e.c = '2 > 4'
        assert e.__repr__(1) == '<e><c>2 > 4</c></e>'
        e.c = 'CDATA sections are <em>closed</em> with ]]>.'
        assert e.__repr__(1) == '<e><c>CDATA sections are &lt;em>closed&lt;/em> with ]]&gt;.</c></e>'
        e.c = parse('<div xmlns="http://www.w3.org/1999/xhtml">i<br /><span></span>love<br />you</div>')
        assert e.__repr__(1) == '<e><c><div xmlns="http://www.w3.org/1999/xhtml">i<br /><span></span>love<br />you</div></c></e>'

        e = Element('e')
        e('c', 'that "sucks"')
        assert e.__repr__(1) == '<e c="that &quot;sucks&quot;"></e>'
        assert quote("]]>") == "]]&gt;"
        assert quote('< dkdkdsd dkd sksdksdfsd fsdfdsf]]> kfdfkg >') == '&lt; dkdkdsd dkd sksdksdfsd fsdfdsf]]&gt; kfdfkg >'
        assert parse('<x a="&lt;"></x>').__repr__(1) == '<x a="&lt;"></x>'
        assert parse('<a xmlns="http://a"><b xmlns="http://b"/></a>').__repr__(1) == '<a xmlns="http://a"><b xmlns="http://b"></b></a>'

def get_youtube_data(video):
    """
    Helper to extract video and thumbnail from youtube
    """
    video.source = 'youtube'
    if 'youtube.com/watch' in video.url:
        parsed = urlparse.urlsplit(video.url)
        query  = urlparse.parse_qs(parsed.query)
        try:
            video.key  = query.get('v')[0]
        except IndexError:
            video.key = None
    else:
        video.key = video.url.rsplit('/', 1)[1]
    video.embed_src = 'http://www.youtube.com/embed/'
    #http://gdata.youtube.com/feeds/api/videos/Agdvt9M3NJA
    api_url = 'http://gdata.youtube.com/feeds/api/videos/%s' % video.key
    video_data = urllib.urlopen(api_url).read()
    xml = xmltramp.parse(video_data)

    video.title = unicode(xml.title)
    video.slug = slugify(video.title)
    video.summary = unicode(xml.content)
    video.thumb_url = xml[xml_media.group][xml_media.thumbnail:][1]('url')
    return video

 def getPhotoListingFromPhotoSet( self, photoSetId, page=1):
     #print "Getting photo listing for photoset", photoSetId, "for page", page, "...",
     d = {
         api.method   : "flickr.photosets.getPhotos",
         api.token    : str(self.token),
         api.perms    : str(self.perms),
         "photoset_id": str( photoSetId ),
         "per_page"   : str( 500 ),
         "page"       : str( page )
     }
     sig = self.signCall( d )
     d[ api.sig ] = sig
     d[ api.key ] = FLICKR[ api.key ]
     url = self.build_request(api.rest, d, ())
     xml = urllib2.urlopen( url ).read()
     res = xmltramp.parse(xml)
     if ( self.isGood( res ) ):
         #print "successful."
         photos = []
         for photo in res.photoset:
             photos.append(photo('title').encode('ascii', 'ignore'))
         if photoSetId in self.listings :
             self.listings[photoSetId].extend(photos)
         else :
             self.listings[photoSetId] = photos
         if int(res.photoset('page')) < int(res.photoset('pages')) :
             self.getPhotoListingFromPhotoSet(photoSetId, page+1)
     else :
         print "Problem while getting photo listing for photoset",photoSetId
         self.reportError( res )
     sys.stdout.flush()

 def createPhotoSet( self, photoSetIdFile, directoryName, photoId):
     print "Creating photoSet for folder", directoryName , "...",
     d = {
         api.method   : "flickr.photosets.create",
         api.token    : str(self.token),
         api.perms    : str(self.perms),
         "title"      : str(directoryName),
         "description": str(directoryName),
         "primary_photo_id"   : str(photoId)
     }
     sig = self.signCall( d )
     d[ api.sig ] = sig
     d[ api.key ] = FLICKR[ api.key ]        
     url = self.build_request(api.rest, d, ())    
     xml = urllib2.urlopen( url ).read()
     res = xmltramp.parse(xml)
     if ( self.isGood( res ) ):
         print "successful."
         photoSetId  = res.photoset('id')
         self.listings[photoSetId] = [];
         self.setCachedPhotoSetId(photoSetIdFile, photoSetId)
     else :
         print "problem.."
         self.reportError( res )
     sys.stdout.flush()

 def uploadImage( self, image ):
     if ( not self.isAlreadyUploaded(image) ):
         print "Uploading ", image , "...",
         try:
             photo = ('photo', image, open(image,'rb').read())
             filename = self.getImageTitle(image)
             d = {
                 api.token   : str(self.token),
                 api.perms   : str(self.perms),
                 "tags"      : str( FLICKR["tags"] ),
                 "is_public" : str( FLICKR["is_public"] ),
                 "is_friend" : str( FLICKR["is_friend"] ),
                 "is_family" : str( FLICKR["is_family"] ),
                 "title"     : str( filename )
             }
             sig = self.signCall( d )
             d[ api.sig ] = sig
             d[ api.key ] = FLICKR[ api.key ]        
             url = self.build_request(api.upload, d, (photo,))    
             xml = urllib2.urlopen( url ).read()
             res = xmltramp.parse(xml)
             if ( self.isGood( res ) ):
                 print "successful."
                 self.addImageToFlickrSet( res.photoid, image )
             else :
                 print "problem.."
                 self.reportError( res )
             sys.stdout.flush()
         except:
             print str(sys.exc_info())
     else:
         #print "Already uploaded image", image
         pass

    def getResponse( self, url ):
        """
        Send the url and get a response.  Let errors float up
        """

        xml = urllib2.urlopen( url ).read()
        return xmltramp.parse( xml )

 def uploadImage( self, image ):
     if ( not self.uploaded.has_key( image ) ):
         print "Uploading ", image , "...",
         try:
             photo = ('photo', image, open(image,'rb').read())
             d = {
                 api.token   : str(self.token),
                 api.perms   : str(self.perms),
                 "tags"      : str( FLICKR["tags"] ),
                 "is_public" : str( FLICKR["is_public"] ),
                 "is_friend" : str( FLICKR["is_friend"] ),
                 "is_family" : str( FLICKR["is_family"] )
             }
             sig = self.signCall( d )
             d[ api.sig ] = sig
             d[ api.key ] = FLICKR[ api.key ]        
             url = self.build_request(api.upload, d, (photo,))    
             xml = urllib2.urlopen( url ).read()
             res = xmltramp.parse(xml)
             if ( self.isGood( res ) ):
                 print "successful."
                 self.logUpload( res.photoid, image )
                 message = "Uploaded photo " + image
                 #self.growlNotify( "Photo uploaded", message)
                 return True
             else :
                 print "problem.."
                 self.reportError( res )
                 return False
         except:
             print str(sys.exc_info())

    def uploadImage( self, image ):
        """ uploadImage
        """
	if self.history.uploaded(image):
	  print "Already uploaded ", image, ", skipping."
	else:
	    success = False

	    while not success:
	      print "Uploading ", image , "..."
	      try:
		  photo = ('photo', image, open(image,'rb').read())
		  d = {
		      api.token   : str(self.token),
		      api.perms   : str(self.perms),
		      "tags"      : str( FLICKR["tags"] ),
		      "is_public" : str( FLICKR["is_public"] ),
		      "is_friend" : str( FLICKR["is_friend"] ),
		      "is_family" : str( FLICKR["is_family"] )
		  }
		  sig = self.signCall( d )
		  d[ api.sig ] = sig
		  d[ api.key ] = FLICKR[ api.key ]
		  url = self.build_request(api.upload, d, (photo,))
		  xml = urllib2.urlopen( url ).read()
		  res = xmltramp.parse(xml)
		  if self.isGood(res):
		      print "successful."
		      self.logUpload(res.photoid, image)
		      success = True
		  else:
		      print "problem."
		      self.reportError( res )
	      except:
		  logging.exception('Failed upload')

    def uploadImage(self, image):
        """ uploadImage
        """

        if not self.uploaded.has_key(image):
            print("Uploading ", image, "...")
            try:
                photo = ("photo", image, open(image, "rb").read())
                d = {
                    api.token: str(self.token),
                    api.perms: str(self.perms),
                    "title": str(FLICKR["title"]),
                    "description": str(FLICKR["description"]),
                    "tags": str(FLICKR["tags"]),
                    "is_public": str(FLICKR["is_public"]),
                    "is_friend": str(FLICKR["is_friend"]),
                    "is_family": str(FLICKR["is_family"]),
                }
                sig = self.signCall(d)
                d[api.sig] = sig
                d[api.key] = FLICKR[api.key]
                url = self.build_request(api.upload, d, (photo,))
                xml = urllib2.urlopen(url).read()
                res = xmltramp.parse(xml)
                if self.isGood(res):
                    print("successful.")
                    self.logUpload(res.photoid, image)
                else:
                    print("problem..")
                    self.reportError(res)
            except:
                print(str(sys.exc_info()))

    def uploadImage( self, image ):
        if ( not (LOG_UPLOADED and self.uploaded.has_key( image ) ) ):
            print "Uploading ", image , "...",
            try:
                photo = ('photo', image, open(image,'rb').read())
                d = {
                    api.token   : str(self.token),
                    api.perms   : str(self.perms),
                    "tags"      : str( FLICKR["tags"] ),
                    "is_public" : str( FLICKR["is_public"] ),
                    "is_friend" : str( FLICKR["is_friend"] ),
                    "is_family" : str( FLICKR["is_family"] )
                }
                sig = self.signCall( d )
                d[ api.sig ] = sig
                d[ api.key ] = FLICKR[ api.key ]        
                url = self.build_request(api.upload, d, (photo,))    
                xml = urllib2.urlopen( url ).read()
                res = xmltramp.parse(xml)
                if ( self.isGood( res ) ):
                    print "successful."
                    if ( LOG_UPLOADED ):
                        self.logUpload( res.photoid, image )
                    return res.photoid
                else :
                    print "problem.."
                    self.reportError( res )
	    except KeyboardInterrupt:
		print "stopping:";
		if (LOG_UPLOADED): self.uploaded.close();
		sys.exit(-1);		    
            except:
        	print "ERROR"
                print str(sys.exc_info())

	def post(self, conn=None, alwaysReturnList=False):
		headers = { "User-Agent": "BeatBox/" + __version__,
					"SOAPAction": "\"\"",
					"Content-Type": "text/xml; charset=utf-8" }
		if gzipResponse:
			headers['accept-encoding'] = 'gzip'
		if gzipRequest:
			headers['content-encoding'] = 'gzip'					
		close = False
		(scheme, host, path, params, query, frag) = urlparse(self.serverUrl)
		if conn == None:
			conn = makeConnection(scheme, host)
			close = True
		conn.request("POST", path, self.makeEnvelope(), headers)
		response = conn.getresponse()
		rawResponse = response.read()
		if response.getheader('content-encoding','') == 'gzip':
			rawResponse = gzip.GzipFile(fileobj=StringIO(rawResponse)).read()
		if close:
			conn.close()
		tramp = xmltramp.parse(rawResponse)
		try:
			faultString = str(tramp[_tSoapNS.Body][_tSoapNS.Fault].faultstring)
			faultCode   = str(tramp[_tSoapNS.Body][_tSoapNS.Fault].faultcode).split(':')[-1]
			raise SoapFaultError(faultCode, faultString)
		except KeyError:
			pass
		# first child of body is XXXXResponse
		result = tramp[_tSoapNS.Body][0]
		# it contains either a single child, or for a batch call multiple children
		if alwaysReturnList or len(result) > 1:
			return result[:]
		else:
			return result[0]

 def test_uploading_job_with_invalid_hostRequires_raises_exception(self):
     session.begin()
     try:
         xmljob = XmlJob(xmltramp.parse('''
             <job>
                 <whiteboard>job with invalid hostRequires</whiteboard>
                 <recipeSet>
                     <recipe>
                         <distroRequires>
                             <distro_name op="=" value="BlueShoeLinux5-5" />
                         </distroRequires>
                         <hostRequires>
                             <memory op=">=" value="500MB" />
                         </hostRequires>
                         <task name="/distribution/install" role="STANDALONE">
                             <params/>
                         </task>
                     </recipe>
                 </recipeSet>
             </job>
             '''))
         self.assertRaises(BX, lambda: self.controller.process_xmljob(xmljob, self.user))
     finally:
         session.rollback()
         session.close()

def main():
    """
    Search for wishlists using command line arguments.
    """
    # Leaving out the program name, grab all space-separated arguments.
    name   = " ".join(sys.argv[1:])
    
    # Construct the list of arguments for the AWS query
    args = {
        'Service'        : 'AWSECommerceService',
        'Operation'      : 'ListSearch',
        'ListType'       : 'WishList',
        'SubscriptionId' : AWS_ID,
        'Name'           : name
    }
    
    # Build the URL for the API call using the base URL and params.
    url = "%s?%s" % (AWS_URL, urllib.urlencode(args))
    
    # Perform the query, fetch and parse the results.
    data  = HTTPCache(url).content()
    doc   = xmltramp.parse(data)
    
    # Print out the list IDs found.
    lists = [ x for x in doc.Lists if 'List' in x._name ]
    for list in lists:
        print '%15s: %s' % ( list.ListId, list.CustomerName )

 def uploadImage( self, image ):
     print "Uploading ", image , "..."
     try:
         photo = ('photo', image, open(image,'rb').read())
         d = {
             api.token   : str(self.token),
             api.perms   : str(self.perms),
             "tags"      : str( FLICKR["tags"] ),
             "is_public" : str( FLICKR["is_public"] ),
             "is_friend" : str( FLICKR["is_friend"] ),
             "is_family" : str( FLICKR["is_family"] )
         }
         sig = self.signCall( d )
         d[ api.sig ] = sig
         d[ api.key ] = FLICKR[ api.key ]        
         url = self.build_request(api.upload, d, (photo,))    
         xml = urllib2.urlopen( url ).read()
         res = xmltramp.parse(xml)
         if ( self.isGood( res ) ):
             print "successful."
             self.logUpload( image )
             self.addPhotoToSet(image, res.photoid)
         else :
             print "problem.."
             self.reportError( res )
     except:
         print str(sys.exc_info())

    def test_abort_recipe_bubbles_status_to_job(self):
        xmljob = XmlJob(xmltramp.parse('''
            <job>
                <whiteboard>job </whiteboard>
                <recipeSet>
                    <recipe>
                        <distroRequires>
                            <distro_name op="=" value="BlueShoeLinux5-5" />
                        </distroRequires>
                        <hostRequires/>
                        <task name="/distribution/install" role="STANDALONE">
                            <params/>
                        </task>
                    </recipe>
                </recipeSet>
                <recipeSet>
                    <recipe>
                        <distroRequires>
                            <distro_name op="=" value="BlueShoeLinux5-5" />
                        </distroRequires>
                        <hostRequires/>
                        <task name="/distribution/install" role="STANDALONE">
                            <params/>
                        </task>
                    </recipe>
                </recipeSet>
            </job>
            '''))
        job = self.controller.process_xmljob(xmljob, self.user)
        session.flush()
        for recipeset in job.recipesets:
            for recipe in recipeset.recipes:
                recipe.process()
                recipe.queue()
                recipe.schedule()
                recipe.waiting()

        # Abort the first recipe.
        job.recipesets[0].recipes[0].abort()
        job.update_status()

        # Verify that it and its children are aborted.
        self.assertEquals(job.recipesets[0].recipes[0].status, TaskStatus.aborted)
        for task in job.recipesets[0].recipes[0].tasks:
            self.assertEquals(task.status, TaskStatus.aborted)

        # Verify that the second recipe and its children are still waiting.
        self.assertEquals(job.recipesets[1].recipes[0].status, TaskStatus.waiting)
        for task in job.recipesets[1].recipes[0].tasks:
            self.assertEquals(task.status, TaskStatus.waiting)

        # Verify that the job still shows waiting.
        self.assertEquals(job.status, TaskStatus.waiting)

        # Abort the second recipe now.
        job.recipesets[1].recipes[0].abort()
        job.update_status()

        # Verify that the whole job shows aborted now.
        self.assertEquals(job.status, TaskStatus.aborted)

 def fetch_items(self):
     """
     Grab search result items for given index and keywords.
     """
     # Construct the list of arguments for the AWS query
     args = {
         'Service'        : 'AWSECommerceService',
         'Operation'      : 'ListLookup',
         'ResponseGroup'  : 'Medium,ListFull,ItemAttributes',
         'Sort'           : 'LastUpdated',
         'ListType'       : 'WishList',
         'ListId'         : self.wishlist_id,
         'SubscriptionId' : self.aws_id,
     }
     
     # Build the URL for the API call using the base URL and params.
     url = "%s?%s" % (self.AWS_URL, urllib.urlencode(args))
     
     # Perform the query, fetch and parse the results.
     data  = HTTPCache(url).content()
     doc   = xmltramp.parse(data)
     
     # Update the feed link and title from search result metadata
     self.FEED_META['feed.link']  = doc.Lists.List.ListURL
     self.FEED_META['feed.title'] = \
         'Amazon.com wishlist items for "%s"' % \
         doc.Lists.List.CustomerName
     
     # Fetch first page of items.
     return [ x.Item for x in doc.Lists.List if 'ListItem' in x._name ]

def character( guild, character, force = False ):
    if not guild and not character.guild:
        # erase characters without a guild
        logging.info("deleting guildless character %s"%character.name)
        character.delete()
        return

    try:
        raw_xml = fetch_raw( character.armory_url() )
    except FetchError:
        logging.info("fetcherror")
        return # normally this is an armory failure. I'm not going to put clever handling in here.
    
    char_xml = xmltramp.parse( raw_xml )
    try:
        char = char_xml['characterInfo']['character']
    except KeyError:
        logging.info("keyerror")
        return # normally armoury error

    character.achPoints = long(char('points'))
    
    added_count = add_achievements( character, raw_xml )
    if added_count >= 5:
        logging.info("Seen at least 5 new achievements - running backfill")
        added_count = backfill( character, char_xml )
    
    logging.info("added %d achievements"%added_count)

    # otherwise we're done.
    character.last_fetch = datetime.utcnow()
    character.put()

    guild.update_achievements_cache_for( character )
    guild.put()

 def addSetToCollection( self, set_id , collection_id ):
     success = False
     print("Adding set with id " + str(set_id) + " to collection with id " + str(collection_id) + "...")
     try:
         d = {
             api.token       : str(self.token),
             api.perms       : str(self.perms),
             "method"        : "flickr.collections.addSet",
             "collection_id" : str(collection_id),
             "photoset_id"   : str(set_id)
         }
         sig = self.signCall( d )
         d[ api.sig ] = sig
         d[ api.key ] = FLICKR[ api.key ]
         url = self.build_request(api.rest, d, ())
         xml = urllib2.urlopen( url ).read()
         res = xmltramp.parse(xml)
         if ( self.isGood( res ) ):
             print("    Success.")
             success = True
         else :
             print("    Problem:")
             self.reportError( res )
     except KeyboardInterrupt:
         flick.printStats()
         print "\nUploading session interrupted by user..."
         sys.exit()
     except:
         print(str(sys.exc_info()))
     return success

def serviceGetParameters():
    printDebugMessage('serviceGetParameters', 'Begin', 1)
    requestUrl = baseUrl + '/parameters'
    printDebugMessage('serviceGetParameters', 'requestUrl: ' + requestUrl, 2)
    xmlDoc = restRequest(requestUrl)
    doc = xmltramp.parse(xmlDoc)
    printDebugMessage('serviceGetParameters', 'End', 1)
    return doc['id':]