def on_task_finished(self, task):
        task.sound_file.progress = 1.0

        if task.error:
            debug("error in task, skipping rename:", task.output_filename)
            if vfs_exists(task.output_filename):
                vfs_unlink(task.output_filename)
            self.errors.append(task.error)
            self.error_count += 1
            return

        duration = task.get_duration()
        if duration:
            self.duration_processed += duration

        # rename temporary file
        newname = self.window.prefs.generate_filename(task.sound_file)
        log(beautify_uri(task.output_filename), "->", beautify_uri(newname))

        # safe mode. generate a filename until we find a free one
        p, e = os.path.splitext(newname)
        p = p.replace("%", "%%")
        p = p + " (%d)" + e
        i = 1
        while vfs_exists(newname):
            newname = p % i
            i += 1

        task.error = vfs_rename(task.output_filename, newname)
        if task.error:
            self.errors.append(task.error)
            self.error_count += 1

    def getImagesMjpegInterlace(self, camera, url, control, prefix):
        """ Update camera position with interlaced mjpeg frames """

        try:
            stream = requests.get(url, stream=True, timeout=TIMEOUT).raw

        except requests.RequestException as e:
            utils.log(3, e)
            control[0].setImage(_error, useCache=False)
            return

        x = 0
        while not monitor.abortRequested() and self.isRunning:

            filename = os.path.join(_datapath, "%s_%s.%d.jpg") % (prefix, camera.number, x)
            filename_exists = utils.get_mjpeg_frame(stream, filename)

            if filename_exists:
                if x % 2 == 0:  # Interlacing for flicker reduction/elimination
                    control[0].setImage(filename, useCache=False)
                else:
                    control[1].setImage(filename, useCache=False)
                xbmcvfs.delete(os.path.join(_datapath, "%s_%s.%d.jpg") % (prefix, camera.number, x - 2))
                x += 1

            else:
                utils.log(3, "Camera %s - Error on MJPEG" % camera.number)
                control[0].setImage(_error, useCache=False)
                return

def answer_from_div(div):
    a = Answer()
    a.author = div.xpath('.//a[@class="author-link"]')[0].text
    log('author,', a)
    content = div.xpath('.//div[@class="zm-editable-content clearfix"]/text()')
    a.content = '\n'.join(content)
    return a

def delete_temp_files():
    log('Deleting temporary files...')

    os.remove(settings.BROKEN_CSV_FILE_NAME)
    os.remove(settings.FIXED_CSV_FILE_NAME)

    log('Done')

def ebsmount_add(devname, mountdir):
    """ebs device attached"""

    matching_devices = []
    for device in udevdb.query():
        if device.name.startswith(basename(devname)):
            matching_devices.append(device)

    for device in matching_devices:
        devpath = join('/dev', device.name)
        mountpath = join(mountdir, device.env.get('ID_FS_UUID', devpath[-1])[:6])
        mountoptions = ",".join(config.mountoptions.split())

        filesystem = device.env.get('ID_FS_TYPE', None)
        if not filesystem:
            log(devname, "could not identify filesystem: %s" % devpath)
            continue

        if not filesystem in config.filesystems.split():
            log(devname, "filesystem (%s) not supported: %s" % (filesystem,devpath))
            continue

        if is_mounted(devpath):
            log(devname, "already mounted: %s" % devpath)
            continue

        mount(devpath, mountpath, mountoptions)
        if exists(config.postmountscript):
          log(devname, "Executing: %s -m %s -d %s" % (config.postmountscript, mountpath, devname))
          res = system("exec '%s' -m %s -d %s" % (config.postmountscript, mountpath, devname))
        else:
          log(devname, "Script does not exist at %s" % config.postmountscript)

    def export_from_db(self, full_path):
        temp_path = os.path.join(xbmc.translatePath("special://profile"), 'temp_export_%s.csv' % (int(time.time())))
        with open(temp_path, 'w') as f:
            writer = csv.writer(f)
            f.write('***VERSION: %s***\n' % self.__get_db_version())
            if self.__table_exists('favorites'):
                f.write(CSV_MARKERS.FAVORITES + '\n')
                for fav in self.get_favorites():
                    writer.writerow(fav)
            if self.__table_exists('subscriptions'):
                f.write(CSV_MARKERS.SUBSCRIPTIONS + '\n')
                for sub in self.get_subscriptions():
                    writer.writerow(sub)
            if self.__table_exists('new_bkmark'):
                f.write(CSV_MARKERS.BOOKMARKS + '\n')
                for bookmark in self.get_bookmarks():
                    writer.writerow(bookmark)
            if self.__table_exists('external_subs'):
                f.write(CSV_MARKERS.EXT_SUBS + '\n')
                for sub in self.get_external_subs():
                    writer.writerow(sub)

        utils.log('Copying export file from: |%s| to |%s|' % (temp_path, full_path), xbmc.LOGDEBUG)
        if not xbmcvfs.copy(temp_path, full_path):
            raise Exception('Export: Copy from |%s| to |%s| failed' % (temp_path, full_path))

        if not xbmcvfs.delete(temp_path):
            raise Exception('Export: Delete of %s failed.' % (temp_path))

def ensure_mongo_indexes():
    """
    Runs commands on the mongo indexes to ensure that they are set
    :return: None
    """
    utils.log("Setting index for countries")
    cmd = config.MONGO_PATH + "mongo  --quiet " + config.MONGO_HOST + "/logs --eval \"db.clickstream.ensureIndex({country:1})\""

	def process(self, message):
		
		end = message.get('end')
		start = message.get('start')
		startDoy = message.get('startDoy')
		startYear = message.get('startYear')
		layer = message.get('layer')
		region = message.get('region')
		productName = message.get('productName').replace('.','_')

		outputFilename = "_".join([productName, layer['name'], startYear + startDoy , region]) + '.tif'
		outputFilepath = os.path.join(self.module_path, outputFilename)

		utils.removeFileIfExist(outputFilepath)

		utils.log(self.name, 'Generating', outputFilename, 'merging', str(len(layer['files'])), 'files')
		gdal_utils.mosaic(layer['files'], outputFilepath, layer['nodata'])

		tmpFiles = message.get('tmpFiles')
		tmpFiles += layer['files']
		message.set('tmpFiles', tmpFiles)

		del layer['files']
		layer['file'] = outputFilepath
		message.set('layer', layer)

		utils.log(self.name, 'Forward message (', outputFilename, ')')
		self.publish(message)

 def _run(self):
     utils.log("[%s] pulling from source '%s'" % (self.name, self._source.name))
     
     self._source.startProducing()
     self.processQueue(self._source._output)
     self._source.join()
     self._output.put(StopIteration)

def crossValidate(opts):
	"""
	Starts the tree cross validation.
	The subset tree has a depth defined by the the division-depth opttion.
	Each dataset will be split into a number of subsets defined by the division-factor option.
	
	@type	opts: Options
	@param	opts: Options object
	"""
	totaltime = time.time()
	opts.training_file = scale(opts, opts.training_file)
	log("Scaling input data")
	while opts.division_depth >= 0:
		starttime = time.time()
		log("Range: C={0}; gamma={1}".format(opts.c_range,opts.g_range))
		res = crossValidationRound(opts)
		cstart,cstop,cstep = opts.c_range
		gstart,gstop,gstep = opts.g_range
		cmin,cmax,gmin,gmax = calcNewRange(res, opts)
		opts.division_depth = opts.division_depth -1
		opts.c_range = cmin,cmax,cstep
		opts.g_range = gmin,gmax,gstep
		endtime = time.time()
		log("Time for this round: {0}".format(time.strftime("%H:%M:%S", time.gmtime(endtime-starttime))))
		log("Remaining division steps: {0}".format(opts.division_depth+1))
		
	log("Total time: {}".format(time.strftime("%H:%M:%S", time.gmtime(time.time()-totaltime))))
			

def upload_log():
    """
        Upload our full XBMC log as a GitHub gist
    """
    try:
        log_content = get_xbmc_log()
    except Exception as e:
        utils.log("Failed to read log: %s" % e)
        return None

    utils.log("Uploading xbmc.log")
    try:
        response = urllib2.urlopen(make_request(config.GIST_API_URL), json.dumps({
            "files": {
                "xbmc.log": {
                    "content": log_content
                }
            }
        }))
    except urllib2.HTTPError as e:
        print e
        utils.log("Failed to save log: HTTPError %s" % e.code)
        return False
    except urllib2.URLError as e:
        print e
        utils.log("Failed to save log: URLError %s" % e.reason)
        return False
    try:
        return json.load(response)["html_url"]
    except:
        utils.log("Failed to parse API response: %s" % response.read())

def get_latest_version():
    """
        Sort the list, and get the latest version
    """
    versions = get_versions()
    utils.log('Version check found versions: %s' % versions)
    return sorted(versions, reverse=True)[0]

    def setup(self):
        if(APP_KEY == '' or APP_SECRET == ''):
            xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30058),utils.getString(30059))
            return
        
        user_token_key,user_token_secret = self.getToken()
        
        sess = session.DropboxSession(APP_KEY,APP_SECRET,"app_folder")

        if(user_token_key == '' and user_token_secret == ''):
            token = sess.obtain_request_token()
            url = sess.build_authorize_url(token)

            #print url in log
            utils.log("Authorize URL: " + url)
            xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30056),utils.getString(30057))  
            
            #if user authorized this will work
            user_token = sess.obtain_access_token(token)
            self.setToken(user_token.key,user_token.secret)
            
        else:
            sess.set_token(user_token_key,user_token_secret)
        
        self.client = client.DropboxClient(sess)

        try:
            utils.log(str(self.client.account_info()))
        except:
            #this didn't work, delete the token file
            self.deleteToken()

 def _get_totals(self):
     utils.log('calculating totals')
     df = self.df[self.independent_vars].sum().to_frame('total')
     df = df.set_value('tds_0','total',None)
     df = df.set_value('tds_1','total',None)
     df = df.reset_index().rename(columns={'index':'independent_variable'})
     return df

    def connect_to_sql(self, sql_connect, db_name="", force_reconnect=False, create_db=True):
        """
        Connect to SQL database or create the database and connect
        :param sql_connect: the variable to set
        :param db_name: the name of the database
        :param force_reconnect: force the database connection
        :param create_db: create the database
        :return the created SQL connection
        """
        print self
        if sql_connect is None or force_reconnect:
            try:
                sql_connect = MySQLdb.connect(host=config.SQL_HOST, user=config.SQL_USERNAME, passwd=config.SQL_PASSWORD, db=db_name)
                return sql_connect
            except Exception, e:
                # Create the database
                if e[0] and create_db and db_name != "":
                    if sql_connect is None:
                        sql_connect = MySQLdb.connect(host=config.SQL_HOST, user=config.SQL_USERNAME, passwd=config.SQL_PASSWORD)
                    utils.log("Creating database " + db_name)

                    cur = sql_connect.cursor()
                    cur.execute("CREATE DATABASE " + db_name)
                    sql_connect.commit()
                    sql_connect.select_db(db_name)
                    return sql_connect
                else:
                    utils.log("Could not connect to MySQL: %s" % e)
                    return None

def parse(queue):
    utils.log('start...')
    request = queue.get()
    success, response = page_hybridflightresults_aspx(request)
    print success, response
    utils.log('done')
    queue.put({ 'success': success, 'flags':['no_range'], 'response': response })

def playlist_item(play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history):
    log("playlist_item: play_list: %s rar_file_list: %s folder: %s sab_nzo_id: %s sab_nzo_id_history: %s" %\
       (play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history))
    new_play_list = play_list[:]
    for arch_rar, movie_file in zip(play_list[0::2], play_list[1::2]):
        info = nfo.ReadNfoLabels(folder)
        xurl = "%s?mode=%s" % (sys.argv[0],MODE_LIST_PLAY)
        url = (xurl + "&nzoid=" + str(sab_nzo_id) + "&nzoidhistory=" + str(sab_nzo_id_history)) +\
              "&play_list=" + utils.quote_plus(';'.join(new_play_list)) + "&folder=" + utils.quote_plus(folder) +\
              "&file_list=" + utils.quote_plus(';'.join(rar_file_list))
        new_play_list.remove(arch_rar)
        new_play_list.remove(movie_file)
        item = xbmcgui.ListItem(movie_file, iconImage='DefaultVideo.png', thumbnailImage=info.thumbnail)
        item.setInfo(type="Video", infoLabels=info.info_labels)
        item.setProperty("Fanart_Image", info.fanart)
        item.setPath(url)
        isfolder = False
        # item.setProperty("IsPlayable", "true")
        cm = []
        if sab_nzo_id_history:
            cm_url_repair = sys.argv[0] + '?' + "mode=repair" + "&nzoidhistory=" + str(sab_nzo_id_history) + "&folder=" + utils.quote_plus(folder)
            cm.append(("Repair" , "XBMC.RunPlugin(%s)" % (cm_url_repair)))
        cm_url_delete = sys.argv[0] + '?' + "mode=delete" + "&nzoid=" + str(sab_nzo_id) + "&nzoidhistory=" + str(sab_nzo_id_history) + "&folder=" + utils.quote_plus(folder)
        cm.append(("Delete" , "XBMC.RunPlugin(%s)" % (cm_url_delete)))
        item.addContextMenuItems(cm, replaceItems=True)
        xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=url, listitem=item, isFolder=isfolder)
    xbmcplugin.setContent(HANDLE, 'movies')
    xbmcplugin.endOfDirectory(HANDLE, succeeded=True, cacheToDisc=True)
    return

def main():
    parser = argparse.ArgumentParser(description="Retrieve petitions from We The People")
    parser.add_argument(
        "-m",
        "--max",
        metavar="INTEGER",
        dest="max",
        type=int,
        default=None,
        help="maximum number of petitions to retrieve",
    )
    parser.add_argument(
        "-s",
        "--start",
        metavar="INTEGER",
        dest="start",
        type=int,
        default=1,
        help="starting page, 20 per page, default is 1",
    )
    args = parser.parse_args()

    if args.max is not None and args.max < 1:
        parser.error("How can I scrape less than one petition? You make no sense! --max must be one or greater.")

    if args.start < 1:
        parser.error("--start must be one or greater.")

    log("Found %i petitions" % (petitions(args.start, args.max)))

    # write log
    scrapelog["end"] = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
    write(json.dumps(scrapelog, indent=2), "log-wh-" + scrapelog["begin"] + ".json", log_dir())

def the_end_dialog(params, **kwargs):
    log("the_end_dialog: params: %s kwargs: %s" %(params, kwargs))
    dialog = xbmcgui.Dialog()
    if 'is_stopped' in kwargs:
        is_stopped = kwargs['is_stopped']
    else:
        is_stopped = False
    if 'progressing' in kwargs:
        progressing = kwargs['progressing']
    else:
        progressing = False
    if progressing:
        options = ['Delete', 'Just download']
        if is_stopped:
            heading = 'Downloading, what do you want to do?'
        else:
            heading = 'Still downloading, what do you want to do?'
    else:
        heading = 'Download finished, what do you want to do?'
        options = ['Delete', 'Repair']
    ret = dialog.select(heading, options)
    if ret == 0:
        delete(params)
    if ret == 1 and progressing:
        just_download(params)
    elif ret == 1 and not progressing:
        repair(params)
    return

    def import_into_db(self, full_path):
        temp_path = os.path.join(xbmc.translatePath("special://profile"), 'temp_import_%s.csv' % (int(time.time())))
        utils.log('Copying import file from: |%s| to |%s|' % (full_path, temp_path), xbmc.LOGDEBUG)
        if not xbmcvfs.copy(full_path, temp_path):
            raise Exception('Import: Copy from |%s| to |%s| failed' % (full_path, temp_path))

        try:
            with open(temp_path, 'r') as f:
                    reader = csv.reader(f)
                    mode = ''
                    _ = f.readline()  # read header
                    for line in reader:
                        if CSV_MARKERS.FAVORITES in line[0] or CSV_MARKERS.SUBSCRIPTIONS in line[0] or CSV_MARKERS.BOOKMARKS in line[0] or CSV_MARKERS.EXT_SUBS in line[0]:
                            mode = line[0]
                            continue
                        elif mode == CSV_MARKERS.FAVORITES:
                            try:
                                self.save_favorite(line[0], line[1], line[2], line[3])
                            except: pass  # save_favorite throws exception on dupe
                        elif mode == CSV_MARKERS.SUBSCRIPTIONS:
                            # don't allow import of days with values other than 0-6
                            if line[5].translate(None, '0123456'): line[5] = '0123456'
                            self.add_subscription(line[0], line[1], line[2], line[3], line[4], line[5])
                        elif mode == CSV_MARKERS.BOOKMARKS:
                            self.set_bookmark(line[0], line[1])
                        elif mode == CSV_MARKERS.EXT_SUBS:
                            self.add_ext_sub(line[0], line[1], line[2], line[3])
                        else:
                            raise Exception('CSV line found while in no mode')
        finally:
            if not xbmcvfs.delete(temp_path):
                raise Exception('Import: Delete of %s failed.' % (temp_path))